text:
import tensorflow as tf
import numpy as np
from collections import deque

from Model import build_actor
from network_utils import copy_vars, get_vars
from Environment import Environment

from settings import Settings

class Agent:
    """
    This class builds an agent that interacts with an environment to gather
    experiences and put them into a buffer.
    """

    def __init__(self, sess, n_agent, gui, buffer):
        print("Initializing agent %i..." % n_agent)

        self.n_agent = n_agent
        self.sess = sess
        self.gui = gui
        self.buffer = buffer

        self.env = Environment()

        self.build_actor()
        self.build_update()

        print("Agent initialized !\n")

    def build_actor(self):
        """
        Build a copy of the learner's actor network to allow the agent to
        interact with the environment on its own.
        """
        scope = 'worker_agent_' + str(self.n_agent)
        self.state_ph = tf.placeholder(dtype=tf.float32,
                                       shape=[None, *Settings.STATE_SIZE],
                                       name='state_ph')

        # Get the policy prediction network
        self.policy = build_actor(self.state_ph, trainable=False, scope=scope)
        self.vars = get_vars(scope, trainable=False)

    def build_update(self):
        """
        Build the operation to copy the weights of the learner's actor network
        in the agent's network.
        """
        with self.sess.as_default(), self.sess.graph.as_default():

            self.network_vars = get_vars('learner_actor', trainable=True)
            self.update = copy_vars(self.network_vars, self.vars,
                                    1, 'update_agent_'+str(self.n_agent))

    def predict_action(self, s):
        """
        Wrapper method to get the action outputted by the actor network.
        """
        return self.sess.run(self.policy, feed_dict={self.state_ph: s[None]})[0]

    def run(self):
        """
        Method to run the agent in the environment to collect experiences.
        """
        print("Beginning of the run agent {}...".format(self.n_agent))

        self.sess.run(self.update)

        self.total_steps = 0
        self.nb_ep = 1

        while self.nb_ep < Settings.TRAINING_EPS and not self.gui.STOP:

            s = self.env.reset()
            episode_reward = 0
            done = False

            memory = deque()
            episode_step = 1
            # The more episodes the agent performs, the longer they are
            max_step = Settings.MAX_EPISODE_STEPS
            if Settings.EP_ELONGATION > 0:
                max_step += self.nb_ep // Settings.EP_ELONGATION

            noise_scale = Settings.NOISE_SCALE * Settings.NOISE_DECAY**(self.nb_ep//20)

            while episode_step < max_step and not done and not self.gui.STOP:

                a = np.clip(self.predict_action(s),
                            Settings.LOW_BOUND, Settings.HIGH_BOUND)

                # Add gaussian noise
                noise = np.random.normal(size=Settings.ACTION_SIZE)

                a += noise_scale * noise
                s_, r, done, _ = self.env.act(a)
                episode_reward += r

                memory.append((s, a, r))

                # Keep the experience in memory until 'N_STEP_RETURN' steps has
                # passed to get the delayed return r_1 + ... + gamma^n r_n
                if len(memory) >= Settings.N_STEP_RETURN:
                    s_mem, a_mem, discount_r = memory.popleft()
                    for i, (si, ai, ri) in enumerate(memory):
                        discount_r += ri * Settings.DISCOUNT ** (i + 1)
                    self.buffer.add((s_mem, a_mem, discount_r, s_, 1 if not done else 0))

                s = s_
                episode_step += 1
                self.total_steps += 1

            # Periodically update agents on the network
            if self.nb_ep % Settings.UPDATE_ACTORS_FREQ == 0:
                self.sess.run(self.update)

            if not self.gui.STOP:
                if self.n_agent == 1 and self.gui.ep_reward.get(self.nb_ep):
                    print("Episode %i : reward %i, steps %i, noise scale %f" % (self.nb_ep, episode_reward, episode_step, noise_scale))

                self.nb_ep += 1

        self.env.close()
        if self.n_agent == 1:
            self.gui.STOP = True

max_stars_repo_name: SuReLI/Deep-RL-agents

max_issues_repo_name: SuReLI/Deep-RL-agents

max_forks_repo_name: SuReLI/Deep-RL-agents


text:
#
# Copyright (c) 2020. Asutosh Nayak (nayak.asutosh@ymail.com)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#

import sys
sys.path.append('../src')
import re
import calendar
import unicodedata
from collections import Counter

import numpy as np
import torch
import time
from nltk.tokenize import sent_tokenize, word_tokenize
from constants import *
# from src.constants import *
import multiprocessing as mp
import random
from pathlib import Path
from torchtext.data.metrics import bleu_score

def unicodeToAscii(s):
    return ''.join(
        c for c in unicodedata.normalize('NFD', s)
        if unicodedata.category(c) != 'Mn'
    )

def clean_text(text):
    text = text.lower()
    text = re.sub(r'[\"\'\`\~\#\$\%\&\^\*\“\”\’\‘]', '', text)
    text = re.sub(r'[-—_,]', ' ', text)
    text = re.sub(r'((\?+)|(\!+)|(;+)|(\.+))', '.', text)
    text = re.sub(r'[()]', '', text)
    # text = re.sub(r'\d{1,2}:\d{2} (p\.\s?m\.|a\.\s?m\.)?', '<time>', text)
    text = re.sub(r"\n+\s*\n+", ' <p> ', text)
    text = re.sub(r"\s+", ' ', text)
    text = re.sub(r'to\smorrow', 'tomorrow', text)

    month_names = '|'.join([calendar.month_name[i].lower()
                            for i in range(1, 13)])
    day_regex = r'\d{1,2}(st|nd|rd|th)'
    date_regex = day_regex+'\s('+month_names+')'

    text = re.sub(date_regex, '<date>', text)
    text = re.sub(r'\d+', '<number>', text)

    return text

def clean_text_yelp(text):
    # text = ''.join(text)
    if text.startswith('URL:') or \
            re.search(r'http(s?):\/\/[^\n\s]*', text) is not None:
        return None
    text = text.lower()
    text = text.replace(r'\/', ' ')  # e.g. staff/owner => staff owner
    text = re.sub(r'(.)\1{2,}', r'\1', text)  # ohhhhkay => okay
    text = ' '.join(word_tokenize(text))
    text = text.replace('_num_', 'NUMBER')
    text = text.replace('can\'t', 'can not')
    text = text.replace(" n't", ' not')
    text = text.replace(" 's", ' is')
    text = text.replace(" s ", ' is ')
    text = text.replace(" re ", ' are ')
    text = text.replace(" 've ", ' have ')
    text = text.replace(" ve ", ' have ')
    text = text.replace(" 'd ", ' would ')
    text = text.replace(" 'll ", " will ")
    text = text.replace(" 'm ", ' am ')
    text = text.replace(" m ", ' am ')
    text = text.replace("&", 'and')
    text = text.replace('\\n', ' ').replace("\\","")
    text = re.sub(r'[\"\'\`\~\#\$\%\&\+\^\*\“\”\’\‘\:\/]', ' ', text)
    text = re.sub(r'[-—_,]', ' ', text)
    text = re.sub(r'((\?+)|(\!+)|(;+)|(\.+))', '.', text)
    text = re.sub(r'[()]', '', text)
    text = re.sub(r'(\.\s?\.)', '.', text)
    # text = re.sub(r'\d{1,2}:\d{2} (p\.\s?m\.|a\.\s?m\.)?', '<time>', text)
    # text = re.sub(r"\n+\s*\n+", ' <p> ', text)
    text = re.sub(r"\s+", ' ', text)
    text = re.sub(r'to\smorrow', 'tomorrow', text)

    month_names = '|'.join([calendar.month_name[i].lower()
                            for i in range(1, 13)])
    day_regex = r'\d{1,2}(st|nd|rd|th)'
    date_regex = day_regex+'\s('+month_names+')'

    text = re.sub(date_regex, 'DATE', text)
    text = re.sub(r'\d+(\.\d+)?', 'NUMBER', text)
    # print(text.count('NUMBER'), len(text.split(' ')), text.count('NUMBER')/len(text.split(' ')))
    if text.count('NUMBER')/len(text.split(' ')) >= 0.3:
        return None
    if text.count('UNK')/len(text.split(' ')) >= 0.3:
        return None
    text = re.sub(r'(NUMBER)+', 'NUMBER', text)
    text = re.sub(r'(UNK)+', 'UNK', text)
    return text

def clean_text_de(text):
    # text = ''.join(text)
    if text.startswith('URL:') or text.startswith('Adresse:') or \
            re.search(r'http(s?):\/\/.*\.html', text) is not None:
        return None
    text = text.lower()
    text = text.replace(r'\/', ' ')  # e.g. staff/owner => staff owner
    text = re.sub(r'(.)\1{2,}', r'\1', text)  # ohhhhkay => okay
    text = ' '.join(word_tokenize(text))
    text = text.replace('\\n', ' ').replace("\\","")
    text = re.sub(r'[\"\'\`\~\#\$\%\&\+\^\*\“\”\’\‘\:\/]', ' ', text)
    text = re.sub(r'[-—_,]', ' ', text)
    text = re.sub(r'((\?+)|(\!+)|(;+)|(\.+))', '.', text)
    text = re.sub(r'[()]', '', text)
    text = re.sub(r'(\.\s?\.)', '.', text)
    text = re.sub(r"\s+", ' ', text)

    month_names = '|'.join([calendar.month_name[i].lower()
                            for i in range(1, 13)])
    day_regex = r'\d{1,2}(st|nd|rd|th)'
    date_regex = day_regex+'\s('+month_names+')'

    text = re.sub(date_regex, 'DATUM', text)
    text = re.sub(r'\d+(\.\d+)?', 'NUMMER', text)
    if text.count('NUMMER')/len(text.split(' ')) >= 0.3:
        return None
    if text.count('UNK')/len(text.split(' ')) >= 0.3:
        return None
    text = re.sub(r'(NUMMER)+', 'NUMMER', text)
    text = re.sub(r'(UNK)+', 'UNK', text)
    return text

def clean_text_fr(text):
    # text = ''.join(text)
    if text.startswith('URL:') or text.startswith('Adresse:') or \
            re.search(r'http(s?):\/\/.*\.html', text) is not None:
        return None
    text = text.lower()
    text = text.replace(r'\/', ' ')  # e.g. staff/owner => staff owner
    text = re.sub(r'(.)\1{2,}', r'\1', text)  # ohhhhkay => okay
    text = ' '.join(word_tokenize(text))
    text = text.replace('\\n', ' ').replace("\\","")
    text = re.sub(r'[\"\'\`\~\#\$\%\&\+\^\*\“\”\’\‘\:\/]', ' ', text)
    text = re.sub(r'[-—_,]', ' ', text)
    text = re.sub(r'((\?+)|(\!+)|(;+)|(\.+))', '.', text)
    text = re.sub(r'[()]', '', text)
    text = re.sub(r'(\.\s?\.)', '.', text)
    text = re.sub(r"\s+", ' ', text)

    month_names = '|'.join([calendar.month_name[i].lower()
                            for i in range(1, 13)])
    day_regex = r'\d{1,2}(st|nd|rd|th)'
    date_regex = day_regex+'\s('+month_names+')'

    text = re.sub(date_regex, 'DATE', text)
    text = re.sub(r'\d+(\.\d+)?', 'NOMBRE', text)
    if text.count('NOMBRE')/len(text.split(' ')) >= 0.3:
        return None
    if text.count('UNK')/len(text.split(' ')) >= 0.3:
        return None
    text = re.sub(r'(NOMBRE)+', 'NOMBRE', text)
    text = re.sub(r'(UNK)+', 'UNK', text)
    return text

def get_subclauses(sent):
    clauses = []
    for word in sent:
        if word.dep_ in ('xcomp', 'ccomp'):
            subtree_span = sent[word.left_edge.i: word.right_edge.i + 1]
            clauses.append(subtree_span.text)
    if len(clauses) == 0:
        clauses.append(sent.text)
    return clauses

def get_vocab_dicts(words):
    word2idx = {'SOS': len(words)+0, 'EOS': len(words)+1,
                'PAD': len(words)+2, 'UNK': len(words)+3}
    idx2word = {v: k for k, v in word2idx.items()}

    for w in words:
        word2idx[w] = len(word2idx)
        idx2word[len(idx2word)] = w

    return word2idx, idx2word

def vocab_from_pretrained_emb(emb_path, words, start=0, end=0, batch_num=0,
                              batch_size=0, emb_dim=-1):
    word2idx = {}
    idx2word = {}
    word_emb = []
    offset = (batch_size*batch_num)
    pretrained_emb_dim = -1
    if emb_path is not None and emb_path != 'random':
        with open(emb_path) as file:
            for i, line in enumerate(file):
                split = line.split()
                if len(split) == 2:  # skip header
                    continue
                word = split[0]
                if word not in words:
                    continue
                emb = split[1:]
                idx = len(word_emb)+offset
                word2idx[word] = idx
                idx2word[idx] = word
                word_emb.append([float(i) for i in emb])
                pretrained_emb_dim = len(emb)
                if len(emb) < emb_dim:
                    word_emb[-1].extend([random.uniform(0, 1) for d in
                                     range(emb_dim-len(emb))])
    else:
        assert emb_dim > 0, "if embedding vectors file is not provided, " \
                            "embedding dimension has to be passed explicitly."
        pretrained_emb_dim = emb_dim
        for word in words:
            idx = len(word_emb) + offset
            word2idx[word] = idx
            idx2word[idx] = word
            word_emb.append(np.random.randn(emb_dim))

    return word2idx, idx2word, word_emb, pretrained_emb_dim

def vocab_from_pretrained_emb_parallel(emb_path, words, pool, extra_tokens,
                                       workers=3, emb_dim=-1, skip_oov=True):
    word2idx = {}
    idx2word = {}
    word_emb = []
    results = []

    offset = len(words) // workers
    for i in range(workers):
        s = i*offset
        e = s+offset-1
        results.append(pool.apply_async(vocab_from_pretrained_emb,
                                   args=[emb_path, words[s:e+1], s, e, i,
                                         offset, emb_dim]))
    [res.wait() for res in results]
    prev = None
    # print('words', len(words))
    for i, res in enumerate(results):
        _ = res.get()
        if prev:
            prev_len = len(word2idx)
            diff = i*offset - prev_len
            _[1].clear()
            temp = _[0].copy()
            _[0].clear()
            for k, v in temp.items():  # word2idx
                _[0][k] = v-diff
                _[1][v-diff] = k  # idx2word

        word2idx.update(_[0])
        idx2word.update(_[1])
        word_emb.extend(_[2])
        pretrained_emb_dim = _[3]
        prev = _[0]

    # now add words from corpus which are missing in Glove embeddings
    diff = []
    if not skip_oov:
        diff.extend(list(set(words).difference(word2idx.keys()))) # this includes extra_tokens
        print('diff', diff[:200], len(diff))
    else:
        diff = extra_tokens.copy()
    for word in diff:
        word2idx[word] = len(word_emb)
        if len(word_emb) in idx2word:
            raise Exception("word index already exists", len(word_emb),
                            idx2word[len(word_emb)], word)
        idx2word[len(word_emb)] = word
        word_emb.append(np.random.uniform(0, 1, len(word_emb[-1])))
    return word2idx, idx2word, word_emb, len(diff), pretrained_emb_dim

def roll_prepend(tensor, prefix_num, dim=1):
    b = torch.roll(tensor, 1, dim)
    b[:, 0] = prefix_num
    return b

def permute_items(collection, k=3):
    q = np.random.uniform(0, k+1, len(collection)) + np.arange(len(collection))
    return list(np.array(collection)[np.argsort(q)])

# def permute_tensor(tensor, k=3):
#     c = torch.FloatTensor(tensor.size(0)).uniform_(
#         0, k+1)+torch.arange(tensor.size(0))
#     return tensor[torch.argsort(c)]

def permute_tensor(tensor, k=3):
    c = torch.FloatTensor(tensor.size(1)).uniform_(
        0, k+1)+torch.arange(tensor.size(1))
    return tensor[:, torch.argsort(c)]

def get_readable_ctime():
    return time.strftime("%d-%m-%Y %H_%M_%S")

# small functions to enable parallelization
def is_sent_shorter(sent, max_len):
    return sent is not None and len(word_tokenize(sent)) <= max_len

def len_word_tokenize(sent):
    return len(word_tokenize(sent))

def clean_text_wrapper(text, clean_text_func, max_len):
    # print(text, 'func=',clean_text_func, 'max_len',max_len)
    sent = clean_text_func(text)
    is_short = is_sent_shorter(sent, max_len) if max_len > 0 else True
    return sent if is_short else None

def iter_apply(func, coll, kwargs):
    res = []
    for item in coll:
        res.append(func(item, **kwargs))
    return res

def parallelize(func, collection, pool, parts, **kwargs):
    batch_size = len(collection)//parts
    results = []
    output = []

    for i in range(parts):
        s = i*batch_size
        e = s+batch_size-1
        results.append(pool.apply_async(iter_apply, (func, collection[s:e+1], kwargs)))

    for res in results:
        output.extend(res.get())

    return output

def tensor_to_sentence(sent_tensor, idx2word):
    sent_tensor = sent_tensor.squeeze().cpu().numpy()
    sent = []
    for idx in sent_tensor:
        sent.append(idx2word[idx])
    return ' '.join(sent)

def sent_to_tensor(sentence, **kwargs):
    word2idx = kwargs['word2idx'] if 'word2idx' in kwargs else word2idx
    max_len = kwargs['max_len'] if 'max_len' in kwargs else max_len
    prefix = kwargs['prefix'] if 'prefix' in kwargs else None
    shuffle = kwargs['shuffle'] if 'shuffle' in kwargs else False
    type = kwargs['type'] if 'type' in kwargs else 'src'
    dropout = kwargs['dropout'] if 'dropout' in kwargs else False
    word_dropout = kwargs['word_dropout'] if 'word_dropout' in kwargs else 0
    shuffle_prob = kwargs['shuffle_prob'] if 'shuffle_prob' in kwargs else 0

    temp = []
    sos = word2idx[SOS_SRC] if type == 'src' else word2idx[SOS_TGT]
    temp.append(sos)
    if prefix:
        for _ in prefix.split():
            temp.append(word2idx[_])
    words = word_tokenize(sentence.strip())

    if dropout and np.random.uniform(0, 1) < word_dropout:
        drop_idx = np.random.randint(len(words))
        # don't drop NER mask token
        if not words[drop_idx].isupper() and \
                not words[drop_idx] == '.' and not words[drop_idx] == '?':
            words = words[:drop_idx] + words[drop_idx + 1:]

    if shuffle and np.random.uniform(0,1) < shuffle_prob:
        words = permute_items(words, k=4)

    temp.extend([word2idx.get(w, word2idx['UNK']) for w in words])
    temp.append(word2idx['EOS'])
    temp.extend([word2idx['PAD']] * (max_len - len(temp)))
    return torch.tensor(temp)

def vocab_from_sents(sents, pool, extra_tokens, emb_path=None, emb_dim=-1,
                     skip_oov=True, min_freq=0):
    words = pool.map(word_tokenize, sents)
    words = [w for l in words for w in l]
    count_dict = Counter(words)
    print('total words before filtering', len(count_dict))
    words = [w for w in count_dict.keys() if count_dict[w] >= min_freq]
    print('total words after filtering', len(words))
    words = list(set(words))  # TODO remove redundant call to set. Already used counter
    words.extend(extra_tokens)
    word2idx, idx2word, word_emb, oov_words_size, pretrained_emb_size = vocab_from_pretrained_emb_parallel(
        emb_path, words, pool, extra_tokens, mp.cpu_count(), emb_dim=emb_dim,
    skip_oov=skip_oov)

    return word2idx, idx2word, word_emb, oov_words_size, pretrained_emb_size

def get_dir_size_mb(path):
    # st_size returns in bytes
    size_bytes = sum(f.stat().st_size for f in path.glob('**/*') if f.is_file())
    return size_bytes/(1024**2)

def get_bleu_score(candidates, references, n=4, w=[1/4]*4):
    return bleu_score(candidates, references, max_n=n, weights=w)

ext: py

max_stars_repo_name: nayash/unsupervised-text-style-transfer

max_issues_repo_name: nayash/unsupervised-text-style-transfer

max_forks_repo_name: nayash/unsupervised-text-style-transfer

reason: import numpy


text:
"""
 @author Zhenyu Jiang
 @email stevetod98@gmail.com
 @date 2022-01-12
 @desc Point cloud utilities
"""

import numpy as np
import open3d as o3d

def assert_array_shape(xyz, shapes=((-1, 3),)):
    """check array shape

    Args:
        xyz (np.ndarray): array
        shape (tuple of tuple of ints, optional): possible target shapes, -1 means arbitrary. Defaults to ((-1, 3)).

    Raises:
        ValueError.
    """
    flags = {x: True for x in range(len(shapes))}
    for idx, shape in enumerate(shapes):
        if len(xyz.shape) != len(shape):
            flags[idx] = False

        for dim, num in enumerate(shape):
            if num == -1:
                continue
            elif xyz.shape[dim] != num:
                flags[idx] = False
    if sum(flags.values()) == 0:  # None of the possible shape works
        raise ValueError(f"Input array {xyz.shape} is not in target shapes {shapes}!")

def np_to_o3d_pointcloud(xyz, color=None):
    """convert numpy array to open3d point cloud

    Args:
        xyz (np.ndarray): point cloud
        color (np.ndarray, optional): colors of input point cloud. Can be N*3 or 3. Defaults to None.

    Returns:
        o3d.geometry.PointCloud: open3d point cloud
    """
    assert_array_shape(xyz)
    pcd = o3d.geometry.PointCloud()
    pcd.points = o3d.utility.Vector3dVector(xyz)

    # add color
    if color is not None:
        if len(color.shape) == 2:
            # same number of colors as the points
            assert_array_shape(color, shapes=(xyz.shape,))
            pcd.colors = o3d.utility.Vector3dVector(color)
        elif len(color.shape) == 1:
            # N*3
            color = np.tile(color, (xyz.shape[0], 1))
            assert_array_shape(color, shapes=(xyz.shape,))
            pcd.colors = o3d.utility.Vector3dVector(color)
        else:
            raise ValueError(f"Bad color with shape {color.shape}")

    return pcd

def normalize_pointcloud(xyz, padding=0.0):
    """normalize point cloud to [-0.5, 0.5]

    Args:
        xyz (np.ndarray): input point cloud, N*3
        padding (float, optional): padding. Defaults to 0.0.

    Returns:
        np.ndarray: normalized point cloud
        np.ndarray: original center
        float: original scale
    """
    assert_array_shape(xyz)
    bound_max = xyz.max(0)
    bound_min = xyz.min(0)
    center = (bound_max + bound_min) / 2
    scale = (bound_max - bound_min).max()
    scale = scale * (1 + padding)
    normalized_xyz = (xyz - center) / scale
    return normalized_xyz, center, scale

def sample_pointcloud(xyz, color=None, num_points=2048):
    """random subsample point cloud

    Args:
        xyz (np.ndarray): input point cloud of N*3.
        color (np.ndarray, optional): color of the points, N*3 or None. Defaults to None.
        num_points (int, optional): number of subsampled point cloud. Defaults to 2048.

    Returns:
        np.ndarray: subsampled point cloud
    """
    assert_array_shape(xyz)
    replace = num_points > xyz.shape[0]
    sample_idx = np.random.choice(np.arange(xyz.shape[0]), size=(num_points,), replace=replace)

    if color is not None:
        assert_array_shape(color, shapes=(xyz.shape,))
        color = color[sample_idx]
    xyz = xyz[sample_idx]
    return xyz, color


text:
# Copyright (C) 2017-2020 Intel Corporation
#
# SPDX-License-Identifier: MIT

import argparse
from bench import (
    parse_args, measure_function_time, load_data, print_output,
    getFPType
)
import numpy as np
from daal4py import pca, pca_transform, normalization_zscore
from sklearn.utils.extmath import svd_flip

parser = argparse.ArgumentParser(description='daal4py PCA benchmark')
parser.add_argument('--svd-solver', type=str,
                    choices=['daal', 'full', 'correlation'],
                    default='daal', help='SVD solver to use')
parser.add_argument('--n-components', type=int, default=None,
                    help='Number of components to find')
parser.add_argument('--whiten', action='store_true', default=False,
                    help='Perform whitening')
parser.add_argument('--write-results', action='store_true', default=False,
                    help='Write results to disk for verification')
params = parse_args(parser, size=(10000, 1000))

# Load data
X_train, X_test, _, _ = load_data(params, generated_data=['X_train'],
                                  add_dtype=True)

if params.n_components is None:
    p, n = X_train.shape
    params.n_components = min((n, (2 + min((n, p))) // 3))

# Define how to do our scikit-learn PCA using DAAL...
def pca_fit_daal(X, n_components, method):

    if n_components < 1:
        n_components = min(X.shape)

    fptype = getFPType(X)

    centering_algo = normalization_zscore(
        fptype=fptype,
        doScale=False
    )

    pca_algorithm = pca(
        fptype=fptype,
        method=method,
        normalization=centering_algo,
        resultsToCompute='mean|variance|eigenvalue',
        isDeterministic=True,
        nComponents=n_components
    )

    pca_result = pca_algorithm.compute(X)
    eigenvectors = pca_result.eigenvectors
    eigenvalues = pca_result.eigenvalues.ravel()
    singular_values = np.sqrt((X.shape[0] - 1) * eigenvalues)

    return pca_result, eigenvalues, eigenvectors, singular_values

def pca_transform_daal(pca_result, X, n_components, fit_n_samples,
                       eigenvalues, eigenvectors,
                       whiten=False, scale_eigenvalues=False):

    fptype = getFPType(X)

    tr_data = {}
    tr_data['mean'] = pca_result.dataForTransform['mean']

    if whiten:
        if scale_eigenvalues:
            tr_data['eigenvalue'] = (fit_n_samples - 1) \
                * pca_result.eigenvalues
        else:
            tr_data['eigenvalue'] = pca_result.eigenvalues
    elif scale_eigenvalues:
        tr_data['eigenvalue'] = np.full((1, pca_result.eigenvalues.size),
                                        fit_n_samples - 1, dtype=X.dtype)

    transform_algorithm = pca_transform(
        fptype=fptype,
        nComponents=n_components
    )
    transform_result = transform_algorithm.compute(X, pca_result.eigenvectors,
                                                   tr_data)
    return transform_result.transformedData

def pca_fit_full_daal(X, n_components):

    fit_result, eigenvalues, eigenvectors, S = pca_fit_daal(
      X, min(X.shape), 'svdDense')
    U = pca_transform_daal(fit_result, X, min(X.shape), X.shape[0],
                           eigenvalues, eigenvectors,
                           whiten=True, scale_eigenvalues=True)
    V = fit_result.eigenvectors

    U, V = svd_flip(U, V)

    eigenvalues = fit_result.eigenvalues[:n_components].copy()
    eigenvectors = fit_result.eigenvectors[:n_components].copy()

    return fit_result, eigenvalues, eigenvectors, U, S, V

def test_fit(X):
    if params.svd_solver == 'full':
        return pca_fit_full_daal(X, params.n_components)
    else:
        method = 'correlationDense' if params.svd_solver == 'correlation' else 'svdDense'
        return pca_fit_daal(X, params.n_components, method)

def test_transform(Xp, pca_result, eigenvalues, eigenvectors):
    return pca_transform_daal(pca_result, Xp, params.n_components,
                              X_train.shape[0], eigenvalues,
                              eigenvectors, whiten=params.whiten)

columns = ('batch', 'arch', 'prefix', 'function', 'threads', 'dtype', 'size',
           'svd_solver', 'n_components', 'whiten', 'time')

# Time fit
fit_time, res = measure_function_time(test_fit, X_train, params=params)

# Time transform
transform_time, tr = measure_function_time(
    test_transform, X_test, *res[:3], params=params)

print_output(library='daal4py', algorithm='pca',
             stages=['training', 'transformation'], columns=columns,
             params=params, functions=['PCA.fit', 'PCA.transform'],
             times=[fit_time, transform_time], accuracy_type=None,
             accuracies=[None, None], data=[X_train, X_test],
             alg_params={'svd_solver': params.svd_solver,
                         'n_components': params.n_components})

if params.write_results:
    np.save('pca_daal4py_X_train.npy', X_train)
    np.save('pca_daal4py_X_test.npy', X_test)
    np.save('pca_daal4py_eigvals.npy', res[1])
    np.save('pca_daal4py_eigvecs.npy', res[2])
    np.save('pca_daal4py_tr.npy', tr)


text:
# -*- coding:utf-8 -*-
import PCF8591 as ADC
import csv
import time
import socket
import numpy as np

def map(x,in_min,in_max,out_min,out_max,compensation):
    data_raw = (x-in_min) * (out_max-out_min) / (in_max-in_min) + out_min + compensation
    if data_raw > 0 and data_raw < 10:
        data = format(data_raw, '.4f')
    elif data_raw < 0:
        data = format(data_raw, '.3f')
    elif data_raw > 10 and data_raw < 100:
        data = format(data_raw, '.3f')
    else:
        data = format(data_raw, '.2f')
    return data

def setup():
    ADC.setup(0x48)

def loop():
    i = 0
    j = 0
    TIME = 0

    VCC = 5.0
    R_div=100000.0
    straight1 = 30114.0
    bend1 = 55800.0
    straight5 = 29500.0
    bend5 = 48500.0

    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(('192.168.43.51', 6666))  #绑定ip和端口号（IP为发送数据的树莓派ip，端口号自己指定）
    s.listen(5)
    c, address = s.accept()      #等待别的树莓派接入

    while True:
		#part of getting local time
        ct = time.time()
        local_time = time.localtime(ct)
        data_head = time.strftime('%Y-%m-%d %H:%M:%S',local_time)
        data_secs = (ct - int(ct)) * 1000
        time_finial = "%s.%03d" % (data_head, data_secs)

        degree1_flexV = ADC.read(0) * VCC /256
        degree1_flexR = R_div * (VCC/degree1_flexV - 1.0)
        degree1 = map(degree1_flexR,straight1,bend1,0,90.0,-11.280)

        degree2_flexV = ADC.read(1) * VCC /256
        degree2_flexR = R_div * (VCC/degree2_flexV - 1.0)
        degree2 = map(degree2_flexR,straight5,bend5,0,90.0,4.058)

        TIME = TIME + 0.5
        time.sleep(0.5)
		#mode1
        if(TIME%4 == 0):

            if(float(degree1) < 10.000 and float(degree2) < 10.000):

                print(time_finial, 'finger1 degree = ', degree1)
                print(time_finial, 'finger2 degree = ', degree2)

                msg = degree1 + ' ' + degree2

                c.send(msg.encode('utf-8'))   #编码

                with open("test_data1.csv", "a") as csvfile:
                    writer = csv.writer(csvfile)
                    writer.writerow(["Time", "ID", "Value"])
                    writer.writerows([[time_finial, 'finger1', degree1], [time_finial, 'finger2', degree2]])

            elif(float(degree1) > 10.000 and float(degree2) > 10.000):

                print(time_finial, 'finger1 degree = ', degree1)
                print(time_finial, 'finger2 degree = ', degree2)

                msg = degree1 + ' ' + degree2

                c.send(msg.encode('utf-8'))   #编码

                with open("test_data1.csv", "a") as csvfile:
                    writer = csv.writer(csvfile)
                    writer.writerow(["Time", "ID", "Value"])
                    writer.writerows([[time_finial, 'finger1', degree1], [time_finial, 'finger2', degree2]])

            elif(float(degree1) > 10.000 and i < 15):
                for i in range(15):

                    print(time_finial, 'finger1 degree = ', degree1)

                    msg = degree1 + ' ' + degree2

                    c.send(msg.encode('utf-8'))   #编码

                    with open("test_data1.csv", "a") as csvfile:
                        writer = csv.writer(csvfile)
                        writer.writerow(["Time", "ID", "Value"])
                        writer.writerows([[time_finial, 'finger1',degree1]])

                i = 0

                print(time_finial, 'finger2 degree = ', degree2)

                msg = degree1 + ' ' + degree2

                c.send(msg.encode('utf-8'))   #编码

                with open("test_data1.csv", "a") as csvfile:
                    writer = csv.writer(csvfile)
                    writer.writerow(["Time", "ID", "Value"])
                    writer.writerows([[time_finial, 'finger2',degree2]])

            elif(float(degree2) > 10.000 and j < 15):
                for j in range(15):

                    print(time_finial, 'finger2 degree = ', degree2)

                    msg = degree1 + ' ' + degree2

                    c.send(msg.encode('utf-8'))   #编码

                    with open("test_data1.csv", "a") as csvfile:
                        writer = csv.writer(csvfile)
                        writer.writerow(["Time", "ID", "Value"])
                        writer.writerows([[time_finial, 'finger2',degree2]])

                j = 0

                print(time_finial, 'finger1 degree = ', degree1)

                msg = degree1 + ' ' + degree2

                c.send(msg.encode('utf-8'))   #编码

                with open("test_data1.csv", "a") as csvfile:
                    writer = csv.writer(csvfile)
                    writer.writerow(["Time", "ID", "Value"])
                    writer.writerows([[time_finial, 'finger1',degree1]])

            else:
                print("Warning! The data is missing")

		#mode2
            '''elif(TIME%2 == 0):

			if(float(degree1) < 10.000 and float(degree2) < 10.000):
				print(time_finial, 'finger1 degree = ', degree1)
				print(time_finial, 'finger2 degree = ', degree2)
				with open("test_data1.csv", "a") as csvfile:
					writer = csv.writer(csvfile)
					writer.writerow(["Time", "ID", "Value"])
					writer.writerows([[time_finial, 'finger1', degree1], [time_finial, 'finger2', degree2]])

			elif(float(degree1) > 10.000 and float(degree2) > 10.000):
				print(time_finial, 'finger1 degree = ', degree1)
				print(time_finial, 'finger2 degree = ', degree2)
				with open("test_data1.csv", "a") as csvfile:
					writer = csv.writer(csvfile)
					writer.writerow(["Time", "ID", "Value"])
					writer.writerows([[time_finial, 'finger1', degree1], [time_finial, 'finger2', degree2]])
			elif(float(degree1) > 10.000 and i < 15):
				print(time_finial, 'finger1 degree = ', degree1)
				i = i + 1
				with open("test_data1.csv", "a") as csvfile:
					writer = csv.writer(csvfile)
					writer.writerow(["Time", "ID", "Value"])
					writer.writerows([[time_finial, 'finger1',degree1]])
			elif(float(degree1) > 10.000 and i == 15):
				print(time_finial, 'finger2 degree = ', degree2)
				i = 0
				with open("test_data1.csv", "a") as csvfile:
					writer = csv.writer(csvfile)
					writer.writerow(["Time", "ID", "Value"])
					writer.writerows([[time_finial, 'finger2',degree2]])
			elif(float(degree2) > 10.000 and j < 15):
				print(time_finial, 'finger2 degree = ', degree2)
				j = j + 1
				with open("test_data1.csv", "a") as csvfile:
					writer = csv.writer(csvfile)
					writer.writerow(["Time", "ID", "Value"])
					writer.writerows([[time_finial, 'finger2',degree2]])
			elif(float(degree2) > 10.000 and j == 15):
				print(time_finial, 'finger1 degree = ', degree1)
				j = 0
				with open("test_data1.csv", "a") as csvfile:
					writer = csv.writer(csvfile)
					writer.writerow(["Time", "ID", "Value"])
					writer.writerows([[time_finial, 'finger1',degree1]])
			else:
				print("Warning! The data is missing")

			for i in range(2):
				if(i == 0):
					print(time_finial, 'finger1 degree = ', degree1)
				else:
					print(time_finial, 'finger2 degree = ', degree2)
			with open("test_data1.csv","a") as csvfile:
				writer = csv.writer(csvfile)
				writer.writerow(["Time","ID","Value"])
				writer.writerows([[time_finial,'finger1',degree1],[time_finial,'finger2',degree2]])
				'''

		#mode3
        else:

            for i in range(2):
                if(i == 0):

                    print(time_finial, 'finger1 degree = ', degree1)

                    msg = degree1 + ' ' + degree2

                    c.send(msg.encode('utf-8'))   #编码

                else:

                    print(time_finial, 'finger2 degree = ', degree2)

                    msg = degree1 + ' ' + degree2

                    c.send(msg.encode('utf-8'))   #编码

            with open("test_data1.csv","a") as csvfile:
                writer = csv.writer(csvfile)
                writer.writerow(["Time","ID","Value"])
                writer.writerows([[time_finial,'finger1',degree1],[time_finial,'finger2',degree2]])

  #打印电位计电压大小A/D转换后的数字值（从AIN0借口输入的）
  #范围是0~255,0时LED灯熄灭，255时灯最亮

  #将0通道输入的电位计电压数字值转化成模拟值从AOUT输出
  #给LED灯提供电源VCC输入

def destroy():
	ADC.write(0)

if __name__ == "__main__":
	try:
		setup()
		loop()
	except KeyboardInterrupt:
		destroy()

max_stars_repo_name: MithunHub/TINGLE

max_issues_repo_name: MithunHub/TINGLE

max_forks_repo_name: MithunHub/TINGLE


text:
#!/usr/bin/env python
# encoding: utf-8

import numpy as np
import iconstrain
import egp.icgen, egp.io, egp.toolbox
import sys

test_id = "2_P3M"

run_dir_base = "/Users/users/pbos/dataserver/sims"
nproc = 2

# ---- INPUT ----
cosmo = egp.icgen.Cosmology('wmap7', trans=8)
ps = egp.icgen.CosmoPowerSpectrum(cosmo)
boxlen = 200. # Mpc h^-1
redshift = 63.
gridsize = 256
gridsize_iter = 64
nf_tile_I = 2 # zet op 3 bij 32, anders raakt ie alle particles kwijt.

seed = 2522572538
try:
    seed += int(sys.argv[1])
except:
    print "Geef een getal op om op te tellen bij de seed!"
    raise SystemExit

deltaU = egp.icgen.GaussianRandomField(ps, boxlen, gridsize_iter, seed=seed)

# constrained peak position:
target_pos = np.array([20.,40.,70.])

# constrained peak mass:
filename = "/Users/users/pbos/code/egp/testing/icgen/MCXC+xyz_SCLx4.1.csv" # kapteyn
#filename = "/Users/patrick/astro/code/egp/testing/icgen/MCXC+xyz_SCLx4.1.csv" # mac
clusters = egp.toolbox.load_csv_catalog(filename)
mass0 = np.array([x[12] for x in clusters], dtype='float64')[0] # 10^14 Msun
# ... leading to the peak's height:
peak_height = iconstrain.mass_to_peak_height(mass0, cosmo) # 1.0000000000000002
# ... and the scale of the peak in mpc:
scale_mpc = iconstrain.mass_to_scale(mass0, cosmo) # 4.0225468329142506

# Multiply height by second command line parameter:
# try:
#     height_multiplier = float(sys.argv[2])
# except:
#     print "Geef een getal op om te vermenigvuldigen met de height!"
#     raise SystemExit

# peak_height *= height_multiplier

shape_seed = 10
shape_constraints = iconstrain.set_shape_constraints(ps, boxlen, peak_height, scale_mpc, shape_seed)
one_step_result = iconstrain.iteration_mean_P3M(target_pos, peak_height, scale_mpc, boxlen, gridsize_iter, deltaU, ps, cosmo, shape_constraints, nf_tile_I = nf_tile_I)

raise SystemExit

result = iconstrain.run(cosmo, ps, boxlen, gridsize_iter, seed, target_pos, peak_height, scale_mpc, initial_guess, iconstrain.iterate_PM, constrain_shape=True)

print "Uiteindelijk gebruiken we ", result
# result = np.array([ 19.98012509,  38.26122861,  67.98546924])

# Plot
#~ iteration_mean(result, mass0, boxlen, gridsize_iter, rhoU, ps, cosmo, shape_constraints, True, pos0)

run_name = "run%i_%i_hx%f" % ((100+int(test_id)), seed, height_multiplier)

setup_gadget_run(cosmo, ps, boxlen, gridsize, seed, result, peak_height, scale_mpc, redshift, test_id, run_dir_base, nproc, run_name = run_name)

print("Saving not iterated version for comparison (run100)...")

run_name = "run100_%i_hx%f" % (seed, height_multiplier)

setup_gadget_run(cosmo, ps, boxlen, gridsize, seed, target_pos, peak_height, scale_mpc, redshift, test_id, run_dir_base, nproc, run_name = run_name)

max_stars_repo_name: egpbos/egp

max_issues_repo_name: egpbos/egp

max_forks_repo_name: egpbos/egp


text:
"""ATORtf_RF.py

# Author:
Richard Bruce Baxter - Copyright (c) 2021 Baxter AI (baxterai.com)

# License:
MIT License

# Installation:
See ATORtf.tf

# Usage:
See ATORtf.tf

# Description:
ATORtf RF (receptive field)

"""

import tensorflow as tf
import numpy as np
import copy

import ATORtf_ellipseProperties
import ATORtf_RFellipse
import ATORtf_RFtri
import ATORtf_RFfilter
import ATORtf_RFproperties
import ATORtf_operations

class RFneuronClass():
	def __init__(self, resolutionProperties, RFproperties, RFfilter, RFImage):
		self.resolutionIndex = resolutionProperties.resolutionIndex
		self.RFproperties = RFproperties
		#if(debugSaveRFfiltersAndImageSegments):
		self.RFfilter = RFfilter
		self.RFneuronParents = []
		self.RFneuronComponents = []

def normaliseRFComponentWRTparent(resolutionProperties, neuronComponent, RFpropertiesParent):
	neuronComponent.RFpropertiesNormalisedWRTparent = ATORtf_RFproperties.generateRFtransformedProperties(neuronComponent, RFpropertiesParent)
	if(resolutionProperties.debugSaveRFfiltersAndImageSegments):
		neuronComponent.RFfilterNormalisedWRTparent = ATORtf_RFfilter.transformRFfilterTF(neuronComponent.RFfilter, RFpropertiesParent)
		neuronComponent.RFImageNormalisedWRTparent = ATORtf_RFfilter.transformRFfilterTF(neuronComponent.RFImage, RFpropertiesParent)

#TODO: this needs to be upgraded with a robust method
def childRFoverlapsParentRF(neuronRFpropertiesNormalisedGlobal, lowerNeuronRFpropertiesNormalisedGlobal):
	if(neuronRFpropertiesNormalisedGlobal.RFtype == ATORtf_RFproperties.RFtypeEllipse):
		return ATORtf_ellipseProperties.centroidOverlapsEllipse(neuronRFpropertiesNormalisedGlobal, lowerNeuronRFpropertiesNormalisedGlobal)
	elif(neuronRFpropertiesNormalisedGlobal.RFtype == ATORtf_RFproperties.RFtypeTri):
		#CHECKTHIS is appropriate for tri;
		return ATORtf_ellipseProperties.centroidOverlapsEllipse(neuronRFpropertiesNormalisedGlobal, lowerNeuronRFpropertiesNormalisedGlobal)

def normaliseGlobalRFproperties(RFproperties, resolutionFactor):
	#normalise RF respect to original image size
	if(RFproperties.RFtype == ATORtf_RFproperties.RFtypeEllipse):
		RFpropertiesNormalisedGlobal = ATORtf_RFellipse.normaliseGlobalEllipseProperties(RFproperties, resolutionFactor)
	elif(RFproperties.RFtype == ATORtf_RFproperties.RFtypeTri):
		RFpropertiesNormalisedGlobal = ATORtf_RFtri.normaliseGlobalTriProperties(RFproperties, resolutionFactor)
	return RFpropertiesNormalisedGlobal

def normaliseLocalRFproperties(RFproperties):
	#normalise ellipse respect to major/minor ellipticity axis orientation (WRT self)
	if(RFproperties.RFtype == ATORtf_RFproperties.RFtypeEllipse):
		ATORtf_RFellipse.normaliseLocalEllipseProperties(RFproperties)
	elif(RFproperties.RFtype == ATORtf_RFproperties.RFtypeTri):
		#CHECKTHIS is appropriate for tri; (ellipseNormalisedAxesLength, ellipseNormalisedAxesLength/q) might create equilateral triangle
		ATORtf_RFtri.normaliseLocalTriProperties(RFproperties)

def generateRFfilters(resolutionProperties, generateRFfiltersEllipse, generateRFfiltersTri):

	RFfiltersList = []
	RFfiltersPropertiesList = []

	if(generateRFfiltersEllipse):
		ATORtf_RFellipse.generateRFfiltersEllipse(resolutionProperties, RFfiltersList, RFfiltersPropertiesList)
	if(generateRFfiltersTri):
		ATORtf_RFtri.generateRFfiltersTri(resolutionProperties, RFfiltersList, RFfiltersPropertiesList)

	return RFfiltersList, RFfiltersPropertiesList

def applyRFfilters(resolutionProperties, inputImageSegments, RFfiltersTensor, numberOfDimensions, RFfiltersPropertiesList2):

	#perform convolution for each filter size;
	axesLengthMax, filterRadius, filterSize = ATORtf_RFfilter.getFilterDimensions(resolutionProperties)

	#filterApplicationResultList = []
	RFpropertiesList = []

	#print("RFfiltersPropertiesList2[0].isColourFilter = ", RFfiltersPropertiesList2[0].isColourFilter)
	#print("inputImageSegments.shape = ", inputImageSegments.shape)
	#print("RFfiltersTensor.shape = ", RFfiltersTensor.shape)

	#inputImageSegments dim: num inputImageSegments, x, y, c
	#RFfiltersTensor dim: num RFfilters, x, y, c
	#print("inputImageSegments.shape = ", inputImageSegments.shape)
	#print("RFfiltersTensor.shape = ", RFfiltersTensor.shape)

	inputImageSegmentsPixelsFlattened = tf.reshape(inputImageSegments, [inputImageSegments.shape[0], inputImageSegments.shape[1]*inputImageSegments.shape[2]*inputImageSegments.shape[3]])
	RFfiltersTensorPixelsFlattened = tf.reshape(RFfiltersTensor, [RFfiltersTensor.shape[0], RFfiltersTensor.shape[1]*RFfiltersTensor.shape[2]*RFfiltersTensor.shape[3]])

	filterApplicationResult = tf.matmul(inputImageSegmentsPixelsFlattened, tf.transpose(RFfiltersTensorPixelsFlattened))	#dim: num inputImageSegments, num RFfilters
	filterApplicationResult	= tf.reshape(filterApplicationResult, [filterApplicationResult.shape[0]*filterApplicationResult.shape[1]]) #flatten	#dim: num inputImageSegments * num RFfilters
	#filterApplicationResultList.append(filterApplicationResult)

	isColourFilter = RFfiltersPropertiesList2[0].isColourFilter
	numberOfDimensions = RFfiltersPropertiesList2[0].numberOfDimensions
	RFtype = RFfiltersPropertiesList2[0].RFtype
	filterApplicationResultThreshold = ATORtf_RFfilter.calculateFilterApplicationResultThreshold(filterApplicationResult, ATORtf_RFfilter.minimumFilterRequirement, filterSize, isColourFilter, numberOfDimensions, RFtype)
	filterApplicationResultThresholdIndices = tf.where(filterApplicationResultThreshold)	#returns [n, 1] tensor
	filterApplicationResultThresholdIndices = tf.squeeze(filterApplicationResultThresholdIndices, axis=1)

	#print("filterApplicationResult = ", filterApplicationResult)
	#print("filterApplicationResult.shape = ", filterApplicationResult.shape)
	#print("filterApplicationResultThreshold.shape = ", filterApplicationResultThreshold.shape)
	#print("filterApplicationResultThresholdIndices.shape = ", filterApplicationResultThresholdIndices.shape)

	if(not ATORtf_operations.isTensorEmpty(filterApplicationResultThresholdIndices)):

		filterApplicationResultThresholded = tf.gather(filterApplicationResult, filterApplicationResultThresholdIndices)

		#print("filterApplicationResultThresholded = ", filterApplicationResultThresholded)
		#print("filterApplicationResultThresholded.shape = ", filterApplicationResultThresholded.shape)

		filterApplicationResultThresholdIndicesNP = filterApplicationResultThresholdIndices.numpy()	#verify 1D
		filterApplicationResultThresholdIndicesList = filterApplicationResultThresholdIndicesNP.tolist()
		filterApplicationResultThresholdedNP = filterApplicationResultThresholded.numpy()	#verify 1D
		filterApplicationResultThresholdedList = filterApplicationResultThresholdedNP.tolist()

		for RFthresholdedListIndex, RFlistIndex in enumerate(filterApplicationResultThresholdIndicesList):
			#RFlistIndex = imageSegmentIndex*len(RFfiltersPropertiesList2) +  RFfiltersPropertiesList2Index

			imageSegmentIndex, RFfilterIndex = divmod(RFlistIndex, len(RFfiltersTensor))
			centerCoordinates1, centerCoordinates2 = divmod(imageSegmentIndex, resolutionProperties.imageSize[1])
			centerCoordinates = (centerCoordinates1, centerCoordinates2)

			#print("adding RFproperties")
			RFImage = inputImageSegments[imageSegmentIndex]
			RFfiltersProperties = RFfiltersPropertiesList2[RFfilterIndex]
			RFproperties = copy.deepcopy(RFfiltersProperties)
			RFproperties.centerCoordinates = centerCoordinates
			RFproperties.filterIndex = RFfilterIndex
			RFproperties.imageSegmentIndex = imageSegmentIndex
			RFpropertiesList.append(RFproperties)

		#inefficient:
		#imageSegmentIndex = 0
		#for centerCoordinates1 in range(0, resolutionProperties.imageSize[0], ellipseCenterCoordinatesResolution):
		#	for centerCoordinates2 in range(0, resolutionProperties.imageSize[1], ellipseCenterCoordinatesResolution):
		#		centerCoordinates = (centerCoordinates1, centerCoordinates2)
		#		allFilterCoordinatesWithinImageResult, imageSegmentStart, imageSegmentEnd = ATORtf_RFfilter.allFilterCoordinatesWithinImage(centerCoordinates, filterRadius, resolutionProperties.imageSize)
		#		if(allFilterCoordinatesWithinImageResult):
		#			for RFfilterIndex, RFfiltersProperties in enumerate(RFfiltersPropertiesList2):
		#				RFproperties = copy.deepcopy(RFfiltersProperties)
		#				RFproperties.centerCoordinates = centerCoordinates
		#				RFproperties.filterIndex = RFfilterIndex
		#				RFproperties.imageSegmentIndex = imageSegmentIndex
		#				RFpropertiesList.append(RFproperties)
		#			imageSegmentIndex = imageSegmentIndex+1

		#verify these match:
		#print("len(filterApplicationResultThresholdedList) = ", len(filterApplicationResultThresholdedList))
		#print("len(filterApplicationResultThresholdIndicesList) = ", len(filterApplicationResultThresholdIndicesList))
		#print("len(RFpropertiesList) = ", len(RFpropertiesList))
	else:
		filterApplicationResultThresholdIndicesList = []
		filterApplicationResultThresholdedList = []

	return filterApplicationResultThresholdIndicesList, filterApplicationResultThresholdedList, RFpropertiesList	#filterApplicationResultList

max_stars_repo_name: baxterai/ATORtf

max_issues_repo_name: baxterai/ATORtf

max_forks_repo_name: baxterai/ATORtf


text:
import sys
import numpy as np

from .base import Classifier

class Movelets(Classifier):
    """MOVELETS: exploring relevant subtrajectories for robust trajectory
    classification.

    Parameters
    ----------
    dist_functions : array-like, shape (n_features)
        Specifies the distance functions used for each trajectory
        attribute.
    norm_distances : array-like, shape (n_features) (default=None)
        Specifies the maximum values to use for distance normalization. For
        the i-th feature, if norm_distances[i] <= 0, then the distance is not
        normalized.

    References
    ----------
    `Ferrero, C. A., Alvares, L. O., Zalewski, W., & Bogorny, V. (2018,
    April). MOVELETS: exploring relevant subtrajectories for robust trajectory
    classification. In Proceedings of the 33rd Annual ACM Symposium on Applied
    Computing (pp. 849-856). ACM.
    <https://dl.acm.org/citation.cfm?doid=3167132.3167225>`__
    """

    def __init__(self, dist_functions, norm_distances=None):
        self.distances = dist_functions

        if norm_distances is not None:
            for i in range(0, len(dist_functions)):
                if norm_distances[i] > 0:
                    self.distances[i] = \
                        lambda x, y: self.distances[i] / norm_distances[i]

    def fit(self, X, y):
        for i, traj in enumerate(X):
            break

    def predict(self, X):
        pass

    def score(self, X, y):
        pass

    def _best_alignment(self, subtraj, traj):
        if len(subtraj) > len(traj):
            return sys.float_info.max

        comp = len(traj) - len(subtraj)
        minDist = sys.float_info.max
        position = -1

        for i in range(0, comp + 1):
            curDist = 0
            for j in range(0, len(subtraj)):
                pointDist = 0

                for dist in self.distances:
                    pointDist += dist(subtraj[j], traj[i + j])
                curDist += pointDist * pointDist

            if curDist < minDist:
                minDist = curDist
                position = i

        minDist = np.sqrt(minDist / len(subtraj))
        position = [position, position + len(subtraj)]
        return minDist, position


text:
# Copyright 2017 The Rudders Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import item_graph
import tensorflow as tf
import numpy as np

class TestItemGraph(unittest.TestCase):

    def test_build_graph_similarity_below_threshold(self):
        iids = ["a", "b", "c", "d", "e"]
        cossim_matrix = tf.zeros((len(iids), len(iids)), dtype=tf.float16)

        graph = item_graph.build_graph(iids, cossim_matrix, threshold=0.5, use_distance=False)

        self.assertEqual(len(graph), 0)

    def test_build_graph_use_distance(self):
        iids = ["a", "b", "c", "d", "e"]
        cossim_matrix = tf.ones((len(iids), len(iids)), dtype=tf.float16) * 0.7

        graph = item_graph.build_graph(iids, cossim_matrix, threshold=0.5, use_distance=True)

        self.assertGreater(len(graph), 0)
        for edge in graph.edges(data=True):
            self.assertAlmostEqual(edge[2]["weight"], 0.3, places=3)

    def test_build_graph_no_distance(self):
        iids = ["a", "b", "c", "d", "e"]
        cossim_matrix = tf.ones((len(iids), len(iids)), dtype=tf.float16) * 0.7

        graph = item_graph.build_graph(iids, cossim_matrix, threshold=0.5, use_distance=False)

        self.assertGreater(len(graph), 0)
        for edge in graph.edges(data=True):
            self.assertEqual(edge[2]["weight"], 1)

    def test_build_graph_only_two_nodes(self):
        iids = ["a", "b", "c", "d", "e"]
        cossim_matrix = np.ones((len(iids), len(iids))) * 0.3
        cossim_matrix[0, 1] = 1     # this represents (a, b)
        cossim_matrix[1, 0] = 1     # the matrix should be symmetric
        cossim_matrix = tf.convert_to_tensor(cossim_matrix, dtype=tf.float16)

        graph = item_graph.build_graph(iids, cossim_matrix, threshold=0.5, use_distance=False)

        self.assertEqual(len(graph), 2)
        for edge in graph.edges(data=True):
            self.assertEqual(edge[2]["weight"], 1)
        self.assertIn("a", graph.nodes())
        self.assertIn("b", graph.nodes())

max_stars_repo_name: LeoLaugier/recommendation-rudders

max_issues_repo_name: LeoLaugier/recommendation-rudders

max_forks_repo_name: LeoLaugier/recommendation-rudders


text:
import math
import numpy as np
import matplotlib.pyplot as plt
from numpy.linalg import solve

N = 10
M = 4

X = np.linspace(0, 1, N, endpoint=True)
S = np.sin(2*np.pi*X) + np.random.rand(N)
z = np.linspace(0, 1, N*10, endpoint=True)

a = np.zeros([M,M])
b = np.zeros(M)

for i in range(0, M):
    for j in range(0, M):
        a[i,j] = math.fsum(map(lambda x: x**(i+j), X))
    b[i] = sum([m*n for (m, n) in zip(map(lambda x: x**i, X), S)])

w = solve(a, b)

plt.plot(z, np.polyval(w[::-1], z))
plt.plot(X, S, 'o')
plt.show()

max_stars_repo_name: hyoiutu/PRMLPractice

max_issues_repo_name: hyoiutu/PRMLPractice

max_forks_repo_name: hyoiutu/PRMLPractice

reason: import numpy,from numpy


text:
#!/usr/bin/env python
# coding=utf-8

import pyarrow
import argparse
import pandas as pd
import numpy as np
import random
import sys
import re
import os
from typing import Any, Dict, List, Optional, Union
from tqdm import tqdm
import json
from collections import Counter

import soundfile as sf
import librosa

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F

from transformers import Wav2Vec2CTCTokenizer
from transformers import Wav2Vec2FeatureExtractor
from transformers import Wav2Vec2Processor
from transformers import Wav2Vec2ForCTC
from transformers import TrainingArguments
from datasets import load_dataset, load_metric, Dataset, concatenate_datasets
from dataclasses import dataclass, field
from datasets import ClassLabel

import pyarrow

def print_(*string):
    print(*string)
    sys.stdout.flush()

def fix_seed(seed):
    # Random
    random.seed(seed)
    # Numpy
    np.random.seed(seed)
    # PyTorch
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False

@dataclass
class DataCollatorCTCWithPadding:
    """
    Data collator that will dynamically pad the inputs received.
    Args:
        processor (:class:`~transformers.Wav2Vec2Processor`)
            The processor used for proccessing the data.
        padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`):
            Select a strategy to pad the returned sequences (according to the model's padding side and padding index)
            among:
            * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
              sequence if provided).
            * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the
              maximum acceptable input length for the model if that argument is not provided.
            * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
              different lengths).
        max_length (:obj:`int`, `optional`):
            Maximum length of the ``input_values`` of the returned list and optionally padding length (see above).
        max_length_labels (:obj:`int`, `optional`):
            Maximum length of the ``labels`` returned list and optionally padding length (see above).
        pad_to_multiple_of (:obj:`int`, `optional`):
            If set will pad the sequence to a multiple of the provided value.
            This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >=
            7.5 (Volta).
    """

    processor: Wav2Vec2Processor
    padding: Union[bool, str] = True
    max_length: Optional[int] = None
    max_length_labels: Optional[int] = None
    pad_to_multiple_of: Optional[int] = None
    pad_to_multiple_of_labels: Optional[int] = None

    def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:
        # split inputs and labels since they have to be of different lenghts and need
        # different padding methods
        input_features = [{"input_values": feature["input_values"]} for feature in features]
        label_features = [{"input_ids": feature["labels"]} for feature in features]

        batch = self.processor.pad(
            input_features,
            padding=self.padding,
            max_length=self.max_length,
            pad_to_multiple_of=self.pad_to_multiple_of,
            return_tensors="pt",
        )
        with self.processor.as_target_processor():
            labels_batch = self.processor.pad(
                label_features,
                padding=self.padding,
                max_length=self.max_length_labels,
                pad_to_multiple_of=self.pad_to_multiple_of_labels,
                return_tensors="pt",
            )

        # replace padding with -100 to ignore loss correctly
        labels = labels_batch["input_ids"].masked_fill(labels_batch.attention_mask.ne(1), -100)  ## what is attention_mask.ne(1)?

        batch["labels"] = labels

        return batch

# ------ DATA LOADER --------

def load_LibriSpeech(data_path):

    dev_clean = Dataset.load_from_disk(os.path.join(data_path, "dev-clean"))
    dev_other = Dataset.load_from_disk(os.path.join(data_path, "dev-other"))
    test_clean = Dataset.load_from_disk(os.path.join(data_path, "test-clean"))
    test_other = Dataset.load_from_disk(os.path.join(data_path, "test-other"))

    return dev_clean, dev_other, test_clean, test_other

def load_ARCTIC(split_path, purposes=["train", "dev", "test"], L1="all", \
      removed_ids=[]):
    """Given a list of removed_ids, load the corresponsing train, dev and test
    set where test set has two types: `test small` only contains speakers in
    removed_ids while `test big` contains all speakers of L1 type.

    Args:
        L1 (str) -- L1 type of speakers. Default: "all", means load all speakers
                    except those to be removed
        removed_ids (list) -- A list of speakers to be removed. Default: [],
                    means nothing to be removed and no small test set
    """
    train_set, dev_set, test_set, test_unseen_set = [], [], [], []
    train, dev, test, test_unseen = [], [], [], []

    # Collect file paths
    for d in os.listdir(split_path):
        if d.startswith("."):
            continue
        else:
            speaker_path = os.path.join(split_path, d)
            cur_L1, cur_speaker = d.split("_")
            if L1 == "all":
                test_set.append(os.path.join(speaker_path, "test"))
                if cur_speaker in removed_ids:  # add to unseen; no train/dev
                    test_unseen_set.append(os.path.join(speaker_path, "test"))
                else:  # add to train/dev
                    train_set.append(os.path.join(speaker_path, "train"))
                    dev_set.append(os.path.join(speaker_path, "dev"))
            elif L1 == cur_L1:
                test_set.append(os.path.join(speaker_path, "test"))
                if cur_speaker in removed_ids:  # add to unseen; no train/dev
                    test_unseen_set.append(os.path.join(speaker_path, "test"))
                else:  # add to train/dev
                    train_set.append(os.path.join(speaker_path, "train"))
                    dev_set.append(os.path.join(speaker_path, "dev"))

    # Create dataset objects
    if "train" in purposes:
        print_(f"train loading... {[p.split('/')[-2] for p in train_set]}")
        train = concatenate_datasets(list(map(Dataset.load_from_disk, train_set)))
    if "dev" in purposes:
        print_(f"dev loading... {[p.split('/')[-2] for p in dev_set]}")
        dev = concatenate_datasets(list(map(Dataset.load_from_disk, dev_set)))
    if "test" in purposes:
        print_(f"test loading... {[p.split('/')[-2] for p in test_set]}")
        test = concatenate_datasets(list(map(Dataset.load_from_disk, test_set)))
        if len(test_unseen_set):
            print_(f"test_unseen loading... {[p.split('/')[-2] for p in test_unseen_set]}")
            test_unseen = concatenate_datasets(
                list(map(Dataset.load_from_disk, test_unseen_set))
                )

    # Verify L1
    if not L1 == "all":
        if len(train): assert train[0]["L1"] == L1
        if len(dev): assert dev[0]["L1"] == L1
    print_(f"train {len(train)} dev {len(dev)} test {len(test)} test_unseen {len(test_unseen)}")

    # Remove columns that are not necessary
#    removed_cols = ['L1', 'gender', 'path', 'speaker_id']
    removed_cols = ['path']
    if len(train):
        train = train.remove_columns(removed_cols+['sentence_id'])
    if len(dev):
        dev = dev.remove_columns(removed_cols)
    if len(test):
        test = test.remove_columns(removed_cols)
    if len(test_unseen):
        test_unseen = test_unseen.remove_columns(removed_cols)

#    return {"train": train, "dev": dev, "test": test, "test_unseen": test_unseen}
    return train, dev, test, test_unseen


text:
import numpy as np
import sklearn
import torch.nn as nn
import torch.nn.functional as F
from binaryClassification import loadBinaryModel

class_dict = {
    0: "W",
    1: "N1",
    2: "N2",
    3: "N3",
    4: "REM",
    5: "UNKNOWN"
}

class_to_num_dict = {
    "W":0,
    "N1":1,
    "N2":2,
    "N3":3,
    "REM":4,
    "UNKNOWN":5
}

class binaryHierarchy:

    # classifier order is a list of strings of classifiers order
    # from shallowest to deepest
    def __init__(self, name, classifierOrder, subjectNum, modelTypes):
        self.name = name
        self.order = classifierOrder
        self.subjectNum = subjectNum
        self.models = []
        self.modelTypes = modelTypes
        self.buildModel()

    def buildModel(self):
        for i, s in enumerate(self.order):
            pos = class_to_num_dict[s]
            m = loadBinaryModel(self.modelTypes, self.subjectNum, pos)
            self.models.append(m)

    def predict(self,x):
        numSamples = x.shape[0]
        predictions = []

        classified = False

        for i,sample in enumerate(x):
            # print("on ", i)
            classified = False
            for modelNum, model in enumerate(self.models): # for every binary classifier
                # print("\tmodel", self.order[modelNum])
                ans = model.predict(sample.reshape(1,-1)) # predict class
                # print("\tpredicted", ans)
                if ans: # if is that class
                    # print("adding ", class_to_num_dict[self.order[modelNum]])
                    # exit()
                    predictions.append(class_to_num_dict[self.order[modelNum]]) # save answers
                    classified = True
                    break # stop evaluating lower layers
            if not classified: # sample not classified as any class, set y = -1
                # print("not classified, adding -1")
                predictions.append(-1)
                classified = False

        preds = np.array(predictions)
        # print(preds.shape)
        # print(preds)

        return preds

    def __str__(self):
        string = "Model: " + self.name + " (Subject "+ str(self.subjectNum) + ")"
        for s in self.order:
            string += "\n\t" + s

        return string

class binarySleepNet(nn.Module):
    def __init__(self):
        super().__init__()
        # each input is 3000 long

        self.conv1 = nn.Conv1d(1, 1, 2)
        self.mp1 = nn.MaxPool1d(2)

        self.conv2 = nn.Conv1d(1, 1, 10)
        self.mp2 = nn.MaxPool1d(2)

        self.conv3 = nn.Conv1d(1, 1, 50)
        self.mp3 = nn.MaxPool1d(2)

        self.conv4 = nn.Conv1d(1, 1, 100)
        self.mp4 = nn.MaxPool1d(2)

        self.conv5 = nn.Conv1d(1, 1, 100)
        self.mp5 = nn.MaxPool1d(2)

        # self.conv6 = nn.Conv1d(1, 1, 100)
        # self.mp6 = nn.MaxPool1d(2)

        # self.conv7 = nn.Conv1d(1, 1, 1000)
        # self.mp7 = nn.MaxPool1d(4)

        self.r = nn.ReLU()
        self.fc1 = nn.Linear(12, 1)
        self.logsoftmax = nn.LogSoftmax()

    def forward(self, x):
        x = self.conv1(x)
        x = self.r(x)
        x = self.mp1(x)

        x = self.conv2(x)
        x = self.r(x)
        x = self.mp2(x)

        x = self.conv3(x)
        x = self.r(x)
        x = self.mp3(x)

        x = self.conv4(x)
        x = self.r(x)
        x = self.mp4(x)

        x = self.conv5(x)
        x = self.r(x)
        x = self.mp5(x)

        x = self.fc1(x)
        x = self.logsoftmax(x)
        x = self.r(x)
        return x

class binarySleepNetSimple(nn.Module):
    def __init__(self):
        super().__init__()
        # each input is 3000 long

        self.conv1 = nn.Conv1d(1, 1, 2996)
        self.r = nn.ReLU()
        self.fc1 = nn.Linear(5, 1)
        self.logsoftmax = nn.LogSoftmax()

    def forward(self, x):
        x = self.conv1(x)
        x = self.r(x)
        x = self.fc1(x)
        x = self.r(x)
        x - self.logsoftmax(x)
        return x

max_stars_repo_name: ereoh/Binary-Sleep-Classification

max_issues_repo_name: ereoh/Binary-Sleep-Classification

max_forks_repo_name: ereoh/Binary-Sleep-Classification


text:
import pytest

import numpy as np
import torch
import math

from proxssi.tests import prox_fn
from proxssi.mcp_l2 import _newton_raphson, prox_mcp_l2_weighted

def mcp_l2(lambda_, gamma):
    def _mcp_l2(z):
        norm = torch.linalg.norm(z, dim=(0, 1))
        mask = (norm <= gamma * lambda_)

        result = z.new_full((norm.numel(),), 0.5 * gamma * (lambda_ ** 2))
        result[mask] = lambda_ * norm[mask] - 0.5 * (norm[mask] ** 2) / gamma

        return torch.sum(result)
    return _mcp_l2

def test_newton_raphson():
    x = torch.randn(3, 5).unsqueeze(0)
    weights = torch.rand(3, 5).unsqueeze(0)
    gamma, lr, lambda_ = 1.7, 1.11, 1.13

    # First condition to apply Newton-Raphson: ||x|| <= gamma * lambda
    x.mul_(0.8 * gamma * lambda_ / torch.linalg.norm(x, dim=(0, 1)))
    assert torch.all(torch.linalg.norm(x, dim=(0, 1)) <= (gamma * lambda_))

    # Second condition to apply Newton-Raphson: ||Dx|| > lr * lambda
    weights.mul_(1.2 * lr * lambda_ / torch.linalg.norm(x * weights, dim=(0, 1)))
    assert torch.all(torch.linalg.norm(x * weights, dim=(0, 1)) > (lr * lambda_))

    def theta_function_mcp(theta):
        num = (weights * x) ** 2
        tmp = gamma * weights - lr
        den = theta * tmp + lr * gamma * lambda_
        return (gamma ** 2) * torch.sum(num / (den ** 2), dim=(0, 1)) - 1

    theta = _newton_raphson(x, weights, lr, lambda_, gamma,
        atol=1e-7, rtol=1e-7, max_iters=5)
    func = theta_function_mcp(theta)

    np.testing.assert_allclose(func.abs().numpy(), 0., atol=1e-6, rtol=1e-6)

def test_prox_mcp_l2_weighted_case1():
    x = torch.randn(3, 5).unsqueeze(0)
    weights = torch.rand(3, 5).unsqueeze(0)
    gamma, lr, lambda_ = 1.7, 1.11, 1.13

    # Case 1: ||x|| <= gamma * lambda & ||Dx|| > lr * lambda
    x.mul_(0.8 * gamma * lambda_ / torch.linalg.norm(x, dim=(0, 1)))
    assert torch.all(torch.linalg.norm(x, dim=(0, 1)) <= (gamma * lambda_))

    weights.mul_(1.2 * lr * lambda_ / torch.linalg.norm(x * weights, dim=(0, 1)))
    assert torch.all(torch.linalg.norm(x * weights, dim=(0, 1)) > (lr * lambda_))
    x_0 = x.clone()

    prox = prox_mcp_l2_weighted(x, weights, lr, lambda_, gamma,
        atol=1e-7, rtol=1e-7, max_iters=100)
    prox.requires_grad_()

    # Inplace changes
    np.testing.assert_array_equal(x.detach().numpy(), prox.detach().numpy())

    # Prox is the minimizer of the proximal function
    func = prox_fn(x_0, prox, weights, lr, mcp_l2(lambda_, gamma))
    func.backward()

    np.testing.assert_allclose(torch.linalg.norm(prox.grad, dim=(0, 1)).numpy(), 0.,
                               atol=1e-6)

def test_prox_mcp_l2_weighted_case2():
    x = torch.randn(3, 5).unsqueeze(0)
    weights = torch.rand(3, 5).unsqueeze(0)
    gamma, lr, lambda_ = 1.7, 1.11, 1.13

    # Case 2: ||x|| <= gamma * lambda & ||Dx|| <= lr * lambda
    x.mul_(0.8 * gamma * lambda_ / torch.linalg.norm(x, dim=(0, 1)))
    assert torch.all(torch.linalg.norm(x, dim=(0, 1)) <= (gamma * lambda_))

    weights.mul_(0.8 * lr * lambda_ / torch.linalg.norm(x * weights, dim=(0, 1)))
    assert torch.all(torch.linalg.norm(x * weights, dim=(0, 1)) <= (lr * lambda_))
    x_0 = x.clone()

    prox = prox_mcp_l2_weighted(x, weights, lr, lambda_, gamma,
        atol=1e-7, rtol=1e-7, max_iters=100)

    # Inplace changes
    np.testing.assert_array_equal(x.detach().numpy(), prox.detach().numpy())

    np.testing.assert_array_equal(prox.numpy(), 0.)

def test_prox_mcp_l2_weighted_case3():
    x = torch.randn(3, 5).unsqueeze(0)
    weights = torch.rand(3, 5).unsqueeze(0)
    gamma, lr, lambda_ = 1.7, 1.11, 1.13

    # Case 3: ||x|| > gamma * lambda
    x.mul_(1.2 * gamma * lambda_ / torch.linalg.norm(x, dim=(0, 1)))
    assert torch.all(torch.linalg.norm(x, dim=(0, 1)) > (gamma * lambda_))
    x_0 = x.clone()

    prox = prox_mcp_l2_weighted(x, weights, lr, lambda_, gamma,
        atol=1e-7, rtol=1e-7, max_iters=100)
    prox.requires_grad_()

    # Prox leaves x unchanged
    np.testing.assert_array_equal(prox.detach().numpy(), x_0.numpy())

    # Inplace changes
    np.testing.assert_array_equal(x.detach().numpy(), prox.detach().numpy())

    # Prox is the minimizer of the proximal function
    func = prox_fn(x_0, prox, weights, lr, mcp_l2(lambda_, gamma))
    func.backward()

    np.testing.assert_allclose(prox.grad.norm('fro').abs().item(), 0.,
                               atol=1e-6)

def test_prox_mcp_l2_weighted_multiple():
    x = torch.randn(3, 5).unsqueeze(0)
    weights = torch.rand(3, 5).unsqueeze(0)
    gamma, lr, lambda_ = 1.7, 1.11, 1.13

    norms = torch.linalg.norm(x, dim=(0, 1))

    # Case 1: ||x|| <= gamma * lambda & ||Dx|| > lr * lambda
    factor = torch.ones_like(norms)
    factor[[0, 3, 4]] = 0.8 * gamma * lambda_ / norms[[0, 3, 4]]
    x.mul_(factor)

    norms_weighted = torch.linalg.norm(x * weights, dim=(0, 1))

    # Case 1: ||x|| <= gamma * lambda & ||Dx|| > lr * lambda
    factor = torch.ones_like(norms_weighted)
    factor[[4]] = 1.2 * lr * lambda_ / norms_weighted[[4]]
    weights.mul_(factor)

    # Case 2: ||x|| <= gamma * lambda & ||Dx|| <= lr * lambda
    factor = torch.ones_like(norms_weighted)
    factor[[0, 3]] = 0.8 * lr * lambda_ / norms_weighted[[0, 3]]
    weights.mul_(factor)

    # Case 3: ||x|| > gamma * lambda
    factor = torch.ones_like(norms)
    factor[[1, 2]] = 1.2 * gamma * lambda_ / norms[[1, 2]]
    x.mul_(factor)

    # Check the conditions are satisfied
    norms = torch.linalg.norm(x, dim=(0, 1))
    norms_weighted = torch.linalg.norm(x * weights, dim=(0, 1))
    assert torch.all(norms[[1, 2]] > (gamma * lambda_))
    assert torch.all(norms[[0, 3, 4]] <= (gamma * lambda_))
    assert torch.all(norms_weighted[[4]] > (lr * lambda_))
    assert torch.all(norms_weighted[[0, 3]] <= (lr * lambda_))
    x_0 = x.clone()

    prox = prox_mcp_l2_weighted(x, weights, lr, lambda_, gamma,
        atol=1e-7, rtol=1e-7, max_iters=100)
    prox.requires_grad_()

    # Inplace changes
    np.testing.assert_array_equal(x.detach().numpy(), prox.detach().numpy())

    # Prox is zero for indices [0, 3]
    np.testing.assert_array_equal(prox[..., [0, 3]].detach().numpy(), 0.)

    # Prox is unchanged for indices [1, 2]
    np.testing.assert_array_equal(prox[..., [1, 2]].detach().numpy(),
                                  x_0[..., [1, 2]].numpy())

    # Prox is the minimizer of the proximal function
    func = prox_fn(x_0, prox, weights, lr, mcp_l2(lambda_, gamma))
    func.backward()

    np.testing.assert_allclose(prox.grad[..., [4]].norm('fro').abs().item(), 0.,
                               atol=1e-6)

def test_newton_raphson_convolution():
    x = torch.randn(3, 5, 2, 7).view((3, 5, -1)).transpose(1, 2)
    weights = torch.rand(3, 5, 2, 7).view((3, 5, -1)).transpose(1, 2)
    gamma, lr, lambda_ = 1.7, 1.11, 1.13

    # First condition to apply Newton-Raphson: ||x|| <= gamma * lambda
    x.mul_(0.8 * gamma * lambda_ / torch.linalg.norm(x, dim=(0, 1)))
    assert torch.all(torch.linalg.norm(x, dim=(0, 1)) <= (gamma * lambda_))

    # Second condition to apply Newton-Raphson: ||Dx|| > lr * lambda
    weights.mul_(1.2 * lr * lambda_ / torch.linalg.norm(x * weights, dim=(0, 1)))
    assert torch.all(torch.linalg.norm(x * weights, dim=(0, 1)) > (lr * lambda_))

    def theta_function_mcp(theta):
        num = (weights * x) ** 2
        tmp = gamma * weights - lr
        den = theta * tmp + lr * gamma * lambda_
        return (gamma ** 2) * torch.sum(num / (den ** 2), dim=(0, 1)) - 1

    theta = _newton_raphson(x, weights, lr, lambda_, gamma,
        atol=1e-7, rtol=1e-7, max_iters=5)
    func = theta_function_mcp(theta)

    np.testing.assert_allclose(func.abs().numpy(), 0., atol=1e-6, rtol=1e-6)

def test_prox_mcp_l2_weighted_multiple_convolution():
    x = torch.randn(3, 5, 2, 7).view((3, 5, -1)).transpose(1, 2)
    weights = torch.rand(3, 5, 2, 7).view((3, 5, -1)).transpose(1, 2)
    gamma, lr, lambda_ = 1.7, 1.11, 1.13

    norms = torch.linalg.norm(x, dim=(0, 1))

    # Case 1: ||x|| <= gamma * lambda & ||Dx|| > lr * lambda
    factor = torch.ones_like(norms)
    factor[[0, 3, 4]] = 0.8 * gamma * lambda_ / norms[[0, 3, 4]]
    x.mul_(factor)

    norms_weighted = torch.linalg.norm(x * weights, dim=(0, 1))

    # Case 1: ||x|| <= gamma * lambda & ||Dx|| > lr * lambda
    factor = torch.ones_like(norms_weighted)
    factor[[4]] = 1.2 * lr * lambda_ / norms_weighted[[4]]
    weights.mul_(factor)

    # Case 2: ||x|| <= gamma * lambda & ||Dx|| <= lr * lambda
    factor = torch.ones_like(norms_weighted)
    factor[[0, 3]] = 0.8 * lr * lambda_ / norms_weighted[[0, 3]]
    weights.mul_(factor)

    # Case 3: ||x|| > gamma * lambda
    factor = torch.ones_like(norms)
    factor[[1, 2]] = 1.2 * gamma * lambda_ / norms[[1, 2]]
    x.mul_(factor)

    # Check the conditions are satisfied
    norms = torch.linalg.norm(x, dim=(0, 1))
    norms_weighted = torch.linalg.norm(x * weights, dim=(0, 1))
    assert torch.all(norms[[1, 2]] > (gamma * lambda_))
    assert torch.all(norms[[0, 3, 4]] <= (gamma * lambda_))
    assert torch.all(norms_weighted[[4]] > (lr * lambda_))
    assert torch.all(norms_weighted[[0, 3]] <= (lr * lambda_))
    x_0 = x.clone()

    prox = prox_mcp_l2_weighted(x, weights, lr, lambda_, gamma,
        atol=1e-7, rtol=1e-7, max_iters=100)
    prox.requires_grad_()

    # Inplace changes
    np.testing.assert_array_equal(x.detach().numpy(), prox.detach().numpy())

    # Prox is zero for indices [0, 3]
    np.testing.assert_array_equal(prox[..., [0, 3]].detach().numpy(), 0.)

    # Prox is unchanged for indices [1, 2]
    np.testing.assert_array_equal(prox[..., [1, 2]].detach().numpy(),
                                  x_0[..., [1, 2]].numpy())

    # Prox is the minimizer of the proximal function
    func = prox_fn(x_0, prox, weights, lr, mcp_l2(lambda_, gamma))
    func.backward()

    np.testing.assert_allclose(prox.grad[..., [4]].norm('fro').abs().item(), 0.,
                               atol=1e-6)


text:
# Code that gets the images that are misclassified by our model and convert it into a single figure

import numpy as np
import matplotlib.pyplot as plt
import os

os.chdir('d:\Python Projects\EVA')
cwd = os.getcwd()
img_dir = os.path.join(cwd, 'Assignment-6/saved_images')

def plot_misclassified(misclassified:list):
    plt.figure(figsize=(20,20))
    for plotIndex, badIndex in enumerate(misclassified[0:25]):
        plt.subplot(5, 5, plotIndex + 1,)
        plt.axis('off')
        plt.imshow(np.reshape(misclassified[plotIndex][0].cpu(), (28,28)), cmap=plt.cm.gray)
        plt.title("Predicted: {}, Actual: {}".format(misclassified[plotIndex][1], misclassified[plotIndex][2].cpu().numpy(), fontsize = 20))
        plt.savefig(os.path.join(img_dir,'misclassified.png'))


text:
from PoseEngine import PoseEngine
import numpy as np
from PIL import Image

class pose_detection():
	MODEL_S = 'models/posenet_mobilenet_v1_075_353_481_quant_decoder_edgetpu.tflite'
	MODEL_M = 'models/posenet_mobilenet_v1_075_481_641_quant_decoder_edgetpu.tflite'
	MODEL_L = 'models/posenet_mobilenet_v1_075_721_1281_quant_decoder_edgetpu.tflite'

	def __init__(self, mirror=False, model=MODEL_S):
		self.engine = PoseEngine(model, mirror)
		self.resize = {
			pose_detection.MODEL_S: (481, 353),
			pose_detection.MODEL_M: (641, 481),
			pose_detection.MODEL_L: (1281, 721),
		}[model]
		self.poses = None

	def detect(self, img):
		img = Image.fromarray(img)
		img.resize(self.resize, Image.NEAREST)
		self.poses, _ = self.engine.DetectPosesInImage(np.uint8(img))
		return self.poses

	def get_poses(self):
		return self.poses