text:
# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.

import torch
import torch.optim as optim
import torch.nn as nn
import model
from collections import deque
import numpy as np
from model import repackage_hidden
import train_weights
from .base_mixture_learner import BaseMixtureOfRNNsLearner
from observer import Observable

class CloneLearner(BaseMixtureOfRNNsLearner):
    def __init__(self, rnn_type, consolidate_moment, consolidate_threshold, nin, nout, ninp, nhid, nlayers,
            max_memory_size, max_ltm_size, init_stm_size, lr, batch_size, clip, optimizer, ltm_reinstatement, stm_consolidation,
            train_weights_before_predict, weights_trainer, residual_weights_trainer=None,
            tie_weights=False, w_window=20, recent_window_size=5, dropout=0.2, is_cuda = True):
        self.optimizers = []
        self.consolidate_moment = consolidate_moment
        self.consolidation_threshold = consolidate_threshold
        self.reinstatement_threshold = consolidate_threshold
        self.consolidate_moment_ticks = 1
        self.residual_weights_trainer = None
        self.ltm_size = 0
        self.memory_size = 0
        self.max_ltm_size = max_ltm_size
        self.init_stm_size = init_stm_size
        self.max_memory_size = max_memory_size
        self.stm_consolidation = stm_consolidation
        self.ltm_reinstatement = ltm_reinstatement
        self.recent_window_size = recent_window_size
        self.weight_history = []
        self.weights_copied = Observable()
        self.weights_moved = Observable()
        self.ltm_size_updated = Observable()
        super(CloneLearner, self).__init__(rnn_type, nin, nout, ninp, nhid, nlayers,
            max_memory_size, lr, batch_size, clip, optimizer,
            train_weights_before_predict, weights_trainer, tie_weights, w_window, dropout, is_cuda)
        self.weights_updated.register(self.save_weights_history)

    def cuda(self):
        super(CloneLearner, self).cuda()
        if self.residual_weights_trainer:
            self.residual_weights_trainer = self.residual_weights_trainer.cuda()

    def _initialize_model(self):
        memory_size = self.init_stm_size or self.max_memory_size
        for i in range(memory_size):
            self._create_new_rnn()

    def learn(self, data, targets):
        loss, prediction, outputs = self.predict_and_compute_loss(data, targets)
        self.train_modules(data, outputs, targets)
        self.train_weights(data, self.get_outputs().detach(), targets)
        self.consolidate_if_needed(data)
        return loss

    def train_modules(self, data, outputs, targets):
        if self.residual_weights_trainer:
            get_loss = self.get_loss_closure()
            n_modules = self.get_n_modules()
            self.residual_weights_trainer.optimize(
                    data, get_loss, until_convergence=True)
            weights = self.residual_weights_trainer.get_weights(data)
        else:
            weights = self.get_weights(data)
        prediction = self.get_prediction(weights.detach(), outputs)
        loss = self.get_loss(prediction, targets)
        self.backpropagate_and_train_modules(loss)

    def backpropagate_and_train_modules(self, loss):
        n_modules = self.get_n_modules()
        stm_start, stm_end = self._get_stm_interval()
        for opt in self.optimizers[stm_start:stm_end]:
            opt.zero_grad()
        loss.backward()
        for i,rnn in enumerate(self.rnns[stm_start:stm_end]):
            assert next(rnn.parameters()).requires_grad
            torch.nn.utils.clip_grad_norm_(rnn.parameters(), self.clip)
        for opt in self.optimizers[stm_start:stm_end]:
            opt.step()

    def consolidate_if_needed(self, data):
        if self.consolidate_moment_ticks > 20:
            #weight_diff = self.get_weight_diff(weights)
            #if (self.consolidate_moment and self.consolidate_moment_ticks % self.consolidate_moment == 0) or (self.consolidate_threshold and weight_diff > self.consolidate_threshold):
            self.consolidate_and_reinstate(data)
        self.consolidate_moment_ticks += 1

    def get_weight_diff(self, weights):
        current_weights= weights.detach().cpu().numpy()
        current_weights /= np.linalg.norm(current_weights)
        recent_weights = self.get_recent_weights_mean()
        recent_weights  /= np.linalg.norm(recent_weights)
        #cosine =  np.dot(current_weights, recent_weights) / np.linalg.norm(current_weights) / np.linalg.norm(recent_weights)
        #weight_diff = 1 - cosine
        weight_diff = np.linalg.norm(current_weights - recent_weights)
        return weight_diff

    def consolidate_and_reinstate(self, data):
        ids_at_start = list(self.ids)
        consolidated = self.consolidate(data)
        if consolidated:
            print("Consolidated", self.get_memory_str())
        reinstated = self.reinstate(data)
        if reinstated:
            print("Reinstatated", self.get_memory_str())
            if self.ids == ids_at_start:
                print('Warning: CYCLING')

    def get_memory_str(self):
        return " ".join(self.ids[:self.ltm_size]) + "|" \
                + " ".join(self.ids[self.ltm_size:])

    def consolidate(self, data):
        consolidated = 0
        if self.stm_consolidation == 'fifo':
            raise NotImplementedError()
            #stm_start, stm_end = self._get_stm_interval()
            #self._consolidate_stm2ltm(stm_start)
            #stm_start, stm_end = self._get_stm_interval()
            #self._move_rnn(stm_start, stm_end)
        elif self.stm_consolidation == 'relevance':
            weights = self.get_weights(data).detach().cpu().numpy()
            while self._has_consolidation_candidates(weights):
                stm_idx = self._get_consolidation_candidate(weights)
                self._consolidate_stm2ltm(stm_idx)
                weights = self.get_weights(data).detach().cpu().numpy()
                consolidated += 1
        else:
            raise NotImplementedError(self.stm_consolidation)
        return consolidated

    def _has_consolidation_candidates(self, weights):
        if self._get_stm_size() == 1:
            return False
        consolidation_scores = self._get_consolidation_scores(weights)
        return (consolidation_scores > self.consolidation_threshold).any()

    def _get_consolidation_candidate(self, weights):
        stm_start, stm_end = self._get_stm_interval()
        consolidation_scores = self._get_consolidation_scores(weights)
        return stm_start+consolidation_scores.argmax()

    def _get_consolidation_scores(self, weights):
        stm_start, stm_end = self._get_stm_interval()
        scores = self.get_usage_scores(weights, stm_start, stm_end)
        return -scores

    def _consolidate_stm2ltm(self, stm_idx):
        ltm_start, ltm_end = self._get_ltm_interval()
        self._move_rnn(stm_idx, ltm_end)
        self._freeze_rnn(ltm_end)
        self.ltm_size += 1
        self.ltm_size_updated(self.ltm_size)
        print(f"Moving stm@{stm_idx} to ltm@{ltm_end} (ltm: {self.ltm_size})")
        if self.max_ltm_size is not None and self.ltm_size > self.max_ltm_size:
            self._delete_rnn(ltm_start)
            self.ltm_size -= 1
            self.ltm_size_updated(self.ltm_size)
            print("Reduced LTM size")

    def reinstate(self, data):
        reinstated = False
        if self.ltm_reinstatement == 'fifo':
            raise NotImplementedError()
        elif self.ltm_reinstatement == 'relevance':
            weights = self.get_weights(data).detach().cpu().numpy()
            while self._has_reinstatement_candidates(weights):
                self._reinstate_by_relevance(weights)
                weights = self.get_weights(data).detach().cpu().numpy()
                reinstated += 1
        elif self.ltm_reinstatement == 'none':
            raise NotImplementedError()
        else:
            raise NotImplementedError()
        return reinstated

    def _reinstate_by_relevance(self, weights):
        ltm_idx = self._get_reinstatement_candidate(weights)
        self._reinstate_ltm2stm(ltm_idx)

    def _has_reinstatement_candidates(self, weights):
        if self.ltm_size == 0:
            return False
        reinstatement_scores = self._get_reinstatement_scores(weights)
        return (reinstatement_scores > self.reinstatement_threshold).any()

    def _get_reinstatement_candidate(self, weights):
        ltm_start, ltm_end = self._get_ltm_interval()
        reinstatement_scores = self._get_reinstatement_scores(weights)
        return ltm_start + reinstatement_scores.argmax()

    def _get_reinstatement_scores(self, weights):
        ltm_start, ltm_end = self._get_ltm_interval()
        scores = self.get_usage_scores(weights, ltm_start, ltm_end)
        return scores

    def _reinstate_by_none(self, weights, network_full):
        if network_full:
            ltm_start, ltm_end = self._get_ltm_interval()
            self._delete_rnn(ltm_start)
            self._create_new_rnn()
        else:
            pass  #with this policy we don't reinstate until we are full

    def _reinstate_ltm2stm(self, ltm_idx):
        assert self._is_in_ltm(ltm_idx), ltm_module_idx
        _, stm_end = self._get_stm_interval()
        #self._copy_rnn(ltm_idx, stm_end)
        #ltm_start, _ = self._get_ltm_interval()
        #self.weights_trainer.set_weight_parameters(ltm_idx, 0)
        #self.reset_weight_history(ltm_idx)
        #self._move_rnn(ltm_idx, ltm_start)  # prepare to die, if you need to
        self._move_rnn(ltm_idx, stm_end)
        self._unfreeze_rnn(stm_end-1)
        if self.memory_size > self.max_memory_size:
            #ltm_start, _  = self._get_ltm_interval()
            stm_start, _ = self._get_stm_interval()
            self._delete_rnn(stm_start)
            print("Reduced STM size")
        print(f"Copied ltm@{ltm_idx} to stm@{stm_end} (ltm: {self.ltm_size})")

    def _is_in_ltm(self, idx):
        ltm_start, ltm_end = self._get_ltm_interval()
        return ltm_start <= idx < ltm_end

    def _get_ltm_interval(self):
        return (0, self.ltm_size)

    def _get_stm_interval(self):
        return (self.ltm_size, self.memory_size)

    def _get_stm_size(self):
        stm_start, stm_end = self._get_stm_interval()
        return stm_end - stm_start

    def _freeze_rnn(self, idx):
        for i, param in enumerate(self.rnns[idx].parameters()):
            param.requires_grad = False

    def _unfreeze_rnn(self, idx):
        for i, param in enumerate(self.rnns[idx].parameters()):
            param.requires_grad = True

    def _create_new_rnn(self):
        new_rnn_idx = super(CloneLearner, self)._create_new_rnn()
        self.weight_history.append(deque(maxlen=self.window_size))
        self.reset_weight_history(new_rnn_idx)
        self._create_new_optimizer()
        if self.residual_weights_trainer:
            self.residual_weights_trainer.append_weight()
        self.memory_size += 1
        return new_rnn_idx

    def _create_new_optimizer(self):
        opt_class = getattr(torch.optim, self.optimizer_algorithm)
        opt = opt_class(self.rnns[-1].parameters(), lr=self.lr)
        self.optimizers.append(opt)

    def _delete_rnn(self, rnn_idx):
        super(CloneLearner, self)._delete_rnn(rnn_idx)
        del self.optimizers[rnn_idx]
        if self.residual_weights_trainer:
            self.residual_weights_trainer.delete_weight(rnn_idx)
        self.memory_size -= 1

    def _copy_rnn(self, from_idx, to_idx):
        module_idx = self._create_new_rnn()
        self._copy_parameters(from_idx, module_idx)
        self._move_rnn(module_idx, to_idx)

    def _copy_parameters(self, source, dest):
        self.ids[dest] = self.ids[source] + "b"
        self.rnns[dest].load_state_dict(self.rnns[source].state_dict())
        self.optimizers[dest].load_state_dict(self.optimizers[source].state_dict())
        with torch.no_grad():
            for h_source, h_dest in zip(self.hidden[source], self.hidden[dest]):
                h_dest.copy_(h_source)
        self.weight_history[dest] = self.weight_history[source].copy()
        self.weights_trainer.set_weight_parameters(dest, self.weights_trainer.get_weight_parameters(source))
        self.weights_copied(source, dest)

    def _multiply_weights(self, idx, m):
        self.weights_trainer.set_weight_parameters(idx, self.weights_trainer.get_weight_parameters(idx) * m)

    def _move_rnn(self, from_idx, to_idx):
        if from_idx == to_idx:
            return
        self._move(self.optimizers, from_idx, to_idx)
        self._move(self.ids, from_idx, to_idx)
        self._move(self.rnns, from_idx, to_idx)
        self._move(self.hidden, from_idx, to_idx)
        self._move(self.weight_history, from_idx, to_idx)
        self.weights_trainer.move_weight(from_idx, to_idx)
        self.weights_moved(from_idx, to_idx)
        if self.residual_weights_trainer:
            self.residual_weights_trainer.move_weight(from_idx, to_idx)

    def _move(self, lst, from_idx, to_idx):
        if from_idx < to_idx:
            to_idx -= 1
        lst.insert(to_idx, lst.pop(from_idx))

    def get_lr(self):
        return self.optimizers[-1].param_groups[0]['lr']

    def reset_lr(self):
        self.optimizers[-1].param_groups[0]['lr'] = lr

    def save_weights_history(self, weights):
        for i in range(self.get_n_modules()):
            self.weight_history[i].append(weights[i].item())

    def get_most_relevant_module_in_recent_history(self, start=None, end=None):
        """returns the most relevant module"""
        mean_weights = self.get_mean_recent_weights(start, end)
        return start + np.argmax(np.abs(mean_weights))

    def get_usage_scores(self, weights, start=None, end=None):
        previous_mean_weights = self.get_recent_weights_mean(start, end, 0, self.window_size - self.recent_window_size)
        std_weights = self.get_recent_weights_std(start, end, 0, self.window_size - self.recent_window_size)
        mean_weights = self.get_recent_weights_mean(start, end, self.window_size - self.recent_window_size, self.window_size)
        min_std_size = 0.1
        std_weights[std_weights<min_std_size] = min_std_size
        return (np.abs(mean_weights) - np.abs(previous_mean_weights))/std_weights

    def get_recent_weights_mean(self, start=None, end=None, dt_start=None, dt_end=None):
        if start is None:
            start = 0
        if end is None:
            end = len(self.weight_history)
        if end == start:
            return None
        mean_weights = []
        for i in range(start, end):
            mean_weights.append(np.mean(list(itertools.islice(self.weight_history[i], dt_start, dt_end))))
        return np.array(mean_weights)

    def get_recent_weights_std(self, start=None, end=None, dt_start=None, dt_end=None):
        if start is None:
            start = 0
        if end is None:
            end = len(self.weight_history)
        if end == start:
            return None
        std_weights = []
        for i in range(start, end):
            std_weights.append(np.std(list(itertools.islice(self.weight_history[i], dt_start, dt_end))))
        return np.array(std_weights)

    def reset_weight_history(self, idx):
        self.weight_history[idx].extend([0]*self.window_size)

ext: py

max_stars_repo_name: germank/CommAI-env

max_issues_repo_name: germank/CommAI-env

max_forks_repo_name: germank/CommAI-env

reason: import numpy


text:
#-*- coding: utf-8 -*-
import os
import sys
import re
import time
import unittest
import numpy as np

import paddle

import pgl
from pgl.utils.logger import log

class MathTest(unittest.TestCase):
    """MathTest
    """

    def test_segment_softmax(self):
        data = np.array([[1, 2, 3], [3, 2, 1], [4, 5, 6]], dtype="float32")
        seg_ids = np.array([0, 0, 1], dtype="int64")

        ground = np.array(
            [[0.11920292, 0.5, 0.880797], [0.880797, 0.5, 0.11920292],
             [1, 1, 1]],
            dtype="float32")
        ground = ground.reshape(-1, ).tolist()

        data = paddle.to_tensor(data, dtype='float32')
        segment_ids = paddle.to_tensor(seg_ids, dtype='int64')
        out = pgl.math.segment_softmax(data, segment_ids)
        out = out.numpy().reshape(-1, ).tolist()

        self.assertAlmostEqual(out, ground)

        # test nan
        data = np.array(
            [[1, 2, 0.003], [3, 2, 10000000000], [4, 5, 6]], dtype="float32")
        seg_ids = np.array([0, 0, 1], dtype="int64")

        ground = np.array(
            [[0.11920292, 0.5, 0], [0.880797, 0.5, 1], [1, 1, 1]],
            dtype="float32")
        ground = ground.reshape(-1, ).tolist()

        data = paddle.to_tensor(data, dtype='float32')
        segment_ids = paddle.to_tensor(seg_ids, dtype='int64')
        out = pgl.math.segment_softmax(data, segment_ids)
        out = out.numpy().reshape(-1, ).tolist()

        self.assertAlmostEqual(out, ground)

    def test_segment_padding(self):

        data = np.array([[1, 2, 3], [3, 2, 1], [4, 5, 6]], dtype="float32")
        seg_ids = np.array([0, 0, 2], dtype="int64")

        data = paddle.to_tensor(data, dtype='float32')
        segment_ids = paddle.to_tensor(seg_ids, dtype='int64')

        ground = np.array(
            [[[1, 2, 3], [3, 2, 1]], [[0, 0, 0], [0, 0, 0]],
             [[4, 5, 6], [0, 0, 0]]],
            dtype="float32")

        ground = ground.reshape(-1, ).tolist()
        out, _, _ = pgl.math.segment_padding(data, segment_ids)
        out = out.numpy().reshape(-1, ).tolist()
        self.assertAlmostEqual(out, ground)

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: zbmain/PGL

max_issues_repo_name: zbmain/PGL

max_forks_repo_name: zbmain/PGL


text:
import pandas as pd
import numpy as np
from sklearn.metrics import mean_squared_error
from pyspark.ml.evaluation import RegressionEvaluator,Evaluator
from math import sqrt
from sklearn.metrics.pairwise import pairwise_distances
from sklearn.metrics.pairwise import cosine_similarity

class EvaluadorRMSE(Evaluator):
    """
    Evalua RMSE de forma robusta.
    Es Igual que RegressionEvaluator con metric=rmse pero descartando valores no predecidos

    """
    def __init__(self,predictionCol, targetCol):
        super(EvaluadorRMSE, self).__init__()
        self.predictionCol=predictionCol
        self.targetCol=targetCol

    def _evaluate(self, dataset):
        error=rmse(dataset,self.predictionCol,self.targetCol)
        print ("Error: {}".format(error))
        return error

    def isLargerBetter(self):
        return False

class ModelBasedALS(object):
    """
    Envoltorio para la clase ALS de ml de Spark.
    Da soporte a los metodos de ALS de mllib

    """
    def __init__(self,modelALS):
        super(ModelBasedALS, self).__init__()
        """
        Parametros
        ----------
        modelALS : objeto entrenado de pyspark.ml.recommendation.ALS
        """
        self.userIndex,self.userFactors = self.toArray(modelALS.userFactors)
        self.itemIndex,self.itemFactors = self.toArray(modelALS.itemFactors)
        self.prediccion=pd.DataFrame(data=self.userFactors.dot(self.itemFactors.T),columns=self.itemIndex,index=self.userIndex)

        self.relacion_index_user=dict(zip(self.userIndex,range(len(self.userIndex))))
        self.relacion_index_item=dict(zip(self.itemIndex,range(len(self.itemIndex))))

    def predictAll(self,user_item:pd.DataFrame,tag_prediccion='prediccion'):
        """
        Devuelve todas las predicciones dado el par (user,item)
        """
        estimaciones=[]
        for tupla in user_item.values:
            try:
                estimacion=self.prediccion.iloc[self.relacion_index_user[tupla[0]],self.relacion_index_item[tupla[1]]]
                estimaciones.append(estimacion)
            except:
                estimaciones.append(np.nan)

        user_item[tag_prediccion]=estimaciones
        return user_item

    def recommendProducts(self,user:int,n:int=3):
        """
        Devuelve el top de productos recomendados para el usuario
        """
        usuario=self.prediccion.loc[user]
        usuario.sort(ascending=False)
        return usuario.iloc[:n]

    def recommendUsers(self,product:int,n:int=3):
        """
        Devuelve el top de los usuarios de un producto
        """
        productos=self.prediccion.loc[:,product]
        productos.sort(ascending=False)
        return productos.iloc[:n]

    @staticmethod
    def toArray(datos):
        indices=[]
        lista=[]
        aaa=datos.rdd.map(lambda l:(l.id,l.features)).collect()
        for tupla in aaa:
            indices.append(tupla[0])
            lista.append(tupla[1])

        return indices,np.array(lista)

def rmse(dataset,predictionCol,targetCol):
    valores=np.array(dataset.dropna().map(lambda r:[r[predictionCol],r[targetCol]]).collect())
    error = sqrt(mean_squared_error(valores[:,0],valores[:,1]))
    return error


text:
import pandas as pd
import numpy as np
from sklearn.metrics import accuracy_score
import warnings
from sklearn.metrics import fbeta_score, precision_score, recall_score, confusion_matrix,f1_score
import itertools
import pickle
from matplotlib import pyplot as plt,style
from multiprocessing import Pool
import json
import os
import sys
warnings.simplefilter("ignore", category=DeprecationWarning)
style.use('ggplot')
np.random.seed(42)

label_file = sys.argv[5]
labels = []
with open(label_file) as ff:
    for line in ff.readlines():
        line = line.strip()
        if line.startswith('#') or line == '':
            continue
        labels.append(line)

# TODO: Do not hardcode dictionary. Labels need to be taken from the device.
di ={}
reverse_di = {}
for i in range(len(labels)):
    di.update({labels[i]:i})
    reverse_di.update({i:labels[i]})

di.update({'anomaly':len(labels)})

class NumpyEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.ndarray):
            return obj.tolist()
        return json.JSONEncoder.default(self, obj)

def plot_confusion_matrix(cm, classes,
                          recall,precision,f2,f1,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    plt.figure()
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=0)
    plt.yticks(tick_marks, classes)

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.xticks(rotation=90)
    plt.text(12,0, f" Recall:{recall},\n Precision:{precision},\n F2 Score:{f2},\n F1 Score:{f1}", fontsize=12)
    return plt
    #plt.show()

def load_data(path):
    anomaly_data = pd.read_csv(path)
   # anomaly_data = anomaly_data.drop(anomaly_data.columns[0], axis=1)
    return anomaly_data

def filter_anomaly(ss,anomaly_data,multivariate_model_dict,model_path):
    mv_model = multivariate_model_dict['mvmodel']
    treshold = multivariate_model_dict['treshold']
    y_test = anomaly_data['state'].apply(lambda x: 1 if x == 'anomaly' else 0)
    y_predict = (mv_model.logpdf(anomaly_data.drop(['state'], axis=1).values) < treshold).astype(int)
    recall = recall_score(y_pred=y_predict, y_true=y_test, average='weighted')
    precision = precision_score(y_pred=y_predict, y_true=y_test, average='weighted')
    f2 = fbeta_score(y_pred=y_predict, y_true=y_test, average='weighted', beta=2)
    f1 = f1_score(y_pred=y_predict, y_true=y_test, average='weighted')
    _acc_score = accuracy_score(y_test, y_predict)
    cm = confusion_matrix(y_test, y_predict)
    plt = plot_confusion_matrix(cm, classes=['Normal', 'Anomalous'],
                          recall=recall, precision=precision, f2=f2, f1=f1, title='Confusion matrix')
    if not os.path.exists(model_path+'/plots'):
        os.makedirs(model_path+'/plots')
    plt.savefig(model_path+'/plots/anomalous_cm.png',bbox_inches='tight')

    anomaly_data['anomalous'] = y_predict
    normal_data = anomaly_data[anomaly_data['anomalous'] == 0]
    anomalous_data = anomaly_data[anomaly_data['anomalous'] == 1]
    output_dict = {'predictions': y_predict, 'recall': recall, 'precision': precision, 'f1': f1, 'f2': f2}
    if not os.path.exists(model_path+'/results'):
        os.makedirs(model_path+'/results')
    with open(model_path+'/results/anomaly_output.txt','w') as file:
        file.write(json.dumps(output_dict,cls=NumpyEncoder))

    return normal_data,anomalous_data

def action_classification_model(normal_data,action_class_dict):
    ss = action_class_dict['standard_scaler']
    pca = action_class_dict['pca']
    trained_model = action_class_dict['trained_model']
    transformed_data = ss.transform(normal_data.drop(['state','anomalous'], axis=1))
    #TODO: Fix nan value results from transformations

    transformed_data = pca.transform(transformed_data)
    transformed_data = pd.DataFrame(transformed_data)
    transformed_data = transformed_data.iloc[:, :4]
    y_predict = trained_model.predict(transformed_data)
    y_predicted_1d = np.argmax(y_predict, axis=1)
    normal_data['predictions'] = y_predicted_1d
    return normal_data

def final_accuracy(final_data,model_path):
    global di
    y_test = final_data['state'].map(di)
    y_predict = final_data['predictions']
    return y_predict

def main():
    global di,reverse_di,labels
    data_path = '/Volumes/Abhijit-Seagate/Data_iot/Features/required-features-split/google-home-mini-req-april.csv'
    root_model = '/Volumes/Abhijit-Seagate/Data_iot/results/results_google_mini_idle_req'
    base_model_path = '/Volumes/Abhijit-Seagate/Data_iot/models/new-tagged-models/knn/google-home-miniknn.model'
    anomaly_model_path = '/Volumes/Abhijit-Seagate/Data_iot/tagged-models/anomaly_models/multivariate_model_google_home_mini_req.pkl'

    data_path = sys.argv[1]
    root_model = sys.argv[2]
    base_model_path = sys.argv[3]
    anomaly_model_path = sys.argv[4]
    num_pools = 12
    p = Pool(num_pools)

    anomaly_data = load_data(data_path)
    start_time = anomaly_data['start_time']
    end_time = anomaly_data['end_time']
    anomaly_data = anomaly_data.drop(['device'], axis=1)
    action_classification_model_dict = pickle.load(open(base_model_path,'rb'))
    ss = action_classification_model_dict['standard_scaler']
    anomaly_model = pickle.load(open(anomaly_model_path,'rb'))
    normal_data,anomalous_data = filter_anomaly(ss,anomaly_data, anomaly_model,root_model)
    print("Normal")
    print(normal_data.head())
    print("Abnormal")
    print(anomalous_data.head())
    #TODO: The label for anomalous data will be according to the dictionary count for the device.
    anomalous_data['predictions'] = di['anomaly']
    normal_data = action_classification_model(normal_data,action_classification_model_dict)
    final_data = normal_data.append(anomalous_data).sort_index()
    y_predict = final_accuracy(final_data,root_model)
    out_dict = {'start_time':start_time,'end_time':end_time,'tagged':final_data['state'],'prediction':y_predict}
    out_df = pd.DataFrame(out_dict)
    out_df['prediction'] = out_df['prediction'].map(reverse_di).fillna("anomaly")
    out_df.to_csv(root_model+'/results/model_results.csv', index=False)

if __name__ == '__main__':
    main()


text:
#!/usr/bin/env python
"""Creates a Torch Dataset and Dataloader objects with custom tokenizer.

Create a simple data set and loader to work with PyTorch Dataset and Dataloader class. Wanted to use these instead of
the torch.text alternatives to keep things simple. In addition, using Dataloader in combination with DDP will hopefully
be more efficient with parallel processing and GPUs (yet to test)

"""
import collections
from itertools import product
import logging

import numpy as np
from torch.utils.data import DataLoader
import torch

# Helper Functions
class SeqDataset(torch.utils.data.Dataset):
    """
    A class to represent a dataset to store sequence data.

    ...

    Attributes
    ----------
    file_name: str
        Name of the file that contains one sequence per line
    data: [str]
        List of seqeunces read as lines from the file
    transform: str
        Name of the function(s) used to transform the data
    context_size: int
        The number of kmers or words to consider on either side of the target kmer

    Methods
    -------
    __len__():
        Prints the number of lines or seqeunces in the data
    __getitem__():
        Gets the next seqeunce in the dataset and applies transformtaions on the data
    """

    def __init__(self, file_name, kmer_size = 4, context_size=3, num_noise_words=5, transform=None):

        """Constructs the dataset of sequences
        that should be passed to the negative sampling loss.

        Parameters
        ----------
        file_name: str
                Name of the file that contains one sequence per line
        kmer_size: int
            The size of the kmers to extract from the sequence
        context_size: int
            The number of kmers or words to consider on either side of the target kmer
        num_noise_words: int
            The number of noise words to use for negative sampling
        transform: [torchvision.Transform]
            List of mames of the transformer function(s) used to transform the data
        """

        self.file_name = file_name
        self.data = open(self.file_name, 'r').readlines()
        self.kmer_size = kmer_size
        self.transform = transform
        self.context_size = context_size
        self.num_noise_words = num_noise_words
        self.word_probs = None # Initialize this to None to make sure _build_vocab() does not error out
        self.doc_to_ix, self.word_to_ix, self.ix_to_word, self.word_probs = self._build_vocab()
        self.sample_noise = lambda: np.random.choice(
            self.word_probs.shape[0], self.num_noise_words, p=self.word_probs).tolist()

    def __len__(self):
        """Gets the length or number of sequences in the dataset.

        Returns
        ----------
            len(int): The number of lines in the input file
        """
        return len(self.data)

    def __getitem__(self, idx):
        """Gets the next sequence in the file and extracts kmers by applying the specified transformations
        In addition, also outputs the context for each target kmer.

        Parameters
        ----------
        idx: int
            The index of the next sequence in the file
        Returns
        ----------
        [[str], [str], [[str]]]: List of lists of sequence (document) ids, kmers, and kmer contexts
        """
        sample = self.data[idx]
        seq = sample.split(',')[0]
        doc_id = sample.split(',')[1].rstrip()
        # print(self.word_probs)

        if self.transform:
            # Add padding to the sequence so there is enough for all kmers at the beginning and end to have enough context kmers
            seq = 'X' * self.kmer_size * self.context_size + seq + 'Y' * self.kmer_size * self.context_size
            # Kmerize
            seqs = self.transform(seq) # Kmer tokenize the sequence and get a list of sequence(s) back

            batch = []
            for seq in seqs: # For each sequence
                if self.context_size == 0:
                    batch.append([doc_id, seq, []])

                # Add context kmers
                full_context = [] # The context kmers for each target kmer in this sequence
                for in_doc_pos in range(self.context_size, (len(seq)-self.context_size)): # For each kmer find the context
                    context_indices = (in_doc_pos + diff for diff in
                                       range(-self.context_size, self.context_size + 1)
                                       if diff != 0)

                    current_context = []
                    for i in context_indices:
                        context_kmer = seq[i]
                        current_context.append(context_kmer)
                    full_context.append(current_context)

                # Add noise targets
                if self.word_probs is not None:
                    target_noise_ids = []
                    for in_doc_pos in range(self.context_size, (len(seq) - self.context_size)):
                        current_noise = [self.ix_to_word[no] for no in self.sample_noise()]
                        current_noise.insert(0, seq[in_doc_pos])
                        target_noise_ids.append(current_noise)
                else:
                    target_noise_ids = []

                seq = seq[self.context_size:(len(seq) - self.context_size)]
                batch.append([[doc_id]*len(seq), seq, full_context, target_noise_ids])

        if self.word_probs is None:
            return batch
        else:
            if len(batch) > 1:
                batch_flat = [item for sublist in batch for item in sublist]
            doc_ids = torch.tensor([self.doc_to_ix[item] for i,sublist in enumerate(batch_flat) for item in sublist if i % 4 == 0])
            target_ids = torch.tensor([self.word_to_ix[item] for i,sublist in enumerate(batch_flat) for item in sublist if i % 4 == 1])
            context_ids = torch.tensor([[self.word_to_ix[nested_item] for nested_item in item] for i,sublist in enumerate(batch_flat) for item in sublist if i % 4 == 2])
            target_noise_ids = torch.tensor([[self.word_to_ix[nested_item] for nested_item in item] for i,sublist in enumerate(batch_flat) for item in sublist if i % 4 == 3])

            return (doc_ids, target_ids, context_ids, target_noise_ids)

    def _build_vocab(self):
        """Gets the next sequence in the file and extracts kmers by applying the specified transformations
        In addition, also outputs the context for each target kmer.

        Parameters
        ----------

        Returns
        ----------
        ([str], [str], [[str]]]: List of lists of sequence (document) ids, kmers, and kmer contexts
        """

        loader = DataLoader(self, batch_size=4, shuffle=False, collate_fn=my_collate, num_workers=8)

        # Create vocabulary
        count = 0
        docs = set()
        vocab = set()
        char_set = set()
        vocab_freq = collections.Counter(vocab)
        print('Building Voabulary')

        for i, batch in enumerate(loader):
            batch_flat = [item for sublist in batch for item in sublist]
            batch_kmers = [item for sublist in batch_flat for item in sublist[1]]
            docs.update(set([item for sublist in batch_flat for item in sublist[0]]))
            vocab.update(set(batch_kmers))
            char_set.update(set([char for kmer in batch_kmers for char in kmer]))
            vocab_freq.update(collections.Counter(batch_kmers))
            count += 1

        # Add begin and end kmers
        vocab.add('XXXX')
        vocab.add('YYYY')
        vocab_freq['XXXX'] = 0
        vocab_freq['YYYY'] = 0
        # Add all combinations of X and Y as well
        print(''.join(char_set))
        for nx in range(1,self.kmer_size):
            all_comb = [''.join(i) for i in list(product(''.join(char_set), repeat=self.kmer_size-nx))]
            for comb in all_comb:
                vocab.add('X'*nx + comb)
                vocab_freq['X'*nx + comb] = 0
                vocab.add(comb + 'Y'*nx)
                vocab_freq[comb + 'Y'*nx] = 0

        # Create final word to ix dictionary
        doc_to_ix = {doc: i for i, doc in enumerate(docs)}
        word_to_ix = {word: i for i, word in enumerate(vocab)}
        ix_to_word = {i : word for i, word in enumerate(vocab)}
        probs = noise_distribution(vocab_freq, word_to_ix)

        return doc_to_ix, word_to_ix, ix_to_word, probs

class KmerTokenize(object):
    """A custom tokenizer class to parse the sequences into kmers
    Inspired From: https://github.com/fhalab/embeddings_reproduction/
    ...

    Attributes
    ----------
    k: int
        The size of kmers
    overlap: boolean
        Should kmers be calculated with or without overlap between them
    merge: boolean
        Should the different kmer stretches from the same sequence be merged.

    Methods
    -------
    __call__():
        Function to kmerize the sequence and return them
    """

    def __init__(self, kmer_hypers):
        """Constructs the kmer tokenizer to break a sequence into batch of kmers

        Parameters
        ----------
        kmer_hypers: (dict of {str: int, str: bool, str: bool})
                Name of the file that contains one sequence per line
        """
        self.k = kmer_hypers['k']
        self.overlap = kmer_hypers['overlap']
        self.merge = kmer_hypers['merge']

    def __call__(self, seq):
        """Generates kmers from a sequence and returns a list of kmers

        Parameters
        ----------
        seq: str
            The seqeunce to be broken down into kmers

        Returns
        -------
        [[str]]: List of lists of kmers

        """
        N = len(seq)

        if self.overlap:
            seq = seq.rstrip()
            kmers = [[seq[i:i + self.k] for i in range(N - self.k + 1)]]
        else:
            kmers = [[seq[i:i + self.k] for i in range(j, N - self.k + 1, self.k)]
                    for j in range(self.k)]

        if self.merge:
            return kmers
        else: # Not tested
            kms = []
            for km in kmers:
                kms.append(km)
            return kms

# a simple custom collate function
def my_collate(batch):
    """A custom collation function to process batch items
    The default collate function uses zip to process each sequence at the kmer level.
    The current implementation just separates each list

    Parameters
    ----------
    batch: [[[[str], [str], [[str]]]]]
        Nested list of strings

    """
    data = [item for item in batch]

    return data

def noise_distribution(vocab_freq, word_to_ix):
    """ We use a unigram distribution raised to the 3/4rd power,
    as proposed by T. Mikolov et al. in Distributed Representations
    of Words and Phrases and their Compositionality
    Inspired From: https://github.com/fhalab/embeddings_reproduction/

    """

    probs = np.zeros(len(vocab_freq))

    for word, freq in vocab_freq.items():
        probs[word_to_ix[word]] = freq

    probs = np.power(probs, 0.75)
    probs /= np.sum(probs)

    return probs

max_stars_repo_name: kimoyerr/my-dataloader

max_issues_repo_name: kimoyerr/my-dataloader

max_forks_repo_name: kimoyerr/my-dataloader


text:
import pandas as pd
import numpy as np
from scipy.sparse import csr_matrix
from sklearn.metrics import ndcg_score, roc_auc_score
from collections import Counter
import sys

# df_train = pd.read_csv('/home/jj/Documents/Mangaki/mugen_train.csv', sep=' ', names=('user', 'item', 'v'))
# df = pd.read_csv('/home/jj/Documents/Mangaki/mugen_test.csv', sep=' ', names=('user', 'item', 'v'))
df_train = pd.read_csv('/home/jj/code/cfdr/code/gottlieb_train.csv', sep=' ', names=('user', 'item', 'v'))
df = pd.read_csv('/home/jj/code/cfdr/code/gottlieb_test.csv', sep=' ', names=('user', 'item', 'v'))
# df_train = pd.read_csv('all_one_matrix.tr.txt', sep=' ', names=('user', 'item', 'v'))
# df = pd.read_csv('all_one_matrix.te.txt', sep=' ', names=('user', 'item', 'v'))
df['user_id'] = np.unique(df['user'], return_inverse=True)[1]
df['item_id'] = np.unique(df['item'], return_inverse=True)[1]
encode_user = dict(df[['user', 'user_id']].drop_duplicates().to_numpy())
encode_item = dict(df[['item', 'item_id']].drop_duplicates().to_numpy())
df_train['user_id'] = df_train['user'].map(encode_user)
df_train['item_id'] = df_train['item'].map(encode_item)

n_users = df['user'].nunique()
n_items = df['item'].nunique()
n_entries, _ = df.shape
print(n_users, df['user_id'].max())
print(n_items, df['item_id'].max())
truth = csr_matrix((df['v'] * n_entries, (df['user_id'], df['item_id'])), shape=(n_users, n_items))

for filename in sys.argv[1:]:
    out = pd.read_csv(filename, names=('pred',))
    print(filename, out.shape, df['user_id'].shape)
    sparse = csr_matrix((out['pred'], (df['user_id'], df['item_id'])), shape=(n_users, n_items))

    ndcg_values = []
    ndcg10_values = []
    auc_values = []
    for user_id in df['user_id'].unique():
        test_set = list(set(range(n_items)) - set(df_train.query("user_id == @user_id")['item_id'].tolist()))
        # print(n_items - len(test_set))
        user_truth = truth[user_id, test_set].toarray()
        # print(Counter(user_truth.reshape(-1).tolist()))
        user_pred = sparse[user_id, test_set].toarray()
        ndcg_values.append(ndcg_score(user_truth, user_pred))
        ndcg10_values.append(ndcg_score(user_truth, user_pred, k=10))
        try:
            auc_values.append(roc_auc_score(user_truth.reshape(-1),
                                        user_pred.reshape(-1)))
        except ValueError as e:
            print(e)
            print(df.query("user_id == @user_id").shape)
            print(df.query("user_id == @user_id and v == 1"))
            print(df_train.query("user_id == @user_id"))
            print(user_id, len(test_set), Counter(user_truth.tolist()[0]))
            break
            continue
    print(filename)
    print('ndcg =', np.mean(ndcg_values))
    print('ndcg@10 =', np.mean(ndcg10_values))
    print('auc =', np.mean(auc_values))

max_stars_repo_name: jilljenn/libmf

max_issues_repo_name: jilljenn/libmf

max_forks_repo_name: jilljenn/libmf

reason: import numpy,from scipy


text:
import numpy as np

class Othello:
    # 1 白棋, 2 黑棋, 3 下一步可以走
    direction_x = [-1, -1, -1, 0, 0, 1, 1, 1]
    direction_y = [1, 0, -1, 1, -1, -1, 0, 1]

    def __init__(self, board_size=8):
        self.reset(board_size)

    def reset(self, board_size=8):
        # 白棋開始
        self.cur_turn = 1
        # 初始化board
        self.board_size = board_size
        self.state = np.zeros((board_size, board_size), dtype=int)
        l = board_size // 2 - 1
        r = board_size // 2
        self.state[l][l] = self.state[r][r] = 1
        self.state[l][r] = self.state[r][l] = 2
        # 尋找哪些地方可以走
        self.prepare_move()

    def check(self, x, y, replace):
        for i in range(8):
            step = 0
            while True:
                step += 1
                cur_x = x + Othello.direction_x[i] * step
                cur_y = y + Othello.direction_y[i] * step

                if cur_x < 0 or cur_x >= self.board_size or cur_y < 0 or cur_y >= self.board_size:
                    break
                if self.state[cur_x][cur_y] in [0, 3]:
                    break
                if self.state[cur_x][cur_y] == self.cur_turn:
                    if step == 1:
                        break
                    elif replace:
                        for j in range(1, step):
                            temp_x = x + Othello.direction_x[i] * j
                            temp_y = y + Othello.direction_y[i] * j
                            self.state[temp_x][temp_y] = self.cur_turn
                        break
                    else:
                        return True
        return False

    def prepare_move(self):
        for i in range(self.board_size):
            for j in range(self.board_size):
                if self.state[i][j] in [0, 3] and self.check(i, j, False):
                    self.state[i][j] = 3

    def move(self, x, y):
        if self.state[x][y] != 3:
            return
        self.state[x][y] = self.cur_turn
        for i in range(self.board_size):
            for j in range(self.board_size):
                if self.state[i][j] == 3:
                    self.state[i][j] = 0
        self.check(x, y, True)
        self.cur_turn = self.cur_turn % 2 + 1
        self.prepare_move()

        cant_move, end = self.cant_move_or_end()
        if end == 0 and cant_move:
            self.cur_turn = self.cur_turn % 2 + 1
            self.prepare_move()

    def get_status(self):
        return (self.state == 0).sum(), (self.state == 1).sum(), (self.state == 2).sum(), (self.state == 3).sum()

    def cant_move_or_end(self):
        empty, white, black, choice = self.get_status()
        cant_move = (choice == 0)
        end = 0
        if empty + choice == 0 or white == 0 or black == 0:
            if white > black == 0:
                end = 1
            elif black > white == 0:
                end = 2
            else:
                end = 3
        return cant_move, end


text:
import deepThought.stats.Cdf as Cdf
import deepThought.stats.Pmf as Pmf
import numpy as np
import random
import bisect
import math
import sys

class Logger(object):
    log_level = 3
    @staticmethod
    def warning(message):
        if Logger.log_level > 1:
            Logger.log("warning", message)

    @staticmethod
    def error(message):
        Logger.log("error", message)

    @staticmethod
    def info(message):
        if Logger.log_level > 2:
            Logger.log("info", message)
    @staticmethod
    def debug(message):
        if Logger.log_level > 3:
            Logger.log("debug", message)
    @staticmethod
    def log(log_type, message):
        if log_type == "warning":
            print(PrintColors.WARNING),
            print("WARNING: "),
        elif log_type == "error":
            print(PrintColors.FAIL),
            print("ERROR: "),
        elif log_type == "info":
            print("INFO: "),
        elif log_type == "debug":
            print("DEBUG: ")
        print(message),
        print(PrintColors.ENDC)

class PrintColors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

class TypeConversion(object):
    @staticmethod
    def get_float(value):
        try:
            return float(value)
        except (ValueError, TypeError):
            return None

    @staticmethod
    def get_int(value):
        try:
            return int(value)
        except (ValueError, TypeError):
            return None

def pairs_list(list1, list2):
    """Produce all pairs for the two given lists"""
    for i1 in list1:
        for i2 in list2:
            yield i1, i2

def list_to_ccdf(list):
    array_x_axis, array_y_axis = list_to_cdf(list)
    return [array_x_axis, cdf_to_ccdf(array_y_axis)]

def list_to_cdf(list):
    array_hist = Pmf.MakeHistFromList(list)
    array_cdf = Cdf.MakeCdfFromHist(array_hist)
    array_x_axis, array_y_axis = array_cdf.Render()
    return [array_x_axis, array_y_axis]

def cdf_to_ccdf(p):
    ccdf = []
    for x in p:
        ccdf.append(1-x)
    return ccdf

def _cdf(weights):
    total = sum(weights)
    result = []
    cumsum = 0
    for w in weights:
        cumsum += w
        result.append(cumsum / total)
    return result

def choice(population, weights):
    assert len(population) == len(weights)
    cdf_vals = _cdf(weights)
    x = random.random()
    idx = bisect.bisect(cdf_vals, x)
    return population[idx]

Epsilon = sys.float_info.epsilon
def epsilon_compare(A, B):
    return (np.fabs(A) - np.fabs(B)) < np.finfo(np.double).eps

class DeepThoughtException(Exception):
    pass

class UnfeasibleScheduleException(DeepThoughtException):
    pass

max_stars_repo_name: juliusf/Genetic-SRCPSP

max_issues_repo_name: juliusf/Genetic-SRCPSP

max_forks_repo_name: juliusf/Genetic-SRCPSP


text:
import os, shutil
import sys
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
import numpy as np
from torch.utils.data import RandomSampler
from sklearn.metrics import mean_squared_error as MSE
from sklearn.metrics import mean_absolute_error as MAE
from sklearn.preprocessing import LabelBinarizer
import argparse
from sklearn.model_selection import KFold
from   sklearn.model_selection import train_test_split
from   sklearn.metrics import mean_absolute_error as sk_MAE
from   tabulate import tabulate
import random,time
import torch, seaborn as sns

def parity_plot(true_label, pred_label,error2):
    sns.regplot(true_label, pred_label,color='green')
    plt.title(f'MAE:{error2:.3f}')
    plt.xlabel(f'True avg-voltage')
    plt.ylabel(f'Predicted avg-voltage')
    plt.savefig('RESULTS/average--voltage.png')

def save_results_to_file(test_idx, CRYSTAL_DATA, pred_label, true_label, idx_k_fold=''):
    with open(f'RESULTS/{idx_k_fold}voltage--prediction.csv','w') as outfile:
        outfile.write("low_mpid,high_mpid,avg_voltage,pred_voltage\n")
        counter = 0
        for idx in test_idx:
            low_mpid,high_mpid,_,_ = CRYSTAL_DATA.full_data.iloc[idx]
            pred    = pred_label[counter]
            newline = f'{low_mpid},{high_mpid},{true_label[counter]:.3f},{pred:.3f}\n'
            outfile.write(newline)
            counter+=1
    outdf = pd.read_csv(f'RESULTS/{idx_k_fold}voltage--prediction.csv').sort_values(by=['avg_voltage'])
    print(outdf)

def get_dataset(src_folder):
    cifs     = [x for x in os.listdir(src_folder) if x.endswith('.cif')]

    #---- read-in the csv
    electrodes_file = 'DATA/electrodes.csv'
    df              = pd.read_csv(electrodes_file)
    valid_rows      = np.zeros(df.shape[0])

    lowMPID  = (df.low_mpid+'.cif').values.reshape(-1)
    highMPID = (df.high_mpid+'.cif').values.reshape(-1)

    for i  in range(df.shape[0]):
        if lowMPID[i] in cifs and highMPID[i] in cifs:
            valid_rows[i] = 1
    kept     = np.nonzero(valid_rows)[0].reshape(-1)
    df       = df.loc[kept]

    iontype   = df.battery_id.values
    iontype   = [x.split('_')[1] for x in iontype]
    df['ion'] = iontype
    df        = df[['low_mpid','high_mpid','avg_voltage','ion']]

    return df

def torch_MAE(tensor1,tensor2):
    return torch.mean(torch.abs(tensor1-tensor2))

def output_training(metrics_obj,epoch,estop_val,extra='---'):
    header_1, header_2 = 'MSE | e-stop','MAE | TIME'
    if metrics_obj.c_property in ['is_metal','is_not_metal']:
        header_1,header_2     = 'Cross_E | e-stop','Accuracy | TIME'

    train_1,train_2 = metrics_obj.training_loss1[epoch],metrics_obj.training_loss2[epoch]
    valid_1,valid_2 = metrics_obj.valid_loss1[epoch],metrics_obj.valid_loss2[epoch]

    tab_val = [['TRAINING',f'{train_1:.4f}',f'{train_2:.4f}'],['VALIDATION',f'{valid_1:.4f}',f'{valid_2:.4f}'],['E-STOPPING',f'{estop_val}',f'{extra}']]

    output = tabulate(tab_val,headers= [f'EPOCH # {epoch}',header_1,header_2],tablefmt='fancy_grid')
    print(output)

def train(net,train_loader1,train_loader2,metrics,optimizer, device):
    # TRAINING-STAGE
    net.train()
    for i,High_Low_data in enumerate(zip(train_loader1,train_loader2)):
        data0, data1  = High_Low_data
        data0, data1  = data0.to(device), data1.to(device)

        predictions  = net(data0, data1)
        train_label  = metrics.set_label('training',data0)
        loss         = metrics('training',predictions,train_label,1)
        _            = metrics('training',predictions,train_label,2)

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        metrics.training_counter+=1
    return metrics, net

def evaluate(net,valid_loader1,valid_loader2,metrics, device, with_labels=False):
    # EVALUATION
    net.eval()
    out_labels = [[],[]]
    for i,High_Low_data in enumerate(zip(valid_loader1,valid_loader2)):
        data0, data1  = High_Low_data
        data0, data1  = data0.to(device), data1.to(device)
        with torch.no_grad():
            predictions  = net(data0, data1)

        valid_label        = metrics.set_label('validation',data0)

        if with_labels:
            out_labels[0].append(valid_label.detach())
            out_labels[1].append(predictions.detach())
        else:
            _   = metrics('validation',predictions,valid_label,1)
            _   = metrics('validation',predictions, valid_label,2)
            metrics.valid_counter+=1

    return metrics, net, out_labels

    # metrics.reset_parameters('validation',epoch)
    # scheduler.step()
    # end_time         = time.time()
    # e_time           = end_time-start_time
    # metrics.save_time(e_time)

    # # EARLY-STOPPING
    # early_stopping(metrics.valid_loss2[epoch], net)
    # flag_value = early_stopping.flag_value+'_'*(22-len(early_stopping.flag_value))
    # if early_stopping.FLAG == True:    estop_val = flag_value
    # else:
    #     estop_val        = '@best: saving model...'; best_epoch = epoch+1
    # output_training(metrics,epoch,estop_val,f'{e_time:.1f} sec.')

    # if early_stopping.early_stop:
    #     print("> Early stopping")
    #     break

    # # SAVING MODEL
    # print(f"> DONE TRAINING !")
    # shutil.copy2('MODELS/crystal-checkpoint.pt', f'MODELS/{crystal_property}.pt')

max_stars_repo_name: superlouis/GATGNN-VOLTAGE

max_issues_repo_name: superlouis/GATGNN-VOLTAGE

max_forks_repo_name: superlouis/GATGNN-VOLTAGE


text:
#Returns the edge search number of a graph
import networkx as nx
from mixedSearch import ms
import time

def subdivideEdge(G):
    '''
    Given a graph G, returns a graph in which the
    edges of G are subdivided.

    INPUT
    G: A networkx graph

    OUTPUT
    Ge: The edge subdivision of G
    '''
    Ge = G.copy()

    edges = G.edges()

    #Go through every edge, remove it, make a new node, and make two edge
    for edge in edges:
        Ge.remove_edge(*edge)
        Ge.add_node(str(edge))
        Ge.add_edge(edge[0], str(edge))
        Ge.add_edge(edge[1], str(edge))

    return Ge

def es(G):
    '''
    Given a graph, returns it's edge search number. Works
    on the principle that es(G) = ms(G^e), where G^e is G
    with all of it's edges subdivided.

    INPUT
    G: A networkx graph

    OUTPUT
    esG: The edge search number of G
    '''

    #Subdivide the edges of G
    Ge = subdivideEdge(G)
    esG = ms(Ge)

    return esG

max_stars_repo_name: Jabbath/Search-Number-Dynamic

max_issues_repo_name: Jabbath/Search-Number-Dynamic

max_forks_repo_name: Jabbath/Search-Number-Dynamic

reason: import networkx


text:
import torch

from torch.autograd  import  Function

class ACL(torch.nn.Module):
    def __init__(self,queryvectordim, layers,activations,contextvectordim):
        super(ACL, self).__init__()

        layers=layers+[contextvectordim]

        assert len(layers)==len(activations),"number of activations must be one more than number of layers because the output layer also needs a activation which is usally set to none"
        self.linears = torch.nn.ModuleList()
        p_dim=queryvectordim
        for i in range(len(layers)):
            self.linears.append(torch.nn.Linear(p_dim,int(layers[i])))
            p_dim=int(layers[i])
            if activations[i]=="relu":
                self.linears.append(torch.nn.ReLU())
            elif activations[i]=="sigmoid":
                self.linears.append(torch.nn.Sigmoid())
            elif activations[i]=="leaky_relu":
                self.linears.append(torch.nn.LeakyReLU(0.1))
            elif activations[i]=="none":
                pass
            else:
                raise NotImplementedError(
                    f"`activations {activations[i]}` is only suporrted"
                )

    def forward(self,queryvector,yi):  #所有ilme方法的forward参数需要保持一致 yi是为 minilstm保留的
        x=queryvector
        for l in self.linears:
            x=l(x)
        return x

class NACL(torch.nn.Module):

    def __init__(self, dim):
        super(NACL, self).__init__()
        self.psudo_ctxvector = torch.nn.Parameter(torch.zeros((dim)))
        self.ctx_size=dim
    def forward(self, queryvector,yi):
        expand_size=list(queryvector.shape)[:-1]+[self.ctx_size]
        return self.psudo_ctxvector.expand(expand_size)

class RevGrad(torch.nn.Module):
    def __init__(self, alpha=1.0, *args, **kwargs):
        """
        A gradient reversal layer.
        This layer has no parameters, and simply reverses the gradient
        in the backward pass.
        """
        super().__init__(*args, **kwargs)
        self._alpha = torch.tensor(alpha, requires_grad=False)

    def forward(self, input_):
        return RevGrad_F.apply(input_, self._alpha)
class RevGrad_F(Function):
    @staticmethod
    def forward(ctx, input_, alpha_):
        ctx.save_for_backward(input_, alpha_)
        output = input_
        return output

    @staticmethod
    def backward(ctx, grad_output):  # pragma: no cover
        grad_input = None
        _, alpha_ = ctx.saved_tensors
        if ctx.needs_input_grad[0]:
            grad_input = -grad_output * alpha_
        return grad_input, None

if __name__=="__main__":
    import numpy as np
    import random
    seed=0
    torch.manual_seed(seed)  # 为CPU设置随机种子
    torch.cuda.manual_seed(seed)  # 为当前GPU设置随机种子
    torch.cuda.manual_seed_all(seed)  # if you are using multi-GPU，为所有GPU设置随机种子
    np.random.seed(seed)  # Numpy module.
    random.seed(seed)  # Python random module.

    testmodel1=ACL(256,"512_512","relu_relu_none",256)
    grl=RevGrad()
    testmodel2=ACL(256,"512_512","relu_relu_none",1)
    ilme_parameter=list(testmodel1.parameters())

    test_in=torch.rand(256)
    testout1=testmodel1(test_in,0)
    testout1=grl(testout1)
    testout=testmodel2(testout1,0)
    loss=100*testout.square().mean()
    loss.backward()

    ilme_parameter[0].register_hook(lambda grad: grad * -2)  # double the gradient

    testout1=testmodel1(test_in,0)
    testout1=grl(testout1)
    testout=testmodel2(testout1,0)
    loss=100*testout.square().mean()
    loss.backward()

    with torch.no_grad():
        print(ilme_parameter[0].grad.mean(),ilme_parameter[0].grad.std(),torch.abs(ilme_parameter[0].grad).mean())
        debug=np.array(ilme_parameter[0].grad)
    b=1


text:
import scipy
import pandas as pd
import math
import time
import numpy as np

def main(dataset, k, epsilon, iterations):
	start_time = time.time()

	centroids = dataset.sample(k)

	columns = centroids.columns

	sse = 0
	for i in range(0,iterations):
		clusters = []
		for index, row in centroids.iterrows():
			clusters.append([list(row.tolist())])
		old_sse = sse
		sse = 0
		for index, row in dataset.iterrows():
			closest_centroid, dist_all, dist = get_closest_centroid(row, centroids)
			sse = sse + dist
			old_length = len(clusters[closest_centroid])
			clusters[closest_centroid].append(list(row.values))
			new_length = len(clusters[closest_centroid])
			for c in columns:
				centroids.loc[centroids.index[closest_centroid], c] = ((centroids.iloc[closest_centroid][c] * old_length) + row[c]) / new_length

		ssed = abs(old_sse-sse)
		if(ssed < epsilon):
			end_time = time.time()
			return clusters, centroids, start_time, end_time, sse, ssed, i
		# print(ssed)

		# print(len(clusters[0]), len(clusters[1]))

	end_time = time.time()

	return clusters, centroids, start_time, end_time, sse, ssed, i

def get_closest_centroid(item, centroids):
	columns = centroids.columns
	dist_all = []
	for i in range(0,len(centroids)):
		dist_all.append(euclidean_distance(item, centroids.iloc[i], columns))
	return dist_all.index(min(dist_all)), dist_all, min(dist_all)

def euclidean_distance(item, centroid, columns):
	inner_value = 0
	for c in columns:
		inner_value += (item[c] - centroid[c]) ** 2
	return math.sqrt(inner_value)

reason: import numpy,import scipy


text:
import logging
import numpy as np
from rlberry.agents import Agent
from gym.spaces import Discrete
from rlberry.utils.writers import PeriodicWriter
from rlberry.utils.jit_setup import numba_jit

logger = logging.getLogger(__name__)

class KOVIAgent(Agent):
    """
    A version of Kernelized Optimistic Least-Squares Value Iteration (KOVI),
    proposed by Jin et al. (2020).

    If bonus_scale_factor is 0.0, performs random exploration.

    Parameters
    ----------
    env : Model
        Online model of an environment.
    horizon : int
        Maximum length of each episode.
    feature_map_fn : function(env, kwargs)
        Function that returns a feature map instance
        (rlberry.agents.features.FeatureMap class).
    feature_map_kwargs:
        kwargs for feature_map_fn
    n_episodes : int
        number of episodes
    gamma : double
        Discount factor.
    bonus_scale_factor : double
        Constant by which to multiply the exploration bonus.
    reg_factor : double
        Linear regression regularization factor.

    References
    ----------

    """

    name = 'KOVI'

    def __init__(self,
                 env,
                 horizon,
                 pd_kernel_fn,
                 pd_kernel_kwargs=None,
                 n_episodes=100,
                 gamma=0.99,
                 bonus_scale_factor=1.0,
                 reg_factor=0.1,
                 **kwargs):
        Agent.__init__(self, env, **kwargs)

        self.horizon = horizon
        self.n_episodes = n_episodes
        self.gamma = gamma
        self.bonus_scale_factor = bonus_scale_factor
        self.reg_factor = reg_factor
        pd_kernel_kwargs = pd_kernel_kwargs or {}
        self.pd_kernel = pd_kernel_fn

        #
        if self.bonus_scale_factor == 0.0:
            self.name = 'KOVI-Random-Expl'

        # maximum value
        r_range = self.env.reward_range[1] - self.env.reward_range[0]
        if r_range == np.inf:
            logger.warning("{}: Reward range is infinity. ".format(self.name)
                           + "Clipping it to 1.")
            r_range = 1.0

        if self.gamma == 1.0:
            self.v_max = r_range * horizon
        else:
            self.v_max = r_range * (1.0 - np.power(self.gamma, self.horizon))\
                 / (1.0 - self.gamma)

        #
        assert isinstance(self.env.action_space, Discrete), \
            "KOVI requires discrete actions."

        # attributes initialized in reset()
        self.episode = None
        self.gram_mat = None        # Gram matrix
        self.gram_mat_inv = None    # inverse of Gram matrix
        self.alphas = None          #

        self.reward_hist = None     # reward history
        self.state_hist = None      # state history
        self.action_hist = None     # action history
        self.nstate_hist = None     # next state history

        # aux variables (init in reset() too)
        self._rewards = None

        # default writer
        self.writer = PeriodicWriter(self.name,
                                     log_every=15)

        #
        self.reset()

    def reset(self, **kwargs):
        self.episode = 0
        self.total_time_steps = 0
        self.targets = [[] for _ in range(self.horizon)]
        self.gram_mat = [np.array([self.reg_factor]) for _ in range(self.horizon)]  # self.reg_factor * np.eye(self.dim)
        self.gram_mat_inv = [np.array([1.0 / self.reg_factor]) for _ in range(self.horizon)]
        self.alphas = [np.array([1]) for _ in range(self.horizon)]

        self.reward_hist = [[] for _ in range(self.horizon)]
        self.state_hist = [[] for _ in range(self.horizon + 1)]
        self.action_hist = [[] for _ in range(self.horizon)]
        self.nstate_hist = []

        # episode rewards
        self._rewards = np.zeros(self.n_episodes)

    def bonus(self, step, state, action):
        # b_h^t (s, a)
        kh = self.score_vector(step, state, action)

        tmp1 = self.pd_kernel((state, action))

        print('kh', kh)
        print('gram inv', self.gram_mat_inv[step])
        if self.episode == 0:
            tmp2 = kh * self.gram_mat_inv[step] * kh
        else:
            tmp2 = kh.T @ self.gram_mat_inv[step] @ kh

        bonus = np.sqrt((tmp1 - tmp2) / self.reg_factor)
        # bonus = np.sqrt((self.pd_kernel((state, action)) - kh.T @ self.gram_mat_inv[step] @ kh) / self.reg_factor)
        return bonus

    def score_vector(self, step, state, action):
        # k_h^t (s, a)
        previous_states = self.state_hist[step]
        previous_actions = self.action_hist[step]

        similarities = [self.pd_kernel((s, a), (state, action)) for s, a in zip(previous_states, previous_actions)]
        return np.array(similarities)

    def q_function(self, step, state, action):
        # Q_h^t (s, a)
        mean = self.score_vector(step, state, action).T @ self.alphas[step]
        bonus = self.bonus(step, state, action)
        Q = mean + self.bonus_scale_factor * bonus

        return max(min(Q, self.horizon - step + 1), 0)

    def value_function(self, step, state):
        # V_h^t (s)
        if isinstance(state, list):
            n_states = len(state)

            if step == self.horizon:
                return np.zeros(n_states)
            else:
                return np.array([np.max([self.q_function(step, s, a) for a in range(self.env.action_space.n)]) for s in state])

        else:
            if step == self.horizon:
                return 0
            else:
                return np.max([self.q_function(step, state, a) for a in range(self.env.action_space.n)])

    def policy(self, state, **kwargs):
        return

    def _optimistic_policy(self, step, state):
        return np.argmax([self.q_function(step, state, a) for a in range(self.env.action_space.n)])

    def fit(self, **kwargs):
        info = {}
        for ep in range(self.n_episodes):
            state = self.env.reset()

            print('Episode: {}'.format(ep))
            self.episode = ep
            # run episode
            print('Run episode --')
            for step in range(self.horizon):

                print('Step: {}'.format(step))

                if ep == 0:
                    action = 0
                else:
                    action = self._optimistic_policy(step, state)
                next_state, reward, is_terminal, _ = self.env.step(action)

                # update history
                self.reward_hist[step].append(reward)
                self.state_hist[step].append(state)
                self.action_hist[step].append(action)

                self._rewards[ep] += reward

                state = next_state

            # run kovi
            print('Run KOVI --')
            for step in range(self.horizon - 1, -1, -1):
                print('Step: {}'.format(step))

                # y_h^t
                self.targets[step] = self.reward_hist[step] + self.value_function(step + 1, self.state_hist[step])

                if ep > 0:
                    # K_h^t
                    Kaa = self.gram_mat[step]
                    Kab = self.score_vector(step, self.state_hist[step][-1], self.action_hist[step][-1])[:-1].reshape(-1, 1)
                    Kba = Kab.reshape(1, -1)
                    Kbb = np.array([self.pd_kernel((self.state_hist[step][-1], self.action_hist[step][-1]))])

                    tmp1 = np.hstack((Kaa, Kab))
                    tmp2 = np.hstack((Kba, Kbb))

                    self.gram_mat[step] = np.vstack((tmp1, tmp2))

                    # (K_h^t)^-1
                    Kdd = 1.0 / (Kbb + self.reg_factor - Kba @ self.gram_mat_inv[step] @ Kab)
                    Kcc = self.gram_mat_inv[step] + Kdd * self.gram_mat_inv[step] @ np.outer(Kab, Kab) @ self.gram_mat_inv[step]
                    Kcd = - Kdd * self.gram_mat_inv[step] @ Kab
                    Kdc = - Kdd * Kba @ self.gram_mat_inv[step]
                    self.gram_mat_inv[step] = np.vstack((np.hstack((Kcc, Kcd)), np.hstack((Kdc, Kdd))))

                # alpha_h^t
                self.alphas[step] = self.gram_mat_inv[step] @ self.targets[step]
                if ep == 0:
                    self.alphas[step] = np.array([self.alphas[step]])

        info['n_episodes'] = self.n_episodes
        info['episode_rewards'] = self._rewards
        return info

max_stars_repo_name: antoine-moulin/rlberry

max_issues_repo_name: antoine-moulin/rlberry

max_forks_repo_name: antoine-moulin/rlberry

reason: import numpy,import numba


text:
import numpy as np
from lib.config import NUM_PIXELS, PIXEL_PIN, ORDER

MIN_VAL = 10
PREV_VAL = None

lookback_size = 8
lookback = []
def update(samples):
    fft = np.abs(np.fft.fft(samples))
    fft = np.clip((fft / np.amax(fft)) * 255, MIN_VAL, 255)
    if len(lookback) < lookback_size:
        lookback.append(fft)
    else:
        lookback.pop(0)
        lookback.pop(0)
        lookback.pop(0)
        lookback.append(fft)
        lookback.append(fft)
        lookback.append(fft)
    fft = np.mean(lookback, axis=0).astype(int)
    val = np.mean(fft[10])
    g = np.empty(len(fft))
    r = np.empty(len(fft))
    b = np.empty(len(fft))
    for i in range(len(g)):
        if i % 5 == 0:
            # red
            r[i] = val
            g[i] = 0
            b[i] = 0
        elif i % 5 == 1:
            # green
            r[i] = 0
            g[i] = val
            b[i] = 0
        elif i % 5 == 2:
            # blue
            r[i] = 0
            g[i] = 0
            b[i] = val
        elif i % 5 == 3:
            # yellow
            r[i] = val
            g[i] = val
            b[i] = 0
        elif i % 5 == 4:
            # pink
            r[i] = val
            g[i] = int(68 / 255 * val)
            b[i] = int(204 / 255 * val)
    return (g, r, b)

max_stars_repo_name: raymondjacobson/rockridge-lights

max_issues_repo_name: raymondjacobson/rockridge-lights

max_forks_repo_name: raymondjacobson/rockridge-lights


text:
from .TriangularL import *
from .TriangularC import *
from .CuadrilateroL import *
from .SerendipityC import *
from .FEM import progressbar as progressbar
from .FEM import *
from tqdm import tqdm
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits import mplot3d
from matplotlib import tri
import matplotlib as mpl
import subprocess
mpl.style.use('default')

class NoLocal(FEM):
    def __init__(this, geometria):
        super().__init__(2 * len(geometria.gdls))
        this.importarGeometria(geometria)

    def generarElementos(this, gauss=3):
        i = -1
        for d, t in zip(this.geometria.diccionarios, this.geometria.tipos):
            i += 1
            coords = np.array(this.geometria.gdls)[np.ix_(d)]
            if t == 'T1V' or t == 'T2V':
                if len(d) == 3:
                    k = np.array(d)
                    k1 = k * 2
                    k2 = k * 2 + 1
                    d1 = np.array(k1.tolist() + k2.tolist())
                    this.elementos.append(TriangularL(coords, d1, gauss=gauss))
                elif len(d) == 6:
                    k = np.array(d)
                    k1 = k * 2
                    k2 = k * 2 + 1
                    d1 = np.array(k1.tolist() + k2.tolist())
                    this.elementos.append(TriangularC(coords, d1, gauss=gauss))
                else:
                    raise 'algo anda mal'
            elif t == 'C1V' or t == 'C2V':
                if len(d) == 8:
                    k = np.array(d)
                    k1 = k * 2
                    k2 = k * 2 + 1
                    d1 = np.array(k1.tolist() + k2.tolist())

                    this.elementos.append(SerendipityC(
                        coords, np.array(d1), gauss=gauss))
                elif len(d) == 4:
                    k = np.array(d)
                    k1 = k * 2
                    k2 = k * 2 + 1
                    d1 = np.array(k1.tolist() + k2.tolist())
                    this.elementos.append(
                        CuadrilateroL(coords, d1, gauss=gauss))
            else:
                raise Exception(
                    'No se pudo crear el elemento con coordenadas ' + format(coords))
            try:
                this.elementos[-1].elementosnl = this.geometria.diccionariosnl[i]
            except:
                this.elementos[-1].elementosnl = None
        if this.elementos[-1].elementosnl == None:
            for e in this.elementos:
                e.elementosnl = np.linspace(
                    0, len(this.elementos)-1, len(this.elementos)).astype(int).tolist()
        PUNTOS = [-np.sqrt(3.0/5.0), 0, np.sqrt(3.0/5.0)]
        for e in this.elementos:
            e._dominioNaturalZ = PUNTOS
            e._dominioNaturalN = PUNTOS
        this.K = np.zeros([this.n, this.n])
        this.F = np.zeros([this.n, 1])
        this.Q = np.zeros([this.n, 1])
        this.U = np.zeros([this.n, 1])
        this.S = np.zeros([this.n, 1])

    def importarMatricesCarpeta(this, RUTA_M):
        print('Importando matrices...')
        for i, e in tqdm(enumerate(this.elementos), total=len(this.elementos), unit='Elemento'):
            n = len(e.coords)
            K = np.loadtxt(RUTA_M+'/Elemento'+format(i+1)+'/KL_'+format(i+1) +
                           '.csv', delimiter=',').reshape([int(2*n), int(2*n)])
            Q = np.zeros([2 * n, 1])
            F = np.zeros([2 * n, 1])
            e.determinarMatrices(K, F, Q)
            knls = []
            j = 0
            KNLS = np.loadtxt(RUTA_M+'/Elemento'+format(i+1) +
                              '/KNLS.csv', delimiter=',')
            for enl in e.elementosnl:
                KNL = KNLS[j].reshape([int(2*n), int(2*n)]).T
                j += 1
                knls.append(KNL)
            e.KNLS = knls
        # PUNTOS = [-np.sqrt(3.0/5.0),0,np.sqrt(3.0/5.0)]
        # for e in this.elementos:
            # e._dominioNaturalZ = PUNTOS
            # e._dominioNaturalN = PUNTOS
        this.K = np.zeros([this.n, this.n])
        this.F = np.zeros([this.n, 1])
        this.Q = np.zeros([this.n, 1])
        this.U = np.zeros([this.n, 1])
        this.S = np.zeros([this.n, 1])
        print('Las matrices se han cargado con éxito')

    def calcularMatrices(this, E, v, Fx, Fy, t):
        def distancia(x0, y0, x1, y1): return np.sqrt(
            (x1 - x0) ** 2 + (y1 - y0) ** 2)
        count = 0
        for e in this.elementos:
            count += 1
            progressbar(count, len(this.elementos),
                        prefix="Integrando elementos ", size=50)

            def psi(z, n, k): return e.psi[k](z, n)
            dzpsi = e.dzpsi
            dnpsi = e.dnpsi
            gauss = e.gauss
            _J = e._J
            J = e.J
            x = e.Tx
            y = e.Ty

            n = len(e.coords)

            Kuu = np.zeros([n, n])
            Kuv = np.zeros([n, n])
            Kvu = np.zeros([n, n])
            Kvv = np.zeros([n, n])

            Fu = np.zeros([n, 1])
            Fv = np.zeros([n, 1])

            K = np.zeros([2 * n, 2 * n])
            Q = np.zeros([2 * n, 1])
            F = np.zeros([2 * n, 1])
            C11 = E / (1 - v ** 2)
            C12 = v * E / (1 - v ** 2)
            C66 = E / 2 / (1 + v)
            print(J(0, 0))
            for i in range(n):
                for j in range(n):
                    def dfdx(n, z, k):
                        jacobiano = _J(n, z)
                        return dzpsi[k](n, z) * jacobiano[0][0] + dnpsi[k](n, z) * jacobiano[0][1]

                    def dfdy(n, z, k):
                        jacobiano = _J(n, z)
                        return dzpsi[k](n, z) * jacobiano[1][0] + dnpsi[k](n, z) * jacobiano[1][1]

                    def EKuu(z, n): return (C11 * dfdx(z, n, i) * dfdx(z, n, j) + C66 * dfdy(z, n, i) * dfdy(z, n,
                                                                                j)) * np.linalg.det(
                        J(z, n))

                    def EKuv(z, n): return (C12 * dfdx(z, n, i) * dfdy(z, n, j) + C66 * dfdy(z, n, i) * dfdx(z, n,
                                                                                j)) * np.linalg.det(
                        J(z, n))

                    def EKvu(z, n): return (C12 * dfdy(z, n, i) * dfdx(z, n, j) + C66 * dfdx(z, n, i) * dfdy(z, n,
                                                                                j)) * np.linalg.det(
                        J(z, n))

                    def EKvv(z, n): return (C11 * dfdy(z, n, i) * dfdy(z, n, j) + C66 * dfdx(z, n, i) * dfdx(z, n,
                                                                                j)) * np.linalg.det(
                        J(z, n))

                    Kuu[i, j] = e.intGauss2D(gauss, EKuu)
                    Kuv[i, j] = e.intGauss2D(gauss, EKuv)
                    Kvu[i, j] = e.intGauss2D(gauss, EKvu)
                    Kvv[i, j] = e.intGauss2D(gauss, EKvv)

                def EFu(z, n): return (Fx(x(z, n), y(z, n))) * \
                    np.linalg.det(J(z, n))

                def EFv(z, n): return (Fy(x(z, n), y(z, n))) * \
                    np.linalg.det(J(z, n))

                Fu[i] = e.intGauss2D(gauss, EFu)
                Fv[i] = e.intGauss2D(gauss, EFv)

            MD = np.linspace(0, 2 * n - 1, 2 * n).reshape([2, n]).astype(int)
            K[np.ix_(MD[0], MD[0])] = Kuu
            K[np.ix_(MD[0], MD[1])] = Kuv
            K[np.ix_(MD[1], MD[0])] = Kvu
            K[np.ix_(MD[1], MD[1])] = Kvv
            np.savetxt(f'KL_{count}.csv', t*K, delimiter=',')

            F[np.ix_(MD[0])] = Fu
            F[np.ix_(MD[1])] = Fv
            Knl = []
            countnl = 0
            dzpsis = e.dzpsi
            dnpsis = e.dnpsi
            psi = e.psi
            for p in e.elementosnl:
                enl = this.elementos[p]
                countnl += 1
                psisnl = e.psis
                dzpsisnl = enl.dzpsis
                dnpsisnl = enl.dnpsis
                gaussnl = enl.gauss
                _Jnl = enl._J
                Jnl = enl.J
                xnl = enl.Tx
                ynl = enl.Ty

                nnl = len(psisnl(0, 0))

                Kuunl = np.zeros([nnl, nnl])
                Kuvnl = np.zeros([nnl, nnl])
                Kvunl = np.zeros([nnl, nnl])
                Kvvnl = np.zeros([nnl, nnl])

                Knlmn = np.zeros([2 * nnl, 2 * nnl])

                C11 = E / (1 - v ** 2)

                C12 = v * E / (1 - v ** 2)

                C66 = E / 2 / (1 + v)

                def A(X, XP, l=0.1, t=0.5):
                    l0 = (1)/(2*np.pi*l**2*t)
                    return l0 * np.exp(-distancia(x(X[0], X[1]), y(X[0], X[1]), xnl(XP[0], XP[1]), ynl(XP[0], XP[1])) / l)
                dzpsisnl = enl.dzpsi
                dnpsisnl = enl.dnpsi
                psinl = enl.psi
                for i in range(n):
                    for j in range(nnl):

                        def FUNCIONES(z, n, znl, nnl):

                            jacobianonl = Jnl(znl, nnl)
                            jacobiano_nl = _Jnl(znl, nnl)
                            jacobiano_ = _J(z, n)
                            jacobiano = J(z, n)

                            detjacnl = np.linalg.det(jacobianonl)
                            detjac = np.linalg.det(jacobiano)
                            dz_i = dzpsis[i](z, n)
                            dn_i = dnpsis[i](z, n)
                            psis_i = psi[i](z, n)

                            dfdx_i = dz_i * \
                                jacobiano_[0][0] + dn_i * jacobiano_[0][1]
                            dfdy_i = dz_i * \
                                jacobiano_[1][0] + dn_i * jacobiano_[1][1]

                            dznl_j = dzpsisnl[j](znl, nnl)
                            dnnl_j = dnpsisnl[j](znl, nnl)
                            psisnl_j = psinl[j](znl, nnl)

                            dfdxnl_j = dznl_j * \
                                jacobiano_nl[0][0] + \
                                dnnl_j * jacobiano_nl[0][1]
                            dfdynl_j = dznl_j * \
                                jacobiano_nl[1][0] + \
                                dnnl_j * jacobiano_nl[1][1]

                            AZN = A([z, n], [znl, nnl])

                            EKuunl = AZN * \
                                (C11 * dfdx_i * dfdxnl_j + C66 *
                                 dfdy_i * dfdynl_j) * detjac * detjacnl
                            EKuvnl = AZN * \
                                (C12 * dfdx_i * dfdynl_j + C66 *
                                 dfdy_i * dfdxnl_j) * detjac * detjacnl
                            EKvunl = AZN * \
                                (C12 * dfdy_i * dfdxnl_j + C66 *
                                 dfdx_i * dfdynl_j) * detjac * detjacnl
                            EKvvnl = AZN * \
                                (C11 * dfdy_i * dfdynl_j + C66 *
                                 dfdx_i * dfdxnl_j) * detjac * detjacnl
                            return np.array([EKuunl, EKuvnl, EKvunl, EKvvnl])
                        matrices = e.intGauss4DNL(
                            gaussnl, FUNCIONES, shape=[4, 1])
                        Kuunl[i, j] = matrices[0]
                        Kuvnl[i, j] = matrices[1]
                        Kvunl[i, j] = matrices[2]
                        Kvvnl[i, j] = matrices[3]

                Knlmn[np.ix_(MD[0], MD[0])] = Kuunl
                Knlmn[np.ix_(MD[0], MD[1])] = Kuvnl
                Knlmn[np.ix_(MD[1], MD[0])] = Kvunl
                Knlmn[np.ix_(MD[1], MD[1])] = Kvvnl
                Knl.append(Knlmn)
                progressbar(countnl, len(e.elementosnl),
                            prefix="Integrando elementos No Locales   ", size=50)

            e.determinarMatrices(t*K, F, Q)
            e.KNLS = t**2*np.array(Knl)
            KNLS__ = []

            for i in range(len(e.KNLS)):
                KNLS__.append(e.KNLS[i].T.reshape([1, int(4*n**2)])[0])
            np.savetxt(f'KNLS_{count}.csv', KNLS__, delimiter=',')

    def graficarSolucion(this, figsize=[12, 7], cmap='magma', linewidth=2, markersize=2, mask=None, mult=10):
        fig = plt.figure(figsize=figsize)
        ax = fig.add_subplot()
        count = 0
        X_0 = []
        Y_0 = []
        X_F = []
        Y_F = []
        for e in this.elementos:
            count += 1

            Ue = this.U[np.ix_(e.gdl)]

            coords = e.coords

            e.coordsNuevas = np.array(
                coords) + Ue.T.reshape([2, len(e.psis(0, 0))]).T * mult
            coordsNuevas = e.coordsNuevas.tolist()
            coords = np.array(coords)

            X = np.array(e.coords)[:, 0].tolist()
            X.append(e.coords[0][0])
            Y = np.array(e.coords)[:, 1].tolist()
            Y.append(e.coords[0][1])
            e._coordenadas = np.array([X, Y]).T

            X = np.array(e.coordsNuevas)[:, 0].tolist()
            X.append(e.coordsNuevas[0][0])
            Y = np.array(e.coordsNuevas)[:, 1].tolist()
            Y.append(e.coordsNuevas[0][1])
            e._coordsNuevas = np.array([X, Y]).T

            X_0 = e._coordenadas[:, 0].tolist()
            Y_0 = e._coordenadas[:, 1].tolist()

            X_F = e._coordsNuevas[:, 0].tolist()
            Y_F = e._coordsNuevas[:, 1].tolist()

            progressbar(count, len(this.elementos),
                        prefix="Graficando Solucion ", size=50)
            ax.plot(X_0, Y_0, 'o--', color='gray', zorder=0)
            ax.plot(X_F, Y_F, 'o-', color='black', zorder=3)
        ax.set_xlabel('x')
        ax.set_ylabel('y')
        ax.set_title('Solución de elementos finitos')
        ax.legend(['Situacion Inicial', 'Deformada'])

    def defUnitariaX(this, figsize):
        count = 0
        fig = plt.figure(figsize=figsize)
        ax = fig.add_subplot(projection='3d')
        markersize = 2
        cmap = 'magma'
        linewidth = 2
        xtotal = []
        ytotal = []
        ztotal = []
        count = 0
        print('Graficando deformaciones...')
        for e in tqdm(this.elementos, unit='Elemento'):
            count += 1
            x, y, u = postProcesoX(e, this.U)
            xtotal.extend(x)
            ytotal.extend(y)
            ztotal.extend(u)
        surf = ax.plot_trisurf(xtotal, ytotal, ztotal, cmap=cmap, zorder=1)
        cbar = fig.colorbar(surf)
        ax.set_xlabel('x')
        ax.set_ylabel('y')
        ax.set_zlabel(r'$\varepsilon x$')
        ax.set_title(r'$\frac{\partial U}{\partial X}$')
        print('Deformaciones graficadas')
        # plt.show()
        return xtotal, ytotal, ztotal

    def defUnitariaXMOD(this, x0, xf, xx):
        count = 0
        fig = plt.figure()
        ax = fig.add_subplot(projection='3d')
        markersize = 2
        cmap = 'magma'
        linewidth = 2
        xtotal = []
        ytotal = []
        ztotal = []
        count = 0
        print('Graficando deformaciones...')
        for e in tqdm(this.elementos, unit='Elemento'):
            count += 1
            x, y, u = postProcesoX(e, this.U)
            xtotal.extend(x)
            ytotal.extend(y)
            ztotal.extend(u)
        surf = ax.plot_trisurf(xtotal, ytotal, ztotal,
                               cmap=cmap, zorder=1, alpha=0.3)
        _Y = np.linspace(x0, xf, 100).tolist()
        Z = []
        Y = []
        Zlocal = []
        print('Graficando perfil')
        for y in tqdm(_Y):
            for i in range(len(this.elementos)):
                e = this.elementos[i]
                if e.estaDentro(y, xx):
                    U = postProcesoXNodos(e, this.U)
                    Y.append(y)
                    Z.append(e.darSolucionXY(U, y, xx, n=100)[0])
                    break
        ax.plot(Y, [xx]*len(Y), Z)

        ax.set_xlabel('x')
        ax.set_ylabel('y')
        print('Deformaciones graficadas')

    def defUnitariaY(this, figsize):
        count = 0
        fig = plt.figure(figsize=figsize)
        ax = fig.add_subplot(projection='3d')
        markersize = 2
        cmap = 'magma'
        linewidth = 2
        xtotal = []
        ytotal = []
        ztotal = []
        count = 0
        for e in this.elementos:
            count += 1
            x, y, u = postProcesoY(e, this.U)
            xtotal.extend(x)
            ytotal.extend(y)
            ztotal.extend(u)
        surf = ax.plot_trisurf(xtotal, ytotal, ztotal, cmap=cmap, zorder=1)
        cbar = fig.colorbar(surf)
        ax.set_xlabel('x')
        ax.set_ylabel('y')
        ax.set_zlabel(r'$\varepsilon x$')
        ax.set_title(r'$\frac{\partial U}{\partial Y}$')

    def darDeformacionXY(this, U, x, y, n=100):
        this.Ue = U[np.ix_(this.gdl)]
        this.U = lambda z, n: this.Ue.T[0] @ this.psis(z, n)
        zeta, eta = this.mappingInverso(x, y, n).reshape(1, 2)[0].tolist()
        return this.U(zeta, eta)

    def ensamblar(this):
        z1 = this.z1
        z2 = 1 - z1
        print('Ensamblando...')
        for e in tqdm(this.elementos, unit='Elemento'):
            this.K[np.ix_(e.gdl, e.gdl)] += e.Ke*z1
            for i, enl in enumerate(np.array(this.elementos)[e.elementosnl]):
                this.K[np.ix_(e.gdl, enl.gdl)] += e.KNLS[i]*z2
            this.F[np.ix_(e.gdl)] += e.Fe
            this.Q[np.ix_(e.gdl)] += e.Qe
        this._K = np.copy(this.K)
        print('Sistema de ecuaciones ensamblado')

    def perfilX(Objeto_FEM, x, yi=0.00016, yf=0.0004, x0=0, xf=1, filename='', acum=False, label=''):
        _Y = np.linspace(x0, xf, 100).tolist()
        Z = []
        Y = []
        Zlocal = []
        print('Graficando perfil')
        for y in tqdm(_Y):
            for i in range(len(Objeto_FEM.elementos)):
                e = Objeto_FEM.elementos[i]
                if e.estaDentro(x, y):
                    print(i, x, y)
                    U = postProcesoXNodos(e, Objeto_FEM.U)
                    try:
                        Z.append(e.darSolucionXY(U, x, y, n=100))
                        Y.append(y)
                    except:
                        print('No se pudo graficár la coordenada',
                              x, y, "del elemento", i)
                    break
        plt.plot(Y, Z, label=label)
        print('Perfil graficado')

        if acum:
            plt.grid()
            plt.legend()
            plt.xlabel('y')
            if not filename == '':
                plt.savefig(filename, transparent=True)
            # plt.show()

    def perfilY(Objeto_FEM, y, yi=0.00016, yf=0.0004, x0=0, xf=50, filename='', acum=False, label=''):
        _X = np.linspace(x0, xf, 100).tolist()
        Z = []
        X = []
        Zlocal = []
        print('Graficando perfil')
        for x in tqdm(_X):
            for i in range(len(Objeto_FEM.elementos)):
                e = Objeto_FEM.elementos[i]
                if e.estaDentro(x, y):
                    print(i, x, y)
                    U = postProcesoXNodos(e, Objeto_FEM.U)
                    try:
                        Z.append(e.darSolucionXY(U, x, y, n=100))
                        X.append(x)
                    except:
                        print('No se pudo graficár la coordenada',
                              x, y, "del elemento", i)
                    break
        plt.plot(X, Z, label=label)
        print('Perfil graficado')

        if acum:
            plt.legend()
            plt.grid()
            plt.xlabel('x')
            if not filename == '':
                plt.savefig(filename, transparent=True)
            # plt.show()
        return X, Z

    def moduloIntegrador(this, enmallado, npg, E, V, t, l, RUTA_M, tfa=1):
        subprocess.run("index.exe"+" " + enmallado + " "+format(int(npg))+" "+format(E)+" "+format(
            V)+" "+format(t)+" "+format(l)+" "+RUTA_M+" "+format(int(tfa)), shell=True, check=True)

    def moduloIntegradorParalelo(this, enmallado, npg, E, V, t, l, RUTA_M, tfa=1):
        subprocess.Popen("index.exe"+" " + enmallado + " "+format(int(npg))+" "+format(
            E)+" "+format(V)+" "+format(t)+" "+format(l)+" "+RUTA_M+" "+format(int(tfa)))

def postProcesoX(this, U):
    this.Ue = U[np.ix_(this.gdl)]
    this._Ue = this.Ue.T[0].tolist()
    this._Ue.append(this.Ue[0][0])
    Z = this._dominioNaturalZ
    N = this._dominioNaturalN
    x = []
    y = []
    u = []
    this.U = lambda z, n: grad(this, z, n)[0]
    for z, n in zip(Z, N):
        x.append(this.Tx(z, n)[0])
        y.append(this.Ty(z, n)[0])
        u.append(this.U(z, n)[0])
    return x, y, u

def postProcesoXNodos(this, U):
    this.Ue = U[np.ix_(this.gdl)]
    this._Ue = this.Ue.T[0].tolist()
    this._Ue.append(this.Ue[0][0])
    Z = this.ZNatural
    N = this.NNatural
    u = []
    this.U = lambda z, n: grad(this, z, n)[0]
    for z, n in zip(Z, N):
        u.append(this.U(z, n)[0])
    return np.array(u).reshape([len(Z), 1])

def postProcesoY(this, U):
    this.Ue = U[np.ix_(this.gdl)]
    this._Ue = this.Ue.T[0].tolist()
    this._Ue.append(this.Ue[0][0])
    Z = this._dominioNaturalZ
    N = this._dominioNaturalN
    x = []
    y = []
    u = []
    for z, n in zip(Z, N):
        x.append(this.Tx(z, n)[0])
        y.append(this.Ty(z, n)[0])
        u.append(grad(this, z, n)[1][0])
    return x, y, u

def grad(this, z, n):
    dz = this.dzpsis(z, n)
    dn = this.dnpsis(z, n)
    result = []
    _j = this._J(z, n)
    for i in range(len(dz)):
        result.append(_j @ np.array([[dz[i][0]], [dn[i][0]]]))
    result = np.array(result)
    n = len(result)
    U = np.linspace(0, n-1, n).astype(int)
    V = U*2+1
    dx = (this.Ue[np.ix_(U)].T @ result[:, 0])[0]
    dy = (this.Ue[np.ix_(U)].T @ result[:, 1])[0]
    return np.array([dx, dy])

max_stars_repo_name: ZibraMax/Tesis-NLFEM

max_issues_repo_name: ZibraMax/Tesis-NLFEM

max_forks_repo_name: ZibraMax/Tesis-NLFEM