text:
import numpy as np
from scipy.stats import gumbel_r

have_sklearn = True
try:
    from .base import (sklearn_regression,
                       sklearn_regression_lagrange,
                       sklearn_classifier,
                       sklearn_classifier_lagrange)
except ImportError:
    have_sklearn = False

from ..smooth.glm import (glm,
                          cloglog_loglike)

if have_sklearn:
    class sklearn_cloglog(sklearn_regression):

        """

        A simple regression mixin for sklearn
        that allows any atom to be used as a regularizer.

        """

        def _loglike_factory(self, X, y):
            response, case_weights_, offset_ = self._check_y_arg(y)

            if response.ndim == 2:
                successes = response[:,0]
                trials = response[:,1]
            else:
                successes = response
                trials = None

            return glm.cloglog(X,
                               successes,
                               trials=trials,
                               case_weights=case_weights_,
                               coef=self.coef,
                               saturated_offset=offset_)

        def _saturated_score(self,
                             predictions,
                             response,
                             case_weights=None):

            if response.ndim == 2:
                successes = response[:,0]
                trials = response[:,1]
            else:
                successes = response
                trials = None

            loss = lambda yhat: cloglog_loglike(successes.shape,
                                                successes,
                                                trials=trials,
                                                case_weights=case_weights).smooth_objective(yhat, 'func')

            # factor of 2 to form proper deviance (default is negative log-likelihood,
            # while deviance is 2 * negative log-likelihood
            # negative sign is to align with sklearn's maximizing a score with grid search

            if self.score_method == 'deviance':
                return -2 * loss(predictions)
            elif self.score_method == 'mean_deviance':
                return -2 * loss(predictions)
            elif self.score_method == 'R2':
                SSE = 2 * loss(predictions)
                pi_0 = response.mean()
                cloglog_0 = gumbel_r.ppf(pi_0)
                SST = 2 * loss(cloglog_0 * np.ones_like(response)) # X: correct for cloglog?
                return 1 - SSE / SST
            elif self.score_method == 'accuracy':
                labels = predictions > 0
                return np.mean(labels == response)
            else:
                return np.nan

    class sklearn_cloglog_lagrange(sklearn_regression_lagrange, sklearn_cloglog):
        pass

    class sklearn_cloglog_classifier(sklearn_classifier):

        """

        A simple regression mixin for sklearn
        that allows any atom to be used as a regularizer.

        """

        def _loglike_factory(self, X, y):
            response, case_weights_, offset_ = self._check_y_arg(y)

            if response.ndim == 2:
                successes = response[:,0]
                trials = response[:,1]
            else:
                successes = response
                trials = None

            return glm.cloglog(X,
                               successes,
                               trials=trials,
                               case_weights=case_weights_,
                               coef=self.coef,
                               saturated_offset=offset_)

        def _saturated_score(self,
                             predictions,
                             response,
                             case_weights=None):

            if response.ndim == 2:
                successes = response[:,0]
                trials = response[:,1]
            else:
                successes = response
                trials = None

            if self.score_method == 'accuracy':
                return np.mean(predictions == successes)
            return np.nan

        def predict_proba(self, X):
            """
            Predict new probabilities in classification setting.

            Parameters
            ----------

            X : np.ndarray((n, p))
                Feature matrix.

            Returns
            -------

            probs : np.ndarray(n)
                Predictions from classification model.

            """
            linpred = X.dot(self._coefs)
            return 1 - np.exp(-np.exp(linpred))

    class sklearn_cloglog_classifier_lagrange(sklearn_cloglog_classifier):
        pass

max_stars_repo_name: vishalbelsare/regreg

max_issues_repo_name: vishalbelsare/regreg

max_forks_repo_name: vishalbelsare/regreg

reason: import numpy,from scipy


text:
import numpy as np
import re
from torch.utils.data import Dataset
import time
from src.dataloader_utils import AA_DICT, MASK_DICT, DSSP_DICT, NUM_DIMENSIONS
from itertools import compress

class Dataset_pnet(Dataset):
    def __init__(self, file, transform=None, transform_target=None, transform_mask=None, max_seq_len=300):
        id,seq,pssm,entropy,dssp,r1,r2,r3,mask = parse_pnet(file,max_seq_len=max_seq_len)
        self.file = file
        self.id = id
        self.seq = seq
        self.pssm = pssm
        self.entropy = entropy
        self.dssp = dssp
        self.mask = mask
        self.r1 = r1
        self.r2 = r2
        self.r3 = r3

        self.transform = transform
        self.transform_target = transform_target
        self.transform_mask = transform_mask
        self.nfeatures = 84

    def __getitem__(self, index):
        features = (self.seq[index], self.pssm[index], self.entropy[index])
        mask = self.mask[index]
        target = (self.r1[index], self.r2[index], self.r3[index], mask)

        if self.transform is not None:
            features = self.transform(features)
        if self.transform_target is not None:
            target = self.transform_target(target)
        if self.transform_mask is not None:
            mask = self.transform_mask(mask) #TODO CHECK THAT THIS IS NOT DOUBLE FLIPPED!

        return features, target, mask

    def __len__(self):
        return len(self.seq)

    def __repr__(self):
        return self.__class__.__name__ + ' (' + self.file + ')'

def separate_coords(full_coords, pos):  # pos can be either 0(n_term), 1(calpha), 2(cterm)
    res = []
    for i in range(len(full_coords[0])):
        if i % 3 == pos:
            res.append([full_coords[j][i] for j in range(3)])

    return res

def flip_multidimensional_list(list_in):  # pos can be either 0(n_term), 1(calpha), 2(cterm)
    list_out = []
    ld = len(list_in)
    for i in range(len(list_in[0])):
        list_out.append([list_in[j][i] for j in range(ld)])
    return list_out

class switch(object):
    """Switch statement for Python, based on recipe from Python Cookbook."""

    def __init__(self, value):
        self.value = value
        self.fall = False

    def __iter__(self):
        """Return the match method once, then stop"""
        yield self.match

    def match(self, *args):
        """Indicate whether or not to enter a case suite"""
        if self.fall or not args:
            return True
        elif self.value in args:  # changed for v1.5
            self.fall = True
            return True
        else:
            return False

def letter_to_num(string, dict_):
    """ Convert string of letters to list of ints """
    patt = re.compile('[' + ''.join(dict_.keys()) + ']')
    num_string = patt.sub(lambda m: dict_[m.group(0)] + ' ', string)
    num = [int(i) for i in num_string.split()]
    return num

def letter_to_bool(string, dict_):
    """ Convert string of letters to list of bools """
    patt = re.compile('[' + ''.join(dict_.keys()) + ']')
    num_string = patt.sub(lambda m: dict_[m.group(0)] + ' ', string)
    num = [bool(int(i)) for i in num_string.split()]
    return num

def read_record(file_, num_evo_entries):
    """ Read all protein records from pnet file. """

    id = []
    seq = []
    pssm = []
    entropy = []
    dssp = []
    coord = []
    mask = []

    t0 = time.time()
    while True:
        next_line = file_.readline()
        for case in switch(next_line):
            if case('[ID]' + '\n'):
                id.append(file_.readline()[:-1])
                if len(id) % 1000 == 0:
                    print("loading sample: {:}, Time: {:2.2f}".format(len(id),time.time() - t0))
            elif case('[PRIMARY]' + '\n'):
                seq.append(letter_to_num(file_.readline()[:-1], AA_DICT))
            elif case('[EVOLUTIONARY]' + '\n'):
                evolutionary = []
                for residue in range(num_evo_entries):
                    evolutionary.append([float(step) for step in file_.readline().split()])
                pssm.append(evolutionary)
                entropy.append([float(step) for step in file_.readline().split()])
            elif case('[SECONDARY]' + '\n'):
                dssp.append(letter_to_num(file_.readline()[:-1], DSSP_DICT))
            elif case('[TERTIARY]' + '\n'):
                tertiary = []
                for axis in range(NUM_DIMENSIONS): tertiary.append([float(coord) for coord in file_.readline().split()])
                coord.append(tertiary)
            elif case('[MASK]' + '\n'):
                mask.append(letter_to_bool(file_.readline()[:-1], MASK_DICT))
            elif case(''):

                return id,seq,pssm,entropy,dssp,coord,mask

def parse_pnet(file,max_seq_len=-1):
    with open(file, 'r') as f:
        t0 = time.time()
        id, seq, pssm, entropy, dssp, coords, mask = read_record(f, 20)
        print("loading data complete! Took: {:2.2f}".format(time.time()-t0))
        r1 = []
        r2 = []
        r3 = []
        pssm2 = []
        for i in range(len(pssm)): #We transform each of these, since they are inconveniently stored
            pssm2.append(flip_multidimensional_list(pssm[i]))
            r1.append(separate_coords(coords[i], 0))
            r2.append(separate_coords(coords[i], 1))
            r3.append(separate_coords(coords[i], 2))
            if i+1 % 1000 == 0:
                print("flipping and separating: {:}, Time: {:2.2f}".format(len(id), time.time() - t0))

        args = (id, seq, pssm2, entropy, dssp, r1,r2,r3, mask)
        if max_seq_len > 0:
            filter = np.full(len(seq), True, dtype=bool)
            for i,seq_i in enumerate(seq):
                if len(seq_i) > max_seq_len:
                    filter[i] = False
            new_args = ()
            for list_i in (id, seq, pssm2, entropy, dssp, r1,r2,r3, mask):
                new_args += (list(compress(list_i,filter)),)
        else:
            new_args = args
        print("parse complete! Took: {:2.2f}".format(time.time() - t0))
    return new_args

ext: py

max_stars_repo_name: OphirGenomica/proteinFolding

max_issues_repo_name: OphirGenomica/proteinFolding

max_forks_repo_name: OphirGenomica/proteinFolding

reason: import numpy


text:
"""
__author__: Jiaming Shen
__description__: Feature extractor with human-designed features
"""
import numpy as np
from gensim.models import KeyedVectors
import networkx as nx
import random

class NegativeQueue(object):
    def __init__(self, queue):
        self.pointer = 0
        self.queue = queue
        random.shuffle(queue)

    def sample(self, query, negative_size):
        if self.pointer == 0:
            random.shuffle(self.queue)

        negatives = [ele for ele in self.queue[self.pointer: self.pointer+negative_size] if ele != query]

        self.pointer += negative_size
        if self.pointer >= len(self.queue):
            self.pointer = 0

        return negatives

    def sample_avoid_positive_set(self, positive_set, negative_size):
        """ positive_set: a set of elements should not be sampled """
        if self.pointer == 0:
            random.shuffle(self.queue)

        negatives = [ele for ele in self.queue[self.pointer: self.pointer+negative_size] if ele not in positive_set]

        self.pointer += negative_size
        if self.pointer >= len(self.queue):
            self.pointer = 0

        return negatives

class FeatureExtractor(object):
    def __init__(self, graph, kv):
        """ Conduct feature extraction

        Parameters
        ----------
        graph : networkx graph
            input full taxonomy graph
        kv : gensim.KeyedVector
            node embedding
        """
        self.graph = graph
        self.kv = kv

    def extract_features(self, query_node, parent_node):
        """
        query_node: a node index
        parent_node: the parent node index of query node

        return: feature vector of <query_node, parent_node> pair
        """
        neighbor = []
        neighbor.append(parent_node)

        grand_parents = [edge[0] for edge in self.graph.in_edges(parent_node)]
        num_gp = len(grand_parents)
        neighbor.extend(grand_parents)

        siblings = [edge[1] for edge in self.graph.out_edges(parent_node) if edge[1] != query_node]
        neighbor.extend(siblings)

        # calculate embedding distances
        distances = self.kv.distances(str(query_node), [str(ele) for ele in neighbor])
        p_distances = self.kv.distances(str(parent_node), [str(ele) for ele in neighbor])

        """
        calculate features
        """
        feat = []
        parent_distance = distances[0]
        # feature 1: query distance to parent node
        feat.append(parent_distance)

        # feature 2-9: query distance to grand parent node(s)
        grand_parent_distances = distances[1:1+num_gp]
        if len(grand_parent_distances) > 0:
            cnt_gp = len(grand_parent_distances)
            min_gp = np.min(grand_parent_distances)
            max_gp = np.max(grand_parent_distances)
            mid_gp = np.median(grand_parent_distances)
            avg_gp = np.mean(grand_parent_distances)
            std_gp = np.std(grand_parent_distances)
            ptp_gp = max_gp - min_gp
            parent_relative_rank = (grand_parent_distances < parent_distance).sum() / cnt_gp
            feat.extend([cnt_gp, min_gp, max_gp, mid_gp, avg_gp, std_gp, ptp_gp, parent_relative_rank])
        else:
            feat.extend([0, -999, -999, -999, -999, -999, -999, -999])

        # feature 10-17: query distance to sibling node(s)
        sibling_distances = distances[1+num_gp:]
        if len(sibling_distances) > 0:
            cnt_sib = len(sibling_distances)
            min_sib = np.min(sibling_distances)
            max_sib = np.max(sibling_distances)
            mid_sib = np.median(sibling_distances)
            avg_sib = np.mean(sibling_distances)
            std_sib = np.std(sibling_distances)
            ptp_sib = max_sib - min_sib
            sibling_relative_rank = (sibling_distances < parent_distance).sum() / cnt_sib
            feat.extend([cnt_sib, min_sib, max_sib, mid_sib, avg_sib, std_sib, ptp_sib, sibling_relative_rank])
        else:
            feat.extend([0, -999, -999, -999, -999, -999, -999, -999])

        # feature 18-25: parent distance to grand parent distance(s)
        parent_to_grand_parent_distances = p_distances[1:1+num_gp]
        if len(parent_to_grand_parent_distances) > 0:
            cnt_pgp = len(parent_to_grand_parent_distances)
            min_pgp = np.min(parent_to_grand_parent_distances)
            max_pgp = np.max(parent_to_grand_parent_distances)
            mid_pgp = np.median(parent_to_grand_parent_distances)
            avg_pgp = np.mean(parent_to_grand_parent_distances)
            std_pgp = np.std(parent_to_grand_parent_distances)
            ptp_pgp = max_pgp - min_pgp
            relative_rank = (parent_to_grand_parent_distances < parent_distance).sum() / cnt_pgp
            feat.extend([cnt_pgp, min_pgp, max_pgp, mid_pgp, avg_pgp, std_pgp, ptp_pgp, relative_rank])
        else:
            feat.extend([0, -999, -999, -999, -999, -999, -999, -999])

        # feature 26-33: parent distance to sibling distance(s)
        parent_to_sibling_distances = p_distances[1+num_gp:]
        if len(parent_to_sibling_distances) > 0:
            cnt_psib = len(parent_to_sibling_distances)
            min_psib = np.min(parent_to_sibling_distances)
            max_psib = np.max(parent_to_sibling_distances)
            mid_psib = np.median(parent_to_sibling_distances)
            avg_psib = np.mean(parent_to_sibling_distances)
            std_psib = np.std(parent_to_sibling_distances)
            ptp_psib = max_psib - min_psib
            relative_rank = (parent_to_sibling_distances < parent_distance).sum() / cnt_psib
            feat.extend([cnt_psib, min_psib, max_psib, mid_psib, avg_psib, std_psib, ptp_psib, relative_rank])
        else:
            feat.extend([0, -999, -999, -999, -999, -999, -999, -999])

        # feature 34-39: query to all local subgraph distances
        min_all = np.min(distances)
        max_all = np.max(distances)
        mid_all = np.median(distances)
        avg_all = np.mean(distances)
        std_all = np.std(distances)
        ptp_all = max_all - min_all
        feat.extend([min_all, max_all, mid_all, avg_all, std_all, ptp_all])

        return feat

    def extract_features_fast(self, query_node, parent_node, ego2parent, ego2children, parent_node_info, all_distances, tx_id2rank_id, rank_id2tx_id):
        """ Optimized version of feature extraction used in the prediction stage.

        More information are cached to reduce repetitive computation. c.f. model_prediction.py script for details

        query_node: a node index
        parent_node: the parent node index of query node

        return: feature vector of <query_node, parent_node> pair
        """

        neighbor = []
        neighbor.append(parent_node)

        grand_parents = ego2parent[parent_node]
        num_gp = len(grand_parents)
        neighbor += grand_parents

        siblings = ego2children[parent_node]
        neighbor += siblings

        # calculate embedding distances
        # distances = kv.distances(str(query_node), [str(ele) for ele in neighbor])
        # p_distances = kv.distances(str(parent_node), [str(ele) for ele in neighbor])
        distances = all_distances[np.array([tx_id2rank_id[tx_id] for tx_id in neighbor])]
        p_distances = parent_node_info['p_distances']

        """
        calculate features
        """
        feat = []
        parent_distance = distances[0]
        # feature 1: query distance to parent node
        feat.append(parent_distance)

        # feature 2-9: query distance to grand parent node(s)
        grand_parent_distances = distances[1:1+num_gp]
        if len(grand_parent_distances) > 0:
            cnt_gp = len(grand_parent_distances)
            min_gp = np.min(grand_parent_distances)
            max_gp = np.max(grand_parent_distances)
            mid_gp = np.median(grand_parent_distances)
            avg_gp = np.mean(grand_parent_distances)
            std_gp = np.std(grand_parent_distances)
            ptp_gp = max_gp - min_gp
            parent_relative_rank = (grand_parent_distances < parent_distance).sum() / cnt_gp
            feat.extend([cnt_gp, min_gp, max_gp, mid_gp, avg_gp, std_gp, ptp_gp, parent_relative_rank])
        else:
            feat.extend([0, -999, -999, -999, -999, -999, -999, -999])

        # feature 10-17: query distance to sibling node(s)
        sibling_distances = distances[1+num_gp:]
        if len(sibling_distances) > 0:
            cnt_sib = len(sibling_distances)
            min_sib = np.min(sibling_distances)
            max_sib = np.max(sibling_distances)
            mid_sib = np.median(sibling_distances)
            avg_sib = np.mean(sibling_distances)
            std_sib = np.std(sibling_distances)
            ptp_sib = max_sib - min_sib
            sibling_relative_rank = (sibling_distances < parent_distance).sum() / cnt_sib
            feat.extend([cnt_sib, min_sib, max_sib, mid_sib, avg_sib, std_sib, ptp_sib, sibling_relative_rank])
        else:
            feat.extend([0, -999, -999, -999, -999, -999, -999, -999])

        # feature 18-25: parent distance to grand parent distance(s)
        parent_to_grand_parent_distances = p_distances[1:1+num_gp]
        if len(parent_to_grand_parent_distances) > 0:
            cnt_pgp = len(parent_to_grand_parent_distances)
            min_pgp = np.min(parent_to_grand_parent_distances)
            max_pgp = np.max(parent_to_grand_parent_distances)
            mid_pgp = np.median(parent_to_grand_parent_distances)
            avg_pgp = np.mean(parent_to_grand_parent_distances)
            std_pgp = np.std(parent_to_grand_parent_distances)
            ptp_pgp = max_pgp - min_pgp
            relative_rank = (parent_to_grand_parent_distances < parent_distance).sum() / cnt_pgp
            feat.extend([cnt_pgp, min_pgp, max_pgp, mid_pgp, avg_pgp, std_pgp, ptp_pgp, relative_rank])
        else:
            feat.extend([0, -999, -999, -999, -999, -999, -999, -999])

        # feature 26-33: parent distance to sibling distance(s)
        parent_to_sibling_distances = p_distances[1+num_gp:]
        if len(parent_to_sibling_distances) > 0:
            cnt_psib = len(parent_to_sibling_distances)
            min_psib = np.min(parent_to_sibling_distances)
            max_psib = np.max(parent_to_sibling_distances)
            mid_psib = np.median(parent_to_sibling_distances)
            avg_psib = np.mean(parent_to_sibling_distances)
            std_psib = np.std(parent_to_sibling_distances)
            ptp_psib = max_psib - min_psib
            relative_rank = (parent_to_sibling_distances < parent_distance).sum() / cnt_psib
            feat.extend([cnt_psib, min_psib, max_psib, mid_psib, avg_psib, std_psib, ptp_psib, relative_rank])
        else:
            feat.extend([0, -999, -999, -999, -999, -999, -999, -999])

        # feature 34-39: query to all local subgraph distances
        min_all = np.min(distances)
        max_all = np.max(distances)
        mid_all = np.median(distances)
        avg_all = np.mean(distances)
        std_all = np.std(distances)
        ptp_all = max_all - min_all
        feat.extend([min_all, max_all, mid_all, avg_all, std_all, ptp_all])

        return feat

reason: import numpy,import networkx


text:
# -*- coding: utf-8 -*-
import dlib  # 人脸处理的库 Dlib
import numpy as np  # 数据处理的库 numpy
import cv2  # 图像处理的库 OpenCv
import pandas as pd  # 数据处理的库 Pandas
import os

# 人脸识别模型,Dlib 检测器和预测器
facerec = dlib.face_recognition_model_v1("model/2.dat")
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor('model/1.dat')
path_faces = "faces/"
# 字体
font = cv2.FONT_HERSHEY_SIMPLEX
# 定义空的list存放人脸的标签
label = []
# 计算两个向量间的欧式距离
# 存储所有标签
all = []

def return_euclidean_distance(feature_1, feature_2):
    feature_1 = np.array(feature_1)
    feature_2 = np.array(feature_2)
    dist = np.sqrt(np.sum(np.square(feature_1 - feature_2)))
    print("e_distance: ", dist)

    if dist > 0.5:
        return "other"
    else:
        return "same"

faces = os.listdir(path_faces)
for person in faces:
    label.append(person)  # 保存标签
# 处理存放所有人脸特征的 CSV
path_features_known_csv = "features_all.csv"
csv_rd = pd.read_csv(path_features_known_csv, header=None)

# 用来存放所有录入人脸特征的数组
features_before = []

# 读取已知人脸数据
# known faces
for i in range(csv_rd.shape[0]):
    features_someone_arr = []
    for j in range(0, len(csv_rd.ix[i, :])):
        features_someone_arr.append(csv_rd.ix[i, :][j])
    #    print(features_someone_arr)
    features_before.append(features_someone_arr)
print("Faces in Database：", len(features_before))

# 创建 cv2 摄像头对象
cap = cv2.VideoCapture(0)

# 设置视频参数
cap.set(3, 640)
cap.set(4, 480)

# cap.isOpened() 返回 true/false 检查初始化是否成功
while cap.isOpened():
    flag, img_rd = cap.read()
    read = cv2.waitKey(1)

    # 取灰度
    img_gray = cv2.cvtColor(img_rd, cv2.COLOR_RGB2GRAY)

    # 人脸数 faces
    faces = detector(img_gray, 1)

    # 存储所有人脸的名字
    pos_namelist = []
    name_namelist = []

    # 按下Esc键退出
    if read == 27:
        break
    else:
        # 检测到人脸
        if len(faces) != 0:
            # 获取当前捕获到的图像的所有人脸的特征，存储到 features_cap_arr
            features_now = []
            for i in range(len(faces)):
                shape = predictor(img_rd, faces[i])
                features_now.append(facerec.compute_face_descriptor(img_rd, shape))

            # 遍历捕获到的图像中所有的人脸
            for k in range(len(faces)):
                # 让人名跟随在矩形框的下方
                # 确定人名的位置坐标
                # 先默认所有人不认识，是 unknown
                name_namelist.append("unknown")

                # 每个捕获人脸的名字坐标
                pos_namelist.append(
                    tuple([faces[k].left(), int(faces[k].bottom() + (faces[k].bottom() - faces[k].top()) / 4)]))

                # 对于某张人脸，遍历所有存储的人脸特征
                for i in range(len(features_before)):
                    print("with person_", str(i + 1), "the ", end='')
                    # 将某张人脸与存储的所有人脸数据进行比对
                    compare = return_euclidean_distance(features_now[k], features_before[i])
                    if compare == "same":  # 找到了相似脸
                        name_namelist[k] = label[i]
                        all.append(name_namelist[k])

                # 矩形框
                for k, d in enumerate(faces):
                    # 绘制矩形框
                    cv2.rectangle(img_rd, tuple([d.left(), d.top()]), tuple([d.right(), d.bottom()]), (0, 255, 255), 2)

            # 在人脸框下面写人脸名字
            for i in range(len(faces)):
                cv2.putText(img_rd, name_namelist[i], pos_namelist[i], font, 0.8, (0, 255, 255), 1, cv2.LINE_AA)

    print("Name list now:", name_namelist, "\n")

    cv2.putText(img_rd, "Face Recognition", (20, 40), font, 1, (0, 0, 0), 1, cv2.LINE_AA)
    cv2.putText(img_rd, "Faces: " + str(len(faces)), (20, 100), font, 1, (0, 0, 255), 1, cv2.LINE_AA)

    # 窗口显示
    cv2.imshow("camera", img_rd)

# 释放摄像头并删除建立的窗口
cap.release()
cv2.destroyAllWindows()

# 去除重复, 同时保持先后顺序不变
all_unique = sorted(set(all), key=all.index)
# print(all)
# print(all_unique)
with open('name.txt', 'a') as f:
    for i in all_unique:
        f.write(i + '\n')


text:
from typing import List, Optional
from . import EntrySettings, AbstractRule, ConditionPayload
import numpy as np
from .enums import Side
import pandas as pd

class Condition(object):
    def __init__(self, settings: EntrySettings, rules: Optional[List[AbstractRule]] = []):
        self.settings = settings
        self.rules = rules

    @property
    def side(self) -> Side:
        return self.settings.side

    def is_valid(self, condition_payload: ConditionPayload, **kwargs) -> bool:
        parameters = dict(
            kwargs,
            condition_payload= condition_payload,
            settings=self.settings,
        )

        output = np.all([
            rule.is_valid(**parameters)
            for rule in self.rules
        ])

        if output:

            has_active_position = condition_payload.has_active_position

            if self.side == Side.Sell and not has_active_position:
                ## cant sell with no active positions
                return False

            if self.side == Side.Buy and has_active_position:
                ## no buying with positions still active
                return False

        return output


text:
#!/usr/bin/python
# -*- coding: utf-8 -*-

import argparse
import os
import subprocess
import sys
import nltk
import time
from shutil import copyfile
from image import ImageRobot
from text import SearchRobot
from video import VideoRobot
from upload import UploadRobot
from urllib.error import HTTPError
import cv2
import numpy as np

def make_project_directory(search_term):
    try:
        search_term = search_term.replace(" ", "_")
        print(os.path.expanduser('~')+ "\\{}".format(search_term))
        directory_path = os.path.expanduser("~") + "\\{}".format(search_term)

        i = 1
        while True:
            s = directory_path + str(i)
            if os.path.exists(s) and os.path.isdir(s):
                i += 1
                continue
            break

        directory_path += str(i)

        print(directory_path)

        os.mkdir(directory_path)
        os.mkdir(directory_path + '\\music\\')
        os.mkdir(directory_path + '\\fonts\\')

        for file in os.listdir('fonts'):
            copyfile(f"fonts\\{file}", f"{directory_path}\\fonts\\{file}")

        copyfile('music.ogg', f'{directory_path}\\music\\song.ogg')

        return directory_path + '\\'

    except OSError as exc:
        if exc.errno != errno.EEXIST:
            print("Creation of the project directory failed.")
            sys.exit(1)

        print("Other error")
    pass

def first(array):
    try:
        return array[0]
    except:
        return array

def init():
    while True:
        try:
            main()
            break
        except:
            main()

def main():
    search_term = input("Wikipedia search term: ")
    if len(search_term) == 0:
        print("Please enter a search term.")
        sys.exit(1)

    print("Avaiable prefixes:\n1. What is\n2. Who is\n3. The history of\n4. Learn more about")
    prefixes = ["What is", "Who is", "The history of", "Learn more about"]
    prefix = input("Prefix: ")
    if not prefix in "1234":
        print("Please enter a prefix.")
        sys.exit(1)

    project_directory = make_project_directory(search_term)
    prefix = prefixes[int(prefix) - 1]

    print("[*] Starting search robot...")
    search_robot = SearchRobot()
    search_result = search_robot.search(search_term)
    keywords_list = search_robot.get_keywords(search_result)
    for i in range(len(search_result)):
        print("[*] Sentence {0}: {1}".format(i + 1, search_result[i]))
        print("[*] Keywords: {0}\n".format(keywords_list[i]))

    print("[*] Starting image robot...")
    image_robot = ImageRobot(project_directory)
    images_list = []
    print(keywords_list)
    for keywords in keywords_list:
        print('here')
        get_img = image_robot.get_image(keywords, search_term)

        print(get_img)
        img = first(get_img)
        images_list.append(img)

        print("[*] Image saved in: " + str(img))

    print(images_list)
    print("[*] Renaming images...")
    images_list = image_robot.rename_files(images_list)

    print("[*] Converting images to JPG...")
    image_robot.convert_to_jpg(images_list)

    print("[*] Resizing images...")
    image_robot.resize(images_list)

    print("[*] Starting video robot...")
    video_robot = VideoRobot(search_result, project_directory, f'{project_directory}\\saida.mkv')
    video_robot.add_subtitles()
    video_robot.make_video()
    video_robot.add_music()

    print(search_result)
    print("[*] Backup files saved in: " + project_directory)

    option = input("Do you want to upload this video? (y/n)")

    if option in 'YySs':
        print("[*] Starting upload robot...")
        upload_robot = UploadRobot()

        title = prefix + " " + search_term
        description = "\n\n".join(search_result)
        keywords = []

        for i in keywords_list:
            keywords += i

        keywords = ",".join(keywords)

        args = argparse.Namespace(
            auth_host_name="localhost",
            auth_host_port=[8080, 8090],
            category="27",
            description=description,
            file="{}/saida.mkv".format(project_directory),
            keywords=keywords,
            logging_level="ERROR",
            noauth_local_webserver=False,
            privacy_status="public",
            title=title)

        try:
            youtube = upload_robot.get_authenticated_service(args)

            print("[*] Uploading video...")
            try:
                upload_robot.initialize_upload(youtube, args)
            except HTTPError as e:
                error("An HTTP error %d occurred:\n%s" % (e.resp.status, e.content))
        except:
            print("Error: Reached video upload limit per day.")

    else:
        play_video(os.path.join(project_directory, "saida.mkv"))

def error(string):
    print(string)
    print("Restarting program... in")
    time.sleep(1)
    print('1...')
    time.sleep(2)

    print('2...')
    time.sleep(2)
    print('3...')
    time.sleep(3)

    sys.stdout.flush()

    restart()
    input()
    exit(1)

def play_video(file):
    print(file)

    os.startfile(file)

def beauty_title():
    return\
'''
   _____ _       __       __  _
  / ___/(_)___  / /____  / /_(_)____
  \__ \/ / __ \/ __/ _ \/ __/ / ___/
 ___/ / / / / / /_/  __/ /_/ / /__
/____/_/_/ /_/\__/\___/\__/_/\___/

'''

if __name__ == "__main__":
    os.system('color b1')
    print(beauty_title())
    print("Descricao: Robos como ajudantes de docentes na elaboracao de materiais audiovisuais")
    print("Autores: Boanerges Rodrigues & Gabriel Albuquerque\n")
    #nltk.download('punkt')
    main()

max_stars_repo_name: ocoiel/sintetic

max_issues_repo_name: ocoiel/sintetic

max_forks_repo_name: ocoiel/sintetic


text:
#!/usr/bin/env python3

from progressbar import ProgressBar
from pretty import *
import argparse
import numpy
import re
import os
import os.path
from scipy import misc
from scipy.ndimage.filters import gaussian_filter

parser = argparse.ArgumentParser()
parser.add_argument('-t', '--tagged', help='load tagged data from this directory', default='tagged/full')
parser.add_argument('-d', '--devkit', help='devkit directory containing categories.txt', default='mp-dev_kit')
parser.add_argument('-i', '--images', help='path to images/', default='mp-data/images')
parser.add_argument('-o', '--outdir', help='directory to save images', default=None)
parser.add_argument('-n', '--network', help='network for choosing outdir', default='base')
parser.add_argument('--serve', help='run http server', action='store_true')
parser.set_defaults(serve=False)
args = parser.parse_args()

if args.outdir is None:
    args.outdir = "exp-{}".format(args.network)

# A clipping rectifier, pegging values less than t1 to 0 and more than t2 to 1.
def peg(ar, t1, t2):
    return numpy.clip((ar - t1) / (t2 - t1), 0, 1)

# regular python argsort, from stack overflow
def argsort(seq):
    return sorted(range(len(seq)), key=seq.__getitem__)

# Category labels
categories = []
for line in open(os.path.join(args.devkit, 'categories.txt')).readlines():
    assert int(line.strip().split()[1]) == len(categories)
    categories.append(re.sub('^/[a-z]/', '', line.strip().split()[0]))
cats = len(categories)

def extract_resp_region(resp):
    avg = numpy.average(resp)
    std = numpy.std(resp)
    respmax = avg + std
    respmin = avg - std
    resp = numpy.minimum(
            numpy.maximum((respmax - resp) /
            (respmax - respmin + 1e-6), 0), 1)
    smeared = numpy.zeros([128, 128])
    smearedd = numpy.ones([128, 128]) * 0.01
    # Some geometric parameters
    res = 16
    pix = 23
    st = 7
    for x in range(res):
        for y in range(res):
            smeared[y*st:y*st+pix, x*st:x*st+pix] += resp[y][x]
            smearedd[y*st:y*st+pix, x*st:x*st+pix] += 1
    return peg(numpy.clip(gaussian_filter(
            numpy.divide(smeared, smearedd), sigma=st), 0, 1), 0.4, 0.6)

def create_response_images(name, subset):
    task("Generating images showing response regions for {}".format(name))
    labels = {}
    for line in open(os.path.join(args.devkit, '%s.txt' % subset)).readlines():
        name, label = line.strip().split()
        labels[name] = int(label)
    filenames = [line.strip() for line in open(os.path.join(args.tagged,
            '%s.filenames.txt' % subset)).readlines()]
    response = numpy.memmap(
            os.path.join(args.outdir, '%s.response.db' % subset),
            dtype=numpy.float32, mode='r')
    response.shape = (response.shape[0] / 256, 16, 16)
    topresponse = numpy.memmap(
            os.path.join(args.outdir, '%s.topresponse.db' % subset),
            dtype=numpy.float32, mode='r')
    topresponse.shape = (topresponse.shape[0] / 256, 16, 16)
    cases = response.shape[0]
    p = progress(cases)
    for index in range(cases):
        # Smear the measured response in the way it was collected:
        # over the image with 16 overlapping blocks with a 6-pixel stride.
        # the blocks are made larger here to fill the entire 128px image,
        # and additionally a gaussian blur is added at the end.
        r = extract_resp_region(response[index])
        tr = extract_resp_region(topresponse[index])
        # now apply the response to create two images.
        r3 = numpy.tile(r.reshape([128, 128, 1]), [1, 1, 3])
        tr3 = numpy.tile(tr.reshape([128, 128, 1]), [1, 1, 3])
        im = misc.imread(os.path.join(args.images, filenames[index]))
        # the "see" image shows where the response was high.
        seegim = numpy.multiply(im, r3)
        seegname = os.path.join(args.outdir, 'resp', 'seeg', filenames[index])
        os.makedirs(os.path.dirname(seegname), exist_ok=True)
        misc.imsave(seegname, seegim)
        # the "seec" image shows where the response was high.
        seecim = numpy.multiply(im, tr3)
        seecname = os.path.join(args.outdir, 'resp', 'seec', filenames[index])
        os.makedirs(os.path.dirname(seecname), exist_ok=True)
        misc.imsave(seecname, seecim)
        # the "ignoreg" image shows where the response was low.
        ignoregim = numpy.multiply(im, 1 - r3)
        ignoregname = os.path.join(
                args.outdir, 'resp', 'ignoregg', filenames[index])
        os.makedirs(os.path.dirname(ignoregname), exist_ok=True)
        misc.imsave(ignoregname, ignoregim)
        # the "tignore" image shows where the response was low.
        ignorecim = numpy.multiply(im, 1 - tr3)
        ignorecname = os.path.join(
                args.outdir, 'resp', 'ignorec', filenames[index])
        os.makedirs(os.path.dirname(ignorecname), exist_ok=True)
        misc.imsave(ignorecname, ignorecim)
        # use a blurred image for visualization
        blurred = numpy.zeros(im.shape)
        for c in range(3):
            blurred[:,:,c] = gaussian_filter(im[:,:,c], sigma=12)
        # the "good" image blurs the ignore image and shows the see image
        blurignore = numpy.multiply(blurred, 1 - r3)
        goalim = blurignore + seegim
        goalname = os.path.join(args.outdir, 'resp', 'goal', filenames[index])
        os.makedirs(os.path.dirname(goalname), exist_ok=True)
        misc.imsave(goalname, goalim)
        # the "notgoal" image shows where the image is confused
        blursee = numpy.multiply(blurred, r3)
        notgoalim = blursee + ignoregim
        notgoalname = os.path.join(
                args.outdir, 'resp', 'notgoal', filenames[index])
        os.makedirs(os.path.dirname(notgoalname), exist_ok=True)
        misc.imsave(notgoalname, notgoalim)
        # the "chosen" image shows where the image chosened a response
        blurtignore = numpy.multiply(blurred, 1 - tr3)
        chosenim = blurtignore + seecim
        chosenname = os.path.join(
                args.outdir, 'resp', 'chosen', filenames[index])
        os.makedirs(os.path.dirname(chosenname), exist_ok=True)
        misc.imsave(chosenname, chosenim)
        # the "notchosen" image shows where the image notchosened a response
        blurtignore = numpy.multiply(blurred, tr3)
        notchosenim = blurtignore + ignorecim
        notchosenname = os.path.join(
                args.outdir, 'resp', 'notchosen', filenames[index])
        os.makedirs(os.path.dirname(notchosenname), exist_ok=True)
        misc.imsave(notchosenname, notchosenim)

        p.update(index + 1)

def create_eval_html(name, subset):
    html = [
        '<!doctype html>',
        '',
        '',
        '',
        '
',
        '
PICKED	CORRECT
'
    ]
    labels = {}
    for line in open(os.path.join(args.devkit, '%s.txt' % subset)).readlines():
        name, label = line.strip().split()
        labels[name] = int(label)
    filenames = [line.strip() for line in open(os.path.join(args.tagged,
            '%s.filenames.txt' % subset)).readlines()]
    predictions = numpy.memmap(os.path.join(args.outdir,
            '%s.confusion.db' % subset), dtype=numpy.float32, mode='r')
    cases = int(predictions.shape[0] / cats)
    predictions.shape = (cases, cats)
    topindex = numpy.argsort(-predictions, axis=1)
    confusion = []
    for index in range(cases):
        top = topindex[index]
        correct = labels[filenames[index]]
        confusion.append(numpy.where(top == correct)[0][0])
    worstfirst = reversed(argsort(confusion))
    for index in worstfirst:
        top = topindex[index]
        correct = labels[filenames[index]]
        score = predictions[index]
        html.append('
{} ({}) [{}]
'.format(
                filenames[index],
                index,
                confusion[index]))
        html.append('
')
        html.append('')
        html.append('
{} ({})
{}	'.format(
                categories[top[0]],
                top[0],
                score[top[0]]))
        html.append('')
        html.append('
NOT {} ({})
{}	'.format(
                categories[top[0]],
                top[0],
                score[top[0]]))
        html.append('')
        html.append('
{} ({})
{}	'.format(
                categories[correct],
                correct,
                score[correct]))
        html.append('')
        html.append('
NOT {} ({})
{}	'.format(
                categories[correct],
                correct,
                score[correct]))
    html.extend(['
', '', ''])
    hfilename = os.path.join(args.outdir, '%s.eval.html' % subset)
    with open(hfilename, 'w') as htmlfile:
        htmlfile.write('\n'.join(html))

create_eval_html("validation set", "val")
create_eval_html("training set", "train")
create_response_images("validation set", "val")
create_response_images("training set", "train")

if args.serve:
    from http.server import HTTPServer
    from http.server import SimpleHTTPRequestHandler
    os.chdir(args.outdir)
    PORT = 8000
    httpd = HTTPServer(('', PORT), SimpleHTTPRequestHandler)
    print("serving at port", PORT)
    try:
      httpd.serve_forever()
    except KeyboardInterrupt:
      pass
    httpd.server_close()

max_stars_repo_name: davidbau/periscope

max_issues_repo_name: davidbau/periscope

max_forks_repo_name: davidbau/periscope


text:
###########################################################################
# Author: Cisco Systems, Inc.
# Change Log:
# Date          Author      Desc
# 2017/11/01    Mike Boyer  Ported to python3 and removed theano refs
# 2017/11/05    Mike Boyer  Removed unused split() function and rewrote
#                           "News" class as fnc1_Data() class.
# 2017/11/16    Mike Boyer  Added vecs to fnc1_Data object
###########################################################################

import csv
import _pickle as pickle
import gzip
from time import time
import numpy as np
from collections import defaultdict
from Vectors import *

chars = set([chr(i) for i in range(32,128)])
#character whitelist
stances = {'agree':0,'disagree':1,'discuss':2}
#set up some values for later

def transform(text):
    #convert a string into a np array of approved character indices, starting at 0
    return np.array([ord(i)-32 for i in text if i in chars])

def pad_char(text, padc=-1):
    #take a set of variable length arrays and convert to a matrix with specified fill value
    maxlen = max([len(i) for i in text])
    tmp = np.ones((len(text), maxlen),dtype='int32')
    tmp.fill(padc)
    for i in range(len(text)):
        data = text[i]
        tmp[i,:len(data)]=data
    return tmp

def proc_bodies(fn):
    #process the bodies csv into arrays
    tmp = {}
    with open(fn,'r') as f:
        reader = csv.reader(f)
        next(reader)
        for line in reader:
            bid, text = line
            tmp[bid]=text
    return tmp

class fnc1_Data(object):
    def __init__(self, stancesFile=None, bodiesFile=None, vecs=None):

        if stancesFile == None or bodiesFile == None:
            raise ValueError('Set the path to the stances bodies!!!!')

        #process files into arrays, etc
        self.bodies = proc_bodies(bodiesFile)
        self.stances = []
        self.body_vecs = {}
        self.stance_vecs = []
        self.vecs = vecs #save the vecs for use in other methods
        self.labels=[]

        #Read stances
        with open(stancesFile,'r') as f:
            reader = csv.reader(f)
            next(reader)
            for line in reader:
                if len(line)==2:
                    hl, bid = line
                    stance = None
                else:
                    hl, bid, stance = line
                    self.labels.append(stances[stance])
                self.stances.append((hl,bid,stance))

        self.n_stances = len(self.stances)

        #Transform the bodies into their vector representation
        for bID, body in self.bodies.items():
            self.body_vecs[bID] = vecs.transform([body])

        #Transform the stance (headline only) into its vector representation
        for stance in self.stances:
            self.stance_vecs.append(vecs.transform([stance[0]]))

        self.batchStartIdx = 0

    def get_one(self, ridx=None):
        #select a single sample either randomly or by index
        if ridx is None:
            ridx = np.random.randint(0,self.n_stances)
        head = self.stances[ridx]
        body = self.bodies[head[1]]

        return head, body

    def sample(self, n=16, ridx=None):
        #select a batch of samples either randomly or by index
        heads = []
        bodies = []
        stances_d = []
        if ridx is not None:
            for r in ridx:
                head, body_text = self.get_one(r)
                head_text, _, stance = head
                heads.append(head_text)
                bodies.append(body_text)
                stances_d.append(stances[stance])
        else:
            for i in range(n):
                head, body_text = self.get_one()
                head_text, _, stance = head
                heads.append(head_text)
                bodies.append(body_text)
                stances_d.append(stances[stance])

        heads = self.vecs.transform(heads)
        bodies = self.vecs.transform(bodies)
        stances_d = np.asarray(stances_d, dtype='int32')
        #clean up everything and return it

        return heads, bodies, stances_d

    def getNextTrainBatch(self, batchSize):
        batchStart = self.batchStartIdx

        if batchStart + batchSize <= self.n_stances:
            heads, bodies, stances_d = self.sample(n=1, ridx=(range(batchStart, batchStart + batchSize)))
            self.batchStartIdx = batchStart + batchSize
        else:
            heads, bodies, stances_d = self.sample(n=1, ridx=(range(batchStart, self.n_stances)))
            for i in range(batchSize - len(heads)):
                head, body, stance_d = self.sample(n=1, ridx=[i])
                np.append(heads, head)
                np.append(bodies, body)
                np.append(stances_d, stance_d)
            self.batchStartIdx = i

        #stances_d is a scalar (e.g. [0] for agree). For training
        #we want it to be a vector that matches a softmax output
        #(e.g. [1, 0, 0] for 'agree')
        stanceVec = np.zeros((len(stances_d), len(stances.keys())))
        for i in range(len(stances_d)):
            s = stances_d[i]
            stanceVec[i][s] = 1.0

        return heads, bodies, stanceVec

    def validate(self):
        #iterate over the dataset in order
        for i in xrange(len(self.headlines)):
            yield self.sample(ridx=[i])

if __name__ == '__main__':
    v = GoogleVec()
    v.load()
    trainData = fnc1_Data(stancesFile='./data/train_stances_related_only.csv', vecs=v, bodiesFile='./data/train_bodies.csv')
    head, body, stance = trainData.sample(1, ridx=[0])
    print(trainData.stances[0])
    print(head)
    print(body)
    print(stance)

    h_vec = v.model.syn0[head]
    b_vec = v.model.syn0[body]
    print(h_vec.shape)
    print(b_vec.shape)

 #   Copyright 2017 Cisco Systems, Inc.
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
 #   You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 #   Unless required by applicable law or agreed to in writing, software
 #   distributed under the License is distributed on an "AS IS" BASIS,
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.


text:
from typing import Tuple, Any

import numpy as np
import sklearn
import time
import torch

from fltk.core.node import Node
from fltk.schedulers import MinCapableStepLR
from fltk.strategy import get_optimizer
from fltk.util.config import FedLearningConfig

class Client(Node):
    """
    Federated experiment client.
    """
    running = False

    def __init__(self, identifier: str, rank: int, world_size: int, config: FedLearningConfig):
        super().__init__(identifier, rank, world_size, config)

        self.loss_function = self.config.get_loss_function()()
        self.optimizer = get_optimizer(self.config.optimizer)(self.net.parameters(),
                                                              **self.config.optimizer_args)
        self.scheduler = MinCapableStepLR(self.optimizer,
                                          self.config.scheduler_step_size,
                                          self.config.scheduler_gamma,
                                          self.config.min_lr)

    def remote_registration(self):
        """
        Function to perform registration to the remote. Currently, this will connect to the Federator Client. Future
        version can provide functionality to register to an arbitrary Node, including other Clients.
        @return: None.
        @rtype: None
        """
        self.logger.info('Sending registration')
        self.message('federator', 'ping', 'new_sender')
        self.message('federator', 'register_client', self.id, self.rank)
        self.running = True
        self._event_loop()

    def stop_client(self):
        """
        Function to stop client after training. This allows remote clients to stop the client within a specific
        timeframe.
        @return: None
        @rtype: None
        """
        self.logger.info('Got call to stop event loop')
        self.running = False

    def _event_loop(self):
        self.logger.info('Starting event loop')
        while self.running:
            time.sleep(0.1)
        self.logger.info('Exiting node')

    def train(self, num_epochs: int, round_id: int):
        """
        Function implementing federated learning training loop, allowing to run for a configurable number of epochs
        on a local dataset. Note that only the last statistics of a run are sent to the caller (i.e. Federator).
        @param num_epochs: Number of epochs to run during a communication round's training loop.
        @type num_epochs: int
        @param round_id: Global communication round ID to be used during training.
        @type round_id: int
        @return: Final running loss statistic and acquired parameters of the locally trained network. NOTE that
        intermediate information is only logged to the STD-out.
        @rtype: Tuple[float, Dict[str, torch.Tensor]]
        """
        start_time = time.time()
        running_loss = 0.0
        final_running_loss = 0.0
        for local_epoch in range(num_epochs):
            effective_epoch = round_id * num_epochs + local_epoch
            progress = f'[RD-{round_id}][LE-{local_epoch}][EE-{effective_epoch}]'
            if self.distributed:
                # In case a client occurs within (num_epochs) communication rounds as this would cause
                # an order or data to re-occur during training.
                self.dataset.train_sampler.set_epoch(effective_epoch)

            training_cardinality = len(self.dataset.get_train_loader())
            self.logger.info(f'{progress}{self.id}: Number of training samples: {training_cardinality}')

            for i, (inputs, labels) in enumerate(self.dataset.get_train_loader(), 0):
                inputs, labels = inputs.to(self.device), labels.to(self.device)

                # zero the parameter gradients
                self.optimizer.zero_grad()

                outputs = self.net(inputs)
                loss = self.loss_function(outputs, labels)

                loss.backward()
                self.optimizer.step()
                running_loss += loss.item()
                # Mark logging update step
                if i % self.config.log_interval == 0:
                    self.logger.info(
                            f'[{self.id}] [{local_epoch}/{num_epochs:d}, {i:5d}] loss: {running_loss / self.config.log_interval:.3f}')
                    final_running_loss = running_loss / self.config.log_interval
                    running_loss = 0.0
            end_time = time.time()
            duration = end_time - start_time
            self.logger.info(f'{progress} Train duration is {duration} seconds')

        return final_running_loss, self.get_nn_parameters(),

    def set_tau_eff(self, total):
        client_weight = self.get_client_datasize() / total
        n = self.get_client_datasize()  # pylint: disable=invalid-name
        E = self.config.epochs  # pylint: disable=invalid-name
        B = 16  # nicely hardcoded :) # pylint: disable=invalid-name
        tau_eff = int(E * n / B) * client_weight
        if hasattr(self.optimizer, 'set_tau_eff'):
            self.optimizer.set_tau_eff(tau_eff)

    def test(self) -> Tuple[float, float, np.array]:
        """
        Function implementing federated learning test loop.
        @return: Statistics on test-set given a (partially) trained model; accuracy, loss, and confusion matrix.
        @rtype: Tuple[float, float, np.array]
        """
        start_time = time.time()
        correct = 0
        total = 0
        targets_ = []
        pred_ = []
        loss = 0.0
        with torch.no_grad():
            for (images, labels) in self.dataset.get_test_loader():
                images, labels = images.to(self.device), labels.to(self.device)

                outputs = self.net(images)

                _, predicted = torch.max(outputs.data, 1)  # pylint: disable=no-member
                total += labels.size(0)
                correct += (predicted == labels).sum().item()

                targets_.extend(labels.cpu().view_as(predicted).numpy())
                pred_.extend(predicted.cpu().numpy())

                loss += self.loss_function(outputs, labels).item()
        # Calculate learning statistics
        loss /= len(self.dataset.get_test_loader().dataset)
        accuracy = 100.0 * correct / total

        confusion_mat = sklearn.metrics.confusion_matrix(targets_, pred_)

        end_time = time.time()
        duration = end_time - start_time
        self.logger.info(f'Test duration is {duration} seconds')
        return accuracy, loss, confusion_mat

    def get_client_datasize(self):  # pylint: disable=missing-function-docstring
        return len(self.dataset.get_train_sampler())

    def exec_round(self, num_epochs: int, round_id: int) -> Tuple[Any, Any, Any, Any, float, float, float, np.array]:
        """
        Function as access point for the Federator Node to kick off a remote learning round on a client.
        @param num_epochs: Number of epochs to run
        @type num_epochs: int
        @return: Tuple containing the statistics of the training round; loss, weights, accuracy, test_loss, make-span,
        training make-span, testing make-span, and confusion matrix.
        @rtype: Tuple[Any, Any, Any, Any, float, float, float, np.array]
        """
        self.logger.info(f"[EXEC] running {num_epochs} locally...")
        start = time.time()
        loss, weights = self.train(num_epochs, round_id)
        time_mark_between = time.time()
        accuracy, test_loss, test_conf_matrix = self.test()

        end = time.time()
        round_duration = end - start
        train_duration = time_mark_between - start
        test_duration = end - time_mark_between
        # self.logger.info(f'Round duration is {duration} seconds')

        if hasattr(self.optimizer, 'pre_communicate'):  # aka fednova or fedprox
            self.optimizer.pre_communicate()
        for k, value in weights.items():
            weights[k] = value.cpu()
        return loss, weights, accuracy, test_loss, round_duration, train_duration, test_duration, test_conf_matrix

    def __del__(self):
        self.logger.info(f'Client {self.id} is stopping')

max_stars_repo_name: JMGaljaard/fltk-testbed

max_issues_repo_name: JMGaljaard/fltk-testbed

max_forks_repo_name: JMGaljaard/fltk-testbed


text:
import pymysql
import configparser
import os
import numpy as np

class Database():
    config = {
            'host': '127.0.0.1',
            'port': 3306,
            'db': 'edgar',
            'charset': 'utf8mb4',
            'cursorclass': pymysql.cursors.DictCursor,
    }

    def __init__(self):
        cfg = configparser.ConfigParser()
        cfg.read(os.path.join(os.path.expanduser('~'), 'database.ini'))
        for section in cfg:
            for key in cfg[section]:
                self.config[key] = cfg[section][key]
    def getConn(self):
        return pymysql.connect(**self.config)

if __name__ == '__main__':
    db = Database()
    conn = db.getConn()
    cursor = conn.cursor()

    result = {
        'KB': [],
        'MB': [],
        'GB': []
    }
    max_id = 0
    while True:
        cursor.execute('SELECT id, filing_desc FROM edgar_company_filing where id > {} limit 1000'.format(max_id))
        filingsRs = cursor.fetchall()
        if len(filingsRs) is 0:
            break
        for filingRs in filingsRs:
            id = filingRs['id']
            if id > max_id:
                max_id = id
            desc = filingRs['filing_desc']
            size = desc[desc.index('Size: ')+len('Size: '):]
            tokens = size.split(" ")
            if len(tokens) < 2:
                print('{} has issue'.format(id))
                continue
            try:
                result[tokens[1]].append(int(tokens[0]))
            except:
                print('id: {} size: {}'.format(id, size))

        print('max id is {}'.format(max_id))

    sizeSum = {
        'KB': np.array(result['KB']).sum(),
        'MB': np.array(result['MB']).sum(),
        'GB': np.array(result['GB']).sum()
    }
    print(sizeSum)

max_stars_repo_name: fubiye/edgar-abs-kg

max_issues_repo_name: fubiye/edgar-abs-kg

max_forks_repo_name: fubiye/edgar-abs-kg


text:
# Imports ---------------------------------------------------------------------
# Python
import argparse
import joblib
import yaml
import os.path as osp
from collections import defaultdict
import joblib

# PyTorch
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torch import autograd
from torch.optim import Adam

# NumPy
import numpy as np
from numpy import array
from numpy.random import choice, randint

# Model Building
from gen_models.vrnn import VRNN
from gen_models.vrnn_flat_latent import VRNN as FlatLatentVRNN
from gen_models.flat_vrnn import VRNN as FlatVRNN
from gen_models.flat_net import VRNN as AE
import rlkit.torch.pytorch_util as ptu

# Data
from gen_models.data_loaders import BasicDataLoader, RandomDataLoader

# Logging
from rlkit.core import logger
from rlkit.launchers.launcher_util import setup_logger, set_seed
from rlkit.core.vistools import generate_gif, save_pytorch_tensor_as_img

import sys

from numpy import pi
from numpy import log as np_log
log_2pi = np_log(2*pi)

LOG_COV_MAX = 2
LOG_COV_MIN = -20

def compute_diag_log_prob(recon_mean, recon_log_cov, obs):
    bs = recon_mean.size(0)
    recon_mean = recon_mean.view(bs, -1)
    recon_log_cov = recon_log_cov.view(bs, -1)
    obs = obs.view(bs, -1)

    recon_cov = torch.exp(recon_log_cov)
    log_prob = -0.5 * torch.sum(
        (recon_mean - obs)**2 / recon_cov
    )
    log_det_temp = torch.sum(recon_log_cov, 1) + log_2pi
    log_prob = log_prob - 0.5 * torch.sum(log_det_temp)

    return log_prob

def experiment(exp_specs):
    ptu.set_gpu_mode(exp_specs['use_gpu'])
    # Set up logging ----------------------------------------------------------
    exp_id = exp_specs['exp_id']
    exp_prefix = exp_specs['exp_name']
    seed = exp_specs['seed']
    set_seed(seed)
    setup_logger(exp_prefix=exp_prefix, exp_id=exp_id, variant=exp_specs)

    # Prep the data -----------------------------------------------------------
    replay_dict = joblib.load(exp_specs['replay_dict_path'])
    next_obs_array = replay_dict['next_observations']
    acts_array = replay_dict['actions']
    data_loader = BasicDataLoader(
        next_obs_array[:40000], acts_array[:40000], exp_specs['episode_length'], exp_specs['batch_size'], use_gpu=ptu.gpu_enabled())
    val_data_loader = BasicDataLoader(
        next_obs_array[40000:], acts_array[40000:], exp_specs['episode_length'], exp_specs['batch_size'], use_gpu=ptu.gpu_enabled())

    # Model Definition --------------------------------------------------------
    conv_channels = 64
    conv_encoder = nn.Sequential(
        nn.Conv2d(3, conv_channels, 4, stride=2, padding=1, bias=False),
        nn.BatchNorm2d(conv_channels),
        nn.ReLU(),
        nn.Conv2d(conv_channels, conv_channels, 4, stride=2, padding=1, bias=False),
        nn.BatchNorm2d(conv_channels),
        nn.ReLU()
    )
    ae_dim = 1024
    gru_dim = 4094
    img_h = 5
    flat_inter_img_dim = img_h * img_h * conv_channels
    act_dim = 64
    act_proc = nn.Linear(4, act_dim, bias=True)
    fc_encoder = nn.Sequential(
        nn.Linear(flat_inter_img_dim + act_dim, ae_dim, bias=False),
        nn.BatchNorm1d(ae_dim),
        nn.ReLU(),
        # nn.Linear(ae_dim, ae_dim, bias=False),
        # nn.BatchNorm1d(ae_dim),
        # nn.ReLU(),
        # nn.Linear(ae_dim, ae_dim, bias=False),
        # nn.BatchNorm1d(ae_dim),
        # nn.ReLU(),
        # nn.Linear(ae_dim, ae_dim, bias=False),
        # nn.BatchNorm1d(ae_dim),
        # nn.ReLU()
    )
    gru = nn.LSTMCell(
        ae_dim, gru_dim, bias=True
    )
    fc_decoder = nn.Sequential(
        nn.Linear(gru_dim + act_dim, flat_inter_img_dim, bias=False),
        nn.BatchNorm1d(flat_inter_img_dim),
        nn.ReLU(),
        # nn.Linear(ae_dim, ae_dim, bias=False),
        # nn.BatchNorm1d(ae_dim),
        # nn.ReLU(),
        # # nn.Linear(ae_dim, ae_dim, bias=False),
        # # nn.BatchNorm1d(ae_dim),
        # # nn.ReLU(),
        # # nn.Linear(ae_dim, ae_dim, bias=False),
        # # nn.BatchNorm1d(ae_dim),
        # # nn.ReLU(),
        # nn.Linear(ae_dim, flat_inter_img_dim, bias=False),
        # nn.BatchNorm1d(flat_inter_img_dim),
        # nn.ReLU(),
    )
    conv_decoder = nn.Sequential(
        nn.ConvTranspose2d(conv_channels, conv_channels, 4, stride=2, padding=1, output_padding=0, bias=False),
        nn.BatchNorm2d(conv_channels),
        nn.ReLU(),
        # nn.Conv2d(conv_channels, conv_channels, 3, stride=1, padding=1, bias=False),
        # nn.BatchNorm2d(conv_channels),
        # nn.ReLU(),
        nn.ConvTranspose2d(conv_channels, conv_channels, 4, stride=2, padding=1, output_padding=0, bias=False),
        nn.BatchNorm2d(conv_channels),
        nn.ReLU(),
        # nn.Conv2d(conv_channels, conv_channels, 3, stride=1, padding=1, bias=False),
        # nn.BatchNorm2d(conv_channels),
        # nn.ReLU(),
    )
    mean_decoder = nn.Sequential(
        nn.Conv2d(conv_channels, 3, 1, stride=1, padding=0, bias=True),
        nn.Sigmoid()
    )
    log_cov_decoder = nn.Sequential(
        nn.Conv2d(conv_channels, 3, 1, stride=1, padding=0, bias=True),
    )
    if ptu.gpu_enabled():
        conv_encoder.cuda()
        fc_encoder.cuda()
        gru.cuda()
        fc_decoder.cuda()
        conv_decoder.cuda()
        mean_decoder.cuda()
        log_cov_decoder.cuda()
        act_proc.cuda()

    # Optimizer ---------------------------------------------------------------
    model_optim = Adam(
        [
            item for sublist in
            map(
                lambda x: list(x.parameters()),
                [fc_encoder, conv_encoder, gru, fc_decoder, conv_decoder, mean_decoder, log_cov_decoder]
            )
            for item in sublist
        ],
        lr=float(exp_specs['model_lr']),
        weight_decay=float(exp_specs['model_wd'])
    )

    # -------------------------------------------------------------------------
    freq_bptt = exp_specs['freq_bptt']
    episode_length = exp_specs['episode_length']
    losses = []
    for iter_num in range(int(float(exp_specs['max_iters']))):
        if iter_num % freq_bptt == 0:
            if iter_num > 0:
                # loss = loss / freq_bptt
                loss.backward()
                model_optim.step()
                prev_h_batch = prev_h_batch.detach()
                prev_c_batch = prev_c_batch.detach()
            loss = 0
        if iter_num % episode_length == 0:
            prev_h_batch = Variable(torch.zeros(exp_specs['batch_size'], gru_dim))
            prev_c_batch = Variable(torch.zeros(exp_specs['batch_size'], gru_dim))
            if ptu.gpu_enabled():
                prev_h_batch = prev_h_batch.cuda()
                prev_c_batch = prev_c_batch.cuda()

            train_loss_print = '\t'.join(losses)
            losses = []

        obs_batch, act_batch = data_loader.get_next_batch()
        act_batch = act_proc(act_batch)

        hidden = fc_decoder(torch.cat([prev_h_batch, act_batch], 1)).view(obs_batch.size(0), conv_channels, img_h, img_h)
        hidden = conv_decoder(hidden)
        recon = mean_decoder(hidden)
        log_cov = log_cov_decoder(hidden)
        log_cov = torch.clamp(log_cov, LOG_COV_MIN, LOG_COV_MAX)

        enc = conv_encoder(obs_batch).view(obs_batch.size(0), -1)
        enc = fc_encoder(torch.cat([enc, act_batch], 1))
        prev_h_batch, prev_c_batch = gru(enc, (prev_h_batch, prev_c_batch))

        losses.append('%.4f' % ((obs_batch - recon)**2).mean())
        if iter_num % episode_length != 0:
            loss = loss + ((obs_batch - recon)**2).sum()/float(exp_specs['batch_size'])
            # loss = loss + compute_diag_log_prob(recon, log_cov, obs_batch)/float(exp_specs['batch_size'])

        if iter_num % (500*episode_length) in range(2*episode_length):
            save_pytorch_tensor_as_img(recon[0].data.cpu(), 'junk_vis/pogrid_4_len_8_scale_4/rnn_recon_%d.png' % iter_num)
            save_pytorch_tensor_as_img(obs_batch[0].data.cpu(), 'junk_vis/pogrid_4_len_8_scale_4/rnn_obs_%d.png' % iter_num)

        if iter_num % exp_specs['freq_val'] == 0:
            print('\nValidating Iter %d...' % iter_num)
            list(map(lambda x: x.eval(), [fc_encoder, conv_encoder, gru, fc_decoder, conv_decoder, mean_decoder, log_cov_decoder, act_proc]))

            val_prev_h_batch = Variable(torch.zeros(exp_specs['batch_size'], gru_dim))
            val_prev_c_batch = Variable(torch.zeros(exp_specs['batch_size'], gru_dim))
            if ptu.gpu_enabled():
                val_prev_h_batch = val_prev_h_batch.cuda()
                val_prev_c_batch = val_prev_c_batch.cuda()

            losses = []
            for i in range(episode_length):
                obs_batch, act_batch = val_data_loader.get_next_batch()
                act_batch = act_proc(act_batch)

                hidden = fc_decoder(torch.cat([val_prev_h_batch, act_batch], 1)).view(obs_batch.size(0), conv_channels, img_h, img_h)
                hidden = conv_decoder(hidden)
                recon = mean_decoder(hidden)
                log_cov = log_cov_decoder(hidden)
                log_cov = torch.clamp(log_cov, LOG_COV_MIN, LOG_COV_MAX)

                enc = conv_encoder(obs_batch).view(obs_batch.size(0), -1)
                enc = fc_encoder(torch.cat([enc, act_batch], 1))
                val_prev_h_batch, val_prev_c_batch = gru(enc, (val_prev_h_batch, val_prev_c_batch))

                # val_loss = compute_diag_log_prob(recon, log_cov, obs_batch)/float(exp_specs['batch_size'])
                losses.append('%.4f' % ((obs_batch - recon)**2).mean())

            loss_print = '\t'.join(losses)
            print('Val MSE:\t' + loss_print)
            print('Train MSE:\t' + train_loss_print)

            list(map(lambda x: x.train(), [fc_encoder, conv_encoder, gru, fc_decoder, conv_decoder, mean_decoder, log_cov_decoder, act_proc]))

if __name__ == '__main__':
    # Arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('-e', '--experiment', help='experiment specification file')
    args = parser.parse_args()
    with open(args.experiment, 'r') as spec_file:
        spec_string = spec_file.read()
        exp_specs = yaml.load(spec_string)

    experiment(exp_specs)

reason: import numpy,from numpy


text:
#!/usr/bin/env python

import matplotlib
import numpy

from matplotlib import pylab
font = {'family' : 'normal',
        'size'   : 16}

matplotlib.rc('font', **font)

import matplotlib.pyplot as plt

# timings in seconds
# niwa-1007520
results = {
	'gcc': {
        'c++': {
          1: 18.722,
          2: 12.441,
          4: 8.896,
          8: 5.592,
          16: 4.830,
        },
        'fortran': {
          1: 14.303,
          2: 6.882,
          4: 3.917,
          8: 3.064,
          16: 3.305,
        },
    },
    'pgi': {
        'c++': {
          1: 22.697,
          2: 14.948,
          4: 9.881,
          8: 6.490,
          16: 7.956,
        },
        'fortran': {
          1: 14.551,
          2: 9.458,
          4: 5.721,
          8: 5.005,
          16: 5.580,
        },
    },
}

fig, ax = plt.subplots()
width = 0.2

count = 0
colors = ['indigo', 'firebrick', 'c', 'orange']
platforms = ('gcc', 'pgi')
compilers = ('c++', 'fortran')
legs = []
for p in platforms:
    for c in compilers:
        legs.append(p + ' ' + c)
        nth = results[p][c].keys()
        nth.sort()
        inds = numpy.array(range(len(nth))) + count*width
        speed = [results['gcc']['c++'][1]/results[p][c][n] for n in nth]
        r = ax.bar(inds, speed, width, color=colors[count])
        count += 1

ax.set_ylabel('speedup')
ax.set_xlabel('number of OpenMP threads')
ax.set_title('C++ vs Fortran')
ax.set_xticks(inds)
ax.set_xticklabels(['{}'.format(n) for n in nth])
ax.legend(legs, loc=2)
plt.show()

max_stars_repo_name: pletzer/fidibench

max_issues_repo_name: pletzer/fidibench

max_forks_repo_name: pletzer/fidibench


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Plot physiological signal vs. time and return acquisition time for each repetition.

Created on Tue Jul  4 17:45:43 2017

@author: slevy
"""

import numpy as np
import matplotlib.pyplot as plt
import pickle
import argparse
import os
from datetime import datetime

class Param:
    def __init__(self):
        self.out_fname = 'physio'
        self.sampling_period = 20  # ms
        self.physiolog_type = 'cmrr'  # type of physiolog, either "slr" (as coded by Simon Levy-Rosetti) or "cmrr" (as coded in CMRR sequences)

def main(log_fname, out_fname):
    """Main."""

    # different processing depending on the physiolog type
    if param_default.physiolog_type == 'slr':

        # extract physio signal
        time, physio_values, epi_acqtime, epi_event, acq_window = read_physiolog(log_fname)

        # sort event times
        reps_table = sort_event_times(epi_acqtime, epi_event)

        # plot physio signal
        plot_physio(time, physio_values, epi_acqtime, reps_table, acq_window, out_fname)

        # write acquisition time of each measurement
        pickle.dump([time, epi_acqtime, reps_table, physio_values, acq_window], open(out_fname+"_trigtimes.pickle", "wb"))

    elif param_default.physiolog_type == 'cmrr':

        # extract physio signal
        time, trigger_start_time, trigger_end_time, physio_values, acq_window = read_physiolog_cmrr(log_fname)

        # plot physio signal along with trigger start and end time
        plot_physio_cmrr(time, trigger_start_time, trigger_end_time, physio_values, acq_window, out_fname)

        # write acquisition time of each measurement
        pickle.dump([time, trigger_start_time, trigger_end_time, physio_values, acq_window], open(out_fname + "_trigtimes.pickle", "wb"))

    print('****Done.****')

def read_physiolog(path, sampling_period=20):
    """
    Read physio logfile and parse it.
    :param path:
           sampling_period: in ms
    :return:
    """

    file = open(path, 'r')
    text = file.readlines()
    physio_sig = np.array(text[2].strip().split(' '), dtype=str)

    # get stats in file footer
    acq_window_idx_line = [text.index(line) for line in text if "AcqWin" in line]
    acq_window = float(text[acq_window_idx_line[0]].strip().split(' ')[-1])

   # get time axis, time of trigger start and time of trigger end
    time, epi_acqtime, epi_event, physio_values = [], [], [], []
    sampling_count = 0
    for i_meas in range(len(physio_sig)):
        if (physio_sig[i_meas] not in ['5000', '6000', '5002', '6002']) and ("[" not in physio_sig[i_meas]):
            time.append(sampling_count*sampling_period)
            physio_values.append(int(physio_sig[i_meas]))
            sampling_count += 1
        elif ("[" in physio_sig[i_meas]) and ("]" in physio_sig[i_meas]):
            epi_acqtime.append((sampling_count-1)*sampling_period)
            epi_event.append(physio_sig[i_meas])

    return np.array(time), np.array(physio_values), np.array(epi_acqtime), np.array(epi_event), acq_window

def read_physiolog_cmrr(path, sampling_period=20):
    """
    Read physio logfile and parse it.
    :param path:
    :return:
    """

    file = open(path, 'r')
    text = file.readlines()
    physio_sig = np.array(text[0].strip().split(' '), dtype=str)

    # get useful data in file footer
    # Acquisition window
    acq_window_idx_line = [text.index(line) for line in text if "AcqWin" in line]
    acq_window = float(text[acq_window_idx_line[0]].strip().split(' ')[-1])
    # Start time of acquisition
    acqStartTime_idx_line = [text.index(line) for line in text if "LogStartMDHTime:" in line]
    # acqStartTime = datetime.strptime(acqStartTime_date+'-'+str(int(acqStartTime_seconds/1000))+'.'+str(acqStartTime_seconds)[-5:], "%Y%m%d-%S.%f")
    # acqStartTime = datetime.timedelta(milliseconds=float(text[acqStartTime_idx_line[0]].strip().split('LogStartMDHTime:')[-1]))
    acqStartTime_time = datetime.utcfromtimestamp(float(text[acqStartTime_idx_line[0]].strip().split('LogStartMDHTime:')[-1]) / 1000.0)
    acqStartTime_date = datetime.strptime(os.path.basename(path).split('_')[2], "%Y%m%d")
    acqStartTime = datetime.combine(acqStartTime_date.date(), acqStartTime_time.time())

    # # remove first (6002=end of info added by sequence, which was at the opening of the logfile) and last (last 5002=section added to indicate acquisition end) elements
    # idx_start, idx_end = np.min(np.where(physio_sig == '6002')), np.max(np.where(physio_sig == '5002'))
    # physio_sig = physio_sig[idx_start+1:idx_end]

    # get time axis, time of trigger start, time of trigger end and physiological values
    time, trigger_start_times, trigger_end_times, physio_values = [], [], [], []
    sampling_count = 0
    logInfo_ON = False
    for i_meas in range(len(physio_sig)):
        if (physio_sig[i_meas] not in ['5000', '6000', '5002', '6002']) and ("[" not in physio_sig[i_meas]) and (not logInfo_ON):
            time.append(sampling_count*sampling_period)
            physio_values.append(int(physio_sig[i_meas]))
            sampling_count += 1
        elif physio_sig[i_meas] == '5000':
            trigger_start_times.append(sampling_count*sampling_period)
        elif physio_sig[i_meas] == '6000':
            trigger_end_times.append(sampling_count*sampling_period)
        elif physio_sig[i_meas] == '5002':
            logInfo_ON = True
        elif physio_sig[i_meas] == '6002':
            logInfo_ON = False

    return np.array(time), np.array(trigger_start_times), np.array(trigger_end_times), np.array(physio_values), acq_window, acqStartTime

def sort_event_times(epi_acqtimes, epi_events):
    """

    :param epi_acqtimes:
    :param epi_events:
    :return:
    """

    if len(epi_acqtimes) != len(epi_events):
        os.error("ERROR: Number of times and events are different.")

    # extract rep and slice numbers of each scan event
    reps, slices = [], []
    for i_event in range(len(epi_acqtimes)):
        reps.append(int(epi_events[i_event].strip().split('Rep#=')[1].split(',')[0]))
        slices.append(int(epi_events[i_event].strip().split('Slice#=')[1].split(',')[0]))
    reps = np.array(reps)
    slices = np.array(slices)

    # get repetitions numbers, acquisition time for each slice of each rep, real or dummy scan
    n_slices = max(slices)+1
    n_reps = int(epi_events.size/n_slices)

    # get acquisition time of each slice for each rep
    slices_table = np.zeros((int(n_reps), n_slices))
    for i_slice in range(n_slices):
        slices_table[:, i_slice] = epi_acqtimes[np.where(slices == i_slice)[0]]

    # get rep number and dummy or real scan
    reps_table = np.zeros((int(n_reps), 2))  # rep number, dummy/real scan (0: dummy scan, 1: real scan)
    idx_new_rep = np.where(slices == min(slices))[0]
    reps_table[:, 0] = reps[idx_new_rep]  # get rep number
    # define for each rep if it is a dummy scan or not
    for i_new_rep in range(0, len(idx_new_rep)):
        if reps_table[i_new_rep, 0] == 1 and reps_table[i_new_rep-1, 0] == 0:
            # if we start the second real scan, the previous one was also a real scan
            reps_table[i_new_rep-1, 1] = 1
            reps_table[i_new_rep, 1] = 1
        elif reps_table[i_new_rep, 0] > 1:
            # if the rep number is more than 1, it was also a real scan
            reps_table[i_new_rep, 1] = 1

    return reps_table, slices_table

def plot_physio(time, physio_sig, epi_acqtime, reps_table, acq_window, out_fname):

    fig = plt.figure(figsize=(20, 9.5))
    plt.title('Saved to: '+out_fname+'_plot.pdf')
    plt.plot(time/1000., physio_sig, '+-', label='physio signal', color='b')
    # add vertical rectangle for each repetition period
    legend_label_counter = [0, 0]
    nSlices = int(len(epi_acqtime)/reps_table.shape[0])
    for i_rep in range(reps_table.shape[0]):
        if reps_table[i_rep, 1] == 1:
            plt.axvspan(epi_acqtime[i_rep*nSlices]/1000., (epi_acqtime[i_rep*nSlices] + acq_window)/1000., facecolor='orange', alpha=0.15, label='acquisition window' if sum(legend_label_counter) == 0 else "_nolegend_")
            plt.axvspan(epi_acqtime[i_rep*nSlices]/1000., epi_acqtime[i_rep*nSlices+nSlices-1]/1000., facecolor='r', alpha=0.25, label='repetitions' if legend_label_counter[0] == 0 else '_nolegend_')
            legend_label_counter[0] += 1
        else:
            plt.axvspan(epi_acqtime[i_rep*nSlices]/1000., (epi_acqtime[i_rep*nSlices] + acq_window)/1000., facecolor='orange', alpha=0.15, label='acquisition window' if sum(legend_label_counter) == 0 else "_nolegend_")
            plt.axvspan(epi_acqtime[i_rep*nSlices]/1000., epi_acqtime[i_rep*nSlices+nSlices-1]/1000., facecolor='gray', alpha=0.25, label='dummy scans' if legend_label_counter[1] == 0 else '_nolegend_')
            legend_label_counter[1] += 1

    # add vertical lines for each slice (each epi event actually)
    for xc in epi_acqtime:
        plt.axvline(x=xc/1000., color='g', label='slices' if np.where(epi_acqtime==xc)[0][0] == 0 else "_nolegend_")
    plt.legend()
    plt.xlabel('Time (s)')
    plt.ylabel('Physio signal')

    plt.show(block=True)
    fig.savefig(out_fname+'_plot.pdf')
    plt.close()

def extract_acqTimes_cmrr(triggerStartTime, acqTime_firstImg, acqStartTime, triggerEndTime):

    """

    :param triggerStartTime: in milliseconds
    :param acqTime_firstImg: datetime object
    :param acqStartTime: datetime object (
    :return: acquisition times in milliseconds
    """

    # remove all triggers start time without paired trigger end time
    idxTrigToKeep = []
    for i_trig in range(len(triggerStartTime)):
        if (i_trig == len(triggerStartTime)-1) and (triggerEndTime > triggerStartTime[i_trig]).any():
            idxTrigToKeep.append(i_trig)
        elif ((triggerEndTime > triggerStartTime[i_trig]) & (triggerEndTime < triggerStartTime[i_trig+1])).any():
            idxTrigToKeep.append(i_trig)
    triggerStartTime_notStopped = triggerStartTime[idxTrigToKeep]

    # get the duration of dummy or auto-calibration scans in microseconds
    seqInitDuration = acqTime_firstImg - acqStartTime

    # only keep trigger times after the sequence initialization period
    triggerFirstImg_idx = np.abs(triggerStartTime_notStopped - seqInitDuration.total_seconds()*1000).argmin()
    acqTimes = triggerStartTime_notStopped[triggerFirstImg_idx:]

    return acqTimes

def plot_physio_cmrr(time, trigger_start_times, trigger_end_times, physio_sig, acq_window, out_fname):

    if trigger_start_times.shape[0] != trigger_end_times.shape[0]:
        os.error("ERROR: Number of start and end times are different.")

    fig = plt.figure("CMRR physiolog signal", figsize=(30, 20))

    plt.plot(time, physio_sig, '+-', label='physio signal', color='b')

    # add vertical rectangle for each trigger signal and add acquisition window
    for i_trig in range(trigger_start_times.shape[0]):
        plt.axvspan(trigger_start_times[i_trig], trigger_start_times[i_trig] + acq_window, facecolor='orange', alpha=0.15, label='acquisition window' if i_trig == 0 else "_nolegend_")
        plt.axvspan(trigger_start_times[i_trig], trigger_end_times[i_trig], facecolor='g', alpha=0.25, label='trigger signal' if i_trig == 0 else "_nolegend_")

    # # add vertical lines for trigger start and end
    # for t_start, t_end in zip(trigger_start_times, trigger_end_times):
    #     plt.axvline(x=t_start, color='g')
    #     plt.axvline(x=t_end, color='r')
    #     plt.text(t_start, 1000, 'Trigger period = '+str(t_end-t_start)+'ms', rotation=90)

    plt.legend()
    plt.xlabel('Time (ms)')
    plt.ylabel('Physio signal')

    plt.show(block=False)

    fig.savefig(out_fname+'_plot.pdf')

if __name__ == "__main__":

    # parse arguments
    parser = argparse.ArgumentParser(description='Plot physiological signal vs. time and return acquisition time for each repetition.')

    optionalArgs = parser._action_groups.pop()
    requiredArgs = parser.add_argument_group('required arguments')

    requiredArgs.add_argument('-i', dest='ifname', help='Path to physio log file.', type=str, required=True)
    requiredArgs.add_argument('-o', dest='ofname', help='Output file name for plot and file storing acquisition times.', type=str, required=True)

    parser._action_groups.append(optionalArgs)

    args = parser.parse_args()

    # params
    param_default = Param()

    # run main
    main(log_fname=args.ifname, out_fname=args.ofname)


text:
import argparse
import numpy as np
from sklearn.naive_bayes import GaussianNB
from joblib import dump

parser = argparse.ArgumentParser()
parser.add_argument('--X_train_dir', dest='X_train_dir', required=True)
parser.add_argument('--y_train_dir', dest='y_train_dir', required=True)
parser.add_argument('--model_dir', dest='model_dir', required=True)

args = parser.parse_args()

print(f"X_train received as {args.X_train_dir}")

x_path = os.path.join(args.X_train_dir, "data.txt")
y_path = os.path.join(args.y_train_dir, "data.txt")

X_train = np.genfromtxt(x_path, delimiter=',')
y_train = np.genfromtxt(y_path, delimiter=',')

# Training (trivial)
model = GaussianNB()
model.fit(X_train, y_train)

model_path = os.path.join(args.model_dir, "model.pkl")
dump(model, model_path)


text:
"""
CPU Scaling Sweeps
"""

# FIXME: broken

import numpy as np

class Scaling():
    def __init__(self, params):
        self.params = [params]
        self.results = []

    def get_args(self):
        for cpus in self.params[0].values:
            yield [(self.params[0].name, cpus)]

    def analyze(self, hf, outdesc):
        import matplotlib
        matplotlib.use('PDF')
        import matplotlib.pyplot as plt

        results = hf['run/time']
        plt.xlabel('Processors')
        plt.ylabel('Time')
        plt.title('Scalability')
        plt.plot(self.params[0].values, results)
        plt.savefig('scale.pdf')
        return results

max_stars_repo_name: zoidy/puq

max_issues_repo_name: zoidy/puq

max_forks_repo_name: zoidy/puq


text:
#%% Imports
from hand_classification_net import handNet, handNet2, trainEpoch, loadDataset, saveObject, loadObject
from HandClassificationDataset import HandClassificationDataset, get_transform

import numpy as np
import matplotlib.pyplot as plt

import torch
import torch.nn as nn
import torch.optim as optim

import random

# torch.cuda.empty_cache()

#%%
reduce_size = (140, 140)

learning_rate = 0.005
n_epoch = 250

#%%

dataset_train = HandClassificationDataset('Train', transforms = get_transform(False))
dataset_test = HandClassificationDataset('Test', transforms = get_transform(False))

b1, b2 = dataset_train[4]

dataset_loader_train = torch.utils.data.DataLoader(dataset_train, batch_size=4, shuffle=True, num_workers=4)
# dataset_loader_test = torch.utils.data.DataLoader(dataset_test, batch_size=4, shuffle=True, num_workers=4)

#%% Training setup

device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")

net = handNet2()

net.train()
net.to(device)

# optimizer = optim.Adam(nn.ParameterList(net.parameters()), lr = learning_rate, weight_decay = 1e-2)
# optimizer = optim.SGD(net.parameters(), lr=0.01, momentum=0.9)
optimizer = torch.optim.Adadelta(net.parameters())

# loss_fn = nn.MSELoss()
loss_fn = nn.CrossEntropyLoss()

# net.computeVisualWeigth()
# net.to(device)

# a = torch.cuda.memory_summary()

#%% Training

test_loss_log = []
train_loss_log = []
train_loss = 0

for epoch in range(n_epoch):
    for local_batch, local_labels in dataset_loader_train:
        # Transfer to GPU
        local_batch, local_labels = local_batch.to(device), local_labels.long().to(device)

        train_loss += trainEpoch(net, [local_batch, local_labels], optimizer, loss_fn)

    train_loss_log.append(float(train_loss))
    if(epoch % 10 == 0): print('EPOCH: ', epoch, ' - Train loss: ', train_loss)

    if(train_loss < 1e0-5): break

    train_loss = 0

#%%

# Plot losses
plt.figure(figsize=(12,8))
plt.semilogy(train_loss_log, label='Train loss')
# plt.semilogy(test_loss_log, label='Test loss')
# plt.semilogy(np.ones(len(test_loss_log)) * 0.8, label = '0.8 line')
plt.xlabel('Epoch')
plt.ylabel('Loss Value')
# plt.xlim(0, 75)
# plt.ylim(0.65, 10)
plt.grid()
plt.legend()
plt.tight_layout()
# plt.savefig("Plot/test_train_error_" + characteristic + "_" + str(ind) + "_" + str(learning_rate) + ".png", bbox_inches  = 'tight')
plt.show()


text:
"""HYPE diagnostic."""
import logging
from pathlib import Path

import dask.array as da
import iris
import numpy
import pandas

from esmvaltool.diag_scripts.shared import (ProvenanceLogger,
                                            get_diagnostic_filename,
                                            group_metadata, run_diagnostic)

logger = logging.getLogger(Path(__file__).name)

def get_provenance_record(attributes):
    """Create a provenance record."""
    ancestor_file = attributes['filename']

    record = {
        'caption': "Forcings for the Hype hydrological model.",
        'domains': ['global'],
        'authors': [
            'pelupessy_inti',
        ],
        'projects': [
            'ewatercycle',
        ],
        'references': [
            'acknow_project',
        ],
        'ancestors': [ancestor_file],
    }
    return record

def get_output_stem(attributes):
    """Get output file stem, specific to HYPE."""
    short_to_stem = dict(tas="Tobs",
                         tasmin="TMINobs",
                         tasmax="TMAXobs",
                         pr="Pobs")

    shortname = attributes["short_name"]
    if shortname in short_to_stem:
        stem = Path(short_to_stem[shortname])
    else:
        stem = Path(attributes['filename']).stem + '_hype'

    stem = attributes['alias'] / stem

    return stem

def get_data_times_and_ids(attributes):
    """Get the data table to be written and the times and indices."""
    input_file = attributes['filename']

    cube = iris.load_cube(input_file)

    data = numpy.array(cube.core_data())

    # Round times to integer number of days
    time_coord = cube.coord('time')
    time_coord.points = da.floor(time_coord.core_points())
    time_coord.bounds = None

    times = [str(x.point.date()) for x in time_coord.cells()]
    ids = cube.coord('shape_id').core_points()

    return data, times, ids

def main(cfg):
    """Process data for use as input to the HYPE hydrological model."""
    input_data = cfg['input_data'].values()
    grouped_input_data = group_metadata(input_data,
                                        'long_name',
                                        sort='dataset')

    for long_name in grouped_input_data:
        logger.info("Processing variable %s", long_name)
        for attributes in grouped_input_data[long_name]:
            logger.info("Processing dataset %s", attributes['dataset'])

            output_file = get_diagnostic_filename(get_output_stem(attributes),
                                                  cfg, 'txt')
            Path(output_file).parent.mkdir(exist_ok=True)

            data, times, ids = get_data_times_and_ids(attributes)

            frame = pandas.DataFrame(data, index=times, columns=ids)

            frame.to_csv(output_file,
                         sep=' ',
                         index_label="DATE",
                         float_format='%.3f')

            # Store provenance
            provenance_record = get_provenance_record(attributes)
            with ProvenanceLogger(cfg) as provenance_logger:
                provenance_logger.log(output_file, provenance_record)

if __name__ == '__main__':

    with run_diagnostic() as config:
        main(config)

max_stars_repo_name: ipelupessy/ESMValTool

max_issues_repo_name: ipelupessy/ESMValTool

max_forks_repo_name: ipelupessy/ESMValTool