text:
from __future__ import division
from __future__ import print_function

import time
import tensorflow as tf
import scipy.sparse as sp

from utils import *
from models import GCN, MLP, GCN_APPRO
import json
from networkx.readwrite import json_graph
import os

# Set random seed
seed = 123
np.random.seed(seed)
tf.set_random_seed(seed)

# Settings
flags = tf.app.flags
FLAGS = flags.FLAGS
flags.DEFINE_string('dataset', 'pubmed', 'Dataset string.')  # 'cora', 'citeseer', 'pubmed'
flags.DEFINE_string('model', 'gcn_appr', 'Model string.')  # 'gcn', 'gcn_appr'
flags.DEFINE_float('learning_rate', 0.001, 'Initial learning rate.')
flags.DEFINE_integer('epochs', 12, 'Number of epochs to train.')
flags.DEFINE_integer('hidden1', 128, 'Number of units in hidden layer 1.')
flags.DEFINE_float('dropout', 0.0, 'Dropout rate (1 - keep probability).')
flags.DEFINE_float('weight_decay', 1e-4, 'Weight for L2 loss on embedding matrix.')
flags.DEFINE_integer('early_stopping', 100, 'Tolerance for early stopping (# of epochs).')
flags.DEFINE_integer('max_degree', 3, 'Maximum Chebyshev polynomial degree.')

# Load data

def iterate_minibatches_listinputs(inputs, batchsize, shuffle=False):
    assert inputs is not None
    numSamples = inputs[0].shape[0]
    if shuffle:
        indices = np.arange(numSamples)
        np.random.shuffle(indices)
    for start_idx in range(0, numSamples - batchsize + 1, batchsize):
        if shuffle:
            excerpt = indices[start_idx:start_idx + batchsize]
        else:
            excerpt = slice(start_idx, start_idx + batchsize)
        yield [input[excerpt] for input in inputs]

def loadRedditFromG(dataset_dir, inputfile):
    f= open(dataset_dir+inputfile)
    objects = []
    for _ in range(pkl.load(f)):
        objects.append(pkl.load(f))
    adj, train_labels, val_labels, test_labels, train_index, val_index, test_index = tuple(objects)
    feats = np.load(dataset_dir + "/reddit-feats.npy")
    return sp.csr_matrix(adj), sp.lil_matrix(feats), train_labels, val_labels, test_labels, train_index, val_index, test_index

def loadRedditFromNPZ(dataset_dir):
    adj = sp.load_npz(dataset_dir+"reddit_adj.npz")
    data = np.load(dataset_dir+"reddit.npz")

    return adj, data['feats'], data['y_train'], data['y_val'], data['y_test'], data['train_index'], data['val_index'], data['test_index']

def transferRedditDataFormat(dataset_dir, output_file):
    G = json_graph.node_link_graph(json.load(open(dataset_dir + "/reddit-G.json")))
    labels = json.load(open(dataset_dir + "/reddit-class_map.json"))

    train_ids = [n for n in G.nodes() if not G.node[n]['val'] and not G.node[n]['test']]
    test_ids = [n for n in G.nodes() if G.node[n]['test']]
    val_ids = [n for n in G.nodes() if G.node[n]['val']]
    train_labels = [labels[i] for i in train_ids]
    test_labels = [labels[i] for i in test_ids]
    val_labels = [labels[i] for i in val_ids]
    feats = np.load(dataset_dir + "/reddit-feats.npy")
    ## Logistic gets thrown off by big counts, so log transform num comments and score
    feats[:, 0] = np.log(feats[:, 0] + 1.0)
    feats[:, 1] = np.log(feats[:, 1] - min(np.min(feats[:, 1]), -1))
    feat_id_map = json.load(open(dataset_dir + "reddit-id_map.json"))
    feat_id_map = {id: val for id, val in feat_id_map.iteritems()}

    # train_feats = feats[[feat_id_map[id] for id in train_ids]]
    # test_feats = feats[[feat_id_map[id] for id in test_ids]]

    # numNode = len(feat_id_map)
    # adj = sp.lil_matrix(np.zeros((numNode,numNode)))
    # for edge in G.edges():
    #     adj[feat_id_map[edge[0]], feat_id_map[edge[1]]] = 1

    train_index = [feat_id_map[id] for id in train_ids]
    val_index = [feat_id_map[id] for id in val_ids]
    test_index = [feat_id_map[id] for id in test_ids]
    np.savez(output_file, feats = feats, y_train=train_labels, y_val=val_labels, y_test = test_labels, train_index = train_index,
             val_index=val_index, test_index = test_index)

def transferLabel2Onehot(labels, N):
    y = np.zeros((len(labels),N))
    for i in range(len(labels)):
        pos = labels[i]
        y[i,pos] =1
    return y

def main(rank1, rank0):

    # config = tf.ConfigProto(device_count={"CPU": 4}, # limit to num_cpu_core CPU usage
    #                 inter_op_parallelism_threads = 1,
    #                 intra_op_parallelism_threads = 4,
    #                 log_device_placement=False)
    adj, features, y_train, y_val, y_test,train_index, val_index, test_index = loadRedditFromNPZ("data/")
    adj = adj+adj.T

    y_train = transferLabel2Onehot(y_train, 41)
    y_val = transferLabel2Onehot(y_val, 41)
    y_test = transferLabel2Onehot(y_test, 41)

    features = sp.lil_matrix(features)

    adj_train = adj[train_index, :][:, train_index]

    numNode_train = adj_train.shape[0]

    train_mask = np.ones((numNode_train,))
    val_mask = np.ones((y_val.shape[0],))
    test_mask = np.ones((y_test.shape[0],))

    # print("numNode", numNode)

    # Some preprocessing
    features = nontuple_preprocess_features(features).todense()
    train_features = features[train_index]

    if FLAGS.model == 'gcn_appr':
        normADJ_train = nontuple_preprocess_adj(adj_train)
        normADJ = nontuple_preprocess_adj(adj)
        # normADJ_val = nontuple_preprocess_adj(adj_val)
        # normADJ_test = nontuple_preprocess_adj(adj_test)

        num_supports = 2
        model_func = GCN_APPRO
    else:
        raise ValueError('Invalid argument for model: ' + str(FLAGS.model))

    # Define placeholders
    placeholders = {
        'support': [tf.sparse_placeholder(tf.float32) for _ in range(num_supports)],
        'features': tf.placeholder(tf.float32, shape=(None, features.shape[1])),
        'labels': tf.placeholder(tf.float32, shape=(None, y_train.shape[1])),
        'labels_mask': tf.placeholder(tf.int32),
        'dropout': tf.placeholder_with_default(0., shape=()),
        'num_features_nonzero': tf.placeholder(tf.int32)  # helper variable for sparse dropout
    }

    # Create model
    model = model_func(placeholders, input_dim=features.shape[-1], logging=True)

    # Initialize session
    sess = tf.Session()

    # Define model evaluation function
    def evaluate(features, support, labels, mask, placeholders):
        t_test = time.time()
        feed_dict_val = construct_feed_dict(features, support, labels, mask, placeholders)
        outs_val = sess.run([model.loss, model.accuracy], feed_dict=feed_dict_val)
        return outs_val[0], outs_val[1], (time.time() - t_test)

    # Init variables
    sess.run(tf.global_variables_initializer())
    saver = tf.train.Saver()

    cost_val = []

    p0 = column_prop(normADJ_train)

    # testSupport = [sparse_to_tuple(normADJ), sparse_to_tuple(normADJ)]
    valSupport = [sparse_to_tuple(normADJ), sparse_to_tuple(normADJ[val_index, :])]
    testSupport = [sparse_to_tuple(normADJ), sparse_to_tuple(normADJ[test_index, :])]

    t = time.time()
    maxACC = 0.0
    # Train model
    for epoch in range(FLAGS.epochs):
        t1 = time.time()

        n = 0
        for batch in iterate_minibatches_listinputs([normADJ_train, y_train, train_mask], batchsize=256, shuffle=True):
            [normADJ_batch, y_train_batch, train_mask_batch] = batch
            if sum(train_mask_batch) < 1:
                continue

            p1 = column_prop(normADJ_batch)
            q1 = np.random.choice(np.arange(numNode_train), rank1, replace=False, p=p1)  # top layer
            # q0 = np.random.choice(np.arange(numNode_train), rank0, p=p0)  # bottom layer
            support1 = sparse_to_tuple(normADJ_batch[:, q1].dot(sp.diags(1.0 / (p1[q1] * rank1))))

            p2 = column_prop(normADJ_train[q1, :])
            q0 = np.random.choice(np.arange(numNode_train), rank0, replace=False, p=p2)

            support0 = sparse_to_tuple(normADJ_train[q1, :][:, q0])
            features_inputs = np.diag(1.0 / (p2[q0] * rank0)).dot(train_features[q0, :])  # selected nodes for approximation

            # distr = np.nonzero(np.sum(normADJ_batch, axis=0))[1]
            # if rank1 > len(distr):
            #     q1 = distr
            # else:
            #     q1 = np.random.choice(distr, rank1, replace=False)  # top layer
            # distr0 = np.nonzero(np.sum(normADJ_train[q1,:], axis=0))[1]
            # if rank0 > len(distr0):
            #     q0 = distr0
            # else:
            #     q0 = np.random.choice(distr0, rank0, replace=False)  # top layer
            #
            # support1 = sparse_to_tuple(normADJ_batch[:, q1].dot(sp.diags(1.0 / (p0[q1] * rank1))))
            # support0 = sparse_to_tuple(normADJ_train[q1, :][:, q0])
            # features_inputs = np.diag(1.0 / (p0[q0] * rank0)).dot(train_features[q0, :])  # selected nodes for approximation

            # Construct feed dictionary
            feed_dict = construct_feed_dict(features_inputs, [support0, support1], y_train_batch, train_mask_batch,
                                            placeholders)
            feed_dict.update({placeholders['dropout']: FLAGS.dropout})

            # Training step
            outs = sess.run([model.opt_op, model.loss, model.accuracy], feed_dict=feed_dict)
            n=n+1

        # Validation
        cost, acc, duration = evaluate(features, valSupport, y_val, val_mask, placeholders)
        cost_val.append(cost)

        if epoch > 50 and acc>maxACC:
            maxACC = acc
            save_path = saver.save(sess, "tmp/tmp_redditModel.ckpt")

        # Print results
        print("Epoch:", '%04d' % (epoch + 1), "train_loss=", "{:.5f}".format(outs[1]),
              "train_acc=", "{:.5f}".format(outs[2]), "val_loss=", "{:.5f}".format(cost),
              "val_acc=", "{:.5f}".format(acc), "time per batch=", "{:.5f}".format((time.time() - t1)/n))

        if epoch > FLAGS.early_stopping and np.mean(cost_val[-2:]) > np.mean(cost_val[-(FLAGS.early_stopping + 1):-1]):
            # print("Early stopping...")
            break

    train_duration = time.time() - t
    # Testing
    if os.path.exists("tmp/tmp_redditModel.ckpt"):
        saver.restore(sess, "tmp/tmp_redditModel.ckpt")
    test_cost, test_acc, test_duration = evaluate(features, testSupport, y_test, test_mask,
                                                  placeholders)
    print("rank1 = {}".format(rank1), "rank0 = {}".format(rank0), "cost=", "{:.5f}".format(test_cost),
          "accuracy=", "{:.5f}".format(test_acc), "training time=", "{:.5f}".format(train_duration),
          "epoch = {}".format(epoch + 1),
          "test time=", "{:.5f}".format(test_duration))

def transferG2ADJ():
    G = json_graph.node_link_graph(json.load(open("reddit/reddit-G.json")))
    feat_id_map = json.load(open("reddit/reddit-id_map.json"))
    feat_id_map = {id: val for id, val in feat_id_map.iteritems()}
    numNode = len(feat_id_map)
    adj = np.zeros((numNode, numNode))
    newEdges0 = [feat_id_map[edge[0]] for edge in G.edges()]
    newEdges1 = [feat_id_map[edge[1]] for edge in G.edges()]

    # for edge in G.edges():
    #     adj[feat_id_map[edge[0]], feat_id_map[edge[1]]] = 1
    adj = sp.csr_matrix((np.ones((len(newEdges0),)), (newEdges0, newEdges1)), shape=(numNode, numNode))
    sp.save_npz("reddit_adj.npz", adj)

if __name__=="__main__":
    # transferRedditDataFormat("reddit/","data/reddit.npz")

    main(100,100)
    # for k in [50, 100, 200, 400]:
    #     main(100, k)

max_stars_repo_name: CEfanmin/DataMiningProjects

max_issues_repo_name: CEfanmin/DataMiningProjects

max_forks_repo_name: CEfanmin/DataMiningProjects

reason: import scipy,from networkx


text:
import os
import random
import numpy as np
import networkx as nx
import matplotlib.pyplot as plt
import time

from mesa import Model
from mesa.datacollection import DataCollector
from mesa.space import MultiGrid
from mesa.time import RandomActivation

from .agent import Human, Wall, FireExit, Furniture, Fire, Door

class FireEvacuation(Model):
    MIN_HEALTH = 0.75
    MAX_HEALTH = 1

    MIN_SPEED = 1
    MAX_SPEED = 2

    MIN_NERVOUSNESS = 1
    MAX_NERVOUSNESS = 10

    MIN_EXPERIENCE = 1
    MAX_EXPERIENCE = 10

    MIN_VISION = 1
    # MAX_VISION is simply the size of the grid

    def __init__(self, floor_plan_file, human_count, collaboration_percentage, fire_probability, visualise_vision, random_spawn, save_plots):
        # Load floorplan
        # floorplan = np.genfromtxt(path.join("fire_evacuation/floorplans/", floor_plan_file))
        with open(os.path.join("fire_evacuation/floorplans/", floor_plan_file), "rt") as f:
            floorplan = np.matrix([line.strip().split() for line in f.readlines()])

        # Rotate the floorplan so it's interpreted as seen in the text file
        floorplan = np.rot90(floorplan, 3)

        # Check what dimension our floorplan is
        width, height = np.shape(floorplan)

        # Init params
        self.width = width
        self.height = height
        self.human_count = human_count
        self.collaboration_percentage = collaboration_percentage
        self.visualise_vision = visualise_vision
        self.fire_probability = fire_probability
        self.fire_started = False  # Turns to true when a fire has started
        self.save_plots = save_plots

        # Set up model objects
        self.schedule = RandomActivation(self)

        self.grid = MultiGrid(height, width, torus=False)

        # Used to start a fire at a random furniture location
        self.furniture_list = []

        # Used to easily see if a location is a FireExit or Door, since this needs to be done a lot
        self.fire_exit_list = []
        self.door_list = []

        # If random spawn is false, spawn_list will contain the list of possible spawn points according to the floorplan
        self.random_spawn = random_spawn
        self.spawn_list = []

        # Load floorplan objects
        for (x, y), value in np.ndenumerate(floorplan):
            value = str(value)
            floor_object = None
            if value is "W":
                floor_object = Wall((x, y), self)
            elif value is "E":
                floor_object = FireExit((x, y), self)
                self.fire_exit_list.append((x, y))
                self.door_list.append((x, y))  # Add fire exits to doors as well, since, well, they are
            elif value is "F":
                floor_object = Furniture((x, y), self)
                self.furniture_list.append((x, y))
            elif value is "D":
                floor_object = Door((x, y), self)
                self.door_list.append((x, y))
            elif value is "S":
                self.spawn_list.append((x, y))

            if floor_object:
                self.grid.place_agent(floor_object, (x, y))
                self.schedule.add(floor_object)

        # Create a graph of traversable routes, used by agents for pathing
        self.graph = nx.Graph()
        for agents, x, y in self.grid.coord_iter():
            pos = (x, y)

            # If the location is empty, or a door
            if not agents or any(isinstance(agent, Door) for agent in agents):
                neighbors = self.grid.get_neighborhood(pos, moore=True, include_center=True, radius=1)

                for neighbor in neighbors:
                    # If there is contents at this location and they are not Doors or FireExits, skip them
                    if not self.grid.is_cell_empty(neighbor) and neighbor not in self.door_list:
                        continue

                    self.graph.add_edge(pos, neighbor)

        # Collects statistics from our model run
        self.datacollector = DataCollector(
            {
                "Alive": lambda m: self.count_human_status(m, Human.Status.ALIVE),
                "Dead": lambda m: self.count_human_status(m, Human.Status.DEAD),
                "Escaped": lambda m: self.count_human_status(m, Human.Status.ESCAPED),
                "Incapacitated": lambda m: self.count_human_mobility(m, Human.Mobility.INCAPACITATED),
                "Normal": lambda m: self.count_human_mobility(m, Human.Mobility.NORMAL),
                "Panic": lambda m: self.count_human_mobility(m, Human.Mobility.PANIC),
                "Verbal Collaboration": lambda m: self.count_human_collaboration(m, Human.Action.VERBAL_SUPPORT),
                "Physical Collaboration": lambda m: self.count_human_collaboration(m, Human.Action.PHYSICAL_SUPPORT),
                "Morale Collaboration": lambda m: self.count_human_collaboration(m, Human.Action.MORALE_SUPPORT)
            }
        )

        # Calculate how many agents will be collaborators
        number_collaborators = int(round(self.human_count * (self.collaboration_percentage / 100)))

        # Start placing human agents
        for i in range(0, self.human_count):
            if self.random_spawn:  # Place human agents randomly
                pos = self.grid.find_empty()
            else:  # Place human agents at specified spawn locations
                pos = random.choice(self.spawn_list)

            if pos:
                # Create a random human
                health = random.randint(self.MIN_HEALTH * 100, self.MAX_HEALTH * 100) / 100
                speed = random.randint(self.MIN_SPEED, self.MAX_SPEED)

                if number_collaborators > 0:
                    collaborates = True
                    number_collaborators -= 1
                else:
                    collaborates = False

                # Vision statistics obtained from http://www.who.int/blindness/GLOBALDATAFINALforweb.pdf
                vision_distribution = [0.0058, 0.0365, 0.0424, 0.9153]
                vision = int(np.random.choice(np.arange(self.MIN_VISION, self.width + 1, (self.width / len(vision_distribution))), p=vision_distribution))

                nervousness_distribution = [0.025, 0.025, 0.1, 0.1, 0.1, 0.3, 0.2, 0.1, 0.025, 0.025]  # Distribution with slight higher weighting for above median nerovusness
                nervousness = int(np.random.choice(range(self.MIN_NERVOUSNESS, self.MAX_NERVOUSNESS + 1), p=nervousness_distribution))  # Random choice starting at 1 and up to and including 10

                experience = random.randint(self.MIN_EXPERIENCE, self.MAX_EXPERIENCE)

                belief_distribution = [0.9, 0.1]  # [Believes, Doesn't Believe]
                believes_alarm = np.random.choice([True, False], p=belief_distribution)

                human = Human(pos, health=health, speed=speed, vision=vision, collaborates=collaborates, nervousness=nervousness, experience=experience, believes_alarm=believes_alarm, model=self)

                self.grid.place_agent(human, pos)
                self.schedule.add(human)
            else:
                print("No tile empty for human placement!")

        self.running = True

    # Plots line charts of various statistics from a run
    def save_figures(self):
        DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
        OUTPUT_DIR = DIR + "/output"

        results = self.datacollector.get_model_vars_dataframe()

        dpi = 100
        fig, axes = plt.subplots(figsize=(1920 / dpi, 1080 / dpi), dpi=dpi, nrows=1, ncols=3)

        status_results = results.loc[:, ['Alive', 'Dead', 'Escaped']]
        status_plot = status_results.plot(ax=axes[0])
        status_plot.set_title("Human Status")
        status_plot.set_xlabel("Simulation Step")
        status_plot.set_ylabel("Count")

        mobility_results = results.loc[:, ['Incapacitated', 'Normal', 'Panic']]
        mobility_plot = mobility_results.plot(ax=axes[1])
        mobility_plot.set_title("Human Mobility")
        mobility_plot.set_xlabel("Simulation Step")
        mobility_plot.set_ylabel("Count")

        collaboration_results = results.loc[:, ['Verbal Collaboration', 'Physical Collaboration', 'Morale Collaboration']]
        collaboration_plot = collaboration_results.plot(ax=axes[2])
        collaboration_plot.set_title("Human Collaboration")
        collaboration_plot.set_xlabel("Simulation Step")
        collaboration_plot.set_ylabel("Successful Attempts")
        collaboration_plot.set_ylim(ymin=0)

        timestr = time.strftime("%Y%m%d-%H%M%S")
        plt.suptitle("Percentage Collaborating: " + str(self.collaboration_percentage) + "%, Number of Human Agents: " + str(self.human_count), fontsize=16)
        plt.savefig(OUTPUT_DIR + "/model_graphs/" + timestr + ".png")
        plt.close(fig)

    # Starts a fire at a random piece of furniture with file_probability chance
    def start_fire(self):
        rand = random.random()
        if rand < self.fire_probability:
            fire_furniture = random.choice(self.furniture_list)
            fire = Fire(fire_furniture, self)
            self.grid.place_agent(fire, fire_furniture)
            self.schedule.add(fire)
            self.fire_started = True
            print("Fire started at:", fire_furniture)

    def step(self):
        """
        Advance the model by one step.
        """

        self.schedule.step()

        # If there's no fire yet, attempt to start one
        if not self.fire_started:
            self.start_fire()

        self.datacollector.collect(self)

        # If no more agents are alive, stop the model and collect the results
        if self.count_human_status(self, Human.Status.ALIVE) == 0:
            self.running = False

            if self.save_plots:
                self.save_figures()

    @staticmethod
    def count_human_collaboration(model, collaboration_type):
        """
        Helper method to count the number of collaborations performed by Human agents in the model
        """

        count = 0
        for agent in model.schedule.agents:
            if isinstance(agent, Human):
                if collaboration_type == Human.Action.VERBAL_SUPPORT:
                    count += agent.get_verbal_collaboration_count()
                elif collaboration_type == Human.Action.MORALE_SUPPORT:
                    count += agent.get_morale_collaboration_count()
                elif collaboration_type == Human.Action.PHYSICAL_SUPPORT:
                    count += agent.get_physical_collaboration_count()

        return count

    @staticmethod
    def count_human_status(model, status):
        """
        Helper method to count the status of Human agents in the model
        """
        count = 0
        for agent in model.schedule.agents:
            if isinstance(agent, Human):
                if agent.get_status() == status:
                    count += 1
        return count

    @staticmethod
    def count_human_mobility(model, mobility):
        """
        Helper method to count the mobility of Human agents in the model
        """
        count = 0
        for agent in model.schedule.agents:
            if isinstance(agent, Human):
                if agent.get_mobility() == mobility:
                    count += 1
        return count

ext: py

reason: import numpy,import networkx


text:
from taylorDiagram import plot_Taylor_graph
import matplotlib.pyplot as plt
from PyEMD import EEMD
from hht import hht
from hht import plot_imfs
from hht import plot_frequency
import waipy
from scipy.stats import norm
import numpy as np
# from matplotlib.ticker import FuncFormatter

def millions(x, pos):
    'The two args are the value and tick position'
    return '%1.1fY' % (x/365.0)

plt.rcParams.update({'font.size': 10})
fontsize = 13
lengendfontsize = 12
col = ['plum', 'darkorchid', 'blue', 'navy', 'deepskyblue', 'darkcyan', 'seagreen', 'darkgreen',
       'olivedrab', 'gold', 'tan', 'red', 'palevioletred', 'm']
start_year = 1991
end_year = 2014

# from matplotlib import colors as mcolors
# colors = dict(mcolors.BASE_COLORS, **mcolors.CSS4_COLORS)
# by_hsv = sorted((tuple(mcolors.rgb_to_hsv(mcolors.to_rgba(color)[:3])), name)
#                 for name, color in colors.items())
# col = [name for hsv, name in by_hsv][]
# def plot_imfs(signal, imfs, time_samples = None, fig=None):
#     ''' Author jaidevd https://github.com/jaidevd/pyhht/blob/dev/pyhht/visualization.py '''
#     '''Original function from pyhht, but without plt.show()'''
#     n_imfs = imfs.shape[0]
#     # print(np.abs(imfs[:-1, :]))
#     # axis_extent = max(np.max(np.abs(imfs[:-1, :]), axis=0))
#     # Plot original signal
#     ax = plt.subplot(n_imfs + 1, 1, 1)
#     ax.plot(time_samples, signal)
#     ax.axis([time_samples[0], time_samples[-1], signal.min(), signal.max()])
#     ax.tick_params(which='both', left=False, bottom=False, labelleft=False,
#                    labelbottom=False)
#     ax.grid(False)
#     ax.set_ylabel('Signal')
#     ax.set_title('Empirical Mode Decomposition')
#
#     # Plot the IMFs
#     for i in range(n_imfs - 1):
#         # print(i + 2)
#         ax = plt.subplot(n_imfs + 1, 1, i + 2)
#         ax.plot(time_samples, imfs[i, :])
#         # ax.axis([time_samples[0], time_samples[-1], -axis_extent, axis_extent])
#         ax.tick_params(which='both', left=False, bottom=False, labelleft=False,
#                        labelbottom=False)
#         ax.grid(False)
#         ax.set_ylabel('imf' + str(i + 1), fontsize=fontsize)
#
#     # Plot the residue
#     ax = plt.subplot(n_imfs + 1, 1, n_imfs + 1)
#     ax.plot(time_samples, imfs[-1, :], 'r')
#     ax.axis('tight')
#     ax.tick_params(which='both', left=False, bottom=False, labelleft=False,
#                    labelbottom=False)
#     ax.grid(False)
#     ax.set_ylabel('res.', fontsize=fontsize)
#     return ax

class spectrum_post(object):

    def __init__(self, filedir, h_site_name_obs, day_obs, day_mod, variable_name):
        [d_obs, d_t_obs, d_unit_obs] = day_obs
        [d_mod, d_t_mod, d_unit_mod] = day_mod
        self.d_obs = d_obs
        self.d_mod = d_mod
        self.d_t_obs = d_t_obs
        self.d_unit_obs = d_unit_obs
        self.sitename = h_site_name_obs
        self.variable = variable_name
        self.filedir = filedir

    def plot_decomposer_imf(self):

        d_obs = self.d_obs
        d_mod = self.d_mod
        d_t_obs = self.d_t_obs
        scores = []

        for j, site in enumerate(self.sitename):

            if self.sitename.mask[j]:
                continue
            print('Process on Decomposer_IMF_' + site + '_No.' + str(j) + '!')
            fig0 = plt.figure(figsize=(8, 4))
            fig3 = plt.figure(figsize=(5, 5))
            data = d_obs[j, :].compressed()
            time = d_t_obs[~d_obs[j, :].mask]
            eemd = EEMD(trials=5)

            if len(data) > 0:
                imfs = eemd.eemd(data)
                # print('obs',imfs.shape)
                if len(imfs) >= 1:
                    ax0 = fig0.add_subplot(1, 2, 1)
                    ax0.plot(time, (imfs[len(imfs) - 1]), 'k-', label='Observed')

                    d_d_obs = np.asarray([str(start_year + int(x) / 365) + (
                        '0' + str(int(x) % 365 / 31 + 1) if int(x) % 365 / 31 < 9 else str(int(x) % 365 / 31 + 1)) for x
                                          in
                                          time])
                    ax0.xaxis.set_ticks(
                        [time[0], time[2 * len(time) / 5],
                         time[4 * len(time) / 5]])
                    ax0.set_xticklabels(
                        [d_d_obs[0], d_d_obs[2 * len(d_d_obs) / 5],
                         d_d_obs[4 * len(d_d_obs) / 5]])

                ## hht spectrum
                if len(imfs) >= 1:
                    fig3, freq = hht(data, imfs, time, 1, fig3)

            fig3.savefig(self.filedir + self.variable + '/' + site + '_hht_IMF_observed' + self.variable + '.png', bbox_inches='tight')

            fig1 = plt.figure(figsize=(4 * (len(d_mod)+1), 8))
            fig2 = plt.figure(figsize=(4 * (len(d_mod)+1), 8))
            fig1.subplots_adjust(wspace=0.5, hspace=0.3)
            fig2.subplots_adjust(wspace=0.5, hspace=0.3)

            if len(data) > 0:
                if len(imfs) >= 1:
                    fig1 = plot_imfs(data, imfs, time_samples=time, fig=fig1, no=1, m=len(d_mod))
                    fig2 = plot_frequency(data, freq.T, time_samples=time, fig=fig2, no=1, m=len(d_mod))

                models1 = []
                datamask = []
                data1 = imfs[len(imfs) - 1]
                for m in range(len(d_mod)):
                    ## hht spectrum
                    eemd = EEMD(trials=5)
                    fig3 = plt.figure(figsize=(5, 5))
                    data2 = d_mod[m][j, :][~d_obs[j, :].mask]
                    imfs = eemd.eemd(data2.compressed())
                    # print('mod'+str(m), imfs.shape)
                    if len(imfs) >= 1:
                        fig3, freq = hht(data2.compressed(), imfs, time[~data2.mask], 1, fig3)
                    fig3.savefig(self.filedir + self.variable + '/' + site + '_hht_IMF_model' + str(m + 1) + self.variable + '.png', bbox_inches='tight')
                    if len(imfs) >= 1:
                        fig1 = plot_imfs(data2.compressed(), imfs, time_samples=time[~data2.mask], fig=fig1, no=m+2, m=len(d_mod))
                        fig2 = plot_frequency(data2.compressed(), freq.T, time_samples=time[~data2.mask], fig=fig2, no=m+2, m=len(d_mod))
                        ax0.plot(time[~data2.mask], (imfs[len(imfs) - 1]), '-', label='Model' + str(m+1), c=col[m])
                        models1.append(imfs[len(imfs) - 1])
                        datamask.append(data2)

                ax0.set_xlabel('Time', fontsize=fontsize)
                ax0.set_ylabel('' + self.variable + '(' + self.d_unit_obs + ')', fontsize=fontsize)
                ax0.yaxis.tick_right()
                ax0.yaxis.set_label_position("right")
                ax0.legend(bbox_to_anchor=(-0.05, 1), shadow=False, fontsize='medium')
                plot_Taylor_graph(data1, models1, fig0, 122, datamask=datamask)
            else:
                print("'Data's length is too short !")

            fig1.savefig(self.filedir + self.variable + '/' + site + '_Decompose_IMF_' + self.variable + '.png', bbox_inches='tight')
            fig2.savefig(self.filedir + self.variable + '/' + site + '_deviation_IMF_' + self.variable + '.png', bbox_inches='tight')

            fig0.subplots_adjust(left=0.1, hspace=0.25, wspace=0.55)
            fig0.savefig(self.filedir  + self.variable + '/' + site + '_' + 'IMF' + '_' + self.variable + '.png', bbox_inches='tight')

            plt.close('all')

        return scores

    def plot_wavelet(self):
        d_obs = self.d_obs
        d_mod = self.d_mod
        d_t_obs = self.d_t_obs

        """ plot data wavelet """
        scores = []
        for j, site in enumerate(self.sitename):
            if self.sitename.mask[j]:
                continue
            print('Process on Wavelet_' + site + '_No.' + str(j) + '!')
            data = d_obs[j, :].compressed()
            fig3 = plt.figure(figsize=(8, 8))
            if len(data) > 0:
                time_data = d_t_obs[~d_obs[j, :].mask]
                # time_data = d_t_obs
                result = waipy.cwt(data, 1, 1, 0.125, 2, 4 / 0.125, 0.72, 6, mother='Morlet', name='Obs')
                waipy.wavelet_plot('Obs', time_data, data, 0.03125, result, fig3, unit=self.d_unit_obs)
                # plt.tight_layout()
            for m in range(len(d_mod)):
                fig4 = plt.figure(figsize=(8, 8))
                data2 = d_mod[m][j, :][~d_obs[j, :].mask]
                data = data2.compressed() - d_obs[j, :].compressed()[~data2.mask]
                if len(data) > 0:
                    result = waipy.cwt(data, 1, 1, 0.125, 2, 4 / 0.125, 0.72, 6, mother='Morlet', name='Obs - Mod' + str(m+1))
                    waipy.wavelet_plot('Obs - Mod' + str(m+1), time_data[~data2.mask], data, 0.03125, result, fig4, unit=self.d_unit_obs, m=m)
                    # plt.tight_layout()
                fig4.savefig(self.filedir + self.variable + '/' + site + 'model' + str(m) + '_wavelet_' + self.variable + '.png', bbox_inches='tight')
            fig3.savefig(self.filedir + self.variable + '/' + site + '_Wavelet_' + self.variable + '.png',
                         bbox_inches='tight')
            plt.close('all')
        return scores

    def plot_spectrum(self):
        import waipy, math
        import numpy as np
        import matplotlib.pyplot as plt
        d_obs = self.d_obs
        d_mod = self.d_mod
        d_t_obs = self.d_t_obs
        scores = []
        """ Plot global wavelet spectrum """
        # col = ['palevioletred', 'm', 'plum', 'darkorchid', 'blue', 'navy', 'deepskyblue', 'darkcyan', 'seagreen',
        #        'darkgreen', 'olivedrab', 'gold', 'tan', 'red']
        for j, site in enumerate(self.sitename):
            if self.sitename.mask[j]:
                continue
            print('Process on Spectrum_' + site + '_No.' + str(j) + '!')
            data = d_obs[j, :].compressed()
            if len(data) > 0:
                result = waipy.cwt(data, 1, 1, 0.125, 2, 4 / 0.125, 0.72, 6, mother='Morlet', name='Data')
                loc_o, scale_o = norm.fit(result['global_ws'])
                fig4 = plt.figure(figsize=(4, 4))
                ax4 = fig4.add_subplot(1, 1, 1)
                # f1, sxx1 = waipy.fft(data)
                # ax.plot(np.log2(1 / f1 * result['dt']), sxx1, 'red', label='Fourier spectrum')
                # plt.suptitle(self.variable + ' ( ' + self.d_unit_obs + ' )', fontsize=8)
                ax4.plot(np.log2(result['period']), result['global_ws'], 'k-', label='Wavelet spectrum')
                ax4.plot(np.log2(result['period']), result['global_signif'], 'r--', label='95% confidence spectrum')
            model_score = []

            for m in range(len(d_mod)):
                data2 = d_mod[m][j, :][~d_obs[j, :].mask]
                data = data2.compressed()
                # data = d_mod[m][j, :][~d_obs[j, :].mask]
                if len(data) > 0:
                    result_temp = waipy.cwt(data, 1, 1, 0.125, 2, 4 / 0.125, 0.72, 6, mother='Morlet', name='Data')
                    loc_m, scale_m = norm.fit(result_temp['global_ws'])
                    model_score.append(abs(loc_m-loc_o))
                    ax4.plot(np.log2(result_temp['period']), result_temp['global_ws'], label='Model' + str(m), c=col[m])
                else:
                    model_score.append(0.5)
            model_score = [i/max(model_score) for i in model_score]
            scores.append(model_score)

            ax4.legend(bbox_to_anchor=(1.05, 1), loc=2, fontsize=lengendfontsize)
            # ax4.set_ylim(0, 1.25 * np.max(result['global_ws']))
            ax4.set_ylabel('Power', fontsize=fontsize)
            ax4.set_title('Global Wavelet Spectrum', fontsize=fontsize)
            y_min = int(min(np.log2(result['period'][0]), np.log2(result_temp['period'][0])))
            y_max = int(max(np.log2(result['period'][-1]) + 1, np.log2(result_temp['period'][-1]) + 1))
            yt = range(y_min, y_max, 3)  # create the vector of period
            Yticks = [float(math.pow(2, p)) for p in yt]  # make 2^periods
            ax4.set_xticks(yt)
            ax4.set_xticklabels(Yticks)
            ax4.set_xlim(xmin=(np.log2(np.min(result['period']))), xmax=(np.log2(np.max(result['period']))))
            plt.tight_layout()
            fig4.savefig(self.filedir  + self.variable + '/' + site + '_spectrum_' + self.variable + '.png', bbox_inches='tight')
            plt.close('all')
        return scores

    def plot_taylor_gram(self):
        d_obs = self.d_obs
        d_mod = self.d_mod
        d_t_obs = self.d_t_obs
        scores = []
        """  Taylor diagram """
        for j, site in enumerate(self.sitename):
            if self.sitename.mask[j]:
                continue
            print('Process on Taylor_' + site + '_No.' + str(j) + '!')
            data1 = d_obs[j, :].compressed()
            models1 = []
            fig7 = plt.figure(figsize=(8, 8))
            for m in range(len(d_mod)):
                models1.append(d_mod[m][j, :][~d_obs[j, :].mask])

            # print(data1.shape, models1[0].shape)
            plot_Taylor_graph(data1, models1, fig7, 111)
            fig7.savefig(self.filedir + self.variable + '/' +site + '_taylor_' + self.variable + '.png', bbox_inches='tight')
            plt.close('all')
        return scores

    def plot_spectrum_score(self):
        import waipy, math
        import numpy as np
        import matplotlib.pyplot as plt
        d_obs = self.d_obs
        d_mod = self.d_mod
        d_t_obs = self.d_t_obs
        scores = []
        """ Plot global wavelet spectrum """
        for j, site in enumerate(self.sitename):
            if self.sitename.mask[j]:
                continue
            print('Process on Spectrum_' + site + '_No.' + str(j) + '!')
            data = d_obs[j, :].compressed()
            result = waipy.cwt(data, 1, 1, 0.125, 2, 4 / 0.125, 0.72, 6, mother='Morlet', name='Data')
            loc_o, scale_o = norm.fit(result['global_ws'])
            model_score = []
            for m in range(len(d_mod)):
                data = d_mod[m][j, :][~d_obs[j, :].mask]
                result_temp = waipy.cwt(data, 1, 1, 0.125, 2, 4 / 0.125, 0.72, 6, mother='Morlet', name='Data')
                loc_m, scale_m = norm.fit(result_temp['global_ws'])
                model_score.append(abs(loc_m - loc_o))
            model_score = model_score / max(model_score)
            scores.append(model_score)
        return scores

def spectrum_analysis(filedir,h_site_name_obs, day_obs, day_mod, variable_name):
    # plot frequency frequency
    f2 = spectrum_post(filedir, h_site_name_obs, day_obs, day_mod, variable_name)
    scores_decomposeimf = f2.plot_decomposer_imf()
    score_wavelet = f2.plot_wavelet()
    score_spectrum = f2.plot_spectrum()
    return scores_decomposeimf, score_wavelet, score_spectrum

max_stars_repo_name: liangliannie/PyCLM

max_issues_repo_name: liangliannie/PyCLM

max_forks_repo_name: liangliannie/PyCLM

reason: import numpy,from scipy


text:
from __future__ import absolute_import, division, print_function, unicode_literals

import sys
import os
import platform
import time
import numpy as np
import pandas as pd
import copy
# import matplotlb.pyplot as plt
import tensorflow as tf
from collections import defaultdict
from tensorflow.python.framework import ops
import pathlib
import random
from model_utils.optimizers import Optimizers
from model_utils.model import Model
from model_utils.losses import Losses
from base_config import get_configs
from data_processing import Dataset

physical_devices = tf.config.experimental.list_physical_devices('GPU')
for physical_device in physical_devices:
    tf.config.experimental.set_memory_growth(physical_device, True)

class Train(object):

    def __init__(self, config, dataset):
        self.config = config
        self.dataset = dataset
        self.dataset.generate_dataset()
        self.model = Model(self.config, self.dataset).get_model()
        self.target_index = self.dataset.target_index
        self.optimizer = Optimizers(self.config).get_optimizer()
        self.losses = Losses(self.config, self.target_index)

        # Load data
        self.train_set = self.dataset.train_set
        self.valid_set = self.dataset.valid_set

        self.train_set = self.train_set.shuffle(buffer_size=10000, seed=self.config.seed)
        self.train_set = self.train_set.batch(batch_size=self.config.batch_size)
        self.valid_set = self.valid_set.batch(batch_size=self.config.batch_size)

        # Create model level directories for chkpts and logs
        self.model_dir = os.path.join(self.config.experiments_dir, self.config.model_dir)
        self.chkpts_dir = os.path.join(self.model_dir, 'chkpts')
        self.train_log_dir = os.path.join(self.model_dir, 'train_log')

        train_dirs = [self.model_dir, self.chkpts_dir, self.train_log_dir]
        print(train_dirs)
        for dirname in train_dirs:
            if not os.path.isdir(dirname):
                print("Creating dir %s" % dirname)
                pathlib.Path(dirname).mkdir(parents=True, exist_ok=True)

        self.min_valid_mse = np.inf
        self.min_valid_uq_loss = np.inf

        # initialize grad norm
        self._grad_norm = tf.constant(1.0)

        # Create Batches
        print("Creating batches ...")
        self._batches = []
        for (batch_n, train_set_items) in enumerate(self.train_set):
            inp_idxs = train_set_items[0]
            tar_idxs = train_set_items[1]
            pre_metadata = train_set_items[2]
            inp, target, metadata = self.dataset.get_batch(inp_idxs, tar_idxs, pre_metadata)
            self._batches.append((inp, target, metadata))

        self._valid_batches = []
        for (batch_n, train_set_items) in enumerate(self.valid_set):
            inp_idxs = train_set_items[0]
            tar_idxs = train_set_items[1]
            pre_metadata = train_set_items[2]
            inp_val, target_val, metadata_val = self.dataset.get_batch(inp_idxs, tar_idxs, pre_metadata)
            self._valid_batches.append((inp_val, target_val, metadata_val))

    def train(self):

        # load from saved weights
        if self.config.load_saved_weights:
            checkpoint_path = os.path.join(self.chkpts_dir, "chkpt")
            self.model.load_weights(checkpoint_path)

        print("Training in progress ...")
        # Training step
        epochs = self.config.max_epoch

        # Name of the checkpoint files
        checkpoint_prefix = os.path.join(self.chkpts_dir, "chkpt")

        # initialize logging
        train_logs_batch = defaultdict(list)
        train_logs_epoch = defaultdict(list)
        self.model.save_weights(checkpoint_prefix)

        start = time.time()
        for epoch in range(epochs):
            # initializing the hidden state at the start of every epoch
            # initially hidden is None
            hidden = self.model.reset_states()

            mse_steps, uq_loss_steps = [], []

            # for (batch_n, train_set_items) in enumerate(self.train_set):
            #     inp_idxs = train_set_items[0]
            #     tar_idxs = train_set_items[1]
            #     pre_metadata = train_set_items[2]
            #     inp, target, metadata = self.dataset.get_batch(inp_idxs, tar_idxs, pre_metadata)

            random.shuffle(self._batches)

            for (batch_n, cur_batch) in enumerate(self._batches):
                inp = cur_batch[0]
                target = cur_batch[1]
                metadata = cur_batch[2]

                if self.config.UQ:
                    uq_loss, mse = self._train_step_uq_range(inp, target)
                else:
                    mse = self._train_step_point(inp, target)
                    uq_loss = None

                # Collect mse, uq_loss for the current epoch
                mse_steps.append(mse)
                uq_loss_steps.append(uq_loss)

                # Log batch level metrics
                if batch_n % self.config.logging_interval == 0:
                    train_logs_batch['batch_n'].append(batch_n)
                    train_logs_batch['time'].append(time.time() - start)
                    train_logs_batch['mse'].append(sum(mse_steps) / len(mse_steps))
                    if uq_loss:
                        train_logs_batch['uq_loss'].append(sum(uq_loss_steps) / len(uq_loss_steps))
                    else:
                        train_logs_batch['uq_loss'].append(None)

                    # validation metrics are only computed at the end of a epoch
                    train_logs_batch['valid_mse'].append(None)
                    train_logs_batch['valid_uq_loss'].append(None)

                    # write files, plots
                    self._write_train_logs(train_logs_batch, 'train-logs-batch')

            if epoch % self.config.epoch_logging_interval == 0:
                # validation metrics
                if self.config.UQ:
                    valid_uq_loss, valid_mse, valid_mse_fcst = self._validation_metrics_uq_range_estimate()
                else:
                    valid_uq_loss, valid_mse, valid_mse_fcst = self._validation_metrics_point_estimate()
                # log epoch level metrics
                train_logs_epoch['epoch'].append(epoch)
                train_logs_epoch['time'].append(time.time() - start)
                train_logs_epoch['mse'].append(sum(mse_steps) / len(mse_steps))
                if uq_loss:
                    train_logs_epoch['uq_loss'].append(sum(uq_loss_steps) / len(uq_loss_steps))
                else:
                    train_logs_epoch['uq_loss'].append(None)
                train_logs_epoch['valid_mse'].append(valid_mse)
                train_logs_epoch['valid_uq_loss'].append(valid_uq_loss)
                train_logs_epoch['valid_mse_fcst'].append(valid_mse_fcst)

                self._write_train_logs(train_logs_epoch, 'train-logs-epoch')

                # saving criteria
                if self._save_criteria(train_logs_epoch):
                    self.model.save_weights(checkpoint_prefix)

                if self._stop_criteria(train_logs_epoch):
                    break

        return valid_mse

    def _train_step_point(self, inp, targets):
        assert not self.config.UQ

        with tf.GradientTape() as tape:
            preds = self.model(inp)
            # re-assign targets, preds as lists as required by loss function
            if self.config.forecast_steps == 1:
                preds = [preds]
                targets = [targets]

            assert isinstance(targets, (list, tuple))
            assert isinstance(preds, (list, tuple))

            loss, mse = self.losses.weight_adjusted_mse(targets, preds)
        grads = tape.gradient(loss, self.model.trainable_variables)

        # gradient clipping
        if self.config.max_grad_norm > 0:
            grads, self._grad_norm = tf.clip_by_global_norm(grads, self.config.max_grad_norm)

        self.optimizer.apply_gradients(zip(grads, self.model.trainable_variables))
        return mse.numpy()

    def _train_step_uq_range(self, inp, targets):
        assert self.config.UQ

        with tf.GradientTape() as tape:
            preds = self.model(inp)
            tar_preds = preds[0::2]  # target preds
            var_preds = preds[1::2]  # variance preds

            if self.config.forecast_steps == 1:
                targets = [targets]

            assert isinstance(targets, (list, tuple))
            assert isinstance(tar_preds, (list, tuple))
            assert isinstance(var_preds, (list, tuple))

            uq_loss, uq_loss_last_tar, mse = self.losses.weight_adjusted_uq_loss(targets, tar_preds, var_preds)

        grads = tape.gradient(uq_loss, self.model.trainable_variables)

        # gradient clipping
        if self.config.max_grad_norm > 0:
            grads, self._grad_norm = tf.clip_by_global_norm(grads, self.config.max_grad_norm)

        self.optimizer.apply_gradients(zip(grads, self.model.trainable_variables))
        return uq_loss_last_tar.numpy(), mse.numpy()

    def _write_train_logs(self, train_logs, name):
        """
        Writes training logs into a csv file
        :param train_logs: training logs dict
        :param name: name of log - _batch, _epoch
        :return:
        """
        df = pd.DataFrame.from_dict(train_logs)
        fname = self.config.name + '-' + name + '.csv'
        fname = os.path.join(self.train_log_dir, fname)
        df.to_csv(fname, sep=',', index=False)
        return

    def _save_criteria(self, train_logs_epoch):
        """
        model saving criteria where min valid mse model is saved
        :param train_logs_epoch:
        :return:
        """
        assert len(train_logs_epoch['valid_mse']) > 0, 'Error in computing valid_mse or incorrect train log dict passed'
        if train_logs_epoch['valid_mse'][-1] < self.min_valid_mse:
            self.min_valid_mse = train_logs_epoch['valid_mse'][-1]
            return True
        else:
            return False

    def _stop_criteria(self, train_logs_epoch):
        """
        Criteria to stop training before hitting max epoch. If `valid_mse` does not improve for consecutive
        `early_stop` epochs, training is stopped
        :param train_logs_epoch:
        :return:
        """
        assert len(train_logs_epoch['valid_mse']) > 0, \
            'Error in computing valid_mse or incorrect train log dict passed'
        valid_mse = np.array(train_logs_epoch['valid_mse'])
        if valid_mse.shape[0] - np.argmin(valid_mse) + 1 >= self.config.early_stop:
            return True
        else:
            return False

    def _validation_metrics_point_estimate(self):
        """
        Returns the loss, accuracy metrics on the validation set using model's current parameters for point estimate
        models
        :return:
        TODO: Add other validation metrics such as accuracy, forecast error, etc. Only mse is implemented at present.
        """
        preds, targets = [], []
        preds_unscaled, targets_unscaled = [], []

        # for (batch_n, valid_set_items) in enumerate(self.valid_set):
        #     inp_idxs = valid_set_items[0]
        #     tar_idxs = valid_set_items[1]
        #     pre_metadata = valid_set_items[2]
        #     inp, batch_target, metadata = self.dataset.get_batch(inp_idxs, tar_idxs, pre_metadata)

        for (batch_n, cur_batch) in enumerate(self._valid_batches):
            inp = cur_batch[0]
            batch_target = cur_batch[1]
            metadata = cur_batch[2]

            batch_pred = self.model.predict(inp)

            # preds, targets need to be a list
            if self.config.forecast_steps == 1:
                batch_pred = [batch_pred]
                batch_target = [batch_target]

            assert isinstance(batch_pred, (list, tuple))
            assert isinstance(batch_target, (list, tuple))

            assert len(batch_pred) == len(batch_target)

            batch_pred_unscaled, batch_target_unscaled = copy.deepcopy(batch_pred), copy.deepcopy(batch_target)

            # unscale preds and targets
            for i in range(len(batch_pred_unscaled)):
                batch_pred_unscaled[i] = self._unscale_preds(batch_pred_unscaled[i])
                batch_target_unscaled[i] = self._unscale_preds(batch_target_unscaled[i])

            preds.append(batch_pred)
            targets.append(batch_target)
            preds_unscaled.append(batch_pred_unscaled)
            targets_unscaled.append(batch_target_unscaled)

        preds_all_batches, targets_all_batches = [], []
        preds_unscaled_all_batches, targets_unscaled_all_batches = [], []

        for i in range(self.config.forecast_steps):
            # scaled
            pred_i = np.vstack([x[i] for x in preds])
            target_i = np.vstack([x[i] for x in targets])
            preds_all_batches.append(pred_i)
            targets_all_batches.append(target_i)

            # unscaled
            pred_unscaled_i = np.vstack([x[i] for x in preds_unscaled])
            target_unscaled_i = np.vstack([x[i] for x in targets_unscaled])
            preds_unscaled_all_batches.append(pred_unscaled_i)
            targets_unscaled_all_batches.append(target_unscaled_i)

        _, valid_mse = self.losses.weight_adjusted_mse(targets_all_batches, preds_all_batches, True)

        # valid_mse_fcst is normalized by seq_norm such as mrkcap
        _, valid_mse_fcst = self.losses.weight_adjusted_mse(targets_unscaled_all_batches,
                                                            preds_unscaled_all_batches,
                                                            True)
        valid_uq_loss = None
        return valid_uq_loss, valid_mse.numpy(), valid_mse_fcst.numpy()

    def _validation_metrics_uq_range_estimate(self):
        """
        Returns the loss, accuracy metrics on the validation set using model's current parameters for uq range estimate
        models
        :return:
        TODO: Add other validation metrics such as accuracy, forecast error, etc. Only mse is implemented at present.
        """
        preds, var, targets = [], [], []
        preds_unscaled, targets_unscaled = [], []

        # for (batch_n, valid_set_items) in enumerate(self.valid_set):
        #     inp_idxs = valid_set_items[0]
        #     tar_idxs = valid_set_items[1]
        #     pre_metadata = valid_set_items[2]
        #     inp, batch_target, metadata = self.dataset.get_batch(inp_idxs, tar_idxs, pre_metadata)

        for (batch_n, cur_batch) in enumerate(self._valid_batches):
            inp = cur_batch[0]
            batch_target = cur_batch[1]
            metadata = cur_batch[2]

            batch_pred = self.model.predict(inp)

            # targets need to be a list
            if self.config.forecast_steps == 1:
                batch_target = [batch_target]

            batch_tar_pred = batch_pred[0::2]  # target preds
            batch_tar_var = batch_pred[1::2]  # variance preds

            assert isinstance(batch_tar_pred, (list, tuple))
            assert isinstance(batch_tar_var, (list, tuple))
            assert isinstance(batch_target, (list, tuple))

            assert len(batch_tar_pred) == len(batch_target)

            batch_tar_pred_unscaled, batch_target_unscaled = copy.deepcopy(batch_tar_pred), copy.deepcopy(batch_target)

            # unscale preds amd targets
            for i in range(len(batch_tar_pred_unscaled)):
                batch_tar_pred_unscaled[i] = self._unscale_preds(batch_tar_pred_unscaled[i])
                batch_target_unscaled[i] = self._unscale_preds(batch_target_unscaled[i])

            preds.append(batch_tar_pred)
            var.append(batch_tar_var)
            targets.append(batch_target)

            preds_unscaled.append(batch_tar_pred_unscaled)
            targets_unscaled.append(batch_target_unscaled)

        preds_all_batches, var_all_batches, targets_all_batches = [], [], []
        preds_unscaled_all_batches, targets_unscaled_all_batches = [], []

        for i in range(self.config.forecast_steps):
            # scaled
            pred_i = np.vstack([x[i] for x in preds]).astype('float32')
            var_i = np.vstack([x[i] for x in var]).astype('float32')
            target_i = np.vstack([x[i] for x in targets]).astype('float32')

            preds_all_batches.append(pred_i)
            var_all_batches.append(var_i)
            targets_all_batches.append(target_i)

            # unscaled
            pred_unscaled_i = np.vstack([x[i] for x in preds_unscaled])
            target_unscaled_i = np.vstack([x[i] for x in targets_unscaled])

            preds_unscaled_all_batches.append(pred_unscaled_i)
            targets_unscaled_all_batches.append(target_unscaled_i)

        # UQ Loss
        _, valid_uq_loss, valid_mse = self.losses.weight_adjusted_uq_loss(targets_all_batches,
                                                                          preds_all_batches,
                                                                          var_all_batches)

        # valid_mse_fcst is normalized by seq_norm such as mrkcap
        _, valid_mse_fcst = self.losses.weight_adjusted_mse(targets_unscaled_all_batches,
                                                            preds_unscaled_all_batches)

        return valid_uq_loss.numpy(), valid_mse.numpy(), valid_mse_fcst.numpy()

    def _unscale_preds(self, arr):
        """
        Unscales the input array by reversing parameter scaling and log squashing.
        Returns raw preds/targets which are normalized by seq_norm
        :param arr:
        :return: raw preds, targets normalized by seq_norm
        """
        arr = np.multiply(arr, self.dataset.scaling_params['scale'][:self.dataset.n_outputs]) + \
              self.dataset.scaling_params['center'][:self.dataset.n_outputs]

        if self.config.log_squasher:
            arr = self.dataset.reverse_log_squasher(arr)
        return arr

if __name__ == '__main__':
    config = get_configs()
    config.train = True
    config.scale_field = 'mrkcap'
    config.log_squasher = True
    config.max_unrollings = 5
    config.forecast_n = 12
    config.forecast_steps = 1
    config.start_date = 197501
    config.end_date = 199912
    # config.batch_size = 1
    config.lr_decay = 1.0
    config.dropout = 0.3
    config.recurrent_dropout = 0.3
    config.nn_type = 'RNNPointEstimate'
    config.UQ = False
    config.max_epoch = 5
    config.early_stop = 10

    # config.datafile = "source-ml-data-v8-100M.dat"
    config.datafile = "sample_data_testing.dat"
    config.data_dir = "../datasets"
    config.model_dir = "../experiments/test-model"
    config.validation_size = 0.3
    D = Dataset(config)
    # print(D._train_gvkeys, D._valid_gvkeys, D._test_gvkeys)

    # rnn_model = RNNPointEstimate(config, D)
    # m = rnn_model.model
    # print(m.summary())
    # train = Train(config, m, D)
    train = Train(config, D)
    train.train()

reason: import numpy


text:
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '0, 1, 2, 3'
import time
import numpy as np
import random
import warnings
from tqdm.auto import tqdm

import torch
import torch.nn as nn
import torch.optim
from torch.optim.lr_scheduler import OneCycleLR
import torch.backends.cudnn as cudnn
import timm
from transformers import get_linear_schedule_with_warmup, set_seed

from loader import getLoader
from config import return_args
from utils import *

best_acc = 0

def main():
    args = return_args()
    if args.seed is not None:
        random.seed(args.seed)
        np.random.seed(args.seed)
        torch.manual_seed(args.seed)
        torch.cuda.manual_seed_all(args.seed)
        cudnn.deterministic = True
        warnings.warn('You have chosen to seed training. '
                      'This will turn on the CUDNN deterministic setting, '
                      'which can slow down your training considerably! '
                      'You may see unexpected behavior when restarting '
                      'from checkpoints.')
    main_worker(args)

def main_worker(args):
    global best_acc
    # adjust batch size
    args.batch_size = args.batch_size * torch.cuda.device_count()
    args.lr = (args.lr / 4) * torch.cuda.device_count()
    train_loader = getLoader(args, 'train')
    val_loader = getLoader(args, 'val')

    model = timm.create_model(
        'swin_base_patch4_window7_224', pretrained=True, num_classes=2)
    model = nn.DataParallel(model).cuda()
    criterion = nn.CrossEntropyLoss().cuda()
    optimizer = torch.optim.AdamW(model.parameters(), args.lr)

    lr_scheduler = OneCycleLR(
        optimizer, max_lr=args.lr, epochs=args.epochs, steps_per_epoch=len(train_loader))

    progress_begin = time.time()

    for epoch in tqdm(range(args.epochs)):
        train_one_epoch(train_loader, model, criterion, optimizer,
                        lr_scheduler, epoch, args)
        acc = validate(val_loader, model, criterion,
                       optimizer, args)
        if acc > best_acc:
            best_acc = acc
            if args.save:
                if not os.path.exists('save'):
                    os.mkdir('save')
                torch.save(model.module.state_dict(), os.path.join(
                    'save', f'ep{epoch}_acc{acc*1000:.0f}.pth'))
    progress_end = time.time()
    print('Dataparallel running time : {} s'.format(progress_end - progress_begin))

def train_one_epoch(train_loader, model, criterion, optimizer, scheduler, epoch, args):
    batch_time = AverageMeter('Time', ':6.3f')
    data_time = AverageMeter('Data', ':6.3f')
    losses = AverageMeter('Loss', ':.4e')
    top1 = AverageMeter('Acc@1', ':6.2f')
    progress = ProgressMeter(
        len(train_loader),
        [batch_time, data_time, losses, top1],
        prefix="Epoch: [{}]".format(epoch))

    model.train()
    end = time.time()
    for i, (img, label) in enumerate(train_loader):
        data_time.update(time.time() - end)

        img, label = img.cuda(), label.cuda()
        output = model(img)
        loss = criterion(output, label)
        loss.backward()
        optimizer.step()
        scheduler.step()
        optimizer.zero_grad()

        acc1 = accuracy(output, label, topk=(1, ))
        losses.update(loss.item(), img.size(0))
        top1.update(acc1[0].item(), img.size(0))
        # top5.update(acc5[0], img.size(0))

        batch_time.update(time.time() - end)
        end = time.time()

        if i % args.print_freq == 0:
            progress.display_common(i)

@torch.no_grad()
def validate(val_loader, model, criterion, optimizer, args):
    batch_time = AverageMeter('Time', ':6.3f', Summary.NONE)
    losses = AverageMeter('Loss', ':.4e', Summary.NONE)
    top1 = AverageMeter('Acc@1', ':6.2f', Summary.AVERAGE)
    progress = ProgressMeter(
        len(val_loader),
        [batch_time, losses, top1],
        prefix='Evaluating: ')

    model.eval()
    end = time.time()
    for i, (img, label) in enumerate(val_loader):
        img, label = img.cuda(), label.cuda()
        output = model(img)
        loss = criterion(output, label)

        acc1 = accuracy(output, label, topk=(1, ))
        losses.update(loss.item(), img.size(0))
        top1.update(acc1[0].item(), img.size(0))
        batch_time.update(time.time() - end)
        end = time.time()

        if i % args.print_freq == 0:
            progress.display_common(i)

    progress.display_summary_common()

    return top1.avg

if __name__ == '__main__':
    main()


text:
# -*- coding: utf-8 -*-
'''
  fastmat/core/calibration.py
 -------------------------------------------------- part of the fastmat package

  Provides generastion and handling of calibration values for estimating the
  runtime properties of transforms.

  Author      : wcw
  Introduced  : 2017-08-15
 ------------------------------------------------------------------------------

   Copyright 2016 Sebastian Semper, Christoph Wagner

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

 ------------------------------------------------------------------------------
'''
import numpy as np
from ..Matrix import MatrixCalibration

# stores calibration data as tuple (offsetFwd/Bwd, gainFwd/Bwd) over class
calData = {}

def saveCalibration(filename):
    import json
    import os
    outData = {target.__name__: cal.export() for target, cal in calData.items()}

    # force existance of output file
    dirname=os.path.dirname(filename)
    if dirname != '' and not os.path.exists(dirname):
        os.makedirs(dirname)

    with open(filename, 'w') as f:
        string = json.dumps(outData)
        f.write(string.replace('],', '],' + os.linesep) + os.linesep)

def loadCalibration(filename):
    import json
    from .. import classes
    classNames = {item.__name__: item for item in classes}

    with open(filename, 'r') as f:
        inData = json.load(f)

    calData.clear()
    for name, data in inData.items():
        item = classNames.get(name, None)
        if item is not None:
            calData[item] = MatrixCalibration(*data)

def getMatrixCalibration(target):
    return calData.get(target, None)

def calibrateClass(target, **options):
    from fastmat.inspect import Benchmark, BENCH
    from .. import flags

    calOptions = {
        BENCH.LIMIT_ITER    : 0.1,
        BENCH.LIMIT_INIT    : 0.1,
        BENCH.LIMIT_SIZE    : 1e6,
        BENCH.LIMIT_MEMORY  : 1e5,
        BENCH.STEP_MINITEMS : 3,
        BENCH.MEAS_MINTIME  : 3e-3,
        BENCH.MEAS_MINRUNS  : 3,
        BENCH.MEAS_MINREPS  : 3,
        BENCH.FUNC_STEP     : lambda x: x + 1,
        'verbose'           : False
    }
    calOptions.update(options)

    # allow the function to skip estimation of calibration values and return
    # benchmark results that were generated with parameters matching the ones
    # that were used for the calibration
    benchmarkOnly = options.get('benchmarkOnly', False)
    if not benchmarkOnly:
        _bypassAllow = flags.bypassAllow
        flags.bypassAllow = False

    B = Benchmark(target)
    B.verbosity = calOptions.get('verbose', False)
    B.run('overhead', **calOptions)

    # transform bypass deactivated when calibrating
    if benchmarkOnly:
        return B
    else:
        flags.bypassAllow = _bypassAllow

    def estimate(nameTime, nameComplexity, nameOvhNested, nameEffNested):
        arr = np.nan_to_num(B.getResult('overhead', 'numN',
                                        nameTime, nameComplexity,
                                        nameOvhNested, nameEffNested))

        # remove runtimes of nested classes from the result
        arrN                = arr[:, 0]
        arrTime             = arr[:, 1]
        arrComplexity       = arr[:, 2]
        arrOverheadNested   = arr[:, 3]
        arrEffortNested     = arr[:, 4]
        arrNested           = np.nan_to_num(arrOverheadNested + arrEffortNested)

        arrTime -= arrNested

        # keep entries with strictly positive runtimes and positive complexity
        arrIter         = arr[(arrTime > 0.) & (arrComplexity >= 0.), 1:3]
        arrTimeIter     = arrIter[:, 0]
        arrCplxIter     = arrIter[:, 1]

        offset, gain = 0., 0.
        if arrIter.shape[0] > 0:
            # estimate gain
            cpxRange = np.ptp(arrCplxIter)
            gain = (np.ptp(arrTimeIter) / cpxRange if cpxRange > 0. else 0.)

            # estimate offset iteratively
            for nn in range(20):
                delta = arrTimeIter - gain * arrCplxIter
                delta = delta[delta > -offset]
                if delta.shape[0] < 3:
                    break

                dOffset = (0 if delta.shape[0] < 1 else np.median(delta))
                offset += dOffset
                arrTimeIter -= dOffset

                if dOffset < offset * 1e-2:
                    break

        return offset, gain

    ovhForward, effForward = estimate('forwardMin',
                                      BENCH.RESULT_COMPLEXITY_F,
                                      BENCH.RESULT_OVH_NESTED_F,
                                      BENCH.RESULT_EFF_NESTED_F)

    ovhBackward, effBackward = estimate('backwardMin',
                                        BENCH.RESULT_COMPLEXITY_B,
                                        BENCH.RESULT_OVH_NESTED_B,
                                        BENCH.RESULT_EFF_NESTED_B)

    cal = MatrixCalibration(ovhForward, ovhBackward, effForward, effBackward)
    calData[target] = cal

    return cal, B


text:
# -*- coding: utf-8 -*-
from numpy.linalg import norm

from .perception import Perception

class KNN(Perception):
    """The class implements a k-nearest neighbors perception of the neighbors."""

    def __init__(self, k, border, perception):
        """Build a new k-nearest neighbors filter.

        Args:
            k (int): The maximum number of neighbors.
            border (Border): The borders of the environment.
            perception (Perception): The wrapped perception.

        """
        super().__init__(border, perception)
        self.k = k
        """int: The maximum number of neighbors."""

    def _filter(self, ind, pop):
        dpop = list()
        # Sort the neighbors by their distance to the individual
        for ind_pop in pop:
            relative_pos = self.border.vector(ind.pos, ind_pop.pos)
            if ind_pop is not ind:
                dpop.append((norm(relative_pos), ind_pop))
        dpop.sort(key=lambda elem: elem[0])
        knn = dpop[: min(self.k, len(dpop))]
        # List the k nearest neighbors
        return [ind for d, ind in knn]

max_stars_repo_name: Bacmel/Boids-Boids-Boids

max_issues_repo_name: Bacmel/Boids-Boids-Boids

max_forks_repo_name: Bacmel/Boids-Boids-Boids

reason: from numpy


text:
import argparse
import time
import os

from model import efficientdet
from utils import preprocess_image
from utils.anchors import anchors_for_shape

import cv2
import numpy as np

parser = argparse.ArgumentParser(description="Process some integers.")

parser.add_argument(
    "--phi", type=int, required=True, help="phi constant between 0 and 7 inclusive"
)

parser.add_argument(
    "--cuda-visible-devices", default="", help="Set the GPU to use or default  to CPU"
)

parser.add_argument(
    "--weighted",
    default=False,
    action="store_true",
    help="Run with an weighted biFPN model." "Defaults to use unweighted BiFPN.",
)

parser.add_argument("--model", required=True, help="path to .h5 file")

parser.add_argument("image")

args = parser.parse_args()

os.environ["CUDA_VISIBLE_DEVICES"] = args.cuda_visible_devices

if not args.weighted and "weighted" in args.model:
    print("WARNING: model", args.model,
          "might be weighted. If so, pass --weighted flag.")

classes = [
    "aeroplane",
    "bicycle",
    "bird",
    "boat",
    "bottle",
    "bus",
    "car",
    "cat",
    "chair",
    "cow",
    "diningtable",
    "dog",
    "horse",
    "motorbike",
    "person",
    "pottedplant",
    "sheep",
    "sofa",
    "train",
    "tvmonitor",
]

num_classes = len(classes)  # 80

score_threshold = 0.5
colors = [np.random.randint(0, 256, 3).tolist() for i in range(num_classes)]

image_sizes = (512, 640, 768, 896, 1024, 1280, 1408)
image_size = image_sizes[args.phi]

anchors = anchors_for_shape((image_size, image_size))

model, prediction_model = efficientdet(
    phi=args.phi,
    weighted_bifpn=args.weighted,
    num_classes=num_classes,
    score_threshold=score_threshold,
    anchors=anchors
)

prediction_model.load_weights(args.model, by_name=True)

image = cv2.imread(args.image)

if image is None:
    raise FileNotFoundError(args.image)

src_image = image.copy()
image = image[:, :, ::-1]
h, w = image.shape[:2]

image, scale, offset_h, offset_w = preprocess_image(
    image, image_size=image_size)

inputs = np.expand_dims(image, axis=0)

# run network
start = time.time()
boxes_list, scores, labels = prediction_model.predict_on_batch(
    [
        np.expand_dims(image, axis=0),
    ]
)

boxes = boxes_list[0]

print("scale", scale, "offset_h", offset_h, "offset_w", offset_w)
print("boxes", boxes.shape, boxes, sep="\n")
print("scores", scores.shape, scores, sep="\n")
print("labels", labels.shape, labels, sep="\n")

print(time.time() - start)

# Subtract offset_w from every predicted x1 and x2
boxes[:, [0, 2]] = boxes[:, [0, 2]] - offset_w
# Subtract offset_w from every predicted y1 and y2
boxes[:, [1, 3]] = boxes[:, [1, 3]] - offset_h

# Divide boxes by scale
boxes /= scale

# ensure that predicted x1 and x2 coordinates between 0 and width-1
boxes[:, 0] = np.clip(boxes[:, 0], 0, w - 1)
boxes[:, 2] = np.clip(boxes[:, 2], 0, w - 1)

# ensure that predicted y1 and y2 coordinates between 0 and height-1
boxes[:, 1] = np.clip(boxes[:, 1], 0, h - 1)
boxes[:, 3] = np.clip(boxes[:, 3], 0, h - 1)

# select indices which have a score above the threshold
indices = np.where(scores[0, :] > score_threshold)[0]

print(indices)
# select those detections
detections = boxes[indices]
scores = scores[0, indices]
labels = labels[0, indices]

print("final")
print(boxes)
print(scores)
print(labels)

for box, score, label in zip(detections, scores, labels):
    xmin = int(round(box[0]))
    ymin = int(round(box[1]))
    xmax = int(round(box[2]))
    ymax = int(round(box[3]))
    score = "{:.4f}".format(score)
    class_id = int(label)
    color = colors[class_id]
    class_name = classes[class_id]
    label = "-".join([class_name, score])

    ret, baseline = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
    cv2.rectangle(src_image, (xmin, ymin), (xmax, ymax), color, 1)
    cv2.rectangle(
        src_image, (xmin, ymax - ret[1] -
                    baseline), (xmin + ret[0], ymax), color, -1
    )

    cv2.putText(
        src_image,
        label,
        (xmin, ymax - baseline),
        cv2.FONT_HERSHEY_SIMPLEX,
        0.5,
        (0, 0, 0),
        1,
    )

cv2.namedWindow("image", cv2.WINDOW_NORMAL)
cv2.imshow("image", src_image)
cv2.waitKey(0)

print("done")

max_stars_repo_name: Ely-S/EfficientDet

max_issues_repo_name: Ely-S/EfficientDet

max_forks_repo_name: Ely-S/EfficientDet


text:
#
# Copyright (c) 2018 Institute for Basic Science
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#

from pysam import BGZFile, faidx
from glob import glob
from functools import partial
from itertools import count
from collections import defaultdict
from shutil import copyfileobj
from threading import Lock
import h5py
import numpy as np
import pandas as pd
import logging
import os
from . import OUTPUT_NAME_FAILED
from .utils import ensure_dir_exists
from .fast5_file import Fast5Reader, DuplicatedReadError

class FASTQWriter:

    def __init__(self, output_dir, output_layout):
        self.output_dir = output_dir
        self.output_layout = output_layout
        self.lock = Lock()

        self.open_streams()

    def open_streams(self):
        self.streams = {
            int_name: BGZFile(self.get_output_path(name), 'w')
            for int_name, name in self.output_layout.items()}

    def close(self):
        for stream in self.streams.values():
            stream.close()

    def get_output_path(self, name):
        output_path = os.path.join(self.output_dir, 'fastq', name + '.fastq.gz')
        ensure_dir_exists(output_path)
        return output_path

    def write_sequences(self, procresult):
        with self.lock:
            for entry in procresult:
                if entry.get('sequence') is not None:
                    seq, qual, adapter_length = entry['sequence']
                    if adapter_length > 0:
                        seq = seq[:-adapter_length]
                        qual = qual[:-adapter_length]

                    output_name = entry['label'], entry.get('barcode')
                    formatted = '@{}\n{}\n+\n{}\n'.format(entry['read_id'], seq, qual)
                    self.streams[output_name].write(formatted.encode('ascii'))

class FAST5Writer:

    def __init__(self, output_dir, output_layout, input_dir, batch_size=4000):
        self.input_dir = input_dir
        self.output_dir = output_dir
        self.output_layout = output_layout
        self.batch_size = batch_size
        self.lock = Lock()

        self.prepare_files()

    def prepare_files(self):
        self.f5files = {}
        self.getf5hdl = {
            int_name: self.generate_rotated_fast5_handle(name)
            for int_name, name in self.output_layout.items()}

    def close(self):
        for hdl in self.f5files.values():
            hdl.close()

    def generate_rotated_fast5_handle(self, name):
        output_path = os.path.join(self.output_dir, 'fast5', name + '_{}.fast5')
        ensure_dir_exists(output_path)

        for fileno in count():
            self.f5files[name] = hdl = h5py.File(output_path.format(fileno), 'w')
            for _ in range(self.batch_size):
                yield hdl

    def transfer_reads(self, procresult):
        with self.lock:
            for entry in procresult:
                output_name = entry.get('label', OUTPUT_NAME_FAILED), entry.get('barcode')
                input_name = os.path.join(self.input_dir, entry['filename'])
                f5rd = Fast5Reader(input_name, entry['read_id'])
                f5wt = next(self.getf5hdl[output_name])
                try:
                    f5rd.copyto(f5wt)
                except DuplicatedReadError as exc:
                    pass # Ignore duplicated reads for now.

class SequencingSummaryWriter:

    SUMMARY_OUTPUT_FIELDS = [
        'filename', 'read_id', 'run_id', 'channel', 'start_time',
        'duration', 'num_events', 'sequence_length', 'mean_qscore',
        'sample_id', 'status', 'label',
    ]

    def __init__(self, config, output_dir, label_mapping, barcode_mapping):
        self.file = open(os.path.join(output_dir, 'sequencing_summary.txt'), 'w')
        self.lock = Lock()
        self.label_mapping = label_mapping

        self.output_fields = self.SUMMARY_OUTPUT_FIELDS[:]

        if config['barcoding']:
            self.barcode_mapping = barcode_mapping
            self.output_fields.extend(['barcode', 'barcode_score',
                                        #'barcode_guess'
                                      ])
        else:
            self.barcode_mapping = None

        if config['measure_polya']:
            self.polya_enabled = True
            self.output_fields.append('polya_dwell')
        else:
            self.polya_enabled = False

        if config['fast5_output']:
            if config['barcoding']:
                self.format_filename = (lambda entry:
                    os.path.join('fast5', entry['label'],
                                 self.barcode_mapping[entry.get('barcode')],
                                 entry['filename']))
            else:
                self.format_filename = (lambda entry:
                    os.path.join('fast5', entry['label'], entry['filename']))
        else:
            self.format_filename = lambda entry: entry['filename']

        print(*self.output_fields, sep='\t', file=self.file)

    def close(self):
        self.file.close()

    def write_results(self, results):
        with self.lock:
            for entry in results:
                if 'label' in entry:
                    output_entry = entry.copy()
                    output_entry['label'] = self.label_mapping[entry['label']]
                    output_entry['filename'] = self.format_filename(output_entry)
                    if self.barcode_mapping is not None:
                        output_entry['barcode'] = self.barcode_mapping[entry.get('barcode')]
                        output_entry['barcode_score'] = entry.get('barcode_score', 0)
                        #output_entry['barcode_guess'] = self.barcode_mapping[
                        #    entry.get('barcode_guess')]
                    if self.polya_enabled:
                        output_entry['polya_dwell'] = (
                            format(entry['polya']['dwell_time'], '.4f')
                            if 'polya' in entry else '')

                    print(*[output_entry[f] for f in self.output_fields],
                          file=self.file, sep='\t')

class NanopolishReadDBWriter:

    def __init__(self, output_dir, output_layout):
        self.output_layout = output_layout
        self.output_dir = os.path.join(output_dir, 'nanopolish')
        self.seqfiles, self.dbfiles = self.open_streams()
        self.lock = Lock()

    def open_streams(self):
        seqfiles, dbfiles = {}, {}
        for groupid, name in self.output_layout.items():
            filepath = os.path.join(self.output_dir, name + '.fasta')
            ensure_dir_exists(filepath)
            seqfiles[groupid] = open(filepath, 'w')
            dbfiles[groupid] = open(filepath + '.index.readdb', 'w')
        return seqfiles, dbfiles

    def close(self):
        for groupid, f in list(self.seqfiles.items()):
            f.close()
            del self.seqfiles[groupid]

        for groupid, f in list(self.dbfiles.items()):
            f.close()
            del self.dbfiles[groupid]

        # Create bgzipped-fasta and indices
        for groupid, name in self.output_layout.items():
            inputfile = os.path.join(self.output_dir, name + '.fasta')
            if os.path.getsize(inputfile) > 0:
                bgzippedfile = inputfile + '.index'
                copyfileobj(open(inputfile, 'rb'), BGZFile(bgzippedfile, 'w'))
                faidx(bgzippedfile)

    def write_sequences(self, procresult):
        with self.lock:
            for entry in procresult:
                if entry.get('sequence') is not None:
                    mappingkey = entry['label'], entry.get('barcode')

                    formatted = '>{}\n{}\n'.format(entry['read_id'], entry['sequence'][0])
                    self.seqfiles[mappingkey].write(formatted)

                    fast5_relpath = os.path.join('fast5', self.output_layout[mappingkey],
                                                 entry['filename'])
                    formatted = '{}\t{}\n'.format(entry['read_id'], fast5_relpath)
                    self.dbfiles[mappingkey].write(formatted)

class FinalSummaryTracker:

    REPORTING_ORDER = ['pass', 'artifact', 'fail']
    FRIENDLY_LABELS = {
        'pass': 'Successfully processed',
        'fail': 'Processing failed',
        'artifact': 'Possible artifact',
    }
    BARCODE_FRIENDLY_NAME = 'Barcoded sample {num} (BC{num})'
    FRIENDLY_STATUS = {
        'fail': {
            'scaler_signal_too_short': 'Signal is too short',
            'sequence_too_short': 'Sequence is too short',
            'irregular_fast5': 'Invalid FAST5 format',
            'basecall_table_incomplete': 'Basecall table does not match',
            'adapter_not_detected': "3' Adapter could not be located",
            'not_basecalled': 'No albacore basecall data found',
            'scaling_qc_fail': 'Signal scaling QC failed',
            'disappeared': 'File is moved to other location',
            'unknown_error': 'File could not be opened due to unknown error',
        },
        'artifact': {
            'unsplit_read': 'Two or more molecules found within a read',
        },
    }

    LABEL_FORMAT = '{:49s} '
    LABEL_BULLET = ' - '
    MINIMUM_COLUMN_WIDTH = 3

    def __init__(self, label_names, barcode_names):
        self.label_names = label_names
        self.barcode_names = barcode_names
        self.counts = defaultdict(int)
        self.label_reporting_order = self.REPORTING_ORDER
        self.barcode_reporting_order = sorted([
            n for n in barcode_names.keys() if n is not None]) + [None]

    def feed_results(self, results):
        for entry in results:
            self.counts[entry.get('label', 'fail'),
                        entry.get('barcode', None),
                        entry['status']] += 1

    def print_results(self, file):
        if hasattr(file, 'write'):
            _ = partial(print, sep='\t', file=file)
        else:
            logger = logging.getLogger('poreplex')
            _ = lambda *args: logger.error(' '.join(map(str, args)))

        _("==== Result Summary ====")
        longest_count_length = len(format(max(self.counts.values()), 'd'))
        column_width = max(self.MINIMUM_COLUMN_WIDTH, longest_count_length)
        column_title_format = '{{:{}s}} '.format(column_width)
        column_numeric_format = '{{:{}d}} '.format(column_width)

        # Show the header
        if len(self.barcode_names) > 1:
            label_fields = [self.LABEL_FORMAT.format('')] + [
                column_title_format.format(self.barcode_names[bc])
                for bc in self.barcode_reporting_order
            ]
            _(''.join(label_fields))

        # Show the table content
        tbl = pd.DataFrame([(k[0], -1 if k[1] is None else k[1], k[2], v)
                            for k, v in self.counts.items()],
                           columns=['label', 'barcode', 'status', 'count'])
        tbl['label_key'] = tbl['label'].apply(self.label_reporting_order.index)
        ordered = (tbl.sort_values(by=['label_key', 'count'], ascending=[True, False])
                      .groupby(by=['label', 'status'], sort=False))
        current_label = None
        for lk, grp in ordered:
            linelabel = None

            # Handle label changes
            if current_label is None or current_label != lk[0]:
                current_label = lk[0]
                if current_label in self.FRIENDLY_STATUS:
                    _(self.LABEL_FORMAT.format(self.FRIENDLY_LABELS[current_label]))
                else:
                    linelabel = self.FRIENDLY_LABELS[current_label]

            # Set the line title for the rows with detailed status explanations
            if linelabel is None:
                linelabel = self.LABEL_BULLET + self.FRIENDLY_STATUS[current_label][lk[1]]

            readcount_by_barcode = grp.set_index('barcode')['count'].to_dict()
            readcounts = [readcount_by_barcode.get(bc if bc is not None else -1, 0)
                          for bc in self.barcode_reporting_order]

            # Print the read counts
            _(self.LABEL_FORMAT.format(linelabel) +
              ''.join(column_numeric_format.format(cnt) for cnt in readcounts))

        _('')

def create_links_rebalanced(desth5, group, infiles):
    desth5.require_group(group)

    for datafile in infiles:
        basename = os.path.basename(datafile)
        with h5py.File(datafile, 'r') as d5:
            for batchid, subgrp in d5[group].items():
                for readid in subgrp.keys():
                    dumpgroup = get_read_id_dump_group(readid)
                    gobj = desth5.require_group(group + '/' + dumpgroup)
                    try:
                        gobj[readid] = h5py.ExternalLink(basename,
                            '{}/{}/{}'.format(group, batchid, readid))
                    except RuntimeError:
                        if readid not in gobj:
                            raise

def create_adapter_dumps_inventory(destfile, filepattern):
    with h5py.File(destfile, 'w') as ivt:
        # Merge items under catalog/adapter group.
        ivt.require_group('catalog')

        fragments = []
        for datafile in glob(filepattern):
            with h5py.File(datafile, 'r') as d5:
                for batchid, tbl in d5['catalog/adapter'].items():
                    fragments.append(tbl[:])

        fulltbl = np.hstack(fragments)
        fulltbl.sort(order='read_id')
        ivt['catalog/adapter'] = fulltbl

        # Create hashed groups of external links for the signal datasets.
        create_links_rebalanced(ivt, 'adapter', glob(filepattern))

def create_events_inventory(destfile, filepattern):
    with h5py.File(destfile, 'w') as ivt:
        create_links_rebalanced(ivt, 'basecalled_events', glob(filepattern))

def get_read_id_dump_group(read_id, grplength=3):
    #hash = unpack('I', sha1(read_id.encode('ascii')).digest()[:4])[0] % hashsize
    #return format(hash, numformat)
    return read_id[:grplength]

max_stars_repo_name: acuacal/poreplex

max_issues_repo_name: acuacal/poreplex

max_forks_repo_name: acuacal/poreplex


text:
import numpy as np

class Reader(object):
    def __init__(self, filename):
        self.filename = filename

    def read(self):
        with open(self.filename) as f:
            lines = f.readlines()
            x, y = [], []
            for line in lines:
                a, b = list(map(float, line.strip().split()))
                x.append(a)
                y.append(b)
            return np.array(x), np.array(y)

if __name__ == "__main__":
    r = Reader("data.txt")
    print(r.read())


text:
#!/usr/bin/env python
# coding: utf-8
import os
import csv
import re
import xml.etree.ElementTree as ET
from xml.etree.ElementTree import ParseError
import pandas as pd
import xml.etree as etree
import glob
import numpy as np

# lists for values
repository = "dbGAP"
xmlpath = "./data/"
# functions

def parse(xmlpath):
    phs = ET.parse(xmlpath)
    root = phs.getroot()
    return(root)

def minedate(root):
    datehere = root.attrib["date_created"]
    date.append(datehere)

n = 0

good_files_count = 0
none_type_error_count = 0
empty_files_error_count = 0
parsed_error_count = 0

pattern = '([Ss][eE][xX]|[Gg][eE][nN][dD][eE][rR])'

n = 0
list_dict = list()
for file in glob.glob('./data/*.xml'):
    dict_data = dict()
    try:
        print(file)
        root = parse(file)
        # Get the date

        dict_data['date'] = root.attrib["date_created"]
        # date.append(NA)
        dict_data['filename'] = file
        for child in root.iter('variable'):
            if re.search(pattern, child.attrib['var_name']):
                for c in child.iter('sex'):
                    for a in c:
                        dict_data.setdefault(a.tag, []).append(a.text)

                for i in child.iter('stat'):
                    info = i.attrib
                    for key in info:
                        dict_data.setdefault(key, []).append(info[key])
        # Getting the max_value for each key if contains a list
        for k in dict_data:
            if isinstance(dict_data[k], list):
                dict_data[k] = max(dict_data[k])
        list_dict.append(dict_data)
    except ParseError:
        pass

list_keys = list()
for dict_ in list_dict:
    for k in dict_:
        list_keys.append(k)
keys = set(list_keys)
print(keys)
with open('summary_second.csv', 'w') as output_file:
    dict_writer = csv.DictWriter(output_file, fieldnames=keys, delimiter=',')
    dict_writer.writeheader()
    dict_writer.writerows(list_dict)


text:
import cv2
import numpy as np
from multiprocessing import shared_memory
import cv2
import socket
import json
from sys import getsizeof

import find_word

import sys
import time
import os

import pybullet
import pybullet_data
import time
from qibullet import SimulationManager
from qibullet import NaoVirtual
from qibullet import PepperVirtual
from qibullet import RomeoVirtual
from gtts import gTTS
from playsound import playsound
from enum import Enum

X_MIN = 0
Y_MIN = 1
X_MAX = 2
Y_MAX = 3

def rotate(pepper):
    pepper.move(0, 0, 0.5)
    # time.sleep(0.1745)
    # pepper.move(0,0,0)

def reading_from_string(text_to_read, filename, language='es', slow_audio_speed=False):
    audio_created = gTTS(text=text_to_read, lang=language,
                         slow=slow_audio_speed)
    audio_created.save(filename)
    playsound(filename)
    os.remove(filename)

cap = cv2.VideoCapture('dang.mp4')
#fps = int(cap.get(60))

host = socket.gethostname()
port = 45678                   # The same port as used by the server
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, port))

language = 'es'
shm = None

#Starting index
object_filename = "object.mp3"
filename = "speech.mp3"

sim_manager = SimulationManager()
client = sim_manager.launchSimulation(gui=True)
# client_direct_1 = sim_manager.launchSimulation(gui=False)

pybullet.setAdditionalSearchPath(pybullet_data.getDataPath())
print(pybullet_data.getDataPath())
f_plane = pybullet.loadURDF('plane.urdf')
textUid = pybullet.loadTexture('enviroment/wood.png')
pybullet.changeVisualShape(f_plane, -1, textureUniqueId=textUid)

plane = pybullet.loadURDF('enviroment/room.urdf', globalScaling=3, basePosition=[0,0,-7])
texUid = pybullet.loadTexture("enviroment/wall_2.png")
pybullet.changeVisualShape(plane, -1, textureUniqueId=texUid)

table = pybullet.loadURDF('table/table.urdf',
 basePosition= [0,5,0],
physicsClientId=client)
table_2 = pybullet.loadURDF('table/table.urdf',
 basePosition= [5,0,0],
 baseOrientation=pybullet.getQuaternionFromEuler([0, 0, 1.5]),
 physicsClientId=client)
table_3 = pybullet.loadURDF('table/table.urdf',
 basePosition= [-5,0,0],
 baseOrientation=pybullet.getQuaternionFromEuler([0, 0, -1.5]),
 physicsClientId=client)
table_4 = pybullet.loadURDF('table/table.urdf',
 basePosition= [0,-5,0],
 physicsClientId=client)

item_in_table_1 = pybullet.loadURDF('enviroment/water_bottle.urdf',
    basePosition= [-0.5,5,0.6],
    globalScaling = 3.0,
    physicsClientId=client)

item_in_table_1 = pybullet.loadURDF('teddy_vhacd.urdf',
basePosition= [0,5,0.6],
globalScaling = 6.0,
 baseOrientation=pybullet.getQuaternionFromEuler([2,0 , 0]),
physicsClientId=client)

item_in_table2 =  pybullet.loadURDF('enviroment/laptop.urdf',
    basePosition= [-5,0,0.7],
    baseOrientation=pybullet.getQuaternionFromEuler([2, 0, 3]),
    globalScaling = 3.5,
    physicsClientId=client)

item_in_table3 =  pybullet.loadURDF('enviroment/knife.urdf',
    basePosition= [-0.5,-5,0.78],
    baseOrientation=pybullet.getQuaternionFromEuler([0, 0, 1.2]),
    globalScaling = 0.3,
    physicsClientId=client)

item_in_table4 =  pybullet.loadURDF('enviroment/cat.urdf',
    basePosition= [5,0,0.7],
    baseOrientation=pybullet.getQuaternionFromEuler([0, 0, -1.5]),
    globalScaling = 0.3,
    physicsClientId=client)

#walls2 = pybullet.loadURDF('C:\\Users\\cjrs2\\Downloads\\keras-yolo3\\proyecto_robo\\walls2.urdf',
#     physicsClientId=client)

# nao = sim_manager.spawnNao(
#    client,
#    translation=[0.5,2,1],
#    quaternion=pybullet.getQuaternionFromEuler([0, 0, 3]))
pepper = sim_manager.spawnPepper(
    client,
    translation=[0, 0, 0],
    quaternion=pybullet.getQuaternionFromEuler([0, 0, 1.5]))

#  nao.goToPosture('StandInit', 1)
pepper.goToPosture("Stand", 1)

#  nao.setAngles('HeadPitch', 0.25, 1)
handle = pepper.subscribeCamera(PepperVirtual.ID_CAMERA_TOP )
print('Retriving camera frame')
resolution = pepper.getCameraResolution(handle)
#print("Resolution: " + str(resolution.width) + "x" + str(resolution.height))
mid_frame = resolution.width / 2
#x = 0
frames = 0

text_to_read = "¿Qué objeto desea encontrar?"
reading_from_string(text_to_read, object_filename)
original_text = input(text_to_read)
valid_object, object_to_find = find_word.find_word(original_text)
object_found = False
coords = []
if valid_object:
    while(not(object_found)):
        t = None
        frame = pepper.getCameraFrame(handle)
        if frames == 0:
            print('Sending initial informantion')
            shm = shared_memory.SharedMemory(create=True, size=frame.nbytes) # name='image_random'
            data=json.dumps({"shape": frame.shape, "name": shm.name, "type": frame.dtype.name, "object":object_to_find})
            s.send(data.encode())

            #data = s.recv(1024)
            #s.close()
            #print('Received', data)
        # Now create a NumPy array backed by shared memory
        b = np.ndarray(frame.shape, dtype=frame.dtype, buffer=shm.buf)
        b[:] = frame[:]  # Copy the original data into shared memory
        #print('Sending')
        s.send(bytes("1",'utf8'))
        t = s.recv(1024)
        found_info = json.loads(t.decode())
        object_found = found_info.get("was_found")
        if object_found:
            coords = found_info.get("coords")
            right_distance_from_middle = abs(coords[X_MAX] - mid_frame)
            left_distance_from_middle = abs(coords[X_MIN] - mid_frame)
            # Este 20 es solo un threshold para que se posicione como debe, se puede cambiar si parece que no está bien
            if abs(right_distance_from_middle - left_distance_from_middle) > 15:
                object_found = False

        frames += 1
        rotate(pepper)

    #print(coords)
    width = coords[ X_MAX ]  - coords[ X_MIN ]
    height = coords[ Y_MAX ] - coords[ Y_MIN]

    # https://github.com/paul-pias/Object-Detection-and-Distance-Measurement
    distance = ((2 * 3.14 * 180) / (width + height * 360) * 1000 + 3) /39.37

    #print("El objeto se encuentra a una distancia de " + str(distance))
    pepper_speed = 1
    # distance
    movement_time = (2.6669/pepper_speed) * 5

    #print("Voy a caminar por " + str(movement_time) + "s")
    pepper.move(pepper_speed, 0, 0)
    #print("Voy a caminar, miher")
    time.sleep(movement_time)
    #print("A mimir")
    pepper.move(0,0,0)
    current_angle = pepper.getAnglesPosition(['RShoulderPitch'])
    # print("Mi brazo derecho está en " + str(current_angle))

    # rotate(pepper)
    pepper.setAngles('RShoulderPitch', 2.08 - current_angle, 1)
    text_to_read = "Aquí está el " + original_text
    reading_from_string(text_to_read, filename)
    print(text_to_read)
else:
    text_to_read = "No sé qué es ese objeto"
    reading_from_string(text_to_read, filename)
    print(text_to_read)

input("Cualquier tecla para salir")

pepper.unsubscribeCamera(handle)
shm.close()
shm.unlink()

cap.release()
cv2.destroyAllWindows()