text:
import numpy as np
import sys
import matplotlib.pyplot as plt

from metod_alg import objective_functions as mt_obj

def plot_functions(obj, test_num, seed):
    """
    Generate contour plot for either the minimum of several quadratic forms
    or sum of gaussians function.

    Parameters
    ----------
    obj : string
          If obj = 'quad', then the minimum of several quadratic forms contour
          plot is generated. Otherwise, if obj = 'sog', then the sum of
          gaussians contour plot is generated.
    test_num : integer
               Number of points to evaluate function to compute contour plot.
    seed : integer
           Seed used to initialize the pseudo random number generator.
    """
    np.random.seed(seed)
    d = 2
    P = 4
    lambda_1 = 1
    lambda_2 = 5
    if obj == 'quad':
        f = mt_obj.several_quad_function
        store_x0, matrix_combined = (mt_obj.function_parameters_several_quad
                                     (P, d, lambda_1, lambda_2))
        store_x0 = np.array([[0.96, 0.09],
                             [0.86, 0.9],
                             [0.2, 0.98],
                             [0.12, 0.22]])
        args = P, store_x0, matrix_combined

    elif obj == 'sog':
        f = mt_obj.sog_function
        store_x0, matrix_combined, store_c = (mt_obj.function_parameters_sog
                                              (P, d, lambda_1, lambda_2))

        store_x0 = np.array([[0.96, 0.09],
                             [0.86, 0.9],
                             [0.2, 0.98],
                             [0.12, 0.22]])
        store_c = np.array([0.8, 0.7, 0.9, 0.75])
        sigma_sq = 0.05
        args = P, sigma_sq, store_x0, matrix_combined, store_c

    x = np.linspace(0, 1, test_num)
    y = np.linspace(0, 1, test_num)
    Z = np.zeros((test_num, test_num))
    X, Y = np.meshgrid(x, y)
    for i in range(test_num):
        for j in range(test_num):
            x1_var = X[i, j]
            x2_var = Y[i, j]
            Z[i, j] = f(np.array([x1_var, x2_var]).reshape(2, ), *args)

    plt.contour(X, Y, Z, 50, cmap='RdGy', alpha=0.5)
    plt.savefig('%s_d=2_rs_%s.png' % (obj, seed))

if __name__ == "__main__":
    obj = str(sys.argv[1])
    test_num = int(sys.argv[2])
    seed = int(sys.argv[3])
    plot_functions(obj, test_num, seed)

max_stars_repo_name: Megscammell/METOD-Algorithm

max_issues_repo_name: Megscammell/METOD-Algorithm

max_forks_repo_name: Megscammell/METOD-Algorithm


text:
# -*- coding:utf-8 -*-
# PLA 算法

import numpy as np
import matplotlib.pyplot as plt
import operator
import time

train_data = array([
	[1, 1, 4],
	[1, 2, 3],
	[1, -2, 3],
	[1, 0, 1],
	[1, 1, 2]
	])

train_label = [1, 1, 1, -1, -1, -1]

def sign(X):
	return 1 if X>0 else -1

ext: py

max_stars_repo_name: heriky/machine-learning-technique

max_issues_repo_name: heriky/machine-learning-technique

max_forks_repo_name: heriky/machine-learning-technique

reason: import numpy


text:
"""ASCII Word to point multi-cloud function
"""
import numpy as np
import random
import math

from mathutils.noise import seed_set, random_unit_vector

def letter_to_vertices(letter, letter_vertices, letter_scale, position_scale, seed):
    vertices = []

    random.seed(letter + seed)
    position = (random.uniform(-1, 1) * position_scale, random.uniform(-1, 1) * position_scale)

    for v in range(letter_vertices):
        vertex = (random_unit_vector() * letter_scale)
        vertex[0] = vertex[0] + position[0]
        vertex[1] = vertex[1] + position[1]
        vertex[2] = 0
        vertices.append(vertex.to_tuple())

    return vertices

def sv_main(letters=[[]], letter_vertices=32, letter_scale=1, position_scale=10, seed=0):
    in_sockets = [
        ['s', 'letters', letters],
        ['s', 'letter_vertices', letter_vertices],
        ['s', 'letter_scale', letter_scale],
        ['s', 'position_scale', position_scale],
        ['s', 'seed', seed]]

    vertices = []

    if len(letters[0]) > 0:
        for letter_index, letter in enumerate(letters[0][0]):
            vertices = vertices + letter_to_vertices(letter,
                                                    letter_vertices,
                                                    letter_scale,
                                                    position_scale,
                                                    seed + letter_index)

    out_sockets = [['v', 'Vertices', [vertices]]]

    return in_sockets, out_sockets

max_stars_repo_name: zeffii/sverchok-nodes

max_issues_repo_name: zeffii/sverchok-nodes

max_forks_repo_name: zeffii/sverchok-nodes


text:
# -*- coding: utf-8 -*-
"""

Created on February 12, 2018

@author:  neerbek

New cut based of exp194 (full 201, emb=100, lr=0.4, bs=50)
"""
# -*- coding: utf-8 -*-
import os
os.chdir("../../taboo-core")
from numpy.random import RandomState  # type: ignore
from sklearn.cluster import KMeans  # type: ignore

import ai_util
import confusion_matrix
import kmeans_cluster_util as kutil
import similarity.load_trees as load_trees

# import pylab  # type: ignore
import matplotlib.pyplot as plt
import importlib
# importlib.reload(kutil)
importlib.reload(confusion_matrix)

#       best    running
# m25K  0.8842, 0.8842
# m50K  0.9211, 0.9211
# m100K 0.9255, 0.9133
# m150K 0.9361, 0.9282
# m200K 0.9393, 0.9289
# m250K 0.9417, 0.9417
# m300K 0.9417, 0.9360 (same as 250K)
# m400K 0.9450, 0.9200
# m500K 0.9499, 0.9479
# m750K 0.9525, 0.9525

# OMP_NUM_THREADS=2 ipython3 functionality/run_model_verbose.py -- -inputtrees ../taboo-jan/functionality/201/data_full_random_cleaned.zip\$train.txt -nx 100 -nh 100 -L1_reg 0 -L2_reg 0 -retain_probability 1 -batch_size 80 -glove_path ../code/glove/ -inputmodel ../taboo-jan/functionality/logs/exp199_800K.zip\$save_exp199_best_250000.txt -output_embeddings -max_tree_count 100 -max_count 100 -max_embedding_count 10000
#
# for realz
# OMP_NUM_THREADS=2 ipython3 functionality/run_model_verbose.py -- -inputtrees ../taboo-jan/functionality/201/data_full_random_cleaned.zip\$train.txt -nx 100 -nh 100 -L1_reg 0 -L2_reg 0 -retain_probability 1 -batch_size 80 -glove_path ../code/glove/ -inputmodel ../taboo-jan/functionality/logs/exp199_800K.zip\$save_exp199_best_250000.txt -output_embeddings > train250K.txt

# OMP_NUM_THREADS=2 ipython3 functionality/run_model_verbose.py -- -inputtrees ../taboo-jan/functionality/201/data_full_random_cleaned.zip\$dev.txt -nx 100 -nh 100 -L1_reg 0 -L2_reg 0 -retain_probability 1 -batch_size 80 -glove_path ../code/glove/ -inputmodel ../taboo-jan/functionality/logs/exp199_800K.zip\$save_exp199_best_250000.txt -output_embeddings > dev.txt
#
# zip -m output/output_embeddings_201_exp199_m25K.zip train.txt dev.txt
# zip -m output/output_embeddings_201_exp199_m50K.zip train.txt dev.txt
# mv train100K.txt train.txt
# zip -m output/output_embeddings_201_exp199_m100K.zip train.txt dev.txt
# zip -m output/output_embeddings_201_exp199_m150K.zip train.txt dev.txt
# mv train200K.txt train.txt
# zip -m output/output_embeddings_201_exp199_m200K.zip train.txt dev.txt
# mv train250K.txt train.txt
# zip -m output/output_embeddings_201_exp199_m250K.zip train.txt dev.txt
# mv train300K.txt train.txt
# zip -m output/output_embeddings_201_exp199_m300K.zip train.txt dev.txt
# mv train400K.txt train.txt
# zip -m output/output_embeddings_201_exp199_m400K.zip train.txt dev.txt
# mv train500K.txt train.txt
# zip -m output/output_embeddings_201_exp199_m500K.zip train.txt dev.txt
# mv train750K.txt train.txt
# zip -m output/output_embeddings_201_exp199_m750K.zip train.txt dev.txt

totaltimer = ai_util.Timer("Total time: ")
traintimer = ai_util.Timer("Train time: ")
totaltimer.begin()

basefile = "output_embeddings_201_exp199_m750K.zip"
basefile = "output_embeddings_201_exp199_m500K.zip"
basefile = "output_embeddings_201_exp199_m300K.zip"
basefile = "output_embeddings_201_exp199_m250K.zip"
basefile = "output_embeddings_201_exp199_m200K.zip"
basefile = "output_embeddings_201_exp199_m150K.zip"
basefile = "output_embeddings_201_exp199_m100K.zip"
basefile = "output_embeddings_201_exp199_m50K.zip"
inputfileTrain = "output/" + basefile + "$train.txt"
print("reading: " + inputfileTrain)
linesTrainFull = confusion_matrix.read_embeddings(inputfileTrain, max_line_count=-1)
linesTrain = [linesTrainFull[i] for i in range(60000)]
inputfileDev = "output/" + basefile + "$dev.txt"
linesDev = confusion_matrix.read_embeddings(inputfileDev, max_line_count=-1)

numberOfClusters = 35
randomSeed = 7485
doShow = True
# from exp194: # (low, acc, fraction) = (5, 99.1%, 23%)
low = 4
high = 27

rng = RandomState(randomSeed)
aTrain = confusion_matrix.get_embedding_matrix(linesTrain, normalize=True)
aTrainFull = confusion_matrix.get_embedding_matrix(linesTrainFull, normalize=True)
aDev = confusion_matrix.get_embedding_matrix(linesDev, normalize=True)

kmeans = KMeans(n_clusters=numberOfClusters, random_state=rng).fit(aTrain)
# kmeans = KMeans(n_clusters=numberOfClusters, random_state=rng).fit(aTrainFull)
# kmeans = KMeans(n_clusters=numberOfClusters, random_state=rng).fit(aDev)
# sort_order = kutil.get_cluster_sen_ratios_sort_order(aTrainFull, linesTrainFull, kmeans)
sort_order = kutil.get_cluster_sen_ratios_sort_order(aTrain, linesTrain, kmeans)

show = kutil.SHOW_ALL
if doShow:
    # plot
    y1 = kutil.get_cluster_sen_ratios(aTrain, linesTrain, kmeans, sort_order)
    y2 = kutil.getScaledSizes(aTrain, kmeans, sort_order)
    print("(low, acc, fraction) = ({}, {:.1f}%, {:.0f}%)".format(low, (1 - y1[low - 1]) * 100, y2[low - 1] * 100))
    # y1 = kutil.get_cluster_sen_ratios(aTrainFull, linesTrainFull, kmeans, sort_order)
    # y2 = getScaledSizes(aTrainFull, kmeans, sort_order)
    # y1 = kutil.get_cluster_sen_ratios(aDev, linesDev, kmeans, sort_order)
    # y2 = kutil.getScaledSizes(aDev, kmeans, sort_order)
    x = kutil.getXRange(show, low, high, numberOfClusters)
    y1 = [y1[i] for i in x]
    y2 = [y2[i] for i in x]
    confusion_matrix.new_graph('Clusters', 'Ratio')
    plt.title('On dataset $201$, using emb from exp194, after $50K$ minibatches')
    plt.plot(x, y1, 'g:', label='Sensitivity')
    plt.plot(x, y2, 'k:', label='Accumulate size')
    if show == kutil.SHOW_ALL:
        plt.plot((low, low), (0, 1), 'k-')
        # plt.plot((high -1 , high - 1), (0, 1), 'k:')
    plt.legend()
    outfile = 'ijcai18_plot_1_201_6_exp199_4.eps'
    print("saving " + outfile)
    plt.savefig(outfile)
    # plt.show() don't call show from an interactive prompt :(
    # https://github.com/matplotlib/matplotlib/issues/8505/

clusterIds = sort_order  # clusterIds == sort_order, it's just syntaxtic sugar
(linesC1, aC1) = kutil.get_sentences_from_clusters(clusterIds[:low], linesTrainFull, aTrainFull, kmeans)
(linesC2, aC2) = kutil.get_sentences_from_clusters(clusterIds[low:], linesTrainFull, aTrainFull, kmeans)
(lines2C1, a2C1) = kutil.get_sentences_from_clusters(clusterIds[:low], linesDev, aDev, kmeans)
(lines2C2, a2C2) = kutil.get_sentences_from_clusters(clusterIds[low:], linesDev, aDev, kmeans)
print("(low, acc, fraction) = ({}, {:.1f}%, {:.0f}%)".format(low, (1 - y1[low - 1]) * 100, y2[low - 1] * 100))
print(len(linesC1), len(linesC2))
print(len(lines2C1), len(lines2C2))

kutil.get_base_accuracy(linesTrainFull, "train").report()
kutil.get_base_accuracy(linesDev, "dev").report()
kutil.get_base_accuracy(linesC1, "train C1").report()
kutil.get_base_accuracy(linesC2, "train C2").report()
kutil.get_base_accuracy(lines2C1, "dev C1").report()
kutil.get_base_accuracy(lines2C2, "dev C2").report()

cmC1 = kutil.get_base_accuracy(lines2C1)
cmC2 = kutil.get_base_accuracy(lines2C2)

formula = "(* (/ {}.0 {}) {})"
formulaC1 = formula.format(len(lines2C1), len(linesDev), "{}")  # little hack "{}" means not setted yet
formulaC2 = formula.format(len(lines2C2), len(linesDev), "{}")
print("formula:  (+ {} {})".format(formulaC1.format(cmC1.get_accuracy()), formulaC2.format(cmC2.get_accuracy())))

load_trees.put_trees("C1.txt", [l.tree for l in linesC1])
load_trees.put_trees("C2.txt", [l.tree for l in linesC2])
load_trees.put_trees("2C1.txt", [l.tree for l in lines2C1])
load_trees.put_trees("2C2.txt", [l.tree for l in lines2C2])

# 50K   #############################################
# (2, 99.3%, 16%), (3, 99.2%, 21%), (4, 98.6%, 27%)

# ijcai18_plot_1_201_6_exp199_4.eps

# (low, acc, fraction) = (3, 99.2%, 21%)
# 22163 82093
# 3096 11904
# Accuracy (train): 0.9318 (0.8689), f1=0.6819 (7623 1065 89520 6048)
# Accuracy (dev): 0.9211 (0.8675), f1=0.6300 (1007 202 12810 981)
# Accuracy (train C1): 0.9928 (0.9928), f1=0.0000 (0 0 22003 160)
# Accuracy (train C2): 0.9153 (0.8354), f1=0.6868 (7623 1065 67517 5888)
# Accuracy (dev C1): 0.9916 (0.9916), f1=0.0000 (0 0 3070 26)
# Accuracy (dev C2): 0.9028 (0.8352), f1=0.6351 (1007 202 9740 955)

# formula:  (+ (* (/ 3096.0 15000) 0.9916020671834626) (* (/ 11904.0 15000) 0.9028057795698925))

# 50K zip -m ../taboo-jan/functionality/201/trees_ijcai18_exp199_50K.zip C1.txt C2.txt 2C1.txt 2C2.txt

# 100K   #############################################
# (3, 99.1%, 20%) (4, 99.0%, 24%) (5, 99.0%, 28%),

# (low, acc, fraction) = (4, 99.0%, 24%)
# 24788 79468
# 3479 11521

# Accuracy (train): 0.9370 (0.8689), f1=0.7128 (8145 1039 89546 5526)
# Accuracy (dev): 0.9255 (0.8675), f1=0.6570 (1070 199 12813 918)
# Accuracy (train C1): 0.9935 (0.9935), f1=0.0000 (0 0 24627 161)
# Accuracy (train C2): 0.9194 (0.8300), f1=0.7178 (8145 1039 64919 5365)
# Accuracy (dev C1): 0.9894 (0.9894), f1=0.0000 (0 0 3442 37)
# Accuracy (dev C2): 0.9063 (0.8307), f1=0.6646 (1070 199 9371 881)

# formula:  (+ (* (/ 3479.0 15000) 0.9893647599885025) (* (/ 11521.0 15000) 0.9062581373144692))

# 100K zip -m ../taboo-jan/functionality/201/trees_ijcai18_exp199_100K.zip C1.txt C2.txt 2C1.txt 2C2.txt

# 150K   #############################################
# (4, 98.9%, 21%), (5, 98.9%, 25%), (6, 98.8%, 27%)

# (low, acc, fraction) = (4, 98.9%, 21%)

# 22360 81896
# 3090 11910
# Accuracy (train): 0.9496 (0.8689), f1=0.7810 (9380 968 89617 4291)
# Accuracy (dev): 0.9361 (0.8675), f1=0.7205 (1236 207 12805 752)
# Accuracy (train C1): 0.9940 (0.9940), f1=0.0000 (0 0 22226 134)
# Accuracy (train C2): 0.9374 (0.8347), f1=0.7854 (9380 968 67391 4157)
# Accuracy (dev C1): 0.9913 (0.9913), f1=0.0000 (0 0 3063 27)
# Accuracy (dev C2): 0.9217 (0.8353), f1=0.7262 (1236 207 9742 725)

# formula:  (+ (* (/ 3090.0 15000) 0.9912621359223301) (* (/ 11910.0 15000) 0.9217464315701092))
# 150K zip -m ../taboo-jan/functionality/201/trees_ijcai18_exp199_150K.zip C1.txt C2.txt 2C1.txt 2C2.txt

# 200K   #############################################
# (4, 99.4%, 19%), (5, 99.0%, 23%), (6, 98.6%, 27%)

# saving ijcai18_plot_1_201_6_exp199_2.eps
# (low, acc, fraction) = (5, 99.0%, 23%)
# 24348 79908
# 3399 11601
# Accuracy (train): 0.9560 (0.8689), f1=0.8176 (10269 1181 89404 3402)
# Accuracy (dev): 0.9393 (0.8675), f1=0.7505 (1370 293 12719 618)
# Accuracy (train C1): 0.9943 (0.9943), f1=0.0000 (0 0 24208 140)
# Accuracy (train C2): 0.9444 (0.8307), f1=0.8221 (10269 1181 65196 3262)
# Accuracy (dev C1): 0.9894 (0.9894), f1=0.0000 (0 0 3363 36)
# Accuracy (dev C2): 0.9246 (0.8317), f1=0.7580 (1370 293 9356 582)

# formula:  (+ (* (/ 3399.0 15000) 0.9894086496028244) (* (/ 11601.0 15000) 0.9245754676321006))

# 200K zip -m ../taboo-jan/functionality/201/trees_ijcai18_exp199_200K.zip C1.txt C2.txt 2C1.txt 2C2.txt

# 250K   #############################################
# (4, 99.2%, 16%), (5, 99.1%, 23%), (6, 99.0%, 27%)

# (low, acc, fraction) = (5, 99.1%, 23%)

# 24228 80028
# 3384 11616
# Accuracy (train): 0.9596 (0.8689), f1=0.8315 (10403 948 89637 3268)
# Accuracy (dev): 0.9417 (0.8675), f1=0.7565 (1359 246 12766 629)
# Accuracy (train C1): 0.9934 (0.9934), f1=0.0000 (0 0 24068 160)
# Accuracy (train C2): 0.9493 (0.8312), f1=0.8369 (10403 948 65569 3108)
# Accuracy (dev C1): 0.9902 (0.9902), f1=0.0000 (0 0 3351 33)
# Accuracy (dev C2): 0.9275 (0.8317), f1=0.7635 (1359 246 9415 596)

# formula:  (+ (* (/ 3384.0 15000) 0.9902482269503546) (* (/ 11616.0 15000) 0.9275137741046832))

# 250K zip -m ../taboo-jan/functionality/201/trees_ijcai18_exp199_250K.zip C1.txt C2.txt 2C1.txt 2C2.txt

# 300K is same as 250K

# 500K   #############################################
# (5, 99.5%, 20%), (6, 99.2%, 24%), (7, 99.1%, 27%)

# saving ijcai18_plot_1_201_6_exp199_3.eps

# (low, acc, fraction) = (6, 99.2%, 24%)
# 25008 79248
# 3575 11425
# Accuracy (train): 0.9671 (0.8689), f1=0.8679 (11261 1017 89568 2410)
# Accuracy (dev): 0.9499 (0.8675), f1=0.7993 (1495 258 12754 493)
# Accuracy (train C1): 0.9954 (0.9954), f1=0.0000 (0 0 24894 114)
# Accuracy (train C2): 0.9582 (0.8289), f1=0.8718 (11261 1017 64674 2296)
# Accuracy (dev C1): 0.9883 (0.9885), f1=0.0000 (0 1 3533 41)
# Accuracy (dev C2): 0.9379 (0.8296), f1=0.8083 (1495 257 9221 452)

# formula:  (+ (* (/ 3575.0 15000) 0.9882517482517482) (* (/ 11425.0 15000) 0.9379431072210066))

# 500K zip -m ../taboo-jan/functionality/201/trees_ijcai18_exp199_500K.zip C1.txt C2.txt 2C1.txt 2C2.txt

# 750K   #############################################
# (3, 99.6%, 7%), (5, 99.2%, 19%), (6, 99.2%, 24%),  (7, 99.2%, 32%)

# saving

# (low, acc, fraction) = (6, 99.2%, 24%)
# 25153 79103
# 3611 11389
# Accuracy (train): 0.9690 (0.8689), f1=0.8712 (10918 475 90110 2753)
# Accuracy (dev): 0.9525 (0.8675), f1=0.8013 (1438 163 12849 550)
# Accuracy (train C1): 0.9942 (0.9942), f1=0.0000 (0 0 25008 145)
# Accuracy (train C2): 0.9610 (0.8290), f1=0.8763 (10918 475 65102 2608)
# Accuracy (dev C1): 0.9886 (0.9886), f1=0.0000 (0 0 3570 41)
# Accuracy (dev C2): 0.9410 (0.8290), f1=0.8106 (1438 163 9279 509)

# formula:  (+ (* (/ 3611.0 15000) 0.9886458044862919) (* (/ 11389.0 15000) 0.9409956976029502))

# 750K zip -m ../taboo-jan/functionality/201/trees_ijcai18_exp199_750K.zip C1.txt C2.txt 2C1.txt 2C2.txt

max_stars_repo_name: neerbek/taboo-selective

max_issues_repo_name: neerbek/taboo-selective

max_forks_repo_name: neerbek/taboo-selective

reason: from numpy


text:
import pathlib
import pickle
import numpy as np
import pytest
try:
    import matplotlib.pyplot
    HAVE_PLT = True
except ImportError:
    HAVE_PLT = False
import gshhg

def get_dirname() -> pathlib.Path:
    return pathlib.Path(__file__).absolute().parent.joinpath('GSHHS_shp')

def test_construct():
    instance = gshhg.GSHHG(get_dirname(), resolution="crude")
    assert instance.polygons() != 0
    assert instance.points() != 0

    instance = gshhg.GSHHG(get_dirname(),
                           resolution="crude",
                           levels=[6],
                           bbox=(-10, -20, 10, 20))
    assert instance.dirname == get_dirname()
    assert instance.resolution == "crude"
    assert instance.levels == [6]
    assert instance.bbox == (-10, -20, 10, 20)

    other = pickle.loads(pickle.dumps(instance))
    assert other.polygons() == instance.polygons()
    assert other.points() == instance.points()

    with pytest.raises(FileNotFoundError):
        gshhg.GSHHG(get_dirname().parent, resolution="crude")

    with pytest.raises(ValueError):
        gshhg.GSHHG(get_dirname(), resolution="CRUDE")

    with pytest.raises(ValueError):
        gshhg.GSHHG(get_dirname(), levels=[0])

    with pytest.raises(ValueError):
        gshhg.GSHHG(get_dirname(), levels=[7])

    with pytest.raises(IndexError):
        gshhg.GSHHG(get_dirname(), bbox=(0, ))

def get_figure_path(path: str) -> pathlib.Path:
    dirname = pathlib.Path(__file__).absolute().parent.joinpath("figures")
    dirname.mkdir(exist_ok=True, parents=True)
    return dirname.joinpath(path)

def test_to_svg():
    instance = gshhg.GSHHG(get_dirname(), resolution="crude")
    figure = get_figure_path("crude.svg")
    if figure.exists():
        figure.unlink()
    assert not figure.exists()
    instance.to_svg(figure)
    assert figure.exists()

def test_nearest():
    instance = gshhg.GSHHG(get_dirname(), resolution="crude")
    lon1 = np.random.uniform(-180.0, 180.0, 1000)
    lat1 = np.random.uniform(-90.0, 90.0, 1000)

    lon2, lat2 = instance.nearest(lon1, lat1, num_threads=0)
    lon3, lat3 = instance.nearest(lon1, lat1, num_threads=1)

    assert np.any(lon1 != lon2)
    assert np.any(lat1 != lat2)
    assert np.all(lon2 == lon3)
    assert np.all(lat2 == lat3)

def test_distance_to_nearest():
    instance = gshhg.GSHHG(get_dirname(), resolution="crude")
    lon = np.random.uniform(-180.0, 180.0, 1000)
    lat = np.random.uniform(-90.0, 90.0, 1000)

    d1 = instance.distance_to_nearest(lon, lat, num_threads=0)
    d2 = instance.distance_to_nearest(lon, lat, num_threads=1)

    assert np.all(d1 == d2)

    d1 = instance.distance_to_nearest(lon, lat, strategy="andoyer")
    d2 = instance.distance_to_nearest(lon, lat, strategy="haversine")
    d3 = instance.distance_to_nearest(lon, lat, strategy="thomas")
    d4 = instance.distance_to_nearest(lon, lat, strategy="vincenty")

    assert np.all(d1 != d2)
    assert np.all(d1 != d3)
    assert np.all(d1 != d4)
    assert np.all(d2 != d3)
    assert np.all(d2 != d4)
    assert np.all(d3 != d4)

def test_mask():
    instance = gshhg.GSHHG(get_dirname(), resolution="crude")

    lon = np.arange(-180, 180, 1, dtype=np.float64)
    lat = np.arange(-90, 90, 1, dtype=np.float64)
    mx, my = np.meshgrid(lon, lat)

    mask1 = instance.mask(mx.flatten(), my.flatten(), num_threads=0)
    mask2 = instance.mask(mx.flatten(), my.flatten(), num_threads=1)

    assert np.all(mask1 == mask2)
    assert set(mask1) == set((0, 1, 2, 3, 5, 6))

def test_grid_mapping_mask():
    instance = gshhg.GSHHG(get_dirname(), resolution="crude")
    ds = instance.grid_mapping_mask(0.25)
    array = ds.mask.data.compute()
    assert set(array.flatten()) == set((0, 1, 2, 3, 5, 6))
    if HAVE_PLT:
        figure = matplotlib.pyplot.figure(figsize=(15, 15), dpi=150)
        axe = figure.add_subplot(2, 1, 1)
        mx, my = np.meshgrid(ds.lon, ds.lat)
        axe.pcolormesh(mx, my, ds.mask, shading='auto')
        figure.savefig(get_figure_path("mask.png"),
                       bbox_inches='tight',
                       pad_inches=0.4)

def test_grid_mapping_distance_to_nearest():
    instance = gshhg.GSHHG(get_dirname(), resolution="crude")
    ds = instance.grid_mapping_distance_to_nearest(0.25)
    array = ds.distance.data.compute()
    assert array.mean() != 0
    if HAVE_PLT:
        figure = matplotlib.pyplot.figure(figsize=(15, 15), dpi=150)
        axe = figure.add_subplot(2, 1, 1)
        mx, my = np.meshgrid(ds.lon, ds.lat)
        axe.pcolormesh(mx, my, ds.distance, shading='auto')
        figure.savefig(get_figure_path("distance_to_nearest.png"),
                       bbox_inches='tight',
                       pad_inches=0.4)

max_stars_repo_name: fbriol/gshhg

max_issues_repo_name: fbriol/gshhg

max_forks_repo_name: fbriol/gshhg


text:
#!/usr/bin/env python3

from collections import defaultdict
from itertools import chain

from pgmpy.models import BayesianModel
from pgmpy.models import MarkovModel
from pgmpy.models import FactorGraph
from pgmpy.models import JunctionTree
from pgmpy.models import DynamicBayesianNetwork
from pgmpy.utils import StateNameInit

class Inference(object):
    """
    Base class for all inference algorithms.

    Converts BayesianModel and MarkovModel to a uniform representation so that inference
    algorithms can be applied. Also it checks if all the associated CPDs / Factors are
    consistent with the model.

    Initialize inference for a model.

    Parameters
    ----------
    model: pgmpy.models.BayesianModel or pgmpy.models.MarkovModel or pgmpy.models.NoisyOrModel
        model for which to initialize the inference object.

    Examples
    --------
    >>> from pgmpy.inference import Inference
    >>> from pgmpy.models import BayesianModel
    >>> from pgmpy.factors import TabularCPD
    >>> student = BayesianModel([('diff', 'grade'), ('intel', 'grade')])
    >>> diff_cpd = TabularCPD('diff', 2, [[0.2, 0.8]])
    >>> intel_cpd = TabularCPD('intel', 2, [[0.3, 0.7]])
    >>> grade_cpd = TabularCPD('grade', 3, [[0.1, 0.1, 0.1, 0.1],
    ...                                     [0.1, 0.1, 0.1, 0.1],
    ...                                     [0.8, 0.8, 0.8, 0.8]],
    ...                        evidence=['diff', 'intel'], evidence_card=[2, 2])
    >>> student.add_cpds(diff_cpd, intel_cpd, grade_cpd)
    >>> model = Inference(student)

    >>> from pgmpy.models import MarkovModel
    >>> from pgmpy.factors import DiscreteFactor
    >>> import numpy as np
    >>> student = MarkovModel([('Alice', 'Bob'), ('Bob', 'Charles'),
    ...                        ('Charles', 'Debbie'), ('Debbie', 'Alice')])
    >>> factor_a_b = DiscreteFactor(['Alice', 'Bob'], cardinality=[2, 2], value=np.random.rand(4))
    >>> factor_b_c = DiscreteFactor(['Bob', 'Charles'], cardinality=[2, 2], value=np.random.rand(4))
    >>> factor_c_d = DiscreteFactor(['Charles', 'Debbie'], cardinality=[2, 2], value=np.random.rand(4))
    >>> factor_d_a = DiscreteFactor(['Debbie', 'Alice'], cardinality=[2, 2], value=np.random.rand(4))
    >>> student.add_factors(factor_a_b, factor_b_c, factor_c_d, factor_d_a)
    >>> model = Inference(student)
    """

    @StateNameInit()
    def __init__(self, model):
        self.model = model
        model.check_model()

        if isinstance(model, JunctionTree):
            self.variables = set(chain(*model.nodes()))
        else:
            self.variables = model.nodes()

        self.cardinality = {}
        self.factors = defaultdict(list)

        if isinstance(model, BayesianModel):
            for node in model.nodes():
                cpd = model.get_cpds(node)
                cpd_as_factor = cpd.to_factor()
                self.cardinality[node] = cpd.variable_card

                for var in cpd.variables:
                    self.factors[var].append(cpd_as_factor)

        elif isinstance(model, (MarkovModel, FactorGraph, JunctionTree)):
            self.cardinality = model.get_cardinality()

            for factor in model.get_factors():
                for var in factor.variables:
                    self.factors[var].append(factor)

        elif isinstance(model, DynamicBayesianNetwork):
            self.start_bayesian_model = BayesianModel(model.get_intra_edges(0))
            self.start_bayesian_model.add_cpds(*model.get_cpds(time_slice=0))
            cpd_inter = [model.get_cpds(node) for node in model.get_interface_nodes(1)]
            self.interface_nodes = model.get_interface_nodes(0)
            self.one_and_half_model = BayesianModel(model.get_inter_edges() + model.get_intra_edges(1))
            self.one_and_half_model.add_cpds(*(model.get_cpds(time_slice=1) + cpd_inter))

max_stars_repo_name: abinashpanda/pgmpy

max_issues_repo_name: abinashpanda/pgmpy

max_forks_repo_name: abinashpanda/pgmpy


text:
import codecs
import copy
import numpy as np
import torch.utils.data as Data
import os
import math
from multiprocessing import Queue,Process,Manager
import torch

# from utils import AttrDict, init_logger, count_parameters, save_model, computer_cer
# import yaml
# import torch
# from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence

base2int = {'A': 1, 'A+': 2, 'C': 3, 'C+': 4, 'G': 5, 'G+': 6, 'T': 7, 'T+': 8, 'N': 9, 'N+': 10}  # padding 0
int2base = {1: 'A', 2: 'a', 3: 'C', 4: 'c', 5: 'G', 6: 'g', 7: 'T', 8: 't', 9: 'N', 10: 'n'}  # padding 0

class FeatureArray():
    refName = ""
    intervalStart = -1
    intervalEnd = -1
    array = None

    def __init__(self, name, s, e, array):
        self.refName = name
        self.intervalStart = s
        self.intervalEnd = e
        self.array = array

def generate_flipflop_sequence(value, length):
    out = []
    for i in range(length):
        if i % 2 == 0:
            out.append(value)
        else:
            out.append(value + 1)
    return out

def rle_label(seq, max_rle):
    base_list, rle_list = [], []
    rle = 1
    for i in range(len(seq)):
        if i > 0:
            prev_base = seq[i - 1]
            cur_base = seq[i]
            if prev_base == cur_base:
                rle += 1
            else:
                base_list.append(prev_base)
                if rle >= max_rle:
                    rle = max_rle
                # rle_list.append(rle_encoding[prev_base][rle - 1])
                rle_list.extend(generate_flipflop_sequence(prev_base, rle))
                rle = 1
        if i == len(seq) - 1:
            base_list.append(seq[i])
            if rle >= max_rle:
                rle = max_rle
            rle_list.extend(generate_flipflop_sequence(seq[i], rle))
    return np.array(base_list), np.array(rle_list)

def load_train_data(datafile, max_rle):
    regions, feats, targets, rle_bases, rles = [], [], [], [], []
    feat_max_length = 0
    target_max_length = 0
    rle_base_max_length = 0
    rle_max_length = 0
    with open(datafile) as fin:
        for line in fin:
            if line.startswith(">"):
                header = line.rstrip().replace('>', '')
                refName, startPos, endPos, label = header.split('\t')
                label = encode(label)
                label_shape = label.shape
                rle_base, rle = rle_label(label, max_rle)
            else:
                array = np.array([float(v) for v in line.split(',')[:-1]]).reshape((-1, 7))
                array_shape = array.shape
                if array_shape[0] > feat_max_length:
                    feat_max_length = array_shape[0]
                if rle_base.shape[0] > array_shape[0] or rle.shape[0] > array_shape[0]:
                    # print(rle_base.shape, rle.shape, array_shape)
                    # filter target length greater than input length
                    continue
                if label_shape[0] > 100 or array_shape[0] > 100:
                    # filter the extra long feature or target
                    continue
                if label_shape[0] > target_max_length:
                    target_max_length = label_shape[0]
                if rle_base.shape[0] > rle_base_max_length:
                    rle_base_max_length = rle_base.shape[0]
                if rle.shape[0] > rle_max_length:
                    rle_max_length = rle.shape[0]
                feats.append(array)
                targets.append(label)
                rle_bases.append(rle_base)
                rles.append(rle)
                regions.append(refName + '\t' + startPos + '\t' + endPos)
    return regions, feats, feat_max_length, targets, target_max_length, rle_bases, rles, rle_base_max_length, rle_max_length

def load_feature_data(datafile):
    regions, feats = [], []
    feat_max_length = 0
    with open(datafile) as fin:
        for line in fin:
            if line.startswith(">"):
                header = line.rstrip().replace('>', '')
                refName, startPos, endPos = header.split('\t')
            else:
                array = np.array([float(v) for v in line.split(',')[:-1]]).reshape((-1, 7))
                array_shape = array.shape
                if array_shape[0] > feat_max_length:
                    feat_max_length = array_shape[0]

                #  超长的简单区域或者复杂区域，分割成小的区域
                if array_shape[0] > 100:
                    split_array_list = []
                    split_region_list = []
                    split_size = math.ceil(array_shape[0] / 100)
                    for i in range(split_size):
                        if i != split_size - 1:
                            split_array_list.append(array[i * 100:(i + 1) * 100])
                        else:
                            split_array_list.append(array[i * 100:array_shape[0]])
                        split_region_list.append(refName + '\t' + startPos + '\t' + endPos + "\t" + str(i))
                    feats.extend(split_array_list)
                    regions.extend(split_region_list)
                    continue
                feats.append(array)
                regions.append(refName + '\t' + startPos + '\t' + endPos + "\t" + str(0))  # contig+起点+终点+重复编号
    return regions, feats, feat_max_length

def encode(seq):
    array = []
    for base in seq:
        array.append(base2int[base])
    return np.array(array)

def pad(inputs, max_length):
    dim = len(inputs.shape)
    if dim == 1:
        pad_zeros_mat = np.zeros([max_length - inputs.shape[0]], dtype=np.int32)
        padded_inputs = np.concatenate([inputs, pad_zeros_mat])
    elif dim == 2:
        feature_dim = inputs.shape[1]
        pad_zeros_mat = np.zeros([max_length - inputs.shape[0], feature_dim])
        padded_inputs = np.row_stack([inputs, pad_zeros_mat])
    else:
        raise AssertionError(
            'Features in inputs list must be one or two dimension matrix! ')
    return padded_inputs

class PolishTrainDataset(Data.Dataset):
    def __init__(self, datafile, max_rle):
        super(PolishTrainDataset, self).__init__()
        self.train_data_file = datafile
        self.regions, self.train_feats, self.feat_max_length, self.train_labels, self.label_max_length, self.rle_bases, self.rles, self.rle_base_max_length, self.rle_max_length = load_train_data(
            datafile, max_rle)  # [array(L,7),array(L,7),...]
        self.lengths = len(self.train_feats)

    def __getitem__(self, index):
        region = self.regions[index]
        feature = self.train_feats[index]
        target = self.train_labels[index]
        rle_base = self.rle_bases[index]
        rle = self.rles[index]
        input_length = np.array(feature.shape[0]).astype(np.int)
        target_length = np.array(target.shape[0]).astype(np.int)
        rle_base_length = np.array(rle_base.shape[0]).astype(np.int)
        rle_length = np.array(rle.shape[0]).astype(np.int)
        feature = pad(feature, self.feat_max_length).astype(np.float32)  # [array(L,7)]
        target = pad(target, self.label_max_length).astype(np.int)  # [array(L)]
        rle_base = pad(rle_base, self.rle_base_max_length).astype(np.int)
        rle = pad(rle, self.rle_max_length).astype(np.int)
        return region, feature, input_length, target, target_length, rle_base, rle_base_length, rle, rle_length

    def __len__(self):
        return self.lengths

def Prod(datafile,batch_size,thread_num,prod_queue):
    """生产者线程"""
    tmp = []
    idx = 0
    with open(datafile) as fin:
        for line in fin:
            if line.startswith(">"):
                header = line.rstrip()
            else:
                feat = line.rstrip()
                tmp.append((idx,header,feat))
                if len(tmp)==batch_size:
                    prod_queue.put(tmp)
                    idx+=1
                    tmp = []
    return 0

def Cust(prod_queue,cust_queue):
    while True:
        batch_data = prod_queue.get()
        if batch_data is None:
            break
        """padding"""
        regions, feats = [],[]
        feat_max_length = 0
        for (idx,header,feat) in batch_data:
            header = header.rstrip().replace('>', '')
            refName, startPos, endPos = header.split('\t')

            array = np.array([float(v) for v in feat.split(',')[:-1]]).reshape((-1, 7))
            array_shape = array.shape
            #  超长的简单区域或者复杂区域，分割成小的区域
            if array_shape[0] > 100:
                split_array_list = []
                split_region_list = []
                split_size = math.ceil(array_shape[0] / 100)
                for i in range(split_size):
                    if i != split_size - 1:
                        sarray = array[i * 100:(i + 1) * 100]
                        if sarray.shape[0] > feat_max_length:
                            feat_max_length = sarray.shape[0]
                        split_array_list.append(sarray)
                    else:
                        sarray = array[i * 100:array_shape[0]]
                        if sarray.shape[0] > feat_max_length:
                            feat_max_length = sarray.shape[0]
                        split_array_list.append(sarray)
                    split_region_list.append(refName + '\t' + startPos + '\t' + endPos + "\t" + str(i))
                feats.extend(split_array_list)
                regions.extend(split_region_list)
                continue
            if array_shape[0] > feat_max_length:
                feat_max_length = array_shape[0]
            feats.append(array)
            regions.append(refName + '\t' + startPos + '\t' + endPos + "\t" + str(0))  # contig+起点+终点+重复编号
        feature_lengths = np.array([v.shape[0] for v in feats]).astype(np.int)
        feature = np.array([pad(v, feat_max_length).astype(np.float32) for v in feats])
        cust_queue.put((idx,feature,regions,feature_lengths))
    return 0

def load_feature_generate_data(datafile,batch_size, thread_num):
    pthread_list = []
    qMar = Manager()
    prod_queue = qMar.Queue(maxsize=100)
    cust_queue = qMar.Queue()

    ##建立1个生产者线程
    thread_pro = Process(target=Prod,args=[datafile,batch_size,thread_num,prod_queue])
    thread_pro.start()

    ##建立多个个消费者线程
    for i in range(thread_num):
        thread_cus = Process(target=Cust, args=[prod_queue,cust_queue])
        thread_cus.start()
        pthread_list.append(thread_cus)

    thread_pro.join()

    for i in range(thread_num):
        prod_queue.put(None)

    for t in pthread_list:
        t.join()

    print("threading done.")
    return cust_queue

class PolishGenerateDataset(Data.Dataset):
    def __init__(self, datafile,batch_size,threads):
        super(PolishGenerateDataset, self).__init__()
        self.train_data_file = datafile
        data_queue = load_feature_generate_data(datafile,batch_size,threads)
        # self.regions, self.train_feats, self.feat_max_length = load_feature_data(datafile)  # [array(L,7),array(L,7),...]
        self.lengths = data_queue.qsize()
        batch_data_dict = {}
        while not data_queue.empty():
            (idx,features,regions,input_lengths) = data_queue.get()
            batch_data_dict[idx] = (features,regions,input_lengths)
        self.batch_data_dict = batch_data_dict

    def __getitem__(self, index):
        batch_data = self.batch_data_dict[index]
        regions = batch_data[1]
        features = batch_data[0]
        input_lengths = batch_data[2]
        return regions, features, input_lengths

    def __len__(self):
        return self.lengths

# if __name__ == '__main__':
#     configfile = open("../config/aishell.yaml")
#
#     config = AttrDict(yaml.load(configfile, Loader=yaml.FullLoader))
#     train_dataset = PolishTrainDataset(config.data)
#     training_data = torch.utils.data.DataLoader(train_dataset, batch_size=10, shuffle=True, num_workers=4)
#     print(train_dataset.feat_max_length, train_dataset.label_max_length)
#     for setp, (regions, feats, inputs_length, targets, targets_length) in enumerate(training_data):
#         feats = torch.FloatTensor(feats)
#         targets = torch.LongTensor(targets)
#         inputs_length = torch.LongTensor(inputs_length)
#         targets_length = torch.LongTensor(targets_length)
#         pack = pack_padded_sequence(feats, inputs_length, batch_first=True, enforce_sorted=False)  # [N, L, C]

if __name__=='__main__':
    generate_dataset = PolishGenerateDataset("/home/user/experiment/BlockPolish/computing_time/BlockPolish/racon/chr22-complex_features.txt",512,40)
    generateg_data = torch.utils.data.DataLoader(generate_dataset, batch_size=1, shuffle=False, num_workers=4)
    for setp, (regions, feats, inputs_length) in enumerate(generateg_data):
        print(feats.shape,inputs_length.shape)
        # print(inputs_length[0])
        print(regions)
        feats = torch.FloatTensor(feats)
        inputs_length = torch.LongTensor(inputs_length)
        print(feats.size())

max_stars_repo_name: huangnengCSU/BlockPolish

max_issues_repo_name: huangnengCSU/BlockPolish

max_forks_repo_name: huangnengCSU/BlockPolish


text:
import numpy as np
import tensorflow as tf
import random

sess = tf.Session()

def loadDataSet():
    # load the dataset
    data = np.load('data_set.npz', encoding='latin1')
    training_data = data['training_data']
    #print(len(training_data))
    validation_data = data['validation_data']
    test_data = data['testing_data']

    # format the dataset into array and output labels in one hot format
    training_inputs = [np.reshape(training_data[x]['img'],(1,1024)) for x in range(len(training_data))]
    training_results = [vectorized_result(training_data[x]['label']) for x in range(len(training_data))]
    training_data = {'training_inputs': training_inputs, 'training_results': training_results}

    validation_inputs =  [np.reshape(validation_data[x]['img'],(1,1024)) for x in range(len(validation_data))]
    validation_results = [vectorized_result(validation_data[x]['label']) for x in range(len(validation_data))]
    validation_data = {'validation_inputs': validation_inputs, 'validation_results': validation_results}

    test_inputs =  [np.reshape(test_data[x]['img'],(1,1024)) for x in range(len(test_data))]
    test_results = [vectorized_result(test_data[x]['label']) for x in range(len(test_data))]
    test_data = {'test_inputs': test_inputs, 'test_results': test_results}
    print(len(test_inputs))

    return [training_data, test_data, validation_data]

def vectorized_result(y):
    e = np.zeros((1,62))
    e[0][y] = 1.0
    return e

training_data, test_data,validation_data =loadDataSet()

n_hidden_1 = 2048
n_hidden_2 = 2048
n_hidden_3 = 2048
n_hidden_4 = 2048
n_classes = 62
batch_size = 10

x = tf.placeholder(tf.float32, [None, 1024])
y = tf.placeholder(tf.float32, [None, n_classes])

def neural_network_model(data):
    hidden_layer_1 = {
        'weights': tf.Variable(tf.random_normal([1024, n_hidden_1])),
        'biases': tf.Variable(tf.random_normal([n_hidden_1]))
    }

    hidden_layer_2 = {
        'weights': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
        'biases': tf.Variable(tf.random_normal([n_hidden_2]))
    }

    hidden_layer_3 = {
        'weights': tf.Variable(tf.random_normal([n_hidden_2, n_hidden_3])),
        'biases': tf.Variable(tf.random_normal([n_hidden_3]))
    }

    hidden_layer_4 = {
        'weights': tf.Variable(tf.random_normal([n_hidden_3, n_hidden_4])),
        'biases': tf.Variable(tf.random_normal([n_hidden_4]))
    }

    output_layer = {
        'weights': tf.Variable(tf.random_normal([n_hidden_4, n_classes])),
        'biases': tf.Variable(tf.random_normal([n_classes]))
    }

    layer_1 = tf.add(tf.matmul(data, hidden_layer_1['weights']), hidden_layer_1['biases'])
    tf.nn.relu(layer_1)

    layer_2 = tf.add(tf.matmul(layer_1, hidden_layer_2['weights']), hidden_layer_2['biases'])
    tf.nn.relu(layer_2)

    layer_3 = tf.add(tf.matmul(layer_2, hidden_layer_3['weights']), hidden_layer_3['biases'])
    tf.nn.relu(layer_3)

    layer_4 = tf.add(tf.matmul(layer_3, hidden_layer_4['weights']), hidden_layer_4['biases'])
    tf.nn.relu(layer_4)

    output = tf.add(tf.matmul(layer_4, output_layer['weights']), output_layer['biases'])

    return output

def train_neural_network(x):

    prediction = neural_network_model(x)
#     print("Pred :",prediction)
    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=prediction, labels=y))

    optimizer = tf.train.AdamOptimizer(0.5).minimize(cost)
    correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))
#     print("Correct:",correct)
#     print( tf.argmax(prediction, 1),"  ",tf.argmax(y, 1))
    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
    saver = tf.train.Saver()

    n_epochs = 30

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        for epoch in range(n_epochs):
            epoch_loss = 0
            total_acc = 0
            total_batches = len(training_data['training_inputs']) // batch_size
            for i in range(total_batches):
                epoch_x = np.squeeze(np.array(training_data['training_inputs'][i * batch_size:(i + 1) * (batch_size)]))
                epoch_y = np.squeeze(np.array(training_data['training_results'][i * batch_size:(i + 1) * (batch_size)]))
                _, c, acc = sess.run([optimizer, cost, accuracy], feed_dict={x: epoch_x, y: epoch_y})
                epoch_loss += c
                total_acc += acc
            save_path = saver.save(sess, "./biases_weights.ckpt")
            print('Epoch ', epoch, ' completed out of ', n_epochs, 'loss : ', epoch_loss, total_acc / total_batches)
        print('optimization finished')

        test_x = np.squeeze(np.array(validation_data['validation_inputs']))
        test_y = np.squeeze(np.array(validation_data['validation_results']))
        acc = sess.run(accuracy, feed_dict={x: test_x, y: test_y})
        print("Validation Accuracy",acc)

train_neural_network(x)


text:
#!/usr/bin/env python
import numpy as np
from nn_numpy import Layer
from nn_numpy import SoftmaxLayer
from mnist import load_mnist
np.random.seed(1)

def main1():
    # 訓練データの画像・ラベルを読み込む (パス名は適宜変更)。
    train_images = load_mnist('./MNIST/train-images-idx3-ubyte.gz')
    train_labels = load_mnist('./MNIST/train-labels-idx1-ubyte.gz')
    # レイヤーを 2つ作成。
    layer1 = Layer(784, 100)
    layer2 = Layer(100, 10)
    n = 0
    for i in range(1):
        for (image,label) in zip(train_images, train_labels):
            # 28×28の画像をフラットな配列に変換。
            x = (image/255).reshape(784)
            # 正解部分だけが 1 になっている 10要素の配列を作成。
            ya = np.zeros(10)
            ya[label] = 1
            # 学習させる。
            y = layer1.forward(x)
            y = layer2.forward(y)
            delta = layer2.mse_loss(ya)
            delta = layer2.backward(delta)
            delta = layer1.backward(delta)
            n += 1
            if (n % 50 == 0):
                print(n, layer2.loss)
                layer1.update(0.01)
                layer2.update(0.01)
    test_images = load_mnist('./MNIST/t10k-images-idx3-ubyte.gz')
    test_labels = load_mnist('./MNIST/t10k-labels-idx1-ubyte.gz')
    correct = 0
    for (image,label) in zip(test_images, test_labels):
        x = (image/255).flatten()
        y = layer1.forward(x)
        y = layer2.forward(y)
        i = np.argmax(y)
        if i == label:
            correct += 1
    print(correct, len(test_images))

def main2():
    # 訓練データの画像・ラベルを読み込む (パス名は適宜変更)。
    train_images = load_mnist('./MNIST/train-images-idx3-ubyte.gz')
    train_labels = load_mnist('./MNIST/train-labels-idx1-ubyte.gz')
    # レイヤーを 3つ作成。
    layer1 = Layer(784, 100)
    layerx = Layer(100, 100)
    layer2 = SoftmaxLayer(100, 10)
    n = 0
    for i in range(1):
        for (image,label) in zip(train_images, train_labels):
            # 28×28の画像をフラットな配列に変換。
            x = (image/255).reshape(784)
            # 正解部分だけが 1 になっている 10要素の配列を作成。
            ya = np.zeros(10)
            ya[label] = 1
            # 学習させる。
            y = layer1.forward(x)
            y = layerx.forward(y)
            y = layer2.forward(y)
            delta = layer2.cross_entropy_loss_backward(ya)
            delta = layerx.backward(delta)
            delta = layer1.backward(delta)
            n += 1
            if (n % 50 == 0):
                print(n, layer2.loss)
                layer1.update(0.01)
                layerx.update(0.01)
                layer2.update(0.01)
    test_images = load_mnist('./MNIST/t10k-images-idx3-ubyte.gz')
    test_labels = load_mnist('./MNIST/t10k-labels-idx1-ubyte.gz')
    correct = 0
    for (image,label) in zip(test_images, test_labels):
        x = (image/255).flatten()
        y = layer1.forward(x)
        y = layerx.forward(y)
        y = layer2.forward(y)
        i = np.argmax(y)
        if i == label:
            correct += 1
    print(correct, len(test_images))

main2()

max_stars_repo_name: euske/introdl

max_issues_repo_name: euske/introdl

max_forks_repo_name: euske/introdl


text:
#!/usr/bin/env python3
"""Test free energy for ideal treatment of unbound electrons."""
from atoMEC import Atom, models, config
import numpy as np

# expected values and tolerance
unbound_expected = -10.091898
accuracy = 1e-3

class TestUnbound:
    """
    Test class for ideal treatment of unbound electrons.

    Checks the free energy for an SCF calculation is given by the expected value.
    Also uses the force_bound parameter.
    """

    def test_unbound(self):
        """Check free energy for SCF calc with unbound=ideal."""
        # test "parallel" with 1 core
        config.numcores = 1

        assert np.isclose(self._run(), unbound_expected, atol=accuracy)

    @staticmethod
    def _run():
        """
        Run an SCF calculation for a Be atom with unbound="ideal".

        Returns
        -------
        F_tot : float
            the total free energy
        """
        # set up the atom and model
        Be_at = Atom(
            "He",
            25,
            radius=3.17506,
            units_temp="eV",
            units_radius="angstrom",
        )
        model = models.ISModel(Be_at, unbound="ideal")

        # run the SCF calculation
        output = model.CalcEnergy(
            3,
            3,
            scf_params={"maxscf": 3},
            grid_params={"ngrid": 1000},
            force_bound=[[0, 0, 0]],
        )

        # extract the total free energy
        F_tot = output["energy"].F_tot
        return F_tot

if __name__ == "__main__":
    print(TestUnbound._run())

max_stars_repo_name: timcallow/atoMEC

max_issues_repo_name: timcallow/atoMEC

max_forks_repo_name: timcallow/atoMEC


text:
#!/usr/local/bin/python2.7

from sys import exit
from os import environ, system
environ['KERAS_BACKEND'] = 'tensorflow'
# environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
# environ["CUDA_VISIBLE_DEVICES"] = ""

import numpy as np
import utils
import adversarial

from keras.models import Model, load_model
from keras.utils import np_utils
import obj
import config
obj.limit = 20
config.DEBUG = True
APOSTLE = 'panda_4b_akt'

if __name__ == '__main__':

    #config.n_truth = 5
    #config.truth = 'resonanceType'

    #classifier = load_model('models/classifier_conv_%s.h5'%APOSTLE)
    regularized = load_model('models/regularized_conv_%s.h5'%APOSTLE)

    basedir = '/fastscratch/snarayan/pandaarrays/v1_akt/'
    system('rm -f %s/test/*%s_conv.npy'%(basedir, APOSTLE))
    coll = obj.PFSVCollection()
    coll.add_categories(['singletons','pf'],
                        basedir+'/PARTITION/*CATEGORY.npy')

    def predict_t(data):
        msd_idx = obj.singletons['msd']
        rho_idx = obj.singletons['rawrho']
        pt_idx = obj.singletons['rawpt']
        if obj.limit:
            inclusive = data['pf'][:,:obj.limit,:]
        else:
            inclusive = data['pf']
        msd = data['singletons'][:,msd_idx] / config.max_mass
        rho = (data['singletons'][:,rho_idx]  - config.min_rho) / config.max_rho
        pt = (data['singletons'][:,pt_idx] - config.min_pt) / config.max_pt
        if pt.shape[0] > 0:
            X = [inclusive, msd, pt]
            try:
               # r_classifier = classifier.predict(X)
               # r_classifier_top = r_classifier[:,config.n_truth-1]
                #return r_classifier_top

                r_regularized = regularized.predict(X)
                r_regularized_top = r_regularized[:,config.n_truth-1]
                return r_regularized_top

                #return np.vstack([r_classifier, r_regularized]).T
                #return np.vstack([r_classifier_top, r_regularized_top]).T
            except Exception as e:
                raise e
        else:
            return np.zeros((0,))

    coll.infer(['singletons', 'pf'], f=predict_t, name='%s_conv'%(APOSTLE), partition='test', ncores=1)

max_stars_repo_name: sammysiegel/SubtLeNet

max_issues_repo_name: sammysiegel/SubtLeNet

max_forks_repo_name: sammysiegel/SubtLeNet


text:
from pandas import read_csv
import numpy as np
from sklearn.utils import shuffle
from matplotlib.image import imread
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, Lambda, Cropping2D, MaxPooling2D

measurments = read_csv('data/driving_log.csv', usecols=[3]).values
C = measurments # Center Camera Measurments
L = measurments + 0.2 # Left Camera Measurments
R = measurments - 0.2 # Right Camera Measurments

# Center Camera Images Paths
images_C = read_csv('data/driving_log.csv', usecols=[0]).values
# Left Camera Images Paths
images_L = read_csv('data/driving_log.csv', usecols=[1]).values
# Right Camera Images Paths
images_R = read_csv('data/driving_log.csv', usecols=[2]).values

# Data Generator Method
def genImages(idx, size):
    images = []

    # import center images
    for path in images_C[idx:idx+size]:
        fname = 'data/IMG/'+ path[0].split('/')[-1]
        images.append(imread(fname))

    # import left images
    for path in images_L[idx:idx+size]:
        fname = 'data/IMG/'+ path[0].split('/')[-1]
        images.append(imread(fname))

    # import right images
    for path in images_R[idx:idx+size]:
        fname = 'data/IMG/'+ path[0].split('/')[-1]
        images.append(imread(fname))

    # convert images list to numpy array
    images = np.array(images)

    # flip all images and add them together
    images = np.concatenate((images, np.fliplr(images)), axis=0)

    # add measurments for:
    # original images (C, L, R)
    # flipped images (-C, -L, -R)
    measurments = np.concatenate((C[idx:idx+size], L[idx:idx+size], R[idx:idx+size],
                                  -C[idx:idx+size], -L[idx:idx+size], -R[idx:idx+size]), axis=0)

    # shuffle
    X_train, Y_train = shuffle(images, measurments)
    return X_train, Y_train

def model():
    #Keras Model Inisilization
    model = Sequential()

    # Pixel Normalization [-1, 1]
    model.add(Lambda(lambda x: x/127.5 - 1., input_shape=(160, 320, 3)))

    # Cropping image to get ride of unwanted stuff (e.g trees, revier)
    model.add(Cropping2D(cropping=((70, 25), (0, 0))))

    ##### Start - NVIDIA Architecture #####
    model.add(Conv2D(24, (5, 5), strides=(2, 2), activation='relu'))
    model.add(Conv2D(36, (5, 5), strides=(2, 2), activation='relu'))
    model.add(Conv2D(48, (5, 5), strides=(2, 2), activation='relu'))
    model.add(Conv2D(64, (3, 3), activation='relu'))
    model.add(Conv2D(64, (3, 3), activation='relu'))

    model.add(Flatten())

    model.add(Dense(100, activation='relu'))
    model.add(Dropout(0.5))

    model.add(Dense(50, activation='relu'))
    model.add(Dropout(0.5))

    model.add(Dense(10, activation='relu'))
    model.add(Dropout(0.5))

    model.add(Dense(1))

    ##### End - NVIDIA Architecture #####

    model.compile('adam', "mse")

    return model

if __name__ == "__main__":
    model = model()

    size = 2500
    rang = 10
    for i in range(rang):
        print('='*50)
        print('='*22,"%d/%d" % (i+1,rang), '='*22)
        print('='*50)
        for idx in range(rang):
                X_train, Y_train = genImages(idx*size, size)
                model.fit(X_train, Y_train, batch_size=600, epochs=1, shuffle=True, validation_split=0.2)


text:
#plotting fluorescence measured with substage camera

import pdb
from py_utilities import tw_filehandling as fh
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.gridspec import GridSpec
from py_utilities import fp_library2 as fpl
from imaging import file_utilities as util
plt.ion()

data_path= '/Volumes/LaCie/2pdata_elife/may20/'
file_names=['Z stack_XY1558372740_Z000_T0_C0.tif']
microns_per_pixel=0.313495

fig=plt.figure(figsize=(2,4))
gs=GridSpec(2,1,figure=fig)
ax=[]

#ax_rthist=fig.add_subplot(gs[5:7,0:4])
#ax2_rthist=fig.add_subplot(gs[1:3,12:])
#ax2_bthist=fig.add_subplot(gs[7:,6:12])
frames_to_plot=[49,50]
for ind,crframe in enumerate(frames_to_plot):
    try:
        ax.append(fig.add_subplot(gs[ind,0]))
    except:
        pdb.set_trace()

for crind in np.arange(len(file_names)):
    plt.set_cmap('viridis')
    dt=util.read_in_tif(data_path+file_names[crind])
    for ind,crframe in enumerate(frames_to_plot):
        #mean_xy=np.mean(dt['tifstack'],axis=0)
    #mean_z=np.mean(dt['tifstack'],axis=1)
        imshowobj=ax[ind].imshow(dt['tifstack'][crframe,:,:])
        #imshowobj.set_clim(20,28)
        fpl.adjust_spines(ax[ind],[])
        ax[ind].set_xlim(200,1150)
        ax[ind].set_ylim(50,200)
        #microns_mean_z=np.shape(mean_z)[1]*microns_per_pixel
        #ax.plot([970,970+20/microns_per_pixel],[775,775],'w')
    #aspect_ratio=microns_mean_z/100.
    #imshowobj2=ax2.imshow(mean_z)
    #imshowobj2.set_clim(14.6,16.5)
    #fpl.adjust_spines(ax2,[])
    #ax2.set_aspect(aspect_ratio)
    #ax2.set_xlim(950,1150)
    #ax2.set_ylim(0,100)
    #mean_meanz=np.mean(mean_z,axis=0)
    #ax2_bthist.plot(mean_meanz)

    #ax2.plot([970,970+20/microns_per_pixel],[20,20],'w')
    #fpl.adjust_spines(ax2_bthist,[])
    #ax2_bthist.set_xlim(950,1150)

    #rtmean_meanz=np.mean(mean_z,axis=1)

    #ax2_rthist.plot(rtmean_meanz,np.arange(len(rtmean_meanz)))
    #fpl.adjust_spines(ax2_rthist,[])
    #ax2_rthist.set_ylim(0,100)

    #rtmean_meanxy=np.mean(mean_xy,axis=1)
    #ax_rthist.plot(rtmean_meanxy,np.arange(len(rtmean_meanxy)))
    #fpl.adjust_spines(ax_rthist,[])
    #ax_rthist.set_ylim(725,850)

max_stars_repo_name: timothylwarren/imaging-routines

max_issues_repo_name: timothylwarren/imaging-routines

max_forks_repo_name: timothylwarren/imaging-routines


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import _init_paths
import os
import pprint
import argparse
import numpy as np
import torch
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader
import torch.optim as optim
from tqdm import tqdm
import lib.datasets as datasets
from lib.core.config import update_config, config

# import datasets
# import models
# from core.config import config, update_config
# from core.engine import Engine
# from core.utils import AverageMeter
# from core import eval
# from core.utils import create_logger
# import models.loss as loss
import math

torch.manual_seed(0)
torch.cuda.manual_seed(0)
torch.autograd.set_detect_anomaly(True)
def parse_args():
    parser = argparse.ArgumentParser(description='Train localization network')

    # general
    #parser.add_argument('--cfg', help='experiment configure file name', required=True, type=str)
    parser.add_argument('--cfg', default="experiments/charades/2D-TAN-16x16-K5L8-pool.yaml")
    args, rest = parser.parse_known_args()

    # update config
    update_config(args.cfg)

    # training
    parser.add_argument('--gpus', help='gpus', type=str)
    parser.add_argument('--workers', help='num of dataloader workers', type=int)
    parser.add_argument('--dataDir', help='data path', type=str)
    parser.add_argument('--modelDir', help='model path', type=str)
    parser.add_argument('--logDir', help='log path', type=str)
    parser.add_argument('--verbose', default=False, action="store_true", help='print progress bar')
    parser.add_argument('--tag', help='tags shown in log', type=str)
    args = parser.parse_args()

    return args

def reset_config(config, args):
    if args.gpus:
        config.GPUS = args.gpus
    if args.workers:
        config.WORKERS = args.workers
    if args.dataDir:
        config.DATA_DIR = args.dataDir
    if args.modelDir:
        config.MODEL_DIR = args.modelDir
    if args.logDir:
        config.LOG_DIR = args.logDir
    if args.verbose:
        config.VERBOSE = args.verbose
    if args.tag:
        config.TAG = args.tag

if __name__ == '__main__':

    args = parse_args()
    reset_config(config, args)

    logger, final_output_dir = create_logger(config, args.cfg, config.TAG)
    logger.info('\n'+pprint.pformat(args))
    logger.info('\n'+pprint.pformat(config))

    # cudnn related setting
    cudnn.benchmark = config.CUDNN.BENCHMARK
    torch.backends.cudnn.deterministic = config.CUDNN.DETERMINISTIC
    torch.backends.cudnn.enabled = config.CUDNN.ENABLED

    dataset_name = config.DATASET.NAME
    model_name = config.MODEL.NAME

    train_dataset = getattr(datasets, dataset_name)('train')
    if config.TEST.EVAL_TRAIN:
        eval_train_dataset = getattr(datasets, dataset_name)('train')
    if not config.DATASET.NO_VAL:
        val_dataset = getattr(datasets, dataset_name)('val')
    test_dataset = getattr(datasets, dataset_name)('test')

    model = getattr(models, model_name)()
    if config.MODEL.CHECKPOINT and config.TRAIN.CONTINUE:
        model_checkpoint = torch.load(config.MODEL.CHECKPOINT)
        model.load_state_dict(model_checkpoint)
    if torch.cuda.device_count() > 1:
        print("Using", torch.cuda.device_count(), "GPUs")
        model = torch.nn.DataParallel(model)
    device = ("cuda" if torch.cuda.is_available() else "cpu" )
    model = model.to(device)

    optimizer = optim.Adam(model.parameters(),lr=config.TRAIN.LR, betas=(0.9, 0.999), weight_decay=config.TRAIN.WEIGHT_DECAY)
    # optimizer = optim.SGD(model.parameters(), lr=config.TRAIN.LR, momentum=0.9, weight_decay=config.TRAIN.WEIGHT_DECAY)
    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=config.TRAIN.FACTOR, patience=config.TRAIN.PATIENCE, verbose=config.VERBOSE)

    def iterator(split):
        if split == 'train':
            dataloader = DataLoader(train_dataset,
                                    batch_size=config.TRAIN.BATCH_SIZE,
                                    shuffle=config.TRAIN.SHUFFLE,
                                    num_workers=config.WORKERS,
                                    pin_memory=False,
                                    collate_fn=datasets.collate_fn)
        elif split == 'val':
            dataloader = DataLoader(val_dataset,
                                    batch_size=config.TEST.BATCH_SIZE,
                                    shuffle=False,
                                    num_workers=config.WORKERS,
                                    pin_memory=False,
                                    collate_fn=datasets.collate_fn)
        elif split == 'test':
            dataloader = DataLoader(test_dataset,
                                    batch_size=config.TEST.BATCH_SIZE,
                                    shuffle=False,
                                    num_workers=config.WORKERS,
                                    pin_memory=False,
                                    collate_fn=datasets.collate_fn)
        elif split == 'train_no_shuffle':
            dataloader = DataLoader(eval_train_dataset,
                                    batch_size=config.TEST.BATCH_SIZE,
                                    shuffle=False,
                                    num_workers=config.WORKERS,
                                    pin_memory=False,
                                    collate_fn=datasets.collate_fn)
        else:
            raise NotImplementedError

        return dataloader

    def network(sample):
        anno_idxs = sample['batch_anno_idxs']
        textual_input = sample['batch_word_vectors'].cuda()
        textual_mask = sample['batch_txt_mask'].cuda()
        visual_input = sample['batch_vis_input'].cuda()
        map_gt = sample['batch_map_gt'].cuda()
        duration = sample['batch_duration']

        prediction, map_mask = model(textual_input, textual_mask, visual_input)
        loss_value, joint_prob = getattr(loss, config.LOSS.NAME)(prediction, map_mask, map_gt, config.LOSS.PARAMS)

        sorted_times = None if model.training else get_proposal_results(joint_prob, duration)

        return loss_value, sorted_times

    def get_proposal_results(scores, durations):
        # assume all valid scores are larger than one
        out_sorted_times = []
        for score, duration in zip(scores, durations):
            T = score.shape[-1]
            sorted_indexs = np.dstack(np.unravel_index(np.argsort(score.cpu().detach().numpy().ravel())[::-1], (T, T))).tolist()
            sorted_indexs = np.array([item for item in sorted_indexs[0] if item[0] <= item[1]]).astype(float)

            sorted_indexs[:,1] = sorted_indexs[:,1] + 1
            sorted_indexs = torch.from_numpy(sorted_indexs).cuda()
            target_size = config.DATASET.NUM_SAMPLE_CLIPS // config.DATASET.TARGET_STRIDE
            out_sorted_times.append((sorted_indexs.float() / target_size * duration).tolist())

        return out_sorted_times

    def on_start(state):
        state['loss_meter'] = AverageMeter()
        state['test_interval'] = int(len(train_dataset)/config.TRAIN.BATCH_SIZE*config.TEST.INTERVAL)
        state['t'] = 1
        model.train()
        if config.VERBOSE:
            state['progress_bar'] = tqdm(total=state['test_interval'])

    def on_forward(state):
        torch.nn.utils.clip_grad_norm_(model.parameters(), 10)
        state['loss_meter'].update(state['loss'].item(), 1)

    def on_update(state):# Save All
        if config.VERBOSE:
            state['progress_bar'].update(1)

        if state['t'] % state['test_interval'] == 0:
            model.eval()
            if config.VERBOSE:
                state['progress_bar'].close()

            loss_message = '\niter: {} train loss {:.4f}'.format(state['t'], state['loss_meter'].avg)
            table_message = ''
            if config.TEST.EVAL_TRAIN:
                train_state = engine.test(network, iterator('train_no_shuffle'), 'train')
                train_table = eval.display_results(train_state['Rank@N,mIoU@M'], train_state['miou'],
                                                   'performance on training set')
                table_message += '\n'+ train_table
            if not config.DATASET.NO_VAL:
                val_state = engine.test(network, iterator('val'), 'val')
                state['scheduler'].step(-val_state['loss_meter'].avg)
                loss_message += ' val loss {:.4f}'.format(val_state['loss_meter'].avg)
                val_state['loss_meter'].reset()
                val_table = eval.display_results(val_state['Rank@N,mIoU@M'], val_state['miou'],
                                                 'performance on validation set')
                table_message += '\n'+ val_table

            test_state = engine.test(network, iterator('test'), 'test')
            loss_message += ' test loss {:.4f}'.format(test_state['loss_meter'].avg)
            test_state['loss_meter'].reset()
            test_table = eval.display_results(test_state['Rank@N,mIoU@M'], test_state['miou'],
                                              'performance on testing set')
            table_message += '\n' + test_table

            message = loss_message+table_message+'\n'
            logger.info(message)

            saved_model_filename = os.path.join(config.MODEL_DIR,'{}/{}/iter{:06d}-{:.4f}-{:.4f}.pkl'.format(
                dataset_name, model_name+'_'+config.DATASET.VIS_INPUT_TYPE,
                state['t'], test_state['Rank@N,mIoU@M'][0,0], test_state['Rank@N,mIoU@M'][0,1]))

            rootfolder1 = os.path.dirname(saved_model_filename)
            rootfolder2 = os.path.dirname(rootfolder1)
            rootfolder3 = os.path.dirname(rootfolder2)
            if not os.path.exists(rootfolder3):
                print('Make directory %s ...' % rootfolder3)
                os.mkdir(rootfolder3)
            if not os.path.exists(rootfolder2):
                print('Make directory %s ...' % rootfolder2)
                os.mkdir(rootfolder2)
            if not os.path.exists(rootfolder1):
                print('Make directory %s ...' % rootfolder1)
                os.mkdir(rootfolder1)

            if torch.cuda.device_count() > 1:
                torch.save(model.module.state_dict(), saved_model_filename)
            else:
                torch.save(model.state_dict(), saved_model_filename)

            if config.VERBOSE:
                state['progress_bar'] = tqdm(total=state['test_interval'])
            model.train()
            state['loss_meter'].reset()

    def on_end(state):
        if config.VERBOSE:
            state['progress_bar'].close()

    def on_test_start(state):
        state['loss_meter'] = AverageMeter()
        state['sorted_segments_list'] = []
        if config.VERBOSE:
            if state['split'] == 'train':
                state['progress_bar'] = tqdm(total=math.ceil(len(train_dataset)/config.TEST.BATCH_SIZE))
            elif state['split'] == 'val':
                state['progress_bar'] = tqdm(total=math.ceil(len(val_dataset)/config.TEST.BATCH_SIZE))
            elif state['split'] == 'test':
                state['progress_bar'] = tqdm(total=math.ceil(len(test_dataset)/config.TEST.BATCH_SIZE))
            else:
                raise NotImplementedError

    def on_test_forward(state):
        if config.VERBOSE:
            state['progress_bar'].update(1)
        state['loss_meter'].update(state['loss'].item(), 1)

        min_idx = min(state['sample']['batch_anno_idxs'])
        batch_indexs = [idx - min_idx for idx in state['sample']['batch_anno_idxs']]
        sorted_segments = [state['output'][i] for i in batch_indexs]
        state['sorted_segments_list'].extend(sorted_segments)

    def on_test_end(state):
        annotations = state['iterator'].dataset.annotations
        state['Rank@N,mIoU@M'], state['miou'] = eval.eval_predictions(state['sorted_segments_list'], annotations, verbose=False)
        if config.VERBOSE:
            state['progress_bar'].close()

    engine = Engine()
    engine.hooks['on_start'] = on_start
    engine.hooks['on_forward'] = on_forward
    engine.hooks['on_update'] = on_update
    engine.hooks['on_end'] = on_end
    engine.hooks['on_test_start'] = on_test_start
    engine.hooks['on_test_forward'] = on_test_forward
    engine.hooks['on_test_end'] = on_test_end
    engine.train(network,
                 iterator('train'),
                 maxepoch=config.TRAIN.MAX_EPOCH,
                 optimizer=optimizer,
                 scheduler=scheduler)


text:
#Import libraries
import numpy
import talib
import config
from numpy import genfromtxt

#--------------------------------
#Perform calculations for bitcoin
#--------------------------------
print("(Time period: 10 days)\n")

my_data = genfromtxt('PARA_BTC_15M.csv', delimiter=',')
close_btc = my_data[:,4]
i=len(close_btc)-1
#Calculate macd and macd signal values
allIndexes=talib.MACD(close_btc, fastperiod=12, slowperiod=26, signalperiod=9)
macd=allIndexes[0]
macdsignal=allIndexes[1]

#print table titles
print('{:-^{width}}'.format('', width=34))
print("|"+'{: ^{width}}'.format('\033[38;5;208m- BTC -\033[0m', width=47)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")

#Calculate alpha profit (if we just invested and left the money)
balance=100
alphaInvestment=round((balance/close_btc[0]*close_btc[i])-balance,2)

#print alpha profit
print("|"+'{: ^{width}}'.format('Alpha profit: '+str(alphaInvestment)+" $", width=32)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")

#Buy bitcoin at buying point and sell at selling point (Simple MACD)
balance=100
btcAmount=0
for x in range(1, i):
    if(macd[x] < macdsignal[x]) and (macd[x-1] > macdsignal[x-1]):
        btcAmount = balance/close_btc[x]
        balance = 0

    elif(macd[x] > macdsignal[x]) and (macd[x-1] < macdsignal[x-1]) and btcAmount != 0:
        balance = btcAmount * close_btc[x]
        btcAmount = 0

#Convert BTC to usdt
if(btcAmount != 0):
    balance = btcAmount * close_btc[i]
simpleInvestment=round(balance-100,2)

#print simple mac profit
print("|"+'{: ^{width}}'.format('Simple MACD profit: '+str(simpleInvestment)+" $", width=32)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")

#Buy bitcoin at buying point and sell at selling point
# with take profits and stop limits (Complex MACD)
balance=100
btcAmount=0
btcPrice=0
takeProfit=5
bestParameter=0
bestBalance=0

for p in range(-6,0):
    for x in range(1, i):
        if(macd[x] < macdsignal[x]) and (macd[x-1] > macdsignal[x-1]):
            btcAmount = balance/close_btc[x]
            btcPrice = close_btc[x]
            balance = 0

        elif(macd[x] > macdsignal[x]) and (macd[x-1] < macdsignal[x-1]) and btcAmount != 0:
            balance = btcAmount * close_btc[x]
            btcAmount = 0

        if(btcAmount > 0):
            percentageChange = 100*(close_btc[x] - btcPrice)/btcPrice
            if (percentageChange < p):
                balance = btcAmount * close_btc[x]
                btcAmount = 0

    if(btcAmount != 0):
        balance = btcAmount * close_btc[i]
        simpleInvestment=round(balance-100,2)
    if(balance > bestBalance):
        bestBalance = balance
        bestParameter = p

complexInvestment=round(bestBalance-100,2)
print("|"+'{: ^{width}}'.format('Complex MACD profit: '+str(complexInvestment)+" $", width=32)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")
print("|"+'{: ^{width}}'.format('Stop Limit: '+str(bestParameter)+" %", width=32)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")

#--------------------------------
#Perform calculations for etherium
#--------------------------------

my_data = genfromtxt('PARA_ETH_15M.csv', delimiter=',')
close_eth = my_data[:,4]

#Calculate macd and macd signal values
allIndexes=talib.MACD(close_eth, fastperiod=12, slowperiod=26, signalperiod=9)
macd=allIndexes[0]
macdsignal=allIndexes[1]

#print table titles
print("|"+'{: ^{width}}'.format('\033[38;5;75m- ETH -\033[0m', width=46)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")
#Calculate alpha profit (if we just invested and left the money)
balance=100
alphaInvestment=round((balance/close_eth[0]*close_eth[i])-balance,2)
print("|"+'{: ^{width}}'.format('Alpha profit: '+str(alphaInvestment)+" $", width=32)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")

balance=100
ethAmount=0

for x in range(1, i):
    if(macd[x] > macdsignal[x]) and (macd[x-1] < macdsignal[x-1]):
        ethAmount = balance/close_eth[x]
        balance = 0

    elif(macd[x] < macdsignal[x]) and (macd[x-1] > macdsignal[x-1]) and ethAmount != 0:
        balance = ethAmount * close_eth[x]
        ethAmount = 0

#Convert ETH to usdt
if(ethAmount != 0):
    balance = ethAmount * close_eth[i]
simpleInvestment=round(balance-100,2)
#print simple mac profit
print("|"+'{: ^{width}}'.format('Simple MACD profit: '+str(simpleInvestment)+" $", width=32)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")

#Buy ethereum at buying point and sell at selling point
# with take profits and stop limits (Complex MACD)
balance=100
ethAmount=0
ethPrice=0
bestParameter=0
bestBalance=0

for p in range(-6,0):
    for x in range(1, i):
        if(macd[x] < macdsignal[x]) and (macd[x-1] > macdsignal[x-1]):
            ethAmount = balance/close_eth[x]
            ethPrice = close_eth[x]
            balance = 0

        elif(macd[x] > macdsignal[x]) and (macd[x-1] < macdsignal[x-1]) and ethAmount != 0:
            balance = ethAmount * close_eth[x]
            ethAmount = 0

        if(ethAmount > 0):
            percentageChange = 100*(close_eth[x] - ethPrice)/ethPrice
            if (percentageChange < p):
                balance = ethAmount * close_eth[x]
                ethAmount = 0

    if(ethAmount != 0):
        balance = ethAmount * close_eth[i]
    simpleInvestment=round(balance-100,2)
    if(balance > bestBalance):
        bestBalance = balance
        bestParameter = p

complexInvestment=round(bestBalance-100,2)
print("|"+'{: ^{width}}'.format('Complex MACD profit: '+str(complexInvestment)+" $", width=32)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")
print("|"+'{: ^{width}}'.format('Stop Limit: '+str(bestParameter)+" %", width=32)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")
#--------------------------------
#Perform calculations for BNB
#--------------------------------

my_data = genfromtxt('PARA_BNB_15M.csv', delimiter=',')
close_bnb = my_data[:,4]

#Calculate macd and macd signal values
allIndexes=talib.MACD(close_bnb, fastperiod=12, slowperiod=26, signalperiod=9)
macd=allIndexes[0]
macdsignal=allIndexes[1]

#print table titles
print("|"+'{: ^{width}}'.format('\033[38;5;226m- BNB -\033[0m', width=47)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")

#Calculate alpha profit (if we just invested and left the money)
balance=100
alphaInvestment=round((balance/close_bnb[0]*close_bnb[i])-balance,2)
print("|"+'{: ^{width}}'.format('Alpha profit: '+str(alphaInvestment)+" $", width=32)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")

balance=100
bnbAmount=0
for x in range(1, i):
    if(macd[x] > macdsignal[x]) and (macd[x-1] < macdsignal[x-1]):
        bnbAmount = balance/close_bnb[x]
        balance = 0

    elif(macd[x] < macdsignal[x]) and (macd[x-1] > macdsignal[x-1]) and bnbAmount != 0:
        balance = bnbAmount * close_bnb[x]
        bnbAmount = 0

#Convert BNB to usdt
if(bnbAmount != 0):
    balance = bnbAmount * close_bnb[i]
simpleInvestment=round(balance-100,2)

#print simple mac profit
print("|"+'{: ^{width}}'.format('Simple MACD profit: '+str(simpleInvestment)+" $", width=32)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")

#Buy BNB at buying point and sell at selling point
# with take profits and stop limits (Complex MACD)
balance=100
bnbAmount=0
bnbPrice=0
bestParameter=0
bestBalance=0

for p in range(-6,0):
    for x in range(1, i):
        if(macd[x] < macdsignal[x]) and (macd[x-1] > macdsignal[x-1]):
            bnbAmount = balance/close_bnb[x]
            bnbPrice = close_bnb[x]
            balance = 0

        elif(macd[x] > macdsignal[x]) and (macd[x-1] < macdsignal[x-1]) and bnbAmount != 0:
            balance = bnbAmount * close_bnb[x]
            bnbAmount = 0

        if(bnbAmount > 0):
            percentageChange = 100*(close_bnb[x] - bnbPrice)/bnbPrice
            if (percentageChange < p):
                balance = bnbAmount * close_bnb[x]
                bnbAmount = 0

    if(bnbAmount != 0):
        balance = bnbAmount * close_bnb[i]
    simpleInvestment=round(balance-100,2)
    if(balance > bestBalance):
        bestBalance = balance
        bestParameter = p

complexInvestment=round(bestBalance-100,2)
print("|"+'{: ^{width}}'.format('Complex MACD profit: '+str(complexInvestment)+" $", width=32)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")
print("|"+'{: ^{width}}'.format('Stop Limit: '+str(bestParameter)+" %", width=32)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")

#--------------------------------
#Perform calculations for LTC
#--------------------------------

my_data = genfromtxt('PARA_LTC_15M.csv', delimiter=',')
close_ltc = my_data[:,4]

#Calculate macd and macd signal values
allIndexes=talib.MACD(close_ltc, fastperiod=12, slowperiod=26, signalperiod=9)
macd=allIndexes[0]
macdsignal=allIndexes[1]

#print table titles
print("|"+'{: ^{width}}'.format('\033[38;5;122m- LTC -\033[0m', width=47)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")

#Calculate alpha profit (if we just invested and left the money)
balance=100
alphaInvestment=round((balance/close_ltc[0]*close_ltc[i])-balance,2)
print("|"+'{: ^{width}}'.format('Alpha profit: '+str(alphaInvestment)+" $", width=32)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")

balance=100
ltcAmount=0
for x in range(1, i):
    if(macd[x] > macdsignal[x]) and (macd[x-1] < macdsignal[x-1]):
        ltcAmount = balance/close_ltc[x]
        balance = 0

    elif(macd[x] < macdsignal[x]) and (macd[x-1] > macdsignal[x-1]) and ltcAmount != 0:
        balance = ltcAmount * close_ltc[x]
        ltcAmount = 0

#Convert LTC to usdt
if(ltcAmount != 0):
    balance = ltcAmount * close_ltc[i]
simpleInvestment=round(balance-100,2)

#print simple mac profit
print("|"+'{: ^{width}}'.format('Simple MACD profit: '+str(simpleInvestment)+" $", width=32)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")

#Buy litecoin at buying point and sell at selling point
# with take profits and stop limits (Complex MACD)
balance=100
ltcAmount=0
ltcPrice=0
bestParameter=0
bestBalance=0

for p in range(-6,0):
    for x in range(1, i):
        if(macd[x] < macdsignal[x]) and (macd[x-1] > macdsignal[x-1]):
            ltcAmount = balance/close_ltc[x]
            ltcPrice = close_ltc[x]
            balance = 0

        elif(macd[x] > macdsignal[x]) and (macd[x-1] < macdsignal[x-1]) and ltcAmount != 0:
            balance = ltcAmount * close_ltc[x]
            ltcAmount = 0

        if(ltcAmount > 0):
            percentageChange = 100*(close_ltc[x] - ltcPrice)/ltcPrice
            if (percentageChange < p):
                balance = ltcAmount * close_ltc[x]
                ltcAmount = 0

    if(ltcAmount != 0):
        balance = ltcAmount * close_ltc[i]
    simpleInvestment=round(balance-100,2)
    if(balance > bestBalance):
        bestBalance = balance
        bestParameter = p

complexInvestment=round(bestBalance-100,2)
print("|"+'{: ^{width}}'.format('Complex MACD profit: '+str(complexInvestment)+" $", width=32)+"|")
print("+"+'{:-^{width}}'.format('', width=32)+"+")
print("|"+'{: ^{width}}'.format('Stop Limit: '+str(bestParameter)+" %", width=32)+"|")
print('{:-^{width}}'.format('', width=34))
print("")

reason: import numpy,from numpy


text:
import torch
import os
import sys
import shutil
import torch.backends.cudnn as cudnn
import argparse
import skimage.metrics
import numpy as np
from torch.utils.data import DataLoader
from utils import write_html, write_loss, get_config, write2images, get_all_data_loaders
from torch.utils.tensorboard import SummaryWriter
from models.LUM_model import DecomNet
from trainer import UNIT_Trainer

# parse options
parser = argparse.ArgumentParser(description='DenoiseNet args setting')
parser.add_argument('--denoise_config', type=str, default='configs/unit_NDM.yaml', help='Path to the config file.')
parser.add_argument('--light_config', type=str, default='configs/unit_LUM.yaml', help='Path to the config file.')
parser.add_argument('--output_path', type=str, default='./denoise', help="outputs path")
parser.add_argument("--resume", action="store_true")
parser.add_argument('--trainer', type=str, default='UNIT', help="UNIT")
parser.add_argument('--light_checkpoint', type=str, default='./checkpoints/LUM_LOL.pth',
                    help="checkpoint of light")
opts = parser.parse_args()
os.environ['CUDA_VISIBLE_DEVICES'] = '0'

def main():
    cudnn.benchmark = True
    # load train setting
    denoise_config = get_config(opts.denoise_config)
    max_iter = denoise_config['max_iter']
    display_size = denoise_config['display_size']
    denoise_config['vgg_model_path'] = opts.output_path

    # pre-trained model set
    light_config = get_config(opts.light_config)
    light = DecomNet(light_config)
    state_dict = torch.load(opts.light_checkpoint, map_location='cpu')
    light.load_state_dict(state_dict)
    light.cuda()
    light.eval()

    # model set and data loader
    trainer = UNIT_Trainer(denoise_config)
    if torch.cuda.is_available():
        trainer.cuda(denoise_config['gpuID'])
        torch.nn.DataParallel(trainer)
    train_loader_x, train_loader_y, test_loader_x, test_loader_y = get_all_data_loaders(denoise_config)

    # set logger and output folder
    writer = SummaryWriter(os.path.join(opts.output_path + "/logs"))
    output_directory = os.path.join(opts.output_path + "/outputs")

    # set checkpoint folder
    checkpoint_directory = os.path.join(output_directory, 'checkpoints_denoise')
    if not os.path.exists(checkpoint_directory):
        print("Creating directory: {}".format(checkpoint_directory))
        os.makedirs(checkpoint_directory)

    # set image folder
    image_directory = os.path.join(output_directory, 'images')
    if not os.path.exists(image_directory):
        print("Creating directory: {}".format(image_directory))
        os.makedirs(image_directory)

    # copy config file to output folder
    shutil.copy(opts.denoise_config, os.path.join(output_directory, 'config_yaml'))

    # start training
    psnr = 0
    ssim = 0
    count = 0
    print('start training')
    iterations = trainer.resume(checkpoint_directory, hyperparameters=denoise_config) if opts.resume else 0
    while True:
        for it, (images_x, images_y, val_x, val_y) in enumerate(zip(train_loader_x, train_loader_y, test_loader_x, test_loader_y)):
            dataX, dataY = images_x.cuda().detach(), images_y.cuda().detach()
            valX, valY = val_x.cuda().detach(), val_y.cuda().detach()
            dataX, _ = light(dataX)
            # main training code
            for _ in range(3):
                trainer.content_update(dataX, dataY, denoise_config)
            trainer.dis_update(dataX, dataY, denoise_config)
            trainer.gen_update(dataX, dataY, denoise_config)
            trainer.update_learning_rate()

            # dump training stats in log file
            if (iterations + 1) % denoise_config['log_iter'] == 0:
                write_loss(iterations, trainer, writer)
            if (iterations + 1) % denoise_config['image_save_iter'] == 0:
                trainer.eval()
                print("[*] Evaluating for phase train / epoch %d..." % (iterations + 1))
                with torch.no_grad():
                    train_image_outputs = trainer.sample(dataX, dataY)
                    val_image_outputs = trainer.sample(valX, valY)
                write2images(train_image_outputs, display_size, image_directory, 'train_%08d' % (iterations + 1))
                write2images(val_image_outputs, display_size, image_directory, 'test_%08d' % (iterations + 1))
                write_html(output_directory + "/index.html", iterations + 1, denoise_config['image_save_iter'],
                           'images')
                print("===> Iteration[{}]: psnr: {}, ssim:{}".format(iterations + 1, psnr / count, ssim / count))
            if (iterations + 1) % denoise_config['snapshot_save_iter'] == 0:
                trainer.save(checkpoint_directory, iterations)
            iterations += 1
            if iterations >= max_iter:
                writer.close()
                sys.exit('Finish training')

if __name__ == '__main__':
    main()