import glob
import os
import warnings

import numpy as np
from tqdm.asyncio import tqdm

np.seterr(divide='ignore')
import pandas as pd
import pickle as pk

mac_path = "/Users/au589901/PycharmProjects/CN_Signatures/"
linux_path = "/home/mateo/pytorch_docker/CNVSig/"

hg19 = pd.read_csv("{}/data/input/hg19.chrom.sizes.csv".format(linux_path), sep=",")
chrom_centromere = pd.read_csv("{}/data/input/chrom_centromere_info.csv".format(linux_path), sep=",")
rep_time = pd.read_csv("{}/data/input/Encode_replication_timing.tsv".format(linux_path), sep="\t")

def readAscatSavePickle(input, output):
    df = pd.read_csv(input, delimiter=",")
    print("Before filtering: ", np.shape(df))
    df = df[~df.Chr.isin([23, 24])]
    df = df.query("ACF < 0.99 and ACF > 0.1")
    # Do we want to remove 'normal' segments?
    # df = df.query("cn != 2 and Ploidy != 2.0")

    # deal with outliers
    df.loc[df["nAraw"] < 0, "nAraw"] = 0.0
    df.loc[df["nBraw"] < 0, "nBraw"] = 0.0
    df.loc[df["cn"] < 0, "cn"] = 0.0
    df.loc[df["cn"] > 10, "cn"] = 10.0
    print("After filtering: ", np.shape(df))
    df['middleOfSegment'] = (df['End'] + df['Start'])/2
    df.to_csv("data/output/ascat.csv")
    with open(output, 'wb') as f:
        pk.dump(df, f, pk.HIGHEST_PROTOCOL)
        f.close()

def readPickle(input):
    return pd.read_pickle(input)

def savePickle(df, output):
    with open(output, 'wb') as f:
        pk.dump(df, f, pk.HIGHEST_PROTOCOL)
        f.close()

def label_loh(row):
    if row['nA'] == 0 and row['nB'] == 1 and row['cn'] == 1:
        return 1
    if row['nA'] == 1 and row['nB'] == 0 and row['cn'] == 1:
        return 1
    else:
        return 0

def makeFilesForEachSampleAndChr(df, IDs, out_folder):
    if not os.path.exists(out_folder):
        os.mkdir(out_folder)
    for id in tqdm(IDs):
        if not os.path.exists("{}/{}".format(out_folder, id)):
            os.mkdir("{}/{}".format(out_folder, id))
        for c in range(1, 23, 1):  # up to 22
            tmp_df = df.loc[df["ID"] == id]
            print("After ID ",tmp_df)
            tmp_df = tmp_df.loc[tmp_df["Chr"] == c]
            #print("After chr: ",tmp_df)
            savePickle(tmp_df, "{}/{}/{}.pickle".format(out_folder, id, c))

def distanceToCentromere(row):
    chrom_centromere_start = chrom_centromere[chrom_centromere['chrom'] == row['Chr']]['chromStart'].values
    chrom_centromere_end = chrom_centromere[chrom_centromere['chrom'] == row['Chr']]['chromEnd'].values
    middle_centromere = (chrom_centromere_end + chrom_centromere_start) / 2
    middle_segment = (row['End'] + row['Start']) / 2
    if middle_segment < middle_centromere:  # long arm
        return np.log10(np.abs(chrom_centromere_start - row['End'] + 1))[0]
    else:  # short arm
        return np.log10(np.abs(row['Start'] - chrom_centromere_end + 1))[0]

def distanceToClosestCNV(df):
    n_row = np.shape(df)[0]
    distances = [0]
    for i in range(1, n_row):  # skip first, distance is 0
        if df.iloc[i]['cn'] == 2:  # if cn is normal:
            distances.append(0)
        else:  # find distance to closes CNV
            for j in range(i, n_row):  # iter from i until next cnv
                if df.iloc[j]['cn'] != 2:  # if cn is not normal
                    distance = np.log10(np.abs(df.iloc[j]['Start'] - df.iloc[i]['End'] + 1))
                    distances.append(distance)
                    break
    # print(distances)
    df['log10_distanceToNearestCNV'] = distances
    #print(df)
    return df

def changePoint(df):
    n_row = np.shape(df)[0]
    changepoint = np.array(df.iloc[range(1, n_row), [15]].values - df.iloc[range(0, n_row - 1), [15]].values)
    changepoint = np.insert(changepoint, 0, 0)
    return changepoint

def allelicImbalance(row):
    if row['nA'] != row['nB']:
        return 1
    else:
        return 0

def replicationTiming(row):
    with warnings.catch_warnings():
        warnings.simplefilter(action='ignore', category=UserWarning)
        start, end = row['Start'], row['End']
        chr = "chr{}".format(row['Chr'])
        rep_time_tmp = rep_time[rep_time['Chromosome'] == chr]
        rep_time_tmp = rep_time_tmp[rep_time['Position'].between(start, end)]
        if np.shape(rep_time_tmp)[0] == 0:
            return 0
        else:
            return rep_time_tmp['Scaled'].mean()

def computeFeatures(df):
    with np.errstate(all="ignore"):
        df['replication_timing'] = df.apply(lambda row: replicationTiming(row), axis=1)
        df['log10_distToCentromere'] = df.apply(lambda row: distanceToCentromere(row), axis=1)
        df = distanceToClosestCNV(df)
        df['logR'] = np.log2(((df['nAraw'] + df['nBraw'] + 0.00001) / (df['Ploidy'] + 0.0001)))
        df.loc[df["logR"] <= -8, "logR"] = -8.0
        df.loc[df["logR"] >= 8, "logR"] = 8.0
        df['changepoint'] = changePoint(df)
        df['log10_segmentSize'] = np.log10(df['End'] - df['Start'] + 1)
        df['loh'] = df.apply(lambda row: label_loh(row), axis=1)
        df['allelicImbalance'] = df.apply(lambda row: allelicImbalance(row), axis=1)
    return df

def generate_short_arm_bins(bin_row):
    bins = pd.DataFrame()
    previous = 0
    for short_bins in range(0, int(bin_row['number_of_bins_on_short_arm'].values[0])):
        bins = pd.concat([bins, pd.DataFrame({'from':previous, 'to':previous + bin_row['short_arm_bin_size'], 'arm':'short'})])
        previous = previous + bin_row['short_arm_bin_size']
    return bins

def generate_long_arm_bins(bin_row):
    bins = pd.DataFrame()
    previous = bin_row['cent_end'].values[0]
    for short_bins in range(0, int(bin_row['number_of_bins_on_long_arm'].values[0])):
        bins = pd.concat([bins, pd.DataFrame({'from': previous, 'to': previous + bin_row['long_arm_bin_size'], 'arm':'long'})])
        previous = previous + bin_row['long_arm_bin_size']
    return bins

def scaleMinMax(x,xmin,xmax,f):
    return (x-xmin)/(xmax-xmin)
pd.options.mode.chained_assignment = None
def checkIfEmpty(tmp, feature, minMax_of_feature):
    if not tmp.empty:
        m = tmp[feature].mean()
        s = tmp[feature].std()
        if tmp[feature].isin([-np.inf]).values.sum() != 0:
            tmp[feature].replace([-np.inf], 0, inplace=True)
            #print(tmp[feature].mean())
        return scaleMinMax(tmp[feature].mean(),minMax_of_feature[0],minMax_of_feature[1],feature)
    else:
        return 0

def computeMeanValueBasedOnBinAndFeature(chr_specific_features, bin_from, bin_to,arm, feature, minMax_of_feature):
    if feature in ['loh', 'allelicImbalance']:
        tmp = chr_specific_features[chr_specific_features['middleOfSegment'].between(bin_from, bin_to) & chr_specific_features[feature] == 1]
        bin_len = np.abs(bin_to - bin_from)
        # if feature == 'allelicImbalance' and len(tmp.index)/bin_len > 6.13e-05:
        #     print(feature, len(tmp.index)/bin_len)
        # if feature == 'loh' and len(tmp.index)/bin_len > 3.6e-06:
        #     print(feature, len(tmp.index)/bin_len)
        #     print(scaleMinMax(len(tmp.index)/bin_len, minMax_of_feature[0], minMax_of_feature[1], feature))
        return scaleMinMax(len(tmp.index)/bin_len, minMax_of_feature[0], minMax_of_feature[1], feature) # number of rows/bin size scaled
    else:
        tmp = chr_specific_features[chr_specific_features['middleOfSegment'].between(bin_from, bin_to)]
        return checkIfEmpty(tmp, feature, minMax_of_feature)

def makeSquareImage(chr_data, chr_specific_bins, feature, minMax_of_feature):
    bin_values = []
    for index, row in chr_specific_bins.iterrows():
        val = computeMeanValueBasedOnBinAndFeature(chr_data, row['from'], row['to'],row['arm'], feature, minMax_of_feature)
        bin_values.append(val)

    return np.array(bin_values)

def minMax(x):
    return pd.Series(index=['min','max'],data=[x.min(),x.max()])
def invertMinMax(scaled_x, min=0, max=10):
    return (scaled_x*(max - min)) + min

# df = readPickle("data/output/merged_features.pickle")
# print(df['log10_distanceToNearestCNV'].isin([-np.inf]).values.sum())
# df.replace([np.inf, -np.inf], 0.0, inplace=True)
# print(min(df['log10_distanceToNearestCNV']))

# minMax_of_feature = pd.read_csv("data/output/min_max_features.csv")
# features = ['cn', 'log10_distanceToNearestCNV', 'logR', 'changepoint', 'log10_segmentSize',
#             'loh', 'allelicImbalance', 'log10_distToCentromere', 'replication_timing']
# order_of_chromosomes = [4,7,2,5,6,13,3,8,9,18,12,1,10,11,14,22,19,17,20,16,15,21]
# all_features = []
# files_in_sample = glob.glob("data/output/compute_features/CPCT02010037T" + "/*.pickle")
# if len(files_in_sample) == 22:
#     path_to_chr_files = files_in_sample[0]
#     path_to_chr_files = "/".join(path_to_chr_files.split("/")[:-1])
#     for f in features:
#         bins_for_specific_chr = [] # make empty list
#         for specific_chr in order_of_chromosomes: # for every chr with specific order
#             chr_data = readPickle("{}/{}.pickle".format(path_to_chr_files,specific_chr)) # read file data/output/compute_fatures/ID/4.pickle
#             chr_specific_bins = pd.read_csv("{}/{}.csv".format("data/output/chromosome_bins_def/", specific_chr)) # read bin file for that chr data/output/chrom_bin_def/4.csv
#             specific_feature_bin_row = makeSquareImage(chr_data, chr_specific_bins, f,minMax_of_feature[f]) # returns 22 values representing all bins for specific features and chr
#             bins_for_specific_chr.append(specific_feature_bin_row) # I append each row (chr) here
#         assert np.shape(bins_for_specific_chr) == (22, 22) # after all chrom run this should be 22,22
#         all_features.append(bins_for_specific_chr) # append feature profile to a list
#     total = np.dstack(all_features) # stack all feature profiles as depth
#     assert np.shape(total) == (22, 22, 9)
#
# else:
#     savePickle(-1, "data/output/make_square_images/{}.pickle".format("data/output/compute_features/CPCT02010003T").split("/")[3])


import random
import numpy
from tkinter import *
from PIL import ImageTk, Image
import random

root = Tk()
root.title("K-Means")
root.geometry("740x580+800+300")

photo = ImageTk.PhotoImage(file='./TestImage.jpg')
label1 = Label(root, image=photo)
label2 = Label(root, text="클러스터 개수 : ")
text = Text(root, height=2,width=5)

label1.pack()
label2.pack()
text.pack()

class Cluster(object):

    def __init__(self):
        self.pixels = []
        self.centroid = None

    def addPoint(self, pixel):
        self.pixels.append(pixel)

    def setNewCentroid(self):

        R = [colour[0] for colour in self.pixels]
        G = [colour[1] for colour in self.pixels]
        B = [colour[2] for colour in self.pixels]

        R = int(sum(R) / len(R))
        G = int(sum(G) / len(G))
        B = int(sum(B) / len(B))

        self.centroid = (R, G, B)
        self.pixels = []

        return self.centroid

class Kmeans(object):

    def __init__(self, k=3, max_iterations=5, min_distance=5.0, size=200):
        self.k = int(text.get(1.0, END+"-1c"))
        self.max_iterations = max_iterations
        self.min_distance = min_distance
        self.size = (size, size)

    def run(self, image):
        self.image = image
        self.image.thumbnail(self.size)
        self.pixels = numpy.array(image.getdata(), dtype=numpy.uint8)

        self.clusters = [None for i in range(self.k)]
        self.oldClusters = None

        randomPixels = random.sample(list(self.pixels), self.k)

        for idx in range(self.k):
            self.clusters[idx] = Cluster()
            self.clusters[idx].centroid = randomPixels[idx]

        iterations = 0

        while self.shouldExit(iterations) is False:

            self.oldClusters = [cluster.centroid for cluster in self.clusters]

            for pixel in self.pixels:
                self.assignClusters(pixel)

            for cluster in self.clusters:
                cluster.setNewCentroid()

            iterations += 1

        return [cluster.centroid for cluster in self.clusters]

    def assignClusters(self, pixel):
        shortest = float('Inf')
        for cluster in self.clusters:
            distance = self.calcDistance(cluster.centroid, pixel)
            if distance < shortest:
                shortest = distance
                nearest = cluster

        nearest.addPoint(pixel)

    def calcDistance(self, a, b):

        result = numpy.sqrt(sum((a - b) ** 2))
        return result

    def shouldExit(self, iterations):

        if self.oldClusters is None:
            return False

        for idx in range(self.k):
            dist = self.calcDistance(
                numpy.array(self.clusters[idx].centroid),
                numpy.array(self.oldClusters[idx])
            )
            if dist < self.min_distance:
                return True

        if iterations <= self.max_iterations:
            return False

        return True

    def showClustering(self):

        localPixels = [None] * len(self.image.getdata())

        for idx, pixel in enumerate(self.pixels):
                shortest = float('Inf')
                for cluster in self.clusters:
                    distance = self.calcDistance(cluster.centroid, pixel)
                    if distance < shortest:
                        shortest = distance
                        nearest = cluster

                localPixels[idx] = nearest.centroid

        w, h = self.image.size
        localPixels = numpy.asarray(localPixels)\
            .astype('uint8')\
            .reshape((h, w, 3))

        colourMap = Image.fromarray(localPixels)
        colourMap.show()

def GetEditedImage():

    image = Image.open("./TestImage.jpg")

    k = Kmeans()

    result = k.run(image)

    k.showClustering()

run_btn = Button(root, padx = 20, text="실행" , command=GetEditedImage)
run_btn.pack()

root.mainloop()


from sympy import sieve

primes = []

uni_primes = []

for i in sieve.primerange(2,100):
    primes.append(i)

for i in range(1,len(primes)):
    if primes[i] == sum(primes[:i-1]):
        uni_primes.append(i)

print(uni_primes[-1])


#!/usr/bin/env python

import argparse
import math
import os
import pathlib
import re

from rich.console import Console
from rich.progress import Progress

def main():
    args = _parse_args()
    generate_data(
        args.total_size,
        args.files_count,
        args.branching_factor,
        args.output_dir,
    )

def generate_data(
    total_size: int,
    files_count: int,
    branching_factor: int,
    output_dir: pathlib.Path,
):
    file_size_bytes = math.ceil(total_size / files_count)
    tree_depth = math.floor(math.log(files_count, branching_factor))
    name_length = len(str(branching_factor))
    console = Console()
    console.log(
        f"Generating {files_count} files {file_size_bytes} bytes each, "
        f"which in total: {total_size} bytes."
    )

    buffer_size = min(file_size_bytes, 16 * 2**20)  # 16MB at max
    garbage = os.urandom(buffer_size)
    write_iterations = file_size_bytes // buffer_size
    tail_size = file_size_bytes % buffer_size

    output_dir.mkdir(parents=True)

    created_files = 0
    folders_counter = 0
    with Progress() as progress:
        data_gen_task = progress.add_task(
            "[green]Generating data...", total=files_count
        )
        file_gen_task = progress.add_task(
            f"[cyan]Generating file...", total=write_iterations
        )
        while created_files < files_count:
            files_count_z = str(folders_counter).zfill(name_length * tree_depth)

            split_path = []
            for level in range(tree_depth):
                split_path.append(
                    files_count_z[name_length * level : name_length * (level + 1)]
                )

            folder_path = output_dir.joinpath(*split_path)

            folder_path.mkdir(parents=True, exist_ok=True)
            folders_counter += 1

            for i in range(branching_factor):
                if created_files < files_count:
                    file_name = str(i).zfill(name_length)
                    full_file_name = folder_path / file_name
                    with full_file_name.open("wb") as file:
                        for iteration in range(write_iterations):
                            file.write(garbage)
                            progress.update(
                                file_gen_task,
                                completed=iteration,
                                description=f"[cyan]Writing file {full_file_name}...",
                            )
                        if tail_size != 0:
                            file.write(garbage[:tail_size])
                    created_files += 1
                    progress.advance(data_gen_task)
                else:
                    break
    console.log("Data generation completed.")

def _parse_args() -> argparse.Namespace:

    parser = argparse.ArgumentParser()
    parser.add_argument("files_count", type=int)
    parser.add_argument("total_size", type=_parse_size)
    parser.add_argument("--branching-factor", type=int, default=100)
    parser.add_argument(
        "--output-dir", type=_parse_dir_path, default=pathlib.Path("./data")
    )
    return parser.parse_args()

def _parse_size(size: str) -> int:
    units = {"B": 1, "KB": 2**10, "MB": 2**20, "GB": 2**30, "TB": 2**40}
    size = size.upper()
    # print("parsing size ", size)
    number, unit = re.fullmatch(r"(\d+(?:\.\d*)?)([KMGT]?B)", size.strip()).groups()
    return int(float(number) * units[unit])

def _parse_dir_path(path: str) -> pathlib.Path:
    if not os.path.exists(path):
        return pathlib.Path(path)
    else:
        raise argparse.ArgumentTypeError(f"{path} already exists, could not overwrite.")

if __name__ == "__main__":
    main()


"""
this programme has for purpose to use the linear regression
to predict the futur grades of the students
"""

import pandas as pd
import numpy as np
import sklearn
from sklearn import linear_model
from sklearn.utils import shuffle
import matplotlib.pyplot as pyplot
import pickle
from matplotlib import style

#read data from the file "student-mat.csv"
data = pd.read_csv("data/student-mat.csv", sep=";")

#just for testing printing our data head from the file
print(data.head())

#data structure am going to use
data = data[["G1", "G2", "G3", "studytime", "failures", "absences"]]

"""
for this program i ll be only using data with integer type
you can see the difference after i ve set the new data structure
"""
print(data.head())

#G3 is the final grade which the program will be predicting based on all the different attributs
predict = "G3"

"""
In this part am going to set up 2 arrays
one for the attributs and the other for the labels
"""

arr_attributs = np.array(data.drop([predict], 1))
arr_labels = np.array(data[predict])

"""
so in this part, i split all of my values into test and train arrays
in order to test 10% of our data so that the computer doesn't memorise the patterns
and we can get better accurate results
"""
arr_attributs_train, arr_attributs_test, arr_labels_train, arr_labels_test = sklearn.model_selection.train_test_split(arr_attributs, arr_labels, test_size=0.1)

#saving models and plotting data
best = 0
for _ in range(30):
    arr_attributs_train, arr_attributs_test, arr_labels_train, arr_labels_test = sklearn.model_selection.train_test_split(arr_attributs, arr_labels, test_size=0.1)

    #create training model
    linear = linear_model.LinearRegression()

    linear.fit(arr_attributs_train, arr_labels_train)
    acc = linear.score(arr_attributs_test, arr_labels_test)
    #this is our accuracy
    print(acc)
    #it's only goning to save a model if it current accuracy is better than the ones before
    if acc > best:
        best = acc
        with open("Model_Save/studentmodel.pickle", "wb") as f:
            pickle.dump(linear, f)

pickle_in = open("Model_Save/studentmodel.pickle", "rb")
linear = pickle.load(pickle_in)

print ('Coefficients : \n', linear.coef_)
print ('Intercept : \n', linear.intercept_)

#this is how this works on students (as a test)

predictions = linear.predict(arr_attributs_test)

for x in range(len(predictions)):
    print (predictions[x], arr_attributs_test[x], arr_labels_test[x])

#drawing the graph
p = "G1"

style.use("ggplot")
pyplot.scatter(data[p],data["G3"])
pyplot.xlabel(p)
pyplot.ylabel("Final Grade")

pyplot.show()


"""
runner for gsnap on methylation data.
gmap_build, gsnap and samtools must be on the PATH of the calling environment
"""

import argparse
import sys
import os
import os.path as op
import subprocess as sp

def sh(cmd, log=sys.stderr, wait=True):
    print >>sys.stderr, "[running command] %s" % cmd
    p = sp.Popen(cmd, shell=True, stderr=sp.PIPE, stdout=sp.PIPE)
    if wait:
        for line in p.stderr:
            print >>log, line,
        p.wait()
        if p.returncode != 0 or "aborted" in p.stderr.read():
            sys.exit(p.returncode)
    return p

def nopen(f, mode="rb"):
    if not isinstance(f, basestring): return f
    return open(f, mode)

def gmap_built(ref_dir, ref_base):
    for ext in ("chromosome", "contig.iit", "ref12153offsetscomp",
                "chromosome.iit", "genomecomp",
                "chrsubset", "maps", "version", "contig",
                "ref12153gammaptrs"):

        f = "%s/%s/%s.%s" % (ref_dir, ref_base, ref_base, ext)
        if not op.exists(f):
            print >>sys.stderr, "%s does not exist" % f
            return False
    return True

def cmetindexed(ref_dir, ref_base, kmer):

    for ext in ("metct12%i3offsetscomp",
                "metct12%i3gammaptrs", "metga12%i3offsetscomp",
                "metga12%i3gammaptrs"):
        ext = ext % kmer
        f = "%s/%s/%s.%s" % (ref_dir, ref_base, ref_base, ext)
        if not op.exists(f):
            print >>sys.stderr, "%s does not exist" % f
            return False
    return True

def gsnap_index(reference, kmer=15):
    ref_dir, ref_base = check_reference(reference)
    ref_base += "." + str(kmer)

    cmd = "gmap_build --no-sarray -k %(kmer)i -D %(ref_dir)s -d %(ref_base)s %(reference)s"
    cmd %= locals()
    if not gmap_built(ref_dir, ref_base):
        sh(cmd)
    else:
        print >>sys.stderr, "[skipping command] %s" % cmd

    cmd_cmet = "cmetindex -d %(ref_base)s -F %(ref_dir)s -k %(kmer)d\n"
    cmd_cmet %= locals()
    if not cmetindexed(ref_dir, ref_base, kmer):
        sh(cmd_cmet)
    else:
        print >>sys.stderr, "[skipping command] %s" % cmd

def check_reference(reference):
    ref_dir = op.dirname(reference)
    ref_base = op.splitext(op.basename(reference))[0] # locals
    assert os.access(reference, os.R_OK), ("reference not found / readable")
    assert os.access(ref_dir, os.W_OK), ("%s must be writable" % (ref_dir))
    return ref_dir, ref_base

def gsnap_meth(reference, reads, prefix, kmer=15, stranded=False,
        extra_args="", threads=1, rg=None):

    ref_dir, ref_base = check_reference(reference)
    ref_base += "." + str(kmer)
    mode = ["cmet-nonstranded", "cmet-stranded"][int(stranded)]
    reads_str = " ".join(reads)
    if any(r.endswith(".gz") for r in reads):
        extra_args = extra_args + " --gunzip"
    cmd_gsnap = "set -eo pipefail;"
    cmd_gsnap += "gsnap -B 4 --npaths 1 --quiet-if-excessive \
        --nthreads {threads} \
        -A sam -k {kmer} -D {ref_dir} -d {ref_base} --mode {mode} \
        --use-sarray 0 \
        --pairexpect 300 \
        --pairdev 250 \
        --read-group-id {rg} --read-group-name {rg} \
         {extra_args}  {reads_str}"
    cmd_gsnap += "| samtools view -bS - | samtools sort - {prefix}"
    cmd_gsnap = cmd_gsnap.format(**locals())
    sh(cmd_gsnap)

    cmd_index = "samtools index {prefix}.bam".format(**locals())
    sh(cmd_index)

def run(args):
    if args.threads is None:
        import multiprocessing
        threads = multiprocessing.cpu_count() # locals
    else:
        threads = args.threads
    reference, kmer = op.abspath(args.reference), args.kmer
    reads = map(op.abspath, args.reads)
    rg = rname(reads[0], reads[1])
    gsnap_meth(reference, reads, args.prefix, args.kmer, args.stranded,
               args.extra_args, threads, rg=rg)

def rname(fq1, fq2):
    def name(f):
        n = op.basename(op.splitext(f)[0])
        if n.endswith('.fastq'): n = n[:-6]
        if n.endswith(('.fq', '.r1', '.r2')): n = n[:-3]
        return n
    return "".join(a for a, b in zip(name(fq1), name(fq2)) if a == b) or 'bm'

def main():
    if len(sys.argv) > 1 and "index" == sys.argv[1]:
        sys.exit(gsnap_index(sys.argv[2], kmer=int(sys.argv[3])))

    p = argparse.ArgumentParser(description=__doc__,
                   formatter_class=argparse.RawDescriptionHelpFormatter)
    p.add_argument("-r", "--reference", help="reference fasta")
    p.add_argument("-k", dest="kmer", help="kmer length for gsnap. default:" \
                   + "%(default)i", type=int, default=15)
    p.add_argument("-t", "--threads", dest="threads", default=None,
            help="number of threads for gsnap to use", type=int)
    p.add_argument("--stranded", action="store_true", default=False, help=\
                   "by default, non-stranded library prep is assumed")
    p.add_argument("--prefix", help="prefix for output bam",
                  default="gsnap-meth")
    p.add_argument("--extra-args", dest="extra_args", help="extra arguments"
                " to send to gsnap", default="")

    p.add_argument('reads', nargs='+', help='reads files (if 2 files are'
                   'they are assumed to be paired end)')

    try:
        args = p.parse_args()
    except:
        p.print_help()
        raise

    if (not len(args.reads) in (1, 2)) or args.reference is None:
        sys.exit(not p.print_help())

    run(args)

if __name__ == "__main__":
    import doctest
    if doctest.testmod(optionflags=doctest.ELLIPSIS |\
                                   doctest.NORMALIZE_WHITESPACE).failed == 0:
        main()


import matplotlib
import numpy as np

from Networks import *
from scipy.io import loadmat
from sklearn.metrics import r2_score, mean_squared_error
import matplotlib.pyplot as plt

matplotlib.use("Agg")
seed_torch(1234)

if __name__ == "__main__":
    # 观测数据准备
    noise_u = Opt.noise
    N_u = 2000
    solution = np.squeeze(loadmat("data/case_field_2/u.mat")["u"])
    positions = np.squeeze(loadmat("data/case_field_2/position.mat")["p"])
    index = np.random.randint(0, 2583, N_u)  # 4960 triangle
    u_star = solution[index].reshape(-1, 1) + np.random.randn(N_u, 1) * noise_u
    x_star = positions[0, :][index]
    y_star = positions[1, :][index]
    X_star = np.vstack([x_star, y_star]).T

    # 网格配点
    nx = 50
    ny = 50
    xx = np.linspace(0, 1, nx)
    yy = np.linspace(0, 1, ny)
    xxx, yyy = np.meshgrid(xx, yy)
    X_f = np.hstack([xxx.flatten()[:, None], yyy.flatten()[:, None]])
    X_f_, index_hole = collocation_check(X_f)
    collocations = np.vstack([X_f_, X_star])

    # fig = plt.figure()
    # plt.scatter(collocations[:, 0], collocations[:, 1], s=5)
    # plt.savefig("peidian.png")

    # bc
    bc_linspace = np.linspace(0.5, 0.7, 21)
    bc_grid_x,  bc_grid_y = np.meshgrid(bc_linspace, bc_linspace)
    bc_f = np.hstack([bc_grid_x.flatten()[:, None], bc_grid_y.flatten()[:, None]])
    index_hole_x = np.where((bc_f[:, 0] == 0.7) | (bc_f[:, 0] == 0.5))[0]
    index_hole_y = np.where((bc_f[:, 1] == 0.7) | (bc_f[:, 1] == 0.5))[0]
    bc_f_x = bc_f[index_hole_x]  # x 固定，x
    bc_f_y = bc_f[index_hole_y]
    # 观察对角线
    x_diag_0 = np.linspace(0, 0.5, 101)
    x_diag_1 = np.linspace(0.7, 1.0, 101)
    xy_diag_0 = np.vstack([x_diag_0, x_diag_0]).T
    xy_diag_1 = np.vstack([x_diag_1, x_diag_1]).T

    # source 场
    source_field_total = source_func_2(X_f)
    source_field_delete = source_func_2(X_f_)

    # numpy to tensor
    device = Opt.device
    X_star_T = torch.tensor(X_star, dtype=torch.float32, requires_grad=True).to(device)
    u_star_T = torch.tensor(u_star, dtype=torch.float32).to(device)
    X_f_T = torch.tensor(X_f_, dtype=torch.float32, requires_grad=True).to(device)
    X_f_T_total = torch.tensor(X_f, dtype=torch.float32, requires_grad=True).to(device)
    collocations_T = torch.tensor(collocations, dtype=torch.float32, requires_grad=True).to(device)
    bc_f_x_T = torch.tensor(bc_f_x, dtype=torch.float32, requires_grad=True).to(device)
    bc_f_y_T = torch.tensor(bc_f_y, dtype=torch.float32, requires_grad=True).to(device)
    xy_diag_0_T = torch.tensor(xy_diag_0, dtype=torch.float32, requires_grad=True).to(device)
    xy_diag_1_T = torch.tensor(xy_diag_1, dtype=torch.float32, requires_grad=True).to(device)

    # 网络相关
    """
    构造一个神经网络Net_u拟合场数据，一个神经网络Net_s拟合热源
    """
    start_time = time.time()
    epochs = Opt.epochs
    layers_s = Opt.layers_s
    layers_u = Opt.layers_u

    Net_s = BNNNet(layers_s, st=True).to(device)
    Net_u = BPINN(layers_u, X_star_T, u_star_T, collocations_T, Net_s, in_test=X_f_T_total, out_test=source_field_total,
                  x_bc=bc_f_x_T, y_bc=bc_f_y_T)
    Net_u.train(epochs)
    end_time = time.time()
    print("耗时：", end_time - start_time)

    #  画图
    plt.style.use("default")
    plt.rcParams['font.family'] = 'Times New Roman'
    save = True
    experiment_tag = "_B-PINN_"
    # 预测热源图
    pred_list_plot = Net_u.predict(X_f_T_total)
    mu_plot = np.mean(pred_list_plot, axis=0)
    mu = np.delete(mu_plot, index_hole, axis=0)
    std_plot = np.std(pred_list_plot, axis=0)
    std = np.delete(std_plot, index_hole, axis=0)
    l2_error = np.linalg.norm(source_field_delete.reshape(-1, 1) - mu, 2) / np.linalg.norm(
        source_field_delete, 2)
    r2 = r2_score(source_field_delete, mu)
    print("Ensemble r2:", r2, "---l2_error:", l2_error)

    mask = np.zeros_like(mu_plot, dtype=bool)
    mask[index_hole] = True
    mu_masked = np.ma.array(mu_plot.reshape(nx, ny), mask=mask.reshape(nx, ny))
    std_masked = np.ma.array(std_plot.reshape(nx, ny), mask=mask.reshape(nx, ny))

    fig, ax = plt.subplots(figsize=[8, 3], ncols=2)
    ctf0 = ax[0].contourf(xxx, yyy, mu_masked, cmap="hot", corner_mask=False, levels=100)
    #ax.contour(xxx, yyy, source_field_masked, colors="k", linestyles="dash", corner_mask=False)
    ax[0].set_xlim([0, 1])
    ax[0].set_ylim([0, 1])
    cb0 = plt.colorbar(ctf0, ax=ax[0])
    ctf1 = ax[1].contourf(xxx, yyy, std_masked, cmap="hot", corner_mask=False, levels=100)
    ax[1].set_xlim([0, 1])
    ax[1].set_ylim([0, 1])
    cb1 = plt.colorbar(ctf1)
    if save:
        plt.savefig("training_results/pred_source" + experiment_tag + ".png", bbox_inches="tight", dpi=300)

    # loss
    loss_list = Net_u.loss_list
    plt.style.use('seaborn')
    plt.rcParams['font.family'] = 'Times New Roman'
    fig2 = plt.figure(figsize=[4, 3])
    plt.semilogy()
    plt.plot(loss_list)
    plt.xlabel("Epochs")
    plt.ylabel("Loss")
    plt.tight_layout()
    if save:
        plt.savefig("training_results/loss" + experiment_tag + ".png", dpi=300, bbox_inches="tight")

    # 对角线
    diag_pred_list_0 = Net_u.predict(xy_diag_0_T)
    diag_pred_list_1 = Net_u.predict(xy_diag_1_T)
    mu_diag_0 = np.mean(diag_pred_list_0, axis=0)
    mu_diag_1 = np.mean(diag_pred_list_1, axis=0)
    std_diag_0 = np.std(diag_pred_list_0, axis=0)
    std_diag_1 = np.std(diag_pred_list_1, axis=0)
    lower_0 = mu_diag_0 - 2 * std_diag_0
    lower_1 = mu_diag_1 - 2 * std_diag_1
    upper_0 = mu_diag_0 + 2 * std_diag_0
    upper_1 = mu_diag_1 + 2 * std_diag_1
    true_diag_0 = source_func_2(xy_diag_0)
    true_diag_1 = source_func_2(xy_diag_1)

    fig3 = plt.figure(figsize=[4, 3])
    plt.fill_between(x_diag_0, lower_0.flatten(), upper_0.flatten(), alpha=0.5, rasterized=True,
                     label="Epistemic uncertainty", color="green")
    plt.fill_between(x_diag_1, lower_1.flatten(), upper_1.flatten(), alpha=0.5, rasterized=True, color="green")
    plt.plot(x_diag_0, true_diag_0, color="xkcd:orange", linestyle="-", linewidth=2, label="Truth")
    plt.plot(x_diag_1, true_diag_1, color="xkcd:orange", linestyle="-", linewidth=2)
    plt.plot(x_diag_0, mu_diag_0, color="xkcd:dark blue", linestyle="--", linewidth=2, label='Identification')
    plt.plot(x_diag_1, mu_diag_1, color="xkcd:dark blue", linestyle="--", linewidth=2)
    plt.vlines(0.5, ymin=0, ymax=1, color="black", linestyle="--")
    plt.vlines(0.7, ymin=0, ymax=1, color="black", linestyle="--")
    plt.legend(loc=2)
    plt.xlabel("x")
    plt.ylabel("Value")
    plt.tight_layout()
    if save:
        plt.savefig("training_results/diag" + experiment_tag + ".png", bbox_inches="tight", dpi=300)

    # 保存数据
    np.savez("training_results/" + experiment_tag + ".npz", total=pred_list_plot, mean=mu_plot, std=std_plot,
             all=pred_list_plot, diag_0=diag_pred_list_0, diag_1=diag_pred_list_1)


from octobot_commons.constants import PORTFOLIO_TOTAL, CONFIG_CRYPTO_CURRENCIES
from octobot_commons.logging.logging_util import get_logger
from octobot_commons.symbol_util import split_symbol, merge_currencies

from octobot_trading.constants import TICKER_CHANNEL
from octobot_trading.channels.exchange_channel import get_chan
from octobot_trading.constants import CONFIG_PORTFOLIO_TOTAL
from octobot_trading.exchanges.exchange_simulator import ExchangeSimulator
from octobot_trading.util import get_reference_market

class PortfolioProfitabilty:
    """
    PortfolioProfitabilty calculates the portfolio profitability
    by subtracting portfolio_current_value and portfolio_origin_value
    """

    def __init__(self, config, trader, portfolio_manager, exchange_manager):
        super().__init__()
        self.config = config
        self.trader = trader
        self.portfolio_manager = portfolio_manager
        self.exchange_manager = exchange_manager
        self.logger = get_logger(f"{self.__class__.__name__}[{self.exchange_manager.exchange_name}]")

        self.profitability = 0
        self.profitability_percent = 0
        self.profitability_diff = 0
        self.market_profitability_percent = 0
        self.initial_portfolio_current_profitability = 0
        self.initializing_symbol_prices = set()

        self.portfolio_origin_value = 0
        self.portfolio_current_value = 0

        self.currencies_last_prices = {}
        self.origin_crypto_currencies_values = {}
        self.current_crypto_currencies_values = {}
        self.origin_portfolio = None

        # buffer of currencies excluding market only used currencies ex: conf = btc/usd, eth/btc, ltc/btc, here usd
        # is market only => not used to compute market average profitability
        self.traded_currencies_without_market_specific = set()

        self.reference_market = get_reference_market(self.config)

    async def handle_mark_price_update(self, symbol, mark_price):
        force_recompute_origin_portfolio = False
        if symbol not in set(self.origin_crypto_currencies_values.keys()):
            # will fail if symbol doesn't have a price in self.origin_crypto_currencies_values and therefore
            # requires the origin portfolio value to be recomputed using this price info in case this price is relevant
            force_recompute_origin_portfolio = True
            self.origin_crypto_currencies_values[symbol] = mark_price
            currency = split_symbol(symbol)[0]
            if currency not in set(self.origin_crypto_currencies_values.keys()):
                self.origin_crypto_currencies_values[currency] = mark_price
        self.currencies_last_prices[symbol] = mark_price
        return await self._update_profitability(force_recompute_origin_portfolio)

    async def handle_balance_update(self, balance):
        return await self._update_profitability()

    """ Get profitability calls get_currencies_prices to update required data
    Then calls get_portfolio_current_value to set the current value of portfolio_current_value attribute
    Returns True if changed else False
    """

    async def _update_profitability(self, force_recompute_origin_portfolio=False):
        self.profitability_diff = self.profitability_percent
        self.profitability = 0
        self.profitability_percent = 0
        self.market_profitability_percent = 0
        self.initial_portfolio_current_profitability = 0

        try:
            await self.update_portfolio_and_currencies_current_value()

            if self.portfolio_origin_value == 0:
                # try to update portfolio origin value if it's not known yet
                await self._init_origin_portfolio_and_currencies_value()
            if force_recompute_origin_portfolio:
                await self._recompute_origin_portfolio_initial_value()

            initial_portfolio_current_value = await self._get_origin_portfolio_current_value()

            self.profitability = self.portfolio_current_value - self.portfolio_origin_value

            if self.portfolio_origin_value > 0:
                self.profitability_percent = (100 * self.portfolio_current_value / self.portfolio_origin_value) - 100
                self.initial_portfolio_current_profitability = \
                    (100 * initial_portfolio_current_value / self.portfolio_origin_value) - 100
            else:
                self.profitability_percent = 0

            # calculate difference with the last current portfolio
            self.profitability_diff = self.profitability_percent - self.profitability_diff

            self.market_profitability_percent = await self.get_average_market_profitability()

            return self.profitability_diff != 0
        except KeyError as e:
            self.logger.warning(f"Missing ticker data to calculate profitability")
            self.logger.warning(f"Missing {e} ticker data to calculate profitability")
        except Exception as e:
            self.logger.exception(e, True, str(e))

    """ Returns the % move average of all the watched cryptocurrencies between bot's start time and now
    """

    async def get_average_market_profitability(self):
        await self.get_current_crypto_currencies_values()

        origin_values = [value / self.origin_crypto_currencies_values[currency]
                         for currency, value
                         in self._only_symbol_currency_filter(self.current_crypto_currencies_values).items()
                         if self.origin_crypto_currencies_values[currency] > 0]

        return sum(origin_values) / len(origin_values) * 100 - 100 if origin_values else 0

    async def get_current_crypto_currencies_values(self):
        if not self.current_crypto_currencies_values:
            await self.update_portfolio_and_currencies_current_value()
        return self.current_crypto_currencies_values

    async def get_current_holdings_values(self):
        holdings = await self.get_current_crypto_currencies_values()

        return {currency: await self._get_currency_value(self.portfolio_manager.portfolio.portfolio, currency, holdings)
                for currency in holdings.keys()}

    def _only_symbol_currency_filter(self, currency_dict):
        if not self.traded_currencies_without_market_specific:
            self._init_traded_currencies_without_market_specific()
        return {currency: v for currency, v in currency_dict.items()
                if currency in self.traded_currencies_without_market_specific}

    def _init_traded_currencies_without_market_specific(self):
        for cryptocurrency in self.config[CONFIG_CRYPTO_CURRENCIES]:
            for pair in self.exchange_manager.exchange_config.get_traded_pairs(cryptocurrency):
                symbol, _ = split_symbol(pair)
                if symbol not in self.traded_currencies_without_market_specific:
                    self.traded_currencies_without_market_specific.add(symbol)

    async def update_portfolio_and_currencies_current_value(self):
        self.portfolio_current_value = await self.update_portfolio_current_value(
            self.portfolio_manager.portfolio.portfolio)

    async def _init_origin_portfolio_and_currencies_value(self):
        self.origin_portfolio = self.origin_portfolio or await self.portfolio_manager.portfolio.copy()
        self.origin_crypto_currencies_values = \
            await self._evaluate_config_crypto_currencies_and_portfolio_values(self.origin_portfolio.portfolio)
        await self._recompute_origin_portfolio_initial_value()

    async def _recompute_origin_portfolio_initial_value(self):
        self.portfolio_origin_value = \
            await self.update_portfolio_current_value(self.origin_portfolio.portfolio,
                                                      currencies_values=self.origin_crypto_currencies_values,
                                                      fill_currencies_values=True)

    async def _get_origin_portfolio_current_value(self, refresh_values=False):
        if refresh_values:
            self.current_crypto_currencies_values = \
                await self._evaluate_config_crypto_currencies_and_portfolio_values(self.origin_portfolio.portfolio)
        return await self.update_portfolio_current_value(self.origin_portfolio.portfolio,
                                                         currencies_values=self.current_crypto_currencies_values)

    async def update_portfolio_current_value(self, portfolio, currencies_values=None, fill_currencies_values=False):
        values = currencies_values
        if values is None or fill_currencies_values:
            self.current_crypto_currencies_values = \
                await self._evaluate_config_crypto_currencies_and_portfolio_values(portfolio)
            if fill_currencies_values:
                for currency, value in self.current_crypto_currencies_values.items():
                    if currency not in currencies_values:
                        currencies_values[currency] = value
            values = self.current_crypto_currencies_values
        return await self._evaluate_portfolio_value(portfolio, values)

    """ try_get_value_of_currency will try to obtain the current value of the currency quantity
    in the reference currency.
    It will try to create the symbol that fit with the exchange logic.
    Returns the value found of this currency quantity, if not found returns 0.
    """

    async def _try_get_value_of_currency(self, currency, quantity, raise_error):
        symbol = merge_currencies(currency, self.reference_market)
        symbol_inverted = merge_currencies(self.reference_market, currency)

        try:
            if self.exchange_manager.symbol_exists(symbol):
                return self.currencies_last_prices[symbol] * quantity

            elif self.exchange_manager.symbol_exists(symbol_inverted) and \
                    self.currencies_last_prices[symbol_inverted] != 0:
                return quantity / self.currencies_last_prices[symbol_inverted]

            self._inform_no_matching_symbol(currency)
            return 0
        except KeyError as e:
            if not self.exchange_manager.is_backtesting:
                symbols_to_add = []
                if self.exchange_manager.symbol_exists(symbol):
                    symbols_to_add = [symbol]
                elif self.exchange_manager.symbol_exists(symbol_inverted):
                    symbols_to_add = [symbol_inverted]

                if symbols_to_add:
                    await get_chan(TICKER_CHANNEL, self.exchange_manager.id).modify(added_pairs=symbols_to_add)
                    self.initializing_symbol_prices.add(currency)
                if raise_error:
                    raise e
            return 0

    def _inform_no_matching_symbol(self, currency):
        # do not log warning in backtesting or tests
        if not self.exchange_manager.is_backtesting:
            self.logger.warning(f"Can't find matching symbol for {currency} and {self.reference_market}")

    async def _evaluate_config_crypto_currencies_and_portfolio_values(self, portfolio):
        values_dict = {}
        evaluated_currencies = set()
        missing_tickers = set()
        # evaluate config currencies
        for cryptocurrency in self.config[CONFIG_CRYPTO_CURRENCIES]:
            pairs = self.exchange_manager.exchange_config.get_traded_pairs(cryptocurrency)
            if pairs:
                currency, market = split_symbol(pairs[0])
                currency_to_evaluate = currency
                try:
                    if currency not in evaluated_currencies:
                        values_dict[currency] = await self.evaluate_value(currency, 1)
                        evaluated_currencies.add(currency)
                    if market not in evaluated_currencies:
                        currency_to_evaluate = market
                        values_dict[market] = await self.evaluate_value(market, 1)
                        evaluated_currencies.add(market)
                except KeyError:
                    missing_tickers.add(currency_to_evaluate)
        # evaluate portfolio currencies
        for currency in portfolio:
            try:
                if currency not in evaluated_currencies:
                    values_dict[currency] = await self.evaluate_value(currency, 1)
                    evaluated_currencies.add(currency)
            except KeyError:
                missing_tickers.add(currency)
        if missing_tickers:
            self.logger.debug(f"Missing price data for {missing_tickers}, impossible to compute all the "
                              f"currencies values for now.")
        return values_dict

    """ evaluate_portfolio_value performs evaluate_value with a portfolio configuration
    Returns the calculated quantity value in reference (attribute) currency
    """

    async def _evaluate_portfolio_value(self, portfolio, currencies_values=None):
        return sum([
            await self._get_currency_value(portfolio, currency, currencies_values)
            for currency in portfolio
        ])

    async def _get_currency_value(self, portfolio, currency, currencies_values=None, raise_error=False):
        if currency in portfolio and portfolio[currency][CONFIG_PORTFOLIO_TOTAL] != 0:
            if currencies_values and currency in currencies_values:
                return currencies_values[currency] * portfolio[currency][CONFIG_PORTFOLIO_TOTAL]
            else:
                return await self.evaluate_value(currency, portfolio[currency][CONFIG_PORTFOLIO_TOTAL], raise_error)
        return 0

    # Evaluate value returns the currency quantity value in the reference (attribute) currency
    async def evaluate_value(self, currency, quantity, raise_error=True):
        # easy case --> the current currency is the reference currency
        if currency == self.reference_market:
            return quantity
        elif quantity == 0:
            return quantity
        else:
            currency_value = await self._try_get_value_of_currency(currency, quantity, raise_error)
            if currency_value > 0 and currency in self.initializing_symbol_prices:
                self.initializing_symbol_prices.remove(currency)
            return currency_value

    async def holdings_ratio(self, currency):
        currency_holdings: float = self.portfolio_manager.portfolio.get_currency_from_given_portfolio(currency,
                                                                                PORTFOLIO_TOTAL)
        currency_value: float = await self.evaluate_value(currency, currency_holdings)
        return currency_value / self.portfolio_current_value if self.portfolio_current_value else 0


from setuptools import setup, find_packages

with open('requirements.txt') as requirements_file:
    install_requirements = requirements_file.read().splitlines()

setup(
    name="samplecli",
    version="0.0.1",
    description="A small package",
    author="hgoe_hoge_bar",
    packages=find_packages(),
    py_modules=["sample_hoge.lib"],
    install_requires=install_requirements,
    entry_points={
        "console_scripts": [
            "sample_hoge=sample_hoge.core:main",
        ]
    },
    classifiers=[
        'Programming Language :: Python :: 3.6',
    ]
)


# Imports
from sys import path
path.append("../Hadamard-Matrix-for-hashing")

import torch
import accimage
import os
import datetime
import numpy as np
import pre_process as prep
from PIL import Image
from torch.autograd import Variable
from pathlib import Path

DATA_DIR = "/mydata/MSCOCO_Dataset/QIK_Image_Data"
# DATA_DIR = "/mydata/MSCOCO_Dataset_Subset"

def pil_loader(path):
    # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835)
    with open(path, 'rb') as f:
        with Image.open(f) as img:
            return img.convert('RGB')

def accimage_loader(path):
    try:
        return accimage.Image(path)
    except IOError:
        # Potentially a decoding problem, fall back to PIL.Image
        return pil_loader(path)

# To obtain the hash code
def predict_hash_code(model, input):
    model.eval()
    y = model(input)
    output = y.data.cpu().float()
    return output

def convert_list(db_lst):
    is_start = True
    for db in db_lst:
        if is_start:
            all_output = db
            is_start = False
        else:
            all_output = torch.cat((all_output, db), 0)
    return all_output.cpu().numpy()

# Compute central similarity
def get_similarity(database_hash_lst, query_hash_lst):
    db = convert_list(database_hash_lst)
    q = convert_list(query_hash_lst)
    sim = np.dot(db, q.T)
    ids = np.argsort(-sim, axis=0)
    return ids

if __name__ == '__main__':
    database_hash_lst = []
    file_lst = []

    # Loading the model
    model = torch.load("coco_64bit_0.8612_resnet50.pkl", map_location='cpu')
    model = torch.nn.DataParallel(model).module.to(torch.device("cpu"))
    model = model.module

    # Iterating over the images in the directory
    for filename in os.listdir(DATA_DIR):
        print("Obtaining the features for the image :: " + filename)
        # Loading the image
        img = pil_loader(DATA_DIR + "/" + filename)

        # Transforming the image
        transform = prep.image_test(resize_size=255, crop_size=224)
        input = transform(img)

        # Hack for batch size
        input = input.unsqueeze(0)

        # Obtaining the hash
        try:
            database_hash = predict_hash_code(model, input)
            database_hash_lst.append(database_hash)
            file_lst.append(filename)
        except Exception as e:
            print("eval_HMFH.py :: Skipping file : ", filename)
            print(e)

    # Clearing the results file, if it exists
    if Path("CSQ_Pre_Results_Dict.txt").exists():
        os.remove("CSQ_Pre_Results_Dict.txt")

    # Iterating over the query images
    for filename in os.listdir(DATA_DIR):
        # Noting the start time
        time = datetime.datetime.now()

        # Initializing an array to hold the search results
        result_lst = []

        # Loading the image
        print("Obtaining the features for the query :: " + filename)
        img = pil_loader(DATA_DIR + "/" + filename)

        # Transforming the image
        transform = prep.image_test(resize_size=255, crop_size=224)
        input = transform(img)

        # Hack for batch size
        input = input.unsqueeze(0)

        try:
            # Obtaining the hash
            query_hash = predict_hash_code(model, input)

            # Creating the query hash list
            query_hash_lst = [query_hash]

            # Obtaining the central similariy
            ids = get_similarity(database_hash_lst, query_hash_lst)

            # Iterating over the ids
            for id in ids:
                result_lst.append(file_lst[id[0]])

            # Removing the query from the result list.
            result_lst.remove(filename)

            # Obtaining the CSQ retrieval time
            csq_time = datetime.datetime.now() - time

            # Writing results to the file
            with open('CSQ_Pre_Results_Dict.txt', 'a+') as f:
                f.write(filename + ":: {\'csq_time\': " + str(csq_time.microseconds) +",\'csq_results\': " + str(result_lst[:16]) + "}\n")

        except Exception as e:
            print("eval_HMFH.py :: Skipping file : ", filename)
            print(e)


""" Module handles common database functionality """
import os
from datetime import datetime
from settings import USE_MONGO
from util.db_util import DBUtil

CLIENT = DBUtil(db_name='nft-alley-users').get_db()
USERS = CLIENT['users']
REVOKED_TOKEN = CLIENT['revoked_token']

def get_admin_users():
    """ Method to get all admin uses """
    admin_users = list(USERS.find({"admin": True}))
    return admin_users

def get_current_user(current_user):
    """ Method to verify is incoming JWT is valid """
    users = USERS.find_one({"userName": current_user})
    return users

def is_admin(current_user):
    """ Method to verify if JWT user identity is admin """
    user_admin_check = USERS.find_one({"userName": current_user,
                                    "admin": True})
    return user_admin_check

def is_api_protected():
    """ Method to verify if api is protected or not """
    api_unprotected = os.environ.get('API_UNPROTECTED', "FALSE").upper()
    if api_unprotected == "TRUE":
        return False
    return True

def insert_a_record(userName, admin, token_expires,
                    current_user=None):
    """ Method to create a user """
    USERS.insert_one({'userName': userName,
                    'admin': admin,
                    'token_expires': token_expires,
                    'created_on': datetime.utcnow(),
                    'created_by': current_user})

def update_a_record(userName, token_expires, current_user=None):
    """ Method to update a user """
    myquery = {"userName": userName}
    newvalues = {"$set": {"token_expires": token_expires,
                        "updated_on": datetime.utcnow(),
                        "updated_by": current_user}}
    USERS.update_one(myquery, newvalues)

def update_user_to_admin(userName, current_user=None):
    """ Method to update a user to admin """
    myquery = {"userName": userName}
    newvalues = {"$set": {"admin": True,
                        "updated_on": datetime.utcnow(),
                        "updated_by": current_user}}
    USERS.update_one(myquery, newvalues)

def delete_a_record(userName):
    """ Method to delete a user """
    myquery = {"userName": userName}
    USERS.delete_one(myquery)


import datetime

class Data:
    def __init__(self, data):
        self.data_str = data

    @classmethod
    def conversion_data(cls, data_str):
        '''Data -"01-01-2000'''
        if Data.validation_data(data_str):
            day, month, year = map(int, data_str.split('-'))
            print(f'Day - {day}, Month - {month}, Year - {year}')
            return cls(data_str)
        else:
            print('Incorrect data')

    @staticmethod
    def validation_data(data_str):
        day, month, year = map(int, data_str.split('-'))
        return 31 >= day > 0 and 0 < month <= 12 and 1900 < year <= 2020

my_data = str(datetime.datetime.now().strftime("%d-%m-%Y"))
print('Today', my_data)
dt1 = Data.conversion_data(my_data)
if dt1:
    print(dt1.data_str)

dt2 = Data.conversion_data('31-12-2021')
if dt2:
    print(dt2.data_str)


import time
import subprocess,uiautomation
file="C:\\Users\\王宇\\Desktop\\开题报告提纲.PNG"
subprocess.Popen("C:\Program Files (x86)\Tencent\WeChat\WeChat.exe")
wechat=uiautomation.WindowControl(searchDepth=1,Name="微信")
wechat.MoveToCenter()
# wechat.ButtonControl(Name="通讯录").Click()
# search_people=wechat.EditControl(Name="搜索")
# search_people.Click()
# search_people.SendKeys('天道酬勤{enter}')
edit_message=wechat.EditControl(Name="输入")
edit_message.Click()
edit_message.SendKeys("hello{enter}")
# wechat.ButtonControl(Name="发送(S)").Click()
wechat.ButtonControl(Name="发送文件").Click()
file_edit=wechat.EditControl(Name="文件名(N):")
file_edit.Click()
file_edit.SendKeys(file+"{enter}")
edit_message.SendKeys("{enter}")
# wechat.ButtonControl(Name="发送(S)").Click()


# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

# useful for handling different item types with a single interface
import copy
from datetime import datetime

import pymongo

class AttachCrawlFieldsPipeline(object):
    def process_item(self, item, spider):
        item = dict(item)
        item['crawling_job_id'] = spider.crawling_job_id
        item['crawler_id'] = spider.crawler_id
        item['spider_name'] = spider.name
        item['crawled_at'] = datetime.utcnow().isoformat()
        return item

class MongoDBPipeline:

    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DATABASE')
        )

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        payload = copy.deepcopy(dict(item))

        payload['crawled_at'] = datetime.fromisoformat(payload['crawled_at'])
        payload['filing_date'] = datetime.fromisoformat(payload['filing_date'])

        for entry in payload['entries']:
            if not entry['filing_date']:
                continue
            entry['filing_date'] = datetime.fromisoformat(entry['filing_date'])

        for party in payload['parties']:
            if not party['end_date']:
                continue
            party['end_date'] = datetime.fromisoformat(party['end_date'])

        self.db[f'job-{spider.crawling_job_id}'].insert_one(payload)
        return item


import numpy as np
import pandas as pd
import sysid as sd
import matplotlib.pyplot as plt

from utils import plot_io, plot_y

def frols(cm, y, tol, max_iter):
    w1i = [cm[:, i] for i in range(cm.shape[1])]
    M = len(w1i)
    g1i = [w1i[i] @ y / (w1i[i] @ w1i[i]) for i in range(len(w1i))]
    ERRi = [g1i[i]**2 * w1i[i] @ w1i[i] / (y @ y) for i in range(len(w1i))]
    selected = [np.argmax(ERRi)]
    W = [w1i[selected[0]]]

    selected_erri = [ERRi[selected[0]]]

    k = 1
    while 1 - np.sum(selected_erri) > tol and k < max_iter:
        ERRi = []
        for i in range(M):
            if i not in selected:
                alpha = [(W[j] @ w1i[i]) / (W[j] @ W[j]) for j in range(k)]
                wki = w1i[i] - sum([alpha[j] * W[j] for j in range(k)])
                gki = wki @ y / (wki @ wki)
                ERRi.append(gki**2 * wki @ wki / (y @ y))
            else:
                ERRi.append(0)
        selected.append(np.argmax(ERRi))
        W.append(w1i[selected[k]] - sum([alpha[j] * W[j] for j in range(k)]))
        selected_erri.append(ERRi[selected[k]])
        k += 1

    selected.pop()
    selected_erri.pop()

    return selected, selected_erri

if __name__ == "__main__":

    # loading the data

    dataset_name = "ball-and-beam"

    if dataset_name != "uniform":
        df = pd.read_csv(f"src/data/{dataset_name}.csv")

    if dataset_name in ["ball-and-beam", "robot-arm"]:
        u = df["u"].values
        y = df["y"].values
    elif dataset_name == "exchanger":
        u = df["q"].values
        y = df["th"].values
    elif dataset_name == "SNLS80mV":
        u = df["V1"].values
        y = df["V2"].values
    elif dataset_name == "tanque":
        u_train, y_train = df["uEst"].values, df["yEst"].values
        u_test, y_test = df["uVal"].values, df["yVal"].values
    elif dataset_name == "uniform":
        u = np.random.uniform(-1, 1, 200)
        y = np.zeros_like(u)

        for k in range(2, len(y)):
            y[k] = -0.605 * y[k - 1] - 0.163 * y[k - 2]**2 + 0.588 * u[k - 1] - 0.240 * u[k - 2]

    # separate the data into training and testing sets

    if dataset_name != "tanque":
        mu, my = np.max(np.abs(u)), np.max(np.abs(y))

        u = u / mu
        y = y / my

        n_train = int(0.2 * u.shape[0])
        u_train, u_test = u[-n_train:], u[:-n_train]
        y_train, y_test = y[-n_train:], y[:-n_train]

        plot_io(
            u=u, y=y, title=dataset_name.replace('-', ' ')
        )
    else:
        mu_train, mu_test = np.max(np.abs(u_train)), np.max(np.abs(u_test))
        my_train, my_test = np.max(np.abs(y_train)), np.max(np.abs(y_test))

        u_train, u_test = u_train / mu_train, u_test / mu_test
        y_train, y_test = y_train / my_train, y_test / my_test

        plot_io(
            u=u_train, y=y_train, title=f"{dataset_name.replace('-', ' ')}"
        )

    if dataset_name == "SNLS80mV":
        dataset_name = "silverbox"

    plt.savefig(f"src/results/{dataset_name}/io.png")
    plt.show()

    # creating the data matrix for the train set

    nu, ny, ne = 5, 5, 0
    dm = sd.data_matrix(u=u_train, y=y_train, nu=nu, ny=ny, ne=ne)

    # creating the candidate matrix for the train set
    nlin = 1
    cm, comb = sd.candidate_matrix(dm, nlin)

    Y = y_train[:-max(nu, ny, ne)]

    tol = 0
    # structure selection with frols
    selected, ERR = frols(cm, Y, tol, 10)

    # parameter estimation on train set
    P = cm[:, selected]

    T = np.linalg.inv(P.T @ P) @ P.T @ Y

    y_pred = P @ T

    plot_y(
        y=Y, y_pred=y_pred,
        title=f"{dataset_name.replace('-', ' ')} - Train"
    )

    plt.savefig(f"src/results/{dataset_name}/FROLS/train.png")
    plt.show()

    print("ERRi = ", ERR)
    print("ESR = %f" % (1 - np.sum(ERR)))
    print("Selected terms: %d of %d" % (len(selected), len(comb)), "with tol = %f" % (tol))
    for i, t in zip(selected, T):
        print("\n", sd.get_model_term(comb[i], nu, ny, ne), t)

    # parameter estimation on test set

    dm = sd.data_matrix(u=u_test, y=y_test, nu=nu, ny=ny, ne=ne)

    # creating the candidate matrix for the test set

    cm, comb = sd.candidate_matrix(dm, nlin)

    Y = y_test[:-max(nu, ny, ne)]

    # parameter estimation on train set
    P = cm[:, selected]

    y_pred = P @ T

    plot_y(
        y=Y, y_pred=y_pred,
        title=f"{dataset_name.replace('-', ' ')} - Validation"
    )

    plt.savefig(f"src/results/{dataset_name}/FROLS/test.png")

    # computes mean squared error

    mse = np.mean((y_pred - Y)**2)
    print("MSE=%f" % (mse))

    plt.show()


from ....core.log.log import Log
from ....core.comm.comm_utils import CommUtils
from ....core.config.config import Config
from ..comm.comm_ctxt import CommCtxt
from .core.workspace_base import WorkspaceBase

from tkinter import ttk
from tkinter import LEFT
from tkinter import Frame

class WsConnStatus(WorkspaceBase):
    def __init__(self, parent_frame, ws_controller, ws_resolution):
        WorkspaceBase.__init__(self, parent_frame, ws_controller, ws_resolution)
        self.Active = False
        self.applications = []
        app_config = Config.get_config_val("application")["comm"]
        for app_name in app_config:
            self.applications.append(app_name)
        self.appl_labels = []

    def update_appl_labels(self):
        curr_index = 0
        for application in self.applications:
            appl_status = "{0} : {1}".format(application, CommUtils.is_connected(CommCtxt.get_comm_if(), application))
            self.appl_labels[curr_index].configure(text = appl_status)
            curr_index += 1

    @staticmethod
    def get_id():
        return "Start"

    def refresh(self):
        if self.active:
            self.after(500, self.refresh)
            self.update_appl_labels()

    def activate(self):
        Log.log("Activate called")
        self.ws_header = ttk.Label(self, text = "Connection status", anchor = "w", width = 20)
        self.ws_header.grid(row = 0, column = 0)
        for application in self.applications:
            appl_label = ttk.Label(self, text = "{0} : {1}".format(application, False), anchor = "w", width = 20)
            appl_label.grid(row = len(self.appl_labels) + 1, column = 0)
            self.appl_labels.append(appl_label)
        self.active = True
        self.after(500, self.refresh)

    def pause(self):
        Log.log("Pause called")
        self.active = False

    def deactivate(self):
        Log.log("Deactivate called")
        self.active = False
        self.appl_labels = []


import cupy as cp
from matplotlib import pyplot as plt
from visualize_net import compare_acts, visualize_multiple_act
from utils import list_files, repair_agent
import seaborn as sns
from mushroom_rl.algorithms.agent import Agent
# from populate_histograms import populate_histograms
import json
from collections import namedtuple
import os, sys

sns.set_style("whitegrid")

games = ["Asterix", "BattleZone", "Breakout", "Enduro", "Jamesbond", "Kangaroo",
         "Pong", "Qbert", "Seaquest", "Skiing", "SpaceInvaders", "Tennis",
         "TimePilot", "Tutankham", "VideoPinball"]

save_folder = 'images/af_comparisons/'
rat_type = "rat"
limit = 20
fig, axes = plt.subplots(len(games), 4, figsize=(10, 1.5*len(games)))
for ax in axes.flatten():
    # ax.xaxis.set_ticklabels([])
    ax.yaxis.set_ticklabels([])
    ax.tick_params(color=[1, 1, 1, 0])
agents_folder = f'updated_populated_agents'
for axs, game in zip(axes, games):
    for seed in range(5):
        filename = f"DQN_{rat_type}_{game}_s{seed}_e500.zip"
        path = f"{agents_folder}/{filename}"
        if os.path.isfile(path):
            print(f"Found agent at {path}")
            ag = Agent.load(path)
            break
    net = ag.approximator.model.network
    acts = [eval(f'net.act_func{act_n+1}') for act_n in range(4)]
    input_dists = [act.distribution for act in acts]
    axs[0].set_ylabel(game)
    if rat_type == "rat":
        from rational.torch import Rational
        Rational.list = acts
        for rat in acts:
            rat.snapshot_list = []
            rat.func_name = "  "
        Rational.show_all(axes=axs, display=False)

plt.tight_layout(pad=0.1)

if "--store" in sys.argv:
    out_filename = f"few_games_{rat_type}.svg"
    fig.savefig(f"{save_folder + out_filename}", format='svg')
    print(f"saved in {save_folder + out_filename}")
else:
    plt.show()


import boto3
import time

class EasyLambdaTrigger:
	def __init__(self, lambda_name, region_name='eu-west-1'):
		self.ENABLING = 'Enabling'
		self.ENABLED = 'Enabled'
		self.DISABLING = 'Disabling'
		self.DISABLED = 'Disabled'
		self._state_values = [
			self.ENABLING, self.ENABLED, self.DISABLING, self.DISABLED
		]
		self.lambda_name = lambda_name
		self.lambda_client = boto3.client(service_name='lambda', region_name=region_name)

	def get_event_sources(self):
		event_source_mapping = self.lambda_client.list_event_source_mappings(FunctionName=self.lambda_name)
		return event_source_mapping['EventSourceMappings']

	def get_event_sources_uuid(self):
		return [e['UUID'] for e in self.get_event_sources()]

	def get_event_source_state(self, trigger_uuid):
		return self.lambda_client.get_event_source_mapping(UUID=trigger_uuid)['State']

	def update_trigger(self, trigger_uuid, enable, timeout=180, wait=False):
		if enable:
			wait_if_before_state = self.DISABLING
			wait_before_state = self.DISABLED
			return_states = [self.ENABLED, self.ENABLING]
			wait_state = self.ENABLED
		else:
			wait_if_before_state = self.ENABLING
			wait_before_state = self.ENABLED
			return_states = [self.DISABLED, self.DISABLING]
			wait_state = self.DISABLED

		current_state = self.get_event_source_state(trigger_uuid)

		if current_state == wait_if_before_state:
			self.wait_state(trigger_uuid, wait_before_state, timeout)
		elif current_state in return_states:
			if wait:
				self.wait_state(trigger_uuid, wait_state, timeout)
			return self.get_event_source_state(trigger_uuid)

		self.lambda_client.update_event_source_mapping(
			UUID=trigger_uuid, FunctionName=self.lambda_name,
			Enabled=enable
		)
		if wait:
			self.wait_state(trigger_uuid, wait_state, timeout)

		return self.get_event_source_state(trigger_uuid)

	def wait_state(self, trigger_uuid, state, timeout=180):
		if state not in self._state_values:
			error_msg = f"The only state values are: {','.join(self._state_values)}"
			raise ValueError(error_msg)

		start_time = time.time()
		while True:
			current_state = self.get_event_source_state(trigger_uuid)
			if current_state == state:
				return current_state
			elif time.time() - start_time > timeout:
				error_msg = f'Timeout {timeout} secs reached. Cannot wait anymore: increase timeout value'
				raise TimeoutError(error_msg)


# -*- coding: utf-8 -*-
#!/usr/bin/python
# package/PYM_MenuBarWidget.py
"""
    MenuBarWidget
    ~~~~~~~~~~~~~~~~~~

    Menu Bar widget designed for frameless UI in PySide2.
    Displays app icon, title and window commands (minimize and close)

    :Author: Michal Rutkowski
    :Created: 2019/03/10
    :Python Version: 3.7
"""

import os
import shutil

# qt modules
import PySide2
from PySide2 import QtWidgets, QtCore, QtSql, QtGui
from PySide2.QtCore import *
from PySide2.QtGui import *

import framelessQt.PYM_MoveableWidget as MoveableWidget

Action_Style = ("""
    QPushButton { font-size: 16px; border: 0px; font-family: Source Sans Pro;background-color: transparent;color : #E6E6E6; }
    QPushButton:hover { font-size: 16px; border: 0px; font-family: Source Sans Pro;background-color: transparent;color : #E6E6E6; }
        """)

class QBaseButton(QtWidgets.QPushButton):
    def __init__(self, request, resourcesPath):
        super(QBaseButton, self).__init__(None)
        self.request = request
        self.iconsPath = os.path.join(resourcesPath, "icons")
        self.setIcon(QIcon(QPixmap(os.path.join(self.iconsPath, self.request + '_icon.png'))))
        self.setIconSize(QSize(22, 22))
        self.setStyleSheet(Action_Style)
        self.setMinimumHeight(22)
        # self.clicked.connect(self.cancel)

    def enterEvent(self, event):
        self.setIcon(QIcon(QPixmap(os.path.join(self.iconsPath, self.request + '_hover_icon.png'))))
        self.setIconSize(QSize(22, 22))

    def leaveEvent(self, event):
        self.setIcon(QIcon(QPixmap(os.path.join(self.iconsPath, self.request + '_icon.png'))))
        self.setIconSize(QSize(22, 22))

class MenuBarWidget(MoveableWidget.MoveableWidget):
    def __init__(self, app, title, resourcesPath, iconPath):
        super(MenuBarWidget, self).__init__(app)
        # Set the top layout
        self.menuBarLayout = QtWidgets.QHBoxLayout()
        self.menuBarLayout.setAlignment(Qt.AlignTop)
        #self.MainLayout.addLayout(self.menuBarLayout)
        self.menuBarLayout.setSpacing(8)

        self.logoIconLayout = QtWidgets.QHBoxLayout()
        self.menuBarLayout.addLayout(self.logoIconLayout)
        self.logoIconLayout.setAlignment(Qt.AlignLeft)
        self.logoIconLayout.setSpacing(8)

        self.logoButton = QtWidgets.QPushButton("")
        self.logoButton.setIcon(QIcon(QPixmap(os.path.join(resourcesPath, iconPath))))
        self.logoButton.setIconSize(QSize(35, 35))
        self.logoButton.setStyleSheet(Action_Style)
        self.logoIconLayout.addWidget(self.logoButton)
        self.titleLabel = QtWidgets.QLabel(title)
        self.logoIconLayout.addWidget(self.titleLabel)

        self.closeMinimizeLayout = QtWidgets.QHBoxLayout()
        self.menuBarLayout.addLayout(self.closeMinimizeLayout)
        self.closeMinimizeLayout.setAlignment(Qt.AlignRight)
        self.closeMinimizeLayout.setSpacing(8)

        self.minimizeButton = QBaseButton("minimize", resourcesPath)
        self.closeMinimizeLayout.addWidget(self.minimizeButton)
        self.minimizeButton.clicked.connect(lambda: self.app.showMinimized())

        self.close_ui = QBaseButton("close", resourcesPath)
        self.closeMinimizeLayout.addWidget(self.close_ui)
        self.close_ui.clicked.connect(self.cancel)

        self.menuBarLayout.addLayout(self.closeMinimizeLayout)

        self.setLayout(self.menuBarLayout)

        self.mouseLock = False

    def cancel(self):
        self.app.close()
        '''
        folder = "cache"
        # remove cache folder contents
        for the_file in os.listdir(folder):
            file_path = os.path.join(folder, the_file)
            try:
                if os.path.isfile(file_path):
                    os.unlink(file_path)
            except Exception as e:
                print(e)
        '''


from flask import jsonify, g, request, url_for, json
from argus import db
from argus.models import Replay, ReplaySectorFever
from argus.api import bp

@bp.route('/api/replay', methods=['POST'])
def post_replay():
    post = request.get_json() or {}
    print(post)
    if 'date' not in post or 'summary' not in post or 'marketFever' not in post:
        # return bad_request('must include username, email and password fields')
        print('error')

    if 'sectorFevers' in post:
        fevers = post['sectorFevers']
        date = post['date']
        for fever in fevers:
            fever['date'] = date
            if 'sectorId' not in fever or 'fever' not in fever:
                continue
            new_sector_fever = ReplaySectorFever()
            new_sector_fever.from_dict(fever)
            db.session.add(new_sector_fever)

    # 大盘
    new_replay = Replay()
    new_replay.from_dict(post)
    db.session.add(new_replay)
    db.session.commit()
    response = jsonify(post)
    response.status_code = 201
    # response.headers['Location'] = url_for('api.post_replay', id=new_replay.id)
    return response

# 根据date获取replay详情

@bp.route('/api/replay', methods=['GET'])
def get_replay():
    params = request.args
    if 'date' not in params:
        print('error')

    date = request.args.get("date")

    resources = Replay.query.filter(
        Replay.date == date
    ).all()

    sectors = ReplaySectorFever.query.filter(
        ReplaySectorFever.date == date
    ).all()

    data = {
        'data': item.to_dict() for item in resources
    }
    # data['data'] = item.to_dict() for item in resources
    data['sector'] = [s.to_dict() for s in sectors]
    return data

@bp.route('/api/replays', methods=['GET'])
def get_replays():
    params = request.args
    if 'startDay' not in params or 'endDay' not in params:
        # 进行数据分页
        page = request.args.get('page', 1, type=int)
        per_page = min(request.args.get('per_page', 10, type=int), 100)
        data = Replay.to_collection_dict(Replay.query, page, per_page, 'api.get_replays')
        return jsonify(data)

    start_day = request.args.get("startDay") or ''
    end_day = request.args.get("endDay") or ''

    resources = Replay.query.filter(
        Replay.date >= start_day,
        Replay.date <= end_day
    ).all()
    data = {
        'data': [item.to_dict() for item in resources]
    }
    return data

@bp.route('/api/replay/fevers', methods=['GET'])
def get_replay_fevers():
    params = request.args
    if 'startDay' not in params or 'endDay' not in params:
        print('error')
    start_day = request.args.get("startDay") or ''
    end_day = request.args.get("endDay") or ''
    sector_ids = request.args.getlist("sectorIds[]") or []

    resources = ReplaySectorFever.query.filter(
        ReplaySectorFever.date >= start_day,
        ReplaySectorFever.date <= end_day,
        ReplaySectorFever.sectorId.in_(sector_ids)
    ).all()
    data = {
        'data': [item.to_dict() for item in resources]
    }
    return data


# This script counts the number of articles in a given dataset.

import csv
import json
import os
import sys
from matplotlib import pyplot
from functions import validateDataset

if __name__ == "__main__":
    """""
        Parameters:
            args[0] --> Controls which dataset is going to be selected. Can be set to "kdd17" or "stocknet"
    """""

    # Handling Parameters
    if len(sys.argv) > 1:
        dataset = validateDataset(sys.argv[1])
    else:
        dataset = "kdd17"

    # Variable Declaration
    counter = 0
    yearlyCounter = {"2007": 0, "2008": 0, "2009": 0, "2010": 0, "2011": 0, "2012": 0, "2013": 0, "2014": 0, "2015": 0, "2016": 0}

    for stock in os.listdir("../data/" + dataset + "/NYT-Business/ourpped"):
        with open("../data/" + dataset + "/NYT-Business/ourpped/" + stock, encoding="utf-8") as file:
            csvReader = csv.reader(file)
            next(csvReader)  # Skip headers
            csvReader = list(csvReader)
            for row in csvReader:

                data = json.loads(row[1])
                if type(data) is list:
                    yearlyCounter[row[0][:4]] = yearlyCounter[row[0][:4]] + len(data)
                    counter = counter + len(data)
                else:
                    counter += 1
                    yearlyCounter[row[0][:4]] += 1
    print("The " + dataset + " dataset contains " + str(counter) + " articles")

    yearlyCounter = {x: y for x, y in yearlyCounter.items() if y != 0}

    figure, axes = pyplot.subplots()
    axes.bar(*zip(*yearlyCounter.items()))

    axes.set_xlabel("Year")
    axes.set_ylabel("Numer of articles")
    pyplot.show()


import librosa
import numpy as np
import os
import pandas as pd
import random
import soundfile
import hydra

from joblib import Parallel, delayed
from omegaconf import DictConfig
from pathlib import Path
from tqdm import tqdm
from typing import List
from .sampling import create_microphone_and_loudspeaker_combinations

from .settings import (
    LOUDSPEAKER_POSITIONS, MIN_DURATION_IN_SECS,
    POSITIONS, RT60_IN_SECS, START_TRIM, END_TRIM, SR, ROOM_DIMS,
)

def sample_adhoc40_phrases(config):
    """randomly select loudspeaker and microphone positions,
    as well as the speech sample used from the LibriAdhoc40 dataset"""

    random.seed(config["random_seed"])

    # 0. Load metadata as a pandas dataframe
    metadata = _load_adhoc40_metadata(config["input_dir"])

    # 1. Get loudspeaker and microphone combinations to use
    combinations = create_microphone_and_loudspeaker_combinations(
                        microphone_groups=config["microphone_groups"],
                        n_mics=config["n_mics"],
                        split_mode=config["mode"],
                        random_seed=config["random_seed"],
                        n=config["n_samples"])

    def get_loudspeaker_phrases(loudspeaker_position_id):
        loudspeaker_phrases = filter(
            lambda x: x["loudspeaker_position"] == loudspeaker_position_id, metadata)
        return list(loudspeaker_phrases)
    loudspeaker_phrases = {
        loudspeaker_position_id: get_loudspeaker_phrases(loudspeaker_position_id)
        for loudspeaker_position_id in LOUDSPEAKER_POSITIONS
    }

    n_mics = len(combinations[0]) - 1 # Every combination has one loudspeaker position and n_mics microphones
    samples = []

    count = 0
    for combination in tqdm(combinations):
        loudspeaker_position = combination[0]
        n_microphones = combination[1]

        for i in range(config["n_phrases_per_positioning"]):
            phrases = loudspeaker_phrases[loudspeaker_position]

            while True: # Loop until finding a signal with a minimal duration
                mic_str = '-'.join(str(x) for x in n_microphones)
                sample_id = f"{loudspeaker_position}-{mic_str}-{i}"
                sample = random.choice(phrases)
                phrase_name = os.path.basename(sample["path"])

                mic_paths = [
                    sample["path"] + "/" + f"{phrase_name}-ch-{n_mic}.wav"
                    for n_mic in n_microphones
                ]
                durations = [
                    librosa.get_duration(filename=mic_path)
                    for mic_path in mic_paths
                ]
                is_min_duration = list(filter(
                    lambda x: x >= MIN_DURATION_IN_SECS,
                    durations
                ))

                if len(is_min_duration) < n_mics:
                    continue

                source_coordinates = POSITIONS[loudspeaker_position]
                mic_coordinates = [
                    POSITIONS[n_mic]
                    for n_mic in n_microphones
                ]

                sample_metadata = {
                    "source_coordinates": source_coordinates,
                    "mic_coordinates": mic_coordinates,
                    "sample_id": sample_id,
                    "mic_paths": mic_paths
                }
                # for i, mic_path in enumerate(mic_paths):
                #     sample_metadata[f"mic_{i}_path"] = mic_path

                samples.append(sample_metadata)

                break

    return samples

def create_dataset(samples: List[dict], config):
    dataset_dir = Path(config["dataset_dir"]) / config["mode"]
    os.makedirs(dataset_dir, exist_ok=True)

    def process(sample):
        sample_id = sample["sample_id"]
        signals_dir = f"samples/{sample_id}"
        os.makedirs(dataset_dir / signals_dir, exist_ok=True)

        mic_signals = np.stack([
            librosa.load(mic_path, sr=SR)[0]
            for mic_path in sample["mic_paths"]
        ])

        mic_signals = _trim_signal(mic_signals.T, START_TRIM, END_TRIM,
                                    config["signal_duration_in_seconds"], SR).T

        soundfile.write(dataset_dir / f"{signals_dir}/0.wav", mic_signals.T, SR)

        # for i, mic_signal in enumerate(mic_signals):
        #     if len(mic_signal.shape) > 0:
        #         mic_signal = mic_signal.T # soundfile expects a NxChannels signal
        #     soundfile.write(dataset_dir / f"{signals_dir}/{i}.wav", mic_signal, SR)

        return {
            "room_dims": ROOM_DIMS,
            "source_coordinates": sample["sources"]["source_coordinates"],
            "mic_coordinates": sample["mic_coordinates"],
            # "mic_delays": delays,
            # "mic_sampling_rate_offsets": sr_offsets,
            # "mic_gains": gains,
            "rt60": RT60_IN_SECS,
            "source_gain": 1,
            "signal_duration_in_seconds": config["signal_duration_in_seconds"],
            "mask_delay": False,
            "anechoic": False,
            "snr_in_db": None,
            "signals_dir": signals_dir,
            "sr": SR
        }

    df = Parallel(n_jobs=config["n_jobs"])(
        delayed(process)(sample)
        for sample in tqdm(samples))

    df = pd.DataFrame(df)
    df.to_csv(dataset_dir / "metadata.csv")

def _load_adhoc40_metadata(dataset_dir):
    "Load dataset metadata (such as path to audio signals)"
    samples = []
    for loudspeaker_position in LOUDSPEAKER_POSITIONS:
        loudspeaker_dir = dataset_dir + "/" + loudspeaker_position
        for speaker in os.listdir(loudspeaker_dir):
            speaker_dir = loudspeaker_dir + "/" + speaker
            if not os.path.isdir(speaker_dir):
                continue
            for chapter in os.listdir(speaker_dir):
                chapter_dir = speaker_dir + "/" + chapter
                if not os.path.isdir(chapter_dir):
                    continue
                for sentence in os.listdir(chapter_dir):
                    sentence_dir = chapter_dir + "/" + sentence
                    if not os.path.isdir(sentence_dir):
                        continue

                    samples.append({
                        "path": sentence_dir,
                        "loudspeaker_position": loudspeaker_position,
                        "speaker": speaker,
                        "chapter": chapter,
                        "sentence": sentence
                    })
    return samples

def _trim_signal(signal, start_trim, end_trim, duration=-1, sr=None):
    if sr is not None:
        start_trim = int(sr*start_trim)
        end_trim = int(sr*end_trim)
        duration = int(sr*duration)

    signal = signal[start_trim:-end_trim]

    if duration > 0:
        if signal.shape[0] < duration:
            return None
        signal = signal[:duration]
    return signal

@hydra.main(config_path="config", config_name="adhoc40_dataset")
def main(config: DictConfig):
    print("Sampling dataset...")
    samples = sample_adhoc40_phrases(config)
    print("Saving signals...")
    create_dataset(samples, config)

if __name__ == "__main__":
    main()


import json
import boto3

TABLE_NAME='fayeez-resume-challenge'
dynamodb = boto3.resource('dynamodb')

def lambda_handler(event, context):
    table = dynamodb.Table(TABLE_NAME)
    item = table.get_item(Key={
        'MyPrimaryKey': 'Number of Hits'
    })

    try:
        count = item['Item']['AnalyticsValue'] + 1
        print(count)
    except KeyError as e:
        print('Count key not found, so assuming it is the first hit')
        count = 1

    res = table.put_item(Item={'MyPrimaryKey': 'Number of Hits', 'AnalyticsValue': count})
    res["Count"] = str(count)

    return {
        'statusCode': res["ResponseMetadata"]["HTTPStatusCode"],
        # I needed to pass the headers like this as this is a lambda proxy integration with API gateway.
        # This headers gets passed directly to the client and without it - I get cors issue.
        'headers': {'Access-Control-Allow-Origin': '*'},
        'body': json.dumps({'count': str(count)})
    }


import torch
import torch.nn.functional as F
from torch import nn
import math

class FinalLayer(nn.Module):
    '''
    Final layer to compute distance based on margin and scale.
    Cross Entropy Loss is used for loss function
    '''
    def __init__(self,
                 in_features,
                 out_features,
                 scale=30.0,
                 margin=0.50,
                 easy_margin=False):
        super(FinalLayer, self).__init__()
        # define nput features
        self.in_features = in_features
        # define output features
        self.out_features = out_features
        # define scale
        self.scale = scale
        # define margin
        self.margin = margin
        # define weight
        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))

        # define easy margin for further computation
        self.easy_margin = easy_margin
        # define cosine margin for further computation
        self.cos_margin = math.cos(margin)
        # define sine margin for further computation
        self.sin_margin = math.sin(margin)
        # define theta based on margin for further computation
        self.theta = math.cos(math.pi - margin)
        # define custom margin
        self.custom_margin = math.sin(math.pi - margin) * margin

    def forward(self, input, label):
        '''
        define forwarding layer
        '''
        # compute cosine based on final linear layer
        cosine = F.linear(F.normalize(input), F.normalize(self.weight))
        # compute sine based on computed cosine value
        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))
        # compute phi based on cosine and margins
        phi = cosine * self.cos_margin - sine * self.sin_margin
        # if needed, recompute phi
        phi = torch.where(cosine > self.theta, phi, cosine - self.custom_margin)
        # one hot encoded values
        one_hot = torch.zeros(cosine.size(), device='cuda')
        # resize
        one_hot.scatter_(1, label.view(-1, 1).long(), 1)
        # compute output
        output = ((one_hot * phi) + ((1.0 - one_hot) * cosine))*self.scale
        # return computed output and loss values
        return output, nn.CrossEntropyLoss()(output,label)


import minkowski_geodesic_construction
import schwarzschild_geodesic_construction
from shutil import copyfile
import warnings
import conversion_formulas
import cv2
import os.path
import math
import numpy as np

# Numpy throws a RuntimeWarning when 0 is divided by 0 in the origin. We simply skip this value.
warnings.filterwarnings("ignore", category=RuntimeWarning)

def crop_to_largest_central_square(input_path):
    Image = cv2.imread(input_path)
    height = Image.shape[0]
    width = Image.shape[1]
    if height >= width:
        new_height = width
        new_width = width
    else:
        new_height = height
        new_width = height
    start_width = round((width - new_width) / 2)
    start_height = round((height - new_height) / 2)
    cropped_image = Image[start_height:start_height + new_height, start_width:start_width + new_width]

    extension = os.path.splitext(input_path)[1]
    cropped_path = input_path.replace(str(extension), '_cropped' + str(extension))

    cv2.imwrite(cropped_path, cropped_image)
    return cropped_image, cropped_path

def blacken_output_image(output_image, output_path):
    x = output_image.shape[0]
    y = output_image.shape[1]
    for i in range(0, x):
        for j in range(0, y):
            output_image[i, j] = [0, 0, 0]
    cv2.imwrite(output_path, output_image)
    return x, y, output_image

def sphere_circle(output_path):
    output_image = cv2.imread(output_path)
    width = output_image.shape[0]
    center_height = math.floor(width / 2)
    extension = os.path.splitext(output_path)[1]
    new_path = output_path.replace(str(extension), '_made_into_sphere' + str(extension))
    for i in range(math.floor(width / 2.0), width):
        if np.any(output_image[center_height, i] != 0):
            radius = i - math.floor(width / 2.0)
            output_image = cv2.circle(output_image, (round(width / 2), round(width / 2)), round(1.3 * radius),
                                      (0, 0, 0), -1)
            cv2.imwrite(new_path, output_image)
            # cv2.imshow('image', output_image)
            # cv2.waitKey(0)
            break
        # print(output_image[i, center_height])

def create_transformed_minkowski_image(dt, t_end, input_path):
    title = input_path.split("//", 1)
    extension = os.path.splitext(input_path)[1]
    output_path = 'output_transformable_images//' + title[1]
    output_path = output_path.replace(str(extension), '_transformed' + str(extension))
    copyfile(input_path, output_path)
    input_image = cv2.imread(input_path)
    output_image = cv2.imread(output_path)
    x_size, y_size, output_image = blacken_output_image(output_image, output_path)
    for y in range(0, y_size):
        for x in range(0, x_size):
            x_start = -10 + 20 * (x / x_size)
            y_start = 10 - 20 * (y / y_size)
            z_start = -10
            try:
                r, theta, phi = minkowski_geodesic_construction.create_specified_geodesic(
                    x_start, y_start, z_start, dt, t_end)
                position_x_end_transposed, position_y_end_transposed, position_z_end_transposed = \
                    conversion_formulas.spherical_to_cartesian(r[-1], theta[-1], phi[-1])
                position_x_end = round((position_x_end_transposed + 10) * x_size / 20)
                position_y_end = round((position_y_end_transposed - 10) * y_size / -20)
                R, G, B = input_image[position_y_end][position_x_end]
            except ZeroDivisionError:
                continue
            # CV2 apparently reverses the order of row and columns which is super weird but ok
            output_image[y, x] = R, G, B
            cv2.imshow('image', output_image)
            cv2.waitKey(1)
    cv2.imwrite(output_path, output_image)
    cv2.imshow('image', output_image)
    cv2.waitKey(0)

def create_transformed_schwarzschild_image(input_path, dt, t_end, r_s, Z_end):
    black_radius = 0
    title = input_path.split("//", 1)
    extension = os.path.splitext(input_path)[1]
    output_path = 'output_transformable_images//' + title[1]
    output_path = output_path.replace(str(extension), '_transformed' + '_r_s='+str(r_s) + '_Z_end=' + str(Z_end)
                                      + str(extension))
    input_image, input_path = crop_to_largest_central_square(input_path)
    copyfile(input_path, output_path)
    output_image = cv2.imread(output_path)
    y_size, x_size, output_image = blacken_output_image(output_image, output_path)
    print(y_size, x_size)
    for y in range(0, y_size):
        for x in range(0, x_size):
            x_start = -10 + 20 * (x / x_size)
            y_start = 10 - 20 * (y / y_size)
            z_start = -10
            try:
                results = schwarzschild_geodesic_construction. \
                    create_specified_geodesic(x_start, y_start, z_start, dt, t_end, r_s, Z_end)
                if results is not None:
                    r, theta, phi = results
                    position_x_end_transposed, position_y_end_transposed, position_z_end_transposed = \
                        conversion_formulas.spherical_to_cartesian(r[-1], theta[-1], phi[-1])
                    position_x_end = round((position_x_end_transposed + 10) * x_size / 20)
                    position_y_end = round((position_y_end_transposed - 10) * y_size / -20)
                    if (position_x_end < 0 or position_x_end >= x_size) or (
                            position_y_end < 0 or position_y_end >= y_size):
                        continue
                    # CV2 apparently reverses the order of row and columns which is super weird but ok
                    R, G, B = input_image[position_y_end][position_x_end]
                    output_image[y, x] = R, G, B
                else:
                    distance_center_x = abs(x - x_size / 2)
                    distance_center_y = abs(y - y_size / 2)
                    if distance_center_x != x_size / 2.0 and distance_center_y != y_size / 2.0:
                        radius = math.sqrt(distance_center_x ** 2 + distance_center_y ** 2)
                        if radius > black_radius and distance_center_x != 50.0:
                            black_radius = radius
                    continue
            except ZeroDivisionError:
                continue
            except IndexError:
                continue
            except ValueError:
                continue
            cv2.imshow('image', output_image)
            cv2.waitKey(1)
        cv2.imwrite(output_path, output_image)
    # For some reason the solver skips the first geodesic, (I have no idea why), this calculates it specifially again
    r, theta, phi = schwarzschild_geodesic_construction.create_specified_geodesic(-10, 10, -10, dt, t_end, r_s, Z_end)
    position_x_end_transposed, position_y_end_transposed, position_z_end_transposed \
        = conversion_formulas.spherical_to_cartesian(r[-1], theta[-1], phi[-1])
    position_x_end = round((position_x_end_transposed + 10) * x_size / 20)
    position_y_end = round((position_y_end_transposed - 10) * y_size / -20)
    R, G, B = input_image[position_x_end][position_y_end]
    output_image[0, 0] = R, G, B
    output_image = cv2.circle(output_image, (round(x_size / 2), round(y_size / 2)), math.ceil(1.3*black_radius), (0, 0, 0),
                              -1)
    sphere_circle(output_path)
    cv2.imwrite(output_path, output_image)
    # cv2.imshow('image', output_image)
    # cv2.waitKey(0)
    return x_size, input_path, output_path

# sphere_circle('output_transformable_images//hubble.jpg')
# sphere_circle('output_transformable_images//600x600_space.jpg')
# sphere_circle('output_transformable_images//605_479_colored_space.jpg')
# sphere_circle('output_transformable_images//747x800_stars.jpg')
# sphere_circle('output_transformable_images//1180x884_space.jpg')
# sphere_circle('output_transformable_images//blue_space.jpg')
# sphere_circle('output_transformable_images//nebula.jpg')
# sphere_circle('output_transformable_images//whatever_this_is.jpg')
# sphere_circle('output_transformable_images//100x100_car_distorted_r_s=0.4_Z_end=20.jpg')


import io
import os.path

from PIL import ImageDraw, ImageFont

def make(data, image, config, logger):
    image_size = image.size
    width = config["img_width"]
    height = int(width * image.size[1] / image.size[0])
    image = image.resize((width, height))

    draw = ImageDraw.ImageDraw(image)

    font_path = os.path.join(config["static_dir"], config["font"])
    font_size = int(data["font_size"])
    draw.font = ImageFont.truetype(font_path, font_size)

    lines = data["quote"].splitlines()
    x = int(data["quote_x"]) + len(lines) * (font_size * 0.6)
    y = int(data["quote_y"])

    for line in lines:
        draw.text((x, y), line, fill="black", direction="ttb")
        x -= font_size + 4

    file = io.BytesIO()
    image.save(file, "png")
    file.seek(0)

    return file