#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Aug  9 16:57:18 2021

@author: linkv
"""

import tskit
import matplotlib.pyplot as plt
import TPhenotypes as pt
import TAssociationTesting as gwas
import TVariants as tvar
import TIndividuals as tind
import TSimulator as tsim
import datetime
import argparse
import logging
import os
import sys
import numpy as np

os.chdir(os.path.dirname(sys.argv[0]))

from numpy.random import RandomState
class randomGenerator:
    def __init__(self, seed):
        self.seed = seed
        self.random = RandomState(seed)

tim = datetime.datetime.now()
seed = tim.hour*10000+tim.minute*100+tim.second
r = randomGenerator(seed)
r.random.get_state()[1][0]

logger = logging.getLogger()

#if you check end of last tree you still get 249,250,621, the length of chr1 in hg19....
# trees.aslist()[trees.num_trees - 1].interval.right
#A site defines a particular location along the genome in which we are interested in observing the allelic state. So I guess that means sites are only defined where there are mutations
# print(trees.tables.sites) #the position of the last site is very close to the end of the interval we kept above

#-----------------------
# Simulate
#-----------------------

simulator = tsim.TSimulatorStdPopsim()
trees = simulator.run_simulation("default", r)

#-----------------------
# Read trees
#-----------------------

directory = "/data/ARGWAS/experiments_N500/"

trees = tskit.load(directory + "test_2.trees")

samp_ids = trees.samples()
N = len(samp_ids)

trees_relate = tskit.load(directory + "relate_test_2_propTyped1/test_2_propTyped1.trees")
variants = tvar.TVariantsFiltered(trees, samp_ids, 0, 1, prop_typed_variants = 1, pos_int = True, random = r, logfile = logger,
                                  filtered_variants_file = directory + "/relate_test_2_propTyped1/test_2_propTyped1_filtered_sample_variants.csv")
print(variants.num_typed)
len(list(trees_relate.variants(samples=samp_ids)))

trees_relate = tskit.load(directory + "relate_test_2_propTyped0.01/test_2_propTyped0.01.trees")
variants = tvar.TVariantsFiltered(trees, samp_ids, 0, 1, prop_typed_variants = 1, pos_int = True, random = r, logfile = logger,
                                  filtered_variants_file = directory + "relate_test_2_propTyped0.01/test_2_propTyped0.01_filtered_sample_variants.csv")
print(variants.num_typed)
len(list(trees_relate.variants(samples=samp_ids)))

trees_relate = tskit.load(directory + "relate_test_2_propTyped0.05/test_2_propTyped0.05.trees")
variants = tvar.TVariantsFiltered(trees, samp_ids, 0, 1, prop_typed_variants = 1, pos_int = True, random = r, logfile = logger,
                                  filtered_variants_file = directory + "relate_test_2_propTyped0.05/test_2_propTyped0.05_filtered_sample_variants.csv")
print(variants.num_typed)
len(list(trees_relate.variants(samples=samp_ids)))

#-----------------------
# simulate pheno based on all variants of a tree
#-----------------------
causal_tree = trees.at(49500000)
len(list(causal_tree.mutations()))
for m in causal_tree.mutations():
    print(m)

for m in causal_tree.sites():
    print(m)

#-----------------------
# create diploids and variants
#-----------------------

inds = tind.Individuals(2, N)
variants = tvar.TVariantsFiltered(trees, samp_ids, 0.01, 1, prop_typed_variants = 1, pos_int = True, random = r, logfile = logger)
variants.num_typed

#see variants from tree directly
list(trees.variants(samples=samp_ids))[0]
len(list(trees.variants(samples=samp_ids)))

variants_relate = tvar.TVariantsFiltered(trees_relate, samp_ids, 0.01, 1, prop_typed_variants = 1, pos_int = True, random = r, logfile = logger)
len(list(trees_relate.variants(samples=samp_ids)))
variants_relate.num_typed
#-----------------------
# GRM
#------------------------
M_sum = np.zeros(shape=(10000, 10000))

for v in range(100):
    # gt = np.array([1,1,0,1,1,1,1,0,0,0,1,1,0,1,1,1,1,0,0,0,1,1,0,1,1,1,1,0,0,0,1,1,0,1,1,1,1,0,0,0,1,1,0,1,1,1,1,0,0,0])
    gt = np.random.binomial(1, 0.1, size=10000)
    af = np.sum(gt) / len(gt)
    first = np.array([gt - af]).T
    second = np.array([gt - af])
    M = np.dot(first, second)
    M = M / (af * (1 - af))
    len(gt)
    M_sum += (M / (100))
np.trace(M_sum)

#--------------------------
# create phenotypes
#-----------------------

# phenotypes with genetic influence
sd_environmental_noise = 1
prop_causal_mutations = 0.002 #this is only for variants found in sampled haplotypes
sd_beta_causal_mutations = 1
pheno_unif = pt.Phenotypes("uniform distr. of causal SNPs", variants, N)
pheno_unif.simulate_env_noise_sd(sd_environmental_noise, r)
pheno_unif.simulate_uniform(variants, prop_causal_mutations=prop_causal_mutations, sd_beta_causal_mutations=sd_beta_causal_mutations, random=r)

# phenotypes with genetic influence, no noise
pheno_unif_noNoise = pt.Phenotypes("uniform distr., no noise", variants, N)
pheno_unif_noNoise.simulate_fixed(pheno_unif.causal_variants, pheno_unif.causal_variant_indeces, pheno_unif.causal_betas)
pheno_unif_noNoise.find_causal_trees(trees)

# random phenotypes
sd_environmental_noise = 1
pheno_random = pt.Phenotypes("random", variants, N)
pheno_random.simulate_env_noise_sd(sd_environmental_noise, r)

# fixed causal variant
sd_environmental_noise = 0
pheno_fixed = pt.Phenotypes("fixed beta -0.67, no noise", variants, N)
pheno_fixed.simulate_fixed([pheno_unif.causal_variants[2]], pheno_unif.causal_variant_indeces[2], [-0.67])

# fixed causal variant with high allele freq
sd_environmental_noise = 0
index = np.where(variants.allele_frequencies > 0.4)[0][int(np.floor(len(np.where(variants.allele_frequencies > 0.4)[0]) / 2))]
pheno_fixed_hp = pt.Phenotypes("fixed high freq beta -0.67, no noise", variants, N)
pheno_fixed_hp.simulate_fixed([variants.variants[index]], index, [-0.67])

# fixed causal variant with high allele freq with noise
sd_environmental_noise = 1
pheno_fixed_hp_wn = pt.Phenotypes("fixed high freq beta -0.67, with noise", variants, N)
pheno_fixed_hp_wn.simulate_env_noise_sd(sd_environmental_noise, r)
pheno_fixed_hp_wn.simulate_fixed([variants.variants[index]], index, [-0.67])

#-----------------------
# run association tests and plot
#-----------------------

pGWAS_unif = gwas.TAssociationTestingGWAS(phenotypes=pheno_unif)
pGWAS_unif.OLS(variants)
pGWAS_unif_noNoise = gwas.TAssociationTestingGWAS(phenotypes=pheno_unif_noNoise)
pGWAS_unif_noNoise.OLS(variants)

pGWAS_random = gwas.TAssociationTestingGWAS(phenotypes=pheno_random)
pGWAS_random.OLS(variants)

pGWAS_fixed = gwas.TAssociationTestingGWAS(phenotypes=pheno_fixed)
pGWAS_fixed.OLS(variants)

pGWAS_fixed_hp = gwas.TAssociationTestingGWAS(phenotypes=pheno_fixed_hp)
pGWAS_fixed_hp.OLS(variants)
pGWAS_fixed_hp_wn = gwas.TAssociationTestingGWAS(phenotypes=pheno_fixed_hp_wn)
pGWAS_fixed_hp_wn.OLS(variants)

fig, ax = plt.subplots(7,figsize=(30,30))
pGWAS_unif.manhattan_plot(variants.positions, ax[0])
# ax[0].axhline(y=30, color="black", lw=0.5)

pGWAS_unif_noNoise.manhattan_plot(variants.positions, ax[1])

pGWAS_random.manhattan_plot(variants.positions, ax[2])

pGWAS_fixed.manhattan_plot(variants.positions, ax[3])

pGWAS_fixed_hp.manhattan_plot(variants.positions, ax[4])

pGWAS_fixed_hp_wn.manhattan_plot(variants.positions, ax[5])

pGWAS_fixed_hp.manhattan_plot_subset(variants.positions, ax[6], pheno_fixed_hp.causal_variant_indeces-200, pheno_fixed_hp.causal_variant_indeces+200, size=1.5)

fig.tight_layout()
fig.set_size_inches(30, 30)
fig.show()
fig.savefig('sims/sims_13_africans.png', bbox_inches='tight')#

#-----------------------
# Mantel
#-----------------------
pheno_fixed_hp.find_causal_trees(trees)
tGWAS_fixed_hp = gwas.TAssociationTestingRegions(trees, pheno_fixed_hp)
tGWAS_fixed_hp.runMantel(trees, pheno_fixed, N)

fig, ax = plt.subplots(2,figsize=(30,30))
pGWAS_fixed_hp.manhattan_plot(variants.positions, ax[0])
tGWAS_fixed_hp.manhattan_plot(range(trees.num_trees), ax[1])

fig.tight_layout()
fig.set_size_inches(30, 30)
fig.show()
fig.savefig('sims/sims_13_randomSeq.png', bbox_inches='tight')#

#-----------------------
# GCTA HE
#-----------------------

pheno_fixed_hp.find_causal_trees(trees)
tGWAS_fixed_hp = gwas.TAssociationTestingRegions(trees, pheno_fixed_hp)
tGWAS_fixed_hp.runCGTA_HE(trees, N)

fig, ax = plt.subplots(5,figsize=(30,30))
pGWAS_fixed_hp.manhattan_plot(variants.positions, ax[0])
tGWAS_fixed_hp.manhattan_plot_special_pvalues(range(trees.num_trees), tGWAS_fixed_hp.p_values_HECP_Jackknife, ax[1], title_supplement = "HECP_Jackknife")
tGWAS_fixed_hp.manhattan_plot_special_pvalues(range(trees.num_trees), tGWAS_fixed_hp.p_values_HECP_OLS, ax[2], title_supplement = "HECP_OLS")
tGWAS_fixed_hp.manhattan_plot_special_pvalues(range(trees.num_trees), tGWAS_fixed_hp.p_values_HESD_Jackknife, ax[3], title_supplement = "HESD_Jackknife")
tGWAS_fixed_hp.manhattan_plot_special_pvalues(range(trees.num_trees), tGWAS_fixed_hp.p_values_HESD_OLS, ax[4], title_supplement = "HESD_OLS")

fig.tight_layout()
fig.set_size_inches(30, 30)
fig.show()
fig.savefig('sims/sims_16_HE.png', bbox_inches='tight')#

# start = time.time()
# tGWAS_fixed_hp_test = gwas.TtGWAS(trees, pheno_fixed_hp)
# tree = trees.at_index(1141)
# tGWAS_fixed_hp_test.runCGTA_HE_one_tree(trees.at_index(1141), N, start)

# tGWAS_fixed_hp_test.runCGTA_HE(trees, N)

# fig, ax = plt.subplots(5,figsize=(30,30))
# pGWAS_fixed_hp.manhattan_plot(variants.positions, ax[0])
# tGWAS_fixed_hp_test.manhattan_plot_special_pvalues(range(trees.num_trees), tGWAS_fixed_hp_test.p_values_HECP_Jackknife, ax[1], title_supplement = "HECP_Jackknife")
# tGWAS_fixed_hp_test.manhattan_plot_special_pvalues(range(trees.num_trees), tGWAS_fixed_hp_test.p_values_HECP_OLS, ax[2], title_supplement = "HECP_OLS")
# tGWAS_fixed_hp_test.manhattan_plot_special_pvalues(range(trees.num_trees), tGWAS_fixed_hp_test.p_values_HESD_Jackknife, ax[3], title_supplement = "HESD_Jackknife")
# tGWAS_fixed_hp_test.manhattan_plot_special_pvalues(range(trees.num_trees), tGWAS_fixed_hp_test.p_values_HESD_OLS, ax[4], title_supplement = "HESD_OLS")

# fig.tight_layout()
# fig.set_size_inches(30, 30)
# fig.show()
# fig.savefig('sims/sims_15_HE.png', bbox_inches='tight')#

#-----------------------
# GCTA REML
#-----------------------

pheno_fixed_hp.find_causal_trees(trees)
tGWAS_fixed_hp_reml = gwas.TAssociationTestingRegions(trees, pheno_fixed_hp)
tGWAS_fixed_hp_reml.runGCTA_REML(trees, N)

fig, ax = plt.subplots(5,figsize=(30,30))
tGWAS_fixed_hp_reml.manhattan_plot(variants.positions, ax[0])
tGWAS_fixed_hp_reml.manhattan_plot_special_pvalues(range(trees.num_trees), tGWAS_fixed_hp_reml.p_values_HECP_Jackknife, ax[1], title_supplement = "HECP_Jackknife")
tGWAS_fixed_hp_reml.manhattan_plot_special_pvalues(range(trees.num_trees), tGWAS_fixed_hp_reml.p_values_HECP_OLS, ax[2], title_supplement = "HECP_OLS")
tGWAS_fixed_hp_reml.manhattan_plot_special_pvalues(range(trees.num_trees), tGWAS_fixed_hp_reml.p_values_HESD_Jackknife, ax[3], title_supplement = "HESD_Jackknife")
tGWAS_fixed_hp_reml.manhattan_plot_special_pvalues(range(trees.num_trees), tGWAS_fixed_hp_reml.p_values_HESD_OLS, ax[4], title_supplement = "HESD_OLS")

fig.tight_layout()
fig.set_size_inches(30, 30)
fig.show()
fig.savefig('sims/sims_15_HE.png', bbox_inches='tight')#

# #-----------------------
# # limix
# #-----------------------

# trees_obj = tt.TTrees(trees)

# def solving_function(array):
#     covariance = trees_obj.TMRCA(trees_obj.trees[0], N)
#     inv = np.linalg.inv(covariance)
#     tmp = np.dot(inv, array)
#     # print("shape of my dot product",np.shape(tmp))
#     return(tmp)

# y = pheno_random.y
# k = 1
# m = 2
# E = random.randn(N,k)
# N = 500
# # F = sp.concatenate([sp.ones((N,1)), random.randn(N,1)], 1)
# F = sp.zeros(N)

# lmm = LMMCore(y.reshape(N,1), F.reshape(N,1), solving_function)

# S = trees_obj.trees[0].interval.right - trees_obj.trees[0].interval.left #this produces a float, but we want number of sites. there is a sites iterator, but dont know how to use it to find number of sites. why are genomic positions floats?
# G = 0.*(random.rand(N,1)<0.2)
# Inter = random.randn(N, m)

# lmm.process(G, Inter) #this needs step param to produce only one p-value per tree. for this i need the number of sites per tree, or just use 1?
# pv = lmm.getPv()
# beta = lmm.getBetaSNP()
# beta_ste = lmm.getBetaSNPste()
# lrt = lmm.getLRT() #likelihood ratio

# tGWAS_fixed = gwas.TtGWAS(trees, pGWAS_fixed)
# F = np.zeros(N)
# tGWAS_fixed.runLimix(trees, N, pheno_fixed.y.reshape(N,1), F.reshape(N,1), random)

#-----------------------
# plot p-values
#-----------------------

# num_bins = 20
# fig, ax = plt.subplots(4,figsize=(15,15))
# # fig, ax = plt.subplots(1,figsize=(15,15))

# pGWAS_unif.p_value_dist(ax[0], num_bins)

# pGWAS_unif_noNoise.p_value_dist(ax[1], num_bins)

# pGWAS_random.p_value_dist(ax[2], num_bins)
# pGWAS_random.chiSquared(num_bins)

# pGWAS_fixed.p_value_dist(ax[3], num_bins)

# fig.tight_layout()
# fig.set_size_inches(10, 20)
# fig.savefig('sims_pvalues_random_pt_2.png', bbox_inches='tight')#

#-----------------------
# plot p-values for random
#-----------------------
# num_bins = 20
# fig, ax = plt.subplots(5,figsize=(15,15))

# for i in range(5):
#     sd_environmental_noise = 1
#     pheno_random = pt.Phenotypes("random",trees)
#     pheno_random.simulateEnvNoise(sd_environmental_noise)
#     pGWAS_random = gwas.TpGWAS(ts_object=trees, phenotypes=pheno_random)
#     pGWAS_random.OLS()

#     x= pGWAS_random.chiSquared(num_bins)
#     pGWAS_random.p_value_dist(ax[i], num_bins)
#     ax[i].set(title=x)

#     fig.tight_layout()
#     fig.set_size_inches(10, 20)
# fig.savefig('sims_pvalues_random_pt.png', bbox_inches='tight')#

# #cumulative p-value distribution
# c_steps = np.arange(0, 1, 0.005)
# c_probs =np.empty(len(c_steps))
# fig, ax = plt.subplots(1,figsize=(15,15))
# for s,c in enumerate(c_steps):
#     c_probs[s] = len(pGWAS_random.p_values[pGWAS_random.p_values <c]) / len(pGWAS_random.p_values)
#     # print(str(c) + ": " + str(len(pGWAS_random.p_values[pGWAS_random.p_values <c]) / len(pGWAS_random.p_values)))
# ax.scatter(c_steps, c_probs)
# ax.set(xlabel='cutoff', ylabel='P(p-value < cutoff)', title="cumulative dist. p-values")
# fig.savefig('sims_pvalues_random_cumulative.png', bbox_inches='tight')#

# fig, ax = plt.subplots(10,figsize=(15,30))
# for i,index in enumerate(range(20000,21000, 100)):
#     pGWAS_random.manhattan_plot_subset(variants.positions, ax[i], index, index+100)
#     ax[i].axhline(y=8, color="black", lw=0.5)
# fig.savefig('manhattan_zooms_random.png', bbox_inches='tight')#

#-----------------------
# test if more causal SNPs bring down p-values
#-----------------------

# # allele_freq = [0.005,0.01, 0.02, 0.05, 0.1, 0.2,0.5]
# num_freqs = 5
# fig, ax = plt.subplots(num_freqs,figsize=(15,15))

# offset = np.floor(len(variants.allele_frequencies)/num_freqs)
# ordered_allele_freq = np.argsort(variants.allele_frequencies)
# for i in range(num_freqs):
#     index = int(offset + i*offset)
#     variant = list(trees.variants(samples=samp_ids))[ordered_allele_freq[index]]
#     freq = variants.allele_frequencies[ordered_allele_freq[index]]
#     pheno = pt.Phenotypes("allele freq=" + str(freq) + ", beta=0.79, with noise" , trees)
#     pheno.simulateEnvNoise(sd_environmental_noise=1)
#     pheno.simulateFixed([variant], [0.79])

#     pGWAS = gwas.TpGWAS(ts_object=trees, phenotypes=pheno)
#     pGWAS.OLS()

#     pGWAS.manhattan_plot(variants.variant_positions, ax[i])

# fig.tight_layout()
# fig.set_size_inches(10, 20)
# fig.show()
# fig.savefig('sims_alleleFreq_africans_withNoise.png', bbox_inches='tight')#

# #
# #
# # y = np.random.normal(scale=environmental_noise_sd, size=N)

# # #add phenotypic effect to mutations that are uniformly distributed
# # betas = [0] * num_variants
# # causal_positions = []
# # variant_positions = []
# # for v, var in enumerate(trees.variants(samples=samp_ids)):

# #     ## Debugging:
# #     ##------------
# #     # print(var.site.mutations[0].node) # the site ID goes from 0 to len(list(trees.variants(samples=samp_ids)))-1
# #     # break

# #     # for v in trees.variants(samples=samp_ids):
# #     #     print(v)

# #     # len(list(trees.variants(samples=samp_ids)))

# #     variant_positions.append(var.site.position)

# #     #causal mutation
# #     r = random.uniform(0,1)
# #     if(r < causal_mutations_prop):

# #         #define beta
# #         beta = random.normalvariate(0, causal_mutation_beta_sd)
# #         betas[v] = beta

# #         #simulate phenotype
# #         y[var.genotypes == 1] += beta

# #         #save causal position
# #         causal_positions.append(var.site.position)

# #-------------------
# # associating
# #-------------------

# #test for associations
# # diffs = ut.diff(y, N)
# # p_variants = []
# # for v in trees.variants(samples=samp_ids):
# #   p_variants.append(sm.OLS(v.genotypes, y).fit().pvalues[0])

# p_trees = []
# for tree in trees.trees():
#     if tree.total_branch_length == 0:
#         continue
#     tmrca = ut.TMRCA(tree, N)
#     p_trees.append(ut.mantel(ut.make(tmrca), diffs))

# # #--------------------
# # # manhattan plot
# # #--------------------
# # q_variants = -np.log10(p_variants)
# # plt.scatter(variant_positions, q_variants, s=0.5)
# # for pos in causal_positions:
# #     plt.axvline(x=pos, color="red", lw=0.5)
# # plt.show()

# #---------------------
# # plot ROC curve
# #---------------------
# """
# https://docs.eyesopen.com/toolkits/cookbook/python/plotting/roc.html
# """

# # if(len(p_variants) != len(causal_positions) != len(p_trees) != len(causal_tree_indeces)):
# #     raise ValueError("lengths of p-values and causal positions/trees are not identical. Cannot create ROC curve.")

# TPR = []
# FPR = []
# cutoffs = np.linspace(start=1, stop=0, num=1000)
# for c in cutoffs:

#     #variants
#     true_positives = np.where((np.array(betas) != 0) & (np.array(p_variants) < c))[0] #predicted to be positive and the actual value is also positive
#     TPR.append(float(len(true_positives)) / float(len(causal_positions)))

#     false_negatives = np.where((np.array(betas) != 0) & (np.array(p_variants) >= c))[0] # predicted to be negative but the actual value is positive
#     FPR.append(float(len(false_negatives)) / float(num_variants - len(causal_positions)))

# plt.plot(FPR,TPR)
# plt.ylabel("FPR")
# plt.xlabel("TPR")
# plt.show()

# df = pd.DataFrame(columns = ["FPR", "TPR"])
# df["FPR"] = FPR
# df["TPR"] = TPR

# df.to_csv("ROC.csv",sep=",", header=True, index=False)
# """
# all inds have phenotye = 0 + noise

# - for each mutation:
#     - choose a beta from N(0,sigma) with certain prob given by prop * 1/ num mutations
#     - check which individuals have the mutation
#     - add the beta to all inds with the mutation
#     - save mutation and the beta associated
#     - keep track of which tree the mutation is located in?

# - for each SNP:
#     - perform OLS

# - for each tree:
#     - perform Mantel

# - plot ROC curve?:
#     - need true positives and false negatives etc.: need to keep track of which trees contain causal mutation

# """

# N = trees.num_samples
# M = trees.num_mutations
# y = np.random.normal(scale=1, size=N)


import torch
from torch_sparse import SparseTensor, SparseStorage
from typing import Tuple

def sprand(dim: Tuple[int, int], nnz: int) -> SparseTensor:
    nu, nv = dim
    row = torch.randint(nu, (nnz,))
    col = torch.randint(nv, (nnz,))

    storage = SparseStorage(row=row, col=col, sparse_sizes=(nu, nv), is_sorted=False)
    storage = storage.coalesce(reduce="max")

    return SparseTensor.from_storage(storage)


from flask import Flask, jsonify, request
from modules.youtube_db import PostSearch
from raven.contrib.flask import Sentry
app = Flask(__name__)

sentry = Sentry(app)

# @app.route('/api/v1/search/<string:q>')
@app.route('/api/v1/search')
def search():
    query = request.args.get('q')
    youtube_api = PostSearch()
    result = youtube_api.db_check(query)
    return jsonify(result)

if __name__ == '__main__':
    app.run(port=5017)


from __future__ import (absolute_import, division,
                        print_function, unicode_literals)
import os
import boto
from boto.s3.key import Key
from future.builtins import (  # noqa
    bytes, dict, int, list, object, range, str,
    ascii, chr, hex, input, next, oct, open,
    pow, round, super,
    filter, map, zip)
from nose.plugins.attrib import attr
from nose.tools import eq_
from orchestrator.cluster_config.s3 import S3ConfigProvider
from tests.helper import dict_compare
from tests.integration.orchestrator.cluster_config import MOCK_CONFIG, \
    MOCK_SERIALIZED_CONFIG

__author__ = 'sukrit'

S3_TEST_BUCKET = os.getenv('S3_TEST_BUCKET', 'totem-integration')
S3_CONFIG_BASE = 'totem-%s/config' % (os.getenv('USER'))

@attr(s3='true')
class TestS3ConfigProvider:
    """
    Integration tests for S3ConfigProvide
    """

    @classmethod
    def teardown_class(cls):
        bucket = boto.connect_s3().get_bucket(S3_TEST_BUCKET)
        for key in bucket.list(prefix=S3_CONFIG_BASE):
            key.delete()

    def setup(self):
        self.provider = S3ConfigProvider(S3_TEST_BUCKET,
                                         config_base=S3_CONFIG_BASE)

    def test_write(self):
        """
        should write config to s3
        """

        # When: I write config using provider
        self.provider.write('totem.yml', MOCK_CONFIG, 'cluster1', 'test_write')

        # Then: Config gets serialized as yaml and written to s3
        key = self.provider._s3_bucket().get_key(
            '/%s/cluster1/test_write/totem.yml' % (S3_CONFIG_BASE))
        eq_(key is not None, True)
        eq_(key.get_contents_as_string().decode(), MOCK_SERIALIZED_CONFIG)

    def test_load(self):
        """
        Should read config from s3
        """

        # Given: Existing config
        key = Key(self.provider._s3_bucket())
        key.key = '/%s/cluster1/test_load/totem.yml' % (S3_CONFIG_BASE)
        key.set_contents_from_string(MOCK_SERIALIZED_CONFIG)

        # When: I load config using provider
        ret_value = self.provider.load('totem.yml', 'cluster1', 'test_load')

        # Then: Config gets loaded
        dict_compare(ret_value, MOCK_CONFIG)

    def test_delete(self):
        """
        Should delete config from s3
        """

        # Given: Existing config
        key = Key(self.provider._s3_bucket())
        key.key = '/%s/cluster1/test_delete/totem.yml' % (S3_CONFIG_BASE)
        key.set_contents_from_string(MOCK_SERIALIZED_CONFIG)

        # When: I load config using provider
        ret_value = self.provider.delete('totem.yml', 'cluster1',
                                         'test_delete')

        # Then: Config gets loaded
        eq_(ret_value, True)
        check_key = self.provider._s3_bucket().get_key(key.key)
        eq_(check_key is None, True)


from models.model import Model
from tqdm import tqdm
import numpy as np
import matplotlib.pyplot as plt
import cv2
from lib.ops import *

class Exponential(Model):
    def __init__(self, **params):
        super(Exponential, self).__init__()
        self.name = "Exponential"
        self.alpha = params.get('lambda', 100)
        self.eps = params.get('eps', 1e-5)
        self.maxiter = params.get('maxiter', 1500)
        self.kappa = params.get('kappa', 1)
        self.dt = params.get('dt', 0.01)
        self.accuracy = []
        self.loss_hist = []
        self.save_path = "dataset/"+self.name+"_"+str(self.maxiter)+".pkl"

    def loss(self):
        ux, uy = gradient(self.u)
        self.loss_hist.append(np.sum(1-np.exp(-1*self.kappa*(ux*ux + uy*uy)) + self.mask*(self.u - self.noisy)**2))

    def solve(self, img, noisy, mask):
        self.img = img
        self.noisy = noisy
        self.mask = mask

        if noisy.ndim==3:
            M,N,C = noisy.shape
        else:
            M,N = noisy.shape
            C = 1

        self.u = noisy.copy()
        self.gen_metrics()
        self.loss()
        delta = 1e-8
        for iter in tqdm(range(0,self.maxiter)):
            ux = derivative(self.u,dir=0,type="forward")
            uy = derivative(self.u,dir=1,type="forward")
            exp = np.exp(-1*self.kappa*(ux*ux + uy*uy))
            uxx = derivative((ux*exp), dir=0, type="backward")
            uyy = derivative((uy*exp), dir=1, type="backward")

            reg = self.kappa * (uxx + uyy)
            u_dash = self.u + self.dt*(reg+self.l2fidelity())

            diff_u = np.linalg.norm(u_dash-self.u/np.linalg.norm(u_dash))
            self.u = u_dash
            self.u[self.u > 1] = 1.0
            self.u[self.u < 0] = 0.0
            self.loss()
            self.gen_metrics()
            if iter%self.update_every == 0:
                self.update_figure()
            if diff_u<self.eps:
                break

        self.save_plots()
        return self.u


from django.conf.urls import patterns, include, url

from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'diloo.views.home', name='home'),
    # url(r'^blog/', include('blog.urls')),

    url(r'^admin/', include(admin.site.urls)),
    url(r'^$','app.views.main', name="main"),
    url(r'^feed$','app.views.feed', name="feed"),
    url(r'^login$', 'app.views.login', name="login"),
    url(r'^register$', 'app.views.register', name="register"),
    url(r'^logout$', 'app.views.logout', name="logout"),
    url(r'^profile/(?P<username>\w+)$', 'app.views.profile', name="profile"),
    url(r'^p/(?P<username>\w+)$', 'app.views.profile', name="short_profile"),
    url(r'^u$', 'app.views.user_profile', name="user_profile"),
    url(r'^u/edit$', 'app.views.user_profile_configuration', name="user_profile_configuration"),
    url(r'^s$', 'app.views.search', name="search_short"),
    url(r'^search$', 'app.views.search', name="search"),
    url(r'^d/(?P<number>\d+)$', 'app.views.review', name="review"),
    url(r'^r/u$', 'app.views.review_upload', name="review_upload"),
    url(r'^r/h$', 'app.views.review_heart', name="review_heart"),
    url(r'^r/h/d$', 'app.views.review_heart_delete', name="review_heart_delete"),
    url(r'^ur$', 'app.views.ur', name="ur"),
    url(r'^categories$', 'app.views.categories', name="categories"),
    url(r'^categories/(?P<name>\w+)$', 'app.views.categories_search', name="categories_search"),
    url(r'^follow$', 'app.views.follow', name="follow"),
    url(r'^unfollow$', 'app.views.unfollow', name="unfollow"),
    url(r'^embed/(?P<number>\d+)$', 'app.reviews.embed_review', name="embed_review"),
)


import json
import os
import smart_open
import logging

from airflow.models import BaseOperator
from airflow.hooks.S3_hook import S3Hook
from airflow.hooks.base_hook import BaseHook
from airflow.utils.decorators import apply_defaults

from mssql_plugin.hooks.astro_mssql_hook import AstroMsSqlHook

class MsSQLToS3Operator(BaseOperator):
    """
    MsSQL to S3 Operator

    NOTE: Because this operator accesses a single database via concurrent
    connections, it is advised that a connection pool be used to control
    requests. - https://airflow.incubator.apache.org/concepts.html#pools

    :param mssql_conn_id:           The input mssql connection id.
    :type mssql_conn_id:            string
    :param mssql_table:             The input MsSQL table to pull data from.
    :type mssql_table:              string
    :param s3_conn_id:              The destination s3 connection id.
    :type s3_conn_id:               string
    :param s3_bucket:               The destination s3 bucket.
    :type s3_bucket:                string
    :param s3_key:                  The destination s3 key.
    :type s3_key:                   string
    :param batchsize                *(optional)* The number of rows you want to
                                    batch inserts with. For files that are too
                                    large for the docker container.
    :type batch:                    string
    :param primary_key:             The key used for batch streaming. Will use
                                    this to chunk your results. Assumes INT.
    :type primary_key:              string
    :param package_schema:          *(optional)* Whether or not to pull the
                                    schema information for the table as well as
                                    the data.
    :type package_schema:           boolean
    :param incremental_key:         *(optional)* The incrementing key to filter
                                    the source data with. Currently only
                                    accepts a column with type of timestamp.
    :type incremental_key:          string
    :param start:                   *(optional)* The start date to filter
                                    records with based on the incremental_key.
                                    Only required if using the incremental_key
                                    field.
    :type start:                    timestamp (YYYY-MM-DD HH:MM:SS)
    :param end:                     *(optional)* The end date to filter
                                    records with based on the incremental_key.
                                    Only required if using the incremental_key
                                    field.
    :type end:                      timestamp (YYYY-MM-DD HH:MM:SS)
    """

    template_fields = ['start', 'end', 's3_key']

    @apply_defaults
    def __init__(self,
                 mssql_conn_id,
                 mssql_table,
                 s3_conn_id,
                 s3_bucket,
                 s3_key,
                 primary_key,
                 batchsize=False,
                 package_schema=False,
                 incremental_key=None,
                 start=None,
                 end=None,
                 *args,
                 **kwargs):
        super().__init__(*args, **kwargs)
        self.mssql_conn_id = mssql_conn_id
        self.mssql_table = mssql_table
        self.s3_conn_id = s3_conn_id
        self.s3_bucket = s3_bucket
        self.s3_key = s3_key
        self.batchsize = batchsize
        self.primary_key = primary_key
        self.package_schema = package_schema
        self.incremental_key = incremental_key
        self.start = start
        self.end = end

    def execute(self, context):
        hook = AstroMsSqlHook(self.mssql_conn_id)
        self.build_fetch_query(hook)
        if self.package_schema:
            self.get_schema(hook, self.mssql_table)

    def get_schema(self, hook, table):
        logging.info('Initiating schema retrieval.')
        results = list(hook.get_schema(table))
        logging.info("Schema:")
        output_dict = {}
        for i in results:
            new = []
            new_dict = {}
            for n in i:
                if n == 'COLUMN_NAME':
                    new.insert(0, i[n])
                else:
                    new.insert(1, i[n])
            # Convert all column names to lower() for easy copy to Redshift.
            new = [i.lower() for i in new]
            if len(new) == 2:
                new_dict[new[0]] = new[1]
                output_dict.update(new_dict)

        output_dict = self.map_datatypes(output_dict)
        logging.info('Mapped Schema:')
        logging.info(output_dict)

        self.s3_upload(str(output_dict), schema=True)

    def map_datatypes(self, schema):
        # Assumes going into Redshift.
        # Maps here to making sinks easier.

        maps = {'smallint': 'INTEGER',
                'varchar': 'VARCHAR',
                'text': 'VARCHAR',
                'int': 'INTEGER',
                'float': 'FLOAT',
                'money': 'FLOAT',
                'datetime': 'TIMESTAMP',
                'bit': 'BOOLEAN',
                'char': 'VARCHAR',
                'tinyint': 'INTEGER',
                'smalldatetime': 'TIMESTAMP',
                'real': 'FLOAT'
                }
        return {v: maps[schema[v]] for v in schema}

    def build_fetch_query(self, hook):
        # Builds the part of the fetch query with the incremental_key

        logging.info('Initiating record retrieval.')
        logging.info('Start Date: {0}'.format(self.start))
        logging.info('End Date: {0}'.format(self.end))

        if all([self.incremental_key, self.start, self.end]):
            query_filter = """ WHERE {0} >= '{1}' AND {0} < '{2}'
                """.format(self.incremental_key, self.start, self.end)

        if all([self.incremental_key, self.start]) and not self.end:
            query_filter = """ WHERE {0} >= '{1}'
                """.format(self.incremental_key, self.start)

        if not self.incremental_key:
            query_filter = ''

        if self.batchsize:
            self.get_records_batch(hook, query_filter)
        else:
            self.get_records_all(hook, query_filter)

    def get_records_all(self, hook, query_filter):
        query = \
            """
            SELECT *
            FROM {0}
            {1}
            """.format(self.mssql_table, query_filter)

        # Perform query and convert returned tuple to list
        results = list(hook.get_records(query))
        logging.info('Successfully performed query.')
        logging.info('QUERY:')
        results = [dict([k.lower(), str(v)] if v is not None else [k, v]
                        for k, v in i.items()) for i in results]
        results = '\n'.join([json.dumps(i) for i in results])
        self.s3_upload(results)
        return results

    def get_records_batch(self, hook, query_filter):
        # Chunks the records and streams to s3 by specified batchsize.

        if query_filter == '':
            query_filter = 'WHERE'
        else:
            query_filter = query_filter + ' AND '

        count_sql_max = """
        SELECT max({0}) as c FROM {1} """.format(
            self.primary_key,
            self.mssql_table)

        count_sql_min = """
        SELECT min({0}) as c FROM {1} """.format(
            self.primary_key,
            self.mssql_table)

        if query_filter != 'WHERE':
            # Remove the AND from the query filter so you're only batching
            # for incremental loads within your timerange. Assumes primary_key
            # is incremental.
            count_sql_max += query_filter.split("AND")[0]
            count_sql_min += query_filter.split("AND")[0]

        count = hook.get_pandas_df(count_sql_max)['c'][0]
        min_count = hook.get_pandas_df(count_sql_min)['c'][0]

        s3_conn = BaseHook('S3').get_connection(self.s3_conn_id)
        s3_creds = s3_conn.extra_dejson

        s3_key = '{}/{}'.format(
            self.s3_bucket,
            self.s3_key
        )

        url = 's3://{}:{}@{}'.format(
            s3_creds['aws_access_key_id'],
            s3_creds['aws_secret_access_key'],
            s3_key
        )

        logging.info('Initiating record retrieval in batches.')
        logging.info('Start Date: {0}'.format(self.start))
        logging.info('End Date: {0}'.format(self.end))
        logging.info('smallest_number: {0}'.format(min_count))
        logging.info('count: {0}'.format(count))

        # Smart Open is a library for efficiently streaming large files to S3.
        # Streaming data to S3 here so it doesn't break the task container.
        # https://pypi.python.org/pypi/smart_open
        # Does this here because smart_open doesn't yet support an
        # append mode and doing it as a function was causing the file to be
        # overwritten every time.

        with smart_open.smart_open(url, 'wb') as fout:
            logging.info("First Row {0}".format(min_count)),
            logging.info("Total Rows: {0}".format(count))
            logging.info("Batch Size: {0}".format(self.batchsize))
            for batch in range(min_count, count, self.batchsize):
                query = \
                    """
                    SELECT  *
                    FROM {table}
                    {query_filter} {primary_key} >= {batch}
                    AND {primary_key} < {batch_two};
                    """.format(count=count,
                               table=self.mssql_table,
                               primary_key=self.primary_key,
                               query_filter=query_filter,
                               batch=batch,
                               batch_two=batch + self.batchsize)

                logging.info(query)

                # Perform query and convert returned tuple to list
                results = list(hook.get_records(query))
                logging.info('Successfully performed query for batch {0}-{1}.'
                             .format(batch, (batch + self.batchsize)))

                results = [dict([k.lower(), str(v)] if v is not
                                None else [k, v]
                                for k, v in i.items()) for i in results]
                results = '\n'.join([json.dumps(i) for i in results])
                # Write the results to bytes.
                results = results.encode('utf-8')
                logging.info("Uploading!")
                fout.write(results)

    def s3_upload(self, results, schema=False):
        s3 = S3Hook(s3_conn_id=self.s3_conn_id)
        key = '{0}'.format(self.s3_key)

        file_name = os.path.splitext(key)[0]
        file_extension = os.path.splitext(key)[1]
        # If the file being uploaded to s3 is a schema, append "_schema" to the
        # end of the file name.
        if schema and file_extension == '.json':
            key = file_name + '_schema' + file_extension
        if schema and file_extension == '.csv':
            key = file_name + '_schema' + file_extension
        s3.load_string(
            string_data=results,
            bucket_name=self.s3_bucket,
            key=key,
            replace=True
        )

        s3.connection.close()
        logging.info('File uploaded to s3')


#!/usr/bin/env python
# -*- coding: iso-8859-15 -*-
processorName = 'Sentinel-2 Level 3 Prototype Processor (SEN2THREE)'
processorVersion = '1.0.0'
processorDate = '2015.09.15'
productVersion = '13'

from tables import *
import sys, os, shutil
import fnmatch
from time import time

from L3_Config import L3_Config
from L2A_Tables import L2A_Tables
from L3_Tables import L3_Tables
from L3_Product import L3_Product
from L2A_Process import L2A_Process
from L3_Synthesis import L3_Synthesis
from L3_Library import stdoutWrite, stderrWrite

class L3_Process(object):
    ''' The main processor module, which coordinates the interaction between the other modules.

            :param config: the config object for the current tile (via __init__)
            :type config: a reference to the config object

    '''
    def __init__(self, config):
        ''' Perform the L3 base initialisation.

            :param config: the config object for the current tile
            :type config: a reference to the config object

        '''

        self._config = config
        self._l3Synthesis = L3_Synthesis(config)

    def get_tables(self):
        return self._tables

    def set_tables(self, value):
        self._tables = value

    def del_tables(self):
        del self._tables

    def get_config(self):
        return self._config

    def set_config(self, value):
        self._config = value

    def del_config(self):
        del self._config

    def __exit__(self):
            sys.exit(-1)

    config = property(get_config, set_config, del_config)
    tables = property(get_tables, set_tables, del_tables)

    def process(self, tables):
        ''' Perform the L3 processing.

            :param tables: the table object for the current tile
            :type tables: a reference to the table object
            :return: true if processing succeeds, false else
            :rtype: bool

        '''
        self._tables = tables
        astr = 'L3_Process: processing with resolution ' + str(self.config.resolution) + ' m'
        self.config.timestamp(astr)
        self.config.timestamp('L3_Process: start of Pre Processing')
        if(self.preprocess() == False):
            return False

        self.config.timestamp('L3_Process: start of Spatio Temporal Processing')
        self.config.logger.info('Performing Spatio Temporal Processing with resolution %d m', self.config.resolution)
        if(self._l3Synthesis.process(self._tables) == False):
            return False

        # append processed tile to list
        processedTile = self.config.product.L2A_TILE_ID + '_' + str(self.config.resolution) + '\n'
        processedFn = self.config.sourceDir + '/' + 'processed'
        try:
            f = open(processedFn, 'a')
            f.write(processedTile)
            f.flush()
            f.close()
        except:
            stderrWrite('Could not update processed tile history.\n')
            self.config.exitError()
            return False
        return True

    def preprocess(self):
        ''' Perform the L3 pre processing,
            currently empty.
        '''
        self.config.logger.info('Pre-processing with resolution %d m', self.config.resolution)
        return True

    def postprocess(self):
        ''' Perform the L3 post processing,
            triggers the export of L3 product, tile metadata and bands

            :return: true if export succeeds, false else.
            :rtype: bool

        '''

        self.config.timestamp('L3_Process: start of Post Processing')
        self.config.logger.info('Post-processing with resolution %d m', self.config.resolution)

        GRANULE = self.config.targetDir + '/' + self.config.product.L3_TARGET_ID + '/GRANULE'
        tilelist = sorted(os.listdir(GRANULE))
        L3_TILE_MSK = 'S2A_*_TL_*'
        res = False
        for tile in tilelist:
            if fnmatch.fnmatch(tile, L3_TILE_MSK) == False:
                continue
            res = self.tables.exportTile(tile)
            if(self.config.resolution == 60):
                self.config.product.postprocess()
        if res == False:
            return res
        res = self.tables.createPreviewImage('L3')
        if res == False:
            return res
        return self._l3Synthesis.postProcessing()

def main(args=None):
    ''' Processes command line,
        initializes the config and product modules and starts sequentially
        the L2A and L3 processing.
    '''
    import argparse
    descr = processorName +', '+ processorVersion +', created: '+ processorDate + \
        ', supporting Level-1C product version: ' + productVersion + '.'

    parser = argparse.ArgumentParser(description=descr)
    parser.add_argument('directory', help='Directory where the Level-2A input files are located')
    parser.add_argument('--resolution', type=int, choices=[10, 20, 60], help='Target resolution, must be 10, 20 or 60 [m]')
    parser.add_argument('--clean', action='store_true', help='Removes all processed files in target directory. Be careful!')
    args = parser.parse_args()

    # SIITBX-49: directory should not end with '/':
    directory = args.directory
    if directory[-1] == '/':
        directory = directory[:-1]

    # check if directory argument starts with a relative path. If not, expand:
    if(os.path.isabs(directory)) == False:
        cwd = os.getcwd()
        directory = os.path.join(cwd, directory)
    elif os.path.exists(args.directory) == False:
        stderrWrite('directory "%s" does not exist\n.' % args.directory)
        return False

    if args.resolution == None:
        resolution = 60
    else:
        resolution = args.resolution

    config = L3_Config(resolution, directory)
    config.init(processorVersion)
    processedTiles = ''
    result = False
    processedFn = directory + '/' + 'processed'

    if args.clean:
        stdoutWrite('Cleaning target directory ...\n')
        try:
            shutil.rmtree(config.targetDir)
        except:
            pass
        try:
            os.remove(processedFn)
        except:
            stdoutWrite('No history file present ...\n')

    HelloWorld = processorName +', '+ processorVersion +', created: '+ processorDate
    stdoutWrite('\n%s started ...\n' % HelloWorld)
    upList = sorted(os.listdir(directory))

    # Check if unprocessed L1C products exist. If yes, process first:
    L1C_mask = 'S2?_*L1C_*'
    product = L3_Product(config)
    processor = L2A_Process(config)
    for L1C_UP_ID in upList:
        if(fnmatch.fnmatch(L1C_UP_ID, L1C_mask) == False):
            continue
        if config.checkTimeRange(L1C_UP_ID) == False:
            continue
        tilelist = product.createL2A_UserProduct(L1C_UP_ID)
        for tile in tilelist:
            # process only L1C tiles:
            if fnmatch.fnmatch(tile, L1C_mask) == False:
                continue
            # ignore already processed tiles:
            if product.tileExists(tile) == True:
                continue
            # finally, process the remaining tiles:
            stdoutWrite('\nL1C tile %s found, will be classified first ...\n' % tile)
            tStart = time()
            tables = L2A_Tables(config, tile)
            if processor.process(tables) == False:
                config.exitError()
                return False
            tile += '_' + str(config.resolution)
            if product.appendTile(tile) == False:
                config.exitError()
                return False

    # Now process all unprocessed L2A products:
    L2A_mask = 'S2?_*L2A_*'
    processor = L3_Process(config)
    for L2A_UP_ID in upList:
        if(fnmatch.fnmatch(L2A_UP_ID, L2A_mask) == False):
            continue
        if config.checkTimeRange(L2A_UP_ID) == False:
            continue

        config.updateUserProduct(L2A_UP_ID)
        GRANULE = directory + '/' + L2A_UP_ID + '/GRANULE'
        tilelist = sorted(os.listdir(GRANULE))
        for tile in tilelist:
            # process only L2A tiles:
            if fnmatch.fnmatch(tile, L2A_mask) == False:
                continue
            # ignore already processed tiles:
            if product.tileExists(tile):
                continue
            tStart = time()
            nrTilesProcessed = product.getNrTilesProcessed(tile)
            config.updateTile(tile, nrTilesProcessed)
            tables = L3_Tables(config)
            tables.init()
            # no processing if first initialisation:
            # check existence of Bands - B2 is always present:
            if tables.testBand('L2A', 2) == False:
                # append processed tile to list
                tile += '_' + str(config.resolution)
                if product.appendTile(tile) == False:
                    config.exitError()
                continue
            result = processor.process(tables)
            if(result == False):
                stderrWrite('Application terminated with errors, see log file and traces.\n')
                return False

            tMeasure = time() - tStart
            config.writeTimeEstimation(resolution, tMeasure)

    if result == True:
        result = processor.postprocess()
        if(result == False):
            stderrWrite('Application terminated with errors, see log file and traces.\n')
            return False

    stdoutWrite('\nApplication terminated successfully.\n')
    return result

if __name__ == "__main__":
    sys.exit(main() or 0)


import torch.nn as nn

from nupic.torch.modules import Flatten, KWinners2d

# import torch
# from torchsummary import summary

__all__ = [
    "wide_conv3x3",
    "WideBasic",
    "WideResNet",
]

def wide_conv3x3(in_planes, out_planes, stride=1):
    return nn.Conv2d(
        in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=True
    )

class WideBasic(nn.Module):
    def __init__(
        self, in_planes, planes, dropout_rate, stride=1, activation_func=nn.ReLU
    ):
        super(WideBasic, self).__init__()

        self.regular_path = nn.Sequential(
            nn.BatchNorm2d(in_planes),
            activation_func(in_planes),
            nn.Conv2d(in_planes, planes, kernel_size=3, padding=1, bias=True),
            nn.Dropout(p=dropout_rate),
            nn.BatchNorm2d(planes),
            activation_func(planes),
            nn.Conv2d(
                planes, planes, kernel_size=3, stride=stride, padding=1, bias=True
            ),
        )

        # resnet shortcut
        self.shortcut = nn.Sequential()
        # never for the first layer
        if stride != 1 or in_planes != planes:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride, bias=True)
            )

    def forward(self, x):

        out = self.regular_path(x)
        out += self.shortcut(x)
        # TODO: no activation function after shortcut? verify
        return out

class WideResNet(nn.Module):
    """
    Config hyperparamaters:
        - Batch norm is not optional
        - Dropout is one single rate, applied at all layers
        - Depth and widen_factor are specific to wide resnet architecture
    """

    def __init__(self, config=None):
        super(WideResNet, self).__init__()

        # update config
        defaults = dict(
            depth=28,
            widen_factor=2,
            num_classes=10,
            dropout_rate=0.3,
            percent_on_k_winner=1.0,
            boost_strength=1.4,
            boost_strength_factor=0.7,
            k_inference_factor=1.0,
        )
        defaults.update(config or {})
        self.__dict__.update(defaults)

        # adds kwinners
        for attr in [
            "percent_on_k_winner",
            "boost_strength",
            "boost_strength_factor",
            "k_inference_factor",
        ]:
            if type(self.__dict__[attr]) == list:
                raise ValueError(
                    """ResNet currently supports only single
                    percentage of activations for KWinners layers"""
                )

        if self.percent_on_k_winner < 0.5:
            self.activation_func = lambda out: self._kwinners(out)
        else:
            self.activation_func = lambda _: nn.ReLU()

        self.in_planes = 16

        assert (self.depth - 4) % 6 == 0, "Wide-resnet depth should be 6n+4"
        n = int((self.depth - 4) / 6)  # 28-4/6 = 4
        k = self.widen_factor

        print("| Wide-Resnet %dx%d" % (self.depth, k))
        n_stages = [16, 16 * k, 32 * k, 64 * k]

        self.features = nn.Sequential(
            wide_conv3x3(3, n_stages[0]),
            self._make_wide_layer(
                WideBasic, n_stages[1], n, self.dropout_rate, stride=1
            ),  # 4x2 (2 convs each)
            self._make_wide_layer(
                WideBasic, n_stages[2], n, self.dropout_rate, stride=2
            ),  # 4x2
            self._make_wide_layer(
                WideBasic, n_stages[3], n, self.dropout_rate, stride=2
            ),  # 4x2
            nn.BatchNorm2d(n_stages[3], momentum=0.9),
            self.activation_func(n_stages[3]),
            nn.AdaptiveAvgPool2d((1, 1)),
            Flatten(),
        )

        self.classifier = nn.Linear(n_stages[3], self.num_classes)  # 1

        # where are the other 2?

    def _kwinners(self, out):
        return KWinners2d(
            out,
            percent_on=self.percent_on_k_winner,
            boost_strength=self.boost_strength,
            boost_strength_factor=self.boost_strength_factor,
            k_inference_factor=self.k_inference_factor,
        )

    def _make_wide_layer(self, block, planes, num_blocks, dropout_rate, stride):
        # first one can define stride - downsampling
        # others are always 1
        strides = [stride] + [1] * (num_blocks - 1)
        layers = []

        # will be the size of N. In WideResnet28, will be 4
        for stride in strides:
            layers.append(
                block(
                    self.in_planes, planes, dropout_rate, stride, self.activation_func
                )
            )
            self.in_planes = planes

        return nn.Sequential(*layers)

    def forward(self, x):

        out = self.features(x)
        out = self.classifier(out)
        return out

# net=WideResNet(config=dict(
#     depth=28,
#     widen_factor=10,
#     num_classes=10,
#     dropout_rate=0.3)
# )
# summary(net, input_size=(3,32,32))


import numpy as np
import astropy.io.fits as fits
# === This scripts counts the amount of rows in two fit tables and appends|
# them in two separate files. These two fit files are suposed to be the   |
# result of matching (correlating) two fit files with the "best" and the  |
# "all" options using topcat or stilts. ================================= |
print("Counting the amount of rows in the tables.")
# --- Open the two fit tables
matchBest=fits.open('matchBest')
BestArray=matchBest[1].data
matchAll=fits.open('matchAll')
AllArray=matchAll[1].data
# --- Write the amount of rows in the files.
file=open("cumulative.txt",'a')
file.write(str(len(BestArray))+","+str(len(AllArray))+"\n")
file.close()
# --- Done.
print("Done.")
# --- DONE.


#
#  Müəllif hüququ (C) 2021-2022 by offlineflood@Github, < https://github.com/ChatBot >.
#
# Bu faylın bir hissəsidir < https://github.com/offlineflood/ChatBot > layihə,
# və "GNU v3.0 Lisenziya Müqaviləsi" əsasında buraxılır".
# Zəhmət olmasa baxın < https://github.com/offlineflood/ChatBot/blob/master/LICENSE >
#
# Bütün hüquqlar qorunur.
#

from os import getenv

from dotenv import load_dotenv

load_dotenv()

# my.telegram.org saytından əldə edin.
API_ID = int(getenv("API_ID"))
API_HASH = getenv("API_HASH")

## Telegram-da @Botfather-dən əldə edin.
BOT_TOKEN = getenv("BOT_TOKEN")

# SUDO İSTİFADƏÇİLƏRİ
SUDO_USER = list(
    map(int, getenv("SUDO_USER", "").split())
)  # Daxiletmə növü tam ədəd olmalıdır.

# Bunun üçün Şəxsi Qrup ID-sinə ehtiyacınız olacaq.
LOG_GROUP_ID = int(getenv("LOG_GROUP_ID"))

# Kimsə botunuzu işə saldıqda göstəriləcək mesaj.
PRIVATE_START_MESSAGE = getenv(
    "PRIVATE_START_MESSAGE",
    "Hello! Welcome to my Personal Assistant Bot",
)

# Söhbətlərinizi və statistikalarınızı saxlamaq üçün verilənlər bazası... MongoDB əldə edin:-  https://offline.gitbook.io/ChatBot/deployment/mongodb#4.-youll-see-a-deploy-cloud-database-option.-please-select-shared-hosting-under-free-plan-here
MONGO_DB_URI = getenv("MONGO_DB_URI", None)


import sys, os
import pandas as pd
import numpy as np
import datetime as dt
from fmiopendata.wfs import download_stored_query
from fmiopendata.utils import read_url
#import xml.etree.ElementTree as ET
from lxml import etree

def enablePrint():
    sys.stdout = sys.__stdout__

def get_places():

    xml = read_url("http://opendata.fmi.fi/wfs?service=WFS&version=2.0.0&request=getFeature&storedquery_id=fmi::ef::stations")
    #stree = etree.parse(xml)
    #root = ET.fromstring(xml)
    root = etree.fromstring(xml)
    locations = dict()
    for name in root.findall('.//{http://www.opengis.net/gml/3.2}name'):
        if name.attrib['codeSpace'] == 'http://xml.fmi.fi/namespace/locationcode/name':
            city = name.text.split(' ', 1)[0]
            id = name.getparent().find('.//{http://www.opengis.net/gml/3.2}identifier').text
            try:
                place = name.text.split(' ', 1)[1]
            except:
                place = ''
            if city in locations:
                locations[city].append((place,id))
            else:
                locations[city] = [(place, id)]

    #res = np.empty([1, 2])

    #for c in cities:
    #    x = np.array(locations[c]).reshape(len(locations[c]),1)
    #    y = np.broadcast_to([c], (len(locations[c]),1))
    #    r = np.hstack((x,y))
    #    res = np.vstack((res,r))

    return locations #res[1:,:]

def get_daily_obs(cities, places):

    # Retrieve the last 10 days daily observations + todays latest 10h observation
    end_time = dt.datetime.utcnow() - dt.timedelta(days=1)
    start_time = end_time - dt.timedelta(days=10)
    # Convert times to properly formatted strings
    start_time = start_time.isoformat(timespec="seconds") + "Z"
    end_time = end_time.isoformat(timespec="seconds") + "Z"

    df = pd.DataFrame({'city': [],'Precipitation amount' : [], 'Air temperature' : [], 'Snow depth' : [], 'Minimum temperature' : [], 'Maximum temperature' : [], 'Ground minimum temperature' : [],})

    for c in cities:
        plcs = places[c]
        appended_data = []
        for p in plcs:
        # For last 10d we get daily values
            obs = download_stored_query("fmi::observations::weather::daily::multipointcoverage",
                                args=["fmisid=" + p[1],
                                    "starttime=" + start_time,
                                    "endtime=" + end_time])
            df2 = pd.DataFrame.from_dict({(i): obs.data[i][j]
                                for i in obs.data.keys()
                                for j in obs.data[i].keys()},
                            orient='index')
            df2 = df2.applymap(lambda x: x.get('value'))
            if df2.empty == False and (df2['Air temperature'].isnull().all() == False or df2['Ground minimum temperature'].isnull().all() == False):
                appended_data.append(df2)
        df2 = pd.concat(appended_data)
        df2['city'] = c
        df2.index = df2.index.date
        #df2 = df2.groupby([df2.index]).mean()
        df = pd.concat([df,df2])

    df = df.groupby([df.index, "city"]).mean().reset_index(level=1)

    return df

def get_hourly_obs(cities, places, cols):
    # Retrieve the last 10 days daily observations + todays latest 10h observation
    end_time = dt.datetime.utcnow()
    start_time = end_time - dt.timedelta(hours=10)
    # Convert times to properly formatted strings
    start_time = start_time.isoformat(timespec="seconds") + "Z"
    end_time = end_time.isoformat(timespec="seconds") + "Z"

    dfh = pd.DataFrame({'city': [], 'Air temperature' : [], 'Precipitation amount' : [],'Snow depth' : []})

    for c in cities:
        plcs = places[c]
        appended_data = []
        for p in plcs:
        # For last 10h we get all observations and use these to calculate the "daily" observations for today
            obs = download_stored_query("fmi::observations::weather::multipointcoverage",
                                args=["fmisid=" + p[1],
                                    "starttime=" + start_time,
                                    "endtime=" + end_time])
            dfh2 = pd.DataFrame.from_dict({(i): obs.data[i][j]
                                for i in obs.data.keys()
                                for j in obs.data[i].keys()},
                            orient='index')
            dfh2 = dfh2.applymap(lambda x: x.get('value'))
            # We have to calculate some new columns for these hourly observations
            if dfh2.empty == False:
                dfh2['Minimum temperature'] = dfh2['Air temperature'].min()
                dfh2['Maximum temperature'] = dfh2['Air temperature'].max()
                dfh2['Air temperature'] = dfh2['Air temperature'].mean()
                dfh2['Ground minimum temperature'] = dfh2['Minimum temperature']
                dfh2['Precipitation amount'] = dfh2['Precipitation amount'].sum()
            # Let's take only last calculated value as daily value for station
            dfh2 = dfh2.sort_index(axis=0, ascending=False).head(1)
            if dfh2.empty == False and dfh2['Air temperature'].isnull().all() == False:
                appended_data.append(dfh2)
        dfh2 = pd.concat(appended_data)
        dfh2['city'] = c
        dfh2.index = dfh2.index.date
        dfh2 = dfh2[cols]
        dfh = pd.concat([dfh,dfh2])

    # Let's take average of station for that city
    dfh = dfh.groupby([dfh.index, "city"]).mean().reset_index(level=1)

    return dfh


import numpy as np
import matplotlib.pyplot as plt
from scipy.integrate import solve_ivp, simps
from scipy.interpolate import CubicSpline
from funcs import *

a = np.loadtxt('e1.txt')
rtab = a[:,0]
utab = a[:,1] / 4.637
u1 = lambda x: np.interp(x, rtab, utab)
u2 = CubicSpline(rtab, utab)
r = np.arange(np.min(rtab), np.max(rtab), 0.005)

# en = np.arange(50., 2050., 2.)
jmax = np.round(np.sqrt(np.max(utab)*np.max(rtab)**2))
js = np.arange(0,int(jmax/2),1)
en = np.arange(2., np.max(utab), 2.)
# phases_lin = np.zeros(len(en))
# phases_cub = np.zeros(len(en))
ejpairs = np.transpose([np.tile(js, len(en)), np.repeat(en, len(js))])
# sm = np.zeros((len(en), 2, 2), dtype=np.complex128)
sol_elastic_cub = solve_ivp(lambda t, y: mulelequations(y, en, js, u2, t), (np.min(rtab), np.max(rtab)),
                                    mul_initial_value_j(np.min(rtab), en, js), t_eval=r)
psi_cub = sol_elastic_cub.y[::2]
dpsi_cub = sol_elastic_cub.y[1::2]

result = np.zeros((len(en), 2))
result[:,0] = en
result[:,1] = mul_sigma_calc(psi_cub, dpsi_cub, r, u2, en, js)

# print(result)
np.savetxt('sigmas.txt', result)

# plt.legend()
# plt.show()


import json
import logging
from datetime import date, timedelta
from os import environ
from typing import Optional

import requests

from scraper.common import ScrapeResults, id_for_transaction

log = logging.getLogger(__name__)

CURRENCY_TO_ID = {}

def api_call(request, params=None, method='GET',
             endpoint='http://localhost:5464', data=None):
    token = environ.get('FIREFLY_TOKEN')
    headers = {
        'Accept': 'application/vnd.api+json, application/json, text/plain, '
                  '*/*',
        'Authorization': f'Bearer {token}',
    }
    if data:
        data = json.dumps(data, default=str, separators=(',', ':'))
        headers['Content-Type'] = 'application/json'
    r = requests.request(method=method, url=f'{endpoint}/api/v1/{request}',
                         params=params, headers=headers, data=data)
    r.raise_for_status()
    return r.json()

def paginated_data_call(request, method='GET',
                        endpoint='http://localhost:5464', **kwargs):
    page1 = api_call(request, {
        'page': '1',
        **kwargs
    }, method, endpoint)
    yield from page1['data']
    for page in range(2, page1['meta']['pagination']['total_pages'] + 1):
        yield from \
        api_call(request, {'page': page, **kwargs}, method, endpoint)['data']

def account_get_all(endpoint: str):
    yield from paginated_data_call('accounts', type='asset', endpoint=endpoint)

def account_create(
        name: str,
        account_type: str = 'asset',
        iban: str = '',
        bic: str = '',
        account_number: str = '',
        opening_balance: Optional[float] = None,
        opening_balance_date: Optional[date] = None,
        virtual_balance: Optional[float] = None,
        currency_id: Optional[int] = None,
        currency_code: str = '',
        active: bool = False,
        order: Optional[int] = None,
        include_net_worth: bool = True,
        account_role: str = '',
        credit_card_type: str = '',
        monthly_payment_date: Optional[date] = None,
        liability_type: str = '',
        liability_direction: str = '',
        interest: Optional[float] = None,
        interest_period: str = '',
        notes: str = '',
        latitude: float = 0.0,
        longitude: float = 0.0,
        zoom_level: int = 0,
        endpoint: str = None,
):
    data = {
        'name': name,
        'type': account_type,
        'iban': iban,
        'bic': bic,
        'account_number': account_number,
        'opening_balance': str(opening_balance) if opening_balance else '',
        'opening_balance_date': opening_balance_date.strftime("%Y-%m-%d")
        if opening_balance_date else '',
        'virtual_balance': str(virtual_balance) if virtual_balance else '',
        'currency_id': str(currency_id) if currency_id else '',
        'currency_code': currency_code.upper(),
        'active': str(active).lower(),
        'order': order,
        'include_net_worth': str(include_net_worth).lower(),
        'account_role': account_role,
        'credit_card_type': credit_card_type,
        'monthly_payment_date': monthly_payment_date.strftime("%Y-%m-%d")
        if monthly_payment_date else None,
        'liability_type': liability_type,
        'liability_direction': liability_direction,
        'interest': str(interest),
        'interest_period': interest_period,
        'notes': notes,
        'latitude': latitude,
        'longitude': longitude,
        'zoom_level': zoom_level,
    }
    cleanup = set()
    for k, v in data.items():
        if v is None:
            cleanup.add(k)
    for k in cleanup:
        data.pop(k)
    return api_call(
        request='accounts',
        method='POST',
        endpoint=endpoint,
        data=data,
    )

def currency_get_all(endpoint: str):
    for currency in paginated_data_call('currencies', endpoint=endpoint):
        CURRENCY_TO_ID[currency['attributes']['code']] = int(currency['id'])

def transaction_get_all(
        endpoint: str,
        start: Optional[date] = None,
        end: Optional[date] = None,
        query_type='all'
):
    # type query_type : all, withdrawal, withdrawals, expense, deposit,
    # deposits, income, transfer, transfers, opening_balance,
    # reconciliation, special, specials, default
    yield from paginated_data_call(
        'transactions',
        endpoint=endpoint,
        start=start.strftime("%Y-%m-%d") if start else '',
        end=end.strftime("%Y-%m-%d") if end else '',
        type=query_type,
    )

def transaction_create(
        endpoint: str,
        t_date: date,
        amount: float,
        description: str,
        currency: str,
        account_name: str,
        account_id: int,
        note: str,
        internal_id: str,
        external_id: str,
        process_date: Optional[date],
):
    if amount > 0:
        t_target = account_name
        t_target_id = account_id
        t_source = "Cash wallet"
        t_source_id = 4
    else:
        t_target = "Cash wallet"
        t_target_id = 4
        t_source = account_name
        t_source_id = account_id
    return api_call(
        request='transactions',
        method='POST',
        endpoint=endpoint,
        data={
            # "group_title": f"{t_date} {t_type} of {amount}",
            "error_if_duplicate_hash": True,
            "apply_rules": True,
            "fire_webhooks": True,
            "transactions": [
                {
                    "type": "transfer",
                    "date": t_date.strftime("%Y-%m-%d") + "T00:00:00+02:00",
                    "amount": abs(amount),
                    "description": description,
                    "currency_code": currency.upper(),
                    "source_id": str(t_source_id),
                    "source_name": t_source,
                    "destination_id": str(t_target_id),
                    "destination_name": t_target,
                    "category_name": "",
                    "interest_date": "",
                    "book_date": "",
                    "process_date": (
                            process_date.strftime("%Y-%m-%d")
                            + "T00:00:00+02:00"
                    ) if process_date else "",
                    "due_date": "",
                    "payment_date": "",
                    "invoice_date": "",
                    "internal_reference": internal_id,
                    "notes": note,
                    "external_id": external_id,
                }
            ]
        }
    )

def update_account(account_id, account_number, result, currency_t, endpoint):
    date_min = None
    date_max = None
    transaction_by_id = set()
    for currency, entries in result.transactions.items():
        if currency != currency_t:
            continue
        for entry in entries:
            if entry['date'] is None:
                continue
            if date_min is None or entry['date'] < date_min:
                date_min = entry['date'] - timedelta(days=1)
            if date_max is None or entry['date'] > date_max:
                date_max = entry['date'] + timedelta(days=1)
            transaction_by_id.add(
                id_for_transaction(entry, currency, account_number)
            )
    for entry in transaction_get_all(endpoint, date_min, date_max):
        for transaction in entry['attributes']['transactions']:
            internal_id = transaction['internal_reference']
            if internal_id in transaction_by_id:
                transaction_by_id.remove(internal_id)
    for currency, entries in result.transactions.items():
        if currency != currency_t:
            continue
        account_name = f'{result.bank} {currency.upper()}'
        for entry in entries:
            if 'serial' not in entry:
                continue
            entry_id = id_for_transaction(entry, currency, account_number)
            if entry_id not in transaction_by_id:
                continue
            transaction_create(
                endpoint,
                entry['date'],
                entry['value'],
                entry['description'],
                currency,
                account_name,
                account_id,
                f"balance: {entry['balance']} serial: {entry['serial']}",
                entry_id,
                "",
                entry['value_date'],
            )

def firefly_upload(result: ScrapeResults, endpoint: str):
    currency_get_all(endpoint)

    for account in account_get_all(endpoint):
        number = account['attributes'].get('account_number')
        cleanup = []
        for currency in result.transactions.keys():
            if number == f'{result.account}-{currency}':
                update_account(account['id'], number, result, currency, endpoint)
                cleanup.append(currency)
        for currency in cleanup:
            result.transactions.pop(currency)

    for currency in result.transactions.keys():
        opening_balance = 0.0,
        opening_balance_date = date.today()
        for t in result.transactions.get(currency, []):
            if t['date'] is not None and t['date'] < opening_balance_date:
                opening_balance_date = t['date'] - timedelta(days=1)
                opening_balance = t['balance'] - t['value']
        number = f'{result.account}-{currency}'
        account = account_create(
            name=f'{result.bank} {currency.upper()}',
            account_type='asset',
            account_role='defaultAsset',
            account_number=number,
            opening_balance=opening_balance,
            opening_balance_date=opening_balance_date if opening_balance else None,
            endpoint=endpoint,
            active=True,
            include_net_worth=True,
            currency_code=currency,
            currency_id=CURRENCY_TO_ID[currency.upper()],
            credit_card_type='monthlyFull',
            monthly_payment_date=date(2020, 1, 1),
            liability_type='loan',
            liability_direction='credit',
            interest=0.0,
            interest_period='monthly',
        )['data']
        update_account(account['id'], number, result, currency, endpoint)

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    log.info('connected to firefly server version {version}'.format(
        **api_call('about')['data']))

    result_x = ScrapeResults()
    from firefly_secret import test_enrich

    test_enrich(result_x)
    firefly_upload(result_x, 'http://localhost:5464')


__author__ = "Derek Gulbranson"
__revision__ = "$Id: nameparser.py 25 2010-08-18 19:57:57Z derek73 $"
__version__ = "0.1.2"
__license__ = "LGPL"
__url__ = "http://code.google.com/p/python-nameparser"

TITLES = [
    'dr','doctor','miss','misses','mr','mister','mrs','ms','sir',
    'rev','madam','madame','AB','2ndLt','Amn','1stLt','A1C','Capt','SrA','Maj',
    'SSgt','LtCol','TSgt','Col','BrigGen','1stSgt','MajGen','SMSgt','LtGen',
    '1stSgt','Gen','CMSgt','1stSgt','CCMSgt','CMSAF','PVT','2LT','PV2','1LT',
    'PFC','CPT','SPC','MAJ','CPL','LTC','SGT','COL','SSG','BG','SFC','MG',
    'MSG','LTG','1SGT','GEN','SGM','CSM','SMA','WO1','WO2','WO3','WO4','WO5',
    'ENS','SA','LTJG','SN','LT','PO3','LCDR','PO2','CDR','PO1','CAPT','CPO',
    'RADM(LH)','SCPO','RADM(UH)','MCPO','VADM','MCPOC','ADM','MPCO-CG','CWO-2',
    'CWO-3','CWO-4','Pvt','2ndLt','PFC','1stLt','LCpl','Capt','Cpl','Maj','Sgt',
    'LtCol','SSgt','Col','GySgt','BGen','MSgt','MajGen','1stSgt','LtGen','MGySgt',
    'Gen','SgtMaj','SgtMajMC','WO-1','CWO-2','CWO-3','CWO-4','CWO-5','ENS','SA',
    'LTJG','SN','LT','PO3','LCDR','PO2','CDR','PO1','CAPT','CPO','RDML','SCPO',
    'RADM','MCPO','VADM','MCPON','ADM','FADM','WO1','CWO2','CWO3','CWO4','CWO5'
]

# QUESTIONABLE_TITLES could be last names or they could be titles
# TODO: need to find best way to deal with these.. http://code.google.com/p/python-nameparser/issues/detail?id=3
QUESTIONABLE_TITLES = ['judge',]

# PUNC_TITLES could be names or titles, but if they have period at the end they're a title
PUNC_TITLES = ['hon.']
PREFICES = [
    'abu','bon','ben','bin','da','dal','de','del','der','de','di','e','ibn',
    'la','le','san','st','ste','van','vel','von'
]
SUFFICES = [
    'esq','esquire','jr','sr','2','i','ii','iii','iv','v','clu','chfc',
    'cfp','md','phd'
]
CAPITALIZATION_EXCEPTIONS = {
    'ii': 'II',
    'iii': 'III',
    'iv': 'IV',
    'md': 'M.D.',
    'phd': 'Ph.D.'
}
CONJUNCTIONS = ['&', 'and', 'et', 'e', 'und', 'y']

ENCODING = 'utf-8'
import re
re_spaces = re.compile(r"\s+")
re_word = re.compile(r"\w+")
re_mac = re.compile(r'^(ma?c)(\w)', re.I)
re_initial = re.compile(r'^(\w\.|[A-Z])?$')

import logging
# logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger('HumanName')

def lc(value):
    '''Lower case and remove any periods to normalize for comparison.'''
    if not value:
        return u''
    return value.lower().replace('.','')

def is_not_initial(value):
    return not re_initial.match(value)

class HumanName(object):

    """
    Parse a person's name into individual components

    Usage::

        >>> name = HumanName("Dr. Juan Q. Xavier de la Vega III")
        >>> name.title
        'Dr.'
        >>> name.first
        'Juan'
        >>> name.middle
        'Q. Xavier'
        >>> name.last
        'de la Vega'
        >>> name.suffix
        'III'
        >>> name2 = HumanName("de la Vega, Dr. Juan Q. Xavier III")
        >>> name == name2
        True
        >>> len(name)
        5
        >>> list(name)
        ['Dr.', 'Juan', 'Q. Xavier', 'de la Vega', 'III']
        >>> name[1:-1]
        [u'Juan', u'Q. Xavier', u'de la Vega']

    """

    def __init__(self, full_name=u"", titles=TITLES, prefices=PREFICES,
        suffices=SUFFICES, punc_titles=PUNC_TITLES, conjunctions=CONJUNCTIONS,
        capitalization_exceptions=CAPITALIZATION_EXCEPTIONS):

        super(HumanName, self).__init__()
        self.titles = titles
        self.punc_titles = punc_titles
        self.conjunctions = conjunctions
        self.prefices = prefices
        self.suffices = suffices
        self.capitalization_exceptions = capitalization_exceptions
        self.full_name = full_name
        self.title = u""
        self.first = u""
        self.suffixes = []
        self.middle_names = []
        self.last_names = []
        self.unparsable = False
        self.count = 0
        self.members = ['title','first','middle','last','suffix']
        if self.full_name:
            self.parse_full_name()

    def __iter__(self):
        return self

    def __len__(self):
        l = 0
        for x in self:
            l += 1
        return l

    def __eq__(self, other):
        """
        HumanName instances are equal to other objects whose
        lower case unicode representations are the same
        """
        return unicode(self).lower() == unicode(other).lower()

    def __ne__(self, other):
        return not unicode(self).lower() == unicode(other).lower()

    def __getitem__(self, key):
        return [getattr(self, x) for x in self.members[key]]

    def next(self):
        if self.count >= len(self.members):
            self.count = 0
            raise StopIteration
        else:
            c = self.count
            self.count = c + 1
            return getattr(self, self.members[c]) or self.next()

    def __unicode__(self):
        return u" ".join(self)

    def __str__(self):
        return self.__unicode__().encode('utf-8')

    def __repr__(self):
        if self.unparsable:
            return u"<%(class)s : [ Unparsable ] >" % {'class': self.__class__.__name__,}
        return u"<%(class)s : [\n\tTitle: '%(title)s' \n\tFirst: '%(first)s' \n\tMiddle: '%(middle)s' \n\tLast: '%(last)s' \n\tSuffix: '%(suffix)s'\n]>" % {
            'class': self.__class__.__name__,
            'title': self.title,
            'first': self.first,
            'middle': self.middle,
            'last': self.last,
            'suffix': self.suffix,
        }

    @property
    def middle(self):
        return u" ".join(self.middle_names)

    @property
    def last(self):
        return u" ".join(self.last_names)

    @property
    def suffix(self):
        return u", ".join(self.suffixes)

    def is_conjunction(self, piece):
        return lc(piece) in self.conjunctions and is_not_initial(piece)

    def is_prefix(self, piece):
        return lc(piece) in self.prefices and is_not_initial(piece)

    def parse_full_name(self):
        if not self.full_name:
            raise AttributeError("Missing full_name")

        if not isinstance(self.full_name, unicode):
            self.full_name = unicode(self.full_name, ENCODING)
        # collapse multiple spaces
        self.full_name = re.sub(re_spaces, u" ", self.full_name.strip() )

        # reset values
        self.title = u""
        self.first = u""
        self.suffixes = []
        self.middle_names = []
        self.last_names = []
        self.unparsable = False

        # break up full_name by commas
        parts = [x.strip() for x in self.full_name.split(",")]

        log.debug(u"full_name: " + self.full_name)
        log.debug(u"parts: " + unicode(parts))

        pieces = []
        if len(parts) == 1:

            # no commas, title first middle middle middle last suffix

            for part in parts:
                names = part.split(' ')
                for name in names:
                    name.replace(',','').strip()
                    pieces.append(name)

            log.debug(u"pieces: " + unicode(pieces))

            for i, piece in enumerate(pieces):
                try:
                    next = pieces[i + 1]
                except IndexError:
                    next = None

                try:
                    prev = pieces[i - 1]
                except IndexError:
                    prev = None

                if lc(piece) in self.titles:
                    self.title = piece
                    continue
                if piece.lower() in self.punc_titles:
                    self.title = piece
                    continue
                if not self.first:
                    self.first = piece.replace(".","")
                    continue
                if (i == len(pieces) - 2) and (lc(next) in self.suffices):
                    self.last_names.append(piece)
                    self.suffixes.append(next)
                    break
                if self.is_prefix(piece):
                    self.last_names.append(piece)
                    continue
                if self.is_conjunction(piece) and i < len(pieces) / 2:
                    self.first += ' ' + piece
                    continue
                if self.is_conjunction(prev) and (i-1) < len(pieces) / 2:
                    self.first += ' ' + piece
                    continue
                if self.is_conjunction(piece) or self.is_conjunction(next):
                    self.last_names.append(piece)
                    continue
                if i == len(pieces) - 1:
                    self.last_names.append(piece)
                    continue
                self.middle_names.append(piece)
        else:
            if lc(parts[1]) in self.suffices:

                # title first middle last, suffix [, suffix]

                names = parts[0].split(' ')
                for name in names:
                    name.replace(',','').strip()
                    pieces.append(name)

                log.debug(u"pieces: " + unicode(pieces))

                self.suffixes += parts[1:]

                for i, piece in enumerate(pieces):
                    try:
                        next = pieces[i + 1]
                    except IndexError:
                        next = None

                    if lc(piece) in self.titles:
                        self.title = piece
                        continue
                    if piece.lower() in self.punc_titles:
                        self.title = piece
                        continue
                    if not self.first:
                        self.first = piece.replace(".","")
                        continue
                    if i == (len(pieces) -1) and self.is_prefix(piece):
                        self.last_names.append(piece + " " + next)
                        break
                    if self.is_prefix(piece):
                        self.last_names.append(piece)
                        continue
                    if self.is_conjunction(piece) or self.is_conjunction(next):
                        self.last_names.append(piece)
                        continue
                    if i == len(pieces) - 1:
                        self.last_names.append(piece)
                        continue
                    self.middle_names.append(piece)
            else:

                # last, title first middles[,] suffix [,suffix]

                names = parts[1].split(' ')
                for name in names:
                    name.replace(',','').strip()
                    pieces.append(name)

                log.debug(u"pieces: " + unicode(pieces))

                self.last_names.append(parts[0])
                for i, piece in enumerate(pieces):
                    try:
                        next = pieces[i + 1]
                    except IndexError:
                        next = None

                    if lc(piece) in self.titles:
                        self.title = piece
                        continue
                    if piece.lower() in self.punc_titles:
                        self.title = piece
                        continue
                    if not self.first:
                        self.first = piece.replace(".","")
                        continue
                    if lc(piece) in self.suffices:
                        self.suffixes.append(piece)
                        continue
                    self.middle_names.append(piece)
                try:
                    if parts[2]:
                        self.suffixes += parts[2:]
                except IndexError:
                    pass

        if not self.first and len(self.middle_names) < 1 and len(self.last_names) < 1:
            self.unparsable = True
            log.error(u"Unparsable full_name: " + self.full_name)

    def cap_word(self, word):
        if self.is_prefix(word) or self.is_conjunction(word):
            return lc(word)
        if word in self.capitalization_exceptions:
            return self.capitalization_exceptions[word]
        mac_match = re_mac.match(word)
        if mac_match:
            def cap_after_mac(m):
                return m.group(1).capitalize() + m.group(2).capitalize()
            return re_mac.sub(cap_after_mac, word)
        else:
            return word.capitalize()

    def cap_piece(self, piece):
        if not piece:
            return ""
        replacement = lambda m: self.cap_word(m.group(0))
        return re.sub(re_word, replacement, piece)

    def capitalize(self):
        name = unicode(self)
        if not (name == name.upper() or name == name.lower()):
            return
        self.title = self.cap_piece(self.title)
        self.first = self.cap_piece(self.first)
        self.middle_names = self.cap_piece(self.middle).split(' ')
        self.last_names = self.cap_piece(self.last).split(' ')
        self.suffixes = self.cap_piece(self.suffix).split(' ')


import numpy as np
import pandas as pd
import seaborn as sn
import matplotlib.pyplot as plt

from sklearn.model_selection import RandomizedSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix, accuracy_score

# Using 3 fold cross validation
def rfc_validation(X_train, y_train):
    # Number of trees in random forest
    n_estimators = [int(x) for x in np.linspace(start = 10, stop = 100, num = 5)]
    # Number of features to consider at every split
    max_features = ['auto', 'sqrt']
    # Maximum number of levels in tree
    # max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]
    # max_depth.append(None)
    # Minimum number of samples required to split a node
    min_samples_split = [2, 5, 10]
    # Minimum number of samples required at each leaf node
    min_samples_leaf = [1, 2, 4]
    # Method of selecting samples for training each tree
    bootstrap = [True, False]
    # Create the random grid
    random_grid = {'n_estimators': n_estimators,
               'max_features': max_features,
               #'max_depth': max_depth,
               'min_samples_split': min_samples_split,
               'min_samples_leaf': min_samples_leaf,
               'bootstrap': bootstrap}

    classifier = RandomForestClassifier()
    # Random search of parameters, using 3 fold cross validation,
    # search across 100 different combinations, and use all available cores
    rf_random = RandomizedSearchCV(estimator = classifier, param_distributions = random_grid, n_iter = 100, cv = 3, verbose=2, random_state=42, n_jobs = -1)
    # Fit the random search model
    rf_random.fit(X_train, y_train)
    return rf_random

def rfc(X_train, y_train, X_test, y_test, labels, n_est, mss, msl, mf, bt):
    # Fitting Random Forest Classification to the Training set
    classifier = RandomForestClassifier(n_estimators = n_est, min_samples_split=mss, min_samples_leaf=msl, max_features=mf, bootstrap= bt, criterion = 'entropy', random_state = 42)
    classifier.fit(X_train, y_train)

    y_pred = classifier.predict(X_test)

    accuracy = accuracy_score(y_test, y_pred)

    # matrix = confusion_matrix(y_test, y_pred)

    y_test = np.vectorize(labels.get)(y_test)
    y_pred = np.vectorize(labels.get)(y_pred)

    # Confusion matrix
    print('Results of Random Forest Classifier-')
    df_cm = pd.crosstab(y_test, y_pred, rownames=['Actual move'], colnames=['Predicted move'])
    print(df_cm)
    print("Accuracy:", accuracy)

    plt.figure(figsize = (12, 10))
    sn.heatmap(df_cm, annot=True)


from django.shortcuts import render, redirect
from django.db.models import Q
from .models import Entry
from .forms import EntryForm

def index(request):
    entries = Entry.objects.order_by('-date_posted')

    context = {'entries' : entries}

    return render(request, 'entries/journal.html', context)

def add(request):
    if request.method == 'POST':
        form = EntryForm(request.POST)
        if form.is_valid():
            form.save()
            return redirect('home')
    else:
        form = EntryForm()

    context = {'form' : form}
    return render(request, 'entries/add.html', context)

def editEntry(request, pk):
    entry = Entry.objects.get(id=pk)

    form = EntryForm(instance=entry)

    if request.method == 'POST':
        form = EntryForm(request.POST, instance=entry)
        if form.is_valid():
            form.save()
            return redirect('/')

    context = {'form' : form}

    return render(request, 'entries/update.html', context)

def deleteEntry(request, pk):
    entry = Entry.objects.get(id=pk)

    if request.method == 'POST':
        entry.delete()
        return redirect('/')

    context = {'entry' : entry}
    return render(request, 'entries/delete.html', context)

def search(request):
    query = request.GET.get('q','')
    if query:
        queryset = (Q(title__icontains=query) | Q(text__icontains=query))
        results = Entry.objects.filter(queryset).distinct()
    else:
       results = []

    return render(request, 'entries/search.html', {'results':results, 'query':query})


import json

import scrapy

from kingfisher_scrapy.base_spiders import SimpleSpider
from kingfisher_scrapy.util import components, handle_http_error

class ItalyANAC(SimpleSpider):
    """
    Domain
      Autorità Nazionale Anticorruzione (ANAC)
    API documentation
      https://dati.anticorruzione.it/opendata/about
    Bulk download documentation
      https://dati.anticorruzione.it/opendata/organization/anticorruzione
    """
    name = 'italy_anac'
    download_timeout = 99999

    # SimpleSpider
    data_type = 'release_package'

    def start_requests(self):
        url = 'https://dati.anticorruzione.it/opendata/api/3/action/package_search?q=ocds'
        yield scrapy.Request(url, meta={'file_name': 'package_search.json'}, callback=self.parse_list)

    @handle_http_error
    def parse_list(self, response):
        data = response.json()
        for result in data['result']['results']:
            for resource in result['resources']:
                if resource['format'].upper() == 'JSON':
                    yield self.build_request(resource['url'], formatter=components(-2))

    @handle_http_error
    def parse(self, response):
        data = response.json()
        for release in data['releases']:
            # Kingfisher Process merges only releases with ocids. Extract the ocid from the release id as a fallback.
            # For example: ocds-hu01ve-7608611 from ocds-hu01ve-7608611-01.
            if 'ocid' not in release:
                release['ocid'] = '-'.join(release['id'].split('-')[:3])
        response = response.replace(body=json.dumps(data))
        yield from super().parse(response)


from django.db.models.aggregates import Count
from django.db.models.query import QuerySet
from whatwasthatbookcalled.books.models import Book

def get_by_id(id):
    return Book.objects.get(id=id)

def get_all():
    return Book.objects.all()

def filter_all_by_GET_params(get_req):
    filter_params = ["language", "genre", "solved"]
    filters = {}
    for f in filter_params:
        value = get_req.get(f)
        if value is not None and value != "":
            filters[f] = value

    books = Book.objects.filter(**filters).only(
        "year_read",
        "plot_details",
        "cover_description",
        "quotes",
        "solved",
        "genre",
        "last_modified",
        "filled_fields_count",
    )

    return books

def sort_filtered_by_GET_params(get_req, filtered_books):
    sort_by = get_req.get("sort_by")
    reverse_order = get_req.get("reverse_order")
    order_prefix = "" if reverse_order else "-"

    if sort_by is None or sort_by == "date":
        books = filtered_books.order_by(order_prefix + "last_modified")
    elif sort_by == "info-amount":
        books = filtered_books.order_by(order_prefix + "filled_fields_count")
    elif sort_by == "popularity":
        books = filtered_books.annotate(comment_count_sort=Count("comment")).order_by(
            order_prefix + "comment_count_sort"
        )

    return books

def get_book_with_user_and_filled_fields(form, user):
    book = form.save(commit=False)

    book.user = user

    filled_fields = [
        x
        for x in form.cleaned_data.values()
        if x != "" and x is not None and not (isinstance(x, QuerySet) and len(x) == 0)
    ]
    book.filled_fields_count = len(filled_fields)

    return book

def get_comment_with_user_and_book(form, user, book):
    comment = form.save(commit=False)
    comment.user = user
    comment.book = book
    return comment