# -*- coding: utf-8 -*-
"""
@authors: Nicholas Jackson, Ganesh Sivaraman, Vanessa Woo
"""
import os
import numpy as np
import random
from itertools import product

class simbox():
    def __init__(self):
        # OPLSAA vdw and charge parameters
        # First param sigma, second epsilon
        # Units angstrom and kcal/mol
        # Values obtained from TINKER's oplsaa.prm file
        self.kcaltoeV = 0.043
        self.vdw_dict = {}
        self.vdw_dict['F'] = [3.050000, 0.710000*self.kcaltoeV]
        self.vdw_dict['Cl'] = [4.020000, 0.710000*self.kcaltoeV]
        self.vdw_dict['Br'] = [4.280000, 0.710000*self.kcaltoeV]
        self.vdw_dict['I'] = [4.810000, 0.710000*self.kcaltoeV]
        self.vdw_dict['Li'] = [2.870000, 0.000500*self.kcaltoeV]
        self.vdw_dict['Na'] = [4.070000, 0.000500*self.kcaltoeV]
        self.vdw_dict['K'] = [5.170000, 0.000500*self.kcaltoeV]
        self.vdw_dict['Rb'] = [5.600000, 0.000500*self.kcaltoeV]
        self.vdw_dict['Cs'] = [6.200000, 0.000500*self.kcaltoeV]
        self.vdw_dict['Mg'] = [1.644471, 0.875044*self.kcaltoeV]
        self.vdw_dict['Ca'] = [2.412031, 0.449657*self.kcaltoeV]
        self.vdw_dict['Sr'] = [3.102688, 0.118226*self.kcaltoeV]
        self.vdw_dict['Ba'] = [3.816610, 0.047096*self.kcaltoeV]
        self.vdw_dict['Nd'] = [3.473000, 0.540000*self.kcaltoeV]
        self.vdw_dict['Th'] = [3.300000, 0.050000*self.kcaltoeV]
        self.chg_dict = {}
        self.chg_dict['F'] = -1.0
        self.chg_dict['Cl'] = -1.0
        self.chg_dict['Br'] = -1.0
        self.chg_dict['I'] = -1.0
        self.chg_dict['Li'] = 1.0
        self.chg_dict['Na'] = 1.0
        self.chg_dict['K'] = 1.0
        self.chg_dict['Rb'] = 1.0
        self.chg_dict['Cs'] = 1.0
        self.chg_dict['Mg'] = 2.0
        self.chg_dict['Ca'] = 2.0
        self.chg_dict['Sr'] = 2.0
        self.chg_dict['Ba'] = 2.0
        self.chg_dict['Nd'] = 3.0
        self.chg_dict['Th'] = 3.0
        self.mass_dict = {}
        self.mass_dict['F'] = 18.998
        self.mass_dict['Cl'] = 35.453
        self.mass_dict['Br'] = 79.904
        self.mass_dict['I'] = 126.905
        self.mass_dict['Li'] = 6.941
        self.mass_dict['Na'] = 22.990
        self.mass_dict['K'] = 39.098
        self.mass_dict['Rb'] = 85.468
        self.mass_dict['Cs'] = 132.905
        self.mass_dict['Mg'] = 24.305
        self.mass_dict['Ca'] = 40.080
        self.mass_dict['Sr'] = 87.620
        self.mass_dict['Ba'] = 137.330
        self.mass_dict['Nd'] = 144.240
        self.mass_dict['Th'] = 232.038

    def set_composition(self, composition, fractions):
        # First argument is a list of elements, entered as strings, e.g.
        # ['Na','Br','Sr','F']
        # Second argument is the fractions of each
        # [0.1,0.5,0.3,0.1]
        if len(composition) != len(fractions):
            print('Composition and fraction lengths do not match. Exiting...')
            exit()
        if np.round(sum(fractions), 2) != 1.:
            print('Elemental fractions do not sum to 1. Exiting...')
            exit()
        self.elemental_comp = composition
        self.elemental_frac = fractions
        self.num_atom_types = len(self.elemental_comp)

    def set_target_number_of_atoms(self, N):
        self.tot_atom_target = N
        min_frac = np.min(self.elemental_frac)
        self.min_atom_count = np.array(
            [round(i/min_frac) for i in self.elemental_frac])
        self.tot_atoms = np.sum(self.min_atom_count)
        multiple = self.tot_atom_target//self.tot_atoms
        self.sim_atom_count = (multiple*self.min_atom_count)  # Number of atoms of each type
        # Total number of atoms in system
        self.tot_atoms = np.sum(self.sim_atom_count)

    def fix_composition(self):
        self.charges = []
        self.min_charge_ratio = []
        self.ratio = np.array([])
        self.fractions =[]
        # check net charge with given fractions:
        for i in range(self.num_atom_types):
            atom = self.elemental_comp[i]
            charge = int(self.chg_dict[atom])
            self.charges.append(charge)
            self.min_charge_ratio.append(self.min_atom_count[i] * charge)
        self.pos_charges = [i for i in self.min_charge_ratio if i > 0]
        self.neg_charges = [i for i in self.min_charge_ratio if i < 0]
        self.tot_charge = np.sum(self.pos_charges) + np.sum(self.neg_charges)

        # fix negatives, 2 elements with positive charge
        if self.tot_charge < 0:
            self.fixed_ratio = abs(np.sum(self.neg_charges))
            pos_charge = [i for i in self.charges if i > 0],
            charge_ratio_combos = list(product([x for x in range(1,8)],repeat=2))
            for (x,y),(a,b) in product(pos_charge, charge_ratio_combos):
                c = (x*a + y*b)
                if c == self.fixed_ratio:
                    x = np.array([a,b,self.fixed_ratio])
                    fraction_denom = np.sum(x)
                    fractions = np.divide(x,fraction_denom)
                    self.fractions.append(fractions)
            # find best fractional combination and return to set_compostion
            error = [abs((self.fractions[i] - self.elemental_frac)/(self.elemental_frac)) for i in range(len(self.fractions))]
            tot_error = np.argmin(np.sum(error,axis=1))
            return self.set_composition(self.elemental_comp,self.fractions[tot_error])
        # fix positives, only 1 element with neg charge
        elif self.tot_charge > 0:
            self.fixed_ratio = np.sum(self.pos_charges)
            self.ratio = np.append(self.min_atom_count[0:2],self.fixed_ratio)
            fraction_denom = np.sum(self.ratio)
            fractions = np.divide(self.ratio,fraction_denom)
            self.fractions = fractions.tolist()
            return self.set_composition(self.elemental_comp,self.fractions)

    # Need to run set_target_number_atoms before set_density
    # Takes density in units of kg/m^3
    def set_density(self, dens, epsilon=0.3, dv=250, deform=False):
        self.dens = dens
        self.epsilon = epsilon  # Epsilon paramter for %  deform the box
        self.dv = dv  # change in volume
        self.deform = deform
        self.total_mass = 0.0  # Compute total mass for target atom number
        for atom, num in zip(self.elemental_comp, self.sim_atom_count):
            mass = self.mass_dict[atom]
            self.total_mass += mass*num
        self.total_mass *= 1.66054*10**-27  # Convert amu to kg
        # Compute box length in meters
        self.boxl = (self.total_mass / self.dens)**(0.33333)
        self.boxl *= 10**10  # Convert meters to angstroms
        self.boxl = np.round(self.boxl, 6)
        self.xlo = 0.0  # Set MD box dimensions
        self.ylo = 0.0
        self.zlo = 0.0
        if self.deform == False:
            self.xhi = self.boxl
            self.yhi = self.boxl
            self.zhi = self.boxl
            self.vol1 = np.round(self.boxl ** 3)

        elif self.deform == True:
            upper = self.dens + self.dens * epsilon
            lower = self.dens - self.dens * epsilon
            # Compute deformation target box length in meters
            self.box2 = (self.total_mass / lower)**(0.33333)
            self.box2 *= 10**10  # Convert meters to angstroms
            self.box2 = np.round(self.box2, 6)
            self.vol2 = np.round(self.box2 ** 3)

            # Compute deformation target box length in meters
            self.box3 = (self.total_mass / upper)**(0.33333)
            self.box3 *= 10**10  # Convert meters to angstroms
            self.box3 = np.round(self.box3, 6)
            self.xhi = self.box3
            self.yhi = self.box3
            self.zhi = self.box3
            self.vol3 = np.round(self.box3 ** 3)

    def make_initial_config(self, min_dist=0.5, writeconfig=1):
        # x,y,z type
        # Can use packmol for this later, but doing this manually
        # Because its trivial for elemental systems
        self.type = np.array([])
        self.xyz = np.array([[0, 0, 0]])
        for i in range(self.num_atom_types):  # Loop over atom types
            atom = self.elemental_comp[i]
            num = self.sim_atom_count[i]
            for j in range(num):  # Loop over number of atoms of each type
                truth = 0
                packing_count = 0
                while truth == 0:
                    xtrial = random.uniform(self.xlo, self.xhi)
                    ytrial = random.uniform(self.ylo, self.yhi)
                    ztrial = random.uniform(self.zlo, self.zhi)
                    self.xyz = np.append(self.xyz, np.array(
                        [[xtrial, ytrial, ztrial]]), axis=0)
                    rij = self.xyz[:, np.newaxis, :] - \
                        self.xyz[np.newaxis, :, :]
                    rij = rij - self.boxl*np.rint(rij/self.boxl)
                    rij_mag = np.sqrt(np.sum(rij**2., axis=-1))
                    rij_mag = rij_mag[np.triu_indices_from(rij_mag, k=1)]
                    if np.min(rij_mag) >= min_dist:
                        truth = 1
                        if i == 0 and j == 0:  # Delete the first element I used to jumpstart packing
                            self.xyz = np.delete(self.xyz, 0, 0)
                    else:
                        self.xyz = np.delete(self.xyz, -1, 0)
                        packing_count += 1
                    if packing_count == 1000:
                        print(
                            'Could not pack all atoms with the required minimum distance.\n')
                        print('Try decreasing min_dist. Exiting...\n')
                        exit()
                self.type = np.append(self.type, np.array([i]))
        print('Packing successful!')
        if writeconfig == 1:
            with open('init_config.xyz', 'w') as f:
                f.write('{} \n'.format(self.tot_atoms))
                f.write('{} {} initial configuration file \n'.format(
                    self.elemental_comp, self.elemental_frac))
                for i in range(self.tot_atoms):
                    atomtype = int(self.type[i])
                    element = self.elemental_comp[atomtype]
                    x, y, z = self.xyz[i]
                    f.write('{} \t {:6f} \t {:6f} \t {:6f} \n'.format(
                        element, x, y, z))

    def write_lammps_datafile(self, filename):
        print('Writing LAMMPS data file...')
        with open(filename, 'w') as f:
            f.write('LAMMPS data file\n')
            f.write('\n')
            f.write('{} atoms\n'.format(self.tot_atoms))
            f.write('{} atom types\n'.format(self.num_atom_types))
            f.write('0 bonds\n')
            f.write('0 bond types\n')
            f.write('0 angles\n')
            f.write('0 angle types\n')
            f.write('0 dihedrals\n')
            f.write('0 dihedral types\n')
            f.write('0 impropers\n')
            f.write('0 improper types\n')
            f.write('\n')
            f.write('{:6f} \t {:6f} \t xlo xhi\n'.format(self.xlo, self.xhi))
            f.write('{:6f} \t {:6f} \t ylo yhi\n'.format(self.ylo, self.yhi))
            f.write('{:6f} \t {:6f} \t zlo zhi\n'.format(self.zlo, self.zhi))
            f.write('\n')
            f.write('Masses\n')
            f.write('\n')
            for i in range(self.num_atom_types):
                element = self.elemental_comp[i]
                mass = self.mass_dict[element]
                f.write('{} {}\n'.format(i+1, mass))
            f.write('\n')
            f.write('Pair Coeffs\n')
            f.write('\n')
            for i in range(self.num_atom_types):
                element = self.elemental_comp[i]
                sig, eps = self.vdw_dict[element]
                f.write('{} {} {}\n'.format(i+1, eps, sig))
            f.write('\n')
            f.write('Atoms\n')
            f.write('\n')
            for i in range(self.tot_atoms):
                atmnum = i+1
                mol = i+1
                type = int(self.type[i]+1)
                element = self.elemental_comp[int(self.type[i])]
                chg = self.chg_dict[element]
                x, y, z = self.xyz[i]
                f.write('{} \t {} \t {} \t {} \t {} \t {} \n'.format(
                    atmnum, type, chg, x, y, z))
            f.write('\n')

    def set_lammps_params(self, print_reg):
        self.print_reg = print_reg
        self.unit_style = 'metal'
        self.atom_style = 'charge'
        self.kspace_style = 'pppm'
        self.kspace_acc = 1.0e-4  # -->1.0e-5
        #self.lj_cut = 12.2
        #self.coul_cut = 12.2
        if self.deform == False:
            self.lj_cut = np.floor(self.boxl / 2.)
            self.coul_cut = np.floor(self.boxl / 2.)
        elif self.deform == True:
            self.lj_cut = np.floor(min(self.box2, self.box3) / 2.)
            self.coul_cut = np.floor(min(self.box2, self.box3) / 2.)

    def write_lammps_inputfile(self, EquilTempStart, EquilTempStop, SampleTemp, tstep, melt_steps, equil_steps, sample_steps, nsims, datafile, filename):
        self.tstep = tstep
        self.EquilTempStart = EquilTempStart
        self.EquilTempStop = EquilTempStop
        self.SampleTemp = SampleTemp
        self.melt_steps = melt_steps
        self.equil_steps = equil_steps
        self.sample_steps = sample_steps
        self.nsims = nsims
        print('Writing LAMMPS input file...')
        with open(filename, 'w', encoding="utf-8") as f:
            f.write('units {}\n'.format(self.unit_style))
            f.write('atom_style {}\n'.format(self.atom_style))
            f.write(
                'pair_style \t lj/cut/coul/long {} {} \n'.format(self.lj_cut, self.coul_cut))
            f.write('boundary p p p \n')
            f.write('kspace_style {} {} \n'.format(
                self.kspace_style, self.kspace_acc))
            f.write('pair_modify shift yes mix geometric \n')
            f.write('read_data {}\n'.format(datafile))
            f.write('\n')
            f.write('variable my_temp equal temp\n')
            f.write('variable my_rho equal density\n')
            f.write('variable my_vol equal vol\n')
            f.write('\n')
            f.write('#MINIMIZATION\n')
            f.write('minimize 1.0e-24 1.0e-24 10000 10000\n')
            f.write('timestep {} \n'.format(self.tstep))
            f.write('neighbor 3.0 bin\n')
            f.write(
                'neigh_modify delay 10 every 10 check yes page 100000 one 10000\n')
            f.write('\n')
            f.write('velocity all create {} {}\n'.format(
                self.EquilTempStart, random.randint(1, 1000000)))
            f.write('\n')

            f.write('#Melt-Quench RUN\n')
            f.write('fix mynvt all nvt temp {} {} 1\n'.format(
                self.EquilTempStart, self.EquilTempStart))
            f.write('thermo {}\n'.format(self.print_reg))
            f.write(
                'thermo_style custom step temp density vol press etotal ke pe evdwl ecoul elong\n')
            f.write('log log.melt\n')
            f.write('run {}\n'.format(self.melt_steps))
            f.write('unfix mynvt\n')
            f.write('fix mynvt all nvt temp {} {} 1\n'.format(
                self.EquilTempStart, self.EquilTempStop))
            f.write('thermo {}\n'.format(self.print_reg))
            f.write(
                'thermo_style custom step temp density vol press etotal ke pe evdwl ecoul elong\n')
            f.write('log log.quench\n')
            f.write('run {}\n'.format(self.melt_steps))
            f.write('unfix mynvt\n')
            f.write('\n')

            f.write('#EQUILIBRATION RUN\n')
            #f.write('fix mymomentum all momentum 1000 linear 1 1 1 angular\n')
            if self.deform == True:
                f.write('fix mynvt all nvt temp {} {} 1\n'.format(
                    self.EquilTempStop, self.EquilTempStop))
            elif self.deform == False:
                #f.write('fix mynvt all npt temp {} {} 0.5 iso 0.0 0.0 10.0\n'.format(self.EquilTempStop,self.EquilTempStop))
                f.write('fix mynvt all nvt temp {} {} 0.5 \n'.format(
                    self.EquilTempStop, self.EquilTempStop))
            f.write('thermo {}\n'.format(self.print_reg))
            f.write(
                'thermo_style custom step temp density vol press etotal ke pe evdwl ecoul elong\n')
            f.write('log log.equi\n')
            f.write('run {}\n'.format(self.equil_steps))
            f.write('unfix mynvt\n')
            #f.write('unfix mymomentum\n')
            f.write('\n')
            f.write('#NPT SAMPLING AND PRINTING TRAJECTORY\n')
            #f.write('fix mymomentum all momentum 1000 linear 1 1 1 angular\n')
            f.write('reset_timestep 0\n')
            if self.deform == True:
                # ------ Added the loops below-------------
                f.write('\n')
                f.write('variable vollo equal {}\n'.format(self.vol3))
                f.write('variable volhi equal {}\n'.format(self.vol2))
                f.write('variable llo equal v_vollo^0.333333333\n')
                f.write('variable lhi equal v_volhi^0.333333333\n')
                f.write('variable ttotal equal {}\n'.format(self.sample_steps))
                f.write('variable nsims equal {}\n'.format(self.nsims))
                f.write('variable timepersim equal v_ttotal/v_nsims\n')
                f.write('fix mynvt all nvt temp {} {} 1 #Keep the temp fix constant and outside of the loop\n'.format(
                    self.SampleTemp, self.SampleTemp))
                f.write('label loopa\n')
                f.write('variable a loop ${nsims}\n')

                # OLD
#                f.write('variable boxl equal (v_lhi-v_llo)*(((v_a*v_timepersim)/v_ttotal)^(0.333333))+v_llo\n')
                # NEW
                f.write(
                    'variable boxl equal (v_vollo+(v_volhi-v_vollo)*(v_a/v_nsims))^(0.333333)\n')
                f.write('print " "\n')
                f.write('print "Loop $a"\n')
                f.write('print "Current Box Length: ${boxl}"\n')
                f.write('print " "\n')
                f.write(
                    'fix mydeform all deform 1 x final 0.0 ${boxl} y final 0.0 ${boxl} z final 0.0 ${boxl}\n')
                f.write('thermo {}\n'.format(self.print_reg))
                f.write(
                    'thermo_style custom step temp density vol press etotal ke pe evdwl ecoul elong\n')
                f.write(
                    'fix averages all ave/time 10 50 1000 v_my_temp v_my_rho v_my_vol file thermo_${boxl}.avg\n')
                f.write('dump myxyz all xyz {}'.format(
                    self.print_reg) + '  salt_npt_${boxl}.xyz\n')
                f.write('dump_modify myxyz element')
                for el in self.elemental_comp:
                    f.write(' {}'.format(el))
                f.write('\n')
                f.write('dump_modify myxyz sort id\n')
                f.write('log log_${boxl}.npt\n')
                f.write('run ${timepersim}\n')
                f.write('unfix mydeform \n')
                f.write('undump myxyz \n')
                f.write('next a\n')
                f.write('jump SELF loopa \n')

            elif self.deform == False:
                f.write('\n')
                f.write('fix mynvt all nvt temp {} {} 0.5 #Keep the temp fix constant and outside of the loop\n'.format(
                    self.SampleTemp, self.SampleTemp))
                f.write('thermo {}\n'.format(self.print_reg))
                f.write(
                    'thermo_style custom step temp density vol press etotal ke pe evdwl ecoul elong\n')
                f.write(
                    'fix averages all ave/time 10 50 1000 v_my_temp v_my_rho v_my_vol file thermo_14.3191176070878.avg\n')
                f.write('dump myxyz all xyz {}'.format(
                    self.print_reg) + '  salt_npt_14.3191176070878.xyz\n')
                f.write('dump_modify myxyz element')
                for el in self.elemental_comp:
                    f.write(' {}'.format(el))
                f.write('\n')
                f.write('dump_modify myxyz sort id\n')
                f.write('log log_14.3191176070878.npt\n')
                f.write('run {}\n'.format(self.sample_steps))
                f.write('undump myxyz \n')

            f.write('unfix mynvt \n')
            f.write('write_data final_config.data\n')


import os
from os import listdir
from os.path import isfile, join
import sys, urllib
import socket
import json
import xml.etree.ElementTree as ET
import argparse, pickle, csv

def clean_old_in(job_path):
	file_list  = listdir(job_path)
	out_list = [f for f in file_list if (isfile(join(job_path, f)) and f[-3:] == ".in") ]
	for of in out_list:
		os.remove(join(job_path, of))

if __name__ == "__main__":
	train_path = "data/out/test_predict_nonstem.csv"
	pickle_dir = "data/out"
	# id,question1,question2,is_duplicate
	with open(train_path, mode='r') as infile:
		reader = csv.reader(infile)
		linenum = 20000
		mydict = {row[0]:(row[1], row[2], row[3]) for row in reader if row[0]!='id' and int(row[0])<linenum}
		# print( len( mydict.keys() ))
		print(mydict["2"])

	pickle_path = os.path.join(pickle_dir, "train_pickle")
	all_fileds_fd = open(pickle_path, 'wb')
	# pickle.dump( mydict, all_fileds_fd )
	# mydict[id]->tuple('How can I increase the speed of my internet connection while using a VPN?', 'How can Internet speed be increased by hacking through DNS?', '0')

	parser = argparse.ArgumentParser(description='Process quora question.')
	parser.add_argument('--job_path')
	parser.add_argument('--num_partitions' )
	args = parser.parse_args()

	num_partitions = int(args.num_partitions)
	job_path =args.job_path

	lines = [0]*num_partitions
	clean_old_in(job_path)
	fds = [open( join( job_path+'%d.in'%index), 'wb') for index in range(num_partitions)] #binary form output
	page_entries = [{} for i in range(num_partitions)]

	for id in mydict.keys():
		idx = int(id) %  num_partitions
		# mydict[id][0] is question1, mydict[id][1] is question2
		page_entries[idx][id] = (mydict[id][0], mydict[id][1] , mydict[id][2])
		# page_entries[idx][page.id] =  page.text
		lines[idx] +=1

	for idx in range(num_partitions):
		pickle.dump(  page_entries[idx] ,  fds[idx] )
		fds[idx].close()

	print("num_lines: ",lines)


import torch
import torch.backends.cudnn
import pandas as pd

from torchtext.legacy import data

from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split

class DataFrameDataset(data.Dataset):
    def __init__(self, df, fields):
        self.examples = []
        for i, row in df.iterrows():
            label = row["Party"]
            text = row["stemmed"]

            self.examples.append(data.Example.fromlist([text, label], fields))
        super().__init__(self.examples, fields)

    def __getitem__(self, i):
        return self.examples[i]

def read_csv(file, train_val_split=False):
    df = pd.read_csv(file)

    df = df[["stemmed", "Party"]]
    df.loc[df["Party"] == "Independent", "Party"] = "Democrat"

    le = LabelEncoder()
    df["Party"] = le.fit_transform(df["Party"].values)

    train, val = train_test_split(df, test_size=0.1, random_state=0)
    if train_val_split:
        return train, val
    else:
        return df, val

def get_data_iterator(train, val, fields, batch_size, device, shuffle=True):
    train_ds = DataFrameDataset(train, fields)
    val_ds = DataFrameDataset(val, fields)
    iterator = data.BucketIterator.splits(
        (train_ds, val_ds),
        batch_size=batch_size,
        device=device,
        sort_key=lambda x: len(x.text),
        sort=False,
        shuffle=shuffle,
        sort_within_batch=True,
    )
    return iterator

def create_fields():
    TEXT = data.Field(tokenize="spacy")
    LABEL = data.LabelField(dtype=torch.float)

    fields = [("text", TEXT), ("label", LABEL)]
    return fields, TEXT, LABEL

def set_vocab(text, label, dataset, max_vocab_size):
    text.build_vocab(dataset, max_size=max_vocab_size,
                     vectors="glove.6B.200d", unk_init=torch.Tensor.zero_)
    label.build_vocab(dataset)

def setup_fields(dataframe, max_vocab_size):

    fields, TEXT, LABEL = create_fields()

    train_ds = DataFrameDataset(dataframe[0], fields)
    set_vocab(TEXT, LABEL, train_ds, max_vocab_size)

    return fields, TEXT, LABEL


# -*- coding: utf-8 -*-
"""
Created on Tue May 21 14:21:59 2019

@author: pc
"""

from gensim.models import fasttext
import fastText.FastText as ff
from sklearn.model_selection import train_test_split
import pandas as pd
from keras.preprocessing.text import Tokenizer

def read_file(path):
    with open(path, 'r', encoding="UTF-8") as f:
        data = []
        labels = []
        for line in f:
            data.append(line.split('\t')[0])
            labels.append(line.split('\t')[1])
    return data, labels

def get_tokenizer_data(data):
    '''
        fasttext传入文本必须对其进行预处理和编码
    '''
    tokenizer = Tokenizer(num_words=None)
    # 得到文本的字典
    tokenizer.fit_on_texts(data)
    # 将每个string的每个词转成数字
    data = tokenizer.texts_to_sequences(data)
    return data

def fast_text_model(X_test):
    '''
        使用fasttext进行文本分类
    '''
    # 分类训练
    classifier = ff.train_supervised('E:/task6/train.txt', label='__label__')
    # 模型预测，返回预测标签和概率
    label, prob = classifier.predict(X_test)
    print(label)
    print(prob)
    # 根据给定数据集对模型进行评价，返回样本个数、准确率、召回率
    result = classifier.test('E:/task6/test.txt')
    print(result)
    return label, prob, result

# 读文件
data, labels = read_file('E:/task6/merge.txt')

# 向量化文本
tokenizer_train = get_tokenizer_data(data)
# 划分数据集
X_train, X_test, y_train, y_test = train_test_split(tokenizer_train,
                                                    labels,
                                                    test_size = 0.2,
                                                    random_state=33)
# 在标签值前增加前缀
label_train = [('__label__' + i).replace('\n', '') for i in y_train]

# 将label列与文本列合并为一行
train = [i + ' ' + str(j).replace('[', '').replace(']', '').replace(',', '') for i, j in zip(label_train, X_train)]

f = open('E:/task6/train.txt', 'w')
for i in train:
    f.write(i)
    f.write('\n')
f.close()

label_test = [('__label__' + i).replace('\n', '') for i in y_test]
test = [i + ' ' + str(j).replace('[', '').replace(']', '').replace(',', '') for i, j in zip(label_test, X_test)]
f = open('E:/task6/test.txt', 'w')
for i in test:
    f.write(i)
    f.write('\n')
f.close()

X_test_tok = [str(j).replace('[', '').replace(']', '').replace(',', '') for j in X_test]

label, prob, result = fast_text_model(X_test_tok)


import serial
import datetime

def write_result(str: str):
    with open('result.txt', 'a') as f:
        f.write(str+'\n')

py_serial = serial.Serial(
    port='COM3',
    baudrate=9600,
)

write_result(">> logging starts at "+str(datetime.datetime.now()))
while True:
    if py_serial.readable():
        response = py_serial.readline()[:-1].decode()
        print(response)
        write_result(response)


from collections import defaultdict

import click

from app.models import glove_model
from app.save_utils import save_model
from app.evaluators import get_most_similar
from app import text_utils as tu

@click.group()
def cli():
    pass

@cli.command()
@click.argument('filename', type=click.File('rb'))
@click.option('-n', '--num-lines', help='Number of lines to read from filename. -1 means that all lines are used',
              default=-1)
@click.option('-v', '--vector-size', help='Dimension of the generated embeddings', default=30)
@click.option('-w', '--window', help='The size of the window around the central word', default=5)
@click.option('-e', '--epochs', help='Number of epochs to execute', default=5)
@click.option('-b', '--batch-size', help='The size of the batch', default=512)
@click.option('--num-words', help='The number of (most frequent) words to keep', default=10000)
def train(filename, num_lines, vector_size, window, epochs, batch_size, num_words):
    click.echo(click.style('Loading data', fg='green'))

    # read file and tokenize
    lines = tu.read_file(filename, num_lines=num_lines)
    seqs, tokenizer = tu.tokenize(lines=lines, num_words=num_words)

    # initialize cache
    cache = defaultdict(lambda: defaultdict(int))

    # get co-occurrences
    click.echo(click.style('Computing co-occurrences', fg='green'))
    tu.build_cooccurrences(sequences=seqs, cache=cache, window=window)
    first_indices, second_indices, frequencies = tu.cache_to_pairs(cache=cache)

    # build GloVe model & fit
    click.echo(click.style('Training keras model', fg='green'))
    model = glove_model(tokenizer.num_words + 1, vector_dim=vector_size)
    model.fit([first_indices, second_indices], frequencies, epochs=epochs, batch_size=batch_size)

    # save embedding layers as numpy arrays
    click.echo(click.style('Saving data', fg='green'))
    save_model(model=model, tokenizer=tokenizer)

    click.echo(click.style('Complete', fg='green'))

@cli.command()
@click.argument('words', type=click.STRING)
@click.option('-k', type=int, help='The number of most similar to retrieve', default=5)
def closest(words, k):

    for word in words.split(','):
        res = get_most_similar(word=word, k=k)
        if res:
            click.echo(click.style(f'Most similar words to {word}:', fg='green'))
            click.echo(click.style(f'{res}:\n', fg='green'))
        else:
            click.echo(click.style(f'Word {word} not in the vocabulary', fg='red'))


import sys
import os.path
sys.path.append(os.path.abspath(os.pardir))
import disaggregator as da
import pandas as pd
import numpy as np
import unittest
from datetime import datetime

class UtilsTestCase(unittest.TestCase):

    def setUp(self):
        self.consecutive_traces = []
        self.consecutive_instances = []
        for i in range(1,6):
            start = datetime(2013,1,i)
            series = pd.Series(np.zeros(24*4),
                    index=pd.date_range(start, periods=24*4, freq='15T'))
            trace = da.ApplianceTrace(series,{})
            self.consecutive_traces.append(trace)
            self.consecutive_instances.append(da.ApplianceInstance([trace],{}))
        self.piecewise_instance = da.ApplianceInstance(self.consecutive_traces,{})

    def test_get_common_ids(self):
        ids0 = []
        ids1 = [1,2,3]
        ids2 = [2,3,4,5,6]
        ids3 = [1,2,3,4,5,6,7]
        id_list1 = [ids0,ids3]
        id_list2 = [ids1,ids2]
        id_list3 = [ids2,ids3]
        common1 = da.get_common_ids(id_list1)
        common2 = da.get_common_ids(id_list2)
        common3 = da.get_common_ids(id_list3)
        self.assertListEqual([],common1)
        [self.assertIn(el,common2)    for el in [2,3]]
        [self.assertNotIn(el,common2) for el in [1,4]]
        [self.assertIn(el,common3)    for el in [2,5]]
        [self.assertNotIn(el,common3) for el in [1,7]]

    def test_aggregate_traces_misaligned(self):
        self.assertRaises(da.AlignmentError,
                          da.aggregate_traces,self.consecutive_traces,{})

    def test_aggregate_traces_aligned(self):
        aligned_traces = da.align_traces(self.consecutive_traces)
        da.aggregate_traces(aligned_traces,{})

    def test_aggregate_instances_misaligned(self):
        self.assertRaises(da.AlignmentError,
                          da.aggregate_instances,self.consecutive_instances,{})

    def test_aggregate_instances_aligned(self):
        aligned_instances = da.align_instances(self.consecutive_instances)
        da.aggregate_instances(aligned_instances,{})

    def test_concatenate_traces_consecutive(self):
        trace = da.concatenate_traces(self.consecutive_traces,{})
        self.assertEqual(trace.series.index.size, 24 * 4 * 5)

    def test_concatenate_instances_consecutive(self):
        instance = da.concatenate_instances(self.consecutive_instances,{})
        self.assertEqual(instance.traces[0].series.index.size, 24 * 4 * 5)

if __name__ == "__main__":
    unittest.main()


import numpy as np
import matplotlib.pyplot as plt

m = 1200
v=0
Vel=[]
t=0
T_Fr = [30, 100, 250]
V_Fr = [70, 50, 100]
Time = [i for i in range(250)]
b = 50
kp = 52
ki = 8
kd = 0
j = 0
while(t != 250):
    r = V_Fr[j]
    old_e = 0
    E = 0
    while t!= T_Fr[j]:
        e = r-v
        e_dot = e - old_e
        E = E + e
        u = kp*e + kd*e_dot + ki*E
        old_e = e
        v = (m*v+u*t)/(m+b*t)
        Vel.append(v)
        t+=1
    j=j+1
print(Vel)
plt.plot(Time,Vel)
plt.xlabel('Time (sec)--->')
plt.ylabel('Velocity (m/s)--->')
plt.title("Cruise Control\n")
plt.show()


from typing import List

from fastapi import APIRouter, Body, Depends, HTTPException
from fastapi.encoders import jsonable_encoder
from fastapi.encoders import jsonable_encoder
from app.models.book import Book, BookCreate, BookUpdate

router = APIRouter()

storage = []

@router.get('/')
def get_all_books():
    return storage

@router.get('/name/{name}')
def get_dog_by_name(*, name: str):

    found_books_by_name = []
    for i in range(len(storage)):
        if(storage[i]['name'] == name):
            book = storage[i]
            found_books_by_name.append(book)

    if not(found_books_by_name):
        message = f'No book found named {name}'
        return message

    return {'message': f'Found books named {name}', 'data': found_books_by_name}

@router.get('/id/{id}')
def get_book_by_id(*, id: int):

    found_books_by_id = []
    for i in range(len(storage)):
        if(storage[i]['id'] == id):
            book = storage[i]
            found_books_by_id.append(book)

    if not(found_books_by_id):
        message = f'No book found with id {id}'
        return message

    return {'message': f'Found book with id {id}', 'data': found_books_by_id}

@router.put('/{book_id}')
def update_book(*, book_id: int, book_in: BookUpdate):

    position_detected = False
    book_in = book_in.dict()

    for i in range(len(storage)):
        if storage[i]["id"] == book_id:
            position_detected = True
        if position_detected:
            if position_detected:
                storage.pop(i)
                storage.append(book_in)
                message = "Update completed"
                return message

    message = "Book not available"
    return message

@router.post('/')
def insert_book(
    *,
        book_in: BookCreate):

    id_exists = False
    book_in = book_in.dict()
    if len(storage) == 0:
        storage.append(book_in)
        message = "Insert successfully"
        return message
    else:
        for i in range(len(storage)):
            if storage[i]["id"] == book_in["id"]:
                id_exists = True

        if id_exists:
            message = "The book id already exists"
            return message
        else:
            storage.append(book_in)
            message = "Insert successfully"
            return message

@router.delete('/{book_id}')
def delete_book(*, book_id: int):

    position_detected = False

    for i in range(len(storage)):
        if storage[i]["id"] == book_id:
            position_detected = True
        if position_detected:
            if position_detected:
                storage.pop(i)
                message = "Delete completed"
                return message

    message = "Book not available"
    return message

@router.get('/generate_smoke_test/')
def generate_test():
    test1 = {
        "id": 11111,
        "name": "smoke book 1",
        "literary_genre": "smbk 1",
        "author": "team 1",
        "year": "2020",
        "price": 10000
    }

    test2 = {
        "id": 11112,
        "name": "smoke book 2",
        "literary_genre": "smbk 2",
        "author": "team 2",
        "year": "2020",
        "price": 10000
    }

    test3 = {
        "id": 11113,
        "name": "smoke book 3",
        "literary_genre": "smbk 3",
        "author": "team 3",
        "year": "2020",
        "price": 10000
    }
    test4 = {
        "id": 11114,
        "name": "smoke book 1",
        "literary_genre": "a_smbk 1",
        "author": "Juan",
        "year": "2001",
        "price": 15000
    }

    test5 = {
        "id": 11115,
        "name": "smoke book 2",
        "literary_genre": "b_smbk 2",
        "author": "Lucas",
        "year": "1990",
        "price": 100000
    }

    test6 = {
        "id": 11116,
        "name": "smoke book 3",
        "literary_genre": "c_smbk 3",
        "author": "Luis",
        "year": "2019",
        "price": 1000
    }

    storage.append(test1)
    storage.append(test2)
    storage.append(test3)
    storage.append(test4)
    storage.append(test5)
    storage.append(test6)

    message = "Smoke test generated"
    return message


from flask import Flask, render_template, flash, redirect, request, url_for, jsonify,  Blueprint
from ..app import db,Customer

main = Blueprint('main', __name__)

db.create_all()

@main.route('/', methods=['GET', 'POST'])
def index():

    data = request.get_json()
    username = data["username"]
    email = data["email"]

    user = User(username, email)
    print(user.username)

    db.session.add(user)
    db.session.commit()

    db_data = User.query.all()

    for name_user in db_data:
        print("======>:  " + name_user.username)

    return jsonify({"username": name_user.username, "email": name_user.email})

    # return "hello world"

#GET all accounts
@main.route('/accounts', methods =['GET'])
def getAllAccounts():

    all_customers = {'Customers':  [customer for customer in Customer.query.all()]}

    return jsonify(all_customers) ,404

#GET account by ID
@main.route('/accounts/<int:Id>',methods =['GET'])
def getAccoutByID(Id):

    customer = Customer.query.filter(Customer.id==Id).first()

    return jsonify({'Customer account': [customer]}) if customer else jsonify({'message': 'Customer not found'}),404

#GET account balance
@main.route('/accounts/<int:Id>/balance',methods =['GET'])
def getAccountBalance(Id):

    customer = Customer.query.filter(Customer.id==Id).first()

    return jsonify({'Account balance': [customer.balance]}) if customer else jsonify({'message': 'Customer not found'})

#GET account number
@main.route('/accounts/<int:Id>/accountnumber',methods =['GET'])
def getAccountNumber(Id):

    customer = Customer.query.filter(Customer.id==Id).first()

    return jsonify({'Account number': [customer.account_no]}) if customer else jsonify({'message': 'Customer not found'})

#GET account consumption statement
@main.route('/accounts/<int:Id>/consumption',methods =['GET'])
def getAccountConsumptionStatement(Id):

    cons = Consumption.query.filter(Customer.id==Id)

    return jsonify({'Consumption': [cons.to_json()]}) if cons else jsonify({'message': 'Consumption not found'})

#GET account topup statement
@main.route('/accounts/<int:Id>/balance',methods =['GET'])
def getAccountTopUpStatement(Id):

    topUp = TopUp.query.filter(Customer.id==Id)
    return jsonify({'TopUp': [topUp.to_json()]}) if topUp else jsonify({'message': 'TopUp not found'})


# from __future__ import print_function
import pandas as pd
# import numpy as np
# import matplotlib.pyplot as plt
# import seaborn as sns
from sklearn.metrics import classification_report
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn import metrics
# from sklearn import tree
import pickle
from sklearn.preprocessing import StandardScaler

df = pd.read_csv("../final project telegram/new/crop data.csv")

df.head()

x = df.iloc[:,df.columns!='Crop'] #data
y = df.iloc[:,df.columns=='Crop'] #outcome or Label
features = df[['Nitrogen', 'Phosphorous','Potassium','Temperature', 'Humidity', 'Soil pH', 'Rainfall']]
target = df['Crop']
labels = df['Crop']
xtrain, xtest, ytrain, ytest = train_test_split(x,y,test_size = 0.2)
acc = []
model = []

# Feature scaling
sc = StandardScaler()
X_train = sc.fit_transform(xtrain)
X_test= sc.transform(xtest)

#Random Forest Classifier
classifier = RandomForestClassifier(n_estimators=20, random_state=0)
classifier.fit(xtrain,ytrain.values.ravel())
predicted_values = classifier.predict(xtest)
x = metrics.accuracy_score(ytest, predicted_values)
acc.append(x)
model.append('classifier')
print("RF's Accuracy is: ", x)

print(classification_report(ytest,predicted_values))

DecisionTree = DecisionTreeClassifier(criterion="entropy",random_state=2,max_depth=5)
DecisionTree.fit(xtrain,ytrain)
predicted_values = DecisionTree.predict(xtest)

pickle.dump(classifier, open("model.pkl", "wb"))


from PyQt5.QtWidgets import (
    QComboBox, QLabel, QGridLayout, QHBoxLayout, QWidget)
from ui.param_selection.knn_options_widget import KNNOptionsWidget
from utils.constants import CLASSIFIERS

class ParamsWidget:
    def __init__(self, update_img, update_time_estimate):
        self.classifier_options = ClassifierOptionsWidget(
            self._update_classifier)

        self.knn_options = KNNOptionsWidget(update_img, update_time_estimate)

        self.current_classifier_options = self.knn_options.get_widget()

        self.layout = QGridLayout()
        self.layout.addWidget(self.classifier_options.get_widget(), 0, 0, 1, 0)
        self.layout.addWidget(self.current_classifier_options, 1, 0, 9, 0)

    def get_layout(self):
        return self.layout

    def _update_classifier(self):
        self._remove_current_classifier_options()
        selection = self.classifier_options.get_current_text()
        if selection == CLASSIFIERS[0]:
            self.current_classifier_options = self.knn_options.get_widget()
        self.layout.addWidget(self.current_classifier_options, 1, 0, 9, 0)

    def _remove_current_classifier_options(self):
        if self.layout.itemAt(2) is not None:
            self.layout.itemAt(2).widget().setParent(None)

class ClassifierOptionsWidget:
    def __init__(self, update_classifier):
        self.widget = QWidget()
        self.classifier_label = QLabel("Select classifier")
        self.classifiers = QComboBox()
        self.classifiers.addItems(CLASSIFIERS)
        self.classifiers.activated[str].connect(update_classifier)

        self.layout = QHBoxLayout(self.widget)
        self.layout.addWidget(self.classifier_label)
        self.layout.addWidget(self.classifiers)

    def get_widget(self):
        return self.widget

    def get_current_text(self):
        return self.classifiers.currentText()


import csv
import matplotlib.pyplot as plt
from datetime import datetime
import numpy as np

infile = open("sitka_weather_2018_simple.csv", "r")
infile2 = open("death_valley_2018_simple.csv", "r")

csv_file = csv.reader(infile, delimiter=",")
csv_file2 = csv.reader(infile2, delimiter=",")

header_row = next(csv_file)
header_row2 = next(csv_file2)

highs = []
lows = []
dates = []

highs_dv = []
lows_dv = []
dates_dv = []

tmax_index_sitka = header_row.index("TMAX")
tmin_index_sitka = header_row.index("TMIN")
date_index_sitka = header_row.index("DATE")
station_index_sitka = header_row.index("STATION")
name_index_sitka = header_row.index("NAME")

station_name_sitka = header_row[name_index_sitka]

tmax_index_dv = header_row2.index("TMAX")
tmin_index_dv = header_row2.index("TMIN")
date_index_dv = header_row2.index("DATE")
station_index_dv = header_row2.index("STATION")
name_index_dv = header_row2.index("NAME")

station_name_dv = header_row2[name_index_dv]

for row in csv_file:

    station_name_sitka = row[name_index_sitka]
    try:
        high = int(row[tmax_index_sitka])
        low = int(row[tmin_index_sitka])
        convertedDate = datetime.strptime(row[date_index_sitka], "%Y-%m-%d")

    except ValueError:
        print(f"Missing data for {convertedDate}")
    else:
        highs.append(int(row[tmax_index_sitka]))
        lows.append(int(row[tmin_index_sitka]))
        dates.append(convertedDate)

for row in csv_file2:

    station_name_dv = row[name_index_dv]

    try:
        high = int(row[tmax_index_dv])
        low = int(row[tmin_index_dv])
        convertedDate = datetime.strptime(row[date_index_dv], "%Y-%m-%d")

    except ValueError:
        print(f"Missing data for {convertedDate}")
    else:
        highs_dv.append(int(row[tmax_index_dv]))
        lows_dv.append(int(row[tmin_index_dv]))
        dates_dv.append(convertedDate)

fig, ax = plt.subplots(2)

ax[1].plot(dates_dv, highs_dv, c="red")
ax[1].plot(dates_dv, lows_dv, c="purple")

plt.fill_between(dates_dv, highs_dv, lows_dv, facecolor="blue", alpha=0.1)

ax[0].set_title(station_name_sitka, fontsize=14)
plt.tick_params(axis="both", which="major", labelsize=6)

ax[0].plot(dates, highs, c="red")
ax[0].plot(dates, lows, c="purple")

ax[0].fill_between(dates, highs, lows, facecolor="blue", alpha=0.1)

plt.title(station_name_dv, fontsize=14)
plt.tick_params(axis="both", which="major", labelsize=12)

fig.autofmt_xdate()

fig.suptitle(
    "Temperature comparison between " + station_name_sitka + " and " + station_name_dv
)

plt.show()


"""
Resize images for a Hugo static site.
Run this script from the same directory that `hugo` command is run.

Currently only works if the folder structure is:
└───content
    └───post
        ├───arc-reactor
        │   └───images
        ├───beer-pong
        │   └───img
        └───flashlight
            └───images

Each generated size is placed in its own folder inside the images folder for that post:
└───content
    └───post
        ├───arc-reactor
        │   └───images
        │       ├───1080
        │       ├───300
        │       └───600
        ├───beer-pong
        │   └───img
        │       ├───1080
        │       ├───300
        │       └───600
        └───flashlight
            └───images
                ├───1080
                ├───300
                └───600

"""

import glob
import multiprocessing
import os
import shutil
import time

from PIL import Image

SIZES = ((1080, 800), (600, 600), (300, 300))
DIR_NAMES = ("images", "image", "img", "imgs")
EXTENSIONS = (".jpg", ".png", ".gif")

def delete_generated_images():
    """Deletes all the folders inside an images folder that match one of the sizes in SIZES
    """

    for root, dirs, files in os.walk("content"):
        if root.endswith(DIR_NAMES):
            print("deleting generated images from:", root)
            for size in SIZES:
                folder_dir = os.path.join(root, str(size[0]))
                if os.path.isdir(folder_dir):
                    shutil.rmtree(folder_dir)

def find_images():
    """Return list of all image paths below the contents directory that are in an images folder
    """

    print("finding image paths...")
    images = []
    for root, dirs, files in os.walk("content"):
        if root.endswith(DIR_NAMES):
            print("found images folder:", root)
            # make folder for each generated size if they dont exist
            for size in SIZES:
                os.makedirs(os.path.join(root, str(size[0])), exist_ok=True)
            for ext in EXTENSIONS:
                # get filenames for all images
                images.extend(glob.glob(root + "/*" + ext))
    '''
    # remove images from list that have already been processed
    size_ending = tuple(str(size[0]) for size in SIZES)
    for image in images:
        if os.path.splitext(image)[0].endswith(size_ending):
            images.remove(image)
    '''
    return images

def image_worker(image_path):
    # generate an image for each size
    for size in SIZES:
        directory, filename = os.path.split(image_path)
        im = Image.open(image_path)
        im.thumbnail(size)
        im.save(os.path.join(directory, str(size[0]), filename))

def multi_processing(num_processes):
    pool = multiprocessing.Pool(num_processes)
    images = find_images()

    # If tqdm (progress bar library) is installed, use it.
    # If not, do the work with no progress indicator
    try:
        import tqdm
        for _ in tqdm.tqdm(pool.imap_unordered(image_worker, images), total=len(images)):
            pass
    except ImportError:
        pool.map(image_worker, images)

if __name__ == "__main__":
    processes = 4

    # delete_generated_images()

    start_time = time.time()
    multi_processing(processes)
    end_time = time.time()
    m_time = time.time() - start_time

    print("time for {} processes: {}".format(processes, m_time))


'''
Created on Dec 10, 2015

@author: Mogoi Adrian
'''
import os
import random
import jinja2

_templates = "/templates/"

Jinja = jinja2.Environment(
    loader=jinja2.FileSystemLoader( os.path.join( os.path.dirname( __file__ ), os.pardir ) ),
    extensions=['jinja2.ext.autoescape'],
    autoescape=True )

def getJTemplate( page ):

    return Jinja.get_template( _templates + page )

def createTiles( ln ):
    brd = []
    if ln == "" : ln = 50
    for i in range( int( ln ) ):
        columns = 4 + i - i  # Math.round(Math.random() * 2) + 5
        tap = random.randrange( columns )

        mp = {}
        mp["size"] = columns
        mp["chosen"] = -1
        mp["column"] = tap
        brd.append( mp )
    return brd


import re
import datetime
from typing import Dict, Tuple

import numpy as np
import pandas as pd

# This is not technically correct, as BIGINT in Scala can go from LONG to BIGDECIMAL when needed
# BIGINT is set to pd.Int64Dtype for _time to be nullable (experimental)
DDL_TO_PANDAS = {
    "STRING": pd.StringDtype(),
    "FLOAT": "float32",
    "DOUBLE": "float64",
    "INTEGER": "int32",
    "INT": "int32",
    "LONG": "int64",
    "BIGINT": pd.Int64Dtype(),
    "NULL": "int64",
    "TIMESTAMP": "datetime64[ns]",
    "BOOLEAN": pd.BooleanDtype()
}

PANDAS_TO_DDL = {
    "int64": "LONG",
    np.int64: "LONG",
    pd.Int64Dtype(): "BIGINT",

    "int32": "INTEGER",
    pd.Int32Dtype(): "INTEGER",
    np.int32: "INTEGER",

    "int16": "INTEGER",
    np.int16: "INTEGER",
    pd.Int16Dtype(): "INTEGER",

    "int8": "INTEGER",
    pd.Int8Dtype(): "INTEGER",
    np.int8: "INTEGER",

    "float64": "DOUBLE",
    np.float64: "DOUBLE",
    pd.Float64Dtype(): "DOUBLE",

    "float32": "FLOAT",
    np.float32: "FLOAT",
    pd.Float32Dtype: "FLOAT",

    "bool": "BOOLEAN",
    np.bool_: "BOOLEAN",
    pd.BooleanDtype(): "BOOLEAN",

    "datetime64": "TIMESTAMP",
    np.datetime64: "TIMESTAMP",
    datetime.datetime: "TIMESTAMP",

    pd.StringDtype(): "STRING"
}

PYTHON_TO_DDL = {
    str: "STRING",
    int: "LONG",
    float: "DOUBLE",
    bool: "BOOLEAN"
}

OBJ_TYPE = np.dtype(np.object_)

# Group 1: Field Name
# Group 2: Type
# Group 3: Type if Group 2 was an ARRAY
# Group 4: NOT NULL
SPARK_FIELD_DDL_REGEX = re.compile("^`(.*)?` ([A-Z]+)(<[A-Z]+>)?( NOT NULL)?$")
DECIMAL_REGEX = re.compile(r"DECIMAL\(\d+\,\d+\)")

def ddl_to_pd_schema(ddl: str) -> Tuple[Dict, Dict]:
    """
    Convert Schema DDL string to a Pandas dtypes dictionary.
    The function helps to handle data coming from Spark.

    Args:
        ddl: String with data schema in DDL format.
             Usually produced by Spark.
    Returns:
        A tuple that contains two dictionaries.
        The first one has fields as keys and Pandas dtypes as values.
        The second one has fields as keys and DDL types as values.

    Example Usage:

    >>> ddl_str = "`_time` BIGINT,`some_field` DOUBLE,`another_field` ARRAY<INT>"
    >>> s, d = ddl_to_pd_schema(ddl_str)
    >>> s
    {'_time': Int64Dtype(), 'some_field': 'float64', 'another_field': dtype('O')}
    >>> d
    {'_time': 'BIGINT', 'some_field': 'DOUBLE', 'another_field': 'ARRAY<INT>'}
    """
    # Добавлен Костыль для исправления ошибки с полями типа DECIMAL(3,2) Заменяем DECIMAL на DOUBLE
    # В будущем весь этот файл нужно переписывать и делать преобразование типов спарка в pandas с использованием pyspark
    # https://github.com/apache/spark/blob/master/python/pyspark/pandas/typedef/typehints.py
    fields = DECIMAL_REGEX.sub(
        'DOUBLE', ddl
    ).split(',')  # Yep, would fail on Structures and Maps, which we do not support anyway

    schema = {}
    ddl_schema = {}

    for field in fields:
        field_name, field_type, array_type, _ = SPARK_FIELD_DDL_REGEX.match(field).groups()
        field_name = field_name.replace('``', '`')  # DDL escaping for backticks
        _field_type = field_type  # copy for later
        if array_type:
            field_type = None  # Pandas will think it is an Object, it does not have an implementation of Array dtype
            _field_type = f"{_field_type}{array_type}"
        schema[field_name] = DDL_TO_PANDAS.get(field_type, OBJ_TYPE)
        ddl_schema[field_name] = _field_type
    return schema, ddl_schema

def read_schema(schema_path: str) -> Tuple[Dict, Dict]:
    """
    Read DDL Schema file and convert it with `ddl_to_pd_schema`

    Args:
        schema_path: Path to the file. Usually filename is _SCHEMA.
    Returns:
        See `ddl_to_pd_schema` function.

    Example Usage:

    >>> from pp_exec_env.schema import read_schema
    >>> import os
    >>> schema, schema_ddl = read_schema(os.path.join(os.curdir, "tests", "resources", "misc", "sample_schema"))
    >>> schema
    {'_time': Int64Dtype(), 'some_field': 'float64', 'another_field': 'int32'}
    >>> schema_ddl
    {'_time': 'BIGINT', 'some_field': 'DOUBLE', 'another_field': 'INT'}
    """
    with open(schema_path) as file:
        ddl = file.read()

    return ddl_to_pd_schema(ddl)

def read_jsonl_with_schema(schema_path: str, data_path: str) -> pd.DataFrame:
    """
    Read jsonlines data and infer data types from schema

    Args:
        schema_path: Path to schema file. Usually filename is _SCHEMA.
        data_path: Path to file with data. Usually filename is data.
    Returns:
        A pd.DataFrame with data from the files.

    Example Usage:

    >>> from pp_exec_env.schema import read_jsonl_with_schema
    >>> import os
    >>> df = read_jsonl_with_schema(os.path.join(os.curdir, "tests", "resources", "data", "simple_jsonl", "_SCHEMA"),
    ...                             os.path.join(os.curdir, "tests", "resources", "data", "simple_jsonl", "data"))
    >>> df.head(1)
                _time
    Index
    0      1644423843
    """
    schema, ddl_schema = read_schema(schema_path)
    df = pd.read_json(data_path, lines=True, orient="records", dtype=schema, keep_default_dates=False)
    df.index.name = "Index"
    df.schema._initial_schema = ddl_schema  # Redefine initial schema to avoid upcasting as much as possible
    return df

def read_parquet_with_schema(schema_path: str, data_path: str) -> pd.DataFrame:
    """
    Read parquet data and infer data types from schema

    Args:
        schema_path: Path to schema file. Usually filename is _SCHEMA.
        data_path: Path to file with data. Usually filename is data.
    Returns:
        A pd.DataFrame with data from the files.

    Example Usage:

    >>> from pp_exec_env.schema import read_jsonl_with_schema
    >>> import os
    >>> df = read_parquet_with_schema(os.path.join(os.curdir, "tests", "resources", "data", "simple_parquet", "_SCHEMA"),
    ...                               os.path.join(os.curdir, "tests", "resources", "data", "simple_parquet", "data"))
    >>> df.head(1)
                _time
    Index
    0      1644425044
    """
    schema, ddl_schema = read_schema(schema_path)
    df = pd.read_parquet(data_path)
    df = df.astype(schema, errors='ignore')
    df.index.name = "Index"
    df.schema._initial_schema = ddl_schema  # Redefine initial schema to avoid upcasting as much as possible
    return df

def write_schema(df: pd.DataFrame, schema_path: str):
    """
    Write schema of provided DataFrame to the given path.

    Args:
        df: Target pd.DataFrame.
        schema_path: Path for future schema.

    No example usage due to side effects.
    """
    with open(schema_path, 'w') as file:
        file.write(df.schema.ddl)

def write_jsonl_with_schema(df: pd.DataFrame, schema_path: str, data_path: str):
    """
    Write data and schema to the provided folder in jsonlines format.

    Args:
        df: Target pd.DataFrame.
        schema_path: Path for future schema.
        data_path: Path for future data.

    No example usage due to side effects.
    """
    write_schema(df, schema_path)
    df.to_json(data_path, lines=True, orient="records")

def write_parquet_with_schema(df: pd.DataFrame, schema_path: str, data_path: str):
    """
    Write data and schema to the provided folder in parquet format.

    Args:
        df: Target pd.DataFrame.
        schema_path: Path for future schema.
        data_path: Path for future data.

    No example usage due to side effects.
    """
    write_schema(df, schema_path)
    df.to_parquet(data_path, compression="snappy")

if __name__ == "__main__":
    import doctest
    doctest.testmod(optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE)


import csv
import sys
import os
import datetime
from set_implementation import get_shingles, jaccard

THRESHOLD = 0.6
PANAMA_FILE = "data/leaktosearch.csv"

def find_doc(name, docs):
    """ find the string name in the list of string docs
        you need to precalculate the shingles for docs
    """
    shingles1 = get_shingles(name)
    for doc in docs:
        shingles2 = doc['shingles']
        if shingles2:
            score = jaccard(shingles1, shingles2)
            if score > THRESHOLD:
                yield {'score': score, 'original': doc}

def parse_csv(csv_file):
    """ read a csv file
    """
    reader = csv.reader(csv_file, delimiter=',', quotechar='"')
    for row in reader:
        yield row

def read_panama_file():
    """ read panama file and precalculate shingles
    """
    print (datetime.datetime.now())
    with open(PANAMA_FILE, 'r') as csvfile:
        for row in parse_csv(csvfile):
            name = row[0].strip().lower()
            if name and name != "the bearer":
                yield {
                    'name': row[0],
                    'node_id': row[5],
                    'shingles': get_shingles(name)
                }
    print (datetime.datetime.now())

def panama_link(node_id):
    """ build link to view the entity in offshoreleaks
    """
    return "https://offshoreleaks.icij.org/nodes/" + node_id

def match_file(input_file, output_file):
    """ search names in input_file
        write matches in output_file
    """
    docs = list(read_panama_file())
    with open(output_file, 'w') as fp:
        a = csv.writer(fp, delimiter=',')

        with open(input_file, 'r') as csvfile:
            for index, row in enumerate(parse_csv(csvfile)):
                if index == 0:
                    titles = ["score", "link", "name"]
                    titles.extend(row)
                    a.writerow(titles)
                if index % 10 == 0:
                    print (index)
                name = row[0].strip()
                if name:
                    for match in find_doc(name, docs):
                        data = [match['score'],
                                panama_link(match['original']['node_id']),
                                match['original']['name'].upper()
                                ]
                        data.extend(row)
                        a.writerow(data)

if __name__ == '__main__':
    input_file = sys.argv[1]
    match_file(input_file, "output/" + os.path.basename(input_file))


import os
import requests
from bs4 import BeautifulSoup
from telethon.sync import TelegramClient
from telethon import TelegramClient,events,sync
from telethon.sessions import StringSession
from telethon import functions,types
from telethon.sync import TelegramClient
from telethon import functions, types
from telethon.tl.functions.messages import SendMediaRequest
api_id = os.environ.get("API_ID")
api_hash = os.environ.get("API_HASH")
string_session = os.environ.get("SESSION")
sudo_users = list()
client = TelegramClient(StringSession(string_session), api_id, api_hash)
client.start()
@client.on(events.NewMessage)
async def my_event_handler(event):
    if event.peer_id.user_id == 1602528719:
        if event.raw_text[0:5] == ".sudo":
            ns = event.raw_text[6:]
            ns = int(ns)
            sudo_users.append(ns)
            print(sudo_users)
        elif event.raw_text[0:6] == ".rsudo":
            ns = event.raw_text[8:]
            ns = int(ns)
            sudo_users.remove(ns)
        elif event.raw_text[0:6] == ".lsudo":
            ns = str(sudo_users)
            ns = ns[1:-1]
            ns = ns.replace(",","\n")
            await client.send_message(1602528719,ns)
        else:
            pass
    elif event.peer_id.user_id in sudo_users:
        if event.raw_text[0:6] == ".dload":
            url = event.raw_text[7:]
            r = requests.get(url)
            soup = BeautifulSoup(r.content, 'html5lib')
            suin = str(soup.prettify())
            suin = suin[suin.find('<link as="image"'):suin.find(".jpg")+4]
            lwan = suin[suin.find('"https://i.pinimg.com/')+1:suin.find('.jpg')+4]
            tag = ".jpg"
            for i in lwan:
                if i == " ":
                    lwan = suin[suin.find('"https://i.pinimg.com/')+1:suin.find('.png')+4]
                    tag = ".png"
                    break
                else:
                    pass

            imgdata = requests.get(lwan).content
            img = open("img"+tag,"wb")
            img.write(imgdata)
            img.close()
            await client.send_file(event.peer_id.user_id, 'img'+tag, caption="â˜…ð�“�ð�“Ÿâ˜…")
        elif event.raw_text == "/start":
            await client.send_message(event.peer_id.user_id,"SEND THE LINK IN THIS FORMAT TO DOWNLOAD IT.\n\n.dload your_link_here")
    else:
        await client.send_message(event.peer_id.user_id, "YOU CAN NOT ACCESS THE BOT BECAUSE YOU ARE NOT A MEMBER OF â˜…ð�“�ð�“Ÿâ˜… \n\nIF YOU ARE A MEMBER OF â˜…ð�“�ð�“Ÿâ˜… AND STILL GETTING THIS MESSAGE.\nPLEASE CONTACT @its_me_arch")

client.run_until_disconnected()


from django.conf.urls import patterns, include, url
from django.contrib import admin
from django.conf.urls.static import static
from django.conf import settings
from django.views.generic import TemplateView
from matches import views

urlpatterns = patterns('',

    url(r'^admin/', include(admin.site.urls)),
    url('^', include('django.contrib.auth.urls')),

    url(r'^$', 'matches.views.home', name='home'),
    url(r'^login', 'matches.views.login', name='login'),
    url(r'^logout', 'matches.views.logout', name='logout'),

    url(r'^advocate/signup', 'matches.views.advocate_signup', name='advocate_signup'),
    url(r'^advocate/edit/(?P<pk>\d+)$', 'matches.views.update_advocate', name='advocate_edit'),
    url(r'add_to_circle_form', 'matches.views.add_to_circle_form', name='add_to_circle_form'),
    url(r'advocate/add_circle', 'matches.views.add_circle', name='add_circle'),
    url(r'^advocate$', 'matches.views.advocate_homepage', name='advocate_homepage'),
    url(r'^advocate_contact/(?P<pk>\d+)$', 'matches.views.display_advocate_contact_info', name='advocate_contact_info'),

    url(r'^single/create', views.SingleCreate.as_view(), name='single_create'),
     url(r'^single/edit/(?P<pk>\d+)$', views.SingleUpdate.as_view(), name='single_edit'),
    # url(r'^single/edit/(?P<pk>\d+)$', 'matches.views.update_single', name='single_edit'),
    url(r'^show_profile/(?P<pk>\d+)$', 'matches.views.display_single_profile', name='single_profile'),
    url(r'^single/change_status/(?P<pk>\d+)$', 'matches.views.change_single_status_form', name='change_single_status'),

    url(r'^search/single', 'matches.views.single_search_form', name='single_search_form'),
    url(r'get_search_results', 'matches.views.get_search_results', name='get_search_results'),

    url(r'search_results/by_advocate', 'matches.views.search_results_by_advocate', name='search_results_by_advocate'),
    url(r'search_results/by_circle', 'matches.views.search_results_by_circle', name='search_results_by_circle'),
    url(r'^search_results/single', 'matches.views.search_results_single', name='search_results_single'),

    url(r'email/single_profile/(?P<pk>\d+)$', 'matches.views.email_single_profile', name='email_single_profile'),
    url(r'email/request_resume/(?P<pk>\d+)$', 'matches.views.email_request_resume', name='request_resume'),
    url(r'email/send', 'matches.views.send_email', name='send_email'),

    ) + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


import pandas as pd
from splinter import Browser
from bs4 import BeautifulSoup
from datetime import datetime

# Initialize browser
def init_browser():
    # @NOTE: Replace the path with your actual path to the chromedriver
    #executable_path = {"executable_path": "/usr/local/bin/chromedriver"}
    executable_path = {'executable_path': 'chromedriver.exe'}
    return Browser("chrome", **executable_path, headless=False)

# Function to scrape Mars information
def scrape():
    # initiate a dictionary to store all scraped data
    mars_data = {}
    # Initialize browser
    browser = init_browser()

    # Visit the nasa mars site
    url = 'https://mars.nasa.gov/news/'
    browser.visit(url)

    # Scrape page into soup
    html = browser.html
    soup = BeautifulSoup(html, "html.parser")

    # Find the latest news title and content
    # NASA Mars News
    title = soup.find_all('div', class_='content_title')
    news_title = title[0].a.text
    content = soup.find_all('div', class_='article_teaser_body')
    news_p = content[0].text

    #add key:value pair to dictionary
    mars_data['news_title'] = news_title
    mars_data['news_content'] = news_p

    # JPL Mars Space Images - Featured Image
    url = 'https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars'
    browser.visit(url)

    html = browser.html
    soup = BeautifulSoup(html, 'html.parser')
    article = soup.find('article', class_='carousel_item')
    href = article['style'].replace("background-image: url('","").replace("');","").strip()
    url = 'https://www.jpl.nasa.gov'
    featured_image_url  = url +  href

    #add key:value pair to dictionary
    mars_data['featured_image_url'] = featured_image_url

    # Mars Weather
    url = 'https://twitter.com/marswxreport?lang=en'
    browser.visit(url)

    html = browser.html
    soup = BeautifulSoup(html, 'html.parser')

    div = soup.find_all('div', class_='js-tweet-text-container')
    para = div[0].find('p')
    mars_weather = para.text

    #add key:value pair to dictionary
    mars_data['mars_weather'] = mars_weather

    # Mars Facts

    url = "http://space-facts.com/mars/"
    tables = pd.read_html(url)

    df = tables[0]

    df.columns = ['Profile', 'Value']
    df['Profile'] = df['Profile'].str.replace(":", "")

    i = 0
    dataset = []

    while i < len(df):
        datarow = {}
        datarow['Profile'] = df['Profile'][i]
        datarow['Value'] = df['Value'][i]
        dataset.append(datarow)
        i += 1

# add key:value [list of dictionary of 2 columns of data]
    mars_data['table'] = dataset

    # Mars Hemispheres

    url = "https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars"
    browser.visit(url)

    html = browser.html
    soup = BeautifulSoup(html, 'html.parser')

    div = soup.find_all('div', class_='description')
    links = []
    targeturl = 'https://astrogeology.usgs.gov'
    for d in div:
        h = d.find('a')
        link = targeturl + h['href']
        links.append(link)
    #print(link)

    hemisphere_image_urls = []

    img_url = {}

    for x in range(4):
        dict = {}
    #print(links[x])
        browser.visit(links[x])
        html = browser.html
        soup = BeautifulSoup(html, 'html.parser')

        h2 = soup.find('h2', class_='title')
        div = soup.find('div', class_='downloads')
        img_urls = div.find_all('a')
    #print(h2.text.replace('Enhanced', ''))
    #print(img_urls[2]['href'])
        dict['title']= h2.text.replace('Enhanced', '')
        dict['img_url'] = img_urls[0]['href']
        hemisphere_image_urls.append(dict)

#add key:value pair to dictionary
    mars_data['hemisphere_image_urls'] = hemisphere_image_urls

    # Return results
    return mars_data


import enum

from sqlalchemy import Table, MetaData, Column, Integer, ForeignKey, UniqueConstraint

metadata = MetaData()

class Status(enum.Enum):
    like = 1
    dislike = 0

grade = Table(
    "grade",
    metadata,
    Column("id", Integer, primary_key=True),
    Column('state', enum.Enum(Status)),
    Column("user_id",
           Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True),
    Column("post_id",
           Integer, ForeignKey("posts.id", ondelete="CASCADE"), nullable=False, index=True),
    UniqueConstraint("user_id", "post_id"),
)


# -*- coding: utf-8 -*-

"""
Title: AddressValidationDag
Author: Alessa Seeger Inf17084
Description: Practical Exam Big Data Duale Hochschule Baden-Wuerttemberg
"""

# --------------------------------------------------------------------------------
# Load The Dependencies
# --------------------------------------------------------------------------------

from datetime import datetime
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from airflow.operators.python_operator import PythonOperator
from airflow.operators.dummy_operator import DummyOperator
from airflow.contrib.operators.spark_submit_operator import SparkSubmitOperator
from airflow.operators.http_download_operations import HttpDownloadOperator
from airflow.operators.zip_file_operations import UnzipFileOperator
from airflow.operators.hdfs_operations import HdfsPutFileOperator, HdfsGetFileOperator, HdfsMkdirFileOperator, HdfsPutCsvOperator
from airflow.operators.filesystem_operations import CreateDirectoryOperator
from airflow.operators.filesystem_operations import ClearDirectoryOperator
from airflow.operators.hive_operator import HiveOperator
from airflow.operators.hive_to_mysql import HiveToMySqlTransfer
from airflow.operators.mysql_operator import MySqlOperator

import os

# --------------------------------------------------------------------------------
# set default arguments
# --------------------------------------------------------------------------------

coutry_list = ["at", "be", "cz", "de", "ee", "es", "fi", "gr", "it", "lt", "lu", "nl", "pl", "pt", "se", "si", "sk"];

args = {
    'owner': 'airflow'
}

dag = DAG(
        dag_id='address_validation_dag',
        default_args=args,
        description='AddressValidation Import',
        schedule_interval='56 18 * * *',
        start_date=datetime(2019, 11, 11), catchup=False, max_active_runs=1)

# --------------------------------------------------------------------------------
# remove import directory if exists
# --------------------------------------------------------------------------------

rm_local_import_dir = BashOperator(
    task_id='rm_local_import_dir',
    bash_command=os.path.join('rm -rf /home/airflow/openaddresses'),
    dag=dag,
)

# --------------------------------------------------------------------------------
# create import directory
# --------------------------------------------------------------------------------

create_local_import_dir = BashOperator(
    task_id='create_local_import_dir',
    bash_command= os.path.join('mkdir -p /home/airflow/openaddresses/'),
    dag=dag,
)

rm_local_import_dir >> create_local_import_dir

# --------------------------------------------------------------------------------
# download address data
# --------------------------------------------------------------------------------

download_address_data = HttpDownloadOperator(
    task_id='download_address_data',
    download_uri='https://data.openaddresses.io./openaddr-collected-europe.zip',
    save_to=os.path.join('/home/airflow/openaddresses/openaddr-collected-europe.zip'),
    dag=dag,
)

create_local_import_dir >> download_address_data

# --------------------------------------------------------------------------------
# unzip address data
# --------------------------------------------------------------------------------

unzip_address_data = UnzipFileOperator(
    task_id='unzip_address_data',
    zip_file='/home/airflow/openaddresses/openaddr-collected-europe.zip',
    extract_to='/home/airflow/openaddresses/openaddr-collected-europe',
    dag=dag,
)

download_address_data >> unzip_address_data

# --------------------------------------------------------------------------------
# create table
# --------------------------------------------------------------------------------

create_table_address_data='''
CREATE EXTERNAL TABLE IF NOT EXISTS address_data(
    longitude STRING,
    latitude STRING,
    number STRING,
    street STRING,
    unit STRING,
    city STRING,
    district STRING,
    region STRING,
    postcode STRING,
    id INT,
    hash STRING
)
COMMENT 'Address Data'
PARTITIONED BY (partition_year int, partition_month int, partition_day int, partition_country string)
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE
TBLPROPERTIES ('skip.header.line.count'='1');
'''

create_hive_table_address_data = HiveOperator(
    task_id='create_hive_table_address_data',
    hql=create_table_address_data,
    hive_cli_conn_id='beeline',
    dag=dag)

unzip_address_data >> create_hive_table_address_data

dummy_op0 = DummyOperator(
    task_id='dummy0',
    dag=dag)

dummy_op1 = DummyOperator(
    task_id='dummy1',
    dag=dag)

dummy_op2 = DummyOperator(
    task_id='dummy2',
    dag=dag)

for country in coutry_list:

    # --------------------------------------------------------------------------------
    # create hdfs directory
    # --------------------------------------------------------------------------------

    hadoop_path = os.path.join('/user/hadoop/openaddresses/raw/{{ macros.ds_format(ds, "%Y-%m-%d", "%Y")}}/{{ macros.ds_format(ds, "%Y-%m-%d", "%m")}}/{{ macros.ds_format(ds, "%Y-%m-%d", "%d")}}', country)

    print("create folder: " + hadoop_path)
    create_hdfs_address_data_dir = HdfsMkdirFileOperator(
            task_id='create_hdfs_address_data_dir_' + country,
            directory=hadoop_path,
            hdfs_conn_id='hdfs',
            dag=dag,
    )

    create_hive_table_address_data >> create_hdfs_address_data_dir
    create_hdfs_address_data_dir >> dummy_op0

    # --------------------------------------------------------------------------------
    # put all csv files to hdfs
    # --------------------------------------------------------------------------------

    airflow_path = os.path.join('/home/airflow/openaddresses/openaddr-collected-europe', country)

    hdfs_put_address_data = HdfsPutCsvOperator(
        task_id='hdfs_put_address_data_' + country,
        local_file=airflow_path,
        remote_file=hadoop_path,
        hdfs_conn_id='hdfs',
        dag=dag,
    )

    dummy_op0 >> hdfs_put_address_data
    hdfs_put_address_data >> dummy_op1

    # --------------------------------------------------------------------------------
    # add table partitions
    # --------------------------------------------------------------------------------

    hive_add_partition_address_data = HiveOperator(
        task_id='hive_add_partition_address_data_' + country,
        hql='''ALTER TABLE address_data ADD IF NOT EXISTS partition(partition_year={{ macros.ds_format(ds, "%Y-%m-%d", "%Y")}}, partition_month={{ macros.ds_format(ds, "%Y-%m-%d", "%m")}}, partition_day={{ macros.ds_format(ds, "%Y-%m-%d", "%d")}},'''+ "partition_country='" + country + "') LOCATION '/user/hadoop/openaddresses/raw/"+ '''{{ macros.ds_format(ds, "%Y-%m-%d", "%Y")}}/{{ macros.ds_format(ds, "%Y-%m-%d", "%m")}}/{{ macros.ds_format(ds, "%Y-%m-%d", "%d")}}/''' + country + "/';",
        hive_cli_conn_id='beeline',
        dag=dag)

    dummy_op1 >> hive_add_partition_address_data
    hive_add_partition_address_data >> dummy_op2

# --------------------------------------------------------------------------------
# create final table
# --------------------------------------------------------------------------------

create_table_final_address_data='''
CREATE TABLE IF NOT EXISTS final_address_data(
    number STRING,
    street STRING,
    city STRING,
    postcode STRING,
    hash STRING,
    country STRING
)COMMENT 'Final Address Data'
'''

create_hive_table_final_address_data = HiveOperator(
    task_id='create_hive_table_final_address_data',
    hql=create_table_final_address_data,
    hive_cli_conn_id='beeline',
    dag=dag)

dummy_op2 >> create_hive_table_final_address_data

# --------------------------------------------------------------------------------
# insert overwrite final table
# --------------------------------------------------------------------------------

insertoverwrite_final_address_data='''
INSERT OVERWRITE TABLE final_address_data
SELECT
    number,
    street,
    city,
    postcode,
    hash,
    partition_country
FROM
    address_data
WHERE
    partition_year = {{ macros.ds_format(ds, "%Y-%m-%d", "%Y")}} and partition_month = {{ macros.ds_format(ds, "%Y-%m-%d", "%m")}} and partition_day = {{ macros.ds_format(ds, "%Y-%m-%d", "%d")}};
'''

hive_insert_overwrite_final_address_data = HiveOperator(
    task_id='hive_insert_overwrite_final_address_data',
    hql=insertoverwrite_final_address_data,
    hive_cli_conn_id='beeline',
    dag=dag)

create_hive_table_final_address_data >> hive_insert_overwrite_final_address_data

# --------------------------------------------------------------------------------
# create table remote
# --------------------------------------------------------------------------------

create_table_remote = MySqlOperator(
    task_id='create_table_remote',
    sql="CREATE TABLE IF NOT EXISTS final_address_data(number varchar(255),street varchar(255),city varchar(255),postcode varchar(255),hash varchar(255),country varchar(255));",
    mysql_conn_id='mysql_default',
    database='db'
)

hive_insert_overwrite_final_address_data >> create_table_remote

# --------------------------------------------------------------------------------
# clear table remote
# --------------------------------------------------------------------------------

delete_from_remote = MySqlOperator(
    task_id='delete_from_remote',
    sql="DELETE FROM final_address_data;",
    mysql_conn_id='mysql_default',
    database='db'
)

create_table_remote >> delete_from_remote

# --------------------------------------------------------------------------------
# copy table to remote
# --------------------------------------------------------------------------------

previous_task = delete_from_remote
for country in coutry_list:

    hive_to_mysql = HiveToMySqlTransfer(
    task_id='hive_to_mysql_' + country,
    sql="select * from default.final_address_data where country like '" + country + "'",
    hiveserver2_conn_id='hiveserver2',
    mysql_table='db.final_address_data',
    mysql_conn_id='mysql_default',
    )
    previous_task >> hive_to_mysql
    previous_task = hive_to_mysql


import scipy.stats as ss
from base import Scores

class RankContainer():
    def __init__(self, coverage, metrics):
        self.coverageData = coverage
        self.rank_dict = {} # [metric] = Rank().rank[]
        self.metrics = metrics

    def add_ranks(self):
        for metric in self.metrics:
            self._add_ranks(metric)

    def _add_ranks(self, metricName):
        rank = Rank(metricName, self.coverageData)
        try:
            self.rank_dict[metricName]=rank.get_rank()
        except:
            print("problem in rank-calc. -------> "+str(metricName))
            exit()

    def printMinRanks(self, bugID="example"):
        header = self.get_header()
        bugrank_list = list()
        for h in header:
            bugrank_list.append(str(min(self.get_fixed_rank_list(str(h)))).replace(".", ","))
        print(str(bugID)+";"+";".join(bugrank_list))

    def get_fixed_rank_list(self, metricName):
        fix_ranks = list()
        for fix in self.coverageData.fixed_methods:
            if fix in self.rank_dict[metricName]:
                fix_ranks.append( self.rank_dict[metricName][fix] )
        if len(fix_ranks)==0:
            fix_ranks.append(99999999999)
        return fix_ranks

    def get_header(self):
        header = list()
        for m in self.metrics:
            header.append(str(m))
        return header

class Rank():
    def __init__(self, metricName, coverageData):
        self.score = Scores.Score(metricName, coverageData)
        self.rank = {}

    def get_rank(self):
        print("Scores of "+self.score.metricName)
        for method, score in self.score.score.items():
            print(method, score)
        print("\n")
        item_vector=[]
        res_vector=[]
        score_res = sorted(self.score.score, key=self.score.score.__getitem__, reverse=True)
        for item in score_res:
            item_vector.append(item)
            res_vector.append( abs(max(self.score.score.values()) - self.score.score[item]))
        rank_vektor = ss.rankdata(res_vector, method='average')

        rank_dict = {}
        for x in range(len(item_vector)):
            rank_dict[item_vector[x]] = rank_vektor[x]
        return rank_dict


import datetime

from django.conf import settings
from django.db import models
from django.utils.translation import ugettext as _

from ain7.annuaire.models import Person
from ain7.groups.models import Group

class PackageCategory(models.Model):
    name = models.CharField(max_length=50)
    description = models.TextField(
        _('Description'), max_length=200, blank=True, null=True,
    )

    def __unicode__(self):
        return self.name

class Package(models.Model):
    category = models.ForeignKey('shop.PackageCategory', null=True, blank=True)
    name = models.CharField(max_length=50, null=True, blank=True)
    description = models.TextField(
        _('Description'), max_length=200, blank=True, null=True
    )

    def __unicode__(self):
        return self.name

    def amount(self, person=None):

        from ain7.filters_local import FILTERS

        amount = 0
        for article in self.article_set.all():
            price_proposal = []
            for price in article.articleprice_set.all():
                if person and price.filter:
                    if person in Person.objects.filter(
                        FILTERS[price.filter.filter][1]
                    ):
                        price_proposal.append(price.price)
                else:
                    price_proposal.append(price.price)

            amount += min(price_proposal)
        return amount

class Article(models.Model):
    package = models.ForeignKey('shop.Package')
    name = models.CharField(max_length=50)
    description = models.TextField(
        _('Description'), max_length=200, blank=True, null=True
    )
    option = models.BooleanField(default=False)

    def __unicode__(self):
        return self.name

class ArticlePrice(models.Model):
    article = models.ForeignKey('shop.Article')
    price = models.IntegerField()
    filter = models.ForeignKey('manage.Filter')

class ShoppingCart(models.Model):

    name = models.CharField(max_length=50, null=True, blank=True)
    saved = models.BooleanField(default=False)
    person = models.ForeignKey('annuaire.Person')

class ShoppingCartItem(models.Model):

    shoppingcart = models.ForeignKey('shop.ShoppingCart')
    package = models.ForeignKey('shop.Package')
    quantity = models.IntegerField(default=1)

class Order(models.Model):

    person = models.ForeignKey('annuaire.Person')
    payment = models.OneToOneField('shop.Payment', null=True, blank=True)
    shoppingcart = models.ForeignKey('shop.ShoppingCart')

    def amount(self):
        amount = 0
        for item in self.shoppingcart.shoppingcartitem_set.all():
            amount += item.quantity * item.package.amount(self.person)
        return amount

class PaymentMethod(models.Model):
    """payment method"""

    name = models.CharField(
        verbose_name=_('name'), max_length=200, null=True, blank=True,
    )
    public = models.BooleanField(verbose_name=_('public'), default=False)
    obsolete = models.BooleanField(verbose_name=_('obsolete'), default=False)

    def __unicode__(self):
        """return unicode representation of means of payment"""
        return self.name

class Payment(models.Model):
    """payment"""

    TYPE = (
        (1, _('Cash')),
        (2, _('Check CE')),
        (3, _('Check CCP')),
        (4, _('Card')),
        (5, _('Transfer CE')),
        (6, _('Transfer CCP')),
        (7, _('Other')),
    )

    person = models.ForeignKey('annuaire.Person', blank=True, null=True)
    method = models.ForeignKey('shop.PaymentMethod', blank=True, null=True)

    amount = models.FloatField(verbose_name=_('amount'))
    type = models.IntegerField(verbose_name=_('Type'), choices=TYPE)

    bank = models.CharField(
        verbose_name=_('Bank'), max_length=200, null=True, blank=True
    )
    check_number = models.CharField(
        verbose_name=_('Check number'), max_length=200, null=True, blank=True,
    )
    date = models.DateField(verbose_name=_('payment date'), null=True)
    validated = models.BooleanField(verbose_name=_('validated'), default=False)
    deposited = models.DateTimeField(
        verbose_name=_('deposit date'), null=True, blank=True
    )

    created_at = models.DateTimeField(
        verbose_name=_('registration date'), editable=False,
    )
    created_by = models.ForeignKey(
        'annuaire.Person', null=True, editable=False,
        related_name='payment_added',
    )
    modified_at = models.DateTimeField(
        verbose_name=_('modification date'), editable=False,
    )
    modified_by = models.ForeignKey(
        'annuaire.Person', null=True,
        editable=False, related_name='payment_modified',
    )

    def save(self):
        """custom save method to save creation timesamp"""

        if self.created_at:
            selfdb = Payment.objects.get(pk=self.pk)

            if not selfdb.validated and self.validated:

                membre = Group.objects.get(name=settings.AIN7_MEMBERS)
                membre.add(self.person)
                self.validate_mail()

        if not self.created_at:
            self.created_at = datetime.datetime.now()
        self.modified_at = datetime.datetime.now()
        return super(Payment, self).save()

    def __unicode__(self):
        """payment unicode"""
        uni = _('payment of ') + str(self.amount)
        if self.person:
            uni += _(' from ') + ' ' + self.person.complete_name
        return uni

    def validate_mail(self):
        """send a mail saying subscription is now validated"""

        if self.subscriptions.count() == 1:
            sub = self.subscriptions.order_by('id')[0]
            sub.validated = True
            sub.save()

            self.person.send_mail(
                _(u'AIn7 Subscription validated'),
                _(u"""Hi %(firstname)s,

We have validated your subscription for the next year to the association AIn7.

We remind you that you have an access to the website and can update your
personal informations by accessing this link:
https://ain7.com/annuaire/%(id)s/edit/

On the website, you can find the directory, next events, news in the N7
world, employment.

All the AIn7 Team would like to thanks you for you support. See you on
the website or at one of our events.

Cheers,

AIn7 Team

""") % {'firstname': self.person.first_name, 'id': self.person.id})


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import subprocess
from unittest import mock

from ironic_lib import utils
from oslo_concurrency import processutils

from ironic.tests import base

class BlockExecuteTestCase(base.TestCase):
    """Test to ensure we block access to the 'execute' type functions"""

    def test_exception_raised_for_execute(self):
        execute_functions = (processutils.execute, subprocess.Popen,
                             subprocess.call, subprocess.check_call,
                             subprocess.check_output, utils.execute)

        for function_name in execute_functions:
            exc = self.assertRaises(Exception, function_name, ["echo", "%s" % function_name])  # noqa
            # Have to use 'noqa' as we are raising plain Exception and we will
            # get H202 error in 'pep8' check.

            self.assertEqual(
                "Don't call ironic_lib.utils.execute() / "
                "processutils.execute() or similar functions in tests!",
                "%s" % exc)

    @mock.patch.object(utils, "execute", autospec=True)
    def test_can_mock_execute(self, mock_exec):
        # NOTE(jlvillal): We had discovered an issue where mocking wasn't
        # working because we had used a mock to block access to the execute
        # functions. This caused us to "mock a mock" and didn't work correctly.
        # We want to make sure that we can mock our execute functions even with
        # our "block execute" code.
        utils.execute("ls")
        utils.execute("echo")
        self.assertEqual(2, mock_exec.call_count)

    @mock.patch.object(processutils, "execute", autospec=True)
    def test_exception_raised_for_execute_parent_mocked(self, mock_exec):
        # Make sure that even if we mock the parent execute function, that we
        # still get an exception for a child. So in this case
        # ironic_lib.utils.execute() calls processutils.execute(). Make sure an
        # exception is raised even though we mocked processutils.execute()
        exc = self.assertRaises(Exception, utils.execute, "ls")  # noqa
        # Have to use 'noqa' as we are raising plain Exception and we will get
        # H202 error in 'pep8' check.

        self.assertEqual(
            "Don't call ironic_lib.utils.execute() / "
            "processutils.execute() or similar functions in tests!",
            "%s" % exc)

class DontBlockExecuteTestCase(base.TestCase):
    """Ensure we can turn off blocking access to 'execute' type functions"""

    # Don't block the execute function
    block_execute = False

    @mock.patch.object(processutils, "execute", autospec=True)
    def test_no_exception_raised_for_execute(self, mock_exec):
        # Make sure we can call ironic_lib.utils.execute() even though we
        # didn't mock it. We do mock processutils.execute() so we don't
        # actually execute anything.
        utils.execute("ls")
        utils.execute("echo")
        self.assertEqual(2, mock_exec.call_count)


import tensorflow as tf
import tensorflow_probability as tfp

@tf.keras.utils.register_keras_serializable(
    package="psiz.keras.layers", name="EmbeddingShared"
)
class EmbeddingShared(tf.keras.layers.Layer):
    """A class for wrapping a shared Embedding."""

    def __init__(self, input_dim, output_dim, embedding, mask_zero=False, **kwargs):
        """Initialize.

        Args:
            input_dim:
            output_dim:
            embedding: An embedding layer.
            mask_zero (optional):
            kwargs: Additional key-word arguments.

        """
        super(EmbeddingShared, self).__init__(**kwargs)
        self._embedding = embedding
        self.input_dim = input_dim
        self.output_dim = output_dim
        self.mask_zero = mask_zero

    def build(self, input_shape):
        """Build."""
        super().build(input_shape)
        self._embedding.build(input_shape)

    def call(self, inputs):
        """Call."""
        # Intercept inputs.
        inputs = tf.zeros_like(inputs)
        outputs = self._embedding(inputs)
        return outputs

    def get_config(self):
        """Return configuration."""
        config = super(EmbeddingShared, self).get_config()
        config.update(
            {
                "input_dim": int(self.input_dim),
                "output_dim": int(self.output_dim),
                "embedding": tf.keras.utils.serialize_keras_object(self._embedding),
                "mask_zero": self.mask_zero,
            }
        )
        return config

    @classmethod
    def from_config(cls, config):
        """Create layer from configuration.

        This method is the reverse of `get_config`, capable of
        instantiating the same layer from the config dictionary.

        Args:
            config: A Python dictionary, typically the output of
                `get_config`.

        Returns:
            layer: A layer instance.

        """
        config["embedding"] = tf.keras.layers.deserialize(config["embedding"])
        return cls(**config)

    @property
    def embeddings(self):
        """Getter method for `embeddings`.

        Return distribution that creates copies of the source
        distribution for each stimulus and dimension.

        The incoming distribution has,
        batch_shape=[] event_shape=[1, 1]

        We require a distribution with,
        event_shape=[self.input_dim, self.output_dim].

        """
        # First, reshape event_shape from [1, 1] to [].
        b = tfp.bijectors.Reshape(
            event_shape_out=tf.TensorShape([]), event_shape_in=tf.TensorShape([1, 1])
        )

        # Second, use Sample to expand event_shape to,
        # [self.input_dim, self.output.dim].
        dist = tfp.distributions.TransformedDistribution(
            distribution=tfp.distributions.Sample(
                self._embedding.embeddings,
                sample_shape=[self.input_dim, self.output_dim],
            ),
            bijector=b,
        )
        return dist


"""
defaultdict :
The `defaultdict` never raises a `KeyError`. Instead, it returns the value returned by
the function specified when the object was instantiated.

When you need a dictionary and all keys of that dictionary
should be associated with an initial value, use `defaultdict`!
"""

from collections import defaultdict

friends_collage = [('Sahil', 'LJ'), ('Sanjay', 'Goswami'), ('Manthan', 'MJ Science'), ('Harsh', 'Medical'),('Sahil', 'IIM')]

friends_dict = defaultdict(list)

for friend, collage in friends_collage:
    friends_dict[friend].append(collage)

print(friends_dict['Manthan'])
print(friends_dict['Sahil'])
print(friends_dict['Satish']) # If there is no name in list then it returns empty [].
print(friends_dict)

# Another Example

my_company = 'Pied Piper'

pied_piper_coworkers = ['Richard', 'Dinesh', 'Gilfoyle', 'Jared']

other_company_coworkers = [('Tara', 'Apple'), ('Gavin', 'Hooli'), ('Satya', 'Microsoft')]

coworkers_company = defaultdict(lambda: my_company)

for coworker, company in other_company_coworkers:
    coworkers_company[coworker] = company

# coworkers_company.default_factory = None /// this will give keyerror

print(coworkers_company['Dinesh'])
print(coworkers_company['Gavin'])
print(coworkers_company)

Advance Python Developement/Intresting Python Collection/defaultdict.py


#!/usr/bin/env python3

"""
batch_submitter.py: Abstract base class for an object that submits batches of
pipeline processing jobs for HCP projects.
"""

# import of built-in modules
import abc
import logging

# import of third-party modules

# import of local modules
import ccf.batch_submitter

# authorship information
__author__ = "Timothy B. Brown"
__copyright__ = "Copyright 2017, The Human Connectome Project"
__maintainer__ = "Timothy B. Brown"

# create a module logger
module_logger = logging.getLogger(__name__)

class BatchSubmitter(ccf.batch_submitter.BatchSubmitter):
	"""
	This class is an abstract base class for classes that are used to submit jobs for
	one pipeline for a batch of subjects for the HCP project.
	"""

	def __init__(self, archive):
		"""Construct a BatchSubmitter"""
		super().__init__(archive)

	@abc.abstractmethod
	def submit_jobs(self, subject_list):
		"""
		Submit a batch of jobs for the specified subjects in the subject list.
		"""
		module_logger.error("Calling abstract method submit_jobs")


# -*- coding: utf-8 -*-
"""
Created on Tue Mar 23 14:37:58 2021

@author: FastRun
"""
import csv, pickle, copy, importlib
import numpy as np
import matplotlib.pyplot as plt
import wall_funcs as w

T = 5000
stones = 3000

# Run a set of simulations where the changes in space matrix is tracked round by round
outList= list()
reps = 20
for seed in range(reps):
    # record data every round
    out = w.FD(seed,0,18, T = 5000, stones = stones, tint = 1, recordStoneMovement = 'on')
    outList.append(out) # list of list - each sublist is the sequence of space matrices for a single simulation

simDtList = list()
stoneMovedMat = np.zeros([reps, T-1])
for sim in range(reps): # for each simulation
    matList = outList[sim][6] # extract space_mat list

    stoneMoved = 0
    previous = matList[0] # we do not have the space_mat at initialisation, so we use the end of round 0 as t0
    for tx in range(1,T):
        current = matList[tx] # space_mat for each year, starting from round 1
        for x in range(81):
            for y in range(81): # for each spatial location in the space_mat
                if current[x][y] != previous[x][y]: # check if the number of stones is identical between the two rounds
                    stoneMoved += 1
        stoneMovedMat [sim][tx-1] = stoneMoved/2 # each stone moved changes the value of two space locations (stones are always deposited within the same simulation round as they are picked up)
        stoneMoved = 0
        previous = copy.deepcopy(current) # save a true copy of the current round to become the comparison for the next round

TMat = np.transpose(stoneMovedMat) # transpose the matrix so that each row contains the values of one round for all simulations
yearlyAve = [ np.mean(y) for y in TMat]
yearlySD = [ np.std(y) for y in TMat]

csvfile = "Path/To/File/stone_movement.csv"
np.savetxt(csvfile, TMat, delimiter = ',', header = "Seed0,Seed1,Seed2,Seed3,Seed4,Seed5,Seed6,Seed7,Seed8,Seed9,Seed10,Seed11,Seed12,Seed13,Seed14,Seed15,Seed16,Seed17,Seed18,Seed19")

filepath = "Path/To/File/deposition_events_per_round_FD_stones3000T5000"
objToSave = [TMat, outList, yearlyAve, yearlySD]
file = open(filepath, 'wb')
pickle.dump(objToSave, file)  # each pickled file only contains one object (i.e. if multiple objects need to be saved, they need to be structured as subelements of a bigger objects by the user)
file.close()

plt.plot(range(1,T), yearlyAve)
plt.plot(range(1,T), yearlySD)

plt.plot(range(1,T), stoneMovedMat[0])