import string
import MySQLdb

### 1. read file
### 2. choose to use or not
### 3. extract items
### 4. format(parse?)
### 5. send to database

### the wrapper function for all

def the_data_handler(file_name, ip, user, pw, database):
	wholetext = read_the_file(file_name)
	if(is_useful(wholetext)):
		table_list = mining(wholetext)
		parsing(table_list)

### read from file

def read_the_file(file_name):### return the whole vote file as a string

	file_object = open(file_name, "r")
	wholetext = file_object.read()
	file_object.close()
	return wholetext

### decide usefulness of the file

def is_useful(wholetext):

	if "Measured" not in wholetext:
		return False
	return True

### extract

def mining(wholetext):

	table_list=[]
	Time_String = ""
	Server_Name = ""
	Server_ID = ""
	linelist = wholetext.split('\n')

	for linectr in range(0, len(linelist)):
		### exit when hit signature part
		cur_line = linelist[linectr]

		### extract time
		if cur_line.startswith('published'):
			Time_String = cur_line[10:]

		### extract server ID and name
		if cur_line.startswith('dir-source'):
			templist = cur_line.split(' ')
			Server_Name = templist[1]
			Server_ID = templist[2]

		### extract relay info (ID, name, abw, rbw)
		if cur_line.startswith('r '): ### find a way to jump block
			if "Measured" not in linelist[linectr+3]:
				continue
			templist = cur_line.split(' ')
			Relay_Name = templist[1]
			Relay_ID = templist[2] ### not sure about 3
			if linelist[linectr+3].startswith("w "):

				cur_plus_three = linelist[linectr+3]
				Relay_Advertised = int(cur_plus_three[(cur_plus_three.find("h=")+2):cur_plus_three.find(" M")])
				Relay_Real = int(cur_plus_three[(cur_plus_three.find("d=")+2):])#cur_plus_three.find("\n")])

			### put data in a list
			block_list = [Server_ID, Server_Name, Time_String, Relay_ID, Relay_Name, Relay_Advertised, Relay_Real]
			table_list.append(block_list)

	return table_list

### parsing data into database

def parsing(table_list, ip, user, pw, database):

	db = MySQLdb.connect(ip,user,pw,database)
	cursor = db.cursor()
	for i in range(0, len(table_list)):
		cursor.execute("INSERT INTO tor_oct (ServerID, Server_Name, Time, RelayID, Relay_Name, Advertised_BW, Measured_BW) VALUES(%s, %s, %s, %s, %s, %s, %s)", (table_list[i][0], table_list[i][1], table_list[i][2], table_list[i][3], table_list[i][4], table_list[i][5], table_list[i][6]))
	db.commit()
	db.close()


import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output, State
import dash_bootstrap_components as dbc

from pages import home, optimize
from app import app

PAGES = ['/home', '/optimization']

# the style arguments for the sidebar. We use position:fixed and a fixed width
SIDEBAR_STYLE = {
    "position": "fixed",
    "top": 0,
    "left": 0,
    "bottom": 0,
    "width": "18rem",
    "padding": "2rem 1rem",
    "background-color": "#f8f9fa",
}

# the styles for the main content position it to the right of the sidebar and
# add some padding.
CONTENT_STYLE = {
    "margin-left": "18rem",
    "margin-right": "2rem",
    "padding": "2rem 1rem",
}

sidebar = html.Div([
    html.H1("Greenfield Optimization"),
    html.Hr(),
    # html.P(
        # "", className="lead"
    # ),
    dbc.Nav([
        dbc.NavItem(dbc.NavLink("Home", href="/home")),
        dbc.NavItem(dbc.NavLink("Optimization", href="/optimization"))
        ],
        vertical=True,
        pills=True,
    ),
    ],
    style=SIDEBAR_STYLE,
)

layout = html.Div([
    sidebar,
    dcc.Location(id='base-url', refresh=True),
    dbc.Container(
        html.Div(home.layout,
            id='home-display',
            style={'display': 'none'})
    ),
    dbc.Container(
        html.Div(optimize.layout,
            id='opt-display',
            style={'display': 'none'})
    )
])

@app.callback(
    [Output('home-display', 'style'),
     Output('opt-display', 'style')],
    Input('base-url', 'pathname')
)
def router(pathname):
    output = []
    for p in PAGES:
        if pathname in p and pathname != '/':
            output.append({'display': 'block'})
        elif pathname == '/' and '/home' in pathname:
            output.append({'display': 'block'})
        else:
            output.append({'display': 'none'})

    return output


import numpy as np
from toolz import identity, pluck
import linreg as lr
import glm
import metrics
from utility import Scaler
from ml_util import train_test_split
from out_utils import plot_cost, plot_errors

# Get the data
input = np.loadtxt('./data/Folds.csv', delimiter=',', skiprows=1)
Z = np.array(list(pluck(list(range(0, len(input[0])-1)), input)))
y = np.array(list(pluck(len(input[0])-1, input)))
data = zip(Z, y)
# Split into a train set and test set
train_data, test_data = train_test_split(data, 0.33)
# Scale the training data
scale = Scaler()
Z_train, y_train = zip(*train_data)
scale.fit(Z_train)
X_train = scale.transform(Z_train)
scaledtrain_data = list(zip(X_train, y_train))
# Scale the testing data using the same scaling parameters
# used for the training data
Z_test, y_test = zip(*test_data)
X_test = scale.transform(Z_test)

print('****Minibatch Gradient Descent****')
print('\n--Training--\n')
hyperparam = {'eta': 0.3,
              'epochs': 300,
              'minibatches': 1,
              'adaptive': 0.99}
print('\nHyperparamters\n')
for k, v in hyperparam.items():
    print(k, '\t', v)
print('\nNumber of Training Examples: ', X_train.shape[0], '\n')

h_thetaf, cost = glm.fit(lr.J,
                         lr.gradJ,
                         hyperparam,
                         scaledtrain_data)
plot_cost(cost)
h_thetad = scale.denormalize(h_thetaf)
print('Coefficients\t', h_thetaf)
for i, h_theta in enumerate(h_thetad):
    print('h_theta' + str(i), '\t', h_theta)
yp_train = glm.predict(identity, X_train, h_thetaf)
plot_errors(y_train, yp_train)
corr_train = metrics.r2(X_train, y_train, h_thetaf)
print('R**2\t', corr_train)

print('\n--Testing--')
yp_test = glm.predict(identity, X_test, h_thetaf)
plot_errors(y_test, yp_test)
corr_test = metrics.r2(X_test, y_test, h_thetaf)
print('R**2\t', corr_test)


from collections import namedtuple
n = int(input())
heads = input()
x = namedtuple('record', heads)
records = []
for i in range(n):
    _ = list(map(lambda x: x.strip(), input().split()))
    records.append(x(*_))
total = 0
for r in records:
    total += int(r.MARKS)
print(total/n)


#-*- coding:utf-8 -*-

from sklearn.neighbors import KNeighborsClassifier

class KNNSolver:

    def __init__(self, distance, neighbors):
        self.distance = distance
        self.neighbors = neighbors

    def train_and_decode(self, train_X, test_X, train_y, test_y):
        print('Begin to train\ndistance type:\t{}\tnumber of neighbors:\t{}'.format(self.distance, self.neighbors))
        knn = KNeighborsClassifier(n_neighbors=self.neighbors, metric=self.distance)
        knn.fit(train_X, train_y)
        acc = knn.score(test_X, test_y)
        return acc


import functools
from typing import Tuple, List

from loguru import logger
import numpy as np
import pandas as pd

from covid_model_seiir_pipeline.lib import (
    math,
    parallel,
)

def find_idr_floor(pred: pd.Series,
                   daily_cases: pd.Series,
                   serosurveys: pd.DataFrame,
                   population: pd.Series,
                   hierarchy: pd.DataFrame,
                   test_range: List,
                   progress_bar: bool,
                   num_threads: int) -> Tuple[pd.DataFrame, pd.Series]:
    _tfv = functools.partial(
        _test_floor_value,
        pred=pred.copy(),
        daily_cases=daily_cases.copy(),
        serosurveys=serosurveys.copy(),
        population=population.copy(),
        hierarchy=hierarchy.copy(),
    )
    floors = (np.array(test_range) / 100).tolist()
    rmse = parallel.run_parallel(
        _tfv,
        arg_list=floors,
        num_cores=num_threads,
        progress_bar=progress_bar,
    )
    rmse = pd.concat(rmse).reset_index()

    best_floor = (rmse
                  .groupby('location_id')
                  .apply(lambda x: x.sort_values('rmse')['floor'].values[0])
                  .rename('idr_floor'))

    rmse, best_floor = _manual_floor_setting(
        rmse, best_floor, hierarchy,
        serosurveys.reset_index()['location_id'].unique().tolist()
    )

    return rmse, best_floor

def _test_floor_value(floor: float,
                      pred: pd.Series,
                      daily_cases: pd.Series,
                      serosurveys: pd.Series,
                      population: pd.Series,
                      hierarchy: pd.DataFrame,
                      min_children: int = 3) -> pd.DataFrame:
    pred = (pred
            .groupby(level=0)
            .apply(lambda x: math.scale_to_bounds(x, floor, 1.))
            .rename('idr'))

    daily_infections = (daily_cases / pred).dropna()
    daily_infections = daily_infections.dropna()
    cumulative_infections = daily_infections.groupby('location_id').cumsum()
    seroprevalence = (cumulative_infections / population).rename('seroprevalence')

    residuals = (seroprevalence - serosurveys).dropna().rename('residuals')

    rmses = pd.Series([],
                      name='rmse',
                      dtype='float',
                      index=pd.Index([], name='location_id'))
    location_ids = hierarchy.sort_values(['level', 'sort_order'])['location_id']
    for location_id in location_ids:
        in_path = hierarchy['path_to_top_parent'].apply(lambda x: str(location_id) in x.split(','))
        child_ids = hierarchy.loc[in_path, 'location_id'].to_list()
        if location_id not in [95, 4749, 434]:
            # exclude England and Scotland if level above UK (too much data in those places; swamps algorithm)
            child_ids = [c for c in child_ids if c not in [4749, 434]]
        is_location = hierarchy['location_id'] == location_id
        parent_id = hierarchy.loc[is_location, 'parent_id'].item()
        # check if location_id is present
        if location_id in serosurveys.reset_index()['location_id'].to_list():
            try:
                rmse = np.sqrt((residuals[location_id]**2).mean())
            except KeyError:
                logger.warning(f"Can't find data for location id {location_id}")
                continue
        # check if at least `min_children` children are present
        elif serosurveys.reset_index()['location_id'].drop_duplicates().isin(child_ids).sum() >= min_children:
            child_residuals = residuals.reset_index()
            child_residuals = (child_residuals
                               .loc[child_residuals['location_id'].isin(child_ids)]
                               .set_index('date')
                               .loc[:, 'residuals'])
            rmse = np.sqrt((child_residuals**2).mean())
        # check if parent is present
        elif parent_id in rmses.index:
            rmse = rmses[parent_id]
        # we have a problem
        else:
            raise ValueError(f'No source of seroprevalence RMSE for location_id {location_id}.')
        rmse = pd.Series(rmse,
                         name='rmse',
                         index=pd.Index([location_id], name='location_id'))
        rmses = pd.concat([rmses, rmse])
    rmses = rmses.to_frame()
    rmses['floor'] = floor

    return rmses

def _manual_floor_setting(rmse: pd.DataFrame,
                          best_floor: pd.Series,
                          hierarchy: pd.DataFrame,
                          data_locations: List[int]) -> Tuple[pd.DataFrame, pd.Series]:
    is_ssa_location = hierarchy['path_to_top_parent'].apply(lambda x: '166' in x.split(','))
    ssa_location_ids = hierarchy.loc[is_ssa_location, 'location_id'].to_list()
    ssa_location_ids = [l for l in ssa_location_ids if l not in data_locations]

    flagged = False
    for ssa_location_id in ssa_location_ids:
        if best_floor.loc[ssa_location_id] > 0.001:
            if not flagged:
                logger.warning('Manually setting IDR floor of 0.1% for SSA locations.')
                flagged = True
            best_floor.loc[ssa_location_id] = 0.001
            is_ssa_rmse = rmse['location_id'] == ssa_location_id
            rmse.loc[is_ssa_rmse, 'rmse'] = np.nan
            rmse.loc[is_ssa_rmse, 'floor'] = 0.001

    # Amazonas - 0.2%
    best_floor.loc[4752] = 0.002
    rmse.loc[4752, 'rmse'] = np.nan
    rmse.loc[4752, 'floor'] = 0.002

    # Maranhao - 2.0%
    best_floor.loc[4759] = 0.02
    rmse.loc[4759, 'rmse'] = np.nan
    rmse.loc[4759, 'floor'] = 0.02

    # Afghanistan - 0.2%
    best_floor.loc[160] = 0.002
    rmse.loc[160, 'rmse'] = np.nan
    rmse.loc[160, 'floor'] = 0.002

    # South Africa - 2%
    best_floor.loc[196] = 0.02
    rmse.loc[196, 'rmse'] = np.nan
    rmse.loc[196, 'floor'] = 0.02

    return rmse, best_floor


import numpy as np

import igraph as ig

#im
def sharedNearestNeighbors(adj_matrix_boolean,threshold_neighbors):
    '''
        Input:  adjacency matrix of the graph
                threshold for the neighbors
        -------------------------------------
        Output: graph_shared
                label

    '''

    size = adj_matrix_boolean.shape[0]
    adj_matrix_shared = np.zeros((size, size))
    max_neighbors     = np.zeros(size)
    label = np.zeros(size,dtype=int)
    count_label = 1

    for i in range(0, size):

        if i == 0:
            label[i] = 1
            count_label = count_label + 1  # new label found

        for j in range(0, size):
            # neighbors = adj_matrix_boolean[i,:] == adj_matrix_boolean[j,:]
            temp_i = adj_matrix_boolean[i, :]
            temp_j = adj_matrix_boolean[j, :]
            neighbors = np.array(np.where(temp_i==temp_j ))

            if neighbors.size > threshold_neighbors:

                if label[i] == 0:  # label is empty for i

                    if label[j] != 0:  # label j is not empty

                        # use label j
                        label[i] = label[j]

                    if label[j] == 0:
                        # not label assigned
                        label[j] = count_label
                        label[i] = count_label
                        count_label = count_label + 1 # new label
                        if neighbors.size > max_neighbors[i]:
                            max_neighbors[i] = neighbors.size

                if label[i] !=0:
                    # if i has already a label

                    if  neighbors.size > max_neighbors[i]:
                        # new label
                        max_neighbors[i] = neighbors.size
                        label[i] = label[j]

                adj_matrix_shared[i, j] = 1
                adj_matrix_shared[j, i] = 1

    graph_shared = ig.Graph.Adjacency(adj_matrix_shared.tolist())  # create a graph from adjacency matrix

    return graph_shared,label


import datetime

import numpy as np
from scipy import stats
from statistics import stdev
import statistics
import math

from model.data_types import GridParamType, PupilLearningState

def get_num_disruptors(model):
    return len(
        [p for p in model.schedule.agents if p.learning_state == PupilLearningState.RED]
    )

def get_num_passive(model):
    return len(
        [
            p
            for p in model.schedule.agents
            if p.learning_state == PupilLearningState.YELLOW
        ]
    )

def get_num_learning(model):
    return len(
        [
            p
            for p in model.schedule.agents
            if p.learning_state == PupilLearningState.GREEN
        ]
    )

def get_pupil_data(model, student_id):
    for p in model.schedule.agents:
        if int(p.student_id) == student_id:
            return round(p.e_math, 2)

def get_date_for_chart(model):
    # Gets the date/time to display on the chart, spreading ticks equally
    # through the school day (9am-3pm) and putting the last tick of the day at
    # 6pm.
    ticks_in_day = model.schedule.steps % model.ticks_per_school_day
    if ticks_in_day == 0:
        # Date has updated to next day so set time to 6pm the day before (at
        # end of home learning time)
        time = datetime.time(18, 0, 0)
        date = model.current_date - datetime.timedelta(days=1)
    else:
        # Work out how far through the school day (9am-3pm) we are
        date = model.current_date
        day_fraction = ticks_in_day / model.ticks_per_school_day
        minutes = int(day_fraction * 360)
        time = datetime.time(9 + minutes // 60, minutes % 60, 0)

    date_with_time = datetime.datetime.combine(date, time, tzinfo=datetime.timezone.utc)
    return date_with_time.timestamp()

def compute_ave(model):
    return round(statistics.mean([agent.e_math for agent in model.schedule.agents]), 2)

def get_grid_size(n_agents, max_group_size):
    # Calculate squarest grid size that will fit the given number of agents in
    # the given number of groups
    if max_group_size > n_agents:
        max_group_size = n_agents

    n_groups = math.ceil(n_agents / max_group_size)
    n_full_groups = n_groups
    if n_groups * max_group_size != n_agents:
        # With very large groups the difference between n_groups * max_group_size
        # and n_agents can be more than the number of groups, so to get evenly-sized
        # groups we need to decrease the max group size.
        while n_groups * max_group_size - n_agents > n_groups:
            max_group_size -= 1
        n_full_groups = n_agents % n_groups

    n_group_rows = math.ceil(math.sqrt(n_groups))
    n_group_cols = math.ceil(n_groups / n_group_rows)
    group_width = math.ceil(math.sqrt(max_group_size))
    group_height = math.ceil(max_group_size / group_width)

    grid_width = n_group_cols * (group_width + 1) - 1
    grid_height = n_group_rows * (group_height + 1) - 1
    return GridParamType(
        width=grid_width,
        height=grid_height,
        n_groups=n_groups,
        n_full_groups=n_full_groups,
        max_group_size=max_group_size,
        n_group_cols=n_group_cols,
        n_group_rows=n_group_rows,
        group_width=group_width,
        group_height=group_height,
    )

def min_neighbour_count_to_modify_state(n_neighbours, default_threshold, group_size):
    # if 8 or more neighbours, state should change if 6 or more neighbours are
    # red/green
    if n_neighbours >= 8:
        return default_threshold

    # otherwise we use the same proportion but round down so e.g. only 1 in a
    # pair will change behaviour
    return max(1, math.floor(group_size * default_threshold / 8))

def get_start_state_weights(inattentiveness, hyper_impulsive):
    # Calculate weights for a pupil's starting state, based on their
    # inattentiveness and hyper_impulsive scores.
    # A high inattentiveness score gives a greater chance of being yellow than
    # green; a high hyper_impulsive score gives a greater chance of being red
    # than yellow
    # Returns an array of weights for [green, yellow, red] states
    green_weight = (6 - inattentiveness) / 10
    red_weight = hyper_impulsive / 10
    yellow_weight = 1 - green_weight - red_weight
    return [green_weight, yellow_weight, red_weight]


from args import *

# parse CLI arguments
args, unknown = parser.parse_known_args()

# number of rings
nring = args.nring

# number of cells per ring
ncell = args.ncell

# number of cells per type
ncell_per_type = args.npt

# min, max random number of dend sections (random tree topology)
nbranch = args.branch

# min, max random nseg for each branch
ncompart = args.compart

# if use gap junctions
usegap = args.gap

# stop time of simulation
tstop = args.tstop

# whether to randomize cell parameters
randomize_parameters = args.rparm

# number of distinct cell types (same branching and compartments)
# each cell has random type [0:ntype]
ntype = int((nring * ncell - 1) / ncell_per_type + 1)

# whether to run via in-memory transfer mode or file mode
# Of filemode is true then model is dumped to file and then
# passed to coreneuron. In in-memory mode model is passed to
# coreneuron via in-memory copy.
coreneuron_file_mode = args.filemode

# whether to run coreneuron or neuron
use_coreneuron = args.coreneuron

# whether to run coreneuron on GPU
coreneuron_gpu = args.gpu

# cell permutation type
coreneuron_permute = args.permute

# permute type is default 0 for GPU then choose type 2
if coreneuron_gpu and coreneuron_permute == 0:
    coreneuron_permute = 2

from ring import *
from neuron import h
from commonutils import *
import settings

# initialize global variables
settings.init(usegap, nring)

# note that if branching is small and variation of nbranch and ncompart
# is small then not all types may have distinct topologies
# CoreNEURON will print number of distinct topologies.

# initialize seed
h.Random().Random123_globalindex(args.gran)

h.load_file('stdgui.hoc')

pc = h.ParallelContext()

# set number of threads
pc.nthread(args.nt, 1)

if settings.rank == 0:
    print ("%s %s" % (str(nbranch), str(ncompart)))
    print ("nring=%d\ncell per ring=%d\nncell_per_type=%d" % (nring, ncell, ncell_per_type))
    print ("ntype=%d" % ntype)

#from cell import BallStick
h.load_file("cell.hoc")

typecellcnt = [[i, 0] for i in range(ntype)]

def multisplit():
    h.load_file("parcom.hoc")
    parcom = h.ParallelComputeTool()
    parcom.multisplit(1)
    if settings.rank == 0:
        lb = parcom.lb
        print ('multisplit rank 0: %d pieces  Load imbalance %.1f%%' % (lb.npiece, (lb.thread_cxbal_ -1)*100))

def prun(tstop):
    runtime = h.startsw()
    wait = pc.wait_time()

    # run simulation
    pc.psolve(tstop)

    # calculate various time statistics for profiling/debugging

    wait = pc.wait_time() - wait
    runtime = h.startsw() - runtime

    # Load balance is average computation time / max computation time
    computation_time = pc.step_time()

    avg_comp_time = pc.allreduce(computation_time, 1) / settings.nhost
    max_comp_time = pc.allreduce(computation_time, 2)

    load_balance = (avg_comp_time / max_comp_time) if max_comp_time else 0.0

    # spike communication time
    spk_time = (pc.allreduce(wait, 2), pc.allreduce(wait, 3))

    # gap junction transfer time
    gap_time = (pc.allreduce(pc.vtransfer_time(), 2), pc.allreduce(pc.vtransfer_time(), 3))

    return runtime, load_balance, avg_comp_time, spk_time, gap_time

if __name__ == '__main__':

    ## Create all rings ##

    timeit(None, settings.rank)

    # create network / ring of cells
    ran = h.Random()
    ran.Random123(0, 1)
    types = shuffle([i % ntype for i in range(ncell * nring)], ran)
    rings = [Ring(ncell, nbranch, ncompart, i * ncell, types) for i in range(nring)]

    timeit("created rings", settings.rank)

    # randomize parameters if asked
    if randomize_parameters:
        from ranparm import cellran
        for ring in rings:
            for gid in ring.gids:
                if pc.gid_exists(gid):
                    cellran(gid, ring.nclist)
        timeit("randomized parameters", settings.rank)

    ## CoreNEURON setting ##

    h.cvode.cache_efficient(1)

    if use_coreneuron:
        from neuron import coreneuron
        coreneuron.enable = True
        coreneuron.file_mode = coreneuron_file_mode
        coreneuron.gpu = coreneuron_gpu
        coreneuron.cell_permute = coreneuron_permute

        if args.multisplit is True:
            print("Error: multi-split is not supported with CoreNEURON\n")
            quit()

    ## Record spikes ##

    spike_record()

    ## Various CLI options ##

    if args.multisplit:
        multisplit()

    if args.show:
        h.topology()

    if usegap:
        pc.setup_transfer()

    # register section/segment mapping
    if args.dumpmodel:
        recordlist = setup_nrnbbcore_register_mapping(rings)
        # NOTE: once simulation finishes, one can print
        # voltages as below
        # voltageout("coredat", recordlist)

    ## Initialize ##

    pc.set_maxstep(10)
    h.stdinit()
    timeit("initialized", settings.rank)

    ## Dump model to file ##

    if args.dumpmodel:
        pc.nrnbbcore_write("coredat")

    ##  Run simulation ##

    runtime, load_balance, avg_comp_time, spk_time, gap_time = prun(tstop)
    timeit("run", settings.rank)

    ## Write spike raster ##

    spikeout(".")

    ## Print stats ##

    pc.barrier()
    if settings.rank == 0:
        print("runtime=%g  load_balance=%.1f%%  avg_comp_time=%g" %
              (runtime, load_balance * 100, avg_comp_time))
        print("spk_time max=%g min=%g" % (spk_time[0], spk_time[1]))
        print("gap_time max=%g min=%g" % (gap_time[0], gap_time[1]))

    h.quit()


# pip install telebot
import telebot

import numpy as np

from io import BytesIO
# pip install pillow
from PIL import Image

# pip install fastai
from fastai.vision.all import *

bot_token = '-------------YOUR_TOKEN-------------'
bot = telebot.TeleBot(bot_token, parse_mode='html')

model = load_learner('model.pkl')

def get_image_from_bytes(b):
    '''
    Gets the PIL.Image object from the bytes of photo
    :param b: Bytes of the photo
    :type b: bytes
    '''
    stream = BytesIO(b)
    image = Image.open(stream).convert("RGB")
    stream.close()
    return image

def classify_image(img):
    img = get_image_from_bytes(img)
    categories = ('Car without damage', 'Damaged car')
    pred, idx, probs = model.predict(img)
    return dict(zip(categories, map(float, probs)))

@bot.message_handler(commands=['start'])
def send_welcome(message):
    send = bot.send_message(message.from_user.id, '😃Hello! Please send me a photo of the car so I can determine if it is damaged or not.')
#    bot.register_next_step_handler(send, get_user_pics)

@bot.message_handler(content_types=['text', 'file'])
def send_error(message):
    send = bot.send_message(message.from_user.id, '🙁Invalid input, send a photo of the car, please')

@bot.message_handler(content_types=['photo'])
def get_user_pics(message):
# Получаем id фотографии в Telegram
    photo_id = message.photo[-1].file_id
# Достаём картинку
    photo_file = bot.get_file(photo_id) # <class 'telebot.types.File'>
    photo_bytes = bot.download_file(photo_file.file_path) # <class 'bytes'>
# Отправить в дальнейшем можно таким образом
#    send = bot.send_photo(message.from_user.id, photo=photo_bytes)
    send = bot.send_message(message.from_user.id, f'💥The probability its damaged car is {round(classify_image(photo_bytes).get("Damaged car")*100, 2)}% \n'
                                                  f'🚗The probability its car without damage is {round(classify_image(photo_bytes).get("Car without damage")*100, 2)}%')

bot.polling()


import operator
import math
import random
import numpy as np

from deap import algorithms
from deap import base
from deap import creator
from deap import tools
from deap import gp

import matplotlib.pyplot as plt
import networkx as nx
from networkx.drawing.nx_agraph import graphviz_layout

import re

coeff_scale = 10
max_depth = 10

# Define new functions
def protectedDiv(left, right):
    try:
        return left / right
    except ZeroDivisionError:
        return 1

pset = gp.PrimitiveSet("PerfModel", 6)
pset.addPrimitive(operator.add, 2)
pset.addPrimitive(operator.mul, 2)
pset.addEphemeralConstant("rand101", lambda: coeff_scale*random.randint(-1, 1)*random.random())
pset.renameArguments(ARG0='o1')
pset.renameArguments(ARG1='o2')
pset.renameArguments(ARG2='o3')
pset.renameArguments(ARG3='o4')
pset.renameArguments(ARG4='o5')
pset.renameArguments(ARG5='o6')

def create_indv():
    init_model = gp.PrimitiveTree.from_string("add(mul(o1,o2),mul(o3,o4))", pset)
    return init_model

creator.create("FitnessMin", base.Fitness, weights=(-1.0,))
creator.create("Individual", gp.PrimitiveTree, fitness=creator.FitnessMin)

toolbox = base.Toolbox()
toolbox.register("expr", gp.genHalfAndHalf, pset=pset, min_=2, max_=max_depth)
toolbox.register("individual", tools.initIterate, creator.Individual, toolbox.expr)
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
toolbox.register("compile", gp.compile, pset=pset)

def KLdiv(p, q):
    p = np.asarray(p, dtype=np.float)
    q = np.asarray(q, dtype=np.float)
    div = np.sum(np.where(p != 0, p*np.log(p/q), 0))
    return div

def evalSymbReg(individual, points):
    # Transform the tree expression in a callable function
    func = toolbox.compile(expr=individual)

    # traversing the syntax tree to discover options and interactions

    nodes, edges, labels = gp.graph(expr=individual)

    g = nx.Graph()
    g.add_nodes_from(nodes)
    g.add_edges_from(edges)

    pos = graphviz_layout(g, prog="dot")
    nx.draw_networkx_nodes(g, pos)
    nx.draw_networkx_edges(g, pos)
    nx.draw_networkx_labels(g, pos, labels)
    plt.show()

    degrees = g.degree()
    primitives = nx.dfs_successors(g)

    L = g.size()
    #for i in range(L):
    #    if degrees[i] == 1:

    sqerrors = []
    for point in points:
        x1 = point[0]
        x2 = point[1]
        sqerrors.append((func(x1,x2) - x1 ** 4 - x1 ** 3 - x2 ** 2 - x2) ** 2)
    return math.fsum(sqerrors) / len(points),

toolbox.register("evaluate", evalSymbReg, points=[(2,4), (3,1), (1,1)])
toolbox.register("select", tools.selTournament, tournsize=3)
toolbox.register("mate", gp.cxOnePoint)
toolbox.register("expr_mut", gp.genFull, min_=0, max_=2)
toolbox.register("mutate", gp.mutUniform, expr=toolbox.expr_mut, pset=pset)

toolbox.decorate("mate", gp.staticLimit(key=operator.attrgetter("height"), max_value=30))
toolbox.decorate("mutate", gp.staticLimit(key=operator.attrgetter("height"), max_value=30))

def main():
    random.seed(318)

    pop = toolbox.population(n=300)

    init_model = gp.PrimitiveTree.from_string("add(mul(o1,o2),mul(o3,o4))",pset)
    init_model.fitness = creator.FitnessMin

    perf_pop = []
    perf_pop.append(init_model)

    hof = tools.HallOfFame(1)

    stats_fit = tools.Statistics(lambda ind: ind.fitness.values)
    stats_size = tools.Statistics(len)
    mstats = tools.MultiStatistics(fitness=stats_fit, size=stats_size)
    mstats.register("avg", np.mean)
    mstats.register("std", np.std)
    mstats.register("min", np.min)
    mstats.register("max", np.max)

    pop, log = algorithms.eaSimple(perf_pop, toolbox, 0.5, 0.1, 40, stats=mstats,
                                   halloffame=hof, verbose=True)
    print(pop[1])
    # print log
    return pop, log, hof

if __name__ == "__main__":
    main()


import esphome.codegen as cg
import esphome.config_validation as cv
from esphome.components import switch
from esphome.const import (
    CONF_CHANNEL,
    CONF_GPIO,
    CONF_MODE
)

from . import (
    mpr121_ns,
    MPR121Component,
    CONF_MPR121_ID,
)

CONF_HIGHSIDE = "high_side"
CONF_LOWSIDE = "low_side"

DEPENDENCIES = ["mpr121"]
MPR121Switch = mpr121_ns.class_("MPR121Switch", switch.Switch)

def validate_supports(value):
    num = value[CONF_CHANNEL]
    mode = value[CONF_MODE] if CONF_MODE in value else {}
    is_highside = mode[CONF_HIGHSIDE] if CONF_HIGHSIDE in mode else False
    is_lowside = mode[CONF_LOWSIDE] if CONF_LOWSIDE in mode else False

    if not (num >= 4 and num <= 11):
        raise cv.Invalid(
            "GPIO Channel must be in range of [3..11]",
            [CONF_CHANNEL],
        )

    if is_highside and is_lowside:
        raise cv.Invalid(
            "High-side and Low-side modes can not be used at same time.",
            [CONF_PULLUP, CONF_PULLDOWN]
        )

    return value

CONFIG_SCHEMA = cv.All(
    switch.switch_schema(MPR121Switch).extend({
        cv.GenerateID(CONF_MPR121_ID): cv.use_id(MPR121Component),
        cv.Required(CONF_CHANNEL): cv.int_range(min=4, max=11),
        cv.Optional(CONF_MODE): cv.Schema({
            cv.Optional(CONF_HIGHSIDE, default=False): cv.boolean,
            cv.Optional(CONF_LOWSIDE, default=False): cv.boolean,
        })
    }),
    validate_supports
)

async def to_code(config):
    var = await switch.new_switch(config)
    hub = await cg.get_variable(config[CONF_MPR121_ID])

    modes = config[CONF_MODE] if CONF_MODE in config else {}
    highside = (CONF_HIGHSIDE in modes and modes[CONF_HIGHSIDE])
    lowside = (CONF_LOWSIDE in modes and modes[CONF_LOWSIDE])

    cg.add(var.set_parent(hub))
    cg.add(var.set_output(config[CONF_CHANNEL], highside, lowside))


import pandas as pd
import toolz
import json
from bson import ObjectId
import re

df = pd.read_csv({MaxQuant_document_path.csv}, sep=',')
df = df.set_index('Sequence')

#Cleaning
#Protein names are splitted and saved in an array
df["Proteins"] = df.Proteins.str.split(";")

#replace NaN in Protein name field by an empty array
null_values = df.Proteins.isnull()
for idx in null_values[null_values].index:
    df['Proteins'].set_value(idx, [])

#Extraction of intensity per QBiC samples identifiers (Barcode) and save it in a dataframe
cols = [col for col in df.columns if col.startswith('Intensity ')]
cols = [col.split() for col in cols if len(col.split()) == 2]
samples = toolz.pluck(1, cols)
samples = [sample for sample in samples if sample.startswith('Q')]
intensity_cols = ['Intensity ' + sample for sample in samples]
intensities = df[intensity_cols]
intensities.columns = samples

#Extraction of the fields of interest
cols_selection = df.loc[:,['N-term cleavage window','C-term cleavage window', 'Length', 'Mass', 'Proteins', 'Acetyl (K) site IDs', 'Oxidation (M) site IDs', 'Phospho (STY) site IDs']]

uniprot_re = re.compile('\S*\|([a-zA-Z0-9-]+)\|')

def extract_prot_id(header):
    match = re.match(uniprot_re, header)
    if not match:
        #match = re.match('CON__([a-zA-Z0-9-]+)', header)
        #if not match:
        return None
    return match.groups()[0]

def convert_headers(headers):
    ids = [extract_prot_id(header) for header in headers]
    return [{'header': header,
             'uniprot': id} for header, id in zip(headers, ids)]

#Concatenation of intensity dataframe and cols_selection dataframe in order to get one dataframe with all quantitative informations per sample
#protein list is embedded in a group protein sub-document contatining specific annotations (uniprot code and fasta header)
for col in intensities.columns:
    df3 = intensities[col]
    result = pd.concat([df3, cols_selection], axis=1, join_axes=[cols_selection.index])
    result["sample"] = col
    result.columns = ["intensity", "N-term_cleavage_window", "C-term_cleavage_window", "length", "mass", "proteinIds", "acetyl(K)_siteIDs", "oxidation(M)_siteIDs", "phospho(STY)_siteIDs", "sample"]
    result['proteinGroup'] = result.proteinIds.apply(convert_headers)
    result  = result.drop("proteinIds", 1)
    result = result.reset_index()
    #results are saved in a json file / one file per sample containing the list of sequenced peptides and related quantitative informations
    with open('tmp_%s.json' % col, 'w') as f:
        f.write(result.to_json(orient='records'))


import logging
import pathlib

from airflow.decorators import task
from airflow.models import Variable
from airflow.operators.python import get_current_context
from airflow.operators.trigger_dagrun import TriggerDagRunOperator

from libsys_airflow.plugins.folio.folio_client import FolioClient

from libsys_airflow.plugins.orafin.emails import (
    generate_ap_error_report_email,
    generate_ap_paid_report_email,
    generate_excluded_email,
    generate_summary_email,
)

from libsys_airflow.plugins.orafin.reports import (
    extract_rows,
    filter_files,
    retrieve_invoice,
    retrieve_voucher,
    update_invoice,
    update_voucher,
)

from libsys_airflow.plugins.orafin.payments import (
    generate_file,
    get_invoice,
    init_feeder_file,
    models_converter,
    transfer_to_orafin,
)

logger = logging.getLogger(__name__)

def _folio_client():
    try:
        return FolioClient(
            Variable.get("OKAPI_URL"),
            "sul",
            Variable.get("FOLIO_USER"),
            Variable.get("FOLIO_PASSWORD"),
        )
    except ValueError as error:
        logger.error(error)
        raise

@task
def consolidate_reports_task(ti=None):
    existing_reports = ti.xcom_pull(
        task_ids="filter_files_task", key="existing_reports"
    )
    new_reports = ti.xcom_pull(task_ids="filter_files_task", key="new_reports")
    all_files = []
    for row in existing_reports + new_reports:
        all_files.append(row)
    logger.info(f"Consolidated {all_files}")
    return all_files

@task
def email_errors_task(ti=None):
    folio_url = Variable.get("FOLIO_URL")
    total_errors = generate_ap_error_report_email(folio_url, ti)
    return f"Email {total_errors:,} error reports"

@task
def email_excluded_task(invoices_exclusion_reasons: list):
    folio_url = Variable.get("FOLIO_URL")
    if len(invoices_exclusion_reasons) > 0:
        generate_excluded_email(invoices_exclusion_reasons, folio_url)
    return f"Emailed report for {len(invoices_exclusion_reasons):,} invoices"

@task
def email_paid_task(ti=None):
    folio_url = Variable.get("FOLIO_URL")
    total_invoices = generate_ap_paid_report_email(folio_url, ti)
    return f"Emailed all paid invoices for {folio_url} {total_invoices}"

@task
def email_summary_task(invoices: list):
    folio_url = Variable.get("FOLIO_URL")
    generate_summary_email(invoices, folio_url)
    return f"Emailed summary report for {len(invoices):,} invoices"

@task
def extract_rows_task(report_path):
    return extract_rows(report_path)

@task
def feeder_file_task(invoices: list):
    converter = models_converter()
    folio_client = _folio_client()
    feeder_file = init_feeder_file(invoices, folio_client, converter)

    return converter.unstructure(feeder_file)

@task
def filter_files_task(ti=None):
    ls_output = ti.xcom_pull(task_ids="find_files")
    existing_reports, new_reports = filter_files(ls_output)
    ti.xcom_push(key="existing_reports", value=existing_reports)
    ti.xcom_push(key="new_reports", value=new_reports)

@task(multiple_outputs=True)
def filter_invoices_task(invoices: list):
    feeder_file, excluded = [], []
    for row in invoices:
        if row['exclude'] is True:
            excluded.append(
                {"invoice": row["invoice"], "reason": row["exclusion_reason"]}
            )
        else:
            feeder_file.append(row['invoice'])
    return {"feed": feeder_file, "excluded": excluded}

@task
def generate_feeder_file_task(feeder_file: dict, airflow: str = "/opt/airflow") -> str:
    # Initialize Feeder File Task
    folio_client = _folio_client()
    orafin_path = pathlib.Path(f"{airflow}/orafin-files/data")
    orafin_path.mkdir(exist_ok=True, parents=True)
    feeder_file_instance = generate_file(feeder_file, folio_client)
    feeder_file_path = orafin_path / feeder_file_instance.file_name
    with feeder_file_path.open("w+") as fo:
        fo.write(feeder_file_instance.generate())
    logger.info(f"Feeder-file {feeder_file_path.resolve()}")
    return str(feeder_file_path.resolve())

@task
def get_new_reports_task(ti=None):
    return ti.xcom_pull(task_ids="filter_files_task", key="new_reports")

@task
def init_processing_task():
    context = get_current_context()
    params = context.get("params")
    return params.get("ap_report_path")

@task
def launch_report_processing_task(**kwargs):
    airflow = kwargs.get("airflow", "/opt/airflow")
    ti = kwargs.get("ti")
    new_reports = ti.xcom_pull(task_ids="filter_files_task", key="new_reports")
    for i, report in enumerate(new_reports):
        TriggerDagRunOperator(
            task_id=f"ap_payment_report-{i}",
            trigger_dag_id="ap_payment_report",
            conf={
                "ap_report_path": f"{airflow}/orafin-files/reports/{report['file_name']}"
            },
        ).execute(kwargs)

@task(max_active_tis_per_dagrun=5)
def retrieve_invoice_task(row: dict):
    """
    Retrieves invoice from a row dictionary
    """
    folio_client = _folio_client()
    return retrieve_invoice(row, folio_client)

@task
def retrieve_voucher_task(invoice_id: str):
    """
    Retrieves voucher based on invoice id
    """
    folio_client = _folio_client()
    return retrieve_voucher(invoice_id, folio_client)

# @task -- When SFTP is available on AP server, uncomment this line to make a taskflow task
def sftp_file_task(feeder_file_path: str, sftp_connection: str = None):  # type: ignore
    bash_operator = transfer_to_orafin(feeder_file_path)
    return bash_operator

@task(max_active_tis_per_dag=5)
def transform_folio_data_task(invoice_id: str):
    """
    Takes Invoice ID and retrieves invoice information and tax status
    from the invoice's organization
    """
    folio_client = _folio_client()
    converter = models_converter()
    # Call to Okapi invoice endpoint
    invoice, exclude, reason = get_invoice(invoice_id, folio_client, converter)
    return {
        "invoice": converter.unstructure(invoice),
        "exclude": exclude,
        "exclusion_reason": reason,
    }

@task
def update_invoices_task(invoice: dict):
    if invoice:
        folio_client = _folio_client()
        logger.info(f"Updating Invoice {invoice['id']}")
        update_invoice(invoice, folio_client)
        return invoice['id']
    else:
        logger.error("Invoice is None")

@task
def update_vouchers_task(voucher: dict, ti=None):
    if voucher:
        folio_client = _folio_client()
        logger.info(f"Updating voucher {voucher['id']}")
        voucher = update_voucher(voucher, ti, folio_client)
        return voucher['id']
    else:
        logger.error("Voucher is None")


from django.conf.urls import url

from rest_framework.urlpatterns import format_suffix_patterns

from sites import views

urlpatterns = [
    url(r'^site/list/$', views.SiteListView.as_view(),
        name='site_list'),
    url(r'^site/data/$', views.SiteDataView.as_view(),
        name='site_data'),
]
urlpatterns = format_suffix_patterns(urlpatterns)


from tradingview_ta import TA_Handler, Interval, Exchange
from lnm_client import lnm_client
from time import time, sleep
import logging
import json
import csv
import pandas as pd
import os

class TAS():

  # Connection to LNMarkets API
  def __init__(self, options):
        self.options = options
        self.lnm = lnm_client(self.options)

  # Get trading signal from trading view https://www.tradingview.com/symbols/XBTUSD/technicals/
  def get_ta(symbol, screener, exchange, interval):
    interval_list = ['1m', '5m', '15m', '30m', '1h', '2h', '4h', '1d', '1W', '1M']
    ta_interval_list = [Interval.INTERVAL_1_MINUTE,
                        Interval.INTERVAL_5_MINUTES,
                        Interval.INTERVAL_15_MINUTES,
                        Interval.INTERVAL_30_MINUTES,
                        Interval.INTERVAL_1_HOUR,
                        Interval.INTERVAL_2_HOURS,
                        Interval.INTERVAL_4_HOURS,
                        Interval.INTERVAL_1_DAY,
                        Interval.INTERVAL_1_WEEK,
                        Interval.INTERVAL_1_MONTH]

    ta_interval = ta_interval_list[interval_list.index(interval)]

    return TA_Handler(
    symbol=symbol,
    screener=screener,
    exchange=exchange,
    interval=ta_interval,
    ).get_analysis().summary

  # Output can be a graph showing the evolution of your Balance during the strategy
  def ta_summary(self, quantity, leverage, takeprofit, stoploss, interval, timeout):
    symbol='XBTUSD.P'
    screener='CRYPTO'
    exchange='BITMEX'

    interval_list = ['1m', '5m', '15m', '30m', '1h', '2h', '4h', '1d', '1W', '1M']
    t_interval_list = [60, 300, 900, 1800, 3600, 7200, 14400, 86400, 604800, 2592000]
    t_interval = t_interval_list[interval_list.index(interval)]

    timeout = time() + 60*timeout

    id_list = []

    analysis = TAS.get_ta(symbol, screener, exchange, interval)
    print(analysis)
    if (analysis['RECOMMENDATION'] == 'STRONG_BUY'):
      side = 'long'
      last = json.loads(self.lnm.get_last())['lastPrice']
      tp = round(last * (1 + takeprofit))
      sl = round(last * (1 - stoploss))
      id = json.loads(self.lnm.market_long(quantity = quantity, leverage = leverage, takeprofit = tp, stoploss = sl))['id']
      id_list.append(id)
    elif (analysis['RECOMMENDATION'] == 'STRONG_SELL'):
      side = 'short'
      last = json.loads(self.lnm.get_last())['lastPrice']
      tp = round(last * (1 - takeprofit))
      sl = round(last * (1 + stoploss))
      id = json.loads(self.lnm.market_short(quantity = quantity, leverage = leverage, takeprofit = tp, stoploss = sl))['id']
      id_list.append(id)
    else:
      side = 'neutral'
      id = ''

    sleep(t_interval)

    while True:
      analysis = TAS.get_ta(symbol, screener, exchange, interval)
      print(analysis)

      num_pos_running = len(json.loads(self.lnm.get_positions(type_pos='running')))
      id_running = [json.loads(self.lnm.get_positions(type_pos='running'))[i]['id'] for i in range(num_pos_running)]

      if (id in id_running):
        if side == 'long':
          if ('BUY' in analysis['RECOMMENDATION']):
            logging.info('Keep long open')
          elif (analysis['RECOMMENDATION'] == 'STRONG_SELL'):
            self.lnm.close_position(id)
            side = 'short'
            last = json.loads(self.lnm.get_last())['lastPrice']
            tp = round(last * (1 - takeprofit))
            sl = round(last * (1 + stoploss))
            id = json.loads(self.lnm.market_short(quantity = quantity, leverage = leverage, takeprofit = tp, stoploss = sl))['id']
            id_list.append(id)
          else:
            self.lnm.close_position(id)
            side = 'neutral'
        elif side == 'short':
          if ('SELL' in analysis['RECOMMENDATION']):
            logging.info('Keep short open')
          elif (analysis['RECOMMENDATION'] == 'STRONG_BUY'):
            self.lnm.close_position(id)
            side = 'long'
            last = json.loads(self.lnm.get_last())['lastPrice']
            tp = round(last * (1 + takeprofit))
            sl = round(last * (1 - stoploss))
            id = json.loads(self.lnm.market_long(quantity = quantity, leverage = leverage, takeprofit = tp, stoploss = sl))['id']
            id_list.append(id)
          else:
            self.lnm.close_position(id)
            side = 'neutral'
        elif side == 'neutral':
          if (analysis['RECOMMENDATION'] == 'STRONG_BUY'):
            side = 'long'
            last = json.loads(self.lnm.get_last())['lastPrice']
            tp = round(last * (1 + takeprofit))
            sl = round(last * (1 - stoploss))
            id = json.loads(self.lnm.market_long(quantity = quantity, leverage = leverage, takeprofit = tp, stoploss = sl))['id']
            id_list.append(id)
          elif (analysis['RECOMMENDATION'] == 'STRONG_SELL'):
            side = 'short'
            last = json.loads(self.lnm.get_last())['lastPrice']
            tp = round(last * (1 - takeprofit))
            sl = round(last * (1 + stoploss))
            id = json.loads(self.lnm.market_short(quantity = quantity, leverage = leverage,  takeprofit = tp, stoploss = sl))['id']
            id_list.append(id)
          else:
            side = 'neutral'
            logging.info('Stay neutral')

      else:
        if (analysis['RECOMMENDATION'] == 'STRONG_BUY'):
          side = 'long'
          last = json.loads(self.lnm.get_last())['lastPrice']
          tp = round(last * (1 + takeprofit))
          sl = round(last * (1 - stoploss))
          id = json.loads(self.lnm.market_long(quantity = quantity, leverage = leverage, takeprofit = tp, stoploss = sl))['id']
          id_list.append(id)
        elif (analysis['RECOMMENDATION'] == 'STRONG_SELL'):
          side = 'short'
          last = json.loads(self.lnm.get_last())['lastPrice']
          tp = round(last * (1 - takeprofit))
          sl = round(last * (1 + stoploss))
          id = json.loads(self.lnm.market_short(quantity = quantity, leverage = leverage, takeprofit = tp, stoploss = sl))['id']
          id_list.append(id)
        else:
          side = 'neutral'
          id = ''

      if time() > timeout:
        break

      sleep(t_interval)

    self.lnm.close_position(id)
    id_list.append(id)

    closed_positions = json.loads(self.lnm.get_positions(type_pos='closed'))
    df_closed_positions = pd.DataFrame.from_dict(closed_positions)

    df_closed_pos = df_closed_positions[df_closed_positions['id'].isin(id_list)].copy()

    pl = df_closed_pos['pl'].sum()

    logging.info('Total PL (sats) = ' + str(pl))

    path = os.path.join(os.path.dirname(__file__), "df_closed_pos.csv")
    df_closed_pos.to_csv(path)
    logging.info('df_closed_pos.csv saved in strategies folder')


# -*- coding: utf-8 -*-

# @Author : 石张毅
# @Site :
# @introduce:

import scrapy
import re
import time
import requests
import copy
from copy import deepcopy
from sqlalchemy import create_engine
# from loupan.items import LoupanItem
import numpy as np
import pandas as pd
import scrapy

def list_str(con):
    sss = ''
    for i in con:
        sss = sss + i
    return sss.strip()

def dbz():
    # now = datetime.datetime.now()
    # otherStyleTime = now.strftime("%Y-%m-%d")

    sql1 = f'''SELECT * FROM `job_type`;'''
    sql2 = f'''SELECT * FROM `job_type_copy1`;'''
    # print(sql)
    engine = create_engine('mysql+pymysql://root:123456@127.0.0.1:3306/stu?charset=utf8')
    engine2 = create_engine('mysql+pymysql://root:123456@127.0.0.1:3306/stu?charset=utf8')
    df = pd.read_sql(sql1, engine)
    df = df.drop_duplicates(subset=['id']) # 默认保留一条重复数据
    # print(df)
    df1 = pd.read_sql(sql2, engine2)
    df1 = df1.drop_duplicates(subset=['id'])  # 默认保留一条重复数据
    engine.dispose()
    engine2.dispose()
    # print(df1)
    db2 = pd.concat([df, df1], axis=0, sort=False, ignore_index=True)
    # print(db2)
    dbz = db2.drop_duplicates(subset=['id'], keep=False)
    # print(dbz)
    # print(f'对比保留了{len(dbz)}条')
    dbz = dbz[['big_type','mid_type','small_type','small_code']].values.tolist() # df转列表
    # print(dbz)
    return dbz

dbz()
class Jishu1Spider(scrapy.Spider):
    name = 'jishu1'
    # allowed_domains = ['boss.com']
    # start_urls = ['http://boss.com/']

    def __init__(self, *args, **kwargs):
        super(Jishu1Spider, self).__init__()
        self.result = dbz()

    def start_requests(self):
        for i in self.result[0:1]:
            item = dict()
            item['big_type'] = i[0]
            item['small_type'] = i[2]
            item['small_code'] = i[3]
            url = 'https://www.zhipin.com/c100010000-p{}/?page=2'.format(item['small_code'])
            # print(url)
            yield scrapy.Request(url, callback=self.parse,dont_filter=True,meta={'item':deepcopy(item)})

    def parse(self, response, **kwargs):

        job_urls = response.xpath('//*[@class="primary-box"]/@href').getall()
        job_titles = response.xpath('//*[@class="job-name"]/a/@title').getall()
        # print(job_titles)
        job_area = response.xpath('//*[@class="job-area"]/text()').getall()
        salary = response.xpath('//*[@class="red"]/text()').getall()
        job_welfare = response.xpath('//*[@class="info-desc"]/text()').getall()
        company_url = response.xpath('//*[@class="company-text"]/h3/a/@href').getall()
        job_tag = response.xpath('string(//*[@class="tags"])').getall()
        # print()
        for job_urls,job_titles,job_area,salary,job_welfare,company_url in zip(job_urls,job_titles,job_area,salary,job_welfare,company_url):
            # 详情链接
            item = response.meta['item']
            item['job_urls'] = response.urljoin(job_urls)
            # 工作名称
            item['job_titles'] = job_titles
            # 工作地区
            item['job_area'] = job_area
            # 薪资
            item['salary'] = salary
            # 工作福利
            item['job_welfare'] = job_welfare
            # 公司链接
            item['company_url'] = response.urljoin(company_url)
            print(item['job_urls'])
    #         yield scrapy.Request(item['job_urls'], callback=self.parse_info, dont_filter=True, meta={'item': deepcopy(item)})
    #
    #
    # def parse_info(self,response):
    #     # print(response.text)
    #     item = response.meta['item']
    #     # 城市
    #     item['city'] = response.xpath('//*[@class="text-city"]/text()').get()
    # #     # 工作年限
    #     item['work_time'] = response.xpath('//*[@class="text-city"]/following::text()[1]').get()
    #     print(item['city'],item['work_time'])
        # 学历
    #     item['education'] = response.xpath('//*[@class="text-city"]/following::text()[2]').get()
    #     # hr
    #     item['hr'] = response.xpath('//h2[@class="name"]/text()[1]').get()
    #     # hr岗位
    #     hr_position = response.xpath('//h2[@class="name"]/following::p[1]//text()').getall()
    #     item['hr_position'] = list_str(hr_position)
    #     # 岗位详情
    #     j_content = response.xpath('//h3/following::div[@class="text"]//text()').getall()
    #     item['job_dec'] = list_str(j_content)
    #
    #     yield item


from flask import Flask,render_template ,request,redirect
from flask_sqlalchemy import SQLAlchemy
from datetime import datetime
app = Flask(__name__)
if locals():
    app.config['SQLALCHEMY_DATABASE_URI'] = "mysql://root:@localhost:3306/db"
else:
    app.config['SQLALCHEMY_DATABASE_URI'] = "mysql://root:@localhost:3306/db"

db=SQLAlchemy(app)

class List(db.Model):
    sno=db.Column(db.Integer, primary_key=True)
    title=db.Column(db.String(20), nullable=False)
    content=db.Column(db.String(500), nullable=False)
    phone=db.Column(db.String(12), nullable=False)
    date=db.Column(db.DateTime,default=datetime.utcnow, nullable=False)

    def __repr__(self) ->  str:
        return f"{self.sno} - {self.title}"

@app.route("/", methods=['GET', 'POST'])
def index():
    if request.method=='POST':
        title=request.form['title']
        content=request.form['content']
        phone=request.form['phone']
        entry=List(title=title, content=content, phone=phone)
        db.session.add(entry)
        db.session.commit()
        return redirect("/")
    alllist=List.query.all()
    return render_template('index.html', alllist=alllist)
@app.route("/delete/<int:sno>")
def delete(sno):
    list=List.query.filter_by(sno=sno).first()
    db.session.delete(list)
    db.session.commit()
    return redirect("/")
@app.route("/update/<int:sno>",methods=['GET','POST'])
def update(sno):
    list=List.query.filter_by(sno=sno).first()
    if request.method=='POST':
        title=request.form['title']
        content=request.form['content']
        phone=request.form['phone']
        list=List.query.filter_by(sno=sno).first()
        list.title=title
        list.content=content
        list.phone=phone
        db.session.add(list)
        db.session.commit()
        return redirect("/")
    return render_template('update.html' , list=list)
if __name__== "__main__":
    app.run(debug=True)


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2018/7/30 16:53
# Project:
# @Author: ZQJ
# @Email : zihe@yscredit.com

from pyspider.libs.base_handler import *
from lxml import etree
import re
import logging
from datetime import datetime
import hashlib
import json
import requests

# 陕西-送达公告

# 日志输出等级
logging.basicConfig(level=logging.info)

class Handler(BaseHandler):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
        'Host': 'www.gd.xinhuanet.com'
    }

    crawl_config = {
        'itag': 'v5.3',
        'headers': headers,
        'time_out': 10000,
        # 'proxy': 'H14LXDJ6O07CAFDP:150D24D434AC09EE@proxy.abuyun.com:9010'
    }

    retry_delay = {
        0: 60,
        1: 60 * 5,
        2: 60 * 10,
        3: 60 * 15,
        4: 60 * 20,
        5: 60 * 25,
        6: 60 * 30,
        7: 60 * 35,
        8: 60 * 40,
        9: 60 * 45,
        10: 60 * 50,
        11: 60 * 55,
        12: 60 * 60,
    }

    def escape(self, string):
        return '%s' % string

    def parse_time(self, t):
        t = list(t)
        d = {'零': '0', '一': '1', '二': '2', '三': '3', '四': '4', '五': '5', '六': '6', '七': '7', '八': '8', '九': '9',
             '○': '0', 'Ｏ': '0', '年': '-', '月': '-', '日': '-', '元': '1', '〇': '0', 'Ο': '0', '０': '0',
             'О': '0', '0': '0', 'O': '0', 'o': '0'
             }
        for i in range(len(t)):
            if t[i] in d.keys():
                t[i] = d[t[i]]
        # 处理汉字为十的情况
        if '十' in t:
            for i in range(len(t)):
                if t[i] == '十':
                    if (i - 1) >= 0 and t[i - 1].isalnum():
                        t[i] = '0'
                    if (i + 1) < len(t) and t[i + 1].isalnum():
                        t[i] = '1'
                    if (i - 1) >= 0 and (i + 1) < len(t) and t[i - 1] == '-' and t[i + 1] == '-':
                        t[i] = '10'
                    if (i - 1) >= 0 and (i + 1) < len(t) and t[i - 1].isalnum() and t[i + 1].isalnum():
                        t[i] = ''
        t = ''.join(t)
        t = re.findall(r'(.{4}[-年\.].{1,2}[-月\.].{1,3}?)[-上下日号\s]', t)[0]
        t = datetime.strptime(t, "%Y-%m-%d").strftime('%Y-%m-%dT00:00:00+08:00')
        return t

    schema_def = [{"name": "id", "type": "string"},
                  {"name": "_id_", "type": "string"},
                  {"name": "ann_type", "type": "string"},
                  {"name": "announcer", "type": "string"},
                  {"name": "case_no", "type": "string"},
                  {"name": "ann_content", "type": "string"},
                  {"name": "ann_date", "type": "string"},
                  {"name": "content_url", "type": "string"},
                  {"name": "ann_html", "type": "string"},
                  {"name": "pdf_url", "type": "string"},
                  {"name": "source", "type": "string"},
                  {"name": "defendant", "type": "string"},
                  ]

    def get_md5(self, s):
        """get md5 value
        Args
            s: a bytes, can be None
        Returns
            h.hexdigest(): md5(s) value, 32 bit.
        """
        if isinstance(s, str):
            s = bytes(s, encoding='utf-8')
        if s:
            h = hashlib.md5()
            h.update(s)
            return h.hexdigest()
        else:
            return ''

    @every(minutes=24 * 60)
    @config(age=60 * 60)
    def on_start(self):
        ccs = ['R00', 'R1_', 'R2_', 'R3_', 'R4_', 'R5_', 'R6_', 'R7_', 'R8_', 'R9_', 'RA_', 'RB_']
        for cc in ccs:
            url = 'http://www.sxgaofa.cn/sxgzfww/R00/ggsd_listAjxx.shtml#start#{}'.format(cc)
            data = {
                'fydm': cc,
                'ahqcOrdsr': ''
            }
            self.crawl(url, data=data, method='POST', callback=self.get_lists)

    @config(age=60 * 60)
    def get_lists(self, response):
        try:
            html = etree.HTML(response.text)
            results = html.xpath('//table[@class="mytable"]//tr')
            print(result)
            url = 'http://www.sxgaofa.cn/sxgzfww/R00/ggsd_pjggsd.shtml'
            print(url)
            for result in results:
                list_response = etree.tounicode(result)
                tmp = result.xpath('.//a/@onclick')[0]
                tmp = re.findall(r'tzpjgg(\(.*\))', tmp)[0]
                tmp = list(eval(tmp))
                logging.info(tmp)
                if len(tmp) == 7:
                    ahqc, dybg, dyyg, laaymc, fymc, fbrq, fydm = tmp
                    url = 'http://www.sxgaofa.cn/sxgzfww/R00/ggsd_pjggsd.shtml?ahqc={}&dybg={}&dyyg={}&laaymc={}&fymc={}&ajxx.fbrq={}&ajxx.fydm={}'.format(
                        ahqc, dybg, dyyg, laaymc, fymc, fbrq, fydm)
                    self.crawl(url, callback=self.get_detail,
                               save={'list_response': list_response, 'ahqc': ahqc, 'dybg': dybg, 'fymc': fymc,
                                     'fbrq': fbrq}, retries=12, allow_redirects=False)
        except Exception as e:
            logging.info(e)
            logging.info('当前法院无内容...')

    @config(age=60 * 60)
    @catch_status_code_error
    def get_detail(self, response):
        # 初始化字段
        ann_type = '送达公告'
        announcer = ''
        defendant = ''
        defendant_origin = ''
        ann_date = ''
        ann_content = ''
        ann_html = ''
        content_url = response.url
        pdf_url = ''
        case_no = ''
        source = '陕西法院诉讼服务网'

        list_response = response.save['list_response']
        ahqc = response.save['ahqc']
        dybg = response.save['dybg']
        fymc = response.save['fymc']
        fbrq = response.save['fbrq']
        # html = response.etree
        if response.status_code == 200:
            try:
                html = etree.HTML(response.text)
                detail = html.xpath('//form[@id="form1"]')[0]
                ann_html = etree.tounicode(detail)
                # logging.info('开始结构化..')
                results = html.xpath('//form[@id="form1"]//text()')
                results = [result.strip() for result in results if result.strip()]
                logging.info(results)
                ann_time = fbrq
                ann_date = re.findall(r'(.{4}[-年\.].{1,2}[-月\.].{1,3}?)', ann_time)[0]
                ann_date = datetime.strptime(ann_date, "%Y-%m-%d").strftime('%Y-%m-%dT00:00:00+08:00')
                # if len(results) >= 6:
                # ann_content = ''.join(results)
                announcer = fymc
                defendant_origin = dybg
                for result in results:
                    if dybg in result:
                        if not ann_content:
                            ann_content = ''.join(results[results.index(result) + 1:])
                            case_no = re.findall(r'[(（][1１２2].{3}[）)].{0,8}[刑民行赔执访认送调管脏移引惩保].{2,18}号', ann_content)[
                                0] if re.findall(r'[(（][1１２2].{3}[）)].{0,8}[刑民行赔执访认送调管脏移引惩保].{2,18}号',
                                                 ann_content) else case_no
                            if '、' in defendant_origin:
                                defendants = defendant_origin.split('、')
                            elif ',' in defendant_origin:
                                defendants = defendant_origin.split(',')
                            else:
                                defendants = [defendant_origin]

                            yield {
                                'ann_type': ann_type,
                                'announcer': announcer,
                                'defendant': str(defendants),
                                'ann_date': ann_date,
                                'ann_content': ann_content,
                                'ann_html': ann_html,
                                'content_url': content_url,
                                'pdf_url': pdf_url,
                                'source': source,
                                'case_no': case_no,
                                'id': '',
                                '_id_': ''
                            }
            except Exception as e:
                logging.warning('哪里可能出问题了...')
                logging.warning(e)


import pickle

import matplotlib.pyplot as plt
import mosaiks.plotting.general_plotter as plots
import numpy as np
import seaborn as sns
import sklearn.metrics as metrics

def checkerboard_vs_delta_with_jitter(
    metrics_checkered,
    best_hps,
    deltas,
    crit,
    val_name,
    app_name=None,
    save_dir=None,
    prefix=None,
    suffix=None,
    figsize=(10, 5),
    ax=None,
    overwrite=False,
):

    """Plot metrics (e.g. r2) or vs delta (degree of spatial extrapolation). Used for visualizing results of
    'checkerboard' analyses. Saves the result. Note this is for a single outcome. If you have multiple outcomes,
    you must obtain the proper metrics_checkered object for each outcome and pass to this function.

    Args:
     metrics_checkered (dict of list of dict) : e.g.
         {'random_features': [{'mse': 117.05561471285215, 'r2_score': 0.875037330527241},
                              {'mse': 119.84752736626068, 'r2_score': 0.8735189806862442}],
         'latlon_features':  [{'mse': 117.05561471285215, 'r2_score': 0.875037330527241},
                              {'mse': 119.84752736626068, 'r2_score': 0.8735189806862442}]}
        Keys of outer dict are labels for the lines that will be plotted. The list indexes different deltas,
        as produced by diagnostics.spatial_experiments.checkered_predictions_by_radius. Within each element,
        the keys of the dict are different metric names.
     best_hps (dict) : keys are ``best_lambda_rcf`` and ``best_sigma_smooth``. Values
         are the list of hyperparameters chosen for each value of ``delta``.
     deltas (list of numeric) : values (e.g. [1,2,3,4,5,6,7,8]) where larger values imply more spatial extrapolation
     crit (str) : Name of criteria that you want to plot (e.g. 'r2_score').
     val_name (str) : Name of outcome.
     app_name (str) : The name of the application (e.g. 'housing'). Only needed if saving
     save_dir (str) : Path to directory in which to save output files. If None, no figures will be saved.
     prefix (str) : Filename prefix identifying what is being plotted (e.g. test_outof_cell_r2). Only
         needed if figure is being saved
     suffix (str) : The suffix containing the grid and sample parameters which will be appended to the
         filename when saving, in order to keep track of various sampling and gridding schemes.
     overwrite (bool) : Whether to overwrite data and figure files

    Returns:
     None (plots function)

    """

    # Make the plot:
    if ax is None:
        fig, ax = plt.subplots(figsize=figsize)

    for feat in metrics_checkered.keys():
        metrics_checkered_this = metrics_checkered[feat]
        # Get the values by jitter into one list
        crit_by_delta = np.zeros(
            (len(metrics_checkered_this), len(metrics_checkered_this[0]))
        )
        for i in range(crit_by_delta.shape[0]):
            for j in range(crit_by_delta.shape[1]):
                crit_by_delta[i, j] = metrics_checkered_this[i][j][crit]

        # plot
        ax.plot(deltas, np.average(crit_by_delta, axis=1), label=feat)
        ax.fill_between(
            deltas,
            np.min(crit_by_delta, axis=1),
            np.max(crit_by_delta, axis=1),
            alpha=0.5,
        )

    ax.set_title("Performance vs. checkerboard size")
    ax.set_xlabel("$\delta$")
    ax.set_ylabel(crit)
    ax.legend()

    if save_dir is not None:
        plots._savefig(
            fig, save_dir, app_name, val_name, prefix, suffix, overwrite=overwrite
        )

        # save pickle of data
        to_save = {"metrics": metrics_checkered, "deltas": deltas, **best_hps}
        plots._save_fig_data(
            to_save, save_dir, app_name, val_name, prefix, suffix, overwrite=overwrite
        )


import gmailer
import logging
import resq

from dateutil.parser import parse as date_parse
from pymongo import MongoClient
from models import Course, Sole, User
from utils.format import format_date_readable_with_ordinal, format_possible_sole_time

def send(to, subject, text, cc=None, bcc=None, reply_to=None, attach=None,
         html=None, pre=False, custom_headers=None):
    """ abstracts away the details of our email provider. always use this method
    instead of gmailer or other future mailers
    """
    return gmailer.mail(to, subject, text, cc=cc, bcc=bcc, reply_to=reply_to,
                        attach=attach, html=html, pre=False, custom_headers=custom_headers)

class EmailCreator():
    queue = resq.Q_EMAIL
    @staticmethod
    def perform(sole_id):
        """ Email the creator of a sole about its creation
        """
        sole, course, users = get_sole_course_and_user_information(sole_id)
        if not (sole and course and users):
            return

        user_id = sole.get(Sole.A_USER_ID)
        user = users[user_id]

        params = {
            'course_name': course.get('name'),
            'day': format_date_readable_with_ordinal(date_parse(sole.get(Sole.A_DAY))),
            'time': format_possible_sole_time(sole.get(Sole.A_TIME)),
            'address': sole.get(Sole.A_ADDRESS),
            'first_name': user.get('first_name')
        }

        to = user.get('email')
        subject = "%(course_name)s, on %(day)s %(time)s" % params
        text = """Hello %(first_name)s,

Your study session has been set up.

We'll let you know via email when others join this session.

A reminder:
You're meeting on %(day)s %(time)s
Near %(address)s

Have fun,
Massively Offline
Learning is more fun together
        """
        text = text % params

        send(to, subject, text)

class EmailGroup():
    queue = resq.Q_EMAIL
    @staticmethod
    def perform(action, sole_id, user_id):
        """ Email the group about a person joining or leaving the group
        """
        sole, course, users = get_sole_course_and_user_information(sole_id, user_id)
        if not (sole and course and users):
            return

        # create email
        params = {
            'actor_name': users[user_id].get('name'),
            'course_name': course.get('name'),
            'day': format_date_readable_with_ordinal(date_parse(sole.get(Sole.A_DAY))),
            'time': sole.get(Sole.A_TIME),
            'address': sole.get(Sole.A_ADDRESS),
            'action': action
        }
        member_list = []
        if action == "joined":
            for k, v in users.iteritems():
                member_list.append(v)
        else:
            for k, v in users.iteritems():
                if k != user_id:
                    member_list.append(v)
        members = ', '.join(m.get('name') for m in member_list)
        params['members'] = members
        params['num_members'] = len(member_list)
        if len(member_list) == 0:
            return

        to = [m.get('email') for m in member_list]
        subject = "Re: %(course_name)s, on %(day)s %(time)s" % params
        text = """Hello,

%(actor_name)s has just %(action)s this study session.
Your group now has %(num_members)s members: %(members)s

Teply-all to chat with your fellow students, or to set up a specific time and place to meet.

A reminder:
You're meeting on %(day)s %(time)s
Near %(address)s

Have a good meeting,
Massively Offline
Learning is more fun together
        """
        text = text % params

        send(to, subject, text)

def get_sole_course_and_user_information(sole_id, user_id=None):
    db = MongoClient()

    # find the sole
    sole = Sole.find_by_id(db, sole_id)
    if not sole:
        None, None, None

    # find the course
    cid = sole.get(Sole.A_COURSE_ID)
    course = Course.find_by_id(db, cid)
    if not course:
        sole, None, None

    # find the user who did the action
    user_ids = set(sole.get(Sole.A_STUDENT_IDS))
    if not user_ids:
        # everyone has left the group!
        sole, course, None

    if user_id:
        user_ids.add(user_id)
    users = User.find_by_ids(db, user_ids, as_dict=True)
    return sole, course, users


from django.urls import path, include
from rest_framework.routers import DefaultRouter

from providers.views import ProviderViewSet, send_mail_api

app_name = "providers"

router = DefaultRouter()
router.register(r'providers', ProviderViewSet)

urlpatterns = [
    path('', include(router.urls)),
    path('providers/<int:id>/send_qr_code', send_mail_api,  name='send_qr_code'),
]


from bs4 import BeautifulSoup
import os
import numpy as np
import pandas as pd
from string import punctuation
import re
import numpy as np
import pandas as pd
import torch
import transformers
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
import nltk
import tensorflow as tf

class reuters_util:
    model = ""
    tokenizer = ""
    # Parses a document in sgm (Standard Generalized Markup language)
    # return a pandas dataframe with desired features.
    def parse_doc(self, doc, columns, df):
        try:
            current_doc = BeautifulSoup(open(doc), features='lxml')
        except UnicodeDecodeError:
            current_doc = BeautifulSoup(open(doc, 'rb'), features='lxml', from_encoding="iso-8859-1")
        doc_list = current_doc.find_all('reuters')
        for d in doc_list:
            row=[]
            for col in columns:
                try:
                    if col in ["places","topics", "orgs", "people","exchanges"]:
                        result = [i.text for i in d.find(col).contents]
                        if result:
                            row.append(result)
                        else:
                            row.append(None)
                    elif col in ['cgisplit', 'lewissplit', 'newid', 'topics_bool']:
                        if col == "topics_bool":
                            row.append(d.get("topics"))
                        else:
                            row.append(d.get(col))
                    else:
                        row.append(str(d.find(col).contents[-1]).strip())
                except :
                    row.append(None)
            df.loc[df.shape[0]]= row

        return df

    # clean the data
    def clean_data(self, df):
        special_char = list(punctuation)
        for e in ['.','?']:
            special_char.remove(e)
        special_char.append("\n+")
        special_char.append("\s+")

        def deep_clean(text_str):
            text_str = str(text_str)
            text_str = re.sub('<[^>]*>', '', text_str)
            for char in special_char:
                text_str = text_str.replace(char, '')
            return text_str

        df['text'] = df['text'].apply(deep_clean)
        df['title'] = df['title'].apply(deep_clean)
        return df

    # Configure GPU
    def gpu_config(self):
        # Get the GPU device name.
        device_name = tf.test.gpu_device_name()

        # The device name should look like the following:
        if device_name == '/device:GPU:0':
            print('Found GPU at: {}'.format(device_name))
        else:
            raise SystemError('GPU device not found')

        # If there's a GPU available...
        if torch.cuda.is_available():
            # Tell PyTorch to use the GPU.
            device = torch.device("cuda")
            print('There are %d GPU(s) available.' % torch.cuda.device_count())
            print('We will use the GPU:', torch.cuda.get_device_name(0))

        # If not...
        else:
            print('No GPU available, using the CPU instead.')
            device = torch.device("cpu")

    def initialize_prebuilt_models(self):
        # DistilBERT model
        model_class, pretrained_weights = (transformers.DistilBertModel, 'distilbert-base-uncased')

        # Load pretrained model
        self.model = model_class.from_pretrained(pretrained_weights)

        # Use GPU
        self.model.cuda()

        # Load the BERT tokenizer
        self.tokenizer = transformers.BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)

        # Download NLTK for paragraph tokenization
        nltk.download('punkt')

    def doc_to_vectors(self, doc):
        sentences  = nltk.sent_tokenize(doc)

        def valid_sent(sent):
            return True if len(sent.split(" ")) > 1 else False

        vector = np.array([self.sent_to_vector(s) for s in sentences if valid_sent(s)]).sum(axis=0)
        return vector

    def sent_to_vector(self, sent):
        # Encode the sentence
        encoded = self.tokenizer.encode_plus(
            text=sent,  # the sentence to be encoded
            add_special_tokens=True,  # Add [CLS] and [SEP]
            max_length = 64,  # maximum length of a sentence
            pad_to_max_length=True,  # Add [PAD]s
            truncation = True, # Truncate words beyond the max_length
            return_attention_mask = True,  # Generate the attention mask
            return_tensors = 'pt',  # ask the function to return PyTorch tensors
        )

        # Get the input IDs and attention mask in tensor format
        input_ids = encoded['input_ids']
        attention_mask = encoded['attention_mask']

        with torch.no_grad():
            last_hidden_states = self.model(input_ids.to("cuda"), attention_mask=attention_mask.to("cuda"))
        #BERT adds a token called [CLS] (for classification) at the beginning of every sentence, we take that
        features = last_hidden_states[0][:,0,:].cpu().numpy()
        return features

    def generate_final_embedding(self, df):
        result_data = []

        def padding_na(vector):
            std_vector_length = 768
            if not vector.shape:
                return np.array([0.0]*std_vector_length).reshape(1,std_vector_length)
            else:
                return vector

        for i in range(len(df)):
            try:
                txt_emb = df.iloc[i]['text_embedding']
                title_emb = df.iloc[i]['title_embedding']
                result_data.append(np.concatenate((txt_emb, title_emb), axis=1)[0])
            except ValueError:
                result_data.append(np.concatenate((padding_na(txt_emb), padding_na(title_emb)), axis=1)[0])

        result_df = pd.DataFrame(result_data)
        result_df['topics'] = df['topics']
        return result_df

    # Test data preprocessing function
    def preprocess_data(self, datapath):
        columns = ['title','text','topics']
        test_df = pd.DataFrame(columns=columns)
        test_files = [f for f in os.listdir(datapath) if f.endswith("sgm")]
        if test_files:
            for test_file in test_files:
                print("parsing {}".format(test_file))
                file_path = os.path.join(datapath, test_file)
                test_df = self.parse_doc(file_path, columns, test_df)

                test_df = self.clean_data(test_df.copy())
                test_df['text_embedding'] = test_df['text'].apply(self.doc_to_vectors)
                test_df['title_embedding'] = test_df['title'].apply(self.doc_to_vectors)
                test_df.loc[test_df['topics'].isnull(),'topics']=['0']
                test_df['topics'] = [1 if 'earn' in topic else 0 for topic in test_df['topics']]
                test_df = self.generate_final_embedding(test_df)
        else:
            raise ValueError('No .sgm file in a given path.')
        return test_df


import sqlite3

def updateMultipleColumns(id, salary, email):
    try:
        db_connect = sqlite3.connect('C:\sqlite\database\Egor_test.db')
        cursor = db_connect.cursor()
        print("Connected to SQLite")

        sqlite_update_query = """Update SqliteDb_developers set salary = ?, email = ? where id = ?"""
        columnValues = (salary, email, id)
        cursor.execute(sqlite_update_query, columnValues)
        db_connect.commit()
        print("Multiple columns updated successfully")
        db_connect.commit()
        cursor.close()

    except sqlite3.Error as error:
        print("Failed to update multiple columns of sqlite table", error)
    finally:
        if (db_connect):
            db_connect.close()
            print("sqlite connection is closed")

updateMultipleColumns(3, 6500, 'ben_stokes@gmail.com')