import pandas as pd
from pcf.common.dbconnect import connect
from sqlite3 import Error

def create_connection():
    """ create a database connection to the SQLite database
    """
    conn = None
    try:
        conn = connect()
    except Error as e:
        print(e)

    return conn

def load_entity(fname='../data/entity.csv'):
    dfentity_csv = pd.read_csv(fname)
    #    print(dfentity_csv.head())

    conn = create_connection()
    dfentity_sql = pd.read_sql("select * from entity", conn)
    print(dfentity_sql.head())

    #    print("start iteration")
    #    df = dfentity_sql.assign(rn=dfentity_sql.sort_values('entityId', ascending=False)
    #                                            .groupby(['entityName'])
    #                                            .cumcount() +1)

    dfentity_sql_rnk1 = dfentity_sql.assign(rn=dfentity_sql.sort_values(['entityId'], ascending=False) \
                             .groupby(['entityName']) \
                             .cumcount() + 1) \
        .query('rn == 1 ')

    #print(df_sql_rnk1)

    df = pd.merge(dfentity_csv, dfentity_sql_rnk1 , how='left' , left_on=['entityName'] , right_on=['entityName'])
    print(df)

"""
    # using itertulples to iterate the csv dataframe. when a match is found the flag column will be updated.
    for row in dfentity_csv.itertuples():
        print(getattr(row,'entityName'))
        #if getattr(row,'entityName') in dfentity_sql['entityName'].values:
        #    print ("exists" + str(dfentity_sql['entityId'].values) )
        if dfentity_sql.isin(getattr(row,'entityName')):
            print
        else:
            print("does not exist")
"""

load_entity()


import networkx as nx
from timeit import default_timer as timer
import numpy as np
from .Error import InfeasibleError
from sklearn.utils.extmath import cartesian
import sys

def get_true_keys_in_dict(d):
    """
     get all keys where the value is true
     return list
    """
    keys = []
    for key in d:
        if d[key]:
            keys.append(key)
    return keys

class Model:
    def __init__(self):
        # holds a list of indices where the function should be called
        self.subscribe_list_on = []
        # holds a list of all check constraint functions
        self.subscribe_list_func = []
        self.nof_calls = 0

    def subscribe(self,on,func,*args):
        # add a list of indices to the subscribe list
        self.subscribe_list_on.append(on['idx'])
        # as well as the function call
        self.subscribe_list_func.append((func,args))

    def solved(self,idx):
        """
            Check whether it is solved
            only the idx values have to be solved
        """

        self.nof_found = - self.nof_start_values
        self.nof_missing = 0
        ret = True
        for r in range(len(self.search_space)):
            for c in range(len(self.search_space[r])):
                if 'value' not in self.search_space[r][c] and idx[r][c]:
                    ret = False
                    self.nof_missing += 1
                elif 'value' in self.search_space[r][c]:
                    self.nof_found += 1
        return ret

    def get_backtrack_candidates(self):
        nof_values = sys.maxsize
        best_id = [0,0]
        for r in range(len(self.search_space)):
            for c in range(len(self.search_space[r])):
                if 'values' in self.search_space[r][c] and self.backtracking_space[r][c] == False:
                    l_nof_values = len(self.search_space[r][c]['values'])
                    if l_nof_values < nof_values:
                        nof_values = l_nof_values
                        best_id = [r,c]
        cands = self.search_space[best_id[0]][best_id[1]]['values']
        self.backtracking_space[best_id[0]][best_id[1]] = True
        return {'row':best_id[0],'col':best_id[1],'cands':cands}

    def backtrack(self,solved_idx,counter_bt=0):
        counter_bt += 1
        # save first search_space
        copy_search_space = np.copy(self.search_space)
        # get candidate list
        candidate_list = [self.get_backtrack_candidates()]
        # while candidates in list
        while candidate_list:
            candidate_entry = candidate_list.pop(0)
            cands = candidate_entry['cands']
            row, col = candidate_entry['row'],candidate_entry['col']
            for cand in cands:
                self.search_space = np.copy(copy_search_space)
                self.search_space[row][col] = {'value':cand}
                try:
                    idx = np.full(self.search_space.shape, False, dtype=bool)
                    idx[row,col] = True
                    self.fire(idx)
                    if not self.solved(solved_idx):
                        feasible, counter = self.backtrack(solved_idx,counter_bt)
                        if feasible:
                            return True,counter
                    else:
                        return True,counter_bt
                except InfeasibleError as e:
                    continue
        return False, counter_bt

    def solve(self,idx,no_val=0):
        try:
            self.fire(np.full(self.search_space.shape, True, dtype=bool))
            if not self.solved(idx):
                print("Need to backtrack")
                print("found %d values" % self.nof_found)
                print("%d values are missing" % self.nof_missing)
                feasible,counter = self.backtrack(idx)
                print("Number of backtracks", counter)
                if not feasible:
                    raise InfeasibleError("Infeasible checked backtracking")

        except InfeasibleError as e:
            print(e)
            exit(2)

    def fire(self,idx):
        """
            Check all subscribed indices and check which functions should be
            called
        """
        i = 0
        self.changed = np.full(self.changed.shape, False, dtype=bool)
        for lidx in self.subscribe_list_on:
            if np.any(np.logical_and(lidx,idx)):
                func = self.subscribe_list_func[i][0]
                args = self.subscribe_list_func[i][1]
                try:
                    func(*args)
                except InfeasibleError as e:
                    raise e
                self.nof_calls += 1

            i += 1

        # if something changed because of the initial fire
        # => fire again the changed indices
        if np.any(self.changed):
            try:
                self.fire(self.changed)
            except InfeasibleError as e:
                raise e

    def get_linked_idx(self,linked_to,constraint_type):
        on = self.subscribe_list_on[linked_to]
        nof_true_values = np.sum(on)
        links = []
        for si in range(len(self.subscribe_list_on)):
            args = self.subscribe_list_func[si][1]
            func = args[1]
            if func == constraint_type:
                sum_of_true = np.sum(np.logical_and(self.subscribe_list_on[si],on))
                if sum_of_true == nof_true_values or sum_of_true == np.sum(self.subscribe_list_on[si]):
                    links.append(si)
        return links

    def link(self):
        """
            Links different constraint types together
        """
        for si in range(len(self.subscribe_list_on)):
            on = self.subscribe_list_on[si]
            args = self.subscribe_list_func[si][1]
            func = args[1]
            # if a part is a straight and part of alldifferent
            # get all other straights and link them together
            # => the straight constraints considers what to do with it
            if func == "straight":
                temp_links = self.get_linked_idx(si,"alldifferent")
                links = []
                for t in temp_links:
                    links.extend(self.get_linked_idx(t,"straight"))
                real_links = []
                for l in links:
                    if not np.any(np.logical_and(self.subscribe_list_on[si],self.subscribe_list_on[l])):
                        real_links.append(l)
                self.subscribe_list_func[si][1][0]['links'] = real_links

    def build_search_space(self,grid,values,no_val=0):
        """
            Build the search space
            Each field which has a fixed value is set as 'value'
            and the other fields are assigned with 'values': values
        """
        self.search_space = np.empty(grid.shape,dtype=dict)
        self.backtracking_space = np.full(grid.shape, False, dtype=bool)
        self.changed = np.full(grid.shape, False, dtype=bool)
        sorted_values = sorted(values)
        self.possible_range = values
        self.highest_value = sorted_values[-1]
        self.lowest_value = sorted_values[0]

        self.nof_start_values = 0
        no_val_idx = np.where(grid == no_val)
        no_val_idx_invert = np.where(grid != no_val)
        self.search_space[no_val_idx] = {'values':values[:]}
        for idx in np.transpose(no_val_idx_invert):
            t_idx = tuple(idx)
            self.nof_start_values += 1
            self.search_space[t_idx] = {'value':grid[t_idx]}

    def get_all_reachable_edges(self,G,n,used_edges):
        """
            Get all reachable edges using a start node n
            get only edges which aren't in used_edges
        """

        reachable_edges = []

        queue = [n]
        while len(queue):
            n = queue.pop()
            for s in G.successors(n):
                e = (n,s)
                if e in used_edges:
                    continue
                used_edges[e] = 1
                reachable_edges.append(e)
                queue.append(s)

        return reachable_edges,used_edges

    def alldifferent(self,ss_idx,values):
        G = nx.MultiDiGraph()

        already_know = {}
        for i in range(len(values)):
            if 'values' in values[i]:
                for j in values[i]['values']:
                    G.add_edge('x_'+str(i),j)
            else:
                G.add_edge('x_'+str(i),values[i]['value'])
                already_know[i] = 1

        # get the maximum matching of this graph
        matching = nx.bipartite.maximum_matching(G)

        n_matching = []
        GM = nx.DiGraph()
        possible = np.empty((len(values)),dtype=dict)
        for k in matching:
            if str(k)[:2] == 'x_':
                n_matching.append({k:matching[k]})
                GM.add_edge(k,matching[k])
                possible[int(k[2:])] = {'values':set([matching[k]])}

        if len(n_matching) < len(values):
            raise InfeasibleError("Matching smaller than values")

        # every edge which is not part of the matching is added (reversed)
        for e in G.edges():
            if not GM.has_edge(e[0],e[1]):
                GM.add_edge(e[1],e[0])

        # Use lemmata of Berge
        # find even alternating path
        # find free vertex
        used_edges = {}
        for n in GM.nodes():
            if str(n)[:2] != "x_" and len(GM.predecessors(n)) == 0:
                edges,used_edges = self.get_all_reachable_edges(GM,n,used_edges)

                for e in edges:
                    if str(e[0])[:2] != 'x_':
                        e = (int(e[1][2:]),e[0])
                    else:
                        e = (int(e[0][2:]),e[1])
                    if 'values' not in possible[e[0]]:
                        possible[e[0]] = {'values': set()}
                    possible[e[0]]['values'].add(e[1])

        # find cycles
        scc = nx.strongly_connected_component_subgraphs(GM)
        for scci in scc:
            for e in scci.edges():
                if str(e[0])[:2] != 'x_':
                    e = (int(e[1][2:]),e[0])
                else:
                    e = (int(e[0][2:]),e[1])
                if 'values' not in possible[e[0]]:
                    possible[e[0]] = {'values': set()}
                possible[e[0]]['values'].add(e[1])

        new_possible = []
        new_knowledge = [False]*len(values)
        i = 0
        for p in possible:
            l = list(p['values'])
            if len(l) == 1:
                new_possible.append({'value':l[0]})
                if i not in already_know:
                    new_knowledge[i] = True
            else:
                new_possible.append({'values':l[:]})
                if len(l)<len(values[i]['values']):
                    new_knowledge[i] = True
            i += 1

        return new_knowledge, new_possible

    def array_to_obj(self,array,obj_struct):
        obj = {}
        for x in array:
            obj[x] = obj_struct
        return obj

    def get_straights(self,arr_of_values,len_straight=False,possible=False,used=False,
                    used_pattern=False,all_straights=False,def_used=False):
        """
            get all straights for an array of values
            eg. [[4], [1, 2, 3, 5, 7, 8, 9], [1, 2, 5, 6, 8, 9], [1, 3, 5, 6, 8, 9], [2, 3, 5, 6, 7, 8, 9]]
            return all_straights, new_values and definitely used values
            all_straights: [[4, 1, 2, 3, 5], [4, 1, 2, 5, 3], ... [4, 8, 5, 6, 7], [4, 8, 6, 5, 7]]
            new_values: [{4}, {1, 2, 3, 5, 7, 8}, {8, 1, 2, 5, 6}, {8, 1, 3, 5, 6}, {2, 3, 5, 6, 7, 8}]
            def_used: {1: False, 2: False, 3: False, 4: True, 5: True, 6: False, 7: False, 8: False, 9: False}
        """
        if not used:
            used = self.array_to_obj(self.possible_range,False)
        if not used_pattern:
            used_pattern = []
        if not all_straights:
            all_straights = []
        if not len_straight:
            len_straight = len(arr_of_values)
        if not possible:
            possible = [set() for x in range(len(arr_of_values))]
        if not def_used:
            # assume every digit has to be used
            def_used = self.array_to_obj(self.possible_range,True)

        # take the first entry and iterate over the possible values
        # for each build a new straight (recursive)
        for value in arr_of_values[0]:
            # only use this value if it's not already part of the straight
            if not used[value]:
                copy_used = dict(used)
                copy_used[value] = True

                # set all values which aren't possible anymore to used
                # for example if the straight has a length of 2 and one value is 2
                # only 1 and 3 are possible but not 2 and not 4-...
                for i in range(self.lowest_value,max(self.lowest_value,value-len_straight+1)):
                    copy_used[i] = True
                for i in range(min(self.highest_value+1,value+len_straight),self.highest_value+1):
                    copy_used[i] = True

                copy_used_pattern = used_pattern[:]
                copy_used_pattern.append(value)
                # if there are 2 or more values left
                if len(arr_of_values) > 1:
                    all_straights,possible,def_used = self.get_straights(arr_of_values[1:],len_straight,possible,
                                                            copy_used,copy_used_pattern,all_straights,def_used)
                else:
                    # add the used pattern to all the straights
                    all_straights.append(copy_used_pattern)
                    # check if there are values which are not used in the straight
                    not_used = [x for x in self.possible_range if x not in copy_used_pattern]
                    for nu in not_used:
                        def_used[nu] = False
                    xi = 0
                    # every digit which is used is possible for the defined position
                    for x in copy_used_pattern:
                        possible[xi].add(x)
                        xi += 1
        return all_straights, possible, def_used

    def list2values_structure(self,list_of_values):
        values_struct = []
        for l in list_of_values:
            if len(l) == 1:
                values_struct.append({'value':l[0]})
            else:
                values_struct.append({'values':l})
        return values_struct

    def straight(self, ss_idx, values, opts):
        """
            the straight constraint
            ss_idx: the indices where there has to be a straight
            values: the actual values on these indices
            opts: all options
        """
        fixed_values = []

        fixed_min_val = self.highest_value
        fixed_max_val = self.lowest_value
        len_values = len(values)

        # build and array of already fixed values
        # and get the highest and lowest fixed value
        for i in range(len(values)):
            if 'value' in values[i]:
                fixed_values.append(values[i]['value'])
                if values[i]['value'] > fixed_max_val:
                    fixed_max_val = values[i]['value']
                if values[i]['value'] < fixed_min_val:
                    fixed_min_val = values[i]['value']

        min_val = self.lowest_value
        max_val = self.highest_value

        for i in range(len(values)):
            if 'values' in values[i]:
                c_min = min(values[i]['values'])
                c_max = max(values[i]['values'])
            else:
                c_min = values[i]['value']
                c_max = values[i]['value']
            if c_max+len_values-1 < max_val:
                max_val = c_max+len_values-1
            if c_min-len_values+1 > min_val:
                min_val = c_min-len_values+1

        lowest_possible = min_val
        highest_possible = max_val

        if len(fixed_values):
            # if there is a fixed value we can determine the lowest possible and highest posssible
            # by the length of the straight
            # example: min: 1 max: 3 length: 5 => values 1-5
            # max-length+1 until min+length-1  or lowest_value until highest_value as bounding
            fixed_lowest_possible  = max(fixed_max_val-len_values+1,self.lowest_value)
            lowest_possible = max(lowest_possible,fixed_lowest_possible)
            fixed_highest_possible = min(fixed_min_val+len_values-1,self.highest_value)
            highest_possible = min(highest_possible,fixed_highest_possible)

        # remove impossible
        # example: {1,3,4,6,7,8,9} and {1,3,4,6,7,8} there 1 isn't possible
        # generate arrays:

        new_possible = [False]*len_values
        new_knowledge = [False]*len_values

        arr_of_values = []
        i = 0
        for e in values:
            if 'value' in e:
                arr_of_values.append(np.array([e['value']]))
            else:
                c_new_values = [x for x in e['values'] if  lowest_possible <= x <= highest_possible]
                arr_of_values.append(np.array(c_new_values))
                if len(c_new_values) < len(e['values']):
                    new_knowledge[i] = True
            i += 1

        estimated_nof_straights = np.prod(np.array([len(x) for x in arr_of_values]))

        if estimated_nof_straights > 10000:
            return new_knowledge, self.list2values_structure(arr_of_values)

        if 'links' in opts and len(opts['links']):
            straights, new_values, def_used = self.get_straights(arr_of_values)

            if len(straights) == 0:
                raise InfeasibleError("No straight possible")

            true_keys = get_true_keys_in_dict(def_used)
            if len(true_keys):
                for l in opts['links']:
                    self.check_constraint({'idx':self.subscribe_list_on[l],'against':set(true_keys)},"notInSet")
        else:
            return new_knowledge, self.list2values_structure(arr_of_values)

        # build up the new possible values and the new knowledge we have
        i = 0
        for entry in new_values:
            entry = list(entry)
            if len(entry) > 1:
                if len(entry) < len(values[i]['values']):
                    new_knowledge[i] = True
                if len(entry) == 1:
                    new_possible[i] = {'value':entry[0]}
                else:
                    new_possible[i] = {'values':entry}
            else:
                if 'value' not in values[i]:
                    new_knowledge[i] = True
                new_possible[i] = {'value':entry[0]}

            i += 1

        # update changed and search space
        old_changed = self.changed.copy()
        self.changed[ss_idx] = new_knowledge
        self.changed = np.logical_or(self.changed,old_changed)

        self.search_space[ss_idx] = new_possible

        return new_knowledge, new_possible

    def not_in_set(self,ss_idx,values,against):
        """
            not_in set constraint
            the values shouldn't exist in against
        """
        new_possible = []
        new_knowledge = [False]*len(values)
        i = 0
        for entry in values:
            if 'values' in entry:
                nof_pos = len(entry['values'])
                new_p = set(entry['values'])-against

                if len(new_p) == 1:
                    new_knowledge[i] = True
                    new_possible.append({'value':list(new_p)[0]})
                    continue
                if len(new_p) < nof_pos:
                    new_knowledge[i] = True
                new_possible.append({'values':list(new_p)})
            else:
                new_possible.append({'value':entry['value']})
            i += 1

        # update changed and search space
        old_changed = self.changed.copy()
        self.changed[ss_idx] = new_knowledge
        self.changed = np.logical_or(self.changed,old_changed)

        self.search_space[ss_idx] = new_possible

        return new_knowledge, new_possible

    def check_constraint(self,opts,operator):
            ss_idx = opts['idx']
            values = self.search_space[ss_idx]

            breaked = False
            for v in values:
                if 'values' in v:
                    breaked = True
                    break

            if not breaked:
                return

            try:
                if operator == "notInSet":
                    new_knowledge, new_possible = self.not_in_set(ss_idx,values,opts['against'])

                elif operator == "alldifferent":
                    new_knowledge, new_possible = self.alldifferent(ss_idx,values)

                elif operator == "straight":
                    new_knowledge, new_possible = self.straight(ss_idx,values,opts)

            except InfeasibleError as e:
                raise e

            # update changed and search space
            old_changed = self.changed.copy()
            self.changed[ss_idx] = new_knowledge
            self.changed = np.logical_or(self.changed,old_changed)

            self.search_space[ss_idx] = new_possible

            # some constraints actually need a different constraint:
            try:
                # if we have a straight constraint we also have an alldifferent constraint
                if operator == "straight":
                    self.check_constraint(opts,"alldifferent")

            except InfeasibleError as e:
                raise e

    def get_solution(self):
        grid = [[0]*9 for i in range(9)]
        for r in range(len(self.search_space)):
            for c in range(len(self.search_space[r])):
                if 'value' in self.search_space[r][c]:
                    grid[r][c] = self.search_space[r][c]['value']
        return grid

    def print_search_space(self):
        for r in range(len(self.search_space)):
            row = []
            for c in range(len(self.search_space[r])):
                if 'value' in self.search_space[r][c]:
                    row.append(self.search_space[r][c]['value'])
                else:
                    row.append(self.search_space[r][c]['values'])
            print(row)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from __future__ import print_function
from jinja2 import Environment, FileSystemLoader
import os.path
import numpy as np

# uniform
def gen_popcnt_test_cases(n_bits, n_samples):
    """
    """
    assert n_bits <= 64

    test_in = []
    test_out = []
    if n_bits < 14:
        for x in range(2 ** n_bits):
            x_str = '{0:0{w}b}'.format(x, w=n_bits)
            x_arr = np.array([int(s) for s in x_str])
            test_in.append('0b' + x_str)
            test_out.append(np.sum(x_arr))
    else:
        bits = np.random.randint(0, high=2, size=(n_samples, n_bits))
        for xb in bits:
            test_in.append('0b' + ''.join([str(b) for b in xb]))
            test_out.append(np.sum(xb))
    return test_in, test_out

# uniform
def gen_popcnt_test_cases_vr(n_bits, n_samples, rate=None):
    """
    """
    assert n_bits <= 64
    if (n_bits < 14) or (rate is None) or (rate == 0.5):
        return gen_popcnt_test_cases(n_bits, n_samples)
    # bits = np.random.randint(0, high=2, size=(n_samples, n_bits))
    rv = np.random.rand(n_samples, n_bits)
    bits = np.zeros((n_samples, n_bits), dtype=int)
    bits[rv < rate] = 1
    test_in = []
    test_out = []
    for xb in bits:
        test_in.append('0b' + ''.join([str(b) for b in xb]))
        test_out.append(np.sum(xb))
    return test_in, test_out

def gen_vector(dim, dtype):
    if dtype == 'float':
        return np.array(np.random.randn(dim), dtype=np.float32)
    elif dtype == 'int':
        return np.random.randint(0, high=256, size=dim)
    elif dtype == 'bit' or dtype == 'bit_enc':
        return np.random.randint(0, high=2, size=dim)

def stringtify(vec, dtype, word_size=None):
    """

    :param vec:
    :param dtype:
    :return:
    """
    if (dtype is not 'bit') and (dtype is not 'bit_enc'):
        s = ', '.join(['{}'.format(vi) for vi in vec])
    else:
        dim = len(vec)
        tmp = []
        if word_size is None:
            word_size = 64
        i = 0
        while dim >= word_size:
            tmp.append(int(''.join([str(vi) for vi in vec[i:(i + word_size)]]), 2))
            dim -= word_size
            i += word_size
        if dim > 0:
            tmp.append(int(''.join([str(vi) for vi in vec[i:]]), 2))

        suf = 'u'
        s = ','.join([str(ti) + suf for ti in tmp])
    return s

def gen_dot_prod_test_cases(dim, n_samples, dtype1, dtype2, rtype, word_size=None):
    """

    :param dim:
    :param n_samples:
    :param dtype1:
    :param dtype2:
    :return:
    """
    tests_ = []
    for _ in range(n_samples):
        va = np.array(gen_vector(dim, dtype=dtype1))
        vb = np.array(gen_vector(dim, dtype=dtype2))

        tests_.append(
            {
                'va': {
                    'data': stringtify(va, dtype1, word_size),
                    'dtype': dtype1 if (dtype1 != 'bit' and dtype1 != 'bit_enc') else 'uint{}_t'.format(word_size)
                },
                'vb': {
                    'data': stringtify(vb, dtype2, word_size),
                    'dtype': dtype2 if (dtype2 != 'bit' and dtype2 != 'bit_enc') else 'uint{}_t'.format(word_size)
                },
                'dlen': dim,
                'res': 0,
                'rtype': rtype
            }
        )
        if dtype1 == 'bit_enc':
            va[va == 0] = -1
        if dtype2 == 'bit_enc':
            vb[vb == 0] = -1
        tests_[-1]['res'] = np.dot(va, vb)
    return tests_

# Capture our current directory
THIS_DIR = os.path.dirname(os.path.abspath(__file__))

def generate_dot_prod_cpp_test_file(template_filename, test_cases, n_tests_each=200, out_filename=None):
    # Create the jinja2 environment.
    # Notice the use of trim_blocks, which greatly helps control whitespace.
    j2_env = Environment(loader=FileSystemLoader(os.path.join(THIS_DIR, 'templates')),
                         trim_blocks=True)
    if out_filename is None:
        filename, file_extension = os.path.splitext(template_filename)
        out_filename = filename
    test_dims = (2, 4, 8, 16, 32)
    test_dims_compact = (64, 128, 256, 60, 59, 100, 119, 24, 23, 150)
    config = []
    type_name = {
        'int': 'int /* {} */',
        'float': 'float /* {} */',
        'bit': 'uint{}_t',
        'bit_enc': 'uint{}_t'
    }
    for test_case in test_cases:
        if test_case[1].find('cb') != -1:
            tdims = test_dims_compact
        else:
            tdims = test_dims
        func_suf = {
            'dot_prod': '<{t1}, {t2}, {res}>',
            'dot_prod_cb': '<{t1}, {res}, {t2}>',
            'dot_prod_cb_enc': '<{t1}, {res}, {t2}>',
            'dot_prod_cb_cb_enc': '<{res}, {t}>'
        }
        for d in tdims:
            ws = test_case[-1]
            if isinstance(ws, str):
                ws = None
            tests = gen_dot_prod_test_cases(d, n_tests_each, test_case[2], test_case[3], test_case[4], word_size=ws)
            ws_str = ws if ws is not None else ''
            config.append({
                'description': test_case[0].format(d),
                'tests': tests,
                'func': test_case[1] + func_suf[test_case[1]].format(t1=type_name[test_case[2]].format(ws_str),
                                                                     t2=type_name[test_case[3]].format(ws_str),
                                                                     res=type_name[test_case[4]].format(ws_str)) if
                test_case[
                    1] != 'dot_prod_cb_cb_enc' else
                test_case[1] + func_suf[test_case[1]].format(t=type_name[test_case[2]].format(ws_str),
                                                             res=type_name[test_case[4]].format(ws_str))
            })
    with open(out_filename, 'w') as wfp:
        wfp.write(j2_env.get_template(template_filename).render(all_tests=config))

def generate_cc_test_file(template_filename, n_tests_each=200, out_filename=None):
    # Create the jinja2 environment.
    # Notice the use of trim_blocks, which greatly helps control whitespace.
    j2_env = Environment(loader=FileSystemLoader(os.path.join(THIS_DIR, 'templates')),
                         trim_blocks=True)
    if out_filename is None:
        filename, file_extension = os.path.splitext(template_filename)
        out_filename = filename
    test_cases = (
        (32, 'unsigned_int_tests'), (64, 'unsigned_long_long_tests'),
        (8, 'uint8_t_tests'), (16, 'uint16_t_tests'), (32, 'uint32_t_tests'),
        (64, 'uint64_t_tests'), (16, 'unsigned_short_tests'), (32, 'unsigned_long_tests')
    )
    config = {}
    #
    rate = None
    for nb, key in test_cases:
        k = n_tests_each
        test_in = []
        test_out = []
        for rate in (0.2, 0.4, 0.6, 0.8):
            test_in_, test_out_ = gen_popcnt_test_cases_vr(nb, n_samples=k, rate=rate)
            test_in += test_in_
            test_out += test_out_
        test_in.append('0b0')
        test_out.append(0)
        test_in.append('0b' + ('1' * nb))
        test_out.append(nb)
        config[key] = [
            {'in': i, 'out': o} for i, o in zip(test_in, test_out)
        ]
    with open(out_filename, 'w') as wfp:
        wfp.write(j2_env.get_template(template_filename).render(tests=config))

if __name__ == '__main__':
    generate_cc_test_file("popcnt_test.cpp.template", n_tests_each=10, out_filename="popcnt_s_test.cpp")
    generate_cc_test_file("popcnt_test.cpp.template", n_tests_each=200, out_filename="popcnt_l_test.cpp")
    for test_cases in (
            (
            ('T dot_prod(const T *,const T *, size_t) with T=int, dim={}', 'dot_prod', 'int', 'int', 'int'),
            ('T dot_prod(const T *,const T *, size_t) with T=float, dim={}', 'dot_prod', 'float', 'float',
             'float'),
            ('float dot_prod(const T1 *,const T2 *, size_t) with T1=int, T2=float, dim={}', 'dot_prod', 'int',
             'float',
             'float'),
            ('float dot_prod(const T1 *,const T2 *, size_t) with T1=float, T2=int, dim={}', 'dot_prod', 'float',
             'int',
             'float')),
            (('int dot_prod_cb(const int *,const uint32_t *, size_t), dim={}', 'dot_prod_cb', 'int', 'bit', 'int', 32),
             ('float dot_prod_cb(const float *,const uint32_t *, size_t), dim={}', 'dot_prod_cb', 'float', 'bit',
              'float',
              32),
             ('int dot_prod_cb(const int*, const uint64_t*, size_t), dim={}', 'dot_prod_cb', 'int', 'bit', 'int', 64),
             (
                     'float dot_prod_cb(const float*, const uint64_t*, size_t), dim={}', 'dot_prod_cb', 'float', 'bit',
                     'float',
                     64)),
            (('int dot_prod_cb_enc(const int*, const uint32_t*, size_t), dim={}', 'dot_prod_cb_enc', 'int', 'bit_enc',
              'int',
              32),
             ('float dot_prod_cb_enc(const float*, const uint32_t*, size_t), dim={}', 'dot_prod_cb_enc', 'float',
              'bit_enc',
              'float', 32),
             (
                     'int dot_prod_cb_enc(const int*, const uint64_t*, size_t), dim={}', 'dot_prod_cb_enc', 'int',
                     'bit_enc', 'int',
                     64),
             ('float dot_prod_cb_enc(const float*, const uint64_t*, size_t), dim={}', 'dot_prod_cb_enc', 'float',
              'bit_enc',
              'float', 64)),
            ((
                     'int dot_prod_cb_cb_enc(const uint64_t*, const uint64_t*, size_t), dim={}', 'dot_prod_cb_cb_enc',
                     'bit_enc',
                     'bit_enc',
                     'int', 64),
             ('int dot_prod_cb_cb_enc(const uint32_t*, const uint32_t*, size_t), , dim={}', 'dot_prod_cb_cb_enc',
              'bit_enc',
              'bit_enc', 'int', 32))):
        generate_dot_prod_cpp_test_file('dot_prod_test.cpp.template',
                                        test_cases=test_cases,
                                        n_tests_each=5,
                                        out_filename=test_cases[0][1] + '_s_test.cpp')
        generate_dot_prod_cpp_test_file('dot_prod_test.cpp.template',
                                        test_cases=test_cases,
                                        n_tests_each=100,
                                        out_filename=test_cases[0][1] + '_l_test.cpp')


# coding = utf-8
import json
import cv2
from fastapi import FastAPI, UploadFile
from fastapi.encoders import jsonable_encoder
from fastapi.responses import JSONResponse
import numpy as np
from handler.http_handler import init_http_exception_handler
from models.captcha_models import *
from models.response_models import captcha_response

from hypercorn.asyncio import serve
from hypercorn import Config
import asyncio

ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'JPG', 'JPEG', 'PNG'])

app = FastAPI(
    title="圖片驗證API",
    description='',
    version="0.0.1",
    terms_of_service="",
)

init_http_exception_handler(app)

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS

@app.get("/")
def index():
    return "HI, 歡迎使用圖形驗證碼API，若要看相關文件請至 /docs"

@app.post("/api/captcha_id", response_model= captcha_response)
async def captcha_id(file: UploadFile):
    """
    圖形驗證碼

    - **tags**: 圖片驗證碼五碼六碼通用
    - **consumes**: ["multipart/form-data"]
    - **parameters**:
        - name: file
        - in: formData
        - type: file
        - required: true
    - **responses**:
        - HttpStatus:
            - 200: 回傳運算結果
            - 400: 檔案上傳不成功 / 檔案格式不符
            - 404: 更版或維修中，請通知管理員
            - 500: 服務機出現異常，請通知管理員
        - status: S成功 F失敗
        - res: 運算結果或錯誤資訊
    """

    if not file:
        return captcha_response(status = 400, res = '檔案上傳不成功')
    else:
        if allowed_file(file.filename):
            try:
                contents = await file.read()
                image_bytes = np.frombuffer(contents, np.uint8)
                image = cv2.imdecode(image_bytes, cv2.IMREAD_COLOR)
                image = cv2.resize(image, (200, 60), interpolation=cv2.INTER_AREA)
                model = captcha_model_id()
                model.load_weights('./checkpoints/verificatioin_code-1118-id.h5')
                captcha_text = model.predict(image)
                # return captcha text
                return JSONResponse(jsonable_encoder(captcha_response(status = "S", res = captcha_text)), 200)
            except:
                return JSONResponse(jsonable_encoder(captcha_response(status = "F", res = '服務機出現異常，請通知管理員')), 500)
        else:
            return JSONResponse(jsonable_encoder(captcha_response(status = "F", res = '檔案格式不符')), 400)

if __name__ == "__main__":
    file = open("./configs/hypercorn_config.json")
    config = json.load(file)

    hc_config = Config()
    hc_config.worker_class = config["worker_class"]
    hc_config.workers = config["workers"]
    hc_config.keep_alive_timeout = config["keep_alive_timeout"]

    if config['https']:
        hc_config.bind = [config['ip'] + ':443']
        hc_config.certfile = config['certfile']
        hc_config.keyfile = config['keyfile']
        hc_config.insecure_bind = [config['ip'] + ":" + config['port']]
    else:
        hc_config.bind = [config['ip'] + ":" + config['port']]

    # config定義在 ./config/hypercorn_config
    asyncio.run(serve(app, hc_config))


# -*- coding: utf-8 -*-
# !/usr/bin/env python

import logging
import yaml
import os

from tornado.escape import json_encode
from tornado.ioloop import IOLoop
from tornado.options import define, options, parse_command_line
from tornado.web import Application, RequestHandler

from web3 import Web3
from libs.transaction_maker import TransactionMaker

define('port', default=11211, help="port to listen on")
define('host', default='127.0.0.1', help="host to listen on")
define('debug', default=False, group='application',
       help="run in debug mode (with automatic reloading)")

estconf = yaml.load(open(os.path.dirname(__file__) + '/conf/est.yaml'))

class BaseHandler(RequestHandler):

    def get(self):
        self.write_error(404)

    def write_error(self, status_code, **kwargs):
        data = dict()
        data["ret"] = data["code"] = status_code
        data["msg"] = "error"
        data['data'] = dict()
        self.write(json_encode(data))

class MainHandler(BaseHandler):
    """
    """

    def get(self):
        pass

class EstAccountHandler(BaseHandler):

    def get(self):
        ret = {'ret': 0, 'code': 0, 'msg': '', 'data': dict()}
        address = self.get_argument('address', '', True)
        est = self.get_argument('est', '', True)
        if not address or not est:
            ret['ret'] = ret['code'] = 400
            ret['msg'] = 'address or est cannot be empty'
            self.write(json_encode(ret))
            return

        address = Web3.toChecksumAddress(address)
        if not Web3.isChecksumAddress(address):
            ret['ret'] = ret['code'] = 400
            ret['msg'] = 'address invalid'
            self.write(json_encode(ret))
            return

        base_conf = estconf['base']

        trans = TransactionMaker(provider_url=base_conf['provider_url'], my_wallet_address=base_conf['from_address'])
        trans.set_abi_type(erc20_abi_json=base_conf['erc20_abi'])

        value = int(float(est) * 10 ** 9)
        cost = trans.get_transfer_cost(to=address, value=value, contract_address=base_conf['contract_address'])
        if not cost:
            ret['ret'] = ret['code'] = 400
            ret['msg'] = 'System maintenance'
            self.write(json_encode(ret))
            return
        ret['data'] = {'cost': cost}

        self.write(json_encode(ret))
        return

def main():
    parse_command_line()

    app = Application([(r"/", MainHandler), (r"/est", EstAccountHandler), (r".*", BaseHandler), ])
    print(options.port)
    app.listen(options.port, address='127.0.0.1')
    logging.info('Listening on http://localhost:%d' % options.port)

    IOLoop.current().start()

if __name__ == '__main__':
    main()


import re
from datetime import date, timedelta

from dateutil.relativedelta import relativedelta

from RPA.Robocorp.WorkItems import WorkItems

def get_date_range(months: int) -> tuple:
    """
    Get date range between the last day of current month
    and the first day of X months ago

    Args:
        months: The number of months to look back

    Raises:
        Exception: Invalid month in case of a negative value

    Returns:
        Date range tuple in a string DD/MM/YYYY format
    """
    if months == 0:
        months = 1
    if months < 0:
        raise Exception("Invalid month requested")
    today = date.today()
    start_date = (today - relativedelta(months=months - 1)).replace(day=1)
    end_date = date.today().replace(day=1, month=today.month + 1) - timedelta(days=1)
    start_date = date.strftime(start_date, "%m/%d/%Y")
    end_date = date.strftime(end_date, "%m/%d/%Y")
    return start_date, end_date

def search_currency(string: str) -> bool:
    """
    Search for any currency occurrence in string
    using regex

    Args:
        string: String to search

    Returns:
        Matches or None
    """
    exp = r"(\$[0-9][0-9,]*(.[0-9]+)?)|([0-9][0-9,]*(.[0-9]+)?( dollars| USD))"
    return re.search(exp, string)

def configure():
    config = WorkItems()
    config.get_input_work_item()
    search_phrase = config.get_work_item_variable("search_phrase", "Python")
    section = config.get_work_item_variable("section", "")
    months = int(config.get_work_item_variable("months", 0))
    return search_phrase, section, months


import pandas as pd
from scipy.stats import ttest_ind, ttest_rel
import math
import matplotlib.pyplot as plt
import numpy as np

totalDf = pd.read_csv('Люшер_fatigue_.csv', sep=',', header=0)

results = []

for i in range(0,12):
    data = totalDf[totalDf['Gr_ug'] == i + 1]

    data1 = data['before'].tolist()
    data2 = data['in'].tolist()
    data3 = data['after'].tolist()

    row = []

    stat, p = ttest_rel(data2, data3)
    print(stat, p)
    if math.isnan(p):
        p = 0
    row.append(p)

    results.append(row)

with open('ttest.csv', 'w') as f:
    for r in results:
        f.write(",".join(list(map(str, r))))
        f.write("\n")

fig = plt.figure()
ax = fig.add_subplot(111)
cax = ax.matshow(results,cmap='binary', vmin=0, vmax=1)
fig.colorbar(cax)
ticks = np.arange(0,len(results),1)
ticks_labels = np.arange(1,len(results) + 1,1)
ax.set_xticks(np.arange(1))
plt.xticks(rotation=90)
ax.set_yticks(ticks)
ax.set_xticklabels(np.arange(1))
ax.set_yticklabels(ticks_labels)

plt.show()

print(results)


# -*- coding : utf-8 -*-
# 创建时间：2019年7月20
# 功能：调试出最佳的识别黑色兴趣区域的参数值
import cv2
import numpy as np

# cap = cv2.VideoCapture(0)  # 获取摄像头图像
# cap = cv2.VideoCapture('../video/test_rectangle_1.mp4')

img = cv2.imread("./1.jpg")
img = cv2.resize(img,(960,600))
hsv_frame = cv2.cvtColor(img,cv2.COLOR_BGR2HSV)

def nothing(x):
    pass

def createbars():
    """
    实现创建六个滑块的作用，分别控制H、S、V的最高值与最低值
    H:色调；S：饱和度；V：亮度
    """
    cv2.createTrackbar("H_l", "image", 0, 180, nothing)
    cv2.createTrackbar("H_h", "image", 0, 180, nothing)
    cv2.createTrackbar("S_l", "image", 0, 255, nothing)
    cv2.createTrackbar("S_h", "image", 0, 255, nothing)
    cv2.createTrackbar("V_l", "image", 0, 255, nothing)
    cv2.createTrackbar("V_h", "image", 0, 255, nothing)

cv2.namedWindow("image")
createbars()  # 创建六个滑块

lower = np.array([0, 0, 0])  # 设置初始值
upper = np.array([0, 0, 0])
while True:
    # ret, frame = cap.read()
    # hsv_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)  # 将图片由BGR颜色空间转化成HSV空间，HSV可以更好地分割颜色图形
    hsv_frame = cv2.GaussianBlur(hsv_frame,(9,9),0)
    lower[0] = cv2.getTrackbarPos("H_l", "image")  # 获取"H_l"滑块的实时值
    upper[0] = cv2.getTrackbarPos("H_h", "image")  # 获取"H_h"滑块的实时值
    lower[1] = cv2.getTrackbarPos("S_l", "image")
    upper[1] = cv2.getTrackbarPos("S_h", "image")
    lower[2] = cv2.getTrackbarPos("V_l", "image")
    upper[2] = cv2.getTrackbarPos("V_h", "image")

    mask = cv2.inRange(hsv_frame, lower, upper)  # cv2.inrange()函数通过设定的最低、最高阈值获得图像的掩膜
    # mask = cv2.inRange(hsv_img, lower, upper)  # cv2.inrange()函数通过设定的最低、最高阈值获得图像的掩膜
    # cv2.imshow("img", frame)
    cv2.imshow("img",img)
    cv2.imshow("mask", mask)
    if cv2.waitKey(1) & 0xff == 27:
        break

cv2.destroyAllWindows()


from itertools import combinations
import pathlib
import unittest

PUZZLE_INPUT = "input.txt"
TEST_INPUT = "test-input.txt"

def encoding_error_1(xmas_data: list[int], preamble_len: int) -> int:
    for num in range(preamble_len, len(xmas_data)):
        prev_nums = [xmas_data[n] for n in range(num - preamble_len, num)]
        if not check_sum(prev_nums, xmas_data[num]):
            return xmas_data[num]

def check_sum(nums: list[int], k: int) -> bool:
    return any(sum(pair) == k for pair in combinations(nums, 2))

def encoding_error_2(xmas_data: list[int], preamble_len: int) -> int:
    invalid_num = encoding_error_1(xmas_data, preamble_len)
    con_set = find_contiguous_set(xmas_data, invalid_num)
    return min(con_set) + max(con_set)

def find_contiguous_set(xmas_data: list[int], invalid_num: int) -> list[int]:
    curr_set = []
    for i in range(len(xmas_data)):
        curr_set.append(xmas_data[i])
        curr_sum = sum(curr_set)
        if curr_sum == invalid_num:
            return curr_set
        while curr_sum > invalid_num:
            curr_set.pop(0)
            curr_sum = sum(curr_set)
            if curr_sum == invalid_num:
                return curr_set

def process_input(file_name: str) -> list[int]:
    with (pathlib.Path(__file__).parent / file_name).open() as input_file:
        output = list(map(int, input_file.readlines()))
    # Output: [0, 1, 2, 3,...]
    return output

class Test(unittest.TestCase):
    def test_1(self):
        self.assertEqual(127, encoding_error_1(process_input(TEST_INPUT), 5))
    def test_2(self):
        self.assertEqual(62, encoding_error_2(process_input(TEST_INPUT), 5))

if __name__ == "__main__":
    unittest.main(exit=False)

    pro_input = process_input(PUZZLE_INPUT)
    print(f"Part 1 Answer: {encoding_error_1(pro_input, 25)}")
    print(f"Part 2 Answer: {encoding_error_2(pro_input, 25)}")


import pygame
from collections import deque
from random import choice, randrange

class Ant():
    def __init__(self,app,pos,color):
        self.app = app
        self.color = color
        self.x, self.y = pos
        self.increments = deque([(1,0),(0,1),(-1,0),(0,-1)])
    def run(self): #définition des règles que doit suivre la fourmi pour se déplacer
        value = self.app.grid[self.y][self.x] #position de la fourmi sur l'écran défini par app
        self.app.grid[self.y][self.x] = not value

        SIZE = self.app.CELL_SIZE #On dessine non plus des carrés mais des ronds
        center = self.x * SIZE, self.y * SIZE
        if value:
            pygame.draw.circle(self.app.screen, self.color, center, SIZE)

        self.increments.rotate(1) if value else self.increments.rotate(-1) #rotate deque de droite à gauche si 1 et gauche à droite si -1, dépend de la case sur laquelle se trouve la fourmi à la position d'avant

        dx, dy = self.increments[0] #le premier élément de deque est donc l'incrément du prochain pas de la fourmi
        self.x = (self.x + dx) % self.app.COLS #pour que la fourmi ne quitte pas les frontières du tableau on prend le reste
        self.y = (self.y + dy) % self.app.ROWS

class App():
    def __init__(self,CELL_SIZE=6,COLOR=(0,0,0),SPEED=120,NUMBER=400):
        pygame.init()
        self.screen = pygame.display.set_mode((0,0), pygame.FULLSCREEN)
        WIDTH, HEIGHT = self.screen.get_size()
        self.clock = pygame.time.Clock()
        self.screen.fill(COLOR)
        self.speed = SPEED
        self.number = NUMBER

        self.CELL_SIZE = CELL_SIZE
        self.ROWS, self.COLS = HEIGHT//CELL_SIZE, WIDTH//CELL_SIZE
        self.grid = [[0 for col in range(self.COLS)] for row in range(self.ROWS)]

#On crée 2 groupes de fourmis tels que au sein d'un groupe, les fourmis démarrent toutes du même point
        colors1 = [(250, 150, i) for i in range(256)]
        colors2 = [(50, i, 200) for i in range(256)]
        ants1 = [Ant(self, [self.COLS // 3, self.ROWS // 2],
                     choice(colors1)) for i in range(NUMBER//2)]
        ants2 = [Ant(self, [self.COLS - self.COLS // 3, self.ROWS // 2],
                     choice(colors2)) for i in range(NUMBER//2)]
        self.ant = ants1 + ants2

    @staticmethod
    def get_color():
        channel = lambda: randrange(30,220)
        return channel(), channel(), channel()

    def run(self):
        A = 1
        while A == 1:
            [ant.run() for ant in self.ant]

            for event in pygame.event.get() :
                if event.type == pygame.QUIT :
                    exit()
                if event.type == pygame.KEYDOWN and event.key == pygame.K_ESCAPE :
                    A = 0

            pygame.display.flip()
            self.clock.tick(self.speed)

if __name__ == '__main__':
    app = App()
    app.run()


#
#  Project: MXCuBE
#  https://github.com/mxcube
#
#  This file is part of MXCuBE software.
#
#  MXCuBE is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Lesser General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  MXCuBE is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public License
#  along with MXCuBE.  If not, see <http://www.gnu.org/licenses/>.

import logging
import numpy.oldnumeric as Numeric

from mxcubeqt.utils import qt_import
from mxcubeqt.base_components import BaseWidget

try:
    if qt_import.qt_variant == "PyQt5":
        from PyMca5.PyMca import McaAdvancedFit
        from PyMca5.PyMca import ConfigDict
    else:
        from PyMca import McaAdvancedFit
        from PyMca import ConfigDict
except BaseException:
    pass

__credits__ = ["MXCuBE collaboration"]
__license__ = "LGPLv3+"
__category__ = "General"

class McaSpectrumBrick(BaseWidget):
    def __init__(self, *args):
        BaseWidget.__init__(self, *args)

        self.defineSlot("setData", ())

        self.mcafit = McaAdvancedFit.McaAdvancedFit(self)
        self.mcafit.dismissButton.hide()

        # Layout --------------------------------------------------------------
        main_layout = qt_import.QVBoxLayout(self)
        main_layout.addWidget(self.mcafit)
        # main_layout.addWidget(self.scan_plot_widget)
        main_layout.setSpacing(0)
        main_layout.setContentsMargins(0, 0, 0, 0)
        self.setLayout(main_layout)

    def setData(self, data, calib, config):
        try:
            configured = False
            if config.get("file", None):
                self._configure(config)
                configured = True
            data = Numeric.array(data)
            x = Numeric.array(data[:, 0]).astype(Numeric.Float)
            y = Numeric.array(data[:, 1]).astype(Numeric.Float)
            calib = Numeric.ravel(calib).tolist()
            self.mcafit.setdata(x, y)
            self.mcafit._energyAxis = False
            self.mcafit.toggleEnergyAxis()
            result = self._fit()
            # pyarch file name and directory
            pf = config["legend"].split(".")
            pd = pf[0].split("/")
            outfile = pd[-1]
            outdir = config["htmldir"]
            sourcename = config["legend"]

            if configured:
                report = McaAdvancedFit.QtMcaAdvancedFitReport.QtMcaAdvancedFitReport(
                    None,
                    outfile=outfile,
                    outdir=outdir,
                    fitresult=result,
                    sourcename=sourcename,
                    plotdict={"logy": False},
                    table=2,
                )

                text = report.getText()
                report.writeReport(text=text)

        except BaseException:
            logging.getLogger().exception(
                "McaSpectrumBrick: problem fitting %s %s %s"
                % (str(data), str(calib), str(config))
            )

    def _fit(self):
        return self.mcafit.fit()

    def _configure(self, config):
        d = ConfigDict.ConfigDict()
        d.read(config["file"])
        if "concentrations" not in d:
            d["concentrations"] = {}
        if "attenuators" not in d:
            d["attenuators"] = {}
            d["attenuators"]["Matrix"] = [1, "Water", 1.0, 0.01, 45.0, 45.0]
        if "flux" in config:
            d["concentrations"]["flux"] = float(config["flux"])
        if "time" in config:
            d["concentrations"]["time"] = float(config["time"])
        self.mcafit.mcafit.configure(d)

    def clear(self):
        x = Numeric.array([0]).astype(Numeric.Float)
        y = Numeric.array([0]).astype(Numeric.Float)
        self.mcafit.setdata(x, y)


from django.db import models
app_name = 'example_app'

class GenericMethods():
    @classmethod
    def get_model_urls(cls):
        return {
            'index':    f'{app_name}:{cls.__name__.lower()}_index'
            ,'create':  f'{app_name}:{cls.__name__.lower()}_create'
            ,'detail':  f'{app_name}:{cls.__name__.lower()}_detail'
            ,'update':  f'{app_name}:{cls.__name__.lower()}_update'
            ,'delete':  f'{app_name}:{cls.__name__.lower()}_delete'
            ,'list':    f'{app_name}:{cls.__name__.lower()}_list'
        }

    @classmethod
    def get_model_form_name(cls):
        return f'{cls.__name__}Form'

class GenderExample(models.Model, GenericMethods):
    id          = models.AutoField(primary_key=True)
    uuid        = models.CharField(max_length=50, unique=True)
    description = models.CharField(max_length=50)

    delete_flag = models.BooleanField(default=False)
    created_at  = models.DateTimeField(auto_now_add=True)
    updated_at  = models.DateTimeField(auto_now=True)

    def __str__(self):
        return self.uuid

    @classmethod
    def get_model_info(cls):
        return {
            'title': {
                'main':     ''
                ,'index':   'Género - Gestión'
                ,'detail':  'Género - Detalle'
                ,'create':  'Género - Nuevo'
                ,'update':  'Género - Editar'
                ,'delete':  'Género - Borrar'
                ,'list':    'Género - Listado'

            },
            'index': {
                'data_table': {
                    'objects': cls.objects.order_by('-updated_at'),
                    'object_info': [
                        {'label': 'Género'        ,'attr': 'uuid'         ,'exportable': True},
                        {'label': 'Descripción'   ,'attr': 'description'  ,'exportable': True}
                    ]
                }
            },
            'detail': {
                'object_info': [
                    {'label': 'Género: '        ,'attr': 'uuid'},
                    {'label': 'Descripción: '   ,'attr': 'description'}
                ]
            }
        }

class PersonExample(models.Model, GenericMethods):
    id 			= models.AutoField(primary_key=True)
    uuid 		= models.CharField(max_length=50, unique=True)
    firstName 	= models.CharField(max_length=50)
    lastName  	= models.CharField(max_length=50)
    gender      = models.ForeignKey(GenderExample, on_delete=models.CASCADE, null=True)

    delete_flag = models.BooleanField(default=False)
    created_at 	= models.DateTimeField(auto_now_add=True)
    updated_at 	= models.DateTimeField(auto_now=True)

    def __str__(self):
        return 'DNI: ' + self.uuid + ' - ' + self.firstName + ' ' + self.lastName

    @classmethod
    def get_model_info(cls):
        return {
            'title': {
                'main':     ''
                ,'index':   'Personas - Gestión'
                ,'detail':  'Personas - Detalle'
                ,'create':  'Personas - Nueva'
                ,'update':  'Personas - Editar'
                ,'delete':  'Personas - Borrar'
                ,'list':    'Personas - Listado'

            },
            'index': {
                'data_table': {
                    'objects': cls.objects.order_by('-updated_at'),
                    'object_info': [
                        {'label': 'DNI'                     ,'attr': 'uuid'       ,'exportable': True},
                        {'label': 'Nombre '                 ,'attr': 'firstName'  ,'exportable': True},
                        {'label': 'Apellido'                ,'attr': 'lastName'   ,'exportable': True},
                        {'label': 'Género'                  ,'attr': 'gender'     ,'exportable': True},
                        {'label': 'Actualizado en'          ,'attr': 'updated_at' ,'exportable': True}
                    ]
                }
            },
            'detail': {
                'object_info': [
                    {'label': 'DNI'             ,'attr': 'uuid'},
                    {'label': 'Nombre '         ,'attr': 'firstName'},
                    {'label': 'Apellido'        ,'attr': 'lastName'},
                    {'label': 'Género'          ,'attr': 'gender'},
                    {'label': 'Actualizado en'  ,'attr': 'updated_at'},
                ]
            }
        }


#Create a Database
import sqlite3

#Create a connnection or create a new database if one does not exist
conn = sqlite3.connect('employee.db')

#create a cursor object
cur = conn.cursor()

#Create a Table
create_table = """
            CREATE TABLE employees(
            emp_id integer PRIMARY,
            f_name text,
            l_name test,
            job_title test,
            )

"""

#excute create table statement
cur.execute(create_table)

print('Table has been created')

#commit changes to the database
conn.commit()

#close connection
conn.close()

SQLite Tutorial 1 - Create a Database and Tabels.py


import typing

class Memento:
    def __init__(self, balance: int) -> None:
        self.balance = balance

class BankAccount:
    def __init__(self, balance: int) -> None:
        self.balance = balance
        self.changes: typing.List["Memento"] = [
            Memento(self.balance),
        ]
        self.current: int = 0

    def deposit(self, amount: int) -> "Memento":
        self.balance += amount
        memento = Memento(self.balance)
        self.changes.append(memento)
        self.current += 1
        return memento

    def restore(self, memento: "Memento") -> None:
        if memento:
            self.balance = memento.balance
        self.changes.append(memento)
        self.current = len(self.changes) - 1

    def undo(self) -> "Memento":
        memento = None
        if self.current > 0:
            self.current -= 1
            memento = self.changes[self.current]
            self.balance = memento.balance
        return memento

    def redo(self) -> "Memento":
        memento = None
        if self.current + 1 < len(self.changes):
            self.current += 1
            memento = self.changes[self.current]
            self.balance = memento.balance
        return memento

    def __str__(self) -> str:
        return f"Balance = {self.balance}"

if __name__ == "__main__":
    ba = BankAccount(100)
    m1 = ba.deposit(50)
    m2 = ba.deposit(25)
    print(ba)

    ba.undo()
    print(f"Undo 1: {ba}")
    ba.undo()
    print(f"Undo 2: {ba}")
    ba.redo()
    print(f"Redo 1: {ba}")


import numpy as np
import tensorflow.compat.v1 as tf
from utils.run import run_parser
from collections import OrderedDict

def create_overlapadd_model(pb_file_path, input_size, step):
    ''' Create tensorflow model for overlap_add op.
    '''
    with tf.Session(graph=tf.Graph()) as sess:
        x = tf.placeholder(tf.float32, shape=input_size, name='X')
        y = tf.signal.overlap_and_add(signal=x, frame_step=step, name='y')
        relu6 = tf.nn.relu6(y, name='relu6')

        sess.run(tf.global_variables_initializer())
        constant_graph = tf.graph_util.convert_variables_to_constants(
            sess, sess.graph_def, ['relu6'])

        # save to pb file
        with tf.gfile.GFile(pb_file_path, mode='wb') as f:
            f.write(constant_graph.SerializeToString())

TEST_NAME = 'overlap_add'
input_shapes = [[2, 3, 4], [3, 4], [2, 3, 4, 5]]

for frame_step in range(2, 5):
    for input_shape in input_shapes:

        # Generate input data
        feed_dict = dict()
        feed_dict['X:0'] = np.random.ranf(input_shape).astype(np.float32)

        model_path = TEST_NAME + str(frame_step)+'.pb'
        # Create model
        create_overlapadd_model(model_path, input_shape, frame_step)

        # Run tests with parser and compare result with runtime
        exit_status = run_parser(
            model_path, feed_dict, model_type='tensorflow', save_output=True, verify=True)
        assert exit_status


from bs4 import BeautifulSoup
import requests
import time

def find_items():
    global item_link
    html_text = requests.get(
        'https://tr.uspoloassn.com/erkek-tum-urunler/').text
    soup = BeautifulSoup(html_text, 'html.parser')
    cards = soup.find_all('div', class_='product__listing--content')

    for index, card in enumerate(cards):

        item_name = (card.find(class_='product__name')).a.text

        item_price_group = card.find(
            'div', class_='product__listing--basket-price')
        item_price = item_price_group.find(
            'span').text.replace(',', '.').replace(' TL', '')

        item_brand = 'UsPolo'

        item_link = card.find('span', 'product__name').a['href']

    #     with open(f'posts/Trendyol/Erkek/{index}.txt', 'w', encoding="utf-8") as f:
    #         f.write(f"item name: {item_name}\n")
    #         f.write(f"item price: {item_price.strip()}\n")
    #         f.write(f"item brand: {item_brand}\n")
    #         f.write(f"item address: https://tr.uspoloassn.com{item_link}\n")
    #         f.write("_______________________")
    # print("files have been created")

        print(index)
        print(f"item name: {item_name}")
        print(f"item price: {item_price.strip()}")
        print(f"item brand: {item_brand}")
        print(f"item address: https://tr.uspoloassn.com{item_link}")
        print('-----------------------')

find_items()

def item_info():
    new_item_link = "https://tr.uspoloassn.com"+item_link

    html_text = requests.get(new_item_link).text
    selected_part = html_text.find_all('ul', class_='cf')
    print(selected_part)

item_info()


import os
import shutil
import unittest

import yaml

from ymir_exc import env, monitor, settings
from ymir_exc.code import ExecutorReturnCode, ExecutorState

class TestMonitor(unittest.TestCase):
    # life cycle
    def __init__(self, methodName: str = ...) -> None:
        super().__init__(methodName)
        self._test_root = os.path.join('/tmp', 'test_tmi', *self.id().split(".")[-3:])
        self._custom_env_file = os.path.join(self._test_root, 'env.yml')
        self._training_result_file = os.path.join(self._test_root, 'out', 'training-result.yaml')
        self._mining_result_file = os.path.join(self._test_root, 'out', 'mining-result.tsv')
        self._infer_result_file = os.path.join(self._test_root, 'out', 'infer-result.json')
        self._monitor_file = os.path.join(self._test_root, 'out', 'monitor.txt')

    def setUp(self) -> None:
        settings.DEFAULT_ENV_FILE_PATH = self._custom_env_file
        self._prepare_dirs()
        self._prepare_env_config()
        return super().setUp()

    def tearDown(self) -> None:
        self._deprepare_dirs()
        return super().tearDown()

    # protected: setup and teardown
    def _prepare_dirs(self) -> None:
        if os.path.isdir(self._test_root):
            shutil.rmtree(self._test_root)
        os.makedirs(self._test_root)
        os.makedirs(os.path.join(self._test_root, 'out'), exist_ok=True)

    def _prepare_env_config(self) -> None:
        env_obj = {
            'task_id': 'task0',
            'output': {
                'root_dir': os.path.join(self._test_root, 'out'),
                'models_dir': os.path.join(self._test_root, 'out', 'models'),
                'training_result_file': self._training_result_file,
                'mining_result_file': self._mining_result_file,
                'infer_result_file': self._infer_result_file,
                'monitor_file': self._monitor_file,
            },
        }
        with open(self._custom_env_file, 'w') as f:
            yaml.safe_dump(env_obj, f, allow_unicode=True)

    def _deprepare_dirs(self) -> None:
        if os.path.isdir(self._test_root):
            shutil.rmtree(self._test_root)

    # protected: check results
    def _check_monitor(self, percent: float, state: ExecutorState, return_code: ExecutorReturnCode) -> None:
        with open(self._monitor_file, 'r') as f:
            lines = f.read().splitlines()
        task_id, timestamp_str, percent_str, state_str, return_code_str, *_ = lines[0].split()

        self.assertEqual(task_id, env.get_current_env().task_id)
        self.assertTrue(float(timestamp_str) > 0)
        self.assertEqual(percent, float(percent_str))
        self.assertEqual(state, int(state_str))
        self.assertEqual(return_code, int(return_code_str))

    # public: test cases
    def test_write_monitor(self) -> None:
        monitor.write_monitor_logger(percent=0.2)
        self._check_monitor(percent=0.2, state=ExecutorState.ES_RUNNING, return_code=ExecutorReturnCode.RC_EXEC_NO_ERROR)

        monitor.write_monitor_logger(percent=0.5, return_code=ExecutorReturnCode.RC_EXEC_NO_GPU)
        self._check_monitor(percent=0.5, state=ExecutorState.ES_RUNNING, return_code=ExecutorReturnCode.RC_EXEC_NO_ERROR)

        monitor.write_monitor_logger(percent=1, state=ExecutorState.ES_ERROR)
        self._check_monitor(percent=1, state=ExecutorState.ES_ERROR, return_code=ExecutorReturnCode.RC_EXEC_NO_ERROR)

        monitor.write_monitor_logger(percent=1, state=ExecutorState.ES_ERROR, return_code=ExecutorReturnCode.RC_EXEC_OOM)
        self._check_monitor(percent=1, state=ExecutorState.ES_ERROR, return_code=ExecutorReturnCode.RC_EXEC_OOM)


from collections import deque
import threading
import time

lock = threading.Lock()

class Station(threading.Thread):

    start_time = 0

    def __init__(self, name, duration):
        threading.Thread.__init__(self)
        self.name = name
        self.duration = duration
        self.queue = deque([])
        self.isOperate = False
        self.arrEv = threading.Event()

    def run(self):
        while True:
            self.arrEv.wait()
            if len(self.queue) != 0:
                task = self.leave()
                time.sleep(task[0] * self.duration)
                print(f"{time.time() - self.start_time:.2f}: " + task[2].name + " Finished at " + self.name)
                task[1].set()
            else:
                self.arrEv.clear()

    def arrival(self, task):
        lock.acquire()
        self.queue.append(task)
        lock.release()

    def leave(self):
        lock.acquire()
        task = self.queue.pop()
        lock.release()
        return task


import matplotlib, matplotlib.pyplot as plt
import pickle, pandas as pd

# 앞에서 NIAAA 데이터 프레임을 pickle에 저장해 두었다.
alco = pickle.load(open("alco.pickle", "rb"))
del alco["Total"]
columns, years = alco.unstack().columns.levels

# 파일에서 축약된 주 이름을 읽어온다.
states = pd.read_csv("states.csv", index_col="State")

# 2009년을 기준으로 알코올 소비량을 정렬한다.
frames = [pd.merge(alco[column].unstack(), states,
                   left_index=True, right_index=True).sort_values(2009)
          for column in columns]

# 데이터의 총 기간은 몇 년인가?
span = max(years) - min(years) + 1

# 적절해보이는 스타일을 선택한다.
matplotlib.style.use("ggplot")

STEP = 5
# subplot에 각 데이터 프레임을 시각화한다.
for pos, (draw, style, column, frame) in enumerate(zip(
        (plt.contourf, plt.contour, plt.imshow), # (1)
        (plt.cm.autumn, plt.cm.cool, plt.cm.spring),
        columns, frames)):

    # 2개 행과 2개 열을 가진 subplot을 선택한다.
    plt.subplot(2, 2, pos + 1) # (2)

    # 데이터 프레임을 시각화한다.
    draw(frame[frame.columns[:span]], cmap=style, aspect="auto") # (3)

    # 플롯을 꾸민다.
    plt.colorbar() # (4)
    plt.title(column)
    plt.xlabel("Year")
    plt.xticks(range(0, span, STEP), frame.columns[:span:STEP])
    plt.yticks(range(0, frame.shape[0], STEP), frame.Postal[::STEP])
    plt.xticks(rotation=-17)

plt.tight_layout()
plt.savefig("pyplot-all.pdf")
#plt.show()

everybody data science book/DataScience_code/예제파일/pyplot-images.py


# -*- coding: utf-8 -*-

import re
from time import time
from flask import request
from flask import redirect
from flask import Blueprint
from flask import render_template

from db_driver.read_db import read_db
from db_driver.write_db import write_db

add_member_route = Blueprint('add_member_route', __name__)

@add_member_route.route("/add/member", methods=["POST", "GET"])
def add_member():
    db_data = read_db()
    if request.method == "GET":
        return render_template("add_member.html")
    else:
        member_data = {}
        if request.form["position"]:
            member_data["position"] = request.form["position"]
        if request.form["name"]:
            member_data["name"] = request.form["name"]
        if request.form["surname"]:
            member_data["surname"] = request.form["surname"]
        if request.form["phone_number"]:
            edit_phone_number = request.form["phone_number"]
            edit_phone_number.isdigit()
            if edit_phone_number.isdigit() and len(edit_phone_number) == 12:
                edit_phone_number = re.sub(r'(\d{2})(\d{3})(\d{3})(\d{2})(\d{2})',
                                           r'+\1(\2)\3-\4-\5', edit_phone_number)
                member_data["phone_number"] = edit_phone_number
            else:
                error_phone = "border: 1px solid #ff0000"
                return render_template("add_member.html", error_phone=error_phone)
        if "position" in member_data and "name" in member_data and "phone_number" in member_data:
            id_timestamp = ''.join(str(time()).split("."))
            db_data["company"][f"company_member_{str(id_timestamp)}"] = member_data
        else:
            edit_style = "-error"
            return render_template("add_member.html", edit_style=edit_style)
        write_db(db_data)
    return redirect("/")


import json
import os
from typing import Any, Dict, List, Union

from hyperopt import fmin, hp, tpe
from tpe_performance_experiments.constants import MAX_EVALS, N_SEEDS, RESULTS_DIR, TASK_NAMES
from tpe_performance_experiments.utils import ResultsManager

def get_config_space(
    search_space: Dict[str, List[Union[int, str, float]]]
) -> Dict[str, Any]:
    return {
        hp_name: hp.choice(hp_name, list(range(len(choices))))
        if isinstance(choices[0], str)
        else hp.quniform(hp_name, low=0, high=len(choices) - 1, q=1)
        for hp_name, choices in search_space.items()
    }

if __name__ == "__main__":
    os.makedirs(RESULTS_DIR, exist_ok=True)
    n_seeds = N_SEEDS
    data = {}
    for task_name in TASK_NAMES:
        data[task_name] = []
        for seed in range(n_seeds):
            res_manager = ResultsManager(task_name, seed)
            obj_func = res_manager.obj_func
            config_space = get_config_space(res_manager.search_space)
            best = fmin(
                fn=obj_func, space=config_space, algo=tpe.suggest, max_evals=MAX_EVALS
            )
            data[task_name].append(res_manager.loss)
            print(f"{task_name}, seed={seed}: {best}")

    with open(f"{RESULTS_DIR}/hyperopt.json", mode="w") as f:
        json.dump(data, f, indent=4)


from flask import request, jsonify
from flask_smorest import Blueprint

from server.extensions.database import handle_database_connection

searchBlueprint = Blueprint("Search", "search", description="Operations on search")

@searchBlueprint.route('/search/event_start_with', methods=['GET'])
@handle_database_connection
def get_event_start_with(cursor):
    """
    Get events starting with a specific string.
    """

    start_with = request.args.get("start_with")
    query = """
            SELECT * FROM event
            WHERE name LIKE %s;
            """
    cursor.execute(query, (start_with + '%',))
    results = cursor.fetchall()
    return jsonify(results)

@searchBlueprint.route('/search/events_active_selections_count', methods=['GET'])
@handle_database_connection
def get_events_active_selections_count(cursor):
    """
    Get events with a minimum number of active selections greater than or equal to the provided threshold.
    """
    start_with = request.args.get("threshold")
    query = """
            SELECT e.name, e.scheduled_start
            FROM event e
            JOIN (
                SELECT event, COUNT(*) as active_selections_count
                FROM selection
                WHERE active = TRUE
                GROUP BY event
                HAVING COUNT(*) >= %s
            ) s ON e.name = s.event;

            """

    cursor.execute(query, start_with)
    results = cursor.fetchall()
    return jsonify(results)

@searchBlueprint.route('/search/events_scheduled_in_timeframe', methods=['GET'])
@handle_database_connection
def get_events_scheduled_in_timeframe(cursor):
    """
    Get events scheduled to start in a specific timeframe for a specific timezone.
    """
    start_time = request.args.get("start_time")
    end_time = request.args.get("end_time")
    timezone = request.args.get("timezone")

    query = """
            SELECT * FROM event
            WHERE scheduled_start AT TIME ZONE 'UTC' AT TIME ZONE %s
            BETWEEN %s AND %s;
            """
    cursor.execute(query, (timezone, start_time, end_time))
    results = cursor.fetchall()
    return jsonify(results)


from kivy.core.window import Window
from kivymd.app import MDApp
from kivymd.uix.menu import MDDropdownMenu
from kivy.lang import Builder
from kivy.metrics import dp
from kivymd.uix.snackbar import Snackbar
from kivy.uix.screenmanager import Screen, ScreenManager
import math
from kivymd.uix.dialog import MDDialog
from kivymd.uix.button import MDFlatButton
from kivymd.uix.picker import MDThemePicker
from kivy.config import Config
Config.set('graphics', 'resizable', True)

class ScreenManager(ScreenManager):
    pass

class MainScreen(Screen):

    pass

class Screen1(Screen):
    def limpar(self):
        self.ids.rms.text = ''
        self.ids.impedancia.text = ''
        self.ids.result_total_volt.text = ''

    def calcular_voltagem(self):
        self.dialog = MDDialog(
            title = "Ops, Algo Errado!",
            text = 'Repita a Operação',
            buttons= [MDFlatButton(text='OK', on_release = self.liberar)]
        )
        try:
            vl_rms = int(self.ids.rms.text)
            vl_imp = int(self.ids.impedancia.text)
            total = vl_rms * vl_imp
            raiz = int(math.sqrt(total))
            self.ids.result_total_volt.text = f"Módulo manda {raiz} V"
        except:
            self.dialog.open()

    def liberar(self, obj):
        self.dialog.dismiss()

class Screen2(Screen):

    def limpar_2(self):
        self.ids.total_rms.text = ''
        self.ids.result_total_amp.text = ''

    def calcular_amp(self):
        self.dialog = MDDialog(
            title = "Ops, Algo Errado!",
            text = 'Repita a Operação',
            buttons= [MDFlatButton(text='OK', on_release = self.liberar)]
        )
        try:
            valor_bat = int(self.ids.total_rms.text)
            total = int(valor_bat / 20)
            self.ids.result_total_amp.text = f"Bateria Min. {total} Ampéres"
        except:
            self.dialog.open()

    def liberar(self, obj):
        self.dialog.dismiss()

class Screen3(Screen):
    def limpar_3(self):
        self.ids.rms.text = ''
        self.ids.impedancia.text = ''
        self.ids.result_total_volt.text = ''

    def calcular_voltagem(self):
        self.dialog = MDDialog(
            title = "Ops, Algo Errado!",
            text = 'Repita a Operação',
            buttons= [MDFlatButton(text='OK', on_release = self.liberar)]
        )
        try:
            vl_rms = int(self.ids.rms.text)
            vl_imp = int(self.ids.impedancia.text)
            total = vl_rms * vl_imp
            raiz = int(math.sqrt(total))
            self.ids.result_total_volt.text = f"Aplicar até {raiz} V"
        except:
            self.dialog.open()

    def liberar(self, obj):
        self.dialog.dismiss()

class Screen_ajuda(Screen):
    pass

class MyApp(MDApp):

    def build(self):
        self.title = "Calculadora de Potência"
        self.icon = 'icon.png'
        #Window.size = (1080, 2400)
        self.theme_cls.theme_style = "Dark"  # "Light"
        self.theme_cls.primary_palette = "Purple"  # "Green", "Red"

        menu_items = [
            {
                "viewclass": "OneLineListItem",
                "text": "Tema",
                "height": dp(56),
                "on_release": lambda x=f"Tema": self.show_theme_picker(x)
            },
            {
                "viewclass": "OneLineListItem",
                "text": "FAQ",
                "height": dp(56),
                "on_release": lambda x=f"FAQ": self.help_callback(x)
            },
            {
                "viewclass": "TwoLineListItem",
                "icon": 'weight-kilogram',
                "text": "Fechar",
                "height": dp(56),
                "on_release": lambda x=f"Fechar": self.menu_callback(x)
            }
        ]
        self.menu = MDDropdownMenu(
            items=menu_items,
            width_mult=3,
        )

        return Builder.load_file('main.kv')

    def callback(self, button):
        self.menu.caller = button
        self.menu.open()

    def menu_callback(self, text_item):
        self.menu.dismiss()
        Snackbar(text=text_item).open()
        self.stop()

    def help_callback(self, text_item):
        self.menu.dismiss()
        Snackbar(text=text_item).open()
        self.root.current = 'ajuda'

    def show_theme_picker(self, text_item):
        self.menu.dismiss()
        Snackbar(text=text_item).open()
        theme_dialog = MDThemePicker()
        theme_dialog.open()

if __name__ == "__main__":
    MyApp().run()


# -*- coding: utf-8 -*-
"""
    Create a report of box contents.
    Parameters:
        1. Input XML file or CSV file.
           XML  This is a Modes database
           CSV  Heading is required. The first column is the serial number
                and the second column is the location. Typically, this was
                produced by filtering the database by some prior criterion.
        2. Optional output CSV file. If omitted, output is to STDOUT.
"""
import argparse
from collections import defaultdict
import csv
import re
import sys
# noinspection PyPep8Naming
import xml.etree.ElementTree as ET

from utl.cfgutil import expand_idnum, read_include_dict
from utl.excel_cols import col2num
from utl.normalize import normalize_id, denormalize_id
from utl.normalize import sphinxify, if_not_sphinx
from utl.readers import object_reader

def pad_loc(loc):
    """
    If the location field matches the pattern "<Letter(s)><Number(s)>" like
    "S1" then pad the number part so that it sorts correctly. Also convert the
    letter part to upper case
    :param loc: location
    :return: padded location
    """
    m = re.match(r'(\D+)(\d+)', loc)
    if m:
        g1 = m.group(1).upper()
        g2 = int(m.group(2))
        return f'{g1}{g2:03}'
    else:
        return loc

def unpad_loc(loc):
    m = re.match(r'(\D+)(\d+)', loc)
    if m:
        g1 = m.group(1).upper()
        g2 = int(m.group(2))
        return f'{g1}{g2}'
    else:
        return loc

def one_xml_object(elt):
    num = elt.find('./ObjectIdentity/Number').text
    nnum = normalize_id(num)
    if includes:
        if nnum not in includes:
            return
    loc = elt.find('./ObjectLocation[@elementtype="current location"]/Location')
    if loc is not None and loc.text:
        location = pad_loc(loc.text)
    else:
        location = 'unknown'
    title = elt.find('./Identification/Title').text
    if not title:
        title = ''
    briefdes = elt.find('./Identification/BriefDescription').text
    if briefdes is None:
        briefdes = ''
    boxdict[location].append(nnum)
    titledict[nnum] = (title, briefdes)

def handle_csv():
    reader = csv.reader(open(_args.infile))
    header = next(reader)
    print('CSV file header:', header)
    for row in reader:
        loc = pad_loc(row[1])
        nnum = normalize_id(row[0])
        boxdict[loc].append(nnum)
        titledict[nnum] = ('', '')

def handle_xml():
    for _, elem in object_reader(_args.infile):
        one_xml_object(elem)
        elem.clear()

def writerow(row):
    if _args.outcsv:
        writer.writerow(row)
    else:
        print(' '.join(row), file=outfile)

def main():
    scanned = 'scanned'
    notscanned = ' ' * len(scanned)
    if _args.infile.endswith('.csv'):
        handle_csv()
    else:
        handle_xml()
    for box in sorted(boxdict.keys()):
        writerow([''])
        writerow([''])
        writerow(['Box', unpad_loc(box)])
        writerow(['--------------'])
        for nnum in sorted(boxdict[box]):
            # titledict[accn#] = (title, description)
            # print(titledict[nnum])
            comment = f'{titledict[nnum][0]} ({titledict[nnum][1]})'[:50]
            if image_set:
                writerow([f'{denormalize_id(nnum):15}',
                         scanned if nnum in image_set else notscanned, comment])
            else:
                writerow([f'{denormalize_id(nnum):15}', comment])

def getparser() -> argparse.ArgumentParser:
    """
    Called either by getargs() in this file or by Sphinx.
    :return: an argparse.ArgumentParser object
    """
    parser = argparse.ArgumentParser(description='''
    ''')
    parser.add_argument('infile', help=sphinxify('''
        Modes XML database file or CSV file.
        ''', called_from_sphinx))
    parser.add_argument('-c', '--outcsv', action='store_true', help='''
        Output is formatted as a CSV file. If not selected, a text report is
        written.''')
    parser.add_argument('-i', '--imglist', help='''
        Text file with accession numbers of pictures already scanned.''')
    parser.add_argument('--include', required=False, help='''
        A CSV file specifying the accession numbers of objects to be processed.
        If omitted, all records will be processed. In either case, objects will
        be output based on configuration statements. ''')
    parser.add_argument('--include_column', required=False,
                        default='0', type=str, help='''
        The column number containing the accession number in the file
        specified by the --include option.
        The column can be a number or a spreadsheet-style letter.''' +
                                                    if_not_sphinx(f''' The default is 0, the first column.''',
                                                                  called_from_sphinx))

    parser.add_argument('--include_skip', type=int, default=0, help='''
        The number of rows to skip at the front of the include file.''' +
                                                                    if_not_sphinx(f''' The default is 0.
        ''', called_from_sphinx))
    parser.add_argument('-o', '--outfile', type=argparse.FileType('w'),
                        default='-', help='''
        The output CSV file.''')
    parser.add_argument('-s', '--short', action='store_true', help='''
        Only process one object. For debugging.''')
    parser.add_argument('-v', '--verbose', type=int, default=1, help='''
        Set the verbosity. The default is 1 which prints summary
        information.''')
    return parser

def getargs(argv):
    parser = getparser()
    args = parser.parse_args(args=argv[1:])
    args.include_column = col2num(args.include_column)
    return args

def get_imgset() -> set[str]:
    """

    :return: A set of normalized accession numbers from the imglist file.
    """
    imgset = set()
    if not _args.imglist:
        return imgset
    imgfile = open(_args.imglist)
    for row in imgfile:
        expanded = [normalize_id(obj) for obj in expand_idnum(row)]
        imgset.update(expanded)
    return imgset

called_from_sphinx = True

if __name__ == '__main__':
    assert sys.version_info >= (3, 9)
    called_from_sphinx = False
    if len(sys.argv) == 1:
        sys.argv.append('-h')
    _args = getargs(sys.argv)
    outfile = _args.outfile
    if _args.outcsv:
        writer = csv.writer(outfile)
    boxdict = defaultdict(list)
    titledict = dict()
    image_set = get_imgset()
    includes = read_include_dict(_args.include, _args.include_column,
                                 _args.include_skip, allow_blanks=True)
    main()


#!/usr/bin/env python2

import logging
import os
import re
import sys

from lewas.models import Instrument, Measurement
from lewas.parsers import split_parser, field_parser, ParseError
from lewas.leapi import leapiStore
from lewas.types import itemgetter_float as ig_float
from lewas.functools import compose
from lewas.ittools import compact, flatten

logger = logging.getLogger(__name__)

# weather station format is particularly annoying because the symbols -> unit
# depends on the data type, e.g. 'M' means 'mm/h' if data is Ri (rain
# intensity), but 'hits/cm2' for Hc (hail accumulation)
weather_station_metrics = { 'Pa': ( 'air', 'pressure', {'H': 'hPa', 'I': 'inHg', 'P': 'Pascal', 'B': 'Bar', 'M': 'mmHg'} ),
                            'Rc': ( 'rain', 'accumulation', {'M': 'mm', 'I': 'in'} ),
                            'Rd': ( 'rain', 'duration', 's' ),
                            'Ri': ( 'rain', 'intensity', {'M': 'mm/h', 'I': 'in/h'} ),
                            'Ta': ('air', 'temperature', {'F': 'F', 'C': 'C'}),
                            'Hc': ( 'hail', 'accumulation', {'M': 'hits/cm2', 'I': 'hits/in2'} ),
                            'Hd': ( 'hail', 'duration', 's' ),
                            'Hi': ( 'hail', 'intensity', {'M': 'hits/cm2h', 'I': 'hits/in2h'}),
                            'Sm': ('wind', 'speed', 'm/s'),
                            'Vs': ('battery', 'voltage', 'V'),
                            'Dm': ('wind', 'direction', 'degrees'),
                            'Ua': ('air', 'humidity', '%RH')
}

def metric_getter(idx):
    def inner(data):
        try:
            return tuple(weather_station_metrics[data[idx]][0:2])
        except KeyError as e:
            # raise ParseError('Unknown metric key: {}'.format(data[idx]))
            return None # because we just want to ignore metrics we don't account for
        except IndexError as e:
            raise ParseError('Invalid index. Check that {} is a valid index and [0:2] is a valid slide into {}'.format(idx,data))
    return inner

def unit_getter(midx, idx):
    """For the data designation at midx, and unit symbol at 'idx',
    e.g. [ 'Ri', '0.05', 'I' ], set midx=0 to select 'Ri' and idx=2 to select 'I'
    """
    def inner(data):
        logger.log(logging.DEBUG, 'unit_getter({},{})({})'.format(midx,idx,data))
        try:
            unit_dict = weather_station_metrics[data[midx]][2]
        except KeyError:
            return None # ignore metrics we don't account for
        if hasattr(unit_dict, 'items'):
            # could be a dict, in which case it would have an 'items' attribute
            logger.log(logging.DEBUG, 'getting key {} in unit_dict: {}'.format(data[idx], unit_dict))
            return unit_dict[data[idx]]
        # if not, it should be a string, just return it
        return unit_dict
    return inner

def map_parser(fn):
    #parser = lambda d: map(fn, d)
    def inner(data):
        logger.log(logging.DEBUG, 'map_parser({})({})'.format(fn,data))
        result =  map(fn, data)
        logger.log(logging.DEBUG, 'result: {}'.format(result))
        return result
    return inner

# 0R5,Th=25.4242C,Vh=12.4242N,Vs=15.4242V
parse_match = re.compile(r'^0R[1-3],(.*)$')
# Th=25.4242C,Vh=12.4242N,Vs=15.4242V
parsers = [split_parser(',')]
# [ 'Th=25.4242C', 'Vh=12.4242N', 'Vs=15.4242V' ]
value_sep = re.compile(r'(?:=|([a-zA-Z/]+))') # split on a single , or = and strings of one or more letters. group the latter.
# [ '', 'Th', '', None, '25.4242', 'C',
parsers.append(map_parser(split_parser(value_sep)))
# [ [ 'Th', '25.4242', 'C' ], [ 'Vh', '12.4242', 'N' ], [ 'Vs', '15.4242', 'V' ] ]
fields = [ (ig_float(1), metric_getter(0), unit_getter(0,2)) ]
parsers.append(map_parser(field_parser(fields, container=False)))
# [ [measurement], [measurement], [measurement] ]
parsers.append(flatten)
parsers.reverse()

def parser(args, **kwargs):
    return { parse_match: compose(*parsers) }

class WeatherStation(Instrument):
    __name__ = 'weather_station'
    ## let's just use the output of the sensor to determine metrics: one parser to rule them all.
    parsers = parser([])

    def start(self):
        pass

    ## Custom methods.  Anything beyond what is handled by the automatic API
    ## Diagnostic methods

    def check(self, metric):
        codes = { 'wind': '0WU',
          'PTH': '0TU',
          'rain': '0RU',
          'self': '0SU'
        }

        self.cmd_output(codes[metric]+'\n\r')

    def reset(self, code):
        codes = { 'rain': '0XZRU',
                  'intensity': '0XZRI'
              }
        output = self.cmd_output(codes[code]+'\n\r')

if __name__ == "__main__":
    logging.basicConfig(level=getattr(logging, os.environ.get('LOGGING_LEVEL','WARN')))

    import lewas.cli as cli
    args = cli.parse(name='weather_station')
    config = args.config
    weather_station = Instrument(args.reader, parser=args.parser, name=args.instrument_name, site=config['site'])
    kwargs = {}
    weather_station.run(args.datastore, **kwargs)


# Import library
import pandas as pd
import numpy as np

# Import library for modeling
from sklearn.model_selection import cross_validate
from sklearn.linear_model import LogisticRegression

# Load configuration
import utils as utils

CONFIG_DATA = utils.load_config()

# The forward selection procedure's function
def forward(X, y, predictors, scoring='roc_auc', cv=5):
    """Function for carrying out the forward selection process"""

    # Specify the number of all predictors and the sample size.
    n_samples, n_predictors = X.shape

    # Describe the entire predictor list.
    col_list = np.arange(n_predictors)

    # For every k, define the remaining predictors.
    remaining_predictors = [p for p in col_list if p not in predictors]

    # Set the CV Score and predictors' initial values.
    pred_list = []
    score_list = []

    # Every possible pairing of the remaining predictors should be cross-validated.
    for p in remaining_predictors:
        combi = predictors + [p]

        # Combine extract predictors
        X_ = X[:, combi]
        y_ = y

        # Define the estimator
        model = LogisticRegression(penalty = 'l2',
                                   class_weight = 'balanced')

        # Cross-validate the model's recall scores
        cv_results = cross_validate(estimator = model,
                                    X = X_,
                                    y = y_,
                                    scoring = scoring,
                                    cv = cv)

        # Determine the typical CV/recall score.
        score_ = np.mean(cv_results['test_score'])

        # Add the combination of predictors and their CV score to the list.
        pred_list.append(list(combi))
        score_list.append(score_)

    # Total the outcomes.
    models = pd.DataFrame({"Predictors": pred_list,
                           "CV Score": score_list})

    # Select the best model.
    best_model = models.loc[models['CV Score'].argmax()]

    return models, best_model

# The ability to carry out forward selection across all attributes
def run_forward():
    """Function to carry out forward selection based on every attribute"""

    cv = CONFIG_DATA['num_of_cv']
    scoring = CONFIG_DATA['scoring']

    X_train_woe_path = CONFIG_DATA['X_train_woe_path']
    X_train_woe = utils.load_pickle(X_train_woe_path)
    X_train = X_train_woe.to_numpy()

    y_train_path = CONFIG_DATA['train_path'][1]
    y_train = utils.load_pickle(y_train_path)
    y_train = y_train.to_numpy()

    # First, fit the null model
    # Define the null model's predictor.
    predictor = []

    # In the null model, every predictor has a value of zero.
    X_null = np.zeros((X_train.shape[0], 1))

    # Define the estimator
    model = LogisticRegression(penalty = 'l2',
                               class_weight = 'balanced')

    # Cross validate
    cv_results = cross_validate(estimator = model,
                                X = X_null,
                                y = y_train,
                                cv = cv,
                                scoring = scoring)

    # Determine the typical CV score.
    score_ = np.mean(cv_results['test_score'])

    # Make a table with each k predictor's best model.
    # Add the null model results.
    forward_models = pd.DataFrame({"Predictors": [predictor],
                                   "CV Score": [score_]})

    # Proceed with forward selection for each and every predictor.
    # Define the predictor list.
    predictors = []
    n_predictors = X_train.shape[1]

    # Apply the forward selection method to the predictors k=1,...,n_predictors.
    for k in range(n_predictors):
        _, best_model = forward(X = X_train,
                                y = y_train,
                                predictors = predictors,
                                scoring = scoring,
                                cv = cv)

        # List the optimal model for each of the k predictors.
        forward_models.loc[k+1] = best_model
        predictors = best_model['Predictors']

    # Find the best CV score
    best_idx = forward_models['CV Score'].argmax()
    best_cv_score = forward_models['CV Score'].loc[best_idx]
    best_predictors = forward_models['Predictors'].loc[best_idx]

    # Print the summary
    print('===================================================')
    print('Best index            :', best_idx)
    print('Best CV Score         :', best_cv_score)
    print('Best predictors (idx) :', best_predictors)
    print('Best predictors       :')
    print(X_train_woe.columns[best_predictors].tolist())
    print('===================================================')

    print(forward_models)
    print('===================================================')

    forward_models_path = CONFIG_DATA['forward_models_path']
    utils.dump_pickle(forward_models, forward_models_path)

    best_predictors_path = CONFIG_DATA['best_predictors_path']
    utils.dump_pickle(best_predictors, best_predictors_path)

    return forward_models, best_predictors

# Function to fit optimal model across the entire X_train
def best_model_fitting(best_predictors):
    """Function to fit optimal model across the entire X_train"""

    X_train_path = CONFIG_DATA['X_train_woe_path']
    X_train_woe = utils.load_pickle(X_train_path)
    X_train = X_train_woe.to_numpy()

    y_train_path = CONFIG_DATA['train_path'][1]
    y_train = utils.load_pickle(y_train_path)
    y_train = y_train.to_numpy()

    if best_predictors is None:
        best_predictors_path = CONFIG_DATA['best_predictors_path']
        best_predictors = utils.load_pickle(best_predictors_path)
        print(f"Best predictors index   :", best_predictors)
    else:
        print(f"[Adjusted] best predictors index   :", best_predictors)

    # Use the best predictors to define X.
    X_train_best = X_train[:, best_predictors]

    # Fit best model
    best_model = LogisticRegression(penalty = 'l2',
                                    class_weight = 'balanced')
    best_model.fit(X_train_best, y_train)

    print(best_model)

    # Extract parameter estimates from the optimal model.
    best_model_intercept = pd.DataFrame({'Characteristic': 'Intercept',
                                         'Estimate': best_model.intercept_})

    best_model_params = X_train_woe.columns[best_predictors].tolist()

    best_model_coefs = pd.DataFrame({'Characteristic': best_model_params,
                                     'Estimate': np.reshape(best_model.coef_,
                                                            len(best_predictors))})

    best_model_summary = pd.concat((best_model_intercept, best_model_coefs),
                                   axis = 0,
                                   ignore_index = True)

    print('===================================================')
    print(best_model_summary)

    best_model_path = CONFIG_DATA['best_model_path']
    utils.dump_pickle(best_model, best_model_path)

    best_model_summary_path = CONFIG_DATA['best_model_summary_path']
    utils.dump_pickle(best_model_summary, best_model_summary_path)

    return best_model, best_model_summary

# Execute the functions
if __name__ == "__main__":

    # 1. Load config file
    CONFIG_DATA = utils.load_config()

    # 2. Perform model selection on all characteristics
    run_forward()

    # 3. Fit the best model (with adjustment)
    best_model_fitting(best_predictors = [0,1,2,3,4,5,6,7,8,9,10,11])