import spacy
import pandas as pd
import time
from tqdm import trange, tqdm
import pickle
import numpy as np
import re
import os, datetime
from prefixspan import PrefixSpan
from synonyms import synonyms_noun, synonyms_verb, synonyms_adj_adv
import random
from sklearn.metrics import classification_report
from sklearn.model_selection import train_test_split
from pathlib import Path
from sklearn.metrics import classification_report
from operator import itemgetter

nlp = spacy.load("en_core_web_lg")

label_dict = {'resource': 1,
              'security-tradeoff': 2,
              'reliability-tradeoff': 3,
              'limited-side-effect': 4,
              'workload-specific': 5,
              'function-tradeoff': 6,
              'others': 7}

label_dict_r = [''] + list(label_dict.keys())
target_dep = ['nsubj', 'nsubjpass', 'dobj', 'obj', 'pobj', 'ROOT', 'iobj']
reserved_pos = ['ADJ', 'ADV', 'NOUN', 'NUM', 'VERB', 'ADP', 'PROPN', 'SCONJ', 'CCONJ']  # , 'PART'

def name2tokens(name):
    """
    :param name: name of configuration parameter, string
    :return: tokenized name
    e.g.
    input: ConfigurationName
    return: Configuration Name
    """
    patterns = re.compile(r'[-._]|(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])')
    res = patterns.split(name)
    res = ' '.join(res)
    # res = ' '.join([r.lower() for r in res])
    return res

def add_token2dict(pos, lemma, token_dict, uid):
    """
    :param pos: pos (part of speech) of token, if we use svo (subject, verb and object) to mine patterns,
    this should be dep_ (dependency) of the token
    :param lemma: lemma of token
    :param token_dict: return index of (pos, lemma)
    :param uid: a global counter of identical (pos, lemma)
    :return: tid: the index of (pos, lemma) of token_dict; new_uid: updated uid
    """
    if (pos, lemma) in token_dict:
        tid = token_dict[(pos, lemma)]
        new_uid = uid
    else:
        token_dict[(pos, lemma)] = uid
        tid = token_dict[(pos, lemma)]
        new_uid = uid + 1
    return tid, new_uid

def replace_syn(token):
    if token.pos_ in ['NOUN', 'PROPN']:
        for word in synonyms_noun:
            if token.lemma_ in synonyms_noun[word]:
                return word
    if token.pos_ == 'VERB':
        for word in synonyms_verb:
            if token.lemma_ in synonyms_verb[word]:
                return word
    if token.pos_ == 'ADV' or token.pos_ == 'ADJ':
        for word in synonyms_adj_adv:
            if token.lemma_ in synonyms_adj_adv[word]:
                return word
    if token.pos_ == 'NUM':
        return 'NUM'
    return token.lemma_

def tokens2id(data, output_folder, feature='description'):
    """
    Convert tokens in descriptions into numbers
    :param data: data read from datafile, in DataFrame type
    :param output_folder: folder to store the token dict
    :param feature: column in data selected to mine patterns
    :return: seqs
    """
    if feature == 'name':
        description = data[feature].apply(name2tokens)
    else:
        description = data['description']

    labels = list(data['label'])
    uid = 1  # 0 for unknown word
    token_dict = {}
    for label in label_dict:
        print(label, uid)
        token_dict[label] = label_dict[label]
        uid += 1
    seqs = []
    description = list(description)
    pbar = trange(len(description))
    for i in pbar:
        pbar.set_description('Processed Descriptions: %d' % i)
        seq = []
        doc = nlp(description[i].strip().lower())
        for token in doc:
            if feature == 'svo':
                # print('POS ：', token.pos_)
                if token.dep_ in target_dep:
                    tid, uid = add_token2dict(token.dep_, token.lemma_, token_dict=token_dict, uid=uid)
                    # print(tid, uid)
                    seq.append(tid)
            elif feature == 'description':
                if token.pos_ in reserved_pos:
                    lemma = replace_syn(token)
                    tid, uid = add_token2dict(token.pos_, lemma, token_dict=token_dict, uid=uid)
                    seq.append(tid)
                # elif token.pos_ == 'PUNCT' and token.lemma_ in ['.', '?']:
                #     lemma = '.'
                #     tid, uid = add_token2dict(token.pos_, lemma, token_dict=token_dict, uid=uid)
                #     seq.append(tid)
            else:
                tid, uid = add_token2dict(token.lemma_, token.lemma_, token_dict, uid)
                seq.append(tid)
        seq.append(token_dict[labels[i]])
        seqs.append(seq)
    print(uid)
    # from csv to txt, sentence to numbered sequences, e.g. a b c ==> 1 2 3
    with open('%s/seqs.txt' % output_folder, 'w') as f:
        for seq in seqs:
            f.write(' '.join([str(s) for s in seq]) + '\n')
    with open('%s/id2token.pkl' % output_folder, 'wb') as f:
        pickle.dump(token_dict, f)
    with open('%s/words.txt' % output_folder, 'w') as f:
        f.write('\n'.join([str(w) for w in list(token_dict.keys())]))
    return seqs

def is_subsequence(s, t):
    index = 0
    for value in s:
        try:
            index = t.index(value, index)
        except ValueError:
            return False
        index += 1
    return True

def pattern_in_seq(pattern, config):
    for p in pattern[0]:
        if p not in config:
            return False
    return True

def label_config(patterns, config, threshold=0.85):
    """
    use patterns mined to label config
    :param patterns: patterns mined from labeled data (configs),
    a list of patterns in format (seqs, label, support, confidence)
    :param config: config to be labeled, a list of numbers like [1,2,3,4]
    :param threshold: above which the pattern is thought to be valuable
    :return: predicted label (number) and the number of matched patterns
    """
    # prob = np.zeros(len(label_dict_r))
    count = 0
    matched_pattern = [[], 0, 0, 0]
    for pattern in patterns:
        # pattern = (pattern, pred, support, confidence)
        if is_subsequence(pattern[0], config) \
                and pattern[3] >= threshold:
            count += 1
            if (pattern[3] > matched_pattern[3]) or \
                    (pattern[3] == matched_pattern[3] and pattern[2] > matched_pattern[2]):
                matched_pattern = pattern
    return matched_pattern[1], count, matched_pattern

def label_config_2(patterns, config):
    """
    use patterns mined to label config
    :param patterns: patterns mined from labeled data (configs),
    a list of patterns in format (seqs, label, support, confidence)
    :param config: config to be labeled, a list of numbers like [1,2,3,4]
    :return: predicted label (number), the number of matched patterns, weight
    """
    count = 0
    weighted_votes = [[0], [0], [0], [0], [0], [0], [0]]
    weighted_voting_classifier = [0, 0, 0, 0, 0, 0, 0]
    for pattern in patterns:
        # pattern = (pattern, pred_label, support, confidence)
        if is_subsequence(pattern[0], config):
            count += 1
            weighted_votes[pattern[1]-1].append(((pattern[2]+30.0)/30.0) * pattern[3])  # black magic
            weighted_voting_classifier[pattern[1]-1] += ((pattern[2]+30.0)/30.0) * pattern[3]  # black magic
    weighted_voting_classifier = [max(weighted_votes[0]), max(weighted_votes[1]),
                                  max(weighted_votes[2]), max(weighted_votes[3]),
                                  max(weighted_votes[4]), max(weighted_votes[5]),
                                  max(weighted_votes[6])]
    if np.sort(weighted_voting_classifier)[-2] * 1.0 < max(weighted_voting_classifier):  # another black magic
        label_pred = np.argmax(weighted_voting_classifier) + 1
        return label_pred, count, weighted_voting_classifier
    else:
        return -1, 0, [0, 0, 0, 0, 0, 0, 0]

def filter_cars(pattern_pkl, out, threshold):
    """
    clean the mined patterns, filter out those shorter ones with lower confidence (or support)
    :param pattern_pkl: file contains the mined patterns, in .pkl format
    :param out: file to store the cleaned patterns, in .pkl format
    :param threshold: pattern whose confidence is lower than threshold is filtered out
    :return:
    """
    patterns = pickle.load(open(pattern_pkl, 'rb'))
    label_count = [0, 0, 0, 0, 0, 0, 0]
    threshold_for_minor = 0.85
    threshold_for_major = 0.85

    with open('%s-log.txt' % pattern_pkl[:-4], 'a') as f:
        f.write('filter start at: %s\n' % time.asctime(time.localtime(time.time())))
    print('Filtering...')
    # Sorted by: x[3](confidence), if equal,
    #        by: x[2](support);
    # patterns = sorted(patterns, key=lambda x: (x[3], x[2], len(x[0])), reverse=True)
    res = []
    for i, p in enumerate(patterns):
        if (p[1] in [1, 2, 3, 4, 5, 6] and p[3] > threshold_for_minor) or\
                (p[1] == 7 and p[3] > threshold_for_major):
            res.append(p)
            label_count[p[1]-1] += 1

    print('Filtered from %d to %d, distribution: %s. With confidence_threshold=(%.2f, %.2f)' % (len(patterns),
                                                                                len(res),
                                                                                label_count,
                                                                                threshold_for_minor,
                                                                                threshold_for_major))
    pickle.dump(res, open(out, 'wb'))

    with open('%s-log.txt' % pattern_pkl[:-4], 'a') as f:
        f.write('filter end at: %s\n' % time.asctime(time.localtime(time.time())))
        f.write('%d patterns got in total.\n' % len(res))

    return res

def update_patterns(patterns, seqs):
    """
    update the confidence of mined patterns with precision
    :param patterns: patterns mined, a list of patterns
    :param configs_file: configuration sequences, a list of numbers
    :param labels_file: file in .npy format that contains the labels of labeled data
    :return: updated patterns
    """
    configs = [seq[:-1] for seq in seqs]
    labels = [seq[-1] for seq in seqs]
    res = []
    # patterns = sorted(patterns, key=lambda x: x[1])
    pbar = tqdm(patterns)
    for pattern in pbar:
        pbar.set_description('Updating patterns')
        support, conf = 0, 0
        for i in range(len(configs)):
            if is_subsequence(pattern[0], configs[i]):
                support += 1  # number of configs that contain this pattern
                if pattern[1] == labels[i]:
                    conf += 1  # number of configs that have same labels with this pattern
        pattern[2] = conf
        pattern[3] = conf / support if support else 0
        if pattern[3] > 0:  # discard patterns with 0 conf or 0 support
            # print(pattern)
            res.append(pattern)
    return res

def text2configs(file, token_dict, feature='svo'):
    """
    Convert the configuration parameters into sequences
    :param file: file contains all the configuration data in .csv
    :param token_dict: dict of tokens obtained by the data to be mined, type: dict
    :param feature: typically one of ['name', 'description', 'svo']
    :return: sequenced configuration parameters
    """
    data = pd.read_csv(file)
    if feature == 'name':
        texts = data[feature]
    else:
        texts = data['description']
    configs = []
    for text in texts:
        config = []
        doc = nlp(text)
        for token in doc:
            if feature == 'svo':
                if token.dep_ in target_dep:
                    if (token.dep_, token.lemma_) in token_dict:
                        config.append(token_dict[(token.dep_, token.lemma_)])
                    else:
                        config.append(0)
            elif feature == 'description':
                if token.pos_ in reserved_pos:
                    lemma = replace_syn(token)
                    if (token.pos_, lemma) in token_dict:
                        config.append(token_dict[(token.pos_, lemma)])
                    else:
                        config.append(0)
            elif feature == 'name':
                if token.lemma_ in token_dict:
                    config.append(token_dict[token.lemma_])
                else:
                    config.append(0)
        configs.append(config)
    return configs

def patterns2file(patterns, out, token_dict):
    """
    Store patterns to file, in a way human can read
    :param patterns: mined patterns in number, type: list
    :param out: file to store patterns in text
    :param token_dict: dict of tokens, number -> text
    :return:
    """
    with open(out, 'w') as f:
        for pattern in patterns:
            f.write('{')
            for p in pattern[0]:
                f.write(str(token_dict[p]) + ',')
            f.write('}, %s, %d, %.3f\n' % (label_dict_r[pattern[1]], pattern[2], pattern[3]))

def label_software(configs, patterns_all, patterns_minority):
    """
    Predict labels for unlabeled configuration parameters
    :param configs: converted sequences of configuration descriptions
    :param patterns_all: all patterns mined from labeled data
    :param patterns_minority: patterns mined from labeled data in minority classes
    :return: predicted labels, a list
    """
    # print(configs)
    pred_labels = []
    matched_patterns = []
    pbar = tqdm(configs)

    ws = []

    # for every configuration description to be expanded, try to match with patterns mined
    for i, config in enumerate(pbar):

        pbar.set_description('Labeling')

        pred_label,   count,   w   = label_config_2(patterns_all,
                                                    config)
        pred_label_m, count_m, w_m = label_config_2(patterns_minority,
                                                    config)
        # prefer mined patterns especially for those minority classes to prevent from being in performance-unrelated.
        if pred_label != pred_label_m and pred_label == 7 and max(w_m) > max(w)*5.0 and count + count_m != 0:
            pred_label = pred_label_m
            count = count_m
            w = w_m

        if count > 0:
            pred_labels.append(label_dict_r[pred_label])
            #matched_patterns.append(p)
        elif count_m > 0: # count == 0
            pred_labels.append(label_dict_r[pred_label_m])
            w = w_m
            #matched_patterns.append(p_m)
        else:
            pred_labels.append('')
            #matched_patterns.append([[], 0, 0, 0])

        ws.append(w)

    # `pred_labels` are string list!!
    return pred_labels, ws#, matched_patterns

def evaluate(labels_true, labels_pred, label_names, result_filename):
    """
    the evaluation metrics for automatic data expansion
    :param result_filename: output file name
    :param labels_true:
    :param labels_pred:
    :param label_names:
    :return:
        precision = TP/(TP+FP)
        recall = TP/(TP+FN)
    """
    counts = {}
    for label in label_names:
        counts[label] = [0, 0, 0]  # TP, TP+FP, TP+FN
    counts['performance-related'] = [0, 0, 0]
    for i in range(len(labels_pred)):
        # ********* for each type ************
        p_label, t_label = labels_pred[i], labels_true[i]
        counts[t_label][2] += 1
        if p_label != '':
            counts[p_label][1] += 1
            if p_label == t_label:
                counts[p_label][0] += 1
        # ********* for each type ************

        # ********* for performance-related ************
        if t_label != 'others':
            counts['performance-related'][2] += 1
        if p_label != '':
            if p_label != 'others':
                counts['performance-related'][1] += 1
                if t_label != 'others':
                    counts['performance-related'][0] += 1
        # ********* for performance-related ************

    with open(result_filename, 'a') as f:
        f.write('%25s\t%10s\t%10s\n' % ('Type', 'Precision', 'Recall'))
        for label in label_names:
            precision = counts[label][0] / counts[label][1] if counts[label][1] > 0 else 0
            recall = counts[label][0] / counts[label][2] if counts[label][2] > 0 else 0
            f.write('%25s\t%10.4f\t%10.4f\n' % (label, precision, recall))

        # ******* p&r for performance-related **********
        label = 'performance-related'
        precision = counts[label][0] / counts[label][1] if counts[label][1] > 0 else 0
        recall = counts[label][0] / counts[label][2]
        f.write('%25s\t%10.4f\t%10.4f\n' % (label, precision, recall))
        # ******* p&r for performance-related **********

        # ******* micro p&r for side-effect **********
        tot = sum([counts[label][1] for label in label_names if label != 'others'])
        micro_precision_6_types = sum([counts[label][0] for label in label_names if label != 'others']) / tot \
            if tot > 0 else 0

        micro_recall_6_types = sum([counts[label][0] for label in label_names if label != 'others']) / \
                               sum([counts[label][2] for label in label_names if label != 'others'])
        f.write('%25s\t%10.4f\t%10.4f\n' % ('micro_6_types', micro_precision_6_types, micro_recall_6_types))
        # ******* micro p&r for side-effect **********

        # ******* micro p&r **********
        tot = sum([counts[label][1] for label in label_names])
        micro_precision = sum([counts[label][0] for label in label_names]) / tot if tot > 0 else 0
        micro_recall = sum([counts[label][0] for label in label_names]) / \
                       sum([counts[label][2] for label in label_names])
        f.write('%25s\t%10.4f\t%10.4f\n' % ('micro', micro_precision, micro_recall))
        # ******* micro p&r **********
        f.write('*' * 80 + '\n')
        return micro_precision, micro_recall

def prefix_mining(seqs, minsup, out_filename, folder, maxlen=10, confidence_threshold=0.8):

    print('PrefixSpan mining starts at %s with minsup=%d, maxlen=%d' % (time.asctime(time.localtime(time.time())), minsup, maxlen))

    ps = PrefixSpan(seqs)
    ps.maxlen = maxlen
    tmp = ps.frequent(minsup, generator=True, filter=lambda patt, match: 1 <= patt[-1] <= 7)

    # pattern in patterns:
    #   [0] pattern itself,
    #   [1] pattern's label,
    #   [2] pattern's support,
    #   [3] 0(reserved for pattern's confidence)
    patterns = []
    num_p = [0, 0, 0, 0, 0, 0, 0]
    for (fre, pattern) in tmp:
        patterns.append([pattern[:-1], pattern[-1], fre, 0])
        num_p[pattern[-1]-1] += 1

    print('PrefixSpan mining ends at %s, got patterns: %s' % (time.asctime(time.localtime(time.time())), num_p))

    # Calculate confidence for each pattern.
    updated_patterns = update_patterns(patterns, seqs)
    pickle.dump(updated_patterns, open('%s/raw-%s' % (folder, out_filename), 'wb'))

    # Filter out redundant patterns.
    patterns_reserved = filter_cars(pattern_pkl='%s/raw-%s' % (folder, out_filename),
                                    out='%s/%s' % (folder, out_filename),
                                    threshold=confidence_threshold)
    return patterns_reserved

if __name__ == '__main__':

    confidence_threshold = 0.85
    scale = 0.2

    date = '%02d%02d' % (datetime.datetime.now().month,
                         datetime.datetime.now().day)
    data_folder = '.'
    out_folder = 'expand_out_%s' % date
    # os.rmdir(out_folder)

    # label_names = ['others',
    #                'resource',
    #                'limited-side-effect',
    #                'workload-specific',
    #                'function-tradeoff',
    #                'security-tradeoff',
    #                'reliability-tradeoff'
    #                ]
    label_names = list(label_dict.keys())
    minority_labels = label_names[1:-1]

    configs_all_path = Path('%s/configs_all.npy' % out_folder)
    labels_all_path = Path('%s/labels_all.npy' % out_folder)
    token_dict_path = Path('%s/id2token.pkl' % out_folder)  # do not change.

    rest_data = pd.read_csv('%s/data_all.csv' % data_folder)
    counts = rest_data['software'].value_counts()
    number_to_study = int(len(rest_data) * scale)

    if configs_all_path.exists() and labels_all_path.exists() and token_dict_path.exists():
        seqs_DB    = np.load(configs_all_path, allow_pickle=True)
        labels     = np.load(labels_all_path, allow_pickle=True)
    else:
        os.makedirs(out_folder, exist_ok=True)
        seqs_DB = np.array(tokens2id(rest_data, output_folder=out_folder, feature='description'))  # saved: id2token.pkl
        labels = np.array(rest_data['label'])
        np.save(configs_all_path, seqs_DB)
        np.save(labels_all_path, labels)
    token_dict = pickle.load(open(token_dict_path, 'rb'))  # dict[(pos, lemma)] = id

    index = list(range(len(labels)))
    id_dict = dict(zip(token_dict.values(), token_dict.keys()))  # dict[id] = (pos, lemma)

    # ------------------ mining and evaluation ------------------
    for seed in range(101, 110):
        sw = 'all'
        print('Software category: %s' % sw)
        outfolder = '%s/RandSeed-%d/Software-%s' % (out_folder, seed, sw)
        os.makedirs(outfolder, exist_ok=True)

        ###  May have problem
        study_index, expand_index = train_test_split(index, train_size=scale, random_state=seed)
        # study_index = list(np.load('%s/1/study_index.npy' % outfolder, allow_pickle=True))
        # expand_index = list(np.load('%s/1/expand_index.npy' % outfolder, allow_pickle=True))

        study_data, expand_data = rest_data.iloc[study_index], rest_data.iloc[expand_index]
        study_data.to_csv('%s/to_study_Initial.csv' % outfolder, index=False)
        expand_data.to_csv('%s/to_expand_Initial.csv' % outfolder, index=False)

        epochs = 0

        # store data that SUCCESSFULLY to be expanded this epoch
        index_to_add = []
        # store data that FAILED to be expanded this epoch, to be expanded in next epoch
        new_expand_index = expand_index
        # use the predicted label for those in `new_expand_index`
        labels_to_change = []

        while True:

            current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
            print("[%s] Mining epoch: %s" % (current_time, epochs))
            patterns_folder = '%s/epoch-%s' % (outfolder, epochs)
            os.makedirs(patterns_folder, exist_ok=True)

            study_index = study_index + index_to_add
            expand_index = new_expand_index

            np.save('%s/study_index_ep%d.npy' % (patterns_folder, epochs), np.array(study_index))
            np.save('%s/expand_index_ep%d.npy' % (patterns_folder, epochs), np.array(expand_index))
            seq_study, seqs_expand = seqs_DB[study_index], seqs_DB[expand_index]

            for i in range(len(index_to_add)):
                seq_study[i-len(index_to_add)][-1] = labels_to_change[i]

            print('%s configs to study, %s configs to expand' % (len(seq_study), len(seqs_expand)))

            print('\n[===== Mining stage 1/2 =====]')
            patterns_all = prefix_mining(seq_study,
                                         minsup=3,
                                         out_filename='patterns_all.pkl',
                                         folder=patterns_folder,
                                         maxlen=5,
                                         confidence_threshold=confidence_threshold)
            patterns2file(patterns_all,
                          out='%s/patterns_in_text_all.txt' % patterns_folder,
                          token_dict=id_dict)

            print('\n[===== Mining stage 2/2 =====]')
            minority_index = [i for i in study_index if labels[i] in minority_labels]
            minority_seqs = seqs_DB[minority_index]
            patterns_minority = prefix_mining(minority_seqs,
                                              minsup=2,
                                              out_filename='patterns_minority.pkl',
                                              folder=patterns_folder,
                                              maxlen=6,
                                              confidence_threshold=confidence_threshold)
            patterns2file(patterns_minority,
                          out='%s/patterns_in_text_minority.txt' % patterns_folder,
                          token_dict=id_dict)

            print('\n[===== Labeling software =====]')
            pred_labels, ws = label_software(seqs_expand,
                                             patterns_all=patterns_all,
                                             patterns_minority=patterns_minority)

            # store data that SUCCESSFULLY to be expanded this epoch
            index_to_add = []
            # store data that FAILED to be expanded this epoch, to be expanded in next epoch
            new_expand_index = []
            # use the predicted label for those in `new_expand_index`
            labels_to_change = []

            yPred, yTrue = [], []
            label_count = [0, 0, 0, 0, 0, 0, 0]
            for i in range(len(expand_index)):
                if pred_labels[i] != '':    # match this time, expanded SUCCESSFULLY.
                    index_to_add.append(expand_index[i])
                    #labels_to_change.append(label_dict[pred_labels[i]])  # Then, mark those labels.
                    yPred.append(label_dict[pred_labels[i]])
                    label_count[label_dict[pred_labels[i]]-1] += 1
                else:                       # FAILED to match any pattern, to be expanded next epoch.
                    new_expand_index.append(expand_index[i])
            print('  [Preserved] Expand from %d to %d,\n              %d to be expanded.' % (len(seq_study),
                                                                                len(seq_study)+len(index_to_add),
                                                                                len(new_expand_index)))
            print('Numbers of those expanded (from label 1 to label 7): %s' % label_count)
            yTrue = [label_dict[a] for a in labels[index_to_add]]

            np.save('%s/successfully_expanded_at_ep%d(index-of-X).npy' % (patterns_folder, epochs), np.array(index_to_add))
            np.save('%s/successfully_expanded_at_ep%d(label-of-y).npy' % (patterns_folder, epochs), np.array(labels_to_change))
            np.save('%s/to_be_expanded_after_ep%d.npy' % (patterns_folder, epochs), np.array(new_expand_index))

            print('\n[===== Evaluating Results =====]')

            target_names = np.array(['resource', 'security-tradeoff', 'reliability-tradeoff',
                                     'limited-side-effect', 'workload-specific', 'function-tradeoff',
                                     'others'])

            print(classification_report(y_true=yTrue,
                                        y_pred=yPred,
                                        target_names=list(target_names)))

            if len(new_expand_index) < 2:
                break
            else:
                epochs += 1


import argparse
import json
import logging
from typing import List

import requests
from bs4 import BeautifulSoup

logger = logging.getLogger('Team Stats')
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
ch.setFormatter(formatter)
logger.addHandler(ch)

parser = argparse.ArgumentParser()
parser.add_argument(
    '-send',
    '--send',
    type=int,
    default=1,
    choices = [0, 1],
    help='1 send email, 0 not send email')
parser.add_argument(
    '-display',
    '--display',
    type=int,
    default=0,
    choices = [0, 1],
    help='1 display in terminal, 0 not display')
args_list = parser.parse_args()

def get_arguments(filename: str):
    with open(filename, 'r') as f:
            arguments = json.load(f)

    return arguments

def get_first_link(team: str):
    url = 'https://www.google.com/search'
    headers = {
        'Accept' : '*/*',
        'Accept-Language': 'en-US,en;q=0.5',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82',
    }
    parameters = {'q': team + " footystats"}
    content = requests.get(url, headers = headers, params = parameters).text
    soup = BeautifulSoup(content, 'html.parser')

    search = soup.find(id = 'search')
    first_link = search.find('a')
    url = first_link['href']

    if '/es/' in url:
        url = url.replace('/es/','/')
    return url

def create_all_disabled_config_file(all_stats_list: List[list]):
    stats_dict = {}
    for item in all_stats_list[1:]:
        stats_dict.update({item[0]:"Disabled"})

    with open("stats.json", "w") as f:
        json.dump(stats_dict, f)

# Implement setup logging() right here


import os
from typing import Dict, Optional, Union

import numpy as np
import tensorflow as tf

import merlin.io
from merlin.models.io import save_merlin_metadata
from merlin.models.tf.core import combinators
from merlin.models.tf.core.base import NoOp
from merlin.models.tf.core.prediction import TopKPrediction
from merlin.models.tf.inputs.base import InputBlockV2
from merlin.models.tf.inputs.embedding import CombinerType, EmbeddingTable
from merlin.models.tf.loader import Loader
from merlin.models.tf.models.base import BaseModel, get_output_schema
from merlin.models.tf.outputs.topk import TopKOutput
from merlin.models.tf.transforms.features import PrepareFeatures
from merlin.models.tf.utils import tf_utils
from merlin.models.tf.utils.batch_utils import TFModelEncode
from merlin.schema import ColumnSchema, Schema, Tags

@tf.keras.utils.register_keras_serializable(package="merlin.models")
class Encoder(tf.keras.Model):
    """Block that can be used for prediction and evaluation but not for training

    Parameters
    ----------
    inputs: Union[Schema, tf.keras.layers.Layer]
        The input block or schema.
        When a schema is provided, a default input block will be created.
    *blocks: tf.keras.layers.Layer
        The blocks to use for encoding.
    pre: Optional[tf.keras.layers.Layer]
        A block to use before the main blocks
    post: Optional[tf.keras.layers.Layer]
        A block to use after the main blocks
    prep_features: Optional[bool]
        Whether this block should prepare list and scalar features
        from the dataloader format. By default True.
    """

    def __init__(
        self,
        inputs: Union[Schema, tf.keras.layers.Layer],
        *blocks: tf.keras.layers.Layer,
        pre: Optional[tf.keras.layers.Layer] = None,
        post: Optional[tf.keras.layers.Layer] = None,
        prep_features: Optional[bool] = True,
        **kwargs,
    ):
        super().__init__(**kwargs)
        if isinstance(inputs, Schema):
            input_block = InputBlockV2(inputs)
            self._schema = inputs
        else:
            input_block = inputs
            if not hasattr(inputs, "schema"):
                raise ValueError("inputs must have a schema")
            self._schema = inputs.schema

        self.blocks = [input_block] + list(blocks) if blocks else [input_block]
        self.pre = pre
        self.post = post

        self.prep_features = prep_features

        self._prepare_features = PrepareFeatures(self.schema) if self.prep_features else NoOp()

    def encode(
        self,
        dataset: Union[merlin.io.Dataset, Loader],
        index: Union[str, ColumnSchema, Schema, Tags],
        batch_size: int,
        **kwargs,
    ) -> merlin.io.Dataset:
        """Encodes the given dataset and index.

        Parameters
        ----------
        dataset: Union[merlin.io.Dataset, merlin.models.tf.loader.Loader]
            The dataset to encode.
        index: Union[str, ColumnSchema, Schema, Tags]
            The index to use for encoding.
        batch_size: int
            The batch size for encoding.

        Returns
        -------
        merlin.io.Dataset
            The encoded dataset.
        """
        if isinstance(index, Schema):
            output_schema = index
        elif isinstance(index, ColumnSchema):
            output_schema = Schema([index])
        elif isinstance(index, str):
            output_schema = Schema([self.schema[index]])
        elif isinstance(index, Tags):
            output_schema = self.schema.select_by_tag(index)
        else:
            raise ValueError(f"Invalid index: {index}")

        return self.batch_predict(
            dataset,
            batch_size=batch_size,
            output_schema=output_schema,
            index=index,
            output_concat_func=np.concatenate,
            **kwargs,
        )

    def batch_predict(
        self,
        dataset: Union[merlin.io.Dataset, Loader],
        batch_size: int,
        output_schema: Optional[Schema] = None,
        index: Optional[Union[str, ColumnSchema, Schema, Tags]] = None,
        **kwargs,
    ) -> merlin.io.Dataset:
        """Batched prediction using Dask.

        Parameters
        ----------
        dataset: Union[merlin.io.Dataset, merlin.models.tf.loader.Loader]
            Dataset or Loader to predict on.
        batch_size: int
            Batch size to use for prediction.

        Returns
        -------
        merlin.io.Dataset
        """

        if index:
            if isinstance(index, ColumnSchema):
                index = Schema([index])
            elif isinstance(index, str):
                index = Schema([self.schema[index]])
            elif isinstance(index, Tags):
                index = self.schema.select_by_tag(index)
            elif not isinstance(index, Schema):
                raise ValueError(f"Invalid index: {index}")

            if len(index) != 1:
                raise ValueError("Only one column can be used as index")
            index = index.first.name

        dataset_schema = None
        if hasattr(dataset, "schema"):
            dataset_schema = dataset.schema
            data_output_schema = dataset_schema
            if isinstance(dataset, Loader):
                data_output_schema = dataset.output_schema
            if not set(self.schema.column_names).issubset(set(data_output_schema.column_names)):
                raise ValueError(
                    f"Model schema {self.schema.column_names} does not match dataset schema"
                    + f" {data_output_schema.column_names}"
                )

        loader_transforms = None
        if isinstance(dataset, Loader):
            loader_transforms = dataset.transforms
            batch_size = dataset.batch_size
            dataset = dataset.dataset

        # Check if merlin-dataset is passed
        if hasattr(dataset, "to_ddf"):
            dataset = dataset.to_ddf()

        model_encode = TFModelEncode(
            self,
            batch_size=batch_size,
            loader_transforms=loader_transforms,
            schema=dataset_schema,
            **kwargs,
        )

        encode_kwargs = {}
        if output_schema:
            encode_kwargs["filter_input_columns"] = output_schema.column_names

        # Processing a sample of the dataset with the model encoder
        # to get the output dataframe dtypes
        sample_output = model_encode(dataset.head(), **encode_kwargs)
        output_dtypes = sample_output.dtypes.to_dict()

        predictions = dataset.map_partitions(model_encode, meta=output_dtypes, **encode_kwargs)
        if index:
            predictions = predictions.set_index(index)

        return merlin.io.Dataset(predictions)

    def call(self, inputs, *, targets=None, training=False, testing=False, **kwargs):
        """Calls the model on new inputs and returns the outputs as tensors.

        Parameters
        ----------
        inputs : tensor-like or dict/tuple of tensors.
            Tensors or dict/tuple of tensors representing the input batch.
        targets : tensor-like, optional
            Tensors representing the target data.
        training : bool, optional
            Whether the model is in training mode.
        testing : bool, optional
            Whether the model is in testing mode.

        Returns
        -------
        A tensor or dict of tensors corresponding to the result of calling the layer.
        """
        inputs = self._prepare_features(inputs, targets=targets)
        if isinstance(inputs, tuple):
            inputs, targets = inputs
        return combinators.call_sequentially(
            list(self.to_call),
            inputs=inputs,
            features=inputs,
            targets=targets,
            training=training,
            testing=testing,
            **kwargs,
        )

    def __call__(self, inputs, **kwargs):
        """Overrides the default __call__ method to remove "features" from inputs.

        Parameters
        ----------
        inputs : tensor-like or dict/tuple of tensors.
            Tensors or dict/tuple of tensors representing the input batch.

        Returns
        -------
        A tensor or dict of tensors corresponding to the result of calling the layer.
        """
        # We remove features here since we don't expect them at inference time
        # Inside the `call` method, we will add them back by assuming inputs=features
        if "features" in kwargs:
            kwargs.pop("features")

        return super().__call__(inputs, **kwargs)

    def build(self, input_shape):
        """Creates the variables of the layer.

        Parameters
        ----------
        input_shape: Tuple[int]
            The shape of the input data.
        """
        self._prepare_features.build(input_shape)
        input_shape = self._prepare_features.compute_output_shape(input_shape)

        combinators.build_sequentially(self, list(self.to_call), input_shape=input_shape)
        if not hasattr(self.build, "_is_default"):
            self._build_input_shape = input_shape

    def compute_output_shape(self, input_shape):
        """Computes the output shape of the layer.

        Parameters
        ----------
        input_shape: Tuple[int]
            The shape of the input data.

        Returns
        -------
        Tuple[int]
            The output shape of the layer.
        """
        input_shape = self._prepare_features.compute_output_shape(input_shape)
        return combinators.compute_output_shape_sequentially(list(self.to_call), input_shape)

    def train_step(self, data):
        """Performs a training step.

        Train step method is not implemented and Raises an error as the
        Encoder block is not meant to be trained by itself and can only be
        trained as part of a model.
        """
        raise NotImplementedError(
            "This block is not meant to be trained by itself. ",
            "It can only be trained as part of a model.",
        )

    def fit(self, *args, **kwargs):
        """Fits the model.

        Fit method is not implemented and Raises an error as the Encoder block
        is not meant to be trained by itself and can only be trained as part
        of a model.
        """
        raise NotImplementedError(
            "This block is not meant to be trained by itself. ",
            "It can only be trained as part of a model.",
        )

    def save(
        self,
        export_path: Union[str, os.PathLike],
        include_optimizer=True,
        save_traces=True,
    ) -> None:
        """Saves the model to export_path as a Tensorflow Saved Model.
        Along with merlin model metadata.

        Parameters
        ----------
        export_path : Union[str, os.PathLike]
            Path where model will be saved to
        include_optimizer : bool, optional
            If False, do not save the optimizer state, by default True
        save_traces : bool, optional
            When enabled, will store the function traces for each layer. This
            can be disabled, so that only the configs of each layer are
            stored, by default True
        """
        super().save(
            export_path,
            include_optimizer=include_optimizer,
            save_traces=save_traces,
            save_format="tf",
        )
        input_schema = self.schema
        output_schema = get_output_schema(export_path)
        save_merlin_metadata(export_path, input_schema, output_schema)

    @property
    def to_call(self):
        """Provides the list of blocks to be called during the execution of the model."""
        if self.pre:
            yield self.pre

        for block in self.blocks:
            yield block

        if self.post:
            yield self.post

    @property
    def has_schema(self) -> bool:
        """Returns True as this class does contain a schema."""
        return True

    @property
    def schema(self) -> Schema:
        """Returns the schema of the model."""
        return self._schema

    @property
    def first(self):
        """Returns the first block of the model."""
        return self.blocks[0]

    @property
    def last(self):
        """Returns the last block of the model."""
        return self.blocks[-1]

    @classmethod
    def from_config(cls, config, custom_objects=None):
        """Creates a new instance of the class by deserializing.

        Parameters
        ----------
        config: dict
            A dictionary, typically the output of get_config.
        custom_objects: dict, optional
            A dictionary mapping the names of layers to the corresponding
            functions and classes.

        Returns
        -------
        A new instance of Encoder.
        """
        pre = config.pop("pre", None)
        post = config.pop("post", None)
        layers = [
            tf.keras.layers.deserialize(conf, custom_objects=custom_objects)
            for conf in config.values()
        ]

        if pre is not None:
            pre = tf.keras.layers.deserialize(pre, custom_objects=custom_objects)

        if post is not None:
            post = tf.keras.layers.deserialize(post, custom_objects=custom_objects)

        output = Encoder(*layers, pre=pre, post=post)
        output.__class__ = cls

        return output

    def get_config(self):
        """Returns the configuration of the model as a dictionary.

        Returns
        -------
        dict
            The configuration of the model.
        """
        config = tf_utils.maybe_serialize_keras_objects(self, {}, ["pre", "post"])
        for i, layer in enumerate(self.blocks):
            config[i] = tf.keras.utils.serialize_keras_object(layer)

        return config

@tf.keras.utils.register_keras_serializable(package="merlin.models")
class TopKEncoder(Encoder, BaseModel):
    """Block that can be used for top-k prediction & evaluation, initialized
    from a trained retrieval model

    Parameters
    ----------
    query_encoder: Union[Encoder, tf.keras.layers.Layer],
        The layer to use for encoding the query features
    topk_layer: Union[str, tf.keras.layers.Layer, TopKOutput]
        The layer to use for computing the top-k predictions.
        You can also pass the `name` of registered top-k layer.
        The current supported strategies are [`brute-force-topk`]
        By default "brute-force-topk"
    candidates: Union[tf.Tensor, ~merlin.io.Dataset]
        The candidate embeddings to use for the Top-k index.
        You can pass a tensor of pre-trained embeddings
        or a merlin.io.Dataset of pre-trained embeddings, indexed by
        the candidates ids.
        This is required when `topk_layer` is a string
        By default None
    candidate_encoder:  Union[Encoder, tf.keras.layers.Layer],
        The layer to use for encoding the item features
    k: int, Optional
        Number of candidates to return, by default 10
    pre: Optional[tf.keras.layers.Layer]
        A block to use before encoding the input query
        By default None
    post: Optional[tf.keras.layers.Layer]
        A block to use after getting the top-k prediction scores
        By default None
    target: str, optional
        The name of the target. This is required when multiple targets are provided.
        By default None
    """

    def __init__(
        self,
        query_encoder: Union[Encoder, tf.keras.layers.Layer],
        topk_layer: Union[str, tf.keras.layers.Layer, TopKOutput] = "brute-force-topk",
        candidates: Union[tf.Tensor, merlin.io.Dataset] = None,
        candidate_encoder: Union[Encoder, tf.keras.layers.Layer] = None,
        k: int = 10,
        pre: Optional[tf.keras.layers.Layer] = None,
        post: Optional[tf.keras.layers.Layer] = None,
        target: str = None,
        **kwargs,
    ):
        if isinstance(topk_layer, TopKOutput):
            topk_output = topk_layer
        else:
            topk_output = TopKOutput(to_call=topk_layer, candidates=candidates, k=k, target=target)
        self.k = k

        Encoder.__init__(self, query_encoder, topk_output, pre=pre, post=post, **kwargs)
        # The base model is required for the evaluation step:
        BaseModel.__init__(self, **kwargs)

    @classmethod
    def from_candidate_dataset(
        cls,
        query_encoder: Union[Encoder, tf.keras.layers.Layer],
        candidate_encoder: Union[Encoder, tf.keras.layers.Layer],
        dataset: merlin.io.Dataset,
        top_k: int = 10,
        index_column: Optional[Union[str, ColumnSchema, Schema, Tags]] = None,
        **kwargs,
    ):
        """Class method to initialize a TopKEncoder from a dataset of
        raw candidates features.

        Parameters
        ----------
        query_encoder : Union[Encoder, tf.keras.layers.Layer]
            The encoder layer to use for computing the query embeddings.
        candidate_encoder : Union[Encoder, tf.keras.layers.Layer]
            The encoder layer to use for computing the candidates embeddings.
        dataset : merlin.io.Dataset
            Raw candidate features dataset
        index_column : Union[str, ColumnSchema, Schema, Tags], optional
            The column to use as candidates identifiers, this will be used
            for returning the topk ids of candidates with the highest scores.
            If not specified, the candidates indices will be used instead.
            by default None
        top_k : int, optional
            Number of candidates to return, by default 10

        Returns
        -------
        TopKEncoder
            a `TopKEncoder` indexed by the pre-trained embeddings of the candidates
            in the specified `dataset`
        """
        # TODO: Add related unit-test after RetrievalModelV2 is merged
        candidates = cls.encode_candidates(dataset, candidate_encoder)
        topk_output = TopKOutput(
            to_call="topk_layer", candidate_dataset=candidates, k=top_k, **kwargs
        )
        return cls(query_encoder, topk_output, **kwargs)

    def compile(
        self,
        optimizer="rmsprop",
        loss=None,
        metrics=None,
        loss_weights=None,
        weighted_metrics=None,
        run_eagerly=None,
        steps_per_execution=None,
        jit_compile=None,
        k: int = None,
        **kwargs,
    ):
        """Extend the compile method of `BaseModel` to set the threshold `k`
        of the top-k encoder.
        """
        if k is not None:
            self.topk_layer._k = k
            self.k = k
        BaseModel.compile(
            self,
            optimizer=optimizer,
            loss=loss,
            metrics=metrics,
            weighted_metrics=weighted_metrics,
            run_eagerly=run_eagerly,
            loss_weights=loss_weights,
            steps_per_execution=steps_per_execution,
            jit_compile=jit_compile,
            **kwargs,
        )

    @property
    def topk_layer(self):
        return self.blocks[-1].to_call

    def index_candidates(self, candidates, identifiers=None):
        self.topk_layer.index(candidates, identifiers=identifiers)
        return self

    def encode_candidates(
        self,
        dataset: merlin.io.Dataset,
        index_column: Optional[Union[str, ColumnSchema, Schema, Tags]] = None,
        candidate_encoder: Optional[Union[Encoder, tf.keras.layers.Layer]] = None,
        **kwargs,
    ) -> merlin.io.Dataset:
        """Method to generate candidates embeddings

        Parameters
        ----------
        dataset : merlin.io.Dataset
            Raw candidate features dataset
        index_column  : Union[str, ColumnSchema, Schema, Tags], optional
            The column to use as candidates identifiers, this will be used
            for returning the topk ids of candidates with the highest scores.
            If not specified, the candidates indices will be used instead.
            by default None
        candidate_encoder : Union[Encoder, tf.keras.layers.Layer], optional
            The encoder layer to use for computing the candidates embeddings.
            If not specified, the candidate_encoder set in the constructor
            will be used instead.
            by default None
        Returns
        -------
        merlin.io.Dataset
            A merlin dataset of candidates embeddings, indexed by index_column.
        """
        # TODO: Add related unit-test after RetrievalModelV2 is merged
        if not candidate_encoder:
            candidate_encoder = self.candidate_encoder
        assert candidate_encoder is not None, ValueError(
            "You should provide a `candidate_encoder` to compute candidates embeddings"
        )
        return candidate_encoder.encode(dataset=dataset, index=index_column, **kwargs)

    def batch_predict(
        self,
        dataset: Union[merlin.io.Dataset, Loader],
        batch_size: int,
        output_schema: Optional[Schema] = None,
        **kwargs,
    ) -> merlin.io.Dataset:
        """Batched top-k prediction using Dask.

        Parameters
        ----------
        dataset : Union[merlin.io.Dataset, merlin.models.tf.loader.Loader]
            Raw queries features dataset or Loader
        batch_size : int
            The number of queries to process at each prediction step
        output_schema: Schema, optional
            The columns to output from the input dataset
        Returns
        -------
        merlin.io.Dataset
            A merlin dataset with the top-k predictions, the
            candidates identifiers and related scores.
        """
        from merlin.models.tf.utils.batch_utils import TFModelEncode

        loader_transforms = None
        if isinstance(dataset, Loader):
            loader_transforms = dataset.transforms
            batch_size = dataset.batch_size
            dataset = dataset.dataset

        dataset_schema = dataset.schema
        dataset = dataset.to_ddf()

        model_encode = TFModelEncode(
            model=self,
            batch_size=batch_size,
            loader_transforms=loader_transforms,
            schema=dataset_schema,
            output_names=TopKPrediction.output_names(self.k),
            **kwargs,
        )

        encode_kwargs = {}
        if output_schema:
            encode_kwargs["filter_input_columns"] = output_schema.column_names

        # Processing a sample of the dataset with the model encoder
        # to get the output dataframe dtypes
        sample_output = model_encode(dataset.head(), **encode_kwargs)
        output_dtypes = sample_output.dtypes.to_dict()

        predictions = dataset.map_partitions(model_encode, meta=output_dtypes, **encode_kwargs)

        return merlin.io.Dataset(predictions)

    def fit(self, *args, **kwargs):
        """Fit model"""
        raise NotImplementedError(
            "This block is not meant to be trained by itself. ",
            "It can only be trained as part of a model.",
        )

@tf.keras.utils.register_keras_serializable(package="merlin.models")
class EmbeddingEncoder(Encoder):
    """Creates an Encoder from an EmbeddingTable.
    Typically used with RetrievalModelV2.

    Parameters
    ----------
    schema : Union[ColumnSchema, Schema]
        The ColumnSchema of the column for which the
        embedding table needs to be created.
        If a Schema is passed, only the first column
        is considered
    dim : int
        Dimension of the embeddings
    embeddings_initializer : Union[str, tf.keras.layers.Layer], optional
        Initializer for the `embeddings`
        matrix (see `keras.initializers`). By default "uniform"
    embeddings_regularizer : Union[str, tf.keras.layers.Layer], optional
        Regularizer function applied to
        the `embeddings` matrix (see `keras.regularizers`)., by default None
    activity_regularizer : Union[str, tf.keras.layers.Layer], optional
        Sets a layer that applies an update to the cost function based
        input activity, by default None
    embeddings_constraint : Union[str, tf.keras.layers.Layer], optional
        Constraint function applied to
        the `embeddings` matrix (see `keras.constraints`), by default None
    mask_zero : bool, optional
        Whether or not the input value 0 is a special "padding"
        value that should be masked out.
        This is useful when using recurrent layers
        which may take variable length input.
        If this is `True`, then all subsequent layers
        in the model need to support masking or an exception will be raised.
        If mask_zero is set to True, as a consequence, index 0 cannot be
        used in the vocabulary (input_dim should equal size of
        vocabulary + 1), by default False
    input_length : int, optional
        This argument is required if you are going to connect
        `Flatten` then `Dense` layers upstream
        (without it, the shape of the dense outputs cannot be computed),
        by default None
    sequence_combiner : Optional[CombinerType], optional
        A string specifying how to combine embedding results for each
        entry ("mean", "sqrtn" and "sum" are supported) or a layer.
        Default is None (no combiner used), by default None
    trainable : bool, optional
        Whether the layer's variables should be trainable, by default True
    name : str, optional
        String name of the layer, by default None
    dtype : optional
        The dtype of the layer's computations and weights. Can also be a
        `tf.keras.mixed_precision.Policy`, which allows the computation and weight
        dtype to differ. Default of `None` means to use
        `tf.keras.mixed_precision.global_policy()`, which is a float32 policy
        unless set to different value., by default None
    dynamic : bool, optional
        Set this to `True` if your layer should only be run eagerly, and
        should not be used to generate a static computation graph.
        This would be the case for a Tree-RNN or a recursive network,
        for example, or generally for any layer that manipulates tensors
        using Python control flow. If `False`, we assume that the layer can
        safely be used to generate a static computation graph., by default False
    embeddings_l2_batch_regularization : Optional[Union[float, Dict[str, float]]], optional
        Factor for L2 regularization of the embeddings vectors (from the current batch only)
        by default 0.0, by default 0.0
    post : Optional[tf.keras.layers.Layer], optional
        _description_, by default None
    **kwargs: Forwarded Encoder parameters
    """

    def __init__(
        self,
        schema: Union[ColumnSchema, Schema],
        dim: int,
        embeddings_initializer: Optional[Union[str, tf.keras.layers.Layer]] = "uniform",
        embeddings_regularizer: Optional[Union[str, tf.keras.layers.Layer]] = None,
        activity_regularizer: Optional[Union[str, tf.keras.layers.Layer]] = None,
        embeddings_constraint: Optional[Union[str, tf.keras.layers.Layer]] = None,
        mask_zero: bool = False,
        input_length: int = None,
        sequence_combiner: Optional[CombinerType] = None,
        trainable: bool = True,
        name: str = None,
        dtype=None,
        dynamic: bool = False,
        embeddings_l2_batch_regularization: Optional[Union[float, Dict[str, float]]] = 0.0,
        post: Optional[tf.keras.layers.Layer] = None,
        **kwargs,
    ):
        if isinstance(schema, ColumnSchema):
            col = schema
        else:
            col = schema.first
        col_name = col.name

        table = EmbeddingTable(
            dim,
            col,
            embeddings_initializer=embeddings_initializer,
            embeddings_regularizer=embeddings_regularizer,
            activity_regularizer=activity_regularizer,
            embeddings_constraint=embeddings_constraint,
            mask_zero=mask_zero,
            input_length=input_length,
            sequence_combiner=sequence_combiner,
            trainable=trainable,
            name=name,
            dtype=dtype,
            dynamic=dynamic,
            l2_batch_regularization_factor=embeddings_l2_batch_regularization,
        )

        super().__init__(table, tf.keras.layers.Lambda(lambda x: x[col_name]), post=post, **kwargs)

    def to_dataset(self, gpu=None) -> merlin.io.Dataset:
        return self.blocks[0].to_dataset(gpu=gpu)


import argparse
import numpy as np
import pandas as pd
from utilities import *

def clean(args):
    """Load and clean user-supplied data."""

    print('CLEANING DATA....')
    # load data as a pandas data frame
    df = pd.read_csv(args.filePath, low_memory=False)

    # interpolate missing data values
    if args.interpolate or args.all:
        print('\tDetecting missing values...')
        df = fill_empty_with_nan(df)
        print('\tImputing missing values...')
        df = replace_missing_data(df)
        df = drop_missing_data(df)

    # detect and remove outliers
    if args.outliers or args.all:
        print('\tRemoving outliers...')
        df = remove_outliers(df)
        df = normalize(df)

    # one-hot encode the categorical variables

    # delete useless columns
    if args.all:
        print('\tDeleting useless columns...')
        df = combine_first_last_name(df)
        df = drop_useless_columns(df)
        df, date = check_if_dateTime(df)
    # save cleaned data file to same directory as uncleaned version

    if args.categorical or args.all:
        print('\tTransforming categorical data using one-hot encoding...')
        df = one_hot_encode(df)
    df.to_csv(args.filePath[:-4] + '_CLEANED.csv')
    print('DONE.')
    return df

def add_args(parser):
    """Add command line options for customized data preprocessing."""
    parser.add_argument('filePath', metavar='filePath', type=str, help='Path to uncleaned data file')
    parser.add_argument('-a', '--all', action='store_const', const='a')
    parser.add_argument('-c', '--categorical', action='store_const', const='c')
    parser.add_argument('-i', '--interpolate', action='store_const', const='i')
    parser.add_argument('-m', '--missing', action='store_const', const='m')
    parser.add_argument('-o', '--outliers', action='store_const', const='o')
    parser.add_argument('-v', '--version', action='version', version='v1.0.0')

def main():
    """Create command line argument parser and call clean for preprocessing."""
    parser = argparse.ArgumentParser()
    add_args(parser)
    args = parser.parse_args()
    print(args)
    df = clean(args) # clean data according to user-supplied options
    # CorrelationMatrixPlot(df)
if __name__ == '__main__':
    main()


from PyQt5.QtWidgets import QMessageBox, QFileDialog, QErrorMessage

from org.sortition.groupselect.gui.mainwin.TAImportOptionsDialog import TAImportOptionsDialog
from org.sortition.groupselect.gui.mainwin.TAInsertRowsColsDialog import TAInsertRowsColsDialog

class TAMainWindowDataActionHandler:
    def __init__(self, ctx, window):
        self.ctx = ctx
        self.win = window

    def confirm_discard(self):
        if not self.ctx.app_data_manager.peopledata_is_empty():
            reply = QMessageBox.question(self.win, 'Overwrite Content', "The data import will overwrite all the raw data. Proceed?", QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
            if reply == QMessageBox.Yes: return True
            else: return False
        else: return True

    def confirm_discard_results(self):
        if self.ctx.app_data.results:
            reply = QMessageBox.question(self.win, 'Discard Results', "Please be aware that this action will discard your current results. Proceed?", QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
            if reply == QMessageBox.Yes: return True
            else: return False
        else: return True

    def import_raw(self):
        if not self.ctx.get_status(): return
        if self.confirm_discard():
            fname, scheme = QFileDialog.getOpenFileName(self.win, 'Import People Data to CSV', None, "Comma-separated Values Files (*.csv)")
            if not fname: return
            ok, csv_format = TAImportOptionsDialog.get_input(self.win)
            if not ok: return
            try:
                with open(fname, 'r') as handle:
                    self.ctx.app_data_manager.import_raw_from_csv(handle, csv_format)
            except Exception as e:
                error_dialog = QErrorMessage()
                error_dialog.showMessage(str(e))
                return
            self.ctx.set_unsaved()
            self.win.tabs.tab_peopledata.update_table_from_data()
            self.win.tabs.peopledata_updated()

    def export_raw(self):
        if not self.ctx.get_status(): return
        fname, scheme = QFileDialog.getSaveFileName(self.win, 'Export People Data to CSV', None, "Comma-separated Values Files (*.csv)")
        if not fname: return
        if not fname.endswith('.csv'):
            fname += '.csv'
        try:
            with open(fname, 'w') as handle:
                self.ctx.app_data_manager.export_raw_to_csv(handle)
                handle.close()
        except Exception as e:
            error_dialog = QErrorMessage()
            error_dialog.showMessage(str(e))

    def insert_rows(self):
        if not self.ctx.get_status(): return
        if not self.confirm_discard_results(): return
        options = [(i, i+1) for i in range(self.ctx.app_data.m_data)]
        options.append((self.ctx.app_data.m_data, 'end'))
        ok, beforeRow, number = TAInsertRowsColsDialog.get_input(self.win, 'rows', options)
        if not ok: return
        self.ctx.app_data_manager.insert_rows(beforeRow, number)
        self.ctx.set_unsaved()
        self.win.tabs.tab_peopledata.update_table_from_data()
        self.win.tabs.peopledata_updated()
        self.win.tabs.tab_results.display_empty()
        self.win.tabs.results_updated()
        self.win.results_menu.setEnabled(False)

    def insert_cols(self):
        if not self.ctx.get_status(): return
        options = [(j, self.ctx.app_data.peopledata_keys[j]) for j in range(self.ctx.app_data.n_data)]
        options.append((self.ctx.app_data.n_data, 'end'))
        ok, beforeCol, number = TAInsertRowsColsDialog.get_input(self.win, 'cols', options)
        if not ok: return
        self.ctx.app_data_manager.insert_cols(beforeCol, number)
        self.ctx.set_unsaved()
        self.win.tabs.tab_peopledata.update_table_from_data()
        self.win.tabs.peopledata_updated()
        self.win.tabs.tab_results.display_empty()
        self.win.tabs.results_updated()
        self.win.results_menu.setEnabled(False)

    def delete_rows(self):
        if not self.ctx.get_status(): return
        if not self.confirm_discard_results(): return
        selection = self.win.tabs.tab_peopledata.table_widget.selectionModel()
        rows = [index.row() for index in selection.selectedRows()]
        if not rows: return
        self.ctx.app_data_manager.delete_rows(rows)
        self.ctx.set_unsaved()
        self.win.tabs.tab_peopledata.update_table_from_data()
        self.win.tabs.peopledata_updated()
        self.win.tabs.tab_results.display_empty()
        self.win.tabs.results_updated()
        self.win.results_menu.setEnabled(False)

    def delete_cols(self):
        if not self.ctx.get_status(): return
        selection = self.win.tabs.tab_peopledata.table_widget.selectionModel()
        cols = [index.column() for index in selection.selectedColumns()]
        if not cols: return
        must_discard_results = self.ctx.app_data_manager.cols_not_ignored(cols)
        if must_discard_results:
            if not self.confirm_discard_results(): return
        self.ctx.app_data_manager.delete_cols(cols, must_discard_results)
        self.ctx.set_unsaved()
        self.win.tabs.tab_peopledata.update_table_from_data()
        self.win.tabs.peopledata_updated()
        if must_discard_results: self.win.tabs.tab_results.display_empty()
        self.win.tabs.results_updated()
        self.win.results_menu.setEnabled(False)


from . import main
from flask import render_template, redirect, url_for, request, jsonify
from flask_login import login_required
import requests

FLOWER_URL = 'http://127.0.0.1:5555'

@main.route('/')
def index():
    return render_template('cover.html')
    #return redirect(url_for('data.samples'))

@main.route('/task/result/<task_id>/', methods=['GET'])
def show_result(task_id):
    return render_template('celery_task.html', task_id=task_id, state='pending')

@main.route('/task/result/', methods=['POST'])
def celery_task():
    if request.method == 'POST':
        task_id = request.form['task_id']
        url = '{root_url}/api/task/result/{id}'.format(root_url=FLOWER_URL, id=task_id)
        reply = requests.get(url).json()
        if reply['state'] == 'SUCCESS':
            task = reply['result']['task']
            result = reply['result']['result']
            return jsonify({'msg': 'ok', 'task': task, 'result': result})
        elif reply['state'] == 'PENDING':
            return jsonify({'msg': 'pending'})
        else:
            return jsonify({'msg': 'error'})


import time

import cv2
import numpy as np
from mediapipe import solutions
from socket import *
import re

pattern = re.compile(r"x:\s+([\d\\.\-e]*)\s+y:\s+([\d\\.\-e]*)\s+z:\s+([\d\\.\-e]*)")

def bytes_to_image(width, height, bs):
    return np.frombuffer(bs, dtype=np.uint8).reshape((height, width, 3))

def process_result(results):
    if results.multi_hand_landmarks is None:
        return "[]"
    res_list = []
    hands_num = len(results.multi_hand_landmarks)
    for i in range(hands_num):
        hand = results.multi_hand_landmarks[i]
        res = pattern.findall(str(hand))
        tmp = []
        for tup in res:
            tmpmp = list(tup)
            for index in range(3):
                tmpmp[index] = float(tmpmp[index])

            tmp.append(tmpmp)
        res_list.append(tmp)

    return str(res_list)

class Processor(object):
    def __init__(self):
        self.hands = solutions.hands.Hands(static_image_mode=False,
                                           max_num_hands=2,
                                           min_detection_confidence=0.8)

    def image_process(self, img):
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        results = self.hands.process(img)
        return process_result(results)

class Communicator(object):
    def __init__(self):
        self.socket = socket(AF_INET, SOCK_STREAM)
        while True:
            try:
                self.socket.connect(("127.0.0.1", 1234))
                break
            except ConnectionRefusedError:
                time.sleep(1)  # spin

    def send(self, message: str):
        self.socket.send(message.encode())

    def receive_int(self):
        bb = self.socket.recv(10)
        return int(bb)

    def receive_img(self, width, height):
        bb = self.socket.recv(width * height * 3)
        return bytes_to_image(width, height, bb)

    def close(self):
        self.socket.close()

def main():
    processor = Processor()
    communicator = Communicator()

    width = communicator.receive_int()
    height = communicator.receive_int()

    res = "begin"
    communicator.send(res)

    try:
        while True:
            img = communicator.receive_img(width, height)
            res = processor.image_process(img)
            communicator.send(res)
    except ConnectionResetError:
        pass

    communicator.close()

if __name__ == '__main__':
    main()


#!/bin/sh python
import math
import numpy as np
import torch as tc

import argparse

import syft as sf
import grid as gr

from torch import nn
from GRU_RNN import GRU

parser = argparse.ArgumentParser(
    description="Perform training and bechmark of spam dataset.")

parser.add_argument(
    "-gw",
    "--gateway",
    type=str,
    required=True,
    help="Address of the grid gateway",
)

def main(args):
    hook = sf.TorchHook(tc)

    tr_grid = gr.GridNetwork("http://" + args.gateway)
    texts = tr_grid.search("#X", "#spam", "#dataset")
    types = tr_grid.search("#Y", "#spam", "#dataset")
    print("Found datasets:")
    print(texts)
    print(types)

    # Model parameters
    HIDDEN_DIM = 10
    EMBEDDING_DIM = 50
    DROPOUT = 0.2
    EPOCHS = 15
    BATCH_SIZE = 32
    HIDDEN_DIM = 10
    LR = 0.1
    VOCAB_SIZE = 0
    for texts_comp in texts:
        VOCAB_SIZE = max(VOCAB_SIZE, int(texts_comp[0].max().get()))
    VOCAB_SIZE += 1

    # Initiating the model
    model = GRU(vocab_size=VOCAB_SIZE, hidden_dim=HIDDEN_DIM,
                embedding_dim=EMBEDDING_DIM, dropout=DROPOUT)

    # Defining the loss and optimizer
    criterion = nn.BCELoss()
    optimizer = tc.optim.SGD(model.parameters(), lr=LR)

    for epoch in range(EPOCHS):
        dataset_size = sum([len(texts[i][0]) for i in range(len(texts))])
        print("Beginning training ...")
        model.train()

        for i in range(len(texts)):
            loss_acc = 0
            nr_batches = math.ceil(len(texts[i][0]) / BATCH_SIZE)
            for batch_idx in range(nr_batches):
                # Extract the batch for training and target
                texts_batch = texts[i][0][BATCH_SIZE *
                                          batch_idx: BATCH_SIZE * (batch_idx + 1), :]
                types_batch = types[i][0][BATCH_SIZE *
                                          batch_idx: BATCH_SIZE * (batch_idx + 1)]

                # Send the model to the worker
                worker = texts_batch.location
                model.send(worker)
                h = tc.Tensor(tc.zeros((BATCH_SIZE, HIDDEN_DIM))).send(worker)

                optimizer.zero_grad()
                pred, _ = model(texts_batch, h)
                loss = criterion(pred.squeeze(), types_batch.float())
                loss.backward()
                optimizer.step()
                model.get()

                # Accumulate the loss
                loss_acc += loss.get().item()

            print('Epoch: {} [{}/{} ({:.0f}%)]\tTraining loss: {:.6f}'.format(
                epoch, i * texts[i][0].shape[0], dataset_size,
                100. * (i*texts[i][0].shape[0]) / dataset_size, loss_acc))

if __name__ == '__main__':
    main(parser.parse_args())


from collections import deque
from contextlib import contextmanager
from typing import IO, ContextManager, Deque, Generator, List

from .search import find_sum_two

@contextmanager
def file_read(filename: str) -> Generator:
    f = open(filename)
    yield f
    f.close()

def process_list(numbers: List[int], n: int) -> int:
    """
    This function returns the first number in the list (after the preamble)
    which is not a sum of two of the X numbers before it.
    Uses the search function from day1
    """
    d: Deque[int] = deque(numbers[:n], maxlen=n)
    for number in numbers[n:]:
        if len(find_sum_two(list(d), number)) == 2:
            d.append(number)
        else:
            return number
    return -1

def break_encryption(numbers: List[int], n: int) -> int:
    """
    This function returns the sum of the smallest and largest
    number in a contiguous set of at least two numbers from the
    list that sum up to the number `n`.
    """
    # first try 2, then 3, then 4, and so on
    for i in range(2, len(numbers)):
        d: Deque[int] = deque(numbers[:i], maxlen=i)
        for j in numbers[i:]:
            if sum(d) == n:
                return min(d) + max(d)
            else:
                d.append(j)
    return -1

if __name__ == "__main__":
    cm: ContextManager[IO] = file_read("input")
    with cm as input_file:
        numbers: List[int] = [int(line) for line in input_file.read().splitlines()]
    # print(numbers)
    ans: int = process_list(numbers, 25)
    print(ans)
    index: int = numbers.index(ans)
    # only send it the part of the list until that number
    print(break_encryption(numbers[:index], ans))


import subprocess
from datetime import datetime

data_e_hora_atuais = datetime.now()
data_e_hora_em_texto = data_e_hora_atuais.strftime("%d/%m/%Y %H:%M")
data_e_hora_em = data_e_hora_atuais.strftime("%d%m%Y%H%M")

with open("config/pc.txt", "r") as tf:
    lines = tf.read().split('\n')

with open("config/msg.txt", "r") as tf:
    msg = tf.read()

for line in lines:
    Computer_List = [line]
    for Computer in Computer_List:
        print("Desligando o PC:" + Computer)
        a = subprocess.getoutput(
            "shutdown /m \\\\" + Computer + " /f /s /c"+msg+"/t 60"+msg)
        arquivo = open('logs/log['+data_e_hora_em+'].txt', 'a')
        arquivo.write(Computer + ' : ')
        if len(a) > 2:
            print("PC Desligado")
            arquivo.write("Desligado  [" +
                          data_e_hora_em_texto + "]\n")
        else:
            print("Desligando")
            arquivo.write("Ok  [" +
                          data_e_hora_em_texto + "]\n")
        arquivo.close()


from collections import defaultdict
from random import shuffle
from typing import List

def undersample_to_even(corpus: List[dict], y_label_name: str):
    per_category = defaultdict(list)
    for sample in corpus:
        per_category[sample[y_label_name]].append(sample)

    min_samples_in_category = min([len(per_category[category]) for category in per_category])
    print(f'The category with the least amount of samples contains: {min_samples_in_category} samples.')
    undersampled = []
    for category in per_category:
        per_category[category] = per_category[category][:min_samples_in_category]
        shuffle(per_category[category])
        undersampled.extend(per_category[category])
    print(f'Undersampling completed. Returning corpus containing {len(undersampled)} samples.')
    shuffle(undersampled)
    return undersampled

def cut_off_longer_texts(corpus: List[dict], x_name: str, word_limit):
    filtered_by_length = []
    for sample in corpus:
        word_count = len(sample[x_name].split())
        if word_count <= word_limit:
            filtered_by_length.append(sample)
    return filtered_by_length


# pylint: disable=missing-docstring
from unittest.mock import Mock, mock_open, patch

import pytest

from peltak import testing
from peltak.core import versioning

def patch_open(**kw):
    return patch('peltak.core.versioning.version_file.open', mock_open(**kw))

@pytest.mark.parametrize('version_def,expected', [
    ("__version__ = '0.7.4'", "__version__ = '1.0.1'"),
    ("__version__='0.7.4'", "__version__ = '1.0.1'"),
    ('__version__ = "0.7.4"', "__version__ = '1.0.1'"),
    ('__version__ = "0.7"', "__version__ = '1.0.1'"),
    ('__version__ = "1.0"', "__version__ = '1.0.1'"),
    ('__version__ = "1"', "__version__ = '1.0.1'"),
])
@testing.patch_pelconf()
@patch('peltak.core.versioning.version_file.exists', Mock(return_value=True))
@patch('peltak.core.fs.write_file')
def test_correctly_replaces_version(p_write_file: Mock, version_def: str, expected: str):
    file_data = '\n'.join([
        "# -*- coding: utf-8 -*-",
        version_def,
        "",
    ])
    expected_data = '\n'.join([
        "# -*- coding: utf-8 -*-",
        expected,
        "",
    ])

    with patch_open(read_data=file_data):
        storage = versioning.PyVersionFile('fake.py')
        storage.write('1.0.1')

        p_write_file.assert_called_once_with(
            'fake.py',
            expected_data
        )


# for python2
# from Tkinter import *
# import ttk
# import tkFileDialog as askopenfilename

#for python3
from tkinter import *
from tkinter import ttk
from tkinter.filedialog import askopenfilename
from PIL import Image
import pytesseract
import numpy as np
import cv2
import os
import sys

from userimageski import UserData

root = Tk(  )

def readFimage():
    path = PathTextBox.get('1.0','end-1c')
    if path:
        im = Image.open(path)
        # text = pytesseract.image_to_string(im, lang = 'eng')
        # Read image with opencv
        img = cv2.imread(path)
        # Convert to gray
        img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        # Apply dilation and erosion to remove some noise
        kernel = np.ones((1, 1), np.uint8)
        img = cv2.dilate(img, kernel, iterations=1)
        img = cv2.erode(img, kernel, iterations=1)

        # Write the image after apply opencv to do some ...
        cv2.imwrite("thres.png", img)
        # Recognize text with tesseract for python
        result = pytesseract.image_to_string(Image.open("thres.png"))
        print(pytesseract.image_to_data(Image.open('thres.png')))
        os.remove("thres.png")
        text = result
        ResultTextBox.delete('1.0',END)
        ResultTextBox.insert(END,text)

        ##### the following code includes all the steps to get from a raw image to a prediction.
        ##### the working code is the uncommented one.
        ##### the two pickle models which are passed as argument to the select_text_among_candidates
        ##### and classify_text methods are obviously the result of a previously implemented pipeline.
        ##### just for the purpose of clearness below the code is provided.
        ##### I want to emphasize that the commented code is the one necessary to get the models trained.

        # creates instance of class and loads image
        user = UserData(path)
        # plots preprocessed imae
        user.plot_preprocessed_image()
        # detects objects in preprocessed image
        candidates = user.get_text_candidates()
        # plots objects detected
        user.plot_to_check(candidates, 'Total Objects Detected')
        # selects objects containing text
        # maybe_text = user.select_text_among_candidates('linearsvc-hog-fulltrain2-90.pickle')
        # plots objects after text detection
        # user.plot_to_check(maybe_text, 'Objects Containing Text Detected')
        # classifies single characters
        # classified = user.classify_text('linearsvc-hog-fulltrain36-90.pickle')
        # plots letters after classification
        # user.plot_to_check(classified, 'Single Character Recognition')
        # plots the realigned text
        # user.realign_text()
    else:
        ResultTextBox.delete('1.0',END)
        ResultTextBox.insert(END,"FILE CANNOT BE READ")

def OpenFile():
    name = askopenfilename(initialdir="/home/frank/Desktop/OCR_PROJECT/data",
                           filetypes =(("PNG File", "*.png"),("BMP File", "*.bmp"),("JPEG File", "*.jpeg"),("JPG File", "*.jpg")),
                           title = "Choose a file."
                           )
    PathTextBox.delete("1.0",END)
    PathTextBox.insert(END,name)
Title = root.title( "Image Reader!")
path = StringVar()

HeadLabel1 = Label(root,text="Image ")
HeadLabel1.grid(row = 1,column = 1,sticky=(E))
HeadLabel2 = Label(root,text=" Reader")
HeadLabel2.grid(row = 1,column = 2,sticky=(W))

InputLabel = Label(root,text = "INPUT IMAGE:")
InputLabel.grid(row=2,column = 1)

BrowseButton = Button(root,text="Browse",command = OpenFile)
BrowseButton.grid(row=2,column=2)

PathLabel = Label(root,text = "Path:")
PathLabel.grid(row = 3,column=1,sticky=(W))

PathTextBox = Text(root,height = 2)
PathTextBox.grid(row = 4,column = 1,columnspan=2)

ReadButton = Button(root,text="READ FROM IMAGE",command = readFimage)
ReadButton.grid(row = 5,column = 2)

DataLabel = Label(root,text = "DATA IN IMAGE:")
DataLabel.grid(row = 6,column=1,sticky=(W))

ResultTextBox = Text(root,height = 6)
ResultTextBox.grid(row = 7,column = 1,columnspan=2)

root.mainloop()


from xml.etree.ElementInclude import include
from django.urls import path

from .views import LoginView, SignUpView, LocationView, MeetingsAPI,MeetingAPI

urlpatterns = [
    path('signup/', SignUpView.as_view()),
    path('login/', LoginView.as_view()),
    path('meetings/',MeetingsAPI.as_view()),
    path('meeting/ <int:meeting_id>',MeetingAPI.as_view()),
    path('location/', LocationView.as_view()),
]


import tempfile
from typing import Literal

from pybotx.async_buffer import AsyncBufferReadable
from pybotx.client.authorized_botx_method import AuthorizedBotXMethod
from pybotx.client.botx_method import response_exception_thrower
from pybotx.client.exceptions.files import FileTypeNotAllowed
from pybotx.constants import CHUNK_SIZE
from pybotx.models.api_base import VerifiedPayloadBaseModel

class BotXAPIUploadFileResult(VerifiedPayloadBaseModel):
    link: str

class BotXAPIUploadFileResponsePayload(VerifiedPayloadBaseModel):
    result: BotXAPIUploadFileResult
    status: Literal["ok"]

    def to_domain(self) -> str:
        return self.result.link

class UploadFileMethod(AuthorizedBotXMethod):
    status_handlers = {
        **AuthorizedBotXMethod.status_handlers,
        400: response_exception_thrower(FileTypeNotAllowed),
    }

    async def execute(
        self,
        async_buffer: AsyncBufferReadable,
        filename: str,
    ) -> BotXAPIUploadFileResponsePayload:
        path = "/api/v3/botx/smartapps/upload_file"

        with tempfile.SpooledTemporaryFile(max_size=CHUNK_SIZE) as tmp_file:
            chunk = await async_buffer.read(CHUNK_SIZE)
            while chunk:
                tmp_file.write(chunk)
                chunk = await async_buffer.read(CHUNK_SIZE)

            tmp_file.seek(0)

            response = await self._botx_method_call(
                "POST",
                self._build_url(path),
                files={"content": (filename, tmp_file)},
            )

        return self._verify_and_extract_api_model(
            BotXAPIUploadFileResponsePayload,
            response,
        )