import pandas as pd
import numpy as np
from numpy import linalg as LA
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
import random

diseases_abbr = {'Lung Opacity': 'AO',
                'Cardiomegaly': 'Cd',
                'Effusion': 'Ef',
                'Enlarged Cardiomediastinum': 'EC',
                'Lung Lesion': 'LL',
                'Atelectasis': 'A',
                'Pneumonia': 'Pa',
                'Pneumothorax': 'Px',
                'Consolidation': 'Co',
                'Edema': 'Ed',
                'Pleural Effusion': 'Ef',
                'Pleural Other': 'PO',
                'Fracture': 'Fr',
                'Support Devices': 'SD',
                'Airspace Opacity': 'AO',
                'No Finding': 'NF'
                }

ylabel = {'Age': 'AGE',
        'Sex': 'SEX',
        'F': 'FEMALE',
        'M': 'MALE'
        }

def plot_frequency(df, diseases, category, category_name):
    plt.rcParams.update({'font.size': 18})
    df = preprocess(df)
    freq = []
    for d in diseases:
        cate = []
        for c in category:
            cate.append(len(df.loc[(df[d] == 1) & (df[category_name] == c), :]))
        freq.append(cate)
    freq = np.array(freq)
    if category_name == 'Age':
        plt.figure(figsize=(18,9))

        width = 0.075
    elif category_name == 'Sex':
        plt.figure(figsize=(18,9))

        width = 0.35
    ind = np.arange((len(diseases)))
    for i in range(len(category)):
        if category_name == 'Sex':
            plt.bar(ind + width * i, freq[:, i], width, label=ylabel[category[i]])
        else:
            plt.bar(ind + width * i, freq[:, i], width, label=category[i])
    plt.ylabel(str(category_name + ' FREQUENCY IN CXP').upper())
    plt.xticks(ind + width*(len(category)-1)/2, [diseases_abbr[k] for k in diseases])
    plt.legend()
    plt.savefig("./results/Frequency_" + category_name + ".pdf")

def tpr(df, d, c, category_name):
    pred_disease = "bi_" + d
    gt = df.loc[(df[d] == 1) & (df[category_name] == c), :]
    pred = df.loc[(df[pred_disease] == 1) & (df[d] == 1) & (df[category_name] == c), :]
    if len(gt) != 0:
        TPR = len(pred) / len(gt)
        return TPR
    else:
        # print("Disease", d, "in category", c, "has zero division error")
        return -1

def func(x, m, b):
    return m*x + b

#

def distance_max_min(df, diseases, category, category_name):
    df = preprocess(df)
    GAP_total = np.zeros((len(category), len(diseases)))
    percentage_total = np.zeros((len(category), len(diseases)))
    cate = []
    for c in range(len(category)):
        for d in range(len(diseases)):
            pred_disease = "bi_" + diseases[d]
            gt = df.loc[(df[diseases[d]] == 1) & (df[category_name] == category[c]), :]
            pred = df.loc[(df[pred_disease] == 1) & (df[diseases[d]] == 1) & (df[category_name] == category[c]), :]
            n_gt = df.loc[(df[diseases[d]] == 1) & (df[category_name] != category[c]) & (df[category_name] != 0), :]
            n_pred = df.loc[(df[pred_disease] == 1) & (df[diseases[d]] == 1) & (df[category_name] != category[c]) & (df[category_name] != 0), :]
            pi_gy = df.loc[(df[diseases[d]] == 1) & (df[category_name] == category[c]), :]
            pi_y = df.loc[(df[diseases[d]] == 1) & (df[category_name] != 0), :]

            if len(gt) != 0 and len(n_gt) != 0 and len(pi_y) != 0:
                TPR = len(pred) / len(gt)
                n_TPR = len(n_pred) / len(n_gt)
                percentage = len(pi_gy) / len(pi_y)
                if category_name != 'Sex':
                    temp = []
                    for c1 in category:
                        ret = tpr(df, diseases[d], c1, category_name)
                        if ret != -1:
                            temp.append(ret)
                    temp.sort()
                    if len(temp) % 2 == 0:
                        median = (temp[(len(temp) // 2) - 1] + temp[(len(temp) // 2)])/2
                    else:
                        median = temp[(len(temp) // 2)]
                    GAP = TPR - median
                else:
                    GAP = TPR - n_TPR
                GAP_total[c, d] = GAP
                percentage_total[c, d] = percentage
            else:
                GAP_total[c, d] = 51
                percentage_total[c, d] = 0

        category[c] = category[c].replace(' ', '_', 3)
        category[c] = category[c].replace('/', '_', 3)
        cate.append(category[c])

    df = pd.DataFrame(index=['Max', 'Min', 'Distance'])
    for d in range(len(diseases)):
        mask = GAP_total[:, d] < 50
        minimum = np.min(GAP_total[:, d][mask])
        maximum = np.max(GAP_total[:, d][mask])
        distance = maximum - minimum
        df[diseases[d]] = [maximum, minimum, distance]
    df.to_csv("./results/distance_max_min_" + category_name + ".csv")

def plot_TPR_CXP(df, diseases, category, category_name):

    plt.rcParams.update({'font.size': 18})
    df = preprocess(df)
    final = {}
    for c in category:
        result = []
        for d in diseases:
            pred_disease = "bi_" + d
            gt = df.loc[(df[d] == 1) & (df[category_name] == c), :]
            pred = df.loc[(df[pred_disease] == 1) & (df[d] == 1) & (df[category_name] == c), :]
            n_gt = df.loc[(df[d] == 1) & (df[category_name] != c) & (df[category_name] != 0), :]
            n_pred = df.loc[(df[pred_disease] == 1) & (df[d] == 1) & (df[category_name] != c) & (df[category_name] != 0), :]
            pi_gy = df.loc[(df[d] == 1) & (df[category_name] == c), :]
            pi_y = df.loc[(df[d] == 1) & (df[category_name] != 0), :]

            if len(gt) != 0 and len(n_gt) != 0 and len(pi_y) != 0:
                TPR = len(pred) / len(gt)
                n_TPR = len(n_pred) / len(n_gt)
                percentage = len(pi_gy) / len(pi_y)
                if category_name != 'Sex':
                    temp = []
                    for c1 in category:
                        ret = tpr(df, d, c1, category_name)
                        if ret != -1:
                            temp.append(ret)
                    temp.sort()

                    if len(temp) % 2 == 0:
                        median = (temp[(len(temp) // 2) - 1] + temp[(len(temp) // 2)])/2
                    else:
                        median = temp[(len(temp) // 2)]
                    GAP = TPR - median
                else:
                    GAP = TPR - n_TPR
                result.append([percentage, GAP])
            else:
                result.append([50, 50])
        result = np.array(result)
        plt.figure(figsize=(10, 8))
        plt.subplots_adjust()
        mask = result[:, 1] < 50
        plt.scatter(result[:, 0][mask], result[:, 1][mask], label='TPR', color='green')

        params, params_cov = curve_fit(func, result[:, 0][mask], result[:, 1][mask])
        plt.plot(result[:, 0][mask], func(result[:, 0][mask], params[0], params[1]), color='green')
        diseases = np.array(diseases)
        for d, x, y in zip(diseases[mask], result[:, 0][mask], result[:, 1][mask]):
            plt.annotate(diseases_abbr[d], color='green', xy=(x, y), xytext=(-3, 3), textcoords='offset points', ha='right', va='bottom')
        plt.xlabel("% " + c)
        plt.ylabel("TPR " + ylabel[category_name] + " DISPARITY")
        c = c.replace(' ', '_', 3)
        c = c.replace('/', '_', 3)
        c = c.replace('>=', '_', 3)
        plt.savefig("./results/Median_TPR_" + category_name + "_" + c + ".pdf")

        ans = {'result': result,
                'mask': mask}
        final[c] = ans
    return final

def count(list1, l, r):
    c = 0
    # traverse in the list1
    for x in list1:
        # condition check
        if x > l and x < r:
            c+= 1
    return c

def plot_14(df, diseases, category, category_name):
    plt.rcParams.update({'font.size': 18})
    df = preprocess(df)
    GAP_total = []
    percentage_total = []
    cate = []

    print(diseases)

    if category_name == 'Sex':

        Run1_sex = pd.DataFrame(diseases,columns=["diseases"])

    if category_name == 'Age':

        Run1_age = pd.DataFrame(diseases,columns=["diseases"])

    for c in category:
        GAP_y = []
        percentage_y = []
        for d in diseases:
            pred_disease = "bi_" + d
            gt = df.loc[(df[d] == 1) & (df[category_name] == c), :]
            pred = df.loc[(df[pred_disease] == 1) & (df[d] == 1) & (df[category_name] == c), :]
            n_gt = df.loc[(df[d] == 1) & (df[category_name] != c) & (df[category_name] != 0), :]
            n_pred = df.loc[(df[pred_disease] == 1) & (df[d] == 1) & (df[category_name] != c) & (df[category_name] != 0), :]
            pi_gy = df.loc[(df[d] == 1) & (df[category_name] == c), :]
            pi_y = df.loc[(df[d] == 1) & (df[category_name] != 0), :]

            if len(gt) != 0 and len(n_gt) != 0 and len(pi_y) != 0:
                TPR = len(pred) / len(gt)
                n_TPR = len(n_pred) / len(n_gt)
                percentage = len(pi_gy) / len(pi_y)
                if category_name != 'Sex':
                    temp = []
                    for c1 in category:
                        ret = tpr(df, d, c1, category_name)
                        if ret != -1:
                            temp.append(ret)
                    temp.sort()

                    if len(temp) % 2 == 0:
                        median = (temp[(len(temp) // 2) - 1] + temp[(len(temp) // 2)])/2
                    else:
                        median = temp[(len(temp) // 2)]
                    GAP = TPR - median
                else:
                    GAP = TPR - n_TPR
                GAP_y.append(GAP)
                percentage_y.append(percentage)
            else:
                GAP_y.append(51)
                percentage_y.append(0)
        # print('Best Positive ' + c + ' ' + str(count(GAP_y, 0, 51)))
        # print('Worst Positive ' + c + ' ' + str(count(GAP_y, -50, 0)))
        # print('Zero ' + c + ' ' + str(GAP_y.count(0)))
        GAP_total.append(GAP_y)
        percentage_total.append(percentage_y)
        c = c.replace(' ', '_', 3)
        c = c.replace('/', '_', 3)
        cate.append(c)

    GAP_total = np.array(GAP_total)
    x = np.arange(len(diseases))
    fig = plt.figure(figsize=(18,9))
    ax = fig.add_subplot(111)
    for item in x:
        mask = GAP_total[:, item] < 50
        ann = ax.annotate('', xy=(item, np.max(GAP_total[:, item][mask])), xycoords='data',
                  xytext=(item, np.min(GAP_total[:, item][mask])), textcoords='data',
                  arrowprops=dict(arrowstyle="<->",
                                  connectionstyle="bar"))

    for i in range(len(GAP_total)):
        s = np.multiply(percentage_total[i],1000)
        mask = GAP_total[i] < 50
        plt.scatter(x[mask], GAP_total[i][mask], s=s, marker='o', label=cate[i])

        print("Perc", percentage_total[i])
        print("GAPt", GAP_total[i][mask])

        if category_name == 'Age':

            if i == 0:
                Percent6 = pd.DataFrame(percentage_total[i], columns=["%60-80"])
                Run1_age = pd.concat([Run1_age, Percent6.reindex(Run1_age.index)], axis=1)

                Gap6 = pd.DataFrame(GAP_total[i][mask], columns=["Gap_60-80"])
                Run1_age = pd.concat([Run1_age, Gap6.reindex(Run1_age.index)], axis=1)

            if i== 1:
                 Percent4 = pd.DataFrame(percentage_total[i], columns=["%40-60"])
                 Run1_age = pd.concat([Run1_age, Percent4.reindex(Run1_age.index)], axis=1)

                 Gap4 = pd.DataFrame(GAP_total[i][mask], columns=["Gap_40-60"])
                 Run1_age = pd.concat([Run1_age, Gap4.reindex(Run1_age.index)], axis=1)

            if i == 2:
                Percent4 = pd.DataFrame(percentage_total[i], columns=["%20-40"])
                Run1_age = pd.concat([Run1_age, Percent4.reindex(Run1_age.index)], axis=1)

                Gap4 = pd.DataFrame(GAP_total[i][mask], columns=["Gap_20-40"])
                Run1_age = pd.concat([Run1_age, Gap4.reindex(Run1_age.index)], axis=1)

            if i == 3:
                Percent8 = pd.DataFrame(percentage_total[i], columns=["%80-"])
                Run1_age = pd.concat([Run1_age, Percent8.reindex(Run1_age.index)], axis=1)

                Gap8 = pd.DataFrame(GAP_total[i][mask], columns=["Gap_80-"])
                Run1_age = pd.concat([Run1_age, Gap8.reindex(Run1_age.index)], axis=1)

            if i == 4:
                Percent0 = pd.DataFrame(percentage_total[i], columns=["%0-20"])
                Run1_age = pd.concat([Run1_age, Percent0.reindex(Run1_age.index)], axis=1)

                Gap0 = pd.DataFrame(GAP_total[i][mask], columns=["Gap_0-20"])
                Run1_age = pd.concat([Run1_age, Gap0.reindex(Run1_age.index)], axis=1)

            Run1_age.to_csv("./results/Run1_Age.csv")

        if category_name == 'Sex':

            if i == 0:
                MalePercent = pd.DataFrame(percentage_total[i], columns=["%M"])
                Run1_sex = pd.concat([Run1_sex, MalePercent.reindex(Run1_sex.index)], axis=1)

                MaleGap = pd.DataFrame(GAP_total[i][mask], columns=["Gap_M"])
                Run1_sex = pd.concat([Run1_sex, MaleGap.reindex(Run1_sex.index)], axis=1)

            else:
                FeMalePercent = pd.DataFrame(percentage_total[i], columns=["%F"])
                Run1_sex = pd.concat([Run1_sex, FeMalePercent.reindex(Run1_sex.index)], axis=1)

                FeMaleGap = pd.DataFrame(GAP_total[i][mask], columns=["Gap_F"])
                Run1_sex = pd.concat([Run1_sex, FeMaleGap.reindex(Run1_sex.index)], axis=1)

            Run1_sex.to_csv("./results/Run1_sex.csv")

    plt.xticks(x, [diseases_abbr[k] for k in diseases])
    plt.ylabel("TPR " + ylabel[category_name] + " DISPARITY")
    plt.legend()
    plt.savefig("./results/Median_Diseases_x_GAP_" + category_name + ".pdf")

def plot_sort_14(df, diseases, category, category_name):
    df_copy = df
    df = preprocess(df)
    GAP_total = []
    percentage_total = []
    cate = []
    for c in category:
        GAP_y = []
        percentage_y = []
        for d in diseases:
            pred_disease = "bi_" + d
            gt = df.loc[(df[d] == 1) & (df[category_name] == c), :]
            pred = df.loc[(df[pred_disease] == 1) & (df[d] == 1) & (df[category_name] == c), :]
            n_gt = df.loc[(df[d] == 1) & (df[category_name] != c) & (df[category_name] != 0), :]
            n_pred = df.loc[(df[pred_disease] == 1) & (df[d] == 1) & (df[category_name] != c) & (df[category_name] != 0), :]
            pi_gy = df.loc[(df[d] == 1) & (df[category_name] == c), :]
            pi_y = df.loc[(df[d] == 1) & (df[category_name] != 0), :]

            if len(gt) != 0 and len(n_gt) != 0 and len(pi_y) != 0:
                TPR = len(pred) / len(gt)
                n_TPR = len(n_pred) / len(n_gt)
                percentage = len(pi_gy) / len(pi_y)
                if category_name != 'Sex':
                    temp = []
                    for c1 in category:
                        ret = tpr(df, d, c1, category_name)
                        if ret != -1:
                            temp.append(ret)
                    temp.sort()

                    if len(temp) % 2 == 0:
                        median = (temp[(len(temp) // 2) - 1] + temp[(len(temp) // 2)])/2
                    else:
                        median = temp[(len(temp) // 2)]
                    GAP = TPR - median
                else:
                    GAP = TPR - n_TPR
                GAP_y.append(GAP)
                percentage_y.append(percentage)
            else:
                GAP_y.append(51)
                percentage_y.append(0)
        GAP_total.append(GAP_y)
        percentage_total.append(percentage_y)
        c = c.replace(' ', '_', 3)
        c = c.replace('/', '_', 3)
        cate.append(c)

    GAP_total = np.array(GAP_total)
    percentage_total = np.array(percentage_total)

    difference = {}
    for i in range(GAP_total.shape[1]):
        mask = GAP_total[:, i] < 50
        difference[diseases[i]] = np.max(GAP_total[:, i][mask]) - np.min(GAP_total[:, i][mask])
    sort = [(k, difference[k]) for k in sorted(difference, key=difference.get, reverse=False)]
    diseases = []
    for k, _ in sort:
        diseases.append(k)
    df = df_copy
    plot_14(df, diseases, category, category_name)

def preprocess(split):
    details = pd.read_csv("/scratch/gobi2/projects/ml4h/datasets/CheXpert/map.csv")
    if 'Atelectasis' in split.columns:
        details = details.drop(columns=['No Finding',
       'Enlarged Cardiomediastinum', 'Cardiomegaly', 'Lung Opacity',
       'Lung Lesion', 'Edema', 'Consolidation', 'Pneumonia', 'Atelectasis',
       'Pneumothorax', 'Pleural Effusion', 'Pleural Other', 'Fracture',
       'Support Devices'])
    split = split.merge(details, left_on="Path", right_on="Path")
    split['Age'] = np.where(split['Age'].between(0,19), 19, split['Age'])
    split['Age'] = np.where(split['Age'].between(20,39), 39, split['Age'])
    split['Age'] = np.where(split['Age'].between(40,59), 59, split['Age'])
    split['Age'] = np.where(split['Age'].between(60,79), 79, split['Age'])
    split['Age'] = np.where(split['Age']>=80, 81, split['Age'])
    split = split.replace([[None], -1, "[False]", "[True]", "[ True]", 19, 39, 59, 79, 81, 'Male', 'Female'],
                            [0, 0, 0, 1, 1, "0-20", "20-40", "40-60", "60-80", "80-", 'M', 'F'])
    return split

def random_split(map_path, total_subject_id, split_portion):
    df = pd.read_csv(map_path)
    subject_df = pd.read_csv(total_subject_id)
    subject_df['random_number'] = np.random.uniform(size=len(subject_df))

    train_id = subject_df[subject_df['random_number'] <= split_portion[0]]
    valid_id = subject_df[(subject_df['random_number'] > split_portion[0]) & (subject_df['random_number'] <= split_portion[1])]
    test_id = subject_df[subject_df['random_number'] > split_portion[1]]

    train_id = train_id.drop(columns=['random_number'])
    valid_id = valid_id.drop(columns=['random_number'])
    test_id = test_id.drop(columns=['random_number'])

    train_id.to_csv("train_id.csv", index=False)
    valid_id.to_csv("valid_id.csv", index=False)
    test_id.to_csv("test_id.csv", index=False)

    train_df = train_id.merge(df, left_on="subject_id", right_on="subject_id")
    valid_df = valid_id.merge(df, left_on="subject_id", right_on="subject_id")
    test_df = test_id.merge(df, left_on="subject_id", right_on="subject_id")

    print(len(train_df))
    print(len(valid_df))
    print(len(test_df))

    train_df.to_csv("new_train.csv", index=False)
    valid_df.to_csv("new_valid.csv", index=False)
    test_df.to_csv("new_test.csv", index=False)

# random_split("map.csv", "total_subject_id_with_gender.csv", [0.8, 0.9])


from __future__ import division
import argparse
import shutil
import re
import tempfile
import os
import itertools as iter
import subprocess
import sys
from multiprocessing import Pool
from functools import partial

def readContinuedLine(f):
    buffer = ""
    while(True):
        buffer = buffer + f.readline()
        if len(buffer) == 0 or buffer.isspace():
         continue
       # print("%s, %s"%(buffer, buffer[-2]))
        if buffer[-2] == '\\':
            buffer = buffer[:-2]
        else:
            break
    return buffer.strip()

def doCall(values, referenceFile, testFile):
   output = subprocess.check_output(["./abc", "-o", "NUL", "-c", "simulate "+' '.join(values), referenceFile]).decode("utf-8").strip()
   m = re.search("Outputs: ([01 ]+)", output)
   reference = m.group(1).split(' ')
   output = subprocess.check_output(["./abc", "-o", "NUL", "-c", "simulate "+' '.join(values), testFile]).decode("utf-8").strip()
   m = re.search("Outputs: ([01 ]+)", output)
   test = m.group(1).split(' ')
   if reference != test:
      return values
   return True

if __name__ == '__main__':
   parser = argparse.ArgumentParser(description="Exhaustively simulate a purely combinational circuit using ABC")
   parser.add_argument("reference", type=str, help="Reference blif format circuit")
   parser.add_argument("test", type=str, help="Blif format circuit to test")

   args = parser.parse_args()
   failed = False
   file = open(args.reference)
   readContinuedLine(file)
   numInputs = len(readContinuedLine(file).split(' '))-1 #-1 since it includes the .inputs
   total = 2**numInputs
   try:
      dir = tempfile.mkdtemp(dir=os.getcwd())+"/"
      shutil.copy(args.reference, dir+"in.blif") #sis has a bug where it can't handle paths with spaces. So copy the original file to a temp dir and just use that
      pool = Pool(8);
      partialCall = partial(doCall, referenceFile=dir+"in.blif", testFile=args.test)
      for i, res in enumerate(pool.imap_unordered(partialCall, iter.product("01", repeat=numInputs), 4)):
          sys.stderr.write('\r{0:%} done'.format(i/total))
          if res != True:
            print("Failed on values: "+str(res))
   finally:
      shutil.rmtree(dir)


# Time:  O(n)
# Space: O(n)
# Given a string, find the first non-repeating character in it and
# return it's index. If it doesn't exist, return -1.
#
# Examples:
#
# s = "leetcode"
# return 0.  --> since "l" only exists once in the string and is the first one, so return 0
#
# s = "loveleetcode",
# return 2.  --> since "v" only exists once in the string and is the first one, so return 2
#
# Note: You may assume the string contain only lowercase letters.

# V0
import collections
class Solution(object):
    def firstUniqChar(self, s):
        """
        :type s: str
        :rtype: int
        """
        d = collections.Counter(s)
        ans = -1
        for x, c in enumerate(s):
            if d[c] == 1:
                ans = x
                break
        return ans

# V1
# http://bookshadow.com/weblog/2016/08/21/leetcode-first-unique-character-in-a-string/
import collections
class Solution(object):
    def firstUniqChar(self, s):
        """
        :type s: str
        :rtype: int
        """
        d = collections.Counter(s)
        ans = -1
        for x, c in enumerate(s):
            if d[c] == 1:
                ans = x
                break
        return ans

# V1'
# https://www.jiuzhang.com/solution/first-unique-character-in-a-string/#tag-highlight-lang-python
class Solution:
    """
    @param str: str: the given string
    @return: char: the first unique character in a given string
    """
    def firstUniqChar(self, str):
        counter = {}

        for c in str:
            counter[c] = counter.get(c, 0) + 1

        for c in str:
            if counter[c] == 1:
                return c

# V2
class Solution(object):
    def firstUniqChar(self, s):
        s_ = ''.join(set(s))
        non_uniq_list = set([ x for x in s_ if s.count(x) > 1  ])
        for i,j in enumerate(s):
            if j in non_uniq_list:
                pass
            else:
                return i
        return -1

# V3
# IDEA : hash table
class Solution(object):
    def firstUniqChar(self, s):
        counts = {}
        for i, j in enumerate(s):
            if j in counts:
                counts[j] += 1
            else:
                counts[j] = 1
        print (counts)
        for i, j in enumerate(s):
            if counts[s[i]] == 1:
                return i
        return -1

# V4
from collections import defaultdict
class Solution(object):
    def firstUniqChar(self, s):
        """
        :type s: str
        :rtype: int
        """
        lookup = defaultdict(int)
        candidtates = set()
        for i, c in enumerate(s):
            if lookup[c]:
                candidtates.discard(lookup[c])
            else:
                lookup[c] = i+1
                candidtates.add(i+1)
        return min(candidtates)-1 if candidtates else -1


from django.shortcuts import render
from django.http import HttpResponse
import json
from django.forms.models import model_to_dict
from django.core import serializers
from django.views.decorators.csrf import csrf_exempt
import datetime
import time
from django.utils import timezone
from django.db.models import Q
from . import models

# Create your views here.

def index(request):
    articles = models.Article.objects.all()
    return render(request, 'blog/index.html', {'articles': articles})

def hello(request):
    return HttpResponse('helloworld')

def list(request):
    articles = models.Article.objects.all()
    article = models.Article.objects.get(pk=1)
    return render(request, 'blog/list.html',
                  {'articles': articles, 'article': article})

@csrf_exempt
def add(request):
    title = request.POST.get('title', 'defaultTitle')
    content = request.POST.get('content', 'defaultContent')

    pub_time = utc2local(timezone.now())
    LOCAL_FORMAT = "%Y-%m-%d %H:%M:%S"
    pub_time = pub_time.strftime(LOCAL_FORMAT)

    models.Article.objects.create(
        title=title,
        content=content,
        pub_time=pub_time)

    articles = models.Article.objects.all()
    return render(request, 'blog/index.html', {'articles': articles})

@csrf_exempt
def edit(request):
    article_id = request.POST.get('id', 0)
    title = request.POST.get('title', 'defaultTitle')
    content = request.POST.get('content', 'defaultContent')
    pub_time = utc2local(timezone.now())
    LOCAL_FORMAT = "%Y-%m-%d %H:%M:%S"
    pub_time = pub_time.strftime(LOCAL_FORMAT)

    article = models.Article.objects.get(pk=article_id)
    article.title = title
    article.content = content
    article.pub_time = pub_time
    article.save()

    articles = models.Article.objects.all()
    return render(request, 'blog/index.html', {'articles': articles})

@csrf_exempt
def delete(request):
    article_id = request.POST.get('id', 0)
    models.Article.objects.get(pk=article_id).delete()
    result = {'code': 0, 'ext': 'success'}
    return HttpResponse(json.dumps(result, ensure_ascii=False))

class DateEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, datetime.datetime):
            return obj.strftime('%Y-%m-%d %H:%M:%S')
        elif isinstance(obj, date):
            return obj.strftime('%Y-%m-%d')
        else:
            return json.JSONEncoder.default(self, obj)

def detail(request, article_id):
    article = models.Article.objects.get(pk=article_id)
    return render(request, 'blog/article.html',
                  {'article': article})

def toadd(request):
    return render(request, 'blog/add.html')

def toedit(request, article_id):
    article = models.Article.objects.get(pk=article_id)
    return render(request, 'blog/edit.html', {'article': article})

@csrf_exempt
def search(request):
    key = request.POST.get('key')
    articles = models.Article.objects.filter(Q(title__contains=key) | Q(content__contains=key))
    json_data = serializers.serialize("json", articles)
    dict_data = json.loads(json_data)
    result = {
        'code': 0,
        'ext': 'success',
        'articles': dict_data}
    return HttpResponse(json.dumps(result, ensure_ascii=False))

def utc2local(utc_st):
    # UTC时间转本地时间（+8:00）
    now_stamp = time.time()
    local_time = datetime.datetime.fromtimestamp(now_stamp)
    utc_time = datetime.datetime.utcfromtimestamp(now_stamp)
    offset = local_time - utc_time
    local_st = utc_st + offset
    return local_st


# encoding: utf-8
#
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http:# mozilla.org/MPL/2.0/.
#
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import itertools
from copy import copy
from itertools import product

from jx_base import STRUCT
from jx_base.query import QueryOp
from jx_base.schema import Schema
from jx_python import jx
from jx_python import meta as jx_base_meta
from jx_python.containers.list_usingPythonList import ListContainer
from jx_python.meta import ColumnList, metadata_columns, metadata_tables, Column, Table
from mo_dots import Data, relative_field, concat_field, SELF_PATH
from mo_dots import coalesce, set_default, Null, split_field, join_field
from mo_dots import wrap
from mo_json.typed_encoder import EXISTS_TYPE
from mo_kwargs import override
from mo_logs import Log
from mo_logs.strings import quote
from mo_threads import Queue
from mo_threads import THREAD_STOP
from mo_threads import Thread
from mo_threads import Till
from mo_times.dates import Date
from mo_times.durations import HOUR, MINUTE
from mo_times.timer import Timer
from pyLibrary.env import elasticsearch
from pyLibrary.env.elasticsearch import es_type_to_json_type

MAX_COLUMN_METADATA_AGE = 12 * HOUR
ENABLE_META_SCAN = False
DEBUG = False
TOO_OLD = 2*HOUR
OLD_METADATA = MINUTE
TEST_TABLE_PREFIX = "testing"  # USED TO TURN OFF COMPLAINING ABOUT TEST INDEXES

class FromESMetadata(Schema):
    """
    QUERY THE METADATA
    """

    def __new__(cls, *args, **kwargs):
        if jx_base_meta.singlton:
            return jx_base_meta.singlton
        else:
            jx_base_meta.singlton = object.__new__(cls)
            return jx_base_meta.singlton

    @override
    def __init__(self, host, index, alias=None, name=None, port=9200, kwargs=None):
        if hasattr(self, "settings"):
            return

        self.too_old = TOO_OLD
        self.settings = kwargs
        self.default_name = coalesce(name, alias, index)
        self.default_es = elasticsearch.Cluster(kwargs=kwargs)
        self.index_does_not_exist = set()
        self.todo = Queue("refresh metadata", max=100000, unique=True)

        self.es_metadata = Null
        self.abs_columns = set()
        self.last_es_metadata = Date.now()-OLD_METADATA

        self.meta=Data()
        table_columns = metadata_tables()
        column_columns = metadata_columns()
        self.meta.tables = ListContainer("meta.tables", [], wrap({c.names["."]: c for c in table_columns}))
        self.meta.columns = ColumnList()
        self.meta.columns.insert(column_columns)
        self.meta.columns.insert(table_columns)
        # TODO: fix monitor so it does not bring down ES
        if ENABLE_META_SCAN:
            self.worker = Thread.run("refresh metadata", self.monitor)
        else:
            self.worker = Thread.run("refresh metadata", self.not_monitor)
        return

    @property
    def query_path(self):
        return None

    @property
    def url(self):
        return self.default_es.path + "/" + self.default_name.replace(".", "/")

    def get_table(self, table_name):
        with self.meta.tables.locker:
            return wrap([t for t in self.meta.tables.data if t.name == table_name])

    def _upsert_column(self, c):
        # ASSUMING THE  self.meta.columns.locker IS HAD
        existing_columns = self.meta.columns.find(c.es_index, c.names["."])
        for canonical in existing_columns:
            if canonical.type == c.type and canonical is not c:
                set_default(c.names, canonical.names)
                for key in Column.__slots__:
                    canonical[key] = c[key]
                if DEBUG:
                    Log.note("todo: {{table}}::{{column}}", table=canonical.es_index, column=canonical.es_column)
                self.todo.add(canonical)
                break
        else:
            self.meta.columns.add(c)
            self.todo.add(c)

            if ENABLE_META_SCAN:
                if DEBUG:
                    Log.note("todo: {{table}}::{{column}}", table=c.es_index, column=c.es_column)
                # MARK meta.columns AS DIRTY TOO
                cols = self.meta.columns.find("meta.columns", None)
                for cc in cols:
                    cc.partitions = cc.cardinality = None
                    cc.last_updated = Date.now() - TOO_OLD
                self.todo.extend(cols)

    def _get_columns(self, table=None):
        # TODO: HANDLE MORE THEN ONE ES, MAP TABLE SHORT_NAME TO ES INSTANCE
        table_path = split_field(table)
        es_index = table_path[0]
        meta = self.es_metadata.indices[es_index]
        if not meta or self.last_es_metadata < Date.now() - OLD_METADATA:
            self.es_metadata = self.default_es.get_metadata(force=True)
            meta = self.es_metadata.indices[es_index]

        for data_type, properties in meta.mappings.items():
            if data_type == "_default_":
                continue
            properties.properties["_id"] = {"type": "string", "index": "not_analyzed"}
            self._parse_properties(meta.index, properties, meta)

    def _parse_properties(self, abs_index, properties, meta):
        # IT IS IMPORTANT THAT NESTED PROPERTIES NAME ALL COLUMNS, AND
        # ALL COLUMNS ARE GIVEN NAMES FOR ALL NESTED PROPERTIES
        def add_column(c, query_path):
            c.last_updated = Date.now() - TOO_OLD
            if query_path[0] != ".":
                c.names[query_path[0]] = relative_field(c.names["."], query_path[0])

            with self.meta.columns.locker:
                for alias in meta.aliases:
                    c_ = copy(c)
                    c_.es_index = alias
                    self._upsert_column(c_)
                self._upsert_column(c)

        abs_columns = elasticsearch.parse_properties(abs_index, None, properties.properties)
        self.abs_columns.update(abs_columns)
        with Timer("upserting {{num}} columns", {"num": len(abs_columns)}, debug=DEBUG):
            # LIST OF EVERY NESTED PATH
            query_paths = [[c.es_column] for c in abs_columns if c.type == "nested"]
            for a, b in itertools.product(query_paths, query_paths):
                aa = a[0]
                bb = b[0]
                if aa and bb.startswith(aa):
                    for i, b_prefix in enumerate(b):
                        if len(b_prefix) > len(aa):
                            continue
                        if aa == b_prefix:
                            break  # SPLIT ALREADY FOUND
                        b.insert(i, aa)
                        break
            for q in query_paths:
                q.append(".")
            query_paths.append(SELF_PATH)

            # ADD RELATIVE COLUMNS
            for abs_column in abs_columns:
                abs_column = abs_column.__copy__()
                abs_column.type = es_type_to_json_type[abs_column.type]
                for query_path in query_paths:
                    add_column(abs_column, query_path)
        pass

    def query(self, _query):
        return self.meta.columns.query(QueryOp(set_default(
            {
                "from": self.meta.columns,
                "sort": ["table", "name"]
            },
            _query.__data__()
        )))

    def get_columns(self, table_name, column_name=None, force=False):
        """
        RETURN METADATA COLUMNS
        """
        table_path = split_field(table_name)
        es_index_name = table_path[0]
        query_path = join_field(table_path[1:])
        table = self.get_table(es_index_name)[0]
        abs_column_name = None if column_name == None else concat_field(query_path, column_name)

        try:
            # LAST TIME WE GOT INFO FOR THIS TABLE
            if not table:
                table = Table(
                    name=es_index_name,
                    url=None,
                    query_path=None,
                    timestamp=Date.now()
                )
                with self.meta.tables.locker:
                    self.meta.tables.add(table)
                self._get_columns(table=es_index_name)
            elif force or table.timestamp == None or table.timestamp < Date.now() - MAX_COLUMN_METADATA_AGE:
                table.timestamp = Date.now()
                self._get_columns(table=es_index_name)

            with self.meta.columns.locker:
                columns = self.meta.columns.find(es_index_name, column_name)
            if columns:
                columns = jx.sort(columns, "names.\.")
                # AT LEAST WAIT FOR THE COLUMNS TO UPDATE
                while len(self.todo) and not all(columns.get("last_updated")):
                    if DEBUG:
                        Log.note("waiting for columns to update {{columns|json}}", columns=[c.es_index+"."+c.es_column for c in columns if not c.last_updated])
                    Till(seconds=1).wait()
                return columns
        except Exception as e:
            Log.error("Not expected", cause=e)

        if abs_column_name:
            Log.error("no columns matching {{table}}.{{column}}", table=table_name, column=abs_column_name)
        else:
            self._get_columns(table=table_name)  # TO TEST WHAT HAPPENED
            Log.error("no columns for {{table}}?!", table=table_name)

    def _update_cardinality(self, column):
        """
        QUERY ES TO FIND CARDINALITY AND PARTITIONS FOR A SIMPLE COLUMN
        """
        if column.es_index in self.index_does_not_exist:
            return

        if column.type in STRUCT:
            Log.error("not supported")
        try:
            if column.es_index == "meta.columns":
                with self.meta.columns.locker:
                    partitions = jx.sort([g[column.es_column] for g, _ in jx.groupby(self.meta.columns, column.es_column) if g[column.es_column] != None])
                    self.meta.columns.update({
                        "set": {
                            "partitions": partitions,
                            "count": len(self.meta.columns),
                            "cardinality": len(partitions),
                            "multi": 1,
                            "last_updated": Date.now()
                        },
                        "where": {"eq": {"es_index": column.es_index, "es_column": column.es_column}}
                    })
                return
            if column.es_index == "meta.tables":
                with self.meta.columns.locker:
                    partitions = jx.sort([g[column.es_column] for g, _ in jx.groupby(self.meta.tables, column.es_column) if g[column.es_column] != None])
                    self.meta.columns.update({
                        "set": {
                            "partitions": partitions,
                            "count": len(self.meta.tables),
                            "cardinality": len(partitions),
                            "multi": 1,
                            "last_updated": Date.now()
                        },
                        "where": {"eq": {"es_index": column.es_index, "es_column": column.es_column}}
                    })
                return

            es_index = column.es_index.split(".")[0]

            is_text = [cc for cc in self.abs_columns if cc.es_column == column.es_column and cc.type == "text"]
            if is_text:
                # text IS A MULTIVALUE STRING THAT CAN ONLY BE FILTERED
                result = self.default_es.post("/" + es_index + "/_search", data={
                    "aggs": {
                        "count": {"filter": {"match_all": {}}}
                    },
                    "size": 0
                })
                count = result.hits.total
                cardinality = 1001
                multi = 1001
            elif column.es_column == "_id":
                result = self.default_es.post("/" + es_index + "/_search", data={
                    "query": {"match_all": {}},
                    "size": 0
                })
                count = cardinality = result.hits.total
                multi = 1
            else:
                result = self.default_es.post("/" + es_index + "/_search", data={
                    "aggs": {
                        "count": _counting_query(column),
                        "multi": {"max": {"script": "doc[" + quote(column.es_column) + "].values.size()"}}
                    },
                    "size": 0
                })
                r = result.aggregations.count
                count = result.hits.total
                cardinality = coalesce(r.value, r._nested.value, r.doc_count)
                multi = coalesce(r.multi.value, 1)
                if cardinality == None:
                   Log.error("logic error")

            query = Data(size=0)

            if column.es_column == "_id":
                with self.meta.columns.locker:
                    self.meta.columns.update({
                        "set": {
                            "count": cardinality,
                            "cardinality": cardinality,
                            "multi": 1,
                            "last_updated": Date.now()
                        },
                        "clear": ["partitions"],
                        "where": {"eq": {"es_index": column.es_index, "es_column": column.es_column}}
                    })
                return
            elif cardinality > 1000 or (count >= 30 and cardinality == count) or (count >= 1000 and cardinality / count > 0.99):
                if DEBUG:
                    Log.note("{{table}}.{{field}} has {{num}} parts", table=column.es_index, field=column.es_column, num=cardinality)
                with self.meta.columns.locker:
                    self.meta.columns.update({
                        "set": {
                            "count": count,
                            "cardinality": cardinality,
                            "multi": multi,
                            "last_updated": Date.now()
                        },
                        "clear": ["partitions"],
                        "where": {"eq": {"es_index": column.es_index, "es_column": column.es_column}}
                    })
                return
            elif column.type in elasticsearch.ES_NUMERIC_TYPES and cardinality > 30:
                if DEBUG:
                    Log.note("{{field}} has {{num}} parts", field=column.es_index, num=cardinality)
                with self.meta.columns.locker:
                    self.meta.columns.update({
                        "set": {
                            "count": count,
                            "cardinality": cardinality,
                            "multi": multi,
                            "last_updated": Date.now()
                        },
                        "clear": ["partitions"],
                        "where": {"eq": {"es_index": column.es_index, "es_column": column.es_column}}
                    })
                return
            elif len(column.nested_path) != 1:
                query.aggs["_"] = {
                    "nested": {"path": column.nested_path[0]},
                    "aggs": {"_nested": {"terms": {"field": column.es_column}}}
                }
            elif cardinality == 0:
                query.aggs["_"] = {"terms": {"field": column.es_column}}
            else:
                query.aggs["_"] = {"terms": {"field": column.es_column, "size": cardinality}}

            result = self.default_es.post("/" + es_index + "/_search", data=query)

            aggs = result.aggregations._
            if aggs._nested:
                parts = jx.sort(aggs._nested.buckets.key)
            else:
                parts = jx.sort(aggs.buckets.key)

            if DEBUG:
                Log.note("{{field}} has {{parts}}", field=column.names["."], parts=parts)
            with self.meta.columns.locker:
                self.meta.columns.update({
                    "set": {
                        "count": count,
                        "cardinality": cardinality,
                        "multi": multi,
                        "partitions": parts,
                        "last_updated": Date.now()
                    },
                    "where": {"eq": {"es_index": column.es_index, "es_column": column.es_column}}
                })
        except Exception as e:
            # CAN NOT IMPORT: THE TEST MODULES SETS UP LOGGING
            # from tests.test_jx import TEST_TABLE
            TEST_TABLE = "testdata"
            is_missing_index = any(w in e for w in ["IndexMissingException", "index_not_found_exception"])
            is_test_table = any(column.es_index.startswith(t) for t in [TEST_TABLE_PREFIX, TEST_TABLE])
            if is_missing_index and is_test_table:
                # WE EXPECT TEST TABLES TO DISAPPEAR
                with self.meta.columns.locker:
                    self.meta.columns.update({
                        "clear": ".",
                        "where": {"eq": {"es_index": column.es_index}}
                    })
                self.index_does_not_exist.add(column.es_index)
            else:
                self.meta.columns.update({
                    "set": {
                        "last_updated": Date.now()
                    },
                    "clear": [
                        "count",
                        "cardinality",
                        "multi",
                        "partitions",
                    ],
                    "where": {"eq": {"names.\\.": ".", "es_index": column.es_index, "es_column": column.es_column}}
                })
                Log.warning("Could not get {{col.es_index}}.{{col.es_column}} info", col=column, cause=e)

    def monitor(self, please_stop):
        please_stop.on_go(lambda: self.todo.add(THREAD_STOP))
        while not please_stop:
            try:
                if not self.todo:
                    with self.meta.columns.locker:
                        old_columns = [
                            c
                            for c in self.meta.columns
                            if (c.last_updated == None or c.last_updated < Date.now()-TOO_OLD) and c.type not in STRUCT
                        ]
                        if old_columns:
                            if DEBUG:
                                Log.note(
                                    "Old columns {{names|json}} last updated {{dates|json}}",
                                    names=wrap(old_columns).es_column,
                                    dates=[Date(t).format() for t in wrap(old_columns).last_updated]
                                )
                            self.todo.extend(old_columns)
                            # TEST CONSISTENCY
                            for c, d in product(list(self.todo.queue), list(self.todo.queue)):
                                if c.es_column == d.es_column and c.es_index == d.es_index and c != d:
                                    Log.error("")
                        else:
                            if DEBUG:
                                Log.note("no more metatdata to update")

                column = self.todo.pop(Till(seconds=(10*MINUTE).seconds))
                if DEBUG:
                    Log.note("update {{table}}.{{column}}", table=column.es_index, column=column.es_column)
                if column:
                    if column.es_index in self.index_does_not_exist:
                        with self.meta.columns.locker:
                            self.meta.columns.update({
                                "clear": ".",
                                "where": {"eq": {"es_index": column.es_index}}
                            })
                        continue
                    if column.type in STRUCT or column.es_column.endswith("." + EXISTS_TYPE):
                        with self.meta.columns.locker:
                            column.last_updated = Date.now()
                        continue
                    elif column.last_updated >= Date.now()-TOO_OLD:
                        continue
                    try:
                        self._update_cardinality(column)
                        if DEBUG and not column.es_index.startswith(TEST_TABLE_PREFIX):
                            Log.note("updated {{column.name}}", column=column)
                    except Exception as e:
                        Log.warning("problem getting cardinality for {{column.name}}", column=column, cause=e)
            except Exception as e:
                Log.warning("problem in cardinality monitor", cause=e)

    def not_monitor(self, please_stop):
        Log.alert("metadata scan has been disabled")
        please_stop.on_go(lambda: self.todo.add(THREAD_STOP))
        while not please_stop:
            c = self.todo.pop()
            if c == THREAD_STOP:
                break

            if not c.last_updated or c.last_updated >= Date.now()-TOO_OLD:
                continue

            with self.meta.columns.locker:
                self.meta.columns.update({
                    "set": {
                        "last_updated": Date.now()
                    },
                    "clear":[
                        "count",
                        "cardinality",
                        "multi",
                        "partitions",
                    ],
                    "where": {"eq": {"es_index": c.es_index, "es_column": c.es_column}}
                })
            if DEBUG:
                Log.note("Could not get {{col.es_index}}.{{col.es_column}} info", col=c)

def _counting_query(c):
    if c.es_column == "_id":
        return {"filter": {"match_all": {}}}
    elif len(c.nested_path) != 1:
        return {
            "nested": {
                "path": c.nested_path[0]  # FIRST ONE IS LONGEST
            },
            "aggs": {
                "_nested": {"cardinality": {
                    "field": c.es_column,
                    "precision_threshold": 10 if c.type in elasticsearch.ES_NUMERIC_TYPES else 100
                }}
            }
        }
    else:
        return {"cardinality": {
            "field": c.es_column
        }}


from flask import Flask, render_template, send_from_directory
from data import db_session
from frame.bd_func import get_subjects, get_types, get_patterns
from flask import request
from frame.frame import DocGenerator
from data.user import User
from flask import redirect
from flask_login import LoginManager
from registrarion_form import RegisterForm
from login_form import LoginForm
from flask_login import login_user
from flask_login import login_required
from flask_login import logout_user
from flask_login import current_user
from threading import Thread
from clearing import clearing
from data_func import get_docs
import time
import os

app = Flask(__name__)
app.config['SECRET_KEY'] = 'mad'
login_manager = LoginManager()
login_manager.init_app(app)

clearing_thread = Thread(target=clearing)
clearing_thread.start()

@login_manager.user_loader
def load_user(user_id):
    db_sess = db_session.create_session()
    return db_sess.query(User).get(user_id)

@app.route('/')
def welcome():
    return render_template('welcome.html')

@app.route('/viewing')
def viewing():
    docs = get_docs(current_user.name if current_user.is_authenticated else 'anonym')
    return render_template('viewing.html', docs=docs)

@app.route('/logout')
@login_required
def logout():
    logout_user()
    return redirect("/")

@app.route('/login', methods=['GET', 'POST'])
def login():
    form = LoginForm()
    if form.validate_on_submit():
        db_sess = db_session.create_session()
        user = db_sess.query(User).filter(User.email == form.email.data).first()
        if user and user.check_password(form.password.data):
            login_user(user, remember=form.remember_me.data)
            return redirect("/")
        return render_template('login.html',
                               message="Неправильный логин или пароль",
                               form=form)
    return render_template('login.html', form=form)

@app.route('/create', methods=['GET', 'POST'])
def create():
    if request.method == 'GET':
        return render_template('create.html', subjects=get_subjects() + [[0, 'Все']],
                               types=get_types() + [[0, 'Все']], patterns=get_patterns())

    elif request.method == 'POST':
        try:
            if current_user.is_authenticated:
                doc_generator = DocGenerator(request.form, user=current_user.name)
            else:
                doc_generator = DocGenerator(request.form)
        except ValueError:
            return render_template('error.html')
        name = doc_generator.generate_document()
        name += '.zip'
        return send_from_directory(
            directory=f'static/generated_documents/{current_user.name if current_user.is_authenticated else "anonym"}',
            filename=name, as_attachment=True)

@app.route('/register', methods=['GET', 'POST'])
def reqister():
    form = RegisterForm()
    if form.validate_on_submit():
        if form.password.data != form.password_again.data:
            return render_template('register.html', title='Регистрация',
                                   form=form,
                                   message="Пароли не совпадают")
        db_sess = db_session.create_session()
        if db_sess.query(User).filter(User.email == form.email.data).first():
            return render_template('register.html', title='Регистрация',
                                   form=form,
                                   message="Такой пользователь уже есть")
        user = User(
            name=form.name.data,
            email=form.email.data,
            is_admin=False
        )
        user.set_password(form.password.data)
        db_sess.add(user)
        db_sess.commit()
        f = open('log/log.txt', 'a')
        data = time.ctime(time.time())
        f.write(f'register {data} | {user.name}\n')
        f.close()

        return redirect('/login')
    return render_template('register.html', title='Регистрация', form=form)

if __name__ == '__main__':
    db_session.global_init("db/blogs.db")
    port = int(os.environ.get("PORT", 5000))
    app.run(host='0.0.0.0', port=port)

# if __name__ == '__main__':
#     db_session.global_init("db/blogs.db") если запускаете сами
#     app.run(host='127.0.0.1', port=8000)


from argparse import ArgumentParser
from random import shuffle
from os import get_terminal_size
from time import time

WEIGHTS = 1000, 100, 0.001
"""The weights of dormitory, sex ratio and ability used in calculating conflicts."""

class Student:
    """Main class representing a student."""

    def __init__(self, info: str):
        """`info` should be in the format of `id name gender ability room`."""
        info = info.split()
        self.id = info[0]
        self.name = info[1]
        self.sex = get_gender(info[2])
        self.ability = int(info[3])
        self.dorm = info[4]

    def __repr__(self) -> str:
        return f"<{self.__class__.__name__} id={self.id} name={self.name} sex={'男' if self.sex else '女'} ability={self.ability} dorm={self.dorm}>"

    def __str__(self) -> str:
        return f"#{self.id}\t{self.name if len(self.name) > 3 else self.name + chr(9)}\t{'男' if self.sex else '女'}\t{self.ability}\t{self.dorm}"

def get_gender(s: str) -> bool:
    """Translate genders into booleans."""
    if s == "男":
        return True
    elif s == "女":
        return False
    else:
        raise ValueError("Invalid gender: " + s)

def from_file(path: str) -> list[Student]:
    """Generate a student list from a file."""
    result = []
    with open(path, encoding="utf-8") as f:
        for line in f.readlines():
            student = Student(line)
            result.append(student)
    return result

def average(students: list[Student], attr: str) -> float:
    """Average of given attribute."""
    sum_ = sum(map(lambda stu: getattr(stu, attr), students))
    return sum_ / len(students)

def conflict(
    students: list[Student], group_size: int, sex_ratio: float, average_ability: float
) -> float:
    """Calculate the conflict value, which consists of 3 aspects: dormitory, sex ratio and ability."""
    left = len(students) % group_size
    div = len(students) - left * group_size
    result = 0
    group_sex_ratio = 0
    group_average_ability = 0
    dorm = set()
    for i, student in enumerate(students[:div]):
        if student.dorm in dorm:
            result += WEIGHTS[0]
        else:
            dorm.add(student.dorm)
        group_sex_ratio += student.sex
        group_average_ability += student.ability
        if not (i + 1) % group_size:  # End of a group.
            group_sex_ratio /= group_size
            group_average_ability /= group_size
            result += ((group_sex_ratio - sex_ratio) ** 2) * WEIGHTS[1] + (
                (group_average_ability - average_ability) ** 2
            ) * WEIGHTS[2]
            dorm.clear()
            group_sex_ratio = 0
            group_average_ability = 0
    for i, student in enumerate(students[div:]):
        if student.dorm in dorm:
            result += WEIGHTS[0]
        else:
            dorm.add(student.dorm)
        group_sex_ratio += student.sex
        group_average_ability += student.ability
        if not (i + 1 - div) % (group_size + 1):  # End of a group.
            group_sex_ratio /= (group_size + 1)
            group_average_ability /= (group_size + 1)
            result += ((group_sex_ratio - sex_ratio) ** 2) * WEIGHTS[1] + (
                (group_average_ability - average_ability) ** 2
            ) * WEIGHTS[2]
            dorm.clear()
            group_sex_ratio = 0
            group_average_ability = 0
    return result

def min_conf_point(
    students: list[Student], target: int, base_conf: float, *statistics
) -> tuple:
    """Find the minimum conflict pos for students[target]. Returns the pos and conflict."""
    result = target
    min_conf = base_conf
    for pos in range(0, len(students), statistics[0]):
        students_ = students[:]
        students_[target], students_[pos] = students_[pos], students_[target]
        conf = conflict(students_, *statistics)
        if conf < min_conf:
            min_conf = conf
            result = pos
    return result, min_conf

def group(students: list[Student], size: int) -> float:
    """Main grouping function (in place)."""
    sex_ratio = average(students, "sex")
    average_ability = average(students, "ability")
    statistics = size, sex_ratio, average_ability
    shuffle(students)
    conf = conflict(students, *statistics)
    flag = True
    while flag:
        flag = False
        for target in range(len(students) - 1):
            pos, conf = min_conf_point(students, target, conf, *statistics)
            if pos != target:
                flag = True
                students[target], students[pos] = students[pos], students[target]
    return conf

def show(students: list[Student], group_size: int) -> None:
    """Prints the result."""
    left = len(students) % group_size
    div = len(students) - left * group_size
    cnt = 0
    group_id = 0
    term_size = get_terminal_size()
    print(f"ID\tName\t\tSex\tAbility\tDormitory")
    print("-" * term_size.columns, end="")
    for i, student in enumerate(students):
        if not cnt:
            group_id += 1
            print(f"\nGroup {group_id}:")
        cnt = (cnt + 1) % (group_size + (i > div))
        print(student)
    print("-" * term_size.columns)

def save_as_text(students: list[Student], group_size: int, file_path: str) -> None:
    """Save the result in a txt file."""
    left = len(students) % group_size
    div = len(students) - left * group_size
    cnt = 0
    group_id = 0
    with open(file_path, "w", encoding="utf-8") as f:
        f.write("Group Id Name\n")
        for i, student in enumerate(students):
            if not cnt:
                group_id += 1
            f.write(f"{group_id} {student.id} {student.name}\n")
            cnt = (cnt + 1) % (group_size + (i > div))

if __name__ == "__main__":
    parser = ArgumentParser(description="Simple grouping program.")
    parser.add_argument("-i", "--input", help="Input file path.", default=None)
    parser.add_argument(
        "-s", "--size", help="The expected size of a group.", type=int, default=0
    )
    parser.add_argument(
        "-t",
        "--txt",
        help="Output as txt file at provided path.",
        required=False,
        default=None,
    )
    parser.add_argument(
        "-q",
        "--quiet",
        help="Disable outputting result on terminal.",
        action="store_true",
    )
    args = parser.parse_args()
    if not args.input:
        args.input = input("Please provide the path of input file: ")
    if not args.size:
        args.size = int(input("Please specify the expected size of a group: "))
    start = time()
    print(f'Loading data from "{args.input}"...')
    students = from_file(args.input)
    print(f"Dividing into groups of {args.size}...")
    conf = group(students, args.size)
    if not args.quiet:
        show(students, args.size)
    print("Conflict value:", conf)
    end = time()
    print(f"Time used: {end - start:.5f}s.")
    if args.txt:
        print(f'Saving as txt to "{args.txt}"...')
        save_as_text(students, args.size, args.txt)


# -*- coding: utf-8 -*-
"""
Figure 12b
"""

import matplotlib.pyplot as plt
import pandas as pd

path1 = './inter_games/allshader_gspan_output/25percent_result/intergame_all_node_occurance_gspan_fsm2.csv'
df = pd.read_csv(path1 , header = 'infer')

list_to_plot_nodesize = []

game_set = df.drop_duplicates(subset=['gameid'])['gameid']
print(len(game_set))

for frame in game_set:
    print(frame)
    node_sort = df[df.gameid == frame]
    node_sort = node_sort.sort_values(by=['nodesize'])
    node_set = node_sort.drop_duplicates(subset=['nodesize'])['nodesize'].sort_values(ascending=True)
    # x = np.cumsum(node_sort['occurance'])
    # node_sort['cum_occ'] = x

    temp1 =pd.DataFrame(columns = ['nodesize', 'occurance', 'frame'])
    for each in node_set:
        temp = node_sort[node_sort.nodesize == each]
        temp1 = temp1.append(temp)
    list_to_plot_nodesize.append(temp1['nodesize'])

# Create a figure instance
fig = plt.figure(1, figsize=(12,7))
labelsize=25
ax = fig.add_subplot(111)
bp = ax.boxplot(list_to_plot_nodesize)
ax.set_ylabel("Avg # nodes in each frequent subgraph", fontsize=labelsize)
ax.set_xlabel("game ID", fontsize=labelsize)
ax.tick_params(size=labelsize-20)
ax.tick_params(labelsize=labelsize-3)
ax.xticks=([i+1 for i in range(len(game_set))])
ax.set_xticklabels(ax.set_xticklabels(game_set,ha='center'))

for tick in ax.get_xticklabels():
    tick.set_rotation(60)


"""Integration test for TorizonCore Builder

Note: These tests heavily rely on the environment:
- /workdir needs to exist, and a Tezi image needs to be in /workdir/tezi
- fast-banana.png need to exist for the splash screen test
- /storage needs to exist, /storage/pytest will be used for testing
"""

import logging

import py
import pytest

from tcbuilder.backend import deploy, ostree, splash, images

@pytest.fixture(name="storage_dir", scope="module")
def fixture_storage_dir():
    """Initialize/cleanup storage directory for pytest"""
    storage_dir = py.path.local("/storage/pytest")
    if not storage_dir.isdir():
        storage_dir.mkdir()
    assert not storage_dir.join("tezi").isdir()
    assert not storage_dir.join("sysroot").isdir()
    assert not storage_dir.join("ostree-archive").isdir()
    assert not storage_dir.join("splash").isdir()

    yield storage_dir

    # /storage is bind mounted, we should not rm -rf it...
    logging.info(f"Cleaning up {str(storage_dir)}...")
    storage_dir.join("tezi").remove(rec=1)
    storage_dir.join("sysroot").remove(rec=1)
    storage_dir.join("ostree-archive").remove(rec=1)
    storage_dir.join("splash").remove(rec=1)

@pytest.fixture(name="deploy_dir", scope="module")
def fixture_deploy_dir():
    """Initialize deploy directory for pytest"""
    deploy_dir = py.path.local("/deploy")
    assert not deploy_dir.join("sysroot").isdir()
    deploy_dir.join("sysroot").mkdir()

    yield deploy_dir

    logging.info(f"Cleaning up {str(deploy_dir)}...")
    deploy_dir.join("sysroot").remove(rec=1)

def test_unpack(storage_dir, work_dir):
    """"Test unpack sub-command"""

    tezi = storage_dir.join("tezi")
    sysroot_dir = storage_dir.join("sysroot")
    ostree_archive = storage_dir.join("ostree-archive")

    tezi_src = work_dir.join("tezi")

    images.import_local_image(
        str(tezi_src), str(tezi), str(sysroot_dir), str(ostree_archive))

    sysroot = ostree.load_sysroot(str(sysroot_dir))
    csum, _kargs = ostree.get_deployment_info_from_sysroot(sysroot)

    print(csum)
    assert len(csum) > 1

    config = ostree_archive.join("config")
    assert "mode=archive-z2" in config.read()

def test_splash(storage_dir, work_dir):
    """"Test splash sub-command"""

    splash_work_dir = storage_dir.join("splash")
    ostree_archive = storage_dir.join("ostree-archive")

    splashimage = str(work_dir.join("fast-banana.png"))
    logging.info(f"Using {splashimage} as splash screen image.")
    splash.create_splash_initramfs(str(splash_work_dir), str(splashimage), str(ostree_archive))

def test_deploy_using_csum(storage_dir, deploy_dir, work_dir):
    """"Test deploy Tezi image sub-command"""

    tezi = storage_dir.join("tezi")
    sysroot_dir = storage_dir.join("sysroot")
    ostree_archive = storage_dir.join("ostree-archive")

    deploy_sysroot = deploy_dir.join("sysroot")

    sysroot = ostree.load_sysroot(str(sysroot_dir))
    csum, _kargs = ostree.get_deployment_info_from_sysroot(sysroot)

    output = work_dir.join("output")

    deploy.deploy_tezi_image(str(tezi), str(sysroot_dir), str(ostree_archive),
                             str(output), str(deploy_sysroot), csum)

    files = [str(outfile) for outfile in output.listdir()]

    assert any(("image.json" in outfile) for outfile in files)
    assert any(("bootfs.tar" in outfile) for outfile in files)


from typing import Callable

from ..components import Box, Button, Column, Divider, Row
from ..database import API

class ViewModel:
    id: str
    update: Callable

class ViewController:
    def __init__(self, database: API) -> None:
        self.database = database

        self.__models: dict[str, ViewModel] = {}
        self.__navegation: list[ViewModel] = []

        self.__main_part1 = Column(
            [
                Button("Vendas", lambda _: self.redirect_to("sales")),
                Button("Produtos", lambda _: self.redirect_to("products")),
                Button("Relatórios", lambda _: self.redirect_to("reports")),
            ],
            style={"flex": 1},
        )
        self.__main_part2 = Box()
        self.__main = Row(
            [
                self.__main_part1,
                Divider("v", {"padding_left": 10, "padding_right": 10}),
                self.__main_part2,
            ],
            {"flex": 1, "padding": 10},
        )

    def main(self):
        return self.__main

    def update_main(self, widgets: list):
        self.__main.remove(self.__main_part2)
        self.__main_part2 = Column(widgets, {"flex": 6})
        self.__main.add(self.__main_part2)

    def register_model(self, model: ViewModel):
        self.__models[model.id] = model

    def update_model(self, custom_data=None):
        if not self.__navegation:
            return

        m_model = self.__navegation[-1]

        if custom_data:
            m_model.update(custom_data)
        else:
            m_model.update()

    def redirect_to(self, model_id: str, custom_data=None):
        if model_id not in self.__models:
            return

        if self.__navegation and model_id == self.__navegation[-1].id:
            return

        if len(self.__navegation) > 50:
            self.__navegation = self.__navegation[25:]

        self.__navegation.append(self.__models[model_id])
        self.update_model(custom_data)

    def redirect_to_previous(self):
        if not self.__navegation:
            return

        self.__navegation.pop()
        self.update_model()


# This Python file uses the following encoding: utf-8

# Függvénykönyvtárak betöltése:
from sense_hat import SenseHat # uncomment on rPI
import pygame #pygame eseménykezelés miatt
from pygame.mixer import music as mp3 #zene lejátszáshoz
import os # mappa(ák) fájl(ok) kezelése
import time # log timestamp miatt
import datetime # log timestamp miatt
from time import sleep
import random # shuffle funkció miatt
import sys # csript mappájának meghatározása, szolgáltatásként futattva (sys.path[0])

# LOG FIle kezelés:
timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H.%M.%S')
logpath=os.path.join(sys.path[0], "log", "rPI_music_box_")+timestamp+".log"
def newlogline(string):
    with open(logpath, "a") as log:
        log.write(datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')+" - ")
        log.write(string+"\n")

# Változók és konstansok definiálása:
index=0 # első zeneszám beállítása
vol=0.5 # kezdeti nagerő érték
playing = "" # kezdeti lejtászás státusz

g = (0, 100, 0) # Green
b = (0, 0, 0) # Black

musicpath=os.path.join(sys.path[0], "music") # platform függtlen elérési út definiálása
l = [] # számok listája

SONG_END = pygame.USEREVENT + 1 # szám vége esemény definiálása
mp3.set_endevent(SONG_END)

# Fügvények:
def displayCL(): #conslo-ra és logba írás egyszerűsítése
    global l
    global index
    newlogline("Playing:")
    newlogline(l[index])
    print("Playing:")
    print(l[index])
    print("Next:")
    newlogline("Next:")
    if index == lSize-1: # index határérték kezelése
        print(l[0])
        newlogline(l[0])
    elif index < 0:
        print(l[lSize-1])
        newlogline(l[lSize-1])
    else:
        print(l[index+1])
        newlogline(l[index+1])

def play_pause():
    global playing
    if playing == True:
        mp3.pause()
        playing = False
        print("Paused")
        newlogline("Paused")
        displayH("pause")
    elif playing == False:
        mp3.unpause()
        playing = True
        displayCL()
        displayH("play")
    else:
        mp3.load(l[index])  # az első szám betöltése
        mp3.play()
        playing = True
        displayH("play")
        displayCL()

def next():
    global index
    global playing
    displayH("next")
    mp3.load(l[index])
    mp3.play()
    playing = True
    displayCL()
    sleep(1)
    displayH("play")

def previous():
    global index
    global playing
    displayH("previous")
    mp3.load(l[index])
    mp3.play()
    playing = True
    displayCL()
    sleep(1)
    displayH("play")

def shuffle(): #5 zene lista megkeverése
    global playing
    global index
    random.shuffle(l)
    index = 0
    mp3.load(l[index])
    mp3.play()
    playing = True
    displayH("shuffle") # uncomment on rPI
    newlogline("rPI Shuffled")
    print("rPI Shuffled")
    displayCL()
    sleep(1)
    displayH("play") # uncomment on rPI

def volup():
    global vol
    global playing
    vol += 0.02
    mp3.set_volume(vol)
    #cvol = mp3.get_volume()
    newlogline("Volume up")
    print(mp3.get_volume())
    sense.show_letter("+",text_colour=g,back_colour=b)
    sleep(0.3)
    if playing == False:
        displayH("pause")
    elif playing == True:
        displayH("play")
    else:
        displayH("smile")

def voldown():
    global vol
    global playing
    vol -= 0.02
    mp3.set_volume(vol)
    #cvol = mp3.get_volume()
    newlogline("Volume down")
    print(mp3.get_volume())
    sense.show_letter("-",text_colour=g,back_colour=b)
    sleep(0.3)
    if playing == False:
        displayH("pause")
    elif playing == True:
        displayH("play")
    else:
        displayH("smile")

def displayH(action): # logók mghivása
    sense.clear()
    if action == "play":
        pixels = [
            b, b, b, b, b, b, b, b,
            b, b, g, b, b, b, b, b,
            b, b, g, g, b, b, b, b,
            b, b, g, g, g, b, b, b,
            b, b, g, g, b, b, b, b,
            b, b, g, b, b, b, b, b,
            b, b, b, b, b, b, b, b,
            b, b, b, b, b, b, b, b
        ]
    elif action == "pause":
        pixels = [
            b, b, b, b, b, b, b, b,
            b, g, g, b, g, g, b, b,
            b, g, g, b, g, g, b, b,
            b, g, g, b, g, g, b, b,
            b, g, g, b, g, g, b, b,
            b, g, g, b, g, g, b, b,
            b, b, b, b, b, b, b, b,
            b, b, b, b, b, b, b, b
        ]
    elif action == "next":
        pixels = [
            b, b, b, b, b, b, b, b,
            b, b, b, b, b, g, b, b,
            b, g, b, b, b, g, g, b,
            b, g, g, b, b, g, g, g,
            b, g, g, g, b, g, g, b,
            b, g, g, b, b, g, b, b,
            b, g, b, b, b, b, b, b,
            b, b, b, b, b, b, b, b
        ]
    elif action == "previous":
        pixels = [
            b, b, b, b, b, b, b, b,
            b, b, g, b, b, b, b, b,
            b, g, g, b, b, b, g, b,
            g, g, g, b, b, g, g, b,
            b, g, g, b, g, g, g, b,
            b, b, g, b, b, g, g, b,
            b, b, b, b, b, b, g, b,
            b, b, b, b, b, b, b, b
        ]
    elif action == "shuffle":
        pixels = [
            b, b, b, b, b, b, g, b,
            b, b, b, b, g, g, g, g,
            g, g, b, g, b, b, g, b,
            b, b, g, b, b, b, b, b,
            b, b, g, b, b, b, b, b,
            g, g, b, g, b, b, g, b,
            b, b, b, b, g, g, g, g,
            b, b, b, b, b, b, g, b
        ]
    elif action == "smile":
        pixels = [
            b, b, b, b, b, b, b, b,
            b, b, b, b, b, b, b, b,
            b, b, g, b, g, b, b, b,
            b, b, b, b, b, b, b, b,
            b, g, b, b, b, g, b, b,
            b, b, g, g, g, b, b, b,
            b, b, b, b, b, b, b, b,
            b, b, b, b, b, b, b, b
        ]
    sense.set_pixels(pixels)

# Zeneszámok betöltése, rendezése, lejátszó inicializálása
if os.path.isdir(musicpath) == True: # zene könyvtár meglétének ellenőrzése
    print("==========================================")
    print("The music folder exists")
    print("==========================================")
    newlogline(("=========================================="))
    newlogline("The music folder exists")
    newlogline("==========================================")
    os.chdir(musicpath) # könyvtár váltás, hogy ne kelljen elérési utat definiálni a zenék lejátszásakor
    for files in os.listdir(musicpath): # zene könyvtár listázása
        if files.endswith(".mp3"): # mp3 kiterejsztésű fájlokra szűrés
            l.append(files) # lejatászi lista létrehozása
            print(files)
            newlogline(files)
    l.sort() # számok sorba rendezése
    lSize=len(l) # számok darabszámának tárolása
    newlogline("")
    newlogline("==========================================")
    print("")
    print("==========================================")

    #START - SenseHat event handler
    sense = SenseHat() # semse Hat inicializálása
    # SYSTEMD BugFix
    import signal
    def handler(signum, frame):
        pass

    try:
        signal.signal(signal.SIGHUP, handler)
    except AttributeError:
        pass
    #_________________
    pygame.init() # pygame importált moduljaninak inicializálása
    mp3.set_volume(vol) # kezdeti hangerő beállítása
    sense.show_message("Welcome! Lets start listening...", scroll_speed=0.08, text_colour=g, back_colour=b) # üdvözlő üzenet
    displayH("smile") # smile jelzi, hogy betöltött a program
    while True:
        acceleration = sense.get_accelerometer_raw() # gyorsulámérő gyers adatok lekérése
        x = abs(acceleration['x'])
        y = abs(acceleration['y'])
        z = abs(acceleration['z'])
        if x > 2 or y > 2 or z > 2: # az eszköz "rázásának" érzékelése
            newlogline("==========================================")
            print("==========================================")
            newlogline("Accelerometer triggered shake")
            shuffle()
        else:
            for event in sense.stick.get_events(): # senseHat események figyelése
                if event.action == "pressed":
                    newlogline("==========================================")
                    print("==========================================")
                    if event.direction == "up":
                        newlogline("Joystick up pressed")
                        volup()
                    elif event.direction == "down":
                        newlogline("Joystick down pressed")
                        voldown()
                    elif event.direction == "left":
                        newlogline("Joystick left pressed")
                        index -= 1
                        if index < 0: # index határérték kezelése
                            index = lSize-1
                        previous()
                    elif event.direction == "right":
                        newlogline("Joystick right pressed")
                        index += 1
                        if index == lSize: # index határérték kezelése
                            index = 0
                        next()
                    elif event.direction == "middle":
                        newlogline("Joystick middle pressed")
                        play_pause()
            for event in pygame.event.get(): # pygame események figyelése
                if event.type == SONG_END: # szám végének figyelése
                    newlogline("==========================================")
                    newlogline("The song ended!")
                    print("==========================================")
                    print("The song ended!")
                    index += 1
                    if index == lSize: # index határérték kezelése
                        index = 0
                    next()
    #END - SenseHat event handler
else:
    newlogline("The music folder is not exists")
    print("The music folder is not exists")


"""Restructure Document Indices

Revision ID: 8aabb57f3b49
Revises: 5e84129c8be3
Create Date: 2023-08-18 21:15:57.629515

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = "8aabb57f3b49"
down_revision = "5e84129c8be3"
branch_labels = None
depends_on = None

def upgrade() -> None:
    op.drop_table("chunk")
    op.execute("DROP TYPE IF EXISTS documentstoretype")

def downgrade() -> None:
    op.create_table(
        "chunk",
        sa.Column("id", sa.VARCHAR(), autoincrement=False, nullable=False),
        sa.Column(
            "document_store_type",
            postgresql.ENUM("VECTOR", "KEYWORD", name="documentstoretype"),
            autoincrement=False,
            nullable=False,
        ),
        sa.Column("document_id", sa.VARCHAR(), autoincrement=False, nullable=False),
        sa.ForeignKeyConstraint(
            ["document_id"], ["document.id"], name="chunk_document_id_fkey"
        ),
        sa.PrimaryKeyConstraint("id", "document_store_type", name="chunk_pkey"),
    )


import logging
import time

logging.basicConfig(
    filename="metric.log",
    encoding="utf-8",
    level=logging.DEBUG,
    format="%(asctime)s:%(levelname)s:%(name)s:%(message)s",
)

class MetricMiddleware:
    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        start_time = time.perf_counter()

        response = self.get_response(request)

        end_time = time.perf_counter()

        total_time = end_time - start_time

        logger = logging.getLogger("debug")
        logger.info(f"Total time: {(total_time):.2f}s")
        print(f"Total time: {(total_time):.2f}s")

        return response


# script to run once every day
import requests, lxml
from django.utils import timezone
from bs4 import BeautifulSoup
from decimal import Decimal
from django.core.management import BaseCommand
from Xeemit.models import CurrencyData

class Command(BaseCommand):
    help = "This gathers data on mid-market rates for currencies and updates table with latest rates from www.xe.com"

    def handle(self, *args, **options):
        currency_list = ["AED","AFN","ALL","AMD","ANG","AOA","ARS","AUD","AWG","AZN","BAM","BBD","BDT","BGN","BHD","BIF","BMD","BND","BOB","BRL","BSD","BTN","BWP","BYN","BZD","CAD","CDF","CHF","CLP","CNY","COP","CRC","CUC","CUP","CVE","CZK","DJF","DKK","DOP","DZD","EGP","ERN","ETB","EUR","FJD","FKP","GBP","GEL","GGP","GHS","GIP","GMD","GNF","GTQ","GYD","HKD","HNL","HRK","HTG","HUF","IDR","ILS","IMP","INR","IQD","IRR","ISK","JEP","JMD","JOD","JPY","KES","KGS","KHR","KMF","KPW","KRW","KWD","KYD","KZT","LAK","LBP","LKR","LRD","LSL","LYD","MAD","MDL","MGA","MKD","MMK","MNT","MOP","MRO","MUR","MVR","MWK","MXN","MYR","MZN","NAD","NGN","NIO","NOK","NPR","NZD","OMR","PAB","PEN","PGK","PHP","PKR","PLN","PYG","QAR","RON","RSD","RUB","RWF","SAR","SBD","SCR","SDG","SEK","SGD","SHP","SLL","SOS","SPL*","SRD","STD","SVC","SYP","SZL","THB","TJS","TMT","TND","TOP","TRY","TTD","TVD","TWD","TZS","UAH","UGX","USD","UYU","UZS","VEF","VND","VUV","WST","XAF","XCD","XDR","XOF","XPF","YER","ZAR","ZMW","ZWD"]

        for currency_code in currency_list:
            try:
                today = timezone.now()
                response = requests.get('http://www.xe.com/currencyconverter/convert/?Amount=1&From=USD&To='+currency_code)
                if response.status_code==200:
                    soup = BeautifulSoup(response.text, "lxml")
                    curr_value = soup.find('span', {'class': 'uccResultAmount'})
                    if curr_value:
                        obj, created = CurrencyData.objects.get_or_create(currency_code=currency_code)
                        obj.currency_rate = Decimal(curr_value.text.replace(',', ''))
                        obj.dateUpdated = today
                        obj.save()

            except:
                with open("error.log", 'a+') as f:
                    f.write(str(timezone.now())+"  ---  Error occurred when doing: "+currency_code)


from src.shared.string import EMPTY
from .reagent import Reagent
from .utils import closest_target, count_free_babies, group_target, paths_to_target
from ..shared import count_dirt, get_length, ROBOT, DIRT, BABY, CORRAL, OBSTACLE
from ..logging import LoggerFactory as Logger

log = None

CAUTIOUS = 'CAUTIOUS'
DARED    = 'DARED'

class Practical(Reagent):
    '''
    Practical reasoning agent. Model-based and goal-based reflex agent
    '''
    def __init__(self, pos, t):
        global log
        log = Logger('Kindergarden').getChild('Practical')
        self.t = 0
        self.time = 0
        self.babies = 0
        self.mode = ''
        self.drop = False
        self.old_pos = pos
        super().__init__(pos)

    def action(self, house):
        beliefs   = self.get_beliefs(house)
        options   = self.get_options(beliefs)
        operation = self.filter_options(options)
        self.execute(operation, house)

    def get_beliefs(self, house):
        '''
        Get state of the environment detected by the agent given the latest perception
        '''
        beliefs = dict()
        self.time += 1

        #Learn t
        if self.old_pos != self.pos:
            t = self.t
            self.t = self.time - self.t - 1
            if self.t == 0:
                self.t = t
            self.time = 1
            log.debug(f't value: {self.t}')
        #Top priority intentions
        if self.will_clean or self.carrying_baby:
            return beliefs

        log.info('Updating state of the environment...', 'get_beliefs')
        self.babies = count_free_babies(house)
        n, m = get_length(house)
        beliefs['mess'] = count_dirt(house) * 100 // (n * m)
        beliefs['dirt_groups'] = group_target(house, DIRT, self.t // 2 if self.t > 0 else 10)
        beliefs['baby_groups'] = group_target(house, BABY, 2)

        paths_to_babies = list()
        paths_to_dirt = list()

        max_path = len(closest_target(house, self.pos, DIRT, [OBSTACLE]))
        max_path *= 2
        paths_to_target(house, self.pos, self.pos, DIRT, {self.pos}, dict(), [OBSTACLE, BABY], paths_to_dirt, max_path)

        max_path = len(closest_target(house, self.pos, BABY, [OBSTACLE]))
        max_path += self.t // 2 if self.t > 0 else 5
        paths_to_target(house, self.pos, self.pos, BABY, {self.pos}, dict(), [OBSTACLE], paths_to_babies, max_path)
        # log.debug(f'paths_to_babies: {paths_to_babies}')
        # log.debug(f'paths_to_dirt: {paths_to_dirt}')

        dirt_in_path_to_babies = list()
        for p in paths_to_babies:
            dirt = 0
            for (i, j) in p:
                dirt += 1 if house[i][j].value == DIRT else 0
            dirt_in_path_to_babies.append(dirt)
        beliefs['dirt_in_path_to_babies'] = dirt_in_path_to_babies
        #log.debug(f'dirt_in_path_to_babies: {dirt_in_path_to_babies}')

        beliefs['paths_to_babies'] = paths_to_babies
        beliefs['paths_to_dirt'] = paths_to_dirt
        return beliefs

    def get_options(self, beliefs):
        '''
        Get options(plans) available for the agent given its actual beliefs and its intentions
        '''
        #Top priority intentions
        if self.will_clean or self.carrying_baby:
            return self.options

        options = beliefs

        #Get best path to dirt
        considering = list()
        for p in options['paths_to_dirt']:
            for g in options['dirt_groups']:
                last = list(p)[-1]
                if last in g:
                    #(length of path, concentration of dirt, path)
                    considering.append((len(p), -len(g), p))
                    break
        considering.sort()
        #log.debug(f'Best paths to dirt: {considering}')
        options['best_path_dirt'] = None if considering == [] else considering[0][2]

        #Get best path to baby sorted by occurrence of dirt in the path
        considering.clear()
        for p, d in zip(options['paths_to_babies'], options['dirt_in_path_to_babies']):
            #(length of path, number of dirt in path, path)
            considering.append((len(p), -d, p))
        considering.sort()
        #log.debug(f'Best dirty paths to babies: {considering}')
        options['best_dirty_path_babies'] = None if considering == [] else considering[0][2]

        #Get best path to baby
        considering.clear()
        for p in options['paths_to_babies']:
            for g in options['baby_groups']:
                last = list(p)[-1]
                if last in g:
                    #(length of path, concentration of babies, path)
                    considering.append((len(p), -len(g), p))
                    break
        considering.sort()
        #log.debug(f'Best paths to babies: {considering}')
        options['best_path_babies'] = None if considering == [] else considering[0][2]
        self.options = options
        return options

    def filter_options(self, options):
        '''
        Update intentions and get an action to execute
        '''
        #Top priority intentions
        if self.carrying_baby:
            if self.drop:
                self.mode = CAUTIOUS
                return ('DROP', [])
            return ('GOTO_CORRAL', [])
        if self.will_clean:
            self.garbage_collected += 1
            self.will_clean = False
            return ('CLEAN', [])

        #Goal selection
        if options['mess'] >= 40 or self.babies == 0:
            #CAUTIOUS mode
            self.mode = CAUTIOUS
            log.debug(f'Current mode: CAUTIOUS')
            actions = list()
            if options['best_path_dirt']:
                actions.append(options['best_path_dirt'].copy())
                self.options['best_path_dirt'].pop(0)
            if options['best_dirty_path_babies']:
                actions.append(options['best_dirty_path_babies'].copy())
                self.options['best_dirty_path_babies'].pop(0)
            if actions == []:
                return ('BLOCK', [])
            else:
                actions.sort()
                return ('MOVE', actions[0])
        else:
            #DARED mode
            self.mode = DARED
            log.debug(f'Current mode: DARED')
            if options['best_path_babies']:
                opt = options['best_path_babies'].copy()
                return ('MOVE', opt)
            return ('BLOCK', [])

    def execute(self, operation, house):
        '''
        Execute a action given the planned operation
        '''
        x, y = self.pos
        task, args = operation

        if task is 'CLEAN':
            house[x][y].update(ROBOT)
            house[x][y].dirty = False
            log.debug(f'Dirt cleaned in ({x}, {y})', 'CMD: CLEAN')
        elif task is 'GOTO_CORRAL':
            log.debug('Executing task GOTO_CORRAL', 'execute')
            if CORRAL in house[x][y].value and not f'{CORRAL}-{BABY}' in house[x][y].value:
                self.carrying_baby = False
                house[x][y].update(f'{BABY}-{ROBOT}')
                log.debug(f'Dropped baby in corral at ({x}, {y})', 'CMD: GOTO_CORRAL')
            else:
                if not self.try_move(house, CORRAL, [OBSTACLE, BABY]):
                    self.drop = True
        elif task is 'DROP':
            if DIRT in house[x][y].value:
                self.try_move(house, EMPTY, [OBSTACLE, BABY, DIRT])
            self.drop = False
            self.carrying_baby = False
            house[x][y].update(f'{BABY}-{ROBOT}')
            log.debug(f'Dropped baby at ({x}, {y})', 'CMD: DROP')
        else:
            if self.mode is CAUTIOUS:
                self.execute_move(operation, house)
                x, y = self.pos
                if DIRT in house[x][y].value:
                    self.will_clean = True
                    log.debug(f'I\'ll clean dirt on ({x}, {y}) in the next turn', 'execute')
            elif self.mode is DARED:
                self.execute_move(operation, house)
                x, y = self.pos

            if BABY in house[x][y].value and not house[x][y].isFixed:
                self.carrying_baby = True
                house[x][y].update(f'{ROBOT}-{BABY}')
                log.debug(f'Loaded baby on ({x}, {y})', 'execute')

        self.old_pos = self.pos

    def execute_move(self, operation, house):
        task, args = operation
        if task is 'MOVE':
            log.debug('Executing task MOVE', 'execute_move')
            self.move(house, args)
        else:
            log.debug(f'I can\'t move!!! Analyzing options... Meanwhile waiting for an environment change', 'execute_move')

    def __str__(self):
        return 'Practical'


from __future__ import absolute_import
from __future__ import unicode_literals
import unittest
import mock

from google.datalab import Context
from google.datalab.utils import _utils as du

class TestCases(unittest.TestCase):

  def test_credentials(self):
    dummy_creds = {}
    c = Context('test_project', credentials=dummy_creds)

    self.assertEqual(c.credentials, dummy_creds)

    dummy_creds = {'test': 'test'}
    c.set_credentials(dummy_creds)
    self.assertEqual(c.credentials, dummy_creds)

  def test_config(self):
    dummy_config = {}
    c = Context('test_project', credentials=None, config=dummy_config)

    self.assertEqual(c.config, dummy_config)

    dummy_config = {'test': 'test'}
    c.set_config(dummy_config)
    self.assertEqual(c.config, dummy_config)

    c = Context('test_project', None, None)
    self.assertEqual(c.config, Context._get_default_config())

  def test_project(self):
    dummy_project = 'test_project'
    c = Context(dummy_project, credentials=None, config=None)

    self.assertEqual(c.project_id, dummy_project)

    dummy_project = 'test_project2'
    c.set_project_id(dummy_project)
    self.assertEqual(c.project_id, dummy_project)

    c = Context(None, None, None)
    with self.assertRaises(Exception):
      print(c.project_id)

  @mock.patch('google.datalab.utils._utils.get_credentials')
  @mock.patch('google.datalab.utils._utils.get_default_project_id')
  @mock.patch('google.datalab.utils._utils.save_project_id')
  def test_default_project(self, mock_save_project_id, mock_get_default_project_id,
                           mock_get_credentials):
    # verify setting the project on a default Context object sets
    # the global default project

    global default_project
    default_project = ''

    def save_project(project=None):
      global default_project
      default_project = project

    def get_project():
      global default_project
      return default_project

    mock_save_project_id.side_effect = save_project
    mock_get_default_project_id.side_effect = get_project
    mock_get_credentials.return_value = ''

    c = Context.default()
    dummy_project = 'test_project3'
    c.set_project_id(dummy_project)
    self.assertEqual(du.get_default_project_id(), dummy_project)

  @mock.patch('google.datalab.utils._utils.get_credentials')
  def test_is_signed_in(self, mock_get_credentials):
    mock_get_credentials.side_effect = Exception('No creds!')
    self.assertFalse(Context._is_signed_in())

    mock_get_credentials.return_value = {}
    mock_get_credentials.side_effect = None
    self.assertTrue(Context._is_signed_in())

  @mock.patch('google.datalab.utils._utils.get_credentials')
  @mock.patch('google.datalab.utils._utils.get_default_project_id')
  @mock.patch('google.datalab.utils._utils.save_project_id')
  def test_default_context(self, mock_save_project_id, mock_get_default_project_id,
                           mock_get_credentials):

    mock_get_default_project_id.return_value = 'default_project'
    mock_get_credentials.return_value = ''

    c = Context.default()
    default_project = c.project_id
    self.assertEqual(default_project, 'default_project')

    # deliberately change the default project and make sure it's reset
    c.set_project_id('test_project4')

    self.assertEqual(Context.default().project_id, 'default_project')


import hashlib
import json
import os
import sys

class Block:
    def __init__(self, prev_block_hash: str, block_number: int, transactions: list):
        self._transactions = transactions
        self._block_number = block_number
        self._prev_block_hash = prev_block_hash
        self._block_hash = self.calculate_hash()

    def calculate_hash(self):
        hash_string = '-'.join(self._transactions) + self._prev_block_hash + str(self._block_number)
        hash_encoded = json.dumps(hash_string).encode()
        return hashlib.sha256(hash_encoded).hexdigest()

    @property
    def block_number(self):
        return self._block_number

    @property
    def prev_block_hash(self):
        return str(self._prev_block_hash)

    @property
    def block_hash(self):
        return str(self.calculate_hash())

    @property
    def header(self):
        return {
            'block_number': self._block_number,
            'prev_block_hash': self.prev_block_hash,
            'block_hash': self.block_hash
        }

    @property
    def body(self):
        return self._transactions

class Blockchain:
    def __init__(self):
        self._blocks = []

    def add(self, block: Block):
        self._blocks.append(block)

    def __len__(self):
        return len(self._blocks)

    def __getitem__(self, index):
        return self._blocks[index]

    @property
    def chain(self):
        return [block.block_hash for block in self._blocks]

    def stats(self):
        _numb_of_blocks = str(len(self._blocks))
        _total_size_in_bytes = str(sys.getsizeof(self._blocks))
        print(f'''
        Number of blocks: {_numb_of_blocks}
        Total size in bytes: {_total_size_in_bytes}
        ''')

    def check_integrity(self) -> bool:
        # Old blocks
        _old_blocks_hash = []
        with open('blockchain.txt', 'r') as f:
            for line in f:
                header, body = line[:-1].split('~')
                block_number, prev_block_hash = header.split('.')
                transactions = body.split('$')
                _old_blocks_hash.append(Block(prev_block_hash, int(block_number), transactions).calculate_hash())

        # New blocks
        _new_blocks_hash = []
        for block in self._blocks:
            _new_blocks_hash.append(block.calculate_hash())
        if _old_blocks_hash < _new_blocks_hash:
            n = len(_new_blocks_hash) - len(_old_blocks_hash)
            return _old_blocks_hash == _new_blocks_hash[:-n]
        else:
            return _old_blocks_hash == _new_blocks_hash

    def save_chain(self):
        if self.check_integrity():
            if os.path.exists('blockchain.txt'):
                os.remove('blockchain.txt')
            for block in self._blocks:
                with open('blockchain.txt', 'a') as f:
                    f.write(str(block.block_number) + '.' + block.prev_block_hash + '~' + '$'.join(block.body) + '\n')
        else:
            return 'Inconsistent blockchain'

    def load_chain(self):
        self._blocks = []
        with open('blockchain.txt', 'r') as f:
            for line in f:
                header, body = line[:-1].split('~')
                block_number, prev_block_hash = header.split('.')
                transactions = body.split('$')
                self._blocks.append(Block(prev_block_hash, int(block_number), transactions))

class Buffer:
    def __init__(self, blockchain: Blockchain):
        self._transactions = []
        self._blockchain = blockchain

    def add(self, *transactions):
        self._transactions += [*transactions]

    def show(self):
        for t in self._transactions:
            print(t)

    def generate_block(self):
        prev_block_hash = self._blockchain[-1].block_hash if self._blockchain else 'Init block'
        block_number = len(self._blockchain)
        block = Block(prev_block_hash, block_number, self._transactions[:])
        self._blockchain.add(block)
        self._transactions.clear()


import streamlit as st
import tensorflow as tf
import tensorflow_hub as hub

model_url = "https://tfhub.dev/google/imagenet/mobilenet_v3_small_100_224/classification/5"
model = tf.keras.Sequential([
    hub.KerasLayer("https://tfhub.dev/google/imagenet/mobilenet_v3_small_100_224/classification/5")
])

st.title("AI Image Classifier")
st.write("MAde with Streamlit")

uploaded_image = st.file_uploader("Upload an image", type=["png", "jpg"])

if uploaded_image is not None:
    with st.spinner('Waiting for the prediction'):
        image = tf.keras.preprocessing.image.load_img(uploaded_image, target_size=(224, 224))
        input_array = tf.keras.preprocessing.image.img_to_array(image)
        input_array /= 255.0
        input_array = tf.expand_dims(input_array, 0)

        predictions = model.predict(input_array)
        probabilities = tf.nn.softmax(predictions).numpy()

        top_1_class_index = tf.argmax(predictions[0]).numpy()

        labels_file = "https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt"

        # download labels and creates a maps
        downloaded_file = tf.keras.utils.get_file("labels.txt", origin=labels_file)

        classes = []

        with open(downloaded_file) as f:
            labels = f.readlines()
            classes = [l.strip() for l in labels]

        st.success(f"This is an image of a {classes[top_1_class_index]}")
        st.image(uploaded_image)
        with st.expander("Click her to see more statistics"):
            st.write("more statistics")