"Greedy layerwise cifar training"
from __future__ import print_function

import torch
import torch.nn as nn
import torch.optim as optim
import torch.backends.cudnn as cudnn
import numpy as np

import torchvision
import torchvision.transforms as transforms

import os
import argparse

from model_greedy import *
from torch.autograd import Variable

from utils import progress_bar

from random import randint
import datetime
import json

parser = argparse.ArgumentParser(description='PyTorch CIFAR10 Training')
parser.add_argument('--lr', default=0.1, type=float, help='learning rate')
parser.add_argument('--resume', '-r', action='store_true', help='resume from checkpoint')
parser.add_argument('--ncnn',  default=5,type=int, help='depth of the CNN')
parser.add_argument('--nepochs',  default=50,type=int, help='number of epochs')
parser.add_argument('--epochdecay',  default=15,type=int, help='number of epochs')
parser.add_argument('--avg_size',  default=16,type=int, help='size of averaging ')
parser.add_argument('--feature_size',  default=128,type=int, help='feature size')
parser.add_argument('--ds-type', default=None, help="type of downsampling. Defaults to old block_conv with psi. Options 'psi', 'stride', 'avgpool', 'maxpool'")
parser.add_argument('--nlin',  default=2,type=int, help='nlin')
parser.add_argument('--ensemble', default=1,type=int,help='compute ensemble')
parser.add_argument('--name', default='',type=str,help='name')
parser.add_argument('--batch_size', default=128,type=int,help='batch size')
parser.add_argument('--bn', default=0,type=int,help='use batchnorm')
parser.add_argument('--debug', default=0,type=int,help='debug')
parser.add_argument('--debug_parameters', default=0,type=int,help='verification that layers frozen')
parser.add_argument('-j', '--workers', default=6, type=int, metavar='N',
                    help='number of data loading workers (default: 4)')
parser.add_argument('--width_aux', default=128,type=int,help='auxillary width')
parser.add_argument('--down', default='[1,2]', type=str,
                        help='layer at which to downsample')
parser.add_argument('--seed', default=None, help="Fixes the CPU and GPU random seeds to a specified number")

args = parser.parse_args()
opts = vars(args)
args.ensemble = args.ensemble>0
args.bn = args.bn > 0
args.debug_parameters = args.debug_parameters > 0

if args.debug:
    args.nepochs = 1 # we run just one epoch per greedy layer training in debug mode

downsample =  list(np.array(json.loads(args.down)))
in_size=32
mode=0

if args.seed is not None:
    seed = int(args.seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)

save_name = 'layersize_'+str(args.feature_size)+'width_' \
            + str(args.width_aux) + 'depth_' + str(args.nlin) + 'ds_type_' + str(args.ds_type) +'down_' +  args.down
#logging
time_stamp = str(datetime.datetime.now().isoformat())

name_log_dir = ''.join('{}{}-'.format(key, val) for key, val in sorted(opts.items()))+time_stamp
name_log_dir = 'runs/'+name_log_dir

name_log_txt = time_stamp + save_name + str(randint(0, 1000)) + args.name
debug_log_txt = name_log_txt + '_debug.log'
name_save_model = name_log_txt + '.t7'
name_log_txt=name_log_txt   +'.log'

with open(name_log_txt, "a") as text_file:
    print(opts, file=text_file)

use_cuda = torch.cuda.is_available()
start_epoch = 0  # start from epoch 0 or last checkpoint epoch

# Data
print('==> Preparing data..')
transform_train = transforms.Compose([
    transforms.RandomCrop(32, padding=4),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])

transform_test = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])

trainset_class = torchvision.datasets.CIFAR10(root='.', train=True, download=True,transform=transform_train)
trainloader_classifier = torch.utils.data.DataLoader(trainset_class, batch_size=args.batch_size, shuffle=True, num_workers=args.workers)
testset = torchvision.datasets.CIFAR10(root='.', train=False, download=True, transform=transform_test)
testloader = torch.utils.data.DataLoader(testset, batch_size=256, shuffle=False, num_workers=2)

# Model

print('==> Building model..')
n_cnn=args.ncnn
#if args.ds_type is None:
 #   block_conv_ = block_conv
#else:
 #   from functools import partial
  #  block_conv_ = partial(ds_conv, ds_type=args.ds_type)
net = greedyNet(block_conv, 1, args.feature_size, downsample=downsample, batchnorm=args.bn)

if args.width_aux:
    num_feat = args.width_aux
else:
    num_feat = args.feature_size

net_c = auxillary_classifier(avg_size=args.avg_size, in_size=in_size,
                             n_lin=args.nlin, feature_size=num_feat,
                             input_features=args.feature_size, batchn=args.bn)

with open(name_log_txt, "a") as text_file:
    print(net, file=text_file)
    print(net_c, file=text_file)

net = torch.nn.DataParallel(nn.Sequential(net,net_c)).cuda()
cudnn.benchmark = True

criterion_classifier = nn.CrossEntropyLoss()

net.module[0].unfreezeGradient(0)
optimizer = optim.SGD(filter(lambda p: p.requires_grad, net.parameters()), lr=args.lr, momentum=0.9, weight_decay=5e-4)
criterion = nn.CrossEntropyLoss()

def train_classifier(epoch,n):
    print('\nSubepoch: %d' % epoch)
    net.train()
    for k in range(n):
        net.module[0].blocks[k].eval()

    if args.debug_parameters:
        #This is used to verify that early layers arent updated
        import copy
        #store all parameters on cpu as numpy array
        net_cpu = copy.deepcopy(net).cpu()
        net_cpu_dict = net_cpu.module[0].state_dict()
        with open(debug_log_txt, "a") as text_file:
            print('n: %d'%n)
            for param in net_cpu_dict.keys():
                net_cpu_dict[param]=net_cpu_dict[param].numpy()
                print("parameter stored on cpu as numpy: %s  "%(param),file=text_file)

    train_loss = 0
    correct = 0
    total = 0
    for batch_idx, (inputs, targets) in enumerate(trainloader_classifier):
        if use_cuda:
            inputs, targets = inputs.cuda(), targets.cuda()

        optimizer.zero_grad()
        inputs, targets = Variable(inputs), Variable(targets)
        outputs = net.forward([inputs,n])

        loss = criterion_classifier(outputs, targets)
        loss.backward()
        optimizer.step()
        loss_pers=0

        if args.debug_parameters:

            s_dict = net.module[0].state_dict()
            with open(debug_log_txt, "a") as text_file:
                print("iteration %d" % (batch_idx), file=text_file)
                for param in s_dict.keys():
                    diff = np.sum((s_dict[param].cpu().numpy()-net_cpu_dict[param])**2)
                    print("n: %d parameter: %s size: %s changed by %.5f" % (n,param,net_cpu_dict[param].shape,diff),file=text_file)

        train_loss += loss.data[0]
        _, predicted = torch.max(outputs.data, 1)
        total += targets.size(0)
        correct += predicted.eq(targets.data).cpu().sum()

        progress_bar(batch_idx, len(trainloader_classifier), 'Loss: %.3f | Acc: %.3f%% (%d/%d) |  losspers: %.3f'
            % (train_loss/(batch_idx+1), 100.*float(correct)/float(total), correct, total,loss_pers))

    acc = 100.*float(correct)/float(total)
    return acc

all_outs = [[] for i in range(args.ncnn)]

def test(epoch,n,ensemble=False):
    global acc_test_ensemble
    all_targs = []
    net.eval()
    test_loss = 0
    correct = 0
    total = 0

    all_outs[n] = []
    for batch_idx, (inputs, targets) in enumerate(testloader):
        if use_cuda:
            inputs, targets = inputs.cuda(), targets.cuda()

        inputs, targets = Variable(inputs, volatile=True), Variable(targets)
        outputs = net([inputs,n])

        loss = criterion_classifier(outputs, targets)

        test_loss += loss.data[0]
        _, predicted = torch.max(outputs.data, 1)
        total += targets.size(0)
        correct += predicted.eq(targets.data).cpu().sum()

        progress_bar(batch_idx, len(testloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
            % (test_loss/(batch_idx+1), 100.*float(correct)/float(total), correct, total))

        if args.ensemble:
            all_outs[n].append(outputs.data.cpu())
            all_targs.append(targets.data.cpu())
    acc = 100. * float(correct) / float(total)

    if ensemble:
        all_outs[n] = torch.cat(all_outs[n])
        all_targs = torch.cat(all_targs)
        #This is all on cpu so we dont care
        weight = 2 ** (np.arange(n + 1)) / sum(2 ** np.arange(n + 1))
        total_out = torch.zeros((total,10))

        #very lazy
        for i in range(n+1):
            total_out += float(weight[i])*all_outs[i]

        _, predicted = torch.max(total_out, 1)
        correct = predicted.eq(all_targs).sum()
        acc_ensemble = 100*float(correct)/float(total)
        print('Acc_ensemble: %.2f'%acc_ensemble)
    if ensemble:
        return acc,acc_ensemble
    else:
        return acc

i=0
num_ep = args.nepochs

for n in range(n_cnn):
    net.module[0].unfreezeGradient(n)
    lr = args.lr*5.0# we run at epoch 0 the lr reset to remove non learnable param

    for epoch in range(0, num_ep):
        i=i+1
        print('n: ',n)
        if epoch % args.epochdecay == 0:
            lr=lr/5.0
            to_train = list(filter(lambda p: p.requires_grad, net.parameters()))
            optimizer = optim.SGD(to_train, lr=lr, momentum=0.9, weight_decay=5e-4)
            print('new lr:',lr)

        acc_train = train_classifier(epoch,n)
        if args.ensemble:
            acc_test,acc_test_ensemble = test(epoch,n,args.ensemble)

            with open(name_log_txt, "a") as text_file:
                print("n: {}, epoch {}, train {}, test {},ense {} "
                      .format(n,epoch,acc_train,acc_test,acc_test_ensemble), file=text_file)
        else:
            acc_test = test(epoch, n)
            with open(name_log_txt, "a") as text_file:
                print("n: {}, epoch {}, train {}, test {}, ".format(n,epoch,acc_train,acc_test), file=text_file)

        if args.debug:
            break

    if args.down and n in downsample:
        args.avg_size = int(args.avg_size/2)
        in_size = int(in_size/2)
        args.feature_size = int(args.feature_size*2)
        args.width_aux = args.width_aux * 2

    if args.width_aux:
        num_feat = args.width_aux
    else:
        num_feat = args.feature_size

    net_c = None
    if n < n_cnn-1:
        net_c = auxillary_classifier(avg_size=args.avg_size, in_size=in_size,
                                     n_lin=args.nlin, feature_size=args.width_aux,
                                     input_features=args.feature_size, batchn=args.bn).cuda()
        net.module[0].add_block(n in downsample)
        net = torch.nn.DataParallel(nn.Sequential(net.module[0], net_c)).cuda()

state_final = {
            'net': net,
            'acc_test': acc_test,
            'acc_train': acc_train,
        }
torch.save(state_final,save_name)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# author：albert time:2019/3/11
# coding=utf-8
'''
Created on 2018-10-28
@author: Lee
Project:使用unittest框架编写测试用例。使用pytest执行
'''
import unittest,pytest,allure
import ddt
from selenium import webdriver
from time import sleep
from Page_object.loginpage import Logintest
from Page_object.read_excle import  ExcelTest
from Page_object.case_list import Case_mangage
from Page_object.case_edit import Case_edit
from Page_object.case_add import Case_add
from Page_object.before import New_Before
from Page_object.incall import in_Call
from Page_object.doctor import Doctors
from Logs.logs import Logger
import Obejct_config.config as cfg
from Page_object.base_processed import Base_proce
from Test_case.dc_reject import Case_state
from  Test_case.doc import HandleDoc

# 测试数据
data = ExcelTest(cfg.ExcelTest.filePath, cfg.ExcelTest.sheetName)
testData = data.dict_data()
# 实例化log
log = Logger(cfg.Logger.path,cfg.Logger.filename)

@ddt.ddt
class Casejinhui120(unittest.TestCase):
    """
          登录金汇120的case
    """
    @classmethod
    def setUpClass(cls):
        #初始化浏览器
        cls.driver = webdriver.Chrome()
        cls.driver.implicitly_wait(0)
        cls.url = cfg.HOME_URL
        cls.base=Base_proce(cls.driver, cls.url, u"金汇")
        cls.login_page = Logintest(cls.driver, cls.url, u"金汇")
        cls.case_page = Case_mangage(cls.driver, cls.url, u"金汇")
        cls.case_add = Case_add(cls.driver, cls.url, u"金汇")
        cls.case_edit = Case_edit(cls.driver, cls.url, u"金汇")
        cls.before = New_Before(cls.driver, cls.url, u"金汇")
        cls.call = in_Call(cls.driver, cls.url, u"金汇")
        cls.base=Base_proce(cls.driver, cls.url, u"金汇")
        cls.doctor=Doctors(cls.driver, cls.url, u"金汇")
        cls.Case = Case_state(cls.driver, cls.url, u"金汇")
        # 登录
        cls.login_page.open()
        cls.login_page.input_username("lishaoyang")
        cls.login_page.input_password(123456)
        cls.login_page.input_passcode(6666)
        cls.login_page.button_submit()

    @classmethod
    def tearDownClass(cls):
        cls.driver.quit()

    # 用例执行体
    # @allure.feature  # 用于定义被测试的功能，被测产品的需求点
    # @allure.story  # 用于定义被测功能的用户场景，即子功能点
    # with allure.step  # 用于将一个测试用例，分成几个步骤在报告中输出
    # allure.attach  # 用于向测试报告中输入一些附加的信息，通常是一些测试数据信息
    # @pytest.allure.step  # 用于将一些通用的函数作为测试步骤输出到报告，调用此函数的地方会向报告中输出步骤

    # @ddt.data(*testData)
    # @allure.feature("工单流程")
    # @allure.story("新建院前工单")
    def test_sale_AA(self):     #新建院前工单
        # with allure.step("进入案件新建院前"):
        try:
            while True:
                #进入案件
                sleep(1)
                self.call.case_mages()
                sleep(1.5)
                self.case_page.do_cases()
                sleep(1)
                # self.case_edit.add_Before()
                #医务官处理信息
                sleep(2)
                self.call.processed_Order()
                sleep(2)
                self.Case.Refer_State(num=2)
                self.call.first_Order()
                sleep(2)

        except :
            self.base.get_shot()
            sl = HandleDoc()
            # sl.doc()
            sl.sendemail()
            print("test error")

if __name__ == "__main__":
    pytest.main("--html=test_report.html")


from django.shortcuts import render, redirect
from .models import Data, Genome_version, Publication, Genome
from .search_helper import search_data, search_publications, search_genomes
from django.urls import reverse

def home(request):

    context = {
        "title": "Resources",
        "query": None,
        "data_hits": ""
    }

    if request.method == "POST":

        query = request.POST.get("searchbar") if request.POST.get("searchbar") else None

        data_hits = len(search_data(query))
        publication_hits = len(search_publications(query))
        genome_hits = len(search_genomes(query))

        context.update(
            {
            "query": query, # will be passed on in a link to the sub resource page
            "data_hits": data_hits,
            "publication_hits": publication_hits,
            "genome_hits": genome_hits
            }
        )

    return render(request, "resources/search.html", context)

def data(request, query=None):

    if request.method == "POST":
        query = request.POST.get("searchbar") if request.POST.get("searchbar") else None
        clear = request.POST.get("clear-btn")
        if not clear:
            return redirect('resources-data', query=query)
        else:
            return redirect('resources-data')

    if query:
        results = search_data(query)
    else:
        results = Data.objects.all()

    context = {
        "title": "Sequencing data",
        "data": results,
        "query": query
    }

    return render(request, "resources/data_datatable.html", context)

def publications(request, query=None):

    if request.method == "POST":

        query = request.POST.get("searchbar") if request.POST.get("searchbar") else None
        clear = request.POST.get("clear-btn")
        if not clear:
            return redirect('resources-publications', query=query)
        else:
            return redirect('resources-publications')

    if query:
        results = search_publications(query)
    else:
        results = Publication.objects.all()

    context = {
        "title": "Publications",
        "publications": results,
        "query": query
    }
    return render(request, "resources/publication_datatable.html", context)

def genomes(request, query=None):

    if request.method == "POST":
        query = request.POST.get("searchbar") if request.POST.get("searchbar") else None
        clear = request.POST.get("clear-btn")
        if not clear:
            return redirect('resources-genomes', query=query)
        else:
            return redirect('resources-genomes')

    if query:
        results = search_genomes(query)
    else:
        results = Genome.objects.all()

    context = {
        "title": "Reference genomes",
        "genomes": results, # replace data with genomes
        "query": query,
        "versions": Genome_version.objects.all().order_by('-date')
    }

    return render(request, "resources/genome_datatable.html", context)


import numpy as np
from typing import Union
from scipy.io import loadmat
import matplotlib.pyplot as plt
import matplotlib as mpl
from ConfinedBrownianAnalysis.Analyse import Dedrift
import pickle

def Load(filename):
    """ Simple function to load a pickled object """
    with open(filename, 'rb') as handle:
        return pickle.load(handle)

class Data(np.ndarray):
    def __new__(cls, file: str, fps: int = 100, cutoff: int = -1):
        # Input array is an already formed ndarray instance
        # We first cast to be our class type
        cls.file = loadmat(file)
        initial_array = np.array([cls.file["x"], cls.file["y"], cls.file["z"]])[
            :, 0, :cutoff
        ]
        obj = np.asarray(initial_array).view(cls)

        obj = obj.transpose()
        del cls.file
        return obj

    def __init__(
        self, file: str, fps: int = 100, cutoff: int = -1, dedrift_method="min_z"
    ):

        self.fps = fps
        self.dt = 1 / fps

        # trajectory time
        self.time = np.arange(len(self.x)) / fps
        self._time = np.arange(len(self.x)) / fps

    @property
    def x(self):
        return self.__array__()[:,0]

    @property
    def y(self):
        return self.__array__()[:,1]

    @property
    def z(self):
        return self.__array__()[:,2]

    def plot_3D(self, N: int = 20, N_c: int = 500):
        """
        Plot the trajectory in 3D, using N chunks of N_c points with a gradient of color indicating the time.
        """
        plt.ioff()
        fig = plt.figure()
        plt.ion()
        ax = plt.axes(projection="3d")

        cmap = plt.get_cmap("jet")

        for i in range(N - 1):
            ax.plot(
                self.x[i * N_c : i * N_c + N_c],
                self.y[i * N_c : i * N_c + N_c],
                self.z[i * N_c : i * N_c + N_c],
                color=plt.cm.jet(1 * i / N),
            )

        ax = plt.gca()
        ax.ticklabel_format(style="sci")

        norm = mpl.colors.Normalize(vmin=0, vmax=1)
        sm = plt.cm.ScalarMappable(cmap=cmap, norm=norm)
        sm.set_array([])

        plt.xlabel("$x$ ($\mathrm{\mu m}$)")
        plt.ylabel("$y$ ($\mathrm{\mu m}$)")
        ax.set_zlabel("$z$ ($\mathrm{\mu m}$)")

        ticks_c = []
        for i in np.linspace(0, 1, 5):
            ticks_c.append("{:.0f}".format(N * N_c * i / self.fps / 60))
        cbar = plt.colorbar(
            sm,
            ticks=np.linspace(0, 1, 5),
            format="%.1f",
            shrink=0.5,
            orientation="vertical",
            pad=0.2,
        )
        cbar.set_ticklabels(ticks_c)
        cbar.set_label("$t ~ \mathrm{(s)}$")
        plt.show()

    def plot_1D(self, axis: Union[str, int]):

        if axis not in ["x", "y", "z", 0, 1, 2]:
            raise ValueError(
                "Please choose an axis in ['x', 'y', 'z'] or the numeric value [0, 1, 2]"
            )

        axis_dict = {"x": 0, "y": 1, "z": 2}
        axis_dict_inverted = {"0": "x", "1": "y", "2": "z"}
        if type(axis) == str:
            axis = axis_dict[axis]

        plt.ioff()
        fig = plt.figure()
        plt.ion()

        plt.plot(self.time, self[:, axis])

        plt.xlabel("$t$ ($\mathrm{s}$)")
        plt.ylabel("$" + axis_dict_inverted[str(axis)] + "$" + "($~\mathrm{\mu m}$)")
        plt.tight_layout()
        plt.show()

    def dedrift(self, method: str = "min_z", **kwargs):
        ded = Dedrift(self, method, **kwargs)
        self = ded.traj

    def clear(self):
        plt.close("all")

    # Handeling the picke and slice of the array

    def __getitem__(self, subscript):
        """
        Rewrite the slicing function __getitem__ so that it also slice
        self.time at the same time.
        """
        if isinstance(subscript, slice):

            self.time = self._time.__getitem__(subscript)
            return super().__getitem__(subscript)

        return super().__getitem__(subscript)

    def __array_finalize__(self, obj):
        """
        The behaviour when terminating the class, this also done when slicing or
        multiplying the array. For example, it permits to keep the attribute when slincing
        the Data class.
        """
        # see InfoArray.__array_finalize__ for comments
        if obj is None:
            return

        self.fps = getattr(obj, "fps", None)
        self.time = getattr(obj, "time", None)
        self._time = getattr(obj, "_time", None)
        self.dt = getattr(obj, "dt", None)

    def __reduce__(self):
        """
        np.ndarray uses __reduce__ to pickle itself, so we need to rewrite them in order to save new attributes.
        https://stackoverflow.com/questions/26598109/preserve-custom-attributes-when-pickling-subclass-of-numpy-array
        """
        # Get the parent's __reduce__ tuple
        pickled_state = super().__reduce__()

        # Create our own tuple to pass to __setstate__
        new_state = pickled_state[2] + (self.fps,self.time, self._time, self.dt)
        # print(new_state)
        # Return a tuple that replaces the parent's __setstate__ tuple with our own
        return (pickled_state[0], pickled_state[1], new_state)

    def __setstate__(self, state):
        self.fps = state[-4]  # Set the info attribute
        self.time = state[-3]
        self._time = state[-2]
        self.dt = state[-1]
        # Call the parent's __setstate__ with the other tuple elements.
        super(Data, self).__setstate__(state[0:-4])

    ### Saving with pickle ###

    def save(self, filename):
        with open(filename, "wb") as handle:
            b = pickle.dump(self, handle, protocol=pickle.HIGHEST_PROTOCOL)


import random
import xlsxwriter

# Create a list of 210 random shuffled numbers between 1 and 210
numbers = list(range(1, 211))
random.shuffle(numbers)

# Create an Excel workbook and worksheet
workbook = xlsxwriter.Workbook('random_numbers.xlsx')
worksheet = workbook.add_worksheet()

# Set the row and column variables
row = 0
col = 0

# Loop through the shuffled list and write each number to a new row in the second column of the Excel worksheet
for number in numbers:
    worksheet.write(row, col, "")
    worksheet.write(row, col + 1, number)
    row += 1

# Close the workbook
workbook.close()


"""
Created on Sat May  8 12:00:42 2021
"""
import math
import numpy as np
import pandas as pd
import scipy as sp
import matplotlib.pyplot as plt
import statsmodels.api as sm
from statsmodels.formula.api import ols
from sklearn import linear_model
from sklearn.metrics import mean_squared_error, r2_score, accuracy_score
from sklearn.decomposition import PCA
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
import meow

df = pd.read_csv('middleSchoolData.csv', header=0)
# cols = df.select_dtypes(exclude=['float']).columns
# df[cols] = df[cols].apply(pd.to_numeric, downcast = 'float', errors='coerce')

#%% Question 1
plot1 = plt.figure(1)
plt.scatter(df['applications'], df['acceptances'])
plt.xlabel('application count')
plt.ylabel('acceptances')
acceptancecorr = df['applications'].corr(df['acceptances'], method='spearman')
acceptancecorrpears = df['applications'].corr(df['acceptances'], method='pearson')
acceptCOD = acceptancecorrpears ** 2
plt.savefig('acceptanceApplicationsplot.png')

####

#%% Question 2
regr = linear_model.LinearRegression()
regr2 = linear_model.LinearRegression()
applications = df['applications']
def rateConv(array, divArray):
    divide = divArray
    rates = []
    for index in range(divide.size):
        if((array[index] == 0) and divide[index] == 0):
            rates.append(float('nan'))
        else:
            rates.append((array[index] / divide[index]))
    return rates
applicationRate = pd.Series(rateConv(applications, df['school_size']))
df['application_rate'] = pd.DataFrame(applicationRate)
dfRate = df[['acceptances', 'application_rate']]
dfAccept = df[['acceptances', 'applications']]
ratenotnans = []
for index, row in dfRate.iterrows():
    if(row.notnull().all()):
        ratenotnans.append([row[0], row[1]])
dfRate = pd.DataFrame(ratenotnans)
regr.fit(np.array(dfRate[1]).reshape(-1,1), dfRate[0])
regrPreds = regr.predict(np.array(dfRate[1]).reshape(-1,1))
print(mean_squared_error(dfRate[0], regrPreds, squared=False))
print(regr.coef_)
print(r2_score(dfRate[0], regrPreds))

regr2.fit(np.array(dfAccept['applications']).reshape(-1,1), dfAccept['acceptances'])
regr2Preds = regr2.predict(np.array(dfAccept['applications']).reshape(-1,1))
print(mean_squared_error(dfAccept['acceptances'], regr2Preds, squared=False))
print(regr2.coef_)
print(r2_score(dfAccept['acceptances'], regr2Preds))

appRatecorr = df['acceptances'].corr(applicationRate, method='spearman')
appRatepears = df['acceptances'].corr(applicationRate, method='pearson')

appRateCOD = appRatepears ** 2
plot2 = plt.figure(2)
plt.scatter(applicationRate, df['acceptances'])
plt.xlabel('application rate')
plt.ylabel('acceptances')
plt.savefig('applicationRateplot.png')

####

#%% Question 3
acceptRate = pd.Series(rateConv(df['acceptances'], df['applications']))
studentOdds = []
for index in range(acceptRate.size):
    prob = applicationRate[index] * acceptRate[index]
    if(not (math.isnan(prob))):
        studentOdds.append(prob / (1-prob))
    else:
        studentOdds.append(0)
df['student_odds'] = pd.DataFrame(studentOdds)
plot3 = plt.figure(3)
plt.bar(df.index.values.tolist(), height=pd.Series(studentOdds), width=5, tick_label=None)
plt.xlabel('schools')
plt.ylabel('odds')
plt.savefig('acceptOddsplot.png')
maxOdds = df['student_odds'].idxmax()
####

#%% Question 4
dfClimate = df[[
    'rigorous_instruction',
    'collaborative_teachers',
    'supportive_environment',
    'effective_school_leadership',
    'strong_family_community_ties',
    'trust'
    ]]

dfPerformance = df[[
    'student_achievement',
    'reading_scores_exceed',
    'math_scores_exceed'
    ]]

def remNAN(mat):
    nans=[]
    for index, row in mat.iterrows():
        if(not (row.notnull().all())):
            nans.append(index)
    mat.drop(nans, inplace=True)

dfBothCliPerf = pd.concat([dfClimate, dfPerformance], axis=1)
remNAN(dfBothCliPerf)
dfNoNANClim = dfBothCliPerf[[
    'rigorous_instruction',
    'collaborative_teachers',
    'supportive_environment',
    'effective_school_leadership',
    'strong_family_community_ties',
    'trust'
    ]]
dfNoNANPerf = dfBothCliPerf[[
    'student_achievement',
    'reading_scores_exceed',
    'math_scores_exceed'
    ]]
dfOnlyRatings = dfBothCliPerf[[
    'rigorous_instruction',
    'collaborative_teachers',
    'supportive_environment',
    'effective_school_leadership',
    'strong_family_community_ties',
    'trust',
    'student_achievement'
    ]]

correlationMat = dfBothCliPerf.corr(method='pearson')
zscored = sp.stats.zscore(dfBothCliPerf)
pca = PCA()
pca.fit(zscored)
eigVals = pca.explained_variance_
loadings = pca.components_
rotatedData = pca.fit_transform(zscored)
covarExplained = eigVals/sum(eigVals)*100

zscored2 = sp.stats.zscore(dfNoNANPerf)
pca2 = PCA()
pca2.fit(zscored2)
eigVals2 = pca2.explained_variance_
loadings2 = pca2.components_
rotatedData2 = pca2.fit_transform(zscored2)
covarExplained2 = eigVals2/sum(eigVals2)*100

zscored3 = sp.stats.zscore(dfNoNANClim)
pca3 = PCA()
pca3.fit(zscored3)
eigVals3 = pca3.explained_variance_
loadings3 = pca3.components_
rotatedData3 = pca3.fit_transform(zscored3)
covarExplained3 = eigVals3/sum(eigVals3)*100

correlationComps = np.corrcoef(rotatedData2[:,0], rotatedData3[:,0])

zscored4 = sp.stats.zscore(dfOnlyRatings)
pca4 = PCA()
pca4.fit(zscored4)
eigVals4 = pca4.explained_variance_
loadings4 = pca4.components_
rotatedData4 = pca4.fit_transform(zscored4)
covarExplained4 = eigVals4/sum(eigVals4)*100
correlationMatRatings = dfOnlyRatings.corr(method='pearson')

plot4 = plt.figure(4)
plt.imshow(correlationMat, aspect='auto')
plt.colorbar()

# plot5 = plt.figure(5)
# numClasses = 9
# plt.bar(np.linspace(0,8, 9), height=eigVals)
# plt.xlabel('Principal component')
# plt.ylabel('Eigenvalue')
# plt.plot([0,numClasses],[1,1],color='red',linewidth=1)

plot6 = plt.figure(6)
numClasses2 = 6
plt.bar(np.linspace(0,5,6), height=eigVals3)
plt.xlabel('Principal component')
plt.ylabel('Eigenvalue')
plt.plot([0,numClasses2],[1,1],color='red',linewidth=1)

plot7 = plt.figure(7)
numClasses3 = 3
plt.bar(np.linspace(0,2,3), height=eigVals2)
plt.xlabel('Principal component')
plt.ylabel('Eigenvalue')
plt.plot([0,numClasses3],[1,1],color='red',linewidth=1)

plot8 = plt.figure(12)
numClasses4 = 7
plt.bar(np.linspace(0,6,7), height=eigVals4)
plt.xlabel('Principal component')
plt.ylabel('Eigenvalue')
plt.plot([0,numClasses4],[1,1],color='red',linewidth=1)

# plot9 = plt.figure(8)
# plt.bar(np.linspace(0,8,9),loadings[:,2])
# plt.xlabel('Question')
# plt.ylabel('Loading')

plot10 = plt.figure(9)
plt.bar(np.linspace(0,5,6),loadings3[:,0])
plt.xlabel('Question')
plt.ylabel('Loading')

plot11 = plt.figure(10)
plt.bar(np.linspace(0,2,3),loadings2[:,0])
plt.xlabel('Question')
plt.ylabel('Loading')

plot12 = plt.figure(13)
plt.bar(np.linspace(0,6,7),loadings4[:,1])
plt.xlabel('Question')
plt.ylabel('Loading')

plot13= plt.figure(11)
plt.plot(rotatedData4[:,0],rotatedData4[:,1],'o',markersize=1)

plot14 = plt.figure(14)
plt.imshow(correlationComps, aspect='auto')
plt.colorbar()
####

#%% Question 5
dfSpendingSize = df[[
        'per_pupil_spending',
        'avg_class_size',
        'poverty_percent',
        'student_achievement'
    ]]
remNAN(dfSpendingSize)
# model = sm.OLS(dfSpendingSize['student_achievement'], dfSpendingSize['avg_class_size']).fit()
# modelPredict = model.predict(dfSpendingSize['avg_class_size'])
# print(model.summary())

spendingMed = dfSpendingSize['per_pupil_spending'].median()
classMed = dfSpendingSize['avg_class_size'].median()
povertyMed = dfSpendingSize['poverty_percent'].median()
def categorizeData(dataseries, threshold):
    categorizedlist = []
    for value in dataseries:
        if value >= threshold:
            categorizedlist.append(2)
        else:
            categorizedlist.append(1)
    return categorizedlist
spendingCats = categorizeData(dfSpendingSize['per_pupil_spending'], spendingMed)
sizeCats = categorizeData(dfSpendingSize['avg_class_size'], classMed)
povCats = categorizeData(dfSpendingSize['poverty_percent'], 50.0)
richAchieve = []
poorAchieve = []
achieves = np.array(dfSpendingSize['student_achievement'])
for index in range(len(spendingCats)):
    if spendingCats[index] == 1:
        poorAchieve.append(achieves[index])
    else:
        richAchieve.append(achieves[index])
histo = pd.DataFrame(meow.meow(dfSpendingSize["student_achievement"]))
histo.sort_values(0, ascending=True, inplace=True)
spendgraph = plt.figure(15)
plt.bar(np.linspace(1, 225, 225), histo[1])
#plt.bar(["rich", 'poor'], height=[np.mean(richAchieve), np.mean(poorAchieve)])

####

#%% Question 6
# model2 = ols('student_achievement ~ per_pupil_spending + avg_class_size + per_pupil_spending:avg_class_size', data=dfSpendingSize).fit()
# anova_table2 = sm.stats.anova_lm(model2, typ=2) #Create the ANOVA table. Residual = Within
# print(anova_table2) #Show the ANOVA table

# fig = meansPlot(x=dfSpendingSize['per_pupil_spending'], trace=dfSpendingSize['avg_class_size'], response=dfSpendingSize['student_achievement'])

bigAchieve = []
smallAchieve = []
for index in range(len(sizeCats)):
    if sizeCats[index] == 1:
        smallAchieve.append(achieves[index])
    else:
        bigAchieve.append(achieves[index])

sizegraph = plt.figure(16)
plt.bar(["big", 'small'], height=[np.mean(bigAchieve), np.mean(smallAchieve)])

####

#%% Question 7
acceptances = df[['acceptances', 'school_name']]
sortedAccept = acceptances.sort_values(by=['acceptances'], ascending=False)
totalAccepts = acceptances['acceptances'].sum()
arraySorted = np.array(sortedAccept)
schoolCount = 0
acceptanceThreshold = 0
for row in arraySorted:
    if(acceptanceThreshold < 4015):
        acceptanceThreshold += row[0]
        schoolCount += 1
    else:
        break

AcceptanceBarGraph = plt.figure(50)
plt.bar(acceptances.index.values.tolist(), height=sortedAccept['acceptances'], width=5)
plt.xlabel('schools')
plt.ylabel('acceptances')

####

#%% Question 8
dfFactors = df[[
        'applications',
        'acceptances',
        'per_pupil_spending',
        'avg_class_size',
        'asian_percent',
        'black_percent',
        'hispanic_percent',
        'multiple_percent',
        'white_percent',
        'rigorous_instruction',
        'collaborative_teachers',
        'supportive_environment',
        'effective_school_leadership',
        'strong_family_community_ties',
        'trust',
        'disability_percent',
        'poverty_percent',
        'ESL_percent',
        'school_size',
        'student_achievement',
        'reading_scores_exceed',
        'math_scores_exceed',
        'application_rate',
        'student_odds'
    ]]
remNAN(dfFactors)
factorMat = dfFactors.corr(method='pearson')
def appendEqualCountsClass(df, class_name, feature, num_bins, labels):
    '''Append a new class feature named 'class_name' based on a split of 'feature' into clases with equal sample points.  Class names are in 'labels'.'''

    # Compute the bin boundaries
    percentiles = np.linspace(0,100,num_bins+1)
    bins = np.percentile(df[feature],percentiles)

    # Split the data into bins
    n = pd.cut(df[feature], bins = bins, labels=labels, include_lowest=True)

    # Add the new binned feature to a copy of the data
    c = df.copy()
    c[class_name] = n
    return c

dfFactors = appendEqualCountsClass(dfFactors, "accepted", "student_odds", 2, ["L","H"])
dfFactors2 = appendEqualCountsClass(dfFactors, "achievementlevel", "student_achievement", 2, ["L","H"])
y = dfFactors['accepted']
X = dfFactors[[
        'applications',
        'per_pupil_spending',
        'avg_class_size',
        'asian_percent',
        'black_percent',
        'hispanic_percent',
        'multiple_percent',
        'white_percent',
        'rigorous_instruction',
        'collaborative_teachers',
        'supportive_environment',
        'effective_school_leadership',
        'strong_family_community_ties',
        'trust',
        'disability_percent',
        'poverty_percent',
        'ESL_percent',
        'school_size',
        'student_achievement',
        'reading_scores_exceed',
        'math_scores_exceed',
        'application_rate'
    ]]

y2 = dfFactors2['achievementlevel']
X2 = dfFactors2[[
        'per_pupil_spending',
        'avg_class_size',
        'asian_percent',
        'black_percent',
        'hispanic_percent',
        'multiple_percent',
        'white_percent',
        'rigorous_instruction',
        'collaborative_teachers',
        'supportive_environment',
        'effective_school_leadership',
        'strong_family_community_ties',
        'trust',
        'disability_percent',
        'poverty_percent',
        'ESL_percent',
        'school_size',
        'reading_scores_exceed',
        'math_scores_exceed'
    ]]

scaler = MinMaxScaler(feature_range=(0,1))
rescaledX = scaler.fit_transform(X)
rescaledX2 = scaler.fit_transform(X2)

X = pd.DataFrame(rescaledX, columns=X.columns)
X2 = pd.DataFrame(rescaledX2, columns=X2.columns)

test_size = 0.5
seed = 12345
X_train, X_test, Y_train, Y_test = train_test_split(X, y, test_size=test_size, random_state=seed)
X_train2, X_test2, Y_train2, Y_test2 = train_test_split(X2, y2, test_size=test_size, random_state=seed)
model_lr = linear_model.LogisticRegression(solver='lbfgs', multi_class='auto')
model_lr.fit(X_train, Y_train)
predictions_lr = model_lr.predict(X_train)
print("LogisticRegression", accuracy_score(Y_train, predictions_lr))
predictions_lr = model_lr.predict(X_test)
print("LogisticRegression", accuracy_score(Y_test, predictions_lr))

def logisticRegressionSummary(model, column_names):
    '''Show a summary of the trained logistic regression model'''

    # Get a list of class names
    numclasses = len(model.classes_)
    if len(model.classes_)==2:
        classes =  [model.classes_[1]] # if we have 2 classes, sklearn only shows one set of coefficients
    else:
        classes = model.classes_

    # Create a plot for each class
    for i,c in enumerate(classes):
        # Plot the coefficients as bars
        fig = plt.figure(figsize=(8,len(column_names)/3))
        fig.suptitle('Logistic Regression Coefficients for Class ' + str(c), fontsize=16)
        rects = plt.barh(column_names, model.coef_[i],color="lightblue")

        # Annotate the bars with the coefficient values
        for rect in rects:
            width = round(rect.get_width(),4)
            plt.gca().annotate('  {}  '.format(width),
                        xy=(0, rect.get_y()),
                        xytext=(0,2),
                        textcoords="offset points",
                        ha='left' if width<0 else 'right', va='bottom')
        plt.show()
        #for pair in zip(X.columns, model_lr.coef_[i]):
        #    print (pair)
logisticRegressionSummary(model_lr, X.columns)

model_lr2 = linear_model.LogisticRegression(solver='lbfgs', multi_class='auto')
model_lr2.fit(X_train2, Y_train2)
predictions_lr2 = model_lr2.predict(X_train2)
print("LogisticRegression2", accuracy_score(Y_train2, predictions_lr2))
predictions_lr2 = model_lr2.predict(X_test2)
print("LogisticRegression2", accuracy_score(Y_test2, predictions_lr2))
logisticRegressionSummary(model_lr2, X2.columns)

####


import cv2
import os
import numpy as np
os.chdir('C:/Users/korla/OneDrive/Escriptori/Startup/frameGPT/encoder')

# Read the video file
cap = cv2.VideoCapture('person01_handwaving_d2_uncomp.avi')

# Get the frame rate
fps = cap.get(cv2.CAP_PROP_FPS)

total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))

# Define the codec and create VideoWriter object
fourcc = cv2.VideoWriter_fourcc(*'XVID')
out = cv2.VideoWriter('output.avi', fourcc, fps, (160, 120), False)

def matrix_to_string(matrix):
# Initialize an empty string to store the output
    output = ""

    # Iterate through the rows of the matrix
    for i, row in enumerate(matrix):
        # Iterate through the elements of each row
        for j, element in enumerate(row):
            # Append the element to the output string
            output += str(element)

            # Add a space separator unless this is the last element in the row
            if j < len(row) - 1:
                output += " "

        # Add an underscore separator between rows unless this is the last row
        if i < len(matrix) - 1:
            output += " _ "

    # Call the matrix_to_string function with your matrix argument
    return output

result = ""

# Read the first 10 frames of the video and convert them to grayscale
for i in range(625):

    ret, frame = cap.read()
    gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    # Resize the frame to 160 x 120
    #resized_frame = cv2.resize(gray_frame, (160, 120))

    # Write the frame to the output video file
    #out.write(resized_frame)
    out.write(gray_frame)

    # Convert the frame to a matrix of pixel values
    #pixel_matrix = np.matrix(resized_frame.tolist())
    pixel_matrix = np.matrix(gray_frame.tolist())

    # Convert each element of the pixel matrix to an 8 digit binary number
    binary_matrix = np.vectorize(lambda x: format(x, '08b'))(pixel_matrix)

    #to_text = np.delete(binary_matrix, np.s_[3:119], axis=0)

    # # Delete column 2
    #to_text_this = np.delete(to_text, np.s_[3:159], axis=1)

    result += matrix_to_string(binary_matrix).replace("[", "").replace("]", "").replace("'", "") + " / "

filename = "person01_handwaving_d2_uncomp.txt"  # Replace this with the desired filename

with open(filename, "w") as f:
    f.write(result)

print(result)

# Release the video capture and writer objects
cap.release()
out.release()


import concurrent
import time

from App import App
import textdistance

from Utils import connectToDB, selectFromDB

# longest common subsequence similarity
def compareGotoh(namesList, baseNameList):
    pair = None
    pairList = []
    start = time.time()
    for name in namesList:
        minDistance = float('inf')
        for baseName in baseNameList:
            base = baseName
            res = textdistance.gotoh.distance(name, baseName)
            if res < minDistance:
                minDistance = res
                pair = (name, baseName)
        pairList.append(pair)
        # print(f'Matching Pair is {pair} and the Hamming distance is: {minDistance}')
    runTime = (time.time() - start)/60
    return 'Gotoh Distance', runTime, pairList

def compareStrCmp95(namesList, baseNameList):
    pair = None
    pairList = []
    start = time.time()
    for name in namesList:
        minDistance = float('inf')
        for baseName in baseNameList:
            base = baseName
            res = textdistance.strcmp95.distance(name, baseName)
            if res < minDistance:
                minDistance = res
                pair = (name, baseName)
        pairList.append(pair)
        # print(f'Matching Pair is {pair} and the Hamming distance is: {minDistance}')
    runTime = (time.time() - start)/60
    return 'StrCmp95 Distance', runTime, pairList

if __name__ == '__main__':
    # GUI section #
    app = App()
    app.startApp()
    # dataBaseCon = connectToDB('LAPTOP-VNSLHC31', 'BraudeProject')
    # baseNamesList = selectFromDB(dataBaseCon, 'SELECT * FROM [BraudeProject].[dbo].[AllVegNames]')
    # namesList = selectFromDB(dataBaseCon, 'SELECT Prod_Name FROM [BraudeProject].[dbo].[AllProds]')
    # namesList = [name[0] for name in namesList]
    # baseNamesList = [name[0] for name in baseNamesList]
    # resultSet = []
    # CountAlg1 = CountAlg2 = 0
    # with concurrent.futures.ThreadPoolExecutor() as executor:
    #     workThreads = [executor.submit(compareStrCmp95, namesList, baseNamesList), executor.submit(compareGotoh, namesList, baseNamesList)]
    #     print('Test Results:\n')
    #     for work in concurrent.futures.as_completed(workThreads):
    #         resultSet.append(work.result())
    #     Alg1, Time1, pairList1 = resultSet[0]
    #     Alg2, Time2, pairList2 = resultSet[1]
    #     print(f'Algorithm {Alg1} finish in: {Time1}')
    #     print(f'Algorithm {Alg2} finish in: {Time2}')
    #     print(f'Results Differences:')
    #     for set1, set2 in zip(pairList1, pairList2):
    #         if set1[1] != set2[1]:
    #             print('------------------------------')
    #             print(f'Algorithm {Alg1} result: {set1}')
    #             print(f'Algorithm {Alg2} result: {set2}')
    #             ans = int(input(f'Press 1 for {Alg1} Press 2 for {Alg2}'))
    #             if ans == 1:
    #                 CountAlg1 = CountAlg1 + 1
    #             if ans == 2:
    #                 CountAlg2 = CountAlg2 + 1
    #             print('------------------------------')
    #     if CountAlg1 > CountAlg2:
    #         print(f'Winner is {Alg1} with {CountAlg1} wins compare to {Alg2} with {CountAlg2}')
    #     else:
    #         print(f'Winner is {Alg2} with {CountAlg2} wins compare to {Alg1} with {CountAlg1}')


import bz2
import os
import xml.etree.ElementTree as ET

import time
from typing import Tuple, List

from WikiDBBZ2 import WikiDB, WikiDB_zlib

pathWikiBZ2 = 'C:\\Users\\16507\\Downloads\\enwiki-20201020-pages-articles-multistream.xml.bz2' #.xml.bz2 file path
pathWikiDB = 'C:\\Users\\16507\\Documents\\Projects\\WikipediaProject\\Literature-Biases---Wikipedia\\wiki.db'

FTR = ["novel"] #filter - these words must be in categories to append to the db - make sure the words are in lowercase
NON_FTR = ["births", "musical", "television series", "films"] #filter - these words must NOT be categories to append to the db - make sure the words are in lowercase

start_time = time.time()
bz2_file = bz2.BZ2File(pathWikiBZ2)

def passes_filter(categories: List[str]) -> bool:
    '''
    Returns a bool depending on if the categories passes the filter
    :param categories: a list of the categories
    '''
    crepr = repr(categories)

    if any([i in crepr for i in NON_FTR]): return False
    return all([i in crepr for i in FTR])

def hms_string(sec_elapsed: int) -> str:
    """
    Gets time in Hour:Minutes:Seconds
    :param sec_elapsed: seconds elapsed
    :return: Hour:Minutes:Seconds
    """
    h = int(sec_elapsed / (60 * 60))
    m = int((sec_elapsed % (60 * 60)) / 60)
    s = sec_elapsed % 60
    return "{}:{:>02}:{:>05.2f}".format(h, m, s)

def remove_category_tags(line: str) -> str:
    '''
    Assumes line param is a [[Category: ]] line and removes the tags
    :param line: decoded b'[[Category: ]] line
    '''
    start_tag = line.find('[[')
    end_tag = line.find(']]')
    return line[start_tag+11:end_tag]

def parseBZ2Page(file: bz2.BZ2File, page_line: bytes):
    '''
    Given a file and the bytes of the first line (<page> line) returns the categories, id, title, and decompressed file as a stringthrough a tuple
    :param file: bz2.BZ2File, actual file must be a .xml.bz2 file
    :param page_line: bytes
    '''
    categories = []

    decompressed_file_as_str = page_line.decode("utf-8")

    for line in file:
        decoded = line.decode("utf-8")
        decompressed_file_as_str += decoded

        if b'[[Category:' in line:
            categories.append(remove_category_tags(decoded))

        elif b'</page>' in line: #Once reading a </page> tag, exit.
            break

    root = ET.fromstring(decompressed_file_as_str)

    if root.find('ns').text != '0' or 'redirect title' in decompressed_file_as_str: return None

    title = root.find('title').text
    id = root.find('id').text

    '''
    Debugging statements
    print(decompressed_file_as_str)
    print('NS:', ns, '\n', 'Title:', title, '\n', 'ID:', id)
    raise Exception('Let me just take a peek!')
    '''

    return (categories, id, title, decompressed_file_as_str)

def main(file: bz2.BZ2File, db: WikiDB, compresssion = False) -> WikiDB:
    '''
    Parses a .xml.bz2 file and returns a dictionary {ID, [Categories]} of articles that pass the filter FTR
    :param file: bz2.BZFile object
    '''
    pc = 0 #page count
    ac = 0 #added count

    for line in file:
        if b'<page>' in line: #</page> indicates new Wikipedia page

            parsed_data = parseBZ2Page(file, line)

            if parsed_data: #parsed_data returns None when the page should be excluded
                (categories, id, title, decompressed_article) = parsed_data
                pc += 1
            else:
                continue

            if passes_filter(categories): #if categories passes filter, add to database
                ac += 1
                if compression:
                    db.insert(id, title, repr(categories), decompressed_article)
                else:
                    db.insert(id, title, repr(categories))

            if (pc % 150 == 0): #Print progress every 150 pages
                elapsed_time = time.time() - start_time
                print("{} articles parsed".format(pc), end=" ")
                print("{} articles passes filter".format(ac), end=" ")
                print("Elapsed time: {}".format(hms_string(elapsed_time)))

            #if pc >= 75: #For debugging
            #    break

    db.commit()
    file.close()
    print("Completed! \n")

    return db

if __name__ == '__main__':
    compression = bool(int(input('Do you want compressed articles to be a part of the sqlite database (1 for yes, 0 for no)?: ')))
    if compression:
        database = WikiDB_zlib(pathWikiDB)
    else:
        database = WikiDB(pathWikiDB)

    main(bz2_file, database, compression)


from uk.ac.ebi.brain.core import Brain
import json

all_ont = Brain()
all_ont.learn = ("") # fbbi
all_ont.learn = ("") # fbbt
all_ont.learn = ("") # fbdv

classlist = all_ont.getSubClasses("Thing", 0)

id_name = {}

for c in classlist:
    id_name[c] = all_ont.getLabel(c)

lookup = open("lookup", "w")

lookup.write(json.dump(id_name))


from functools import reduce
from typing import List

import numpy
import numpy as np
from collections import defaultdict

def gen_groups(importance: numpy.ndarray, dependency: numpy.ndarray, head=None):
    # 固定的头部不依赖其他位置点
    assert type(dependency) != numpy.ndarray or np.all(dependency[:, 1] != head)

    data_size, importance_size = importance.shape

    data_group = np.zeros(shape=data_size)
    # 按顺序，每个对象所属的组
    if type(importance) == np.ndarray:
        importance_cof = np.power(10, np.arange(importance_size)[::-1])
        new_importance = (importance * importance_cof[np.newaxis, np.newaxis, :]).sum(axis=-1).reshape(-1)

        importance_order = new_importance.argsort()[::-1]
        sorted_new_importance = new_importance[importance_order]

        group_end = np.argwhere(sorted_new_importance[1:] != sorted_new_importance[:-1]) + 1
        # 每组在排好序的的起止坐标
        group_index = np.stack([np.append(0, group_end), np.append(group_end, data_size)]).swapaxes(0, 1)

        for i in range(len(group_index)):
            st, ed = group_index[i]
            data_group[importance_order[st:ed]] = i
    # 头部分组
    if head != None:
        data_group[head] = -1

    return adjust_by_dependency(data_group, dependency)

def adjust_by_dependency(data_group: numpy.ndarray, dependency: numpy.ndarray):
    dependency_size, _ = dependency.shape
    # 拓扑排序
    graph = {}
    # 为了避免遍历时出错，不用defaultdict
    for i in np.unique(dependency):
        graph[i] = set()

    for depended, depend in dependency:
        graph[depend].add(depended)

    topology_order = get_topology_order(np.unique(dependency), graph)
    # 过滤出有依赖的
    topology_order = tuple(filter(lambda i: graph[i], topology_order))

    for v in topology_order:
        max_depend_group = max(map(lambda i: data_group[i], graph[v]))
        if max_depend_group + 1 > data_group[v]:
            # 因为重要度的限制，只有两种可能，1.前置同一个组，2.前置后面一个组, 这一点很重要，否则复杂度爆炸
            data_group[v] = max_depend_group + np.random.randint(2)
    return adjust_group_internal_order(data_group, graph)

def adjust_group_internal_order(data_group: np.ndarray, graph: dict):
    dependency_groups = {int(i): set() for i in np.unique(data_group)}
    # 领接链表都放进去
    for k, v in graph.items():
        dependency_groups[int(data_group[k])].add(k)
        for i in v:
            dependency_groups[int(data_group[i])].add(i)

    groups_dict = defaultdict(list)
    for i in range(len(data_group)):
        if i not in dependency_groups[data_group[i]]:
            groups_dict[data_group[i]].append(i)

    # 同一组内的依赖相关项按照拓扑排序随机插入
    for i in dependency_groups:
        dependency_list = dependency_groups[i]
        independent_list = groups_dict[i]
        old_in_len = len(independent_list)
        topology_order = get_topology_order(dependency_list, graph)
        insert_position = np.random.choice(np.arange(len(independent_list) + len(dependency_list)),
                                           size=len(dependency_list), replace=False)
        insert_position.sort()
        for i in range(len(dependency_list)):
            independent_list.insert(insert_position[i], topology_order[i])
        assert len(independent_list) == old_in_len + len(dependency_list)

    sorted_group_list = tuple(map(lambda i: groups_dict[i], sorted(groups_dict.keys())))
    assert sum(map(len, sorted_group_list), 0) == len(data_group)
    return sorted_group_list

def get_topology_order(points: set, graph: dict):
    searched = set()
    topology_order = []

    def dfs(g: dict):
        random_start = np.random.permutation(tuple(filter(lambda i: i not in searched, g.keys())))
        # 这里是惰性的，filter 不是一次性生效, searched生效会有影响
        for v in random_start:
            dfs_visit(g, v)

    def dfs_visit(g: dict, v: int):
        if v in searched:
            return
        searched.add(v)
        for i in filter(lambda j: j not in searched, g[v]):
            dfs_visit(g, i)
        if v in points:
            topology_order.append(v)

    dfs(graph)

    return topology_order

if __name__ == '__main__':
    data_size = 10
    importance_size = 2

    importance = np.zeros(shape=[data_size, importance_size])
    data_position = np.random.choice(np.arange(data_size), size=(data_size // 3, importance_size), replace=False)
    importance_value = np.random.choice(np.arange(10), size=(data_size // 3, importance_size))
    importance[data_position, np.arange(importance_size)[np.newaxis, :]] = importance_value

    dependency_size = 4
    dependency = np.stack([np.random.choice(np.arange(data_size), size=(2), replace=False)
                           for i in range(dependency_size)])

    group_list = gen_groups(importance, dependency)


#!/usr/bin/env python
#

"""
This script takes annotation in bed12 format without UTRs and adds UTRs from a transcriptome assembly (e.g. stringtie)
"""

import argparse
from collections import defaultdict
from collections import Counter
import sys

__author__ = "Ekaterina Osipova, 2020."

def get_all_intron_coords(transcript):
    ## Takes bed12 transcript line and returns a list with intron intervals: [(chrom, x1, x2), (chrom, x3, x4), ..]

    chrom = transcript.split()[0]
    start = int(transcript.split()[1])
    exon_number = int(transcript.split()[9])
    intron_coord_list = []
    block_starts = [start + i for i in map(int, transcript.split()[11].rstrip(',').split(','))]
    block_sizes = map(int, transcript.split()[10].rstrip(',').split(','))
    for i in range(exon_number - 1):
        intron_coord_list.append((chrom, block_starts[i] + block_sizes[i], block_starts[i + 1]))
    return intron_coord_list

def read_rnaseq_bed(rnaseq_file):
    ## Reads RNAseq annotation file (no CDS regions) and adds all introns to rnaseq_coord_dict:
    ## {(chrom, x1, x2): [transcript1, transcript2, ..], }

    rnaseq_coord_dict = defaultdict(list)
    with open(rnaseq_file, 'r') as inf:
        for bed_line in inf.readlines():
            intron_coord_list = get_all_intron_coords(bed_line)
            for intron in intron_coord_list:
                rnaseq_coord_dict[intron].append(bed_line)
    return rnaseq_coord_dict

def prepare_5prime_blocks(transcript, cut_coord):
    ## Takes bed12 transcript line and one coord where to cut it;
    ## returns list of starts and sizes of non-coding blocks; works with 5'-UTR

    noncoding_blocks = []
    start = int(transcript.split()[1])
    abs_block_starts_list = [start + int(i) for i in transcript.split()[11].rstrip(',').split(',')]
    block_sizes_list = [int(i) for i in transcript.split()[10].rstrip(',').split(',')]

    i = 0
    while abs_block_starts_list[i] + block_sizes_list[i] < cut_coord:
        noncoding_blocks.append((abs_block_starts_list[i], block_sizes_list[i]))
        i += 1
    # add the cut block
    noncoding_blocks.append((abs_block_starts_list[i], cut_coord - abs_block_starts_list[i]))
    return noncoding_blocks

def prepare_3prime_blocks(transcript, cut_coord):
    ## Takes bed12 transcript line and one coord where to cut it;
    ## returns list of starts and sizes of non-coding blocks; works with 3'-UTR

    noncoding_blocks = []
    start = int(transcript.split()[1])
    abs_block_starts_list = [start + int(i) for i in transcript.split()[11].rstrip(',').split(',')]
    block_sizes_list = [int(i) for i in transcript.split()[10].rstrip(',').split(',')]

    abs_block_starts_list.reverse()
    block_sizes_list.reverse()
    i = 0
    while abs_block_starts_list[i] > cut_coord:
        noncoding_blocks.append((abs_block_starts_list[i], block_sizes_list[i]))
        i += 1
    # add the cut block
    noncoding_blocks.append((cut_coord, block_sizes_list[i] - cut_coord + abs_block_starts_list[i]))
    # do you need to reverse it back??
    noncoding_blocks.reverse()
    return noncoding_blocks

def read_anno_bed(anno_file, rnaseq_coord_dict):
    ## Reads annotation file without UTRs and finds if first/last intron matches perfectly anything in rnaseq_coord_dict

    transcript_dict = defaultdict(list)
    with open(anno_file, 'r') as inf:
        for bed_line in inf.readlines():
            name = bed_line.split()[3]
            start = int(bed_line.split()[1])
            end = int(bed_line.split()[2])
            exon_number = int(bed_line.split()[9])
            transcript_info = bed_line

            ## Initiate and update UTR lists in the transcript_dict (3'UTRs and 5'UTRs separately)
            utrs5_list = []
            utrs3_list = []

            if exon_number > 1:
                intron_coord_list = get_all_intron_coords(bed_line)
                first_intron_coords = intron_coord_list[0]
                last_intron_coords = intron_coord_list[-1]

                if (first_intron_coords in rnaseq_coord_dict):
                    utrs5_list = [i for i in rnaseq_coord_dict[first_intron_coords] if int(i.split()[1]) < start]
                if (last_intron_coords in rnaseq_coord_dict):
                    utrs3_list = [i for i in rnaseq_coord_dict[last_intron_coords] if int(i.split()[2]) > end]

            transcript_dict[name].append((transcript_info, utrs5_list, utrs3_list))
    return transcript_dict

def get_max_most_common(list, utr_type):
    ## Finds most common element in the list; if there're multiple, returns max of those

    count_dict = Counter(list)
    maxcount = count_dict.most_common(1)[0][1]
    if utr_type == 5:
        max_most_common = min([i for i in count_dict if count_dict[i] == maxcount])
    else:
        max_most_common = max([i for i in count_dict if count_dict[i] == maxcount])
    return max_most_common

def add_utr_blocks(transcript_info, utr5_blocks, utr3_blocks):
    ## Given transcript annotation line updates it with utrs

    start = int(transcript_info.split()[1])
    chrom = transcript_info.split()[0]
    exon_number = int(transcript_info.split()[9])
    name = transcript_info.split()[3]
    trans_starts = [i + start for i in map(int, transcript_info.split()[11].rstrip(',').split(','))]
    trans_sizes = map(int, transcript_info.split()[10].rstrip(',').split(','))

    utr5_starts = [i[0] for i in utr5_blocks]
    utr5_sizes = [i[1] for i in utr5_blocks]
    utr3_starts = [i[0] for i in utr3_blocks]
    utr3_sizes = [i[1] for i in utr3_blocks]

    start_update = utr5_starts[0]
    end_update = utr3_starts[-1] + utr3_sizes[-1]

    ## update starts and block sizes
    new_trans_starts = [i - utr5_starts[0] for i in utr5_starts + trans_starts[1:] + utr3_starts[1:]]
    block_starts_update = ','.join(map(str, new_trans_starts)) + ','

    # make updates of block sizes if it's a single exon gene
    if exon_number == 1:
        new_trans_sizes = utr5_sizes[:-1] + [utr5_sizes[-1] + trans_sizes[0] + utr3_sizes[0]] + utr3_sizes[1:]
    # make updates of block sizes if number of exons > 1
    else:
        new_trans_sizes = utr5_sizes[:-1] + [utr5_sizes[-1] + trans_sizes[0]] + trans_sizes[1:-1] + \
                            [trans_sizes[-1] + utr3_sizes[0]] + utr3_sizes[1:]
    block_sizes_update = ','.join(map(str, new_trans_sizes)) + ','

    ## combine all info in a new annotation line
    exon_number_update = len(new_trans_starts)
    bed_line_list_update = [chrom, str(start_update), str(end_update), name] + transcript_info.split()[4:9] + \
                           [str(exon_number_update), block_sizes_update, block_starts_update]
    bed_line_update = '\t'.join(bed_line_list_update)

    return bed_line_update

def update_annotation(transcript_dict):
    ## Adds 5'- and 3'-UTRs for each transcript in given transcript_dict
    ## Runs add_utrs() function that work with an individual transcript

    for name in transcript_dict:
        for transcript in transcript_dict[name]:
            transcript_info = transcript[0]
            cut_utr5 = int(transcript_info.split()[1])
            cut_utr3 = int(transcript_info.split()[2])
            utr5_transcript_list = transcript[1]
            utr3_transcript_list = transcript[2]

            # check if all non-coding starts > start !!! not yet done
            # print('utr5 list: ', utr5_transcript_list)
            # print('utr3 list: ', utr3_transcript_list)

            # if there are updates for this transcript, get the most common coordinate
            if utr5_transcript_list != []:
                utr5_start_list = [int(i.split()[1]) for i in utr5_transcript_list]
                utr5 = get_max_most_common(utr5_start_list, utr_type=5)
                maxcount_transcript_index = [int(i.split()[1]) for i in utr5_transcript_list].index(utr5)
                utr5_blocks = prepare_5prime_blocks(utr5_transcript_list[maxcount_transcript_index], cut_utr5)
            else:
                utr5_blocks = [(int(transcript_info.split()[1]), 0)]

            if utr3_transcript_list != []:
                utr3_end_list = [int(i.split()[2]) for i in utr3_transcript_list]
                utr3 = get_max_most_common(utr3_end_list, utr_type=3)
                maxcount_transcript_index = [int(i.split()[2]) for i in utr3_transcript_list].index(utr3)
                utr3_blocks = prepare_3prime_blocks(utr3_transcript_list[maxcount_transcript_index], cut_utr3)
            else:
                utr3_blocks = [(int(transcript_info.split()[2]), 0)]

            bed_line_update  = add_utr_blocks(transcript_info, utr5_blocks, utr3_blocks)
            print(bed_line_update)
    return

def main():
    ## Parse arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('-r', '--rnaseq', type=str, help='transcripts assembled from RNAseq, e.g. stringtie in bed12 format')
    parser.add_argument('-a', '--anno', type=str, help='bed12 annotation file to add UTRs to')
    args = parser.parse_args()

    ## bed12 format:
    ## chrom[0] start[1] end[2] name[3] score[4] strand[5] cds_start[6] cds_end[7] rgb[8] count[9]\
    ##  block_sizes[10] block_starts[11]

    ## Read stringtie assembly into a dictionary
    rnaseq_coord_dict = read_rnaseq_bed(args.rnaseq)

    ## Read annotation file checking if first/last blocks overlap blocks in rnaseq
    transcript_dict = read_anno_bed(args.anno, rnaseq_coord_dict)

    ## Add UTRs to the transcripts where possible
    update_annotation(transcript_dict)

if __name__ == "__main__":
    main()


from datetime import datetime
from pydantic import BaseModel

class SkillCreate(BaseModel):
    skill_type: str
    skill_name: str

    class Config:
        orm_mode = True

class SkillResponse(SkillCreate):
    id: int
    user_id: int
    created_at: datetime


import matplotlib.pyplot as plt
import numpy as np
from parse_sample import parse_sample
from scipy.stats import lognorm, gamma
from scipy.special import digamma, polygamma
from math import log, sqrt

# Return the log normal maximum likelihood estimators
def estimate_log_normal(data):
    n = len(data)
    mu = sum(np.log(data))/n
    sigma = sum(list(map(lambda x: (log(x) - mu)**2, data)))
    return mu, sqrt(sigma/n)

# Return an approximation of the Gamma maximum likelihood estimators
# Note: scale and shape parametrization is used
def estimate_gamma(data):
    n = len(data)
    x_bar = sum(data)/n
    lnx_bar = sum(np.log(data))/n
    a_i = 0.5/(log(x_bar) - lnx_bar)
    x_i = 1/a_i + (lnx_bar - log(x_bar) + log(a_i) - digamma(a_i))/(a_i**2 * (1/a_i - polygamma(1,a_i)))

    for i in range(4):
        a_i = 1/x_i
        x_i = 1/a_i + (lnx_bar - log(x_bar) + log(a_i) - digamma(a_i))/(a_i**2 * (1/a_i - polygamma(1,a_i)))

    beta = x_bar/a_i
    return  1/x_i,beta

def plot_histogram_with_distr(values):
    # Create the histogram
    n, bins, patches = plt.hist(values, density=True, bins='auto', edgecolor='white')

    # Calculate the bin widths
    bin_widths = bins[1:] - bins[:-1]

    # Overlay gamma distribution
    a, b = estimate_gamma(values)
    x = np.linspace(0, max(values), 100)
    pdf = gamma.pdf(x, a, loc=0, scale=b)
    plt.plot(x, pdf, lw=2, label='Gamma')

    # Overlay lognormal distribution
    u, sigma = estimate_log_normal(values)
    x = np.linspace(0, max(values), 100)
    pdf = lognorm.pdf(x, s=sigma, loc=0, scale=np.exp(u))
    plt.plot(x, pdf, lw=2, label='Lognormal')

    # Set plot properties
    plt.xlabel('Valor')
    plt.ylabel('Frecuencia')
    plt.title('Histograma y distribuciones Gamma y Log Normal')
    plt.legend()

    plt.savefig("../plots/histogram_with_estimated_distributions")
    plt.clf()

plot_histogram_with_distr(parse_sample("../sample23.dat"))


import os
import logging
import uuid
from azure.identity import DefaultAzureCredential
from azure.keyvault.secrets import SecretClient

class KeyVault:
    def __init__(self):
        # DefaultAzureCredential() expects the following environment variables:
        # * AZURE_CLIENT_ID
        # * AZURE_CLIENT_SECRET
        # * AZURE_TENANT_ID

        credential = DefaultAzureCredential()
        self.secret_client = SecretClient(
            vault_url=os.environ["AZURE_PROJECT_URL"], credential=credential
        )

        self.secret_name = "secret-name-" + uuid.uuid1().hex
        self.secret_Value = "secret-value"

    def set_secret(self):
        logging.info("Setting a secret...")
        self.secret_client.set_secret(self.secret_name, self.secret_Value)
        logging.info("\tdone")

    def get_secret(self):
        logging.info("Getting a secret...")
        secret = self.secret_client.get_secret(self.secret_name)
        logging.info("\tdone, secret: (" + secret.name + "," + secret.value + ").")

    def delete_secret(self):
        logging.info("Deleting a secret...")
        deleted_poller = self.secret_client.begin_delete_secret(self.secret_name)
        deleted_secret = deleted_poller.result()
        logging.info("\tdone: " + deleted_secret.name)

    def run(self):
        logging.info("")
        logging.info("------------------------")
        logging.info("Key Vault - Secrets\nIdentity - Credential")
        logging.info("------------------------")
        logging.info("1) Set a secret")
        logging.info("2) Get that secret")
        logging.info("3) Delete that secret (Clean up the resource)")
        logging.info("")

        try:
            self.set_secret()
            self.get_secret()
        finally:
            self.delete_secret()


#
# motandoadm/urls.py
#

from django.urls import path
from django.contrib.auth.decorators import login_required

from . import views

app_name = 'motandoadm'

urlpatterns = [
    path('home', login_required(views.HomeAdminView.as_view()), name='home'),

    path('moto', login_required(views.ListAllMotorcycleAdminView.as_view()), name='motorcycle_all_list'),

    path('moto/marca', login_required(views.ListMotorcycleBrandAdminView.as_view()), name='motorcycle_brand_list'),
    path('moto/marca/novo', login_required(views.AddMotorcycleBrandAdminView.as_view()), name='motorcycle_brand_add'),
    path('moto/marca/<int:brand_id>', login_required(views.EditMotorcycleBrandAdminView.as_view()), name='motorcycle_brand_edit'),
    path('moto/marca/remover/<int:brand_id>', login_required(views.DeleteMotorcycleBrandAdminView.as_view()), name='motorcycle_brand_delete'),

    path('moto/marca/modelo', login_required(views.SelectMotorcycleBrandModelAdminView.as_view()), name='motorcycle_brand_model_select'),
    path('moto/marca/modelo/novo', login_required(views.AddMotorcycleBrandModelAdminView.as_view()), name='motorcycle_brand_model_add'),
    path('moto/marca/<int:brand_id>/modelo', login_required(views.ListMotorcycleBrandModelAdminView.as_view()), name='motorcycle_brand_model_list'),
    path('moto/marca/<int:brand_id>/modelo/<int:model_id>', login_required(views.EditMotorcycleBrandModelAdminView.as_view()), name='motorcycle_brand_model_edit'),
    path('moto/marca/<int:brand_id>/modelo/<int:model_id>/remover', login_required(views.DeleteMotorcycleBrandModelAdminView.as_view()), name='motorcycle_brand_model_delete'),

    path('moto/marca/modelo/versao', login_required(views.SelectMotorcycleBrandModelVersionAdminView.as_view()), name='motorcycle_brand_model_version_select'),
    path('moto/marca/modelo/versao/novo', login_required(views.AddMotorcycleBrandModelVersionAdminView.as_view()), name='motorcycle_brand_model_version_add'),
    path('moto/marca/<int:brand_id>/modelo/<int:model_id>/versao', login_required(views.ListMotorcycleBrandModelVersionAdminView.as_view()), name='motorcycle_brand_model_version_list'),
    path('moto/marca/<int:brand_id>/modelo/<int:model_id>/versao/<int:version_id>', login_required(views.EditMotorcycleBrandModeVersionlAdminView.as_view()), name='motorcycle_brand_model_version_edit'),
    path('moto/marca/modelo/versao/<int:version_id>/remover', login_required(views.DeleteMotorcycleBrandModelVersionAdminView.as_view()), name='motorcycle_brand_model_version_delete'),

    #path('motorcycle/brand/<str:brand_id>/model/<int:model_id>/version/<int:version_id>', login_required(views.EditMotorcycleAdminView.as_view()), name='edit'),
]


import datetime
from decimal import Decimal, ROUND_HALF_UP
from dateutil.parser import parse as time_parser
import pandas as pd

from enums import CapitalType

class LocalCapital:
    def __init__(self):
        self.path = "data/funds.csv"
        # NUMBER,TIME,USER,TYPE,FLOW,STOCK,REMARK
        self.df = pd.read_csv(self.path, dtype={0: int, 1: str, 2: str, 3: str, 4: str, 5: str, 6: str})
        self.administrator = "j32u4ukh"
        self.users = ["ahuayeh"]
        self.number = self.getLastNumber()

    def getLastNumber(self):
        # [146, '2021/9/30', 'j32u4ukh', 'revenue', '2833', '421635', '79']
        data = list(self.df.iloc[-1])

        return data[0]

    def getUsersCapital(self):
        df = self.df[self.df["TYPE"] == CapitalType.Capital.value]
        admin_capital = self.getUserCapital(df=df, user=self.administrator, preprocess=True)
        users_capital = {self.administrator: admin_capital}

        for user in self.users:
            users_capital[user] = self.getUserCapital(df=df, user=user, preprocess=True)

        return users_capital

    @staticmethod
    def getUserCapital(df, user, preprocess=False):
        capital = Decimal("0")

        if preprocess:
            user_df = df[df["USER"] == user]
        else:
            user_df = df[(df["USER"] == user) & (df["TYPE"] == CapitalType.Capital.value)]

        n_data = len(user_df)

        for i in range(n_data):
            # NUMBER,TIME,USER,TYPE,FLOW,STOCK,REMARK
            data = user_df.iloc[i]
            capital += Decimal(data["FLOW"])

        return capital

    def getUsersStock(self, time: datetime.datetime = datetime.datetime.today()):
        self.df["TIME"] = pd.to_datetime(self.df["TIME"])
        df = self.df[self.df["TIME"] <= time]
        admin_stock = self.getUserStock(df=df, user=self.administrator, preprocess=True)
        users_stock = {self.administrator: admin_stock}

        for user in self.users:
            users_stock[user] = self.getUserStock(df=df, user=user, preprocess=True)

        return users_stock

    @staticmethod
    def getUserStock(df, user, time: datetime.datetime = datetime.datetime.today(), preprocess=False):
        if preprocess:
            user_df = df[df["USER"] == user]
        else:
            df["TIME"] = pd.to_datetime(df["TIME"])
            user_df = df[(df["USER"] == user) & (df["TIME"] <= time)]

        if len(user_df) == 0:
            return Decimal("0")

        data = user_df.iloc[-1]
        stock = Decimal(data["STOCK"])

        return stock

    def getCapitalSummary(self, time: datetime.datetime = datetime.datetime.today()):
        summary = dict()
        users = self.users.copy() + [self.administrator]

        capitals = self.getUsersCapital()
        last_time = time - datetime.timedelta(days=1)
        last_stocks = self.getUsersStock(time=last_time)
        curr_stocks = self.getUsersStock(time=time)

        for user in users:
            summary[user] = dict(capital=capitals[user],
                                 last_stock=last_stocks[user],
                                 stock=curr_stocks[user])

        return summary

    def allocateRevenue(self, deal_time: datetime.datetime, remark: str, trade_revenue: Decimal):
        summary = self.getCapitalSummary(time=deal_time)
        # print("summary:", summary)
        # {'j32u4ukh':
        #   {'capital': Decimal('451595.0'), 'last_stock': Decimal('456592.00'), 'stock': Decimal('456592.00')},
        # 'ahuayeh':
        #   {'capital': Decimal('200000.0'), 'last_stock': Decimal('203456'), 'stock': Decimal('203456')}}

        rate_dict = dict()
        total_last_stock = Decimal("0")

        # 取出所有用戶前一天的資金存量
        for user in summary.keys():
            user_summary = summary[user]
            last_stock = user_summary["last_stock"]

            rate_dict[user] = last_stock
            total_last_stock += last_stock

        # 根據用戶前一天的資金存量，計算資金比例
        for user, stock in rate_dict.items():
            rate_dict[user] = stock / total_last_stock
            # print(f"Last {user}: {rate_dict[user]}% ({stock})")

        allocated_revenu = Decimal("0")

        for user in self.users:
            stock = summary[user]["stock"]
            rate = rate_dict[user]

            # 將 trade_revenue 根據 user 交易日前一天的資金存量比例，計算 user 分配到的收益
            user_revenu = (trade_revenue * rate).quantize(Decimal('0'), ROUND_HALF_UP)

            # 累計已分配收益
            allocated_revenu += user_revenu

            # 更新 user 的資金存量
            stock += user_revenu

            self.add(time=deal_time,
                     user=user,
                     capital_type=CapitalType.Revenue,
                     flow=user_revenu,
                     stock=stock,
                     remark=remark)

        stock = summary[self.administrator]["stock"]

        # administrator 的損益為"總損益 - 已分配損益"，確保不因四捨五入的誤差，造成 "總分配損益" 和 "總損益" 有所偏差
        administrator_revenu = (trade_revenue - allocated_revenu).quantize(Decimal('0'), ROUND_HALF_UP)
        stock += administrator_revenu
        stock = stock.quantize(Decimal('0'), ROUND_HALF_UP)

        self.add(time=deal_time,
                 user=self.administrator,
                 capital_type=CapitalType.Revenue,
                 flow=administrator_revenu,
                 stock=stock,
                 remark=remark)

        self.save()

    def add(self, time: datetime.datetime, user: str, capital_type: CapitalType, flow: Decimal, stock: Decimal,
            remark: str):
        # NUMBER,TIME(2020/06/17),USER,TYPE,FLOW,STOCK,REMARK
        self.number += 1
        self.df = self.df.append({"NUMBER": self.number,
                                  "TIME": time.strftime("%Y/%m/%d"),
                                  "USER": user,
                                  "TYPE": capital_type.value,
                                  "FLOW": str(flow),
                                  "STOCK": str(stock),
                                  "REMARK": remark
                                  }, ignore_index=True)

    def save(self):
        self.df.sort_values(by=["NUMBER"], inplace=True)

        self.df["TIME"] = pd.to_datetime(self.df["TIME"])

        # 資金數據更新
        self.df.to_csv(self.path, index=False)

if __name__ == "__main__":
    lc = LocalCapital()
    # lc.save()

    records = """97,3003,2021-10-19,2021-11-23,91.00,95.00,1000.0,91039.00,325,3636.00
98,3588,2021-11-10,2021-11-23,138.50,155.00,1000.0,138559.00,531,15910.00"""

    trade_records = records.split("\n")

    for trade_record in trade_records:
        number, _, _, sell_time, _, _, _, _, _, revenue = trade_record.split(',')
        print(number, sell_time, revenue)
        # time_parser
        lc.allocateRevenue(deal_time=time_parser(sell_time), remark=number, trade_revenue=Decimal(revenue))

    # # NUMBER,TIME,USER,TYPE,FLOW,STOCK,REMARK
    # # 85,3048,2021-08-25,2021-10-12,0.00,29.50,0.1,1.00,8.00,2941.00
    # lc.allocateRevenue(deal_time=datetime.datetime(2021, 10, 12), remark="85", trade_revenue=Decimal("2941"))
    #
    # # 86,3048,2021-08-25,2021-10-12,0,0,0,0,0,1890
    # lc.allocateRevenue(deal_time=datetime.datetime(2021, 10, 12), remark="86", trade_revenue=Decimal("1890"))

    lc.save()


from pprint import pformat, pprint  # noqa
import re
import logging

from sqlalchemy import inspect
from sqlalchemy.ext import hybrid
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import column_property, relationship  # noqa
from sqlalchemy import func
from sqlalchemy import (  # noqa
    Column,
    String,
    Integer,
    ForeignKey,
    DateTime,
)
from sqlalchemy.ext.declarative import declarative_base, declared_attr

log = logging.getLogger(__name__)

Base = declarative_base()

snake_case_re = re.compile('((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))')

class CommonColumns(Base):
    __abstract__ = True
    _created = Column(DateTime, default=func.now())
    _updated = Column(DateTime, default=func.now(), onupdate=func.now())
    _etag = Column(String(40))

    @hybrid_property
    def _id(self):
        """
        Eve backward compatibility
        """
        return self.id

    @declared_attr
    def __tablename__(cls):
        return snake_case_re.sub(r'_\1', cls.__name__).lower()

    def jsonify(self):
        """
        Used to dump related objects to json
        """
        relationships = inspect(self.__class__).relationships.keys()
        mapper = inspect(self)
        attrs = [
            a.key for a in mapper.attrs
            if a.key not in relationships and
            a.key not in mapper.expired_attributes
        ]
        attrs += [
            a.__name__
            for a in inspect(self.__class__).all_orm_descriptors
            if a.extension_type is hybrid.HYBRID_PROPERTY
        ]
        log.debug("{}".format(self.__class__.__name__))
        log.debug(pformat(attrs))
        return dict([(c, getattr(self, c, None)) for c in attrs])

    def register(self):
        pprint(self.__class__)
        pprint(self.__tablename__)


import math

import torch
from torch import nn as nn
from torch.nn import init as init
from torch.nn.modules.batchnorm import _BatchNorm

@torch.no_grad()
def default_init_weights(module_list, scale=1, bias_fill=0, **kwargs):
    """Initialize network weights.

    Args:
        module_list (list[nn.Module] | nn.Module): Modules to be initialized.
        scale (float): Scale initialized weights, especially for residual
            blocks. Default: 1.
        bias_fill (float): The value to fill bias. Default: 0
        kwargs (dict): Other arguments for initialization function.
    """
    if not isinstance(module_list, list):
        module_list = [module_list]
    for module in module_list:
        for m in module.modules():
            if isinstance(m, nn.Conv2d):
                init.kaiming_normal_(m.weight, **kwargs)
                m.weight.data *= scale
                if m.bias is not None:
                    m.bias.data.fill_(bias_fill)
            elif isinstance(m, nn.Linear):
                init.kaiming_normal_(m.weight, **kwargs)
                m.weight.data *= scale
                if m.bias is not None:
                    m.bias.data.fill_(bias_fill)
            elif isinstance(m, _BatchNorm):
                init.constant_(m.weight, 1)
                if m.bias is not None:
                    m.bias.data.fill_(bias_fill)

def make_layer(basic_block, num_basic_block, **kwarg):
    """Make layers by stacking the same blocks.

    Args:
        basic_block (nn.module): nn.module class for basic block.
        num_basic_block (int): number of blocks.

    Returns:
        nn.Sequential: Stacked blocks in nn.Sequential.
    """
    layers = []
    for _ in range(num_basic_block):
        layers.append(basic_block(**kwarg))
    return nn.Sequential(*layers)

class ResidualBlockNoBN(nn.Module):
    """Residual block without BN.

    It has a style of:
        ---Conv-ReLU-Conv-+-
         |________________|

    Args:
        num_feat (int): Channel number of intermediate features.
            Default: 64.
        res_scale (float): Residual scale. Default: 1.
        pytorch_init (bool): If set to True, use pytorch default init,
            otherwise, use default_init_weights. Default: False.
    """

    def __init__(self, num_feat=64, res_scale=1, pytorch_init=False):
        super(ResidualBlockNoBN, self).__init__()
        self.res_scale = res_scale
        self.conv1 = nn.Conv2d(num_feat, num_feat, 3, 1, 1, bias=True)
        self.conv2 = nn.Conv2d(num_feat, num_feat, 3, 1, 1, bias=True)
        self.relu = nn.ReLU(inplace=True)

        if not pytorch_init:
            default_init_weights([self.conv1, self.conv2], 0.1)

    def forward(self, x):
        identity = x
        out = self.conv2(self.relu(self.conv1(x)))
        return identity + out * self.res_scale

class Upsample(nn.Sequential):
    """Upsample module.

    Args:
        scale (int): Scale factor. Supported scales: 2^n and 3.
        num_feat (int): Channel number of intermediate features.
    """

    def __init__(self, scale, num_feat):
        m = []
        if (scale & (scale - 1)) == 0:  # scale = 2^n
            for _ in range(int(math.log(scale, 2))):
                m.append(nn.Conv2d(num_feat, 4 * num_feat, 3, 1, 1))
                m.append(nn.PixelShuffle(2))
        elif scale == 3:
            m.append(nn.Conv2d(num_feat, 9 * num_feat, 3, 1, 1))
            m.append(nn.PixelShuffle(3))
        else:
            raise ValueError(f'scale {scale} is not supported. '
                             'Supported scales: 2^n and 3.')
        super(Upsample, self).__init__(*m)

# TODO: may write a cpp file
def pixel_unshuffle(x, scale):
    """ Pixel unshuffle.

    Args:
        x (Tensor): Input feature with shape (b, c, hh, hw).
        scale (int): Downsample ratio.

    Returns:
        Tensor: the pixel unshuffled feature.
    """
    b, c, hh, hw = x.size()
    out_channel = c * (scale ** 2)
    assert hh % scale == 0 and hw % scale == 0
    h = hh // scale
    w = hw // scale
    x_view = x.view(b, c, h, scale, w, scale)
    return x_view.permute(0, 1, 3, 5, 2, 4).reshape(b, out_channel, h, w)


from . import NeuroBenchProcessor
from torchaudio.transforms import MFCC
import torch

class MFCCProcessor(NeuroBenchProcessor):
    """ Does MFCC computation on dataset using torchaudio.transforms.MFCC.
    Call expects loaded .wav data and targets as a tuple (data, targets).
    Expects sample_rate to be the same for all samples in data.
    """
    def __init__(
        self,
        sample_rate: int = 16000,
        n_mfcc: int = 40,
        dct_type: int = 2,
        norm: str = "ortho",
        log_mels: bool = False,
        melkwargs: dict = None,
    ):
        super(NeuroBenchProcessor).__init__()
        """
        Args:
            sample_rate (int, optional): Sample rate of the audio signal. (Default: 16000)
            n_mfcc (int, optional): Number of MFCC coefficients to retain. (Default: 40)
            dct_type (int, optional): Type of DCT (discrete cosine transform) to use. (Default: 2)
            norm (str, optional): Norm to use. (Default: "ortho")
            log_mels (bool, optional): Whether to use log-mel spectrograms instead of db-scaled. (Default: False)
            melkwargs (dict or None, optional): Arguments for MelSpectrogram. (Default: None)
        """
        self.sample_rate = sample_rate
        self.n_mfcc = n_mfcc
        self.dct_type = 2
        self.norm = norm
        self.log_mels = log_mels
        self.melkwargs = melkwargs

        self.mfcc = MFCC(
            sample_rate=self.sample_rate,
            n_mfcc=self.n_mfcc,
            dct_type=self.dct_type,
            norm=self.norm,
            log_mels=self.log_mels,
            melkwargs=self.melkwargs,
        )

    def __call__(self, dataset):
        """ Executes the MFCC computation on the dataset.

        Args:
            dataset (tuple): A tuple of (data, targets).

        Returns:
            results: mfcc applied on data
            targets: targets from dataset
        """
        self.dataset_validity_check(dataset)

        data, targets = dataset
        if isinstance(data, list):
            data = torch.vstack(data)

        self.results = self.mfcc(data)

        return self.results, targets

    @staticmethod
    def dataset_validity_check(dataset):
        """ Checks if dataset is a tuple with length two.
        """
        if not isinstance(dataset, tuple):
            raise TypeError("Expected dataset to be tuple")

        if not len(dataset) == 2:
            raise ValueError("Dataset tuple should have values as (data, targets)")


from dataclasses import dataclass

from server.service.helper.dict_helper import normalize
from server.service.strategy.base import BaseStrategy

def reset_function(n):
    return 1 / (2 ** n)

@dataclass
class SmoothStrategy(BaseStrategy):
    def update(self, *, indices_selected: list[int], **kwargs) -> list[float]:
        reset_value = reset_function(len(self.weight_list))
        for index_selected in indices_selected:
            self.weight_list[index_selected] = reset_value
        self.weight_list = normalize(self.weight_list)
        return self.weight_list


import datetime
from prime import is_prime

def from_right(num):
    string = str(num)
    length = len(string)
    for beg in range(length):
        if not is_prime(int(string[beg:])):
            return False
    return True

def main():
    L = [[2, 3, 5, 7]]
    size = 1
    while True:
        l = []
        for i in L[size - 1]:
            num = i * 10
            for j in [1, 3, 7, 9]:
                if is_prime(num + j):
                    l.append(num + j)
        if len(l) == 0:
            break
        L.append(l)
        size += 1

    res = 0
    for (i, l) in enumerate(L):
        if i == 0:
            continue
        for x in l:
            if x % 10 in (3, 7) and from_right(x):
                res += x
    return res

beg = datetime.datetime.now()
ans = main()
end = datetime.datetime.now()

print("answer:", ans)
print("time:", end - beg)


from collections import deque

class MyStack:

    def __init__(self):
        self.queue = deque()
        self.size = 0

    def push(self, x: int) -> None:
        size = len(self.queue)
        self.queue.append(x)
        for _ in range(size):
            self.queue.append(self.queue.popleft())

        self.size += 1

    def pop(self) -> int:
        if self.empty():
            return -1

        self.size -= 1
        return self.queue.popleft()

    def top(self) -> int:
        if self.empty():
            return -1

        return self.queue[0]

    def empty(self) -> bool:
        return len(self.queue) == 0

if __name__ == '__main__':
    obj = MyStack()

    obj.push(1)
    obj.push(2)
    assert obj.top() == 2
    assert obj.pop() == 2
    assert obj.empty() == False


import pandas as pd
from astropy import units as u
from astropy.table import QTable

from cluster import Cluster, temp_from_vdisp

def variance(err_neg, err_pos):
    n_temp = temp_from_vdisp(err_neg)
    p_temp = temp_from_vdisp(err_pos)
    return n_temp + p_temp

def load_clusters(nrows=None):
    # given an integer nrows, returns nrows Clusters generated from GalWeight cluster dataset
    galwcls=pd.read_csv('data/galwcls.dat', sep='|', header=None, nrows=nrows)
    cls_data = {'sig500': galwcls[:][8],
            'M500': galwcls[:][11],
            'r200': galwcls[:][13],
            'sig200':galwcls[:][15],
            'err_neg':galwcls[:][16],
            'err_pos':galwcls[:][17],
            'M200':galwcls[:][18]}
    units = {'sig500': u.km/u.s,
            'M500': u.Msun,
            'r200': u.Mpc,
            'sig200': u.km/u.s,
            'err_neg':u.km/u.s,
            'err_pos':u.km/u.s,
            'M200': u.Msun, }
    cls_table = QTable(cls_data, units=units)

    clusters = [Cluster(cls_table['r200'][i], cls_table['M200'][i], cls_table['sig200'][i], m500=cls_table['M500'][i]) for i in range(galwcls.shape[0])]
    variances = variance(cls_table['err_neg'], cls_table['err_pos'])
    return clusters, variances


import requests as req
import pandas as pd
from glob import glob
import os

#execution timer
def ex_time(func):
    def execution_time(*args,**kw):
        start = pd.datetime.now()
        execution = func(*args,**kw)
        end = pd.datetime.now()
        print('''execution time: {0}
        '''.format(end-start))
        return execution
    return execution_time

class retrieve_data():

    data = {
        'g17_industrial_production_and_cap_utilization' : {
            'url':'https://www.federalreserve.gov/datadownload/Output.aspx?rel=G17&series=1aa8d31f88df776eb8dc991ce0b8b974&lastobs=100&from=&to=&filetype=csv&label=include&layout=seriesrow',
            'output':'g17_industrial_production_and_cap_utilization.csv'
            },
        'g19_consumer_credit_outst' : {
            'url':'https://www.federalreserve.gov/datadownload/Output.aspx?rel=G19&series=47b3133fcba3957706678b2a55cb5a97&lastobs=&from=&to=&filetype=csv&label=include&layout=seriescolumn&type=package',
            'output':'g19_consumer_credit_outst.csv'
            },
        'monthly_retail_trade_report' : {
            'url':'https://www.census.gov/retail/mrts/www/mrtssales92-present.xls',
            'output':'monthly_retail_trade_report.xls'
            },
        'consumer_confidence_index' : {
            'url':'https://stats.oecd.org/sdmx-json/data/DP_LIVE/.CCI.../OECD?contentType=csv&detail=code&separator=comma&csv-lang=en',
            'output':'consumer_confidence_index.csv'
            },
        'current_dollar_gdp' : {
            'url':'https://www.bea.gov/system/files/2020-02/gdplev.xlsx',
            'output':'current_dollar_gdp.xlsx'
            }
    }

    @ex_time
    def download_data(self,url,output):
        print('downloading {0}...'.format(url))
        response = req.get(url)
        print('done.')
        print('saving output {0}...'.format(output))
        with open(output,'wb') as f:
            f.write(response.content)
        print('done.')
        return

    def download_med_fam_income_data(self):
        #available range 1998-2019
        start_year = 1997
        for i in range(2019-start_year):
            year = str(start_year + i)[-2:]
            url = 'https://www.ffiec.gov/xls/msa{0}inc.xls'.format(year)
            output = 'med_family_income_report{0}.xls'.format(year)
            self.download_data(url,output)

    def __init__(self):
        data = self.data
        for item in data:
            self.download_data(data[item]['url'],data[item]['output'])
        self.download_med_fam_income_data()

@ex_time
class preprocessing():

    #function to clean gdp file
    def gdp(self):
        cd_gdp = pd.read_excel('current_dollar_gdp.xlsx')
        #specify headers
        cd_gdp_headers = ['year','gdp_in_billions_of_current_dollars','gdp_in_billions_of_2012_dollars','blank','year_quarter','gdp_in_billions_of_current_dollars2','gdp_in_billions_of_2012_dollars2','blank2']
        cd_gdp.columns = cd_gdp_headers
        #split into separate dataframes
        gdp_by_year = cd_gdp[['year','gdp_in_billions_of_current_dollars','gdp_in_billions_of_2012_dollars']][7:]
        gdp_by_quarter = cd_gdp[['year_quarter','gdp_in_billions_of_current_dollars2','gdp_in_billions_of_2012_dollars2']][7:]
        #drop na's
        gdp_by_year.dropna(inplace=True)
        gdp_by_quarter.dropna(inplace=True)
        #dump to csv
        gdp_by_quarter.to_csv('gdp_by_quarter.csv',index=False)
        gdp_by_year.to_csv('gdp_by_year.csv',index=False)
        os.remove('current_dollar_gdp.xlsx')
        return

    #clean up consumer credit outstanding
    def consumer_credit(self):
        g19 = pd.read_csv('g19_consumer_credit_outst.csv')
        multiplier = g19.loc[1][1:].tolist()
        g19 = g19[7:]
        for i,col in enumerate(g19.columns[1:]):
            g19[col] = pd.to_numeric(g19[col],errors='coerce')
            g19[col] = g19[col] * int(multiplier[i])
        g19.to_csv('consumer_credit.csv',index=False)
        os.remove('g19_consumer_credit_outst.csv')
        return

    #function to grab all median family income files and summarize
    def summarize_med_family_inc(self):
        mfi = {
            'year':[],
            'total':[]
        }
        files = glob('*med_family*')
        for f in files:
            try:
                df = pd.read_excel(f)
                total = df[[df.columns[-1]]][1:].sum().values[0]
                year = df[[df.columns[-1]]][0:1].values[0][0][:4]
                if year == 'HUD ':
                    year = df[[df.columns[-1]]][0:1].values[0][0][14:18]
                mfi['year'].append(year)
                mfi['total'].append(total)
                os.remove(f)
            except:
                print('{0} was unable to load'.format(f))
                pass
        mfi = pd.DataFrame(mfi,columns=['year','total'])
        mfi.to_csv('median_family_income.csv',index=False)
        return

    #create function to split up the excel formatting
    def retail_trade(self):
        columns=['business','january','february','march','april',
            'may','jun','july','august','september','october','november','december']
        df = pd.DataFrame(columns=columns)
        start_year = 1992
        for i in range(2020-start_year):
            year = str(start_year + i)
            ndf = pd.read_excel('monthly_retail_trade_report.xls',sheet_name=year)
            ndf = ndf[ndf.columns[1:14]][7:]
            ndf.dropna(inplace=True)
            ndf.columns = columns
            for i,col in enumerate(ndf.columns[1:]):
                ndf[col] = pd.to_numeric(ndf[col],errors='coerce')
            ndf['year'] = year
            df = df.append(ndf)
            ndf = None
        df.dropna(inplace=True)
        df.to_csv('retail_trade.csv',index=False)
        os.remove('monthly_retail_trade_report.xls')
        return

    def __init__(self):
        print('cleaning up files...')
        self.gdp()
        self.consumer_credit()
        self.summarize_med_family_inc()
        self.retail_trade()
        print('done.')

@ex_time
def convert_all_to_json():
    files = glob('*.csv')
    for f in files:
        print('converting {0}...'.format(f))
        df = pd.read_csv(f)
        df.to_json(str(f).replace('.csv','.json'),orient='records')
        os.remove(f)
        print('done.')
    return

if __name__ == "__main__":
    @ex_time
    def run():
        print('processing...')
        retrieve_data()
        preprocessing()
        convert_all_to_json()
        print('run complete.')
    run()


import datetime
from typing import Any, Dict, Type, TypeVar

import attr
from dateutil.parser import isoparse

T = TypeVar("T", bound="CodatPublicApiModelsCompanyCompanyEventStreamItem")

@attr.s(auto_attribs=True)
class CodatPublicApiModelsCompanyCompanyEventStreamItem:
    """
    Attributes:
        type (str):
        description (str):
        event_time_utc (datetime.datetime):
    """

    type: str
    description: str
    event_time_utc: datetime.datetime

    def to_dict(self) -> Dict[str, Any]:
        type = self.type
        description = self.description
        event_time_utc = self.event_time_utc.isoformat()

        field_dict: Dict[str, Any] = {}
        field_dict.update(
            {
                "type": type,
                "description": description,
                "eventTimeUtc": event_time_utc,
            }
        )

        return field_dict

    @classmethod
    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
        d = src_dict.copy()
        type = d.pop("type")

        description = d.pop("description")

        event_time_utc = isoparse(d.pop("eventTimeUtc"))

        codat_public_api_models_company_company_event_stream_item = cls(
            type=type,
            description=description,
            event_time_utc=event_time_utc,
        )

        return codat_public_api_models_company_company_event_stream_item


from aiogram.types import ReplyKeyboardMarkup, ReplyKeyboardRemove
from main import tg_bot_database

user_keyboard = ReplyKeyboardMarkup([["/input_number"]], resize_keyboard=True)

manager_keyboard = ReplyKeyboardMarkup([["/add_employee"], ["/delete_employee"], ["/input_number"]],
                                       resize_keyboard=True)
admin_keyboard = ReplyKeyboardMarkup([["Stores"], ["Managers"], ["Workers"]], resize_keyboard=True)

admin_stores = ReplyKeyboardMarkup([["/add_store"], ["/delete_store"]], resize_keyboard=True)

admin_managers = ReplyKeyboardMarkup([["/add_manager"], ["/delete_manager"]], resize_keyboard=True)

admin_workers = ReplyKeyboardMarkup([["/add_worker"], ["/delete_worker"]], resize_keyboard=True)

async def cities_keyboard():
    cities_list = await tg_bot_database.cities()
    cities_kb = ReplyKeyboardMarkup(resize_keyboard=True)
    for item in cities_list:
        cities_kb.add(item)
    return cities_kb

async def stores_keyboard(city):
    list_of_stores = await tg_bot_database.stores(city)
    stores_kb = ReplyKeyboardMarkup(resize_keyboard=True)
    for item in list_of_stores:
        stores_kb.add(item)
    return stores_kb

async def users_list(city, store, rights):
    list_of_users = await tg_bot_database.get_list(city, store, rights)
    users_kb = ReplyKeyboardMarkup(resize_keyboard=True)
    for item in list_of_users:
        users_kb.add(item)
    return users_kb


import av

def get_video_container(path_to_vid, multi_thread_decode=False, backend="pyav"):
    """
    Given the path to the video, return the pyav video container.
    Args:
        path_to_vid (str): path to the video.
        multi_thread_decode (bool): if True, perform multi-thread decoding.
        backend (str): decoder backend, options include `pyav` and
            `torchvision`, default is `pyav`.
    Returns:
        container (container): video container.
    """
    if backend == "torchvision":
        with open(path_to_vid, "rb") as fp:
            container = fp.read()
        return container
    elif backend == "pyav":
        container = av.open(path_to_vid)
        if multi_thread_decode:
            # Enable multiple threads for decoding.
            container.streams.video[0].thread_type = "AUTO"
        return container
    else:
        raise NotImplementedError("Unknown backend {}".format(backend))