from core.plugin import *
from core.bot import Bot
from plugins.curriculum_reporter.plugin_config import *
from plugins.curriculum_reporter.plugin_doc import PLUGIN_DOC
import yaml
import time
import datetime
from utils.log import Log

class Curriculum_Reporter_Plugin(Plugin):

    def __init__(self):
        super().__init__(
                name = "Curriculum_Reporter",
                requirements = [],
                info = "自动提醒上课的qq机器人插件",
                doc = PLUGIN_DOC
            )

    def setup(self, bot: Bot):
        try:
            with open(CURRICULUM_PATH, "r", encoding="utf-8") as fp:
                self.curriculum = yaml.load(fp.read(), Loader=yaml.FullLoader)
        except:
            raise Exception("找不到课表文件: {}".format(CURRICULUM_PATH))

        super().setup(bot)

    @check_setup
    async def handle_message(self, message: Message) -> Union[Message, List[Message], Error]:
        if message.text is not None:
            cmd_args = message.text.split(" ")

            if cmd_args[0] == SHOW_COMMAND:
                if len(cmd_args) > 2:
                    return Error("参数个数不对!", urge=self.get_name())

                try:
                    total_info = ""
                    for course in self.curriculum['courses']:
                        if len(cmd_args) == 2:
                            if course['name'] == cmd_args[1]:
                                course_info = "课程名: {}\n会议号: {}\n密码: {}\n时间: ".format(course['name'], str(course['meeting']), str(course['passwd']))
                                for one_time in course['time']:
                                    course_info += WEEKNAME[int(one_time['weekday'])] + one_time['clock'] + ", "
                                course_info = course_info[:-2]
                                return Message(course_info)
                        else:
                            total_info += "课程名: {}\t会议号: {}\t密码: {}".format(course['name'], str(course['meeting']), str(course['passwd']))
                            if self.curriculum['courses'].index(course) != len(self.curriculum['courses'])-1:
                                total_info += "\n"
                    if len(cmd_args) == 2:
                        raise Exception("无此课程")
                    else:
                        return Message(total_info)
                except Exception as e:
                    return Error("发生错误: {}".format(e.args), urge=self.get_name())

    @check_setup
    async def plugin_task(self):
        async for _ in Ticker(self, 60):
            next_datetime = datetime.datetime.now() + datetime.timedelta(minutes=5)
            # Log.info(next_datetime)
            try:
                for course in self.curriculum['courses']:
                    for one_time in course['time']:
                        one_clock = time.strptime(one_time['clock'], TIME_FORMAT)
                        # Log.info(one_clock)
                        if next_datetime.weekday() == one_time['weekday'] and next_datetime.hour == one_clock.tm_hour and next_datetime.minute == one_clock.tm_min:
                            await self.send("课程【{}】还有五分钟就要开始啦~\n会议号: {}\n会议密码: {}".format(course['name'], str(course['meeting']), str(course['passwd'])))
            except Exception as e:
                Log.error(e)


import sys
sys.stdin = open('2164input.txt')
from collections import deque
#
N = int(input())
queue = deque([])
for i in range(1,N+1):
    queue.append(i)

# print(len(queue))
while len(queue) >1:
    queue.popleft()
    n = queue.popleft()

    queue.append(n)

print(queue[0])


#coding:utf-8

import sys
import queue
import os
import time

from lib.core.data import root_path,lock
from lib.core.common import get_local_version,get_latest_revision
from lib.core.log import logvuln,logwarning,logunvuln,logverifyerror,logcritical
from lib.core.data import qu,allpoc,now,vulnoutput,unvulnoutput,unreachoutput,scan_path,poc_path,AliveList,AliveTest
from lib.core.threads import run_threads
from lib.core.common import run,GetCommand
from lib.core.log import loglogo
from lib.core.htmloutput import output_html
from lib.fofa import get_ukey,fofa_login,ukey_save,fofa_search
from lib.core.output import Mkdn_output,Txt_output,Csv_output

from requests import ConnectTimeout

ScanMode = {
    "Single_Verify":1,
    "File_Verify":2,
    }

class oFxCenter():
    def __init__(self):
        self.Mode = None
        self.__version = None
        self.__threads = 10

        self.__proxy = None

        self.TargetList = []

        self.command_parser()

    def addpoc(self,pocpath):
        allpoc.put(pocpath)

    def show_version(self):
        LocalVer = get_local_version(root_path + "/info.ini")
        print("当前的本地版本是 {localv}".format(localv = LocalVer))
        print("获取github仓库信息，请稍等.......")
        LatestVer = get_latest_revision()
        if LatestVer == None:
            print("当前网络状况不佳，无法获取最新版本信息")
            exit()
        elif LatestVer and LocalVer != LatestVer:
            print("最新版本是 {latestv}".format(latestv = LatestVer))
            exit()
        else:
            print("目前使用的ofx是最新版本")
            exit()

    def setproxy(self):
        """
        set the proxy for oFx running

        return:None
        """
        if self.CMD_ARGS.proxy.startswith("http://"):
            self.CMD_ARGS.proxy = self.CMD_ARGS.proxy[7:]
        elif self.CMD_ARGS.proxy.startswith("https://"):
            self.CMD_ARGS.proxy = self.CMD_ARGS.proxy[8:]
        else:
            pass

        if self.CMD_ARGS.proxy.endswith("/"):
            self.CMD_ARGS.proxy = self.CMD_ARGS.proxy[:-1]
        else:
            pass

        self.__proxy = {
        "http": "http://%s"%(self.CMD_ARGS.proxy),
        "https": "http://%s"%(self.CMD_ARGS.proxy),
        }
        self.getproxy()

    def getproxy(self):
        return self.__proxy

    def fromfofa(self):
        # verify and get user and key
        fofa_user,fofa_key = get_ukey(root_path+"/lib/fofa.ini")

        # logincheck
        FofaLogin = fofa_login(fofa_user,fofa_key)
        if FofaLogin[0]:
            log_msg = "User : {user} | Key : {key}".format(user = FofaLogin[1],key = FofaLogin[2])
            log_msg += " | Login Success"
            logvuln(log_msg)
            ukey_save(FofaLogin[1],FofaLogin[2],root_path+"/lib/fofa.ini")

        # No or login failure, raw_input function gets user input,
            # Login repeat, loop
        # Login success

            fofa_save_path = scan_path + input("请输入文件名保存结果（不要添加文件后缀）： ") + ".txt"
            FofaDork = input("请输入搜索语句：")
            search_all = input("！是否需要获取全球最大数量资产( 是(y/Y) , 否(n/N/Enter) ):")
            if search_all.lower() == "y":
                search_all = True
            elif search_all.lower() in ["n",""]:
                search_all = False
            else:
                logwarning("输入错误！需要获取最大数量资产请输入：y/Y，不需要请输入：n/N或按Enter键")
                exit()
            loglogo("Fofa搜索语句是：{fofadork}，开始对接 Fofa Api".format(fofadork = FofaDork))
            FofaResultNum = fofa_search(FofaLogin[1],FofaLogin[2],FofaDork,search_all,fofa_save_path)
            if type(FofaResultNum) == int:
                log_msg = "搜索完成，结果保存到 {path}，去重后，一共 {FofaResultNum}条".format(path = fofa_save_path,FofaResultNum = FofaResultNum)
                logvuln(log_msg)
            # Get search results and save to scan directory

    def getmode(self):
        return self.Mode

    def setmode(self,mode):
        global ScanMode
        if mode in ScanMode.keys():
            self.Mode = ScanMode[mode]
        pass

    def Load_POC(self,poc_path):
        sys.path.append(str(poc_path))
        from poc import POC
        logvuln("POC - %s 加载完毕"%(POC._info["name"]))
        return POC,poc_path

    def Unload_POC(self,poc_path):
        sys.path.remove(poc_path)
        del sys.modules["poc"]

    def get_some_poc(self,poc_path):
        poc_path = poc_path[:-7] if poc_path.endswith("poc.py") else poc_path
        poc_path = root_path+"/"+poc_path
        if os.path.exists(poc_path):
            loglogo("POC - %s 有效"%(poc_path))
            self.addpoc(poc_path)
        else:
            logvuln("POC - %s 不存在，请确认路径并重新指定"%(poc_path))
            exit()
        pass

    def get_all_poc(self):
        for app in os.listdir(poc_path):
            if app == "demo":
                continue
            elif app == ".DS_Store":
                continue
            app_path = poc_path +app
            for vulnname in os.listdir(app_path):
                if vulnname.startswith("."):
                    continue
                if vulnname.startswith("_"):
                    continue
                if vulnname.startswith("Url_Alive"):
                    continue
                vuln_path = app_path + "/" +vulnname
                vuln_path = vuln_path.split(root_path)[1][1:]
                self.addpoc(vuln_path)

    def command_parser(self):
        global AliveList,AliveTest
        self.CMD_ARGS = GetCommand()
        if self.CMD_ARGS.version == True:
            self.show_version()

        if self.CMD_ARGS.proxy != False:
            self.setproxy()

        if self.CMD_ARGS.fofa_search:
            self.fromfofa()

        if self.CMD_ARGS.add_poc:
            app_name = input("请输入受漏洞影响的应用名")
            vuln_name = input("请输入漏洞名")

            app_list = os.listdir(poc_path)
            if app_name in app_list:
                pass
            else:
                os.mkdir(poc_path+app_name+"/")
            app_dir = poc_path+app_name+"/"

            vuln_list = os.listdir(app_dir)
            if vuln_name in vuln_list:
                err_msg = "该POC名已存在，请重新确认"
                exit(err_msg)

            else:
                os.mkdir(app_dir+vuln_name+"/")
            vuln_dir = app_dir+vuln_name+"/"

            with open(root_path+"/poc/demo/demo/poc.py","r") as f:
                poc_content = f.read()
            with open(vuln_dir+"poc.py","w") as f:
                f.write(poc_content)

            reference_dir = vuln_dir+"reference/"
            os.mkdir(reference_dir)
            with open(reference_dir+"reference.md","w") as f:
                f.write("``为了帮助笔者快速理解并完成POC贡献提交的测试工作，请在该文件中写入参考文献链接，以及漏洞基本概念和漏洞检测逻辑文字描述，如何判断漏洞存在与否``")
            with open(reference_dir+"test_num_1w.txt","w") as f:
                pass
            with open(reference_dir+"success_30.txt","w") as f:
                pass

            success_msg = """
___________生成POC目录结构如下____________
    |__ {APP_NAME}/
        |__ {VULN_NAME}/
            |__ poc.py
            |__ reference/
                |__ reference.md
                |__ other file
                |__ test_num_1w.txt
                |__ success_30.txt
_________________________________________
POC路径为{VULN_PATH}
            """.format(APP_NAME = app_name,VULN_NAME = vuln_name,VULN_PATH = vuln_dir)
            exit(success_msg)

        if self.CMD_ARGS.url or self.CMD_ARGS.file:
            # mode verify
            if self.CMD_ARGS.url and self.CMD_ARGS.script:
                self.setmode("Single_Verify")
            elif self.CMD_ARGS.file and self.CMD_ARGS.script:
                self.setmode("File_Verify")

            else:
                print("Please confirm the detection mode,\
                     \nMust provide -f or -u parameter to specify the target,\
                     \nMust provide -s parameter to specify POC or [-s all] to load all POC")
                exit()

            # single mode
            if self.getmode() == 1:

                if self.CMD_ARGS.script == "all":
                    err_msg = "single 模式不支持全量POC，如果有需求，请将单个url保存到一个文件中，再用-f去检测"
                    exit(err_msg)
                elif "," in self.CMD_ARGS.script:
                    err_msg = "single 模式不支持多POC，如果有需求，请将单个url保存到一个文件中，再用-f去检测"
                    exit(err_msg)

                self.get_some_poc(self.CMD_ARGS.script)
                POC,POC_Path = self.Load_POC(allpoc.get())

                single_mode = POC(self.CMD_ARGS.url,self.getproxy())
                # 报错信息统一管理
                try:
                    single_verify = single_mode._attack()
                except ConnectTimeout as e:
                    err_msg = "目标连接超时，请重新确认目标是否存在"
                    exit(err_msg)
                except Exception as e:
                    if self.CMD_ARGS.show_error:
                        raise e
                    else:
                        single_verify = [False,""]
                if single_verify[0] == True:
                    print("URL: {url}  || POC: {script} \n服务器返回信息: \n{text} \n【漏洞存在】\n".format(url = self.CMD_ARGS.url,script = self.CMD_ARGS.script,text = single_verify[1]))
                else:
                    print("URL: {url}  || POC: {script} \n服务器返回信息: \n{text} \n【漏洞不存在】\n".format(url = self.CMD_ARGS.url,script = self.CMD_ARGS.script,text = single_verify[1]))

            # enum mode
            elif self.getmode() == 2:
                loglogo("POC 解析中,请稍后....")
                if self.CMD_ARGS.script == "all":
                    self.get_all_poc()
                elif "," in self.CMD_ARGS.script:
                    for i in self.CMD_ARGS.script.split(","):
                        self.get_some_poc(i)
                else:
                    self.get_some_poc(self.CMD_ARGS.script)

                start_time = time.time()

                #### Alpha Future Start ####
                if self.CMD_ARGS.script == "all":
                    with open(self.CMD_ARGS.file,"r") as f:
                        target_list = [i.strip() for i in f.readlines() if "." in i]
                        target_list = set(target_list)
                    for i in target_list:
                        if i.strip() == "":
                            target_list.remove(i)
                        else:
                            AliveTest.put(i)
                    POC,POC_Path = self.Load_POC(root_path+"/poc/common/Url_Alive/")
                    run_threads(num_threads = self.CMD_ARGS.thread,thread_function = run,args=(POC,AliveTest,self.getproxy(),self.CMD_ARGS.output,str(allpoc.qsize()),True))
                    # run(POC,AliveTest,self.getproxy(),self.CMD_ARGS.output,str(allpoc.qsize()),True)
                    self.Unload_POC(POC_Path)
                else:
                    with open(self.CMD_ARGS.file,"r") as f:
                        target_list = [i.strip() for i in f.readlines() if "." in i]
                    for i in target_list:
                        if i.strip() == "":
                            target_list.remove(i)
                        else:
                            AliveList.add(i)
                            # AliveList.append(i)
                #### Alpha Future End ####

                while not allpoc.empty():
                    POC,POC_Path = self.Load_POC(allpoc.get())

                    for i in AliveList:
                        qu.put(i)
                    run_threads(num_threads = self.CMD_ARGS.thread,thread_function = run,args=(POC,qu,self.getproxy(),self.CMD_ARGS.output,str(allpoc.qsize()),False))
                    # run(POC,qu,self.getproxy(),self.CMD_ARGS.output,str(allpoc.qsize()),False)
                    self.Unload_POC(POC_Path)

                end_time = time.time()
                total_time = int(end_time-start_time)

                if self.CMD_ARGS.output != False:

                    # print(vulnoutput)
                    loglogo("扫描结束，结果汇报")
                    if len(vulnoutput) >= 1:
                        txt_output = now + ".txt" if self.CMD_ARGS.output == True else self.CMD_ARGS.output+".txt"
                        txt_output = root_path+"/output/"+txt_output
                        Txt_output(txt_output,vulnoutput,target_list)

                        md_output = now + ".md" if self.CMD_ARGS.output == True else self.CMD_ARGS.output + ".md"
                        md_output = root_path+"/output/"+md_output
                        Mkdn_output(md_output,vulnoutput,target_list = target_list,actual_list = AliveList , total_time = total_time)

                        csv_output = now + ".csv" if self.CMD_ARGS.output == True else self.CMD_ARGS.output + ".csv"
                        csv_output = root_path+"/output/"+csv_output
                        Csv_output(csv_output,vulnoutput,target_list = target_list,actual_list = AliveList , total_time = total_time)

                    else:
                        logverifyerror("目标文件中的url未匹配POC检测逻辑，疑似无漏洞")

                # end_time = time.time()
                loglogo("本次扫描消耗了: %d 秒"%(total_time))

        if self.CMD_ARGS.sound:
            from playsound import playsound
            playsound(root_path+"/data/95955.mp3")


from datetime import datetime
import random
import time

odds = [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57,
        59]

this_minute = datetime.today().minute

for i in range(5):
    if this_minute in odds:
        print("This is an odd minute")
    else:
        print("This is an even minute")
    wait = random.randint(1, 60)
    time.sleep(wait)


import torch
import os
from utils import geo_utils, ba_functions
import numpy as np

def prepare_predictions(data, pred_cam, conf, bundle_adjustment):
    # Take the inputs from pred cam and turn to ndarray
    outputs = {}
    outputs['scan_name'] = data.scan_name
    calibrated = conf.get_bool('dataset.calibrated')

    Ns = data.Ns.cpu().numpy()
    Ns_inv = data.Ns_invT.transpose(1, 2).cpu().numpy()  # Ks for calibrated, a normalization matrix for uncalibrated
    M = data.M.cpu().numpy()
    xs = geo_utils.M_to_xs(M)

    Ps_norm = pred_cam["Ps_norm"].cpu().numpy()  # Normalized camera!!
    Ps = Ns_inv @ Ps_norm  # unnormalized cameras
    pts3D_pred = geo_utils.pflat(pred_cam["pts3D"]).cpu().numpy()

    pts3D_triangulated = geo_utils.n_view_triangulation(Ps, M=M, Ns=Ns)

    outputs['xs'] = xs  # to compute reprojection error later
    outputs['Ps'] = Ps
    outputs['Ps_norm'] = Ps_norm
    outputs['pts3D_pred'] = pts3D_pred  # 4,m
    outputs['pts3D_triangulated'] = pts3D_triangulated  # 4,n

    if calibrated:
        Ks = Ns_inv  # data.Ns.inverse().cpu().numpy()
        outputs['Ks'] = Ks
        Rs_gt, ts_gt = geo_utils.decompose_camera_matrix(data.y.cpu().numpy(), Ks)  # For alignment and R,t errors
        outputs['Rs_gt'] = Rs_gt
        outputs['ts_gt'] = ts_gt

        Rs_pred, ts_pred = geo_utils.decompose_camera_matrix(Ps_norm)
        outputs['Rs'] = Rs_pred
        outputs['ts'] = ts_pred

        Rs_fixed, ts_fixed, similarity_mat = geo_utils.align_cameras(Rs_pred, Rs_gt, ts_pred, ts_gt, return_alignment=True) # Align  Rs_fixed, tx_fixed
        outputs['Rs_fixed'] = Rs_fixed
        outputs['ts_fixed'] = ts_fixed
        outputs['pts3D_pred_fixed'] = (similarity_mat @ pts3D_pred)  # 4,n
        outputs['pts3D_triangulated_fixed'] = (similarity_mat @ pts3D_triangulated)

        if bundle_adjustment:
            repeat = conf.get_bool('ba.repeat')
            triangulation = conf.get_bool('ba.triangulation')
            ba_res = ba_functions.euc_ba(xs, Rs=Rs_pred, ts=ts_pred, Ks=np.linalg.inv(Ns),
                                         Xs_our=pts3D_pred.T, Ps=None,
                                         Ns=Ns, repeat=repeat, triangulation=triangulation, return_repro=True) #    Rs, ts, Ps, Xs
            outputs['Rs_ba'] = ba_res['Rs']
            outputs['ts_ba'] = ba_res['ts']
            outputs['Xs_ba'] = ba_res['Xs'].T  # 4,n
            outputs['Ps_ba'] = ba_res['Ps']

            R_ba_fixed, t_ba_fixed, similarity_mat = geo_utils.align_cameras(ba_res['Rs'], Rs_gt, ba_res['ts'], ts_gt,
                                                                       return_alignment=True)  # Align  Rs_fixed, tx_fixed
            outputs['Rs_ba_fixed'] = R_ba_fixed
            outputs['ts_ba_fixed'] = t_ba_fixed
            outputs['Xs_ba_fixed'] = (similarity_mat @ outputs['Xs_ba'])

    else:
        if bundle_adjustment:
            repeat = conf.get_bool('ba.repeat')
            triangulation = conf.get_bool('ba.triangulation')
            ba_res = ba_functions.proj_ba(Ps=Ps, xs=xs, Xs_our=pts3D_pred.T, Ns=Ns, repeat=repeat,
                                          triangulation=triangulation, return_repro=True, normalize_in_tri=True)   # Ps, Xs
            outputs['Xs_ba'] = ba_res['Xs'].T  # 4,n
            outputs['Ps_ba'] = ba_res['Ps']

    return outputs

def compute_errors(outputs, conf, bundle_adjustment):
    model_errors = {}
    calibrated = conf.get_bool('dataset.calibrated')
    Ps = outputs['Ps']
    pts3D_pred = outputs['pts3D_pred']
    xs = outputs['xs']
    pts3D_triangulated = outputs['pts3D_triangulated']

    model_errors["our_repro"] = np.nanmean(geo_utils.reprojection_error_with_points(Ps, pts3D_pred.T, xs))
    model_errors["triangulated_repro"] = np.nanmean(geo_utils.reprojection_error_with_points(Ps, pts3D_triangulated.T, xs))
    if calibrated:
        Rs_fixed = outputs['Rs_fixed']
        ts_fixed = outputs['ts_fixed']
        Rs_gt = outputs['Rs_gt']
        ts_gt = outputs['ts_gt']
        Rs_error, ts_error = geo_utils.tranlsation_rotation_errors(Rs_fixed, ts_fixed, Rs_gt, ts_gt)
        model_errors["ts_mean"] = np.mean(ts_error)
        model_errors["ts_med"] = np.median(ts_error)
        model_errors["Rs_mean"] = np.mean(Rs_error)
        model_errors["Rs_med"] = np.median(Rs_error)

    if bundle_adjustment:
        Xs_ba = outputs['Xs_ba']
        Ps_ba = outputs['Ps_ba']
        model_errors['repro_ba'] = np.nanmean(geo_utils.reprojection_error_with_points(Ps_ba, Xs_ba.T, xs))
        if calibrated:
            Rs_fixed = outputs['Rs_ba_fixed']
            ts_fixed = outputs['ts_ba_fixed']
            Rs_gt = outputs['Rs_gt']
            ts_gt = outputs['ts_gt']
            Rs_ba_error, ts_ba_error = geo_utils.tranlsation_rotation_errors(Rs_fixed, ts_fixed, Rs_gt, ts_gt)
            model_errors["ts_ba_mean"] = np.mean(ts_ba_error)
            model_errors["ts_ba_med"] = np.median(ts_ba_error)
            model_errors["Rs_ba_mean"] = np.mean(Rs_ba_error)
            model_errors["Rs_ba_med"] = np.median(Rs_ba_error)
    # Rs errors mean, ts errors mean, ba repro, rs ba mean, ts ba mean

    projected_pts = geo_utils.get_positive_projected_pts_mask(Ps @ pts3D_pred, conf.get_float('loss.infinity_pts_margin'))
    valid_pts = geo_utils.xs_valid_points(xs)
    unprojected_pts = np.logical_and(~projected_pts, valid_pts)
    part_unprojected = unprojected_pts.sum() / valid_pts.sum()

    model_errors['unprojected'] = part_unprojected

    return model_errors


from absympla.core.api_router import SymplaAPIRouter
from absympla.core.types.event import Event
from absympla.log import get_logger
from typing import Iterator

class SymplaEvents(SymplaAPIRouter):
    ROUTES = {
        "list_events": "/public/v3/events",
        "get_event": "/public/v3/events/{}"
    }
    logger = get_logger(__name__)

    def get_event(self, event_id: str) -> Event:
        self.logger.info(f'Reading event ({event_id}) info')
        handler = self.get(url=self.ROUTES["get_event"].format(event_id))
        handler.raise_for_status()
        response = handler.json()
        return Event(**response.get("data"))

    def list_events(self) -> Iterator[Event]:
        has_next = True
        page = 1

        while has_next:
            self.logger.info(f'Reading list_events page ({page})')

            handler = self.get(url=self.ROUTES["list_events"], params={
                "page": page
            })

            handler.raise_for_status()

            response = handler.json()

            sorting_field = response.get("sort", {}).get("field_sort", "unknown")
            sorting_method = response.get("sort", {}).get("sort", "ASC")
            self.logger.info(f'Sorting by: {sorting_field} ({sorting_method})')

            has_next = response.get("pagination", {}).get("has_next", False)

            self.logger.info(f"Has next: {'yes' if has_next else 'no'}")

            for event_info in response.get("data"):
                yield Event(**event_info)

            page += 1

        self.logger.info(f'list_of_events has come to an end')


import numpy as np
from dpsniper.mechanisms.abstract import Mechanism

import mmh3     # murmur hash 3, used for Bloom filter

from dpsniper.utils.zero import ZeroNoisePrng

class Rappor(Mechanism):
    """
    Steps 1--3 from:
        Ulfar Erlingsson, Vasyl Pihur, and Aleksandra Korolova. 2014.
        RAPPOR: Randomized Aggregatable Privacy-Preserving Ordinal Response. CCS 2014.
    """

    def __init__(self, n_hashes: int = 4, filter_size: int = 20, f: float = 0.75, p: float = 0.45, q: float = 0.55, prng=None):
        self.f = f
        self.q = q
        self.p = p
        self.n_hashes = n_hashes
        self.filter_size = filter_size

        if prng is None:
            self.prng = np.random.default_rng()
        else:
            self.prng = prng

    def m(self, a, n_samples: int = 1):
        assert(a.shape == (1,) or a.shape == ())
        val = a.item(0)

        # populate bloom filter
        filter = self._populate_bloom_filter(val, n_samples)

        if isinstance(self.prng, ZeroNoisePrng):
            # don't perform any randomization
            return filter

        # permanent randomized response
        self._apply_permanent_randomized_response(filter)

        # instantaneous randomized response
        res = self._get_instantaneous_randomized_response(filter)
        return res

    def _populate_bloom_filter(self, val, n_samples):
        filter = np.zeros((n_samples, self.filter_size))
        for i in range(0, self.n_hashes):
            hashval = mmh3.hash(str(val), seed=i) % self.filter_size
            filter[:, hashval] = 1
        return filter

    def _apply_permanent_randomized_response(self, filter):
        choices = np.random.choice(3, size=filter.shape, p=[0.5*self.f, 0.5*self.f, 1-self.f])
        filter[choices == 0] = 1
        filter[choices == 1] = 0

    def _get_instantaneous_randomized_response(self, filter):
        res = np.zeros(shape=filter.shape)
        set_to_1_q = np.logical_and(filter == 1, self.prng.binomial(n=1, p=self.q, size=filter.shape) == 1)
        set_to_1_p = np.logical_and(filter == 0, self.prng.binomial(n=1, p=self.p, size=filter.shape) == 1)
        res[set_to_1_p] = 1
        res[set_to_1_q] = 1
        return res

class OneTimeRappor(Rappor):
    """
    Steps 1--2 from:
        Ulfar Erlingsson, Vasyl Pihur, and Aleksandra Korolova. 2014.
        RAPPOR: Randomized Aggregatable Privacy-Preserving Ordinal Response. CCS 2014.
    """
    def __init__(self, n_hashes: int = 4, filter_size: int = 20, f: float = 0.95, prng=None):
        super().__init__(n_hashes=n_hashes, filter_size=filter_size, f=f, prng=prng)

    def m(self, a, n_samples=1):
        assert (a.shape == (1,) or a.shape == ())
        val = a.item(0)

        # populate bloom filter
        filter = self._populate_bloom_filter(val, n_samples)

        if isinstance(self.prng, ZeroNoisePrng):
            # don't perform any randomization
            return filter

        # permanent randomized response
        self._apply_permanent_randomized_response(filter)

        return filter


import discord
from discord.ext import commands
listener = commands.Cog.listener

class events(commands.Cog):

	def __init__(self, client):
		self.client = client

	@listener()
	async def on_member_ban(self, server, user):
		channel = self.client.get_channel(869134199855149056)
		await channel.send(f'Пользователь {user.mention} был забанен')

	@listener()
	async def on_message_unban(self, server, user):
		channel = self.client.get_channel(869134199855149056)
		await channel.send(f'Пользователь {user.mention} был разбанен')

	@listener()
	async def on_member_join(self, member):
		channel = self.client.get_channel(869134199855149056)
		role1 = member.guild.get_role(870347664116551771)
		await member.add_roles(role1)
		await channel.send(f"Добро пожаловать в клан 'Берлога', {member.mention}!")
	@listener()
	async def on_member_remove(self, member):
		channel = self.client.get_channel(869134199855149056)
		await channel.send(f"{member.mention} Покинул нас!")

	@listener()
	async def on_button_click(self, res):
		await res.respond(type=6)
	@listener()
	async def on_voice_state_update(self,member,before,after):
		if after.channel.id == 870945646507610122:
			mainCategory = discord.utils.get(member.guild.categories, id=870945493985923123)
			channel2 = await member.guild.create_voice_channel(name=f"Комната {member.display_name}",category=mainCategory)
			await member.move_to(channel2)
			await channel2.set_permissions(member,mute_members=True,move_members=True,manage_channels=True)
			def check(a,b,c):
				return len(channel2.members) == 0
			await self.client.wait_for('voice_state_update', check=check)
			await channel2.delete()

def setup(client):
	client.add_cog(events(client))


#task1, 2

'''Попросите пользователя ввести год, а затем используйте модуль datetime, чтобы проверить,
является ли этот год високосным, и выведите результат.

Попросите пользователя ввести две даты (начальную и конечную), а затем используйте модуль datetime
для подсчета количества рабочих дней между этими датами, исключая выходные дни.'''

#task1

# сделал через calendar так как для datetime нет метода isleap()

import datetime

import calendar

year = int(input("Enter year: "))

is_leap = calendar.isleap(year)

if is_leap:
    print(str(year) + " is leap year")
else:
    print(str(year) + " is not leap year")

#task2

def get_days(start_date, end_date):

    start_date = (datetime.datetime.strptime(start_date, "%Y-%m-%d")).date()
    end_date = (datetime.datetime.strptime(end_date, "%Y-%m-%d")).date()

    all_days = ((end_date - start_date).days + 1)
    print("Общее количество дней в заданном периоде: " + str(all_days))

    days_in_period = (start_date + datetime.timedelta(x) for x in range((end_date - start_date).days + 1))
    work_days = sum(1 for day in days_in_period if day.weekday() < 5)

    print("Количество рабочих дней в заданном периоде: " + str(work_days))

get_days(input('Введите начальную дату в формате "XXXX-XX-XX" (год-месяц-день): '),
         input('Введите конечную дату в формате "XXXX-XX-XX" (год-месяц-день): '))


import findspark
findspark.init()

import re
from pyspark import SparkConf, SparkContext
import collections

''' Set local context '''
conf = SparkConf().setMaster("local").setAppName("WordCount")
sc = SparkContext(conf = conf)

def normalize_words(line):
    return re.compile(r'\W+', re.UNICODE).split(line.lower())

''' Read in the file and get relevant rdd '''
lines = sc.textFile("Book.txt")
rdd = lines.flatMap(normalize_words)

'''
word_counts = rdd.countByValue()
for w, c in word_counts.items():
    clean_word = w.encode('ascii', 'ignore')
    if clean_word:
        print(clean_word, c)
'''

word_counts = rdd.map(lambda x: (x, 1)).reduceByKey(lambda x, y: x + y).collect()
for w, c in sorted(word_counts, key=lambda x: (x[1], x[0])):
    clean_word = w.encode('ascii', 'ignore')
    if clean_word:
        print(clean_word, c)


from datetime import date
from typing import Optional
from pydantic import BaseModel
from datetime import datetime
from infrastructure.models.patient import gender_enum

class Patient(BaseModel):
    id: int
    patient_id: Optional[str] = None
    name: str
    gender: gender_enum
    date_of_birth: Optional[date] = None
    age: str
    address: str
    contact_details: Optional[str] = None
    created_time: Optional[datetime] = None
    updated_time: Optional[datetime] = None
    created_user_id: Optional[int] = None
    updated_user_id: Optional[int] = None
    class Config():
        orm_mode = True


#charcoal, outfitters, gulahmad, chenone, stylo, khaadi, bareeze

url = 'https://www.khaadi.com/pk'
image_folder = './images/'

import requests
from bs4 import BeautifulSoup as soup
import time
import os

image_name = 1
for i in range(1, 12):
    print("retrieving page " + str(i))
    url = 'https://www.khaadi.com/pk/woman.html?p=' + str(i) + '&product_list_limit=45'

    r = requests.get(url, headers={'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'}, timeout=15)
    content = r.content
    r.close()
    page = soup(content, "html.parser")

    product_images = page.find_all("img", class_="product-image-photo")
    print("total images: " + str(len(product_images)))
    for p in product_images:
        print('retrieving image')
        src = p.get('src')
        try:
            imagefile = open(image_folder + str(image_name) + ".jpg", "wb")
            print('saving ' + imagefile.name)
            imagefile.write(requests.get(src, headers={'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'}, timeout=15).content)
            imagefile.close()
            print("[done]")
        except:
            print('an error occured while retrieving file')
        image_name += 1
        time.sleep(1)
    time.sleep(1)

print("Crawling completed")


#!/usr/bin/env python
# coding: utf-8

# In[2]:

import pandas as pd
# Load train and test data frames
train_df = pd.read_pickle("./nvd-train-df.pkl")
eval_df = pd.read_pickle("./nvd-eval-df.pkl")

# In[3]:

# install the simpletransformers and pytorch library to train the model
get_ipython().system(' pip install simpletransformers')
get_ipython().system(' pip install torchvision ')

# In[4]:

from simpletransformers.classification import ClassificationModel, ClassificationArgs
import logging
import torch
import torchvision

# set logggin messages
logging.basicConfig(level=logging.INFO)
transformers_logger = logging.getLogger("transformers")
transformers_logger.setLevel(logging.WARNING)

# Optional model configuration
model_args = ClassificationArgs(num_train_epochs=1)

# In[6]:

# Create a ClassificationModel
model = ClassificationModel(
    'bert',
    'bert-base-cased',
    num_labels=3,
    args=model_args,
#     args={'reprocess_input_data': True},
    use_cuda=False,
)

# Train the model
model.train_model(train_df)

# In[12]:

# once the model runs, it is saved in "./outputs", so you can load the model from there without having to train again
model = ClassificationModel(
    "bert",
    "./outputs",
    num_labels=3,
    use_cuda=False
)

# In[8]:

# Evaluate the model
result, model_outputs, wrong_predictions = model.eval_model(eval_df[:6000])

# In[13]:

# Make predictions with the model
predictions, raw_outputs = model.predict(['INSERT_A_DESCRIPTION_FROM_eval_df'])

# In[7]:

# check accuracy of model
len(wrong_predictions)/len(eval_df[:6000])*100

# In[ ]:


import sys, glob, operator
from ColumnDataFile import ColumnDataFile as CDF
import matplotlib.pyplot as plt
from pylab import *
import PlotUtility
import numpy
import Smoothing

def fixPlot(axs):
	axs.set_xlabel(r'r ($\AA$)', fontsize='36')
	xlim(0.5,5.0)
	ylim(0.0,2.0)
	xticks([1,2,3,4,5],fontsize=36)
	yticks([0.5,1,1.5], fontsize=36)

files = ['malonic-rdf-alcO-cp2k.dat', 'malonic-rdf-carbO-cp2k.dat', 'MalonicRDF-rdf-alcO-oldparms.dat', 'MalonicRDF-rdf-carbO-oldparms.dat', 'malonic-rdf-alcO-amber.dat', 'malonic-rdf-carbO-amber.dat', 'MalonicRDF-rdf-alcO-newparms.dat', 'MalonicRDF-rdf-carbO-newparms.dat', 'MalonicRDF-rdf-alcO-1.dat', 'MalonicRDF-rdf-carbO-1.dat']
cdfs = [CDF(f) for f in files]

fig = plt.figure(num=1, facecolor='w', edgecolor='w', frameon=True)
fig.subplots_adjust(left=0.06, right=0.97)

axs1 = fig.add_subplot(1,2,1)
fixPlot(axs1)
axs2 = fig.add_subplot(1,2,2)
fixPlot(axs2)

for c in range(len(cdfs)):
	x_data = numpy.array(cdfs[c][0])
	y_data = numpy.array(cdfs[c][1])

	if c == 1:
		y_data = y_data * 0.8

	axs = axs1
	if (c+1)%2 == 0:
		axs = axs2

	name = files[c].split('.')[-2].split('-')[-1]
	sys = files[c].split('-')[2]
	axs.plot (x_data, y_data, linewidth=3.0, label=name+sys)

	minorLocator = MultipleLocator(0.25)
	axs.yaxis.set_minor_locator(minorLocator)

	#if c < 2:
	#else:
		#xticks([1,2,3,4,5],fontsize=42)
		#yticks([0.5,1,1.5], fontsize=42)
	#if c > 1:
	#axs.set_ylabel(r'g(r)', fontsize='54')

PlotUtility.ShowLegend(axs1)
PlotUtility.ShowLegend(axs2)
plt.show()


import os
import unittest
import warnings

import torch
import yaml

from warp_drive.training.example_training_script_pycuda import setup_trainer_and_train
from warp_drive.training.utils.device_child_process.child_process_base import ProcessWrapper
from warp_drive.training.utils.distributed_train.distributed_trainer_pycuda import (
    perform_distributed_training,
)
from warp_drive.utils.common import get_project_root

_ROOT_DIR = get_project_root()

def launch_process(func, kwargs):
    """
    Run a Python function on a separate process.
    """
    p = ProcessWrapper(target=func, kwargs=kwargs)
    p.start()
    p.join()
    if p.exception:
        raise p.exception

class MyTestCase(unittest.TestCase):
    """
    Test the end-to-end training loop
    """

    num_gpus = torch.cuda.device_count()

    def get_config(self, env_name):
        config_path = os.path.join(
            _ROOT_DIR, "warp_drive", "training", "run_configs", f"{env_name}.yaml"
        )
        if not os.path.exists(config_path):
            raise ValueError(
                "Invalid environment specified! The environment name should "
                "match the name of the yaml file in training/run_configs/."
            )

        with open(config_path, "r", encoding="utf8") as fp:
            run_config = yaml.safe_load(fp)
        return run_config

    def test_tag_gridworld_training(self):
        run_config = self.get_config("tag_gridworld")
        try:
            launch_process(
                setup_trainer_and_train,
                kwargs={"run_configuration": run_config, "verbose": False},
            )
        except AssertionError:
            self.fail("TagGridWorld environment training failed")

    def test_tag_continuous_training(self):
        run_config = self.get_config("tag_continuous")
        try:
            run_config["env"]["num_taggers"] = 2
            run_config["env"]["num_runners"] = 10
            launch_process(
                setup_trainer_and_train,
                kwargs={"run_configuration": run_config, "verbose": False},
            )
        except AssertionError:
            self.fail("TagContinuous environment training failed")

    def test_tag_gridworld_training_with_multiple_devices(self):
        if self.num_gpus <= 1:
            warnings.warn(
                "Only single GPU is detected, we skip trainer test for multiple devices"
            )
            return

        run_config = self.get_config("tag_gridworld")
        run_config["trainer"]["num_gpus"] = self.num_gpus
        try:
            perform_distributed_training(setup_trainer_and_train, run_config)
        except AssertionError:
            self.fail(
                f"TagGridWorld environment training failed for {self.num_gpus} GPUs"
            )


from netcontrol.api import *
from netcontrol.com import *
import threading
import json

class HealthMonitor (Component):
    name = 'health-monitor'

    def on_starting(self):
        self._cond_refresh = threading.Condition()
        self._cond_refreshed = threading.Condition()
        self._lock_refresh = threading.Lock()
        self._state = {}

    def refresh(self):
        with self._lock_refresh:
            with self._cond_refresh:
                self._cond_refresh.notify()
        with self._cond_refreshed:
            self._cond_refreshed.wait()

    def iterate(self):
        cfg = json.loads(self.app.gconfig.get('meters', 'config'))
        res = {}
        for cls in cfg:
            try:
                inst = self.app.grab_plugins(IMeter, lambda x:x.plugin_id==cls)[0]
                for var in cfg[cls]:
                    constr = cfg[cls][var]
                    i = inst.prepare(var)
                    res[i] = getattr(self, 'validate_%s'%i.type)(i.format_value(), constr)
            except:
                pass
        self._state = res

    def get(self):
        return self._state

    def run(self):
        while True:
            with self._lock_refresh:
                try:
                    self.iterate()
                except:
                    pass
            with self._cond_refreshed:
                self._cond_refreshed.notify()
            with self._cond_refresh:
                self._cond_refresh.wait(1000*60*5)

    def validate_binary(self, val, cfg):
        return 'good' if val['value'] == cfg['good_state'] else 'dang'

    def validate_decimal(self, val, cfg):
        if cfg['limit_susp'] < cfg['limit_dang']:
            if val['value'] < cfg['limit_susp']:
                return 'good'
            if val['value'] > cfg['limit_dang']:
                return 'dang'
        if cfg['limit_susp'] > cfg['limit_dang']:
            if val['value'] > cfg['limit_susp']:
                return 'good'
            if val['value'] < cfg['limit_dang']:
                return 'dang'
        return 'susp'

    validate_linear = validate_decimal

class MetersExporter (Plugin, URLHandler):

    @url('^/api/meters$')
    def export(self, req, sr):
        clss = self.app.grab_plugins(IMeter)
        r = {}

        for cls in clss:
            variants = cls.get_variants()
            for v in variants:
                inst = cls.prepare(v)
                r.setdefault(cls.category, {}).setdefault(cls.plugin_id, {})[v] = \
                    {
                        'name': inst.name,
                        'mtype': inst.type,
                        'text': inst.text,
                        'variant': v,
                        'transform': inst.transform,
                        'data': inst.format_value(),
                    }

        return json.dumps(r)

class HealthExporter (Plugin, URLHandler):

    @url('^/api/health$')
    def export(self, req, sr):
        mon = ComponentManager.get().find('health-monitor')
        data = json.loads(self.app.gconfig.get('meters', 'config', '{}'))
        nd = {}
        mon.refresh()
        for i in mon.get():
            nd.setdefault(i.category, {}).setdefault(i.plugin_id, {})[i.variant] = {
                'value': i.format_value(),
                'info': {
                    'text': i.text,
                    'name': i.name,
                    'type': i.type,
                    'variant': i.variant,
                    'transform': i.transform,
                },
                'contraints': data[i.plugin_id][i.variant],
                'state': mon.get()[i],
            }
        return json.dumps(nd)


from __future__ import print_function
import streamlit as st
import base64
from email.message import EmailMessage
import google.auth
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from google.oauth2.credentials import Credentials
from Utilities import isResponseYes,validateEmail
from test import speakText, transcribe_speech_with_repeat_for_send

def speakAndWrite(val):
    # st.write(val)
    speakText(val)

SCOPES = ['https://www.googleapis.com/auth/gmail.readonly',
          "https://mail.google.com/",
          "https://www.googleapis.com/auth/gmail.modify",
          "https://www.googleapis.com/auth/gmail.compose",
          "https://www.googleapis.com/auth/gmail.send"]

def gmail_send_message():
    """Create and send an email message
    Print the returned  message id
    Returns: Message object, including message id

    Load pre-authorized user credentials from the environment.
    TODO(developer) - See https://developers.google.com/identity
    for guides on implementing OAuth2 for the application.
    """
    creds = Credentials.from_authorized_user_file('token.json', SCOPES)

    try:
        service = build('gmail', 'v1', credentials=creds)
        message = EmailMessage()

        message.set_content('This is automated draft mail')

        message['To'] = 'ayush2000mickey@gmail.com',
        message['From'] = 'saiprashant.saxena@gmail.com'
        message['Subject'] = 'Automated draft'

        # encoded message
        encoded_message = base64.urlsafe_b64encode(message.as_bytes()) \
            .decode()

        create_message = {
            'raw': encoded_message
        }
        # pylint: disable=E1101
        send_message = (service.users().messages().send
                        (userId="me", body=create_message).execute())
        print(F'Message Id: {send_message["id"]}')
    except HttpError as error:
        print(F'An error occurred: {error}')
        send_message = None
    return send_message

if __name__ == '__main__':
    gmail_send_message()

def handleSendMail(creds):
    st.markdown("**:blue[What is the subject of the mail?]**")
    speakAndWrite("What is the subject of the mail?")
    subject = transcribe_speech_with_repeat_for_send()
    st.markdown("**:blue[What is the body of the email?]**")
    speakAndWrite("What is the body of the email")
    body = transcribe_speech_with_repeat_for_send()
    st.markdown("**:blue[What is receivers's mail id?]**")
    speakAndWrite("What is receivers's  mail id")
    mailID = transcribe_speech_with_repeat_for_send()
    mailID = mailID.replace(" ", "").replace("attherate", "@").lower()
    while (not validateEmail(mailID)):
        st.error("This Email is Invalid! Can you please repeat...", icon="ðŸ’€")
        speakAndWrite("This Email is Invalid!")
        speakAndWrite("Can you please repeat?")
        mailID = transcribe_speech_with_repeat_for_send()
        mailID = mailID.replace(" ", "").replace("attherate", "@").lower()

    st.markdown("**:blue[Do you want to add cc to this mail?]**")
    speakAndWrite("Do you want to add cc to this mail?")
    shouldAddCc = transcribe_speech_with_repeat_for_send()
    ccMailID = ""
    if (isResponseYes(shouldAddCc)):
        st.markdown("**:blue[What is the mail id?]**")
        speakAndWrite("What is the mail id?")
        ccMailID = transcribe_speech_with_repeat_for_send()
        ccmailID = ccmailID.replace(" ", "").replace("attherate", "@").lower()
        while (not validateEmail(ccmailID)):
            st.error("This Email is Invalid! Can you please repeat...", icon="ðŸ’€")
            speakAndWrite("This Email is Invalid!")
            speakAndWrite("Can you please repeat?")
            ccmailID = transcribe_speech_with_repeat_for_send()
            ccmailID = ccmailID.replace(" ", "").replace("attherate", "@").lower()

    try:
        service = build('gmail', 'v1', credentials=creds)
        message = EmailMessage()
        message.set_content(body)
        message['To'] = mailID
        if (len(ccMailID) > 0):
            message['cc'] = ccMailID.replace(
                " ", "").replace("attherate", "@").lower()
        print(message['To'])
        message['Subject'] = subject
        # encoded message
        encoded_message = base64.urlsafe_b64encode(message.as_bytes()) \
            .decode()
        create_message = {
            'raw': encoded_message
        }
        send_message = (service.users().messages().send
                        (userId="me", body=create_message).execute())
        print(F'Message Id: {send_message["id"]}')
        st.markdown("**:green[Mail Sent Successfully!]**")
        speakAndWrite("Mail Sent Successfully!")
    except HttpError as error:
        print(F'An error occurred: {error}')
        st.text("Receiver's Email Id Incorrect!")
        speakAndWrite("Receiver's Email Id Incorrect!")
        send_message = None


import pandas as pd
import requests
from statsmodels.tsa.stattools import adfuller
import scipy
import matplotlib.pyplot as plt
import numpy as np

sch, tmkt = pd.read_csv('CSUSHPISA.csv'), pd.read_csv('WILL5000INDFC.csv')
rt, gld = pd.read_csv('WILLREITIND.csv'), pd.read_csv('GOLDPMGBD228NLBM.csv')

def assignDates(df, colIndex):
    '''assign date type'''
    x = pd.to_datetime(df.iloc[:,colIndex], yearfirst = True)
    df.iloc[:,colIndex] = pd.DataFrame(x)
    return df

rt, sch = assignDates(rt, 0), assignDates(sch, 0)
gld, tmkt = assignDates(gld, 0), assignDates(tmkt, 0)

def checkIndexing(df):
    holder, ogDF = list(df.iloc[:,0]), df
    time2 = holder[1:]
    lst = [t2 - t1 for t1, t2 in zip(holder, time2)]

    delta = pd.DataFrame(df.iloc[1:,:])
    delta.iloc[:, 0] = lst
    lst = [int(str(i)[:2]) for i in lst]
    d = int(str(delta['DATE'].std())[0])
    irregular = []
    delta.index = [i for i in range(len(delta))]

    for i, j in enumerate(lst):
        if not j - d <= 0 <= j + d:
            irregular.append(i)

    for i, j in zip(enumerate(df.iloc[:,0]), irregular):
        if i[0] == j:
            df = df.drop(labels = j)

    df.index = [i for i in range(len(df))]

    if delta['DATE'].std() / delta['DATE'].mean() > 1:
        return df
    else:
        return ogDF

rt, gld = checkIndexing(rt), checkIndexing(gld)
tmkt, sch = checkIndexing(tmkt), checkIndexing(sch)

def checkHiddenNulls(df, colIndex):
    '''check whether columns have any non-null meaningless values'''
    cleaning = df.iloc[:, colIndex]
    for idx, value in zip(range(len(cleaning)), cleaning):
        try:
            if float(value) / float(value) != 1: #filter out n/a values
                df = df.drop(index = idx)
        except:
            df = df.drop(index = idx)
            #same statement as above because above one won't be executed
            #if it raises error

    df.iloc[:, colIndex] = df.iloc[:, colIndex].astype(float)
    return df

sch, tmkt = checkHiddenNulls(sch, 1), checkHiddenNulls(tmkt, 1)
reit, gld = checkHiddenNulls(rt, 1), checkHiddenNulls(gld, 1)

def getHTML(url):
    '''extract html via url'''
    connection = requests.get(url)
    if connection.ok:
        txt = connection.text.lower()
        text = txt[txt.find('<head>'): txt.find('</head>')]
        text = text.split(' ')
    return text

gldText = getHTML('https://fred.stlouisfed.org/series/GOLDPMGBD228NLBM')
reitText = getHTML('https://fred.stlouisfed.org/series/WILLREITIND')
tmktText = getHTML('https://fred.stlouisfed.org/series/WILL5000INDFC')
schillerText = getHTML('https://fred.stlouisfed.org/series/CSUSHPISA')

def seasonallyAdjust(df):
    '''create boolean seasonal indicator variables'''
    df['Spring'], df['Summer'] = False, False
    df['Fall'], df['Winter'] = False, False
    spring, summer = list(df['Spring']), list(df['Summer'])
    fall, winter = list(df['Fall']), list(df['Winter'])

    count = 0
    for date in df.iloc[:,0]:
        if date.month == 1 or date.month == 2 or date.month == 3:
            winter[count] = True
            count += 1
        elif date.month == 4 or date.month == 5 or date.month == 6:
            spring[count] = True
            count += 1
        elif date.month == 7 or date.month == 8 or date.month == 9:
            summer[count] = True
            count += 1
        elif date.month == 10 or date.month == 11 or date.month == 12:
            fall[count] = True
            count += 1

    df['Fall'], df['Winter'] = fall, winter
    df['Summer'], df['Spring'] = summer, spring
    return df

reit, gld = seasonallyAdjust(reit), seasonallyAdjust(gld)
schiller, tmkt = seasonallyAdjust(tmkt), seasonallyAdjust(sch)

def firstDifference(df, colIndex):
    '''insure against autocorrelation by first-differencing the data'''
    holder = list(df.iloc[:, colIndex])
    time2 = holder[1:]
    lst = [t2 - t1 for t1, t2 in zip(holder, time2)]

    delta = pd.DataFrame(df.loc[1:,:]) #was raising error w/o pd.DF wrapper
    delta.iloc[:, colIndex] = lst
    return delta

reit, gld = firstDifference(reit, 1), firstDifference(gld, 1)
tmkt, sch = firstDifference(tmkt, 1), firstDifference(schiller, 1)

def unitRootTest(df, colIndex):
    '''Run Dickey-Fuller test on column to check for unit root'''
    x = df.iloc[:, colIndex]
    dfuller = adfuller(x)
    results = {'ADF Statistic': dfuller[0],
            'p-value': dfuller[1], 'Critical Values': dfuller[4]}
    if float(dfuller[1]) < .05:
        results['Reject Null'] = True #no unit root
    else:
        results['Reject Null'] = False #yes unit root
    return results

reitRoot, gldRoot = unitRootTest(reit, 1), unitRootTest(gld, 1)
sRoot, tmktRoot = unitRootTest(schiller, 1), unitRootTest(tmkt, 1)
#if unit root present, too much risk to create linear model

def kurtosisMaxObservation(df, colIndex):
    '''return the percentage of kurtosis due to one observation'''
    data = df.iloc[:, colIndex]
    mean = data.mean()
    num, den = [], []

    for i, j in enumerate(data):
        distance = j - mean
        num.append(distance ** 4)
        den.append(distance ** 2)
    for a, b in enumerate(num):
        if b == max(num):
            location = a
    num, den = np.array(num), np.array(den)
    numAv, denAv = num.mean(), (den.mean()) ** 2
    kurtosis = (numAv / denAv) - 3
    if kurtosis == scipy.stats.kurtosis(df.iloc[:, colIndex]):
        proportion = num.max() / sum(num)
        return proportion * 100, location
    else:
        raise Exception('The kurtosis is incorrect')

sA, rA = kurtosisMaxObservation(sch, 1), kurtosisMaxObservation(reit, 1)
gA, tA = kurtosisMaxObservation(gld, 1), kurtosisMaxObservation(tmkt, 1)

def plotKurtosisObs(df, iloc, text, idx, kurtosis, condition):
    '''visualize vectors with max single-obs. kurtosis'''
    text = str(text) #extract title from html
    t = text[text.find('<title>'):text.find('</title>')].strip("")
    t, var = t.split(','), str()
    for i in t:
        var += i
    t = var[7:]
    figure = plt.figure()
    ax = figure.add_subplot(111)
    ax.plot(df.iloc[:, iloc], 'steelblue', 3)
    ax.annotate('Kurtosis: ' + str(round(kurtosis, 3)) + '%',
                xy = (idx, df.iloc[idx,iloc]))
    plt.ylabel('Delta')
    plt.xlabel('Time 0: ' + str(df.iloc[0, 0])[:-9] + '\n'
               'Time n: ' + str(df.iloc[-1, 0])[:-9] + '\n'
               'Unit Root: ' + str(not condition))
    plt.ylim(top = 1.5 * max(df.iloc[:,iloc]))
    plt.title(t)
    plt.show()

plotKurtosisObs(sch, 1, schillerText, sA[1], sA[0], sRoot['Reject Null'])
plotKurtosisObs(reit, 1, reitText, rA[1], rA[0], reitRoot['Reject Null'])
plotKurtosisObs(gld, 1, gldText, gA[1], gA[0], gldRoot['Reject Null'])
plotKurtosisObs(tmkt, 1, tmktText, tA[1], tA[0], tmktRoot['Reject Null'])


import logging
from util import func

import generalEquations as equ
import mode

class RocketEngine(object):
	def __init__(self, engineStats):
		self.attached = True
		self.throt_cur = 0.0
		self.throt_prev = 0.0
		self.burn_rate = 0.0
		self.messages = []
		self.thrust_total = 0.0
		self.reached_max = False

		# The following are overwritten by engine stats
		self.event = {}
		self.power_down_start_time = None
		self.ignition_time = None
		self.jettison_time = None
		self.min_throt = 0.0
		self.max_throt = 1.0
		self.engine_count = 0
		self.thrust_sl = 0.0
		self.thrust_vac = 0.0
		self.stage = None
		self.throt_rate_of_change_limit = 1.0
		self.name = "Not set yet"
		self.specImp_sl = 0.0
		self.specImp_vac = 0.0

		for key, value in engineStats.items():
			setattr(self, key, value)

		if self.power_down_start_time:
			self.power_down_start_time = round(self.power_down_start_time, 1)
		if self.ignition_time:
			self.ignition_time = round(self.ignition_time, 1)
		if self.jettison_time:
			self.jettison_time = round(self.jettison_time, 1)
	def __str__(self):
		return "Name={} Throt={} Eff Burn Rate={} Thrust={} Count={} Stage={}".format(self.name, self.throt_avg, self.get_eff_fuel_burn_rate(), self.get_thrust_total(), self.engine_count, self.stage)

	@staticmethod
	def factory(engine_data):
		''' creates engines that burn either solid or liquid fuel '''
		if engine_data["type"] == "Solid":
			return SolidRocketEngine(engine_data)
		elif engine_data["type"] == "Liquid":
			return LiquidRocketEngine(engine_data)
		else:
			raise ValueError("Unsupported engine type.")

	def set_fuel_source(self, source):
		self.fuel_source = source

	def engine_stats(self, starting_throttle, power_down_start_time, ignition_time=None, jettison_time=None):

		stat_time = 0.0
		if ignition_time:
			stat_time = ignition_time

		if power_down_start_time == float('inf'):
			sim_end = stat_time + 6
			# 6 seconds is more than enough time for any engine to hit max
		else:
			sim_end = power_down_start_time + 5.0
		time_inc = 0.1
		stats = {
			"max_reached_time": None,
			"engine_cutoff_time": None,
			"min_reached_time" : None,
			"throttle_at_end_time": None
		}
		if starting_throttle == 0 and not ignition_time:
			# if the engine is never turned on, get out of here
			return stats
		max_reached = False
		min_reached = False
		self.setThrottleOverride(starting_throttle)
		self.setThrottleOverride(starting_throttle)
		while stat_time < sim_end:
			#print(stat_time, self.throt_cur)
			if stat_time != 0:
				self.events(stat_time, time_inc)
			if not max_reached:
				if self.throt_cur == self.max_throt:
					max_reached = True
					stats['max_reached_time'] = stat_time

			if stat_time == jettison_time:
				stats['engine_cutoff_time'] = stat_time

				return stats
			if max_reached and not min_reached and func.less_than_or_almost_equals(self.throt_avg, self.min_throt):
				min_reached = True
				stats['min_reached_time'] = stat_time

			if max_reached and self.throt_avg == 0.0:
				stats['engine_cutoff_time'] = stat_time
				return stats
			stats['throttle_at_end_time'] = self.throt_cur # keep updating the throttle at end, until the end
			stat_time += time_inc
		return stats

	def requested_throt_to_float(self, requested_throt):
		''' Converts the requested throttle to a floating point number between 0.0 and 1.0 '''
		assert func.is_float(requested_throt) or requested_throt in ["min", "max", "off"]
		if requested_throt == "max":
			requested_throt = self.max_throt
		elif requested_throt == "min":
			requested_throt = self.min_throt
		elif requested_throt == "off":
			requested_throt = 0.0

		if requested_throt >= self.max_throt:
			requested_throt = self.max_throt
		assert func.is_float(requested_throt) and requested_throt >= 0.0 and requested_throt <= 1.0
		return requested_throt

	def setThrottleOverride(self, requested_throt):
		''' allows setting throttle without regard to the engine's max rate of change '''
		self.throt_prev = self.throt_cur
		requested_throt = self.requested_throt_to_float(requested_throt)
		self.throt_cur = requested_throt

	@property
	def throt_avg(self):
		return equ.average(self.get_throt(), self.get_throt("prev"))

	def get_throt(self, when="current"):
		if when == "current":
			return self.throt_cur
		elif when == "prev":
			return self.throt_prev
		else:
			raise ValueError("Only current and prev values stored")

	def get_burn_rate(self, when="current"):
		return self.burn_rate

	def get_thrust_total(self, when="current"):
		return self.thrust_total

	def get_thrust_per_engine(self):
		return self.thrust_total / self.engine_count

	def setThrottle(self, requested_throt, time_inc):
		self.throt_prev = self.throt_cur
		#Limit the throttle to its max change limit
		requested_throt = self.requested_throt_to_float(requested_throt)

		if requested_throt == self.get_throt():
			self.throt_cur = requested_throt
			return requested_throt

		if requested_throt > self.get_throt():
			direction = 1.0
			verb = "increased"
		else:
			direction = -1.0
			verb = "decreased"
		#direction 1 for pos and -1 for neg changes
		if abs(requested_throt - self.get_throt()) > (self.throt_rate_of_change_limit * time_inc):
			throt = self.get_throt() + time_inc * self.throt_rate_of_change_limit * direction
		else:
			throt = requested_throt
		if throt > self.max_throt:
			logging.debug("Max Throt achieved for %s", self.name)
			self.throt_cur = self.max_throt
		elif throt < self.min_throt:
			if direction < 0:
				logging.debug("%s Engine shutoff", self.name)
				self.throt_cur = 0
			else:
				logging.debug("%s Engine set to min throt", self.name)
				self.throt_cur = self.min_throt
		else:
			self.throt_cur = throt

		if self.get_throt("prev") != self.get_throt():
			self.messages.append("\nEVENT: {} Throttle {} from {} to {} time inc {}".format(self.name, verb, self.get_throt("prev"), self.get_throt(), time_inc))

	def thrustAtAlt(self, alt):
		''' calculates the thrust at altitude and stores and returns that value '''
		patm = equ.percentOfAtmosphericPressure(alt)
		pctVac = 1.0 - patm
		if mode.THROTTLE_FINAL_UP and self.get_throt("prev") < self.get_throt():
			self.thrust_total = self.engine_count * self.get_throt() * (self.thrust_sl + (pctVac * (self.thrust_vac - self.thrust_sl)))
		else:
			self.thrust_total = self.engine_count * self.throt_avg * (self.thrust_sl + (pctVac * (self.thrust_vac - self.thrust_sl)))
		return self.get_thrust_total()

	def get_eff_fuel_burn_rate(self, for_all=True):
		''' returns the average throttle times the burn rate '''
		if for_all:
			return self.throt_avg * self.get_burn_rate() * self.engine_count
		else:
			return self.throt_avg * self.get_burn_rate()

	def burn_fuel(self, time_inc, alt=None):
		''' Burns fuel (increasing the total fuel used)
		Alt is ignored for this engine type
		'''
		if self.throt_avg > 0.0:
			inc_fuel_used = self.get_eff_fuel_burn_rate() * time_inc
			# print self.name, self.stage, "\nthrot_avg", self.throt_avg, "\neff burn rate", self.get_eff_fuel_burn_rate(), "\ntime_inc", time_inc, "\nInc fuel used", inc_fuel_used, "\n"
			self.fuel_source.fuel_used += inc_fuel_used

	def specific_impulse_at_alt(self, alt):
		''' calculates the specific impulse at the given altitude '''
		patm = equ.percentOfAtmosphericPressure(alt)
		pctVac = 1 - patm
		return self.specImp_sl + (pctVac * (self.specImp_vac - self.specImp_sl))

	def events(self, time, time_inc):
		decimal_precision = 3
		if self.throt_avg > 0.0 and not self.reached_max:
			self.setThrottle(self.max_throt, time_inc)
			if self.throt_avg == self.max_throt:
				self.reached_max = True

class SolidRocketEngine(RocketEngine):
	engine_type = "Solid"
	def __init__(self, engine_data):
		RocketEngine.__init__(self, engine_data)
		self.power_down_start_time = None
		self.power_down_fuel_level = engine_data["power_down_fuel_level"]
		self.power_down_burn_rates = engine_data["power_down_burn_rates"]

	def set_assigned_thrust_per_engine(self, thrust):
		self.thrust_total = thrust * self.engine_count

	def burn_fuel(self, time_inc, alt):

		if self.throt_avg > 0.0:
			self.burn_rate = self.get_thrust_total()  / self.specific_impulse_at_alt(alt)
			self.fuel_source.fuel_used += self.get_burn_rate() * time_inc * self.get_throt()
		# print "solid rocket {} at alt {}".format(self.get_burn_rate(), alt)
	def adjust_thrust_to_burn_at_rate_per_engine(self, rate, alt):
		thrust = rate * self.specific_impulse_at_alt(alt)
		message = "EVENT: Set burn rate of {} (thrust = {})".format(rate, thrust)
		self.messages.append(message)
		self.set_assigned_thrust_per_engine(thrust)

	def change_thrust(self, rate, time_inc):
		currentThrust = self.get_thrust_per_engine()
		newThrust = currentThrust + rate * time_inc
		self.set_assigned_thrust_per_engine(newThrust)
		message = None
		if rate < 0:
			message = "\nEVENT: Reduced thrust of {} to {}".format(self.name, newThrust)
		if rate > 0:
			message = "\nEVENT: Increased thrust of {} to {}".format(self.name, newThrust)
		if message:
			self.messages.append(message)

	def thrustAtAlt(self, alt):
		return self.get_thrust_total() * self.get_throt()

	def events(self, time, time_inc):
		super(SolidRocketEngine, self).events(time, time_inc)
		if self.event:
			if func.between_floats(time, self.event["start_time"], self.event["end_time"]):
				if self.event["name"] == "Change Thrust" and time < self.event["end_time"]:
					self.change_thrust(self.event["rate"], time_inc)

	def power_down_by_burn_rates(self, time, alt, stage):
		time_since_start = time - self.power_down_start_time
		if func.almost_equal(time_since_start, round(time_since_start)):
			try:
				self.burn_rate_per_engine = self.power_down_burn_rates.pop(0)
			except IndexError:
				stage.jettison()
		if stage.attached:
			self.adjust_thrust_to_burn_at_rate_per_engine(self.burn_rate_per_engine, alt)

class LiquidRocketEngine(RocketEngine):
	''' Liquid Rocket Engine class '''
	power_down_in_progress = False
	engine_type = "Liquid"
	def events(self, time, time_inc):
		super(LiquidRocketEngine, self).events(time, time_inc)
		if self.ignition_time and func.almost_equal(time, self.ignition_time, 0.1):
			self.setThrottle(self.min_throt, time_inc)

		if self.power_down_in_progress or self.power_down_start_time and func.almost_equal(time, self.power_down_start_time, 0.1):
			self.setThrottle(0, time_inc)
			self.power_down_in_progress = True
			if self.throt_avg == 0:
				self.power_down_in_progress = False


from apscheduler.schedulers.blocking import BlockingScheduler
import requests

sched = BlockingScheduler()

@sched.scheduled_job('interval', id="keepruning_jobs", minutes=20)
def keepruning():
    url = "https://neo-linebot-image.herokuapp.com/"
    r = requests.get(url)

sched.start()


# -*- coding: utf-8 -*-
import common.models as MODELS
import logging
from django.db.models import Q

logger = logging.getLogger('app')

def getUserList():
    """
    ユーザリストを取得する
    """
    logger.info('getUserList')

    return MODELS.User.objects.all().order_by('last_login').reverse()

def getUserProgressList(user):
    """
    ユーザの進捗一覧を取得する
    return progress_managent[]
    """
    logger.info('getUserProgressList: ' + user.username)
    progress_management_list = []

    result = MODELS.Progress_management.objects.filter(responsible_by=user.username).order_by('target', 'registered_at')
    if result:
        progress_management_list = result

    return progress_management_list

def getTeamProgressList(user):
    """
    ユーザからチームの別メンバの進捗を取得する
    return progress_management[]
    """
    logger.info('getTeamProgressList: ' + user.username)
    team_progress_lists = []

    # 自分のメンバーシップ取得
    my_membership = MODELS.Membership.objects.filter(user=user)

    # 1ユーザ1チームにしか対応していないため、とれた1件目のメンバーシップ取得
    if len(my_membership) == 1:
        my_membership = my_membership[0]

        #  チームが同一の自分以外のメンバーシップを取得
        team_membership_list = MODELS.Membership.objects.filter(team=my_membership.team).exclude(user=user)

        # 自分以外のチームのメンバーシップのユーザに紐づく進捗をリストに詰める
        for team_membership in team_membership_list:
            team_progress_list= MODELS.Progress_management.objects.filter(responsible_by=team_membership.user).order_by('target', 'registered_at')
            team_progress_lists.append(team_progress_list)

    else:
        # 自分のメンバーシップが1件以外の場合、何もしない
        pass

    return team_progress_lists

def getNumTarget(user):
    """
    ユーザが登録したターゲット数を返す
    return int
    """

    return  MODELS.Target.objects.filter(target_register__user=user).count()

def selectUserById(user_id):
    logger.info('selectUserById : ' + user_id)
    try:
        user = MODELS.User.objects.get(username=user_id)
        return user
    except MODELS.User.DoesNotExist:
        return None

def selectTeamsByUser(user):
    """
    ユーザが所属するチームのリストを返す
    """
    logger.info('selectTeamsByUser')
    team_list = []
    membership_list = MODELS.Membership.objects.filter(user=user)
    if len(membership_list) >= 1:
        for membership in membership_list:
            team_list.append(membership.team)
    return team_list

def selectRelatedTarget(user):
    """
    ユーザが登録した、もしくは担当しているターゲットを取得する
    return target[]
    """
    return MODELS.Target.objects.filter(Q(progress_management__responsible_by=user) | Q(target_register__user=user))

def selectActiveUser(target):
    """
    ターゲットの現担当者を帰す
    return user or None
    """
    try:
        if target.taken_flg:
            latest_progress_management = MODELS.Progress_management.objects.filter(target=target).latest('registered_at')
            return latest_progress_management.responsible_by
        else:
            return None
    except:
        return None

def deleteUser(user):
    """
    ユーザを削除する
    連動して削除するものは以下
    membership, invitation, progress_management, target_register、target、introducer、
    もしチーム管理者の場合、teamとそれに紐づくmembershipとinvitation全て

    もし削除対象ユーザの登録ターゲットの担当者が別ユーザである場合、ターゲットは担当者の登録ターゲットとなる

    削除順番(チーム関連)
    1.own membership or team's membership
    2.own invitation or team's invitation
    3.team if admin

    削除順番(ターゲット関連)
    1.もし登録ターゲットの担当が別ユーザだった場合、ターゲットは担当者の登録ターゲットとなる = target_registerをupdate
    2.削除対象ターゲットがintroducerのintroduceレコードを全て削除
    3.削除対象ターゲットがターゲットの全てのtarget_register
    4.削除対象がターゲットの全てのprogress_management
    5.削除対象ターゲット

    """

    #チーム関連項目削除
    team_list = selectTeamsByUser(user)

    for team in team_list:
        membership = MODELS.Membership.objects.get(user=user, team=team)
        if membership.team_admin_flg:
            MODELS.Membership.objects.filter(team = team).delete()
            MODELS.Invitation.objects.filter(team = team).delete()
            team.delete()
        else:
            membership.delete()
            MODELS.Invitation.objects.filter(invited_user=user).delete()

    #ターゲット関連項目削除
    target_list = selectRelatedTarget(user)

    for target in target_list:
        register_flg = False
        responsible_flg = False
        if MODELS.User.objects.get(target_register__target=target) == user:
            register_flg = True
        if user in MODELS.User.objects.filter(progress_management__target=target):
            responsible_flg = True

        if register_flg & responsible_flg:
            MODELS.Introduce.objects.filter(Q(target=target) | Q(introduced_by=target)).delete()
            MODELS.Progress_management.objects.filter(target=target).delete()
            MODELS.Target_register.objects.filter(target=target).delete()
            target.delete()
        elif responsible_flg:
            MODELS.Progress_management.objects.filter(target=target,responsible_by=user).delete()
        elif register_flg:
            active_user = selectActiveUser(target)
            if active_user:
                target_register = MODELS.Target_register.objects.get(target=target)
                target_register.user = active_user
                target_register.save()

                MODELS.Progress_management.objects.filter(target=target, responsible_by=user).delete()
            else:
                MODELS.Introduce.objects.filter(Q(target=target) | Q(introduced_by=target)).delete()
                MODELS.Progress_management.objects.filter(target=target).delete()
                MODELS.Target_register.objects.filter(target=target).delete()
                target.delete()

    user.is_active = False
    user.save()
    return "success"

def activateUser(user):
    """
    ユーザを再稼動させる
    return string
    """
    user.is_active = True
    user.save()

    return "success"

def deactivateUser(user):
    """
    ユーザを再稼動させる
    return string
    """
    user.is_active = False
    user.save()

    return "success"


from pydantic import BaseModel, Extra, Field

class Reference(BaseModel):
    """
    A simple object to allow referencing other components in the specification, internally and externally.

    The Reference Object is defined by [JSON Reference](https://tools.ietf.org/html/draft-pbryan-zyp-json-ref-03)
    and follows the same structure, behavior and rules.

    For this specification, reference resolution is accomplished as defined by the JSON Reference specification
    and not by the JSON Schema specification.
    """

    ref: str = Field(alias="$ref")
    """**REQUIRED**. The reference string."""

    class Config:
        extra = Extra.ignore
        allow_population_by_field_name = True
        schema_extra = {
            "examples": [{"$ref": "#/components/schemas/Pet"}, {"$ref": "Pet.json"}, {"$ref": "definitions.json#/Pet"}]
        }


import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler, MinMaxScaler

def read_data(paths, delimiter, skip_header):
    data = []
    for path in paths:
        subdata = np.genfromtxt(path, delimiter=delimiter, skip_header=skip_header)
        data.append(subdata)
    data = np.concatenate(data)
    return data

def read_data_pandas(paths, delimiter):
    data = []
    for path in paths:
        subdata = pd.read_csv(path, delimiter=delimiter)
        data.append(subdata)
    data = pd.concat(data)
    return data

def export_predictions(X, model, filename):
    y_pred = model.predict(X)
    ids = np.array(range(3000))
    df = {'Id': ids,
          'Bound': y_pred.astype(int)}
    df = pd.DataFrame(df).set_index('Id')
    df.to_csv(filename)

def getKmers(sequence, size):
    return [sequence[x:x + size] for x in range(len(sequence) - size + 1)]

def preprocess_tfidf(X_train, X_test, window_size=6, pca_components=100):
    n1 = X_train.shape[0]
    X = pd.concat([X_train, X_test])
    X['kMers'] = X['seq'].apply(lambda x: getKmers(x, size=window_size))
    X['sentences'] = X['kMers'].apply(lambda x: ' '.join(x))

    vectorizer = TfidfVectorizer(min_df=5)
    tfidf = vectorizer.fit_transform(X['sentences']).toarray()

    pca = PCA(n_components=pca_components)
    tfidf_ = pca.fit_transform(tfidf)

    scaler = MinMaxScaler()
    tfidf_ = scaler.fit_transform(tfidf_)

    return tfidf_[:n1], tfidf_[n1:]

import gudhi as gd
import gudhi.representations
from gudhi import plot_persistence_diagram
import tqdm.notebook as tq

def preprocess_TDA(X_train,X_test,base2idx = {'A':0,'C':1,'G':2,'T':3},e = np.eye(4)):

    print('Preprocess training samples : Compute Persistence Diagrams..')
    train_dgms = compute_persistence_diagrams(X_train['seq'].values)
    print('Preprocess test samples : Compute Persistence Diagrams..')
    test_dgms = compute_persistence_diagrams(X_test['seq'].values)
    return train_dgms, test_dgms

def compute_persistence_diagrams(X):
    ftrs = []
    for i,seq in tq.tqdm(enumerate(X),total = len(X)):
        dgm, dim_1_ftrs = seq2pd(seq) #compute persistence diagrams but only use 1-dimensional features for Sliced Wasserstein Kernel
        ftrs.append(dim_1_ftrs)
    return ftrs

def seq2pd(seq, base2idx = {'A':0,'C':1,'G':2,'T':3},e = np.eye(4)):

    b = seq2point_cloud(seq, base2idx = {'A':0,'C':1,'G':2,'T':3},e = np.eye(4))
    skeleton = gd.RipsComplex(points = b, max_edge_length = 0.8)
    Rips_simplex_tree_sample = skeleton.create_simplex_tree(max_dimension = 2)
    #compute persistence diagram for dimensions 0 and 1
    #but only 1-dimensional features will be used for the kernel
    dgm = Rips_simplex_tree_sample.persistence()
    dim_1_ftrs = Rips_simplex_tree_sample.persistence_intervals_in_dimension(1)
    return dgm, dim_1_ftrs

def seq2point_cloud(seq, base2idx = {'A':0,'C':1,'G':2,'T':3},e = np.eye(4)):
    b = np.zeros((len(seq),4))
    for k,base in enumerate(seq):
        if k==0:
            b[k] = e[base2idx[base]]
        else:
            b[k] = 0.5*(b[k-1]+e[base2idx[base]])
    return b


import argparse
import datetime
import subprocess
import psutil
import time
import os
import threading
from tqdm import tqdm
from random import shuffle

FNULL = open(os.devnull, 'w')
facesPath = None
videosPath = None

def maybeCreateDir(path):
    if not os.path.exists(path):
        os.mkdir(path)

def getSize():
    tot = 0
    for f in os.listdir(facesPath):
        fp = os.path.join(facesPath, f)
        tot += os.path.getsize(fp)
    for f in os.listdir(videosPath):
        fp = os.path.join(videosPath, f)
        tot += os.path.getsize(fp)
    return tot

def runCommands(commands, idx):
    shuffle(commands)
    i = 0
    while i < len(commands):
        size = getSize()
        if size > 2 * (1024 ** 3):
            time.sleep(10)
            continue

        print("{}: ({}/{}), size = {}".format(idx, i + 1, len(commands), size))
        subprocess.run(commands[i], stdout=FNULL, stderr=subprocess.STDOUT)
        i += 1

parser = argparse.ArgumentParser()
parser.add_argument("-op", "--outputPath", required = True)
parser.add_argument("-t", "--numThreads", required = True)
args = parser.parse_args()

outputPath = args.outputPath
videosPath = os.path.join(outputPath, "videos")
facesPath = os.path.join(outputPath, "faces")
maybeCreateDir(outputPath)
maybeCreateDir(videosPath)
maybeCreateDir(facesPath)

alreadyThere = set()
for x in os.listdir(facesPath):
    alreadyThere.add(x.rsplit('_', 1)[0])

commands = []
with open("avspeech_train.csv", "r") as f:
    for line in tqdm(f.readlines()):
        line = line.split(',')
        videoId = line[0]

        if videoId in alreadyThere:
            continue

        start = float(line[1])
        end = min(float(line[2]), start + 60)

        start = str(datetime.timedelta(seconds=start))
        end = str(datetime.timedelta(seconds=end))

        command = [ "youtube-dl",
                    "https://www.youtube.com/watch?v={}".format(videoId),
                    "--external-downloader",
                    "ffmpeg",
                    "-f",
                    "webm",
                    "--external-downloader-args",
                    "-ss {} -to {}".format(start, end),
                    "-o",
                    os.path.join(videosPath, "{}.webm".format(videoId))
        ]

        commands.append(command)

numThreads = int(args.numThreads)
chunkSize = len(commands) // numThreads
splits = [[chunkSize * i, chunkSize * (i + 1)] for i in range(numThreads)]
splits[-1][-1] = len(commands)

for i, (l, r) in enumerate(splits):
    t = threading.Thread(target=runCommands, args=(commands[l:r], i))
    t.start()


from typing import Union, List, Callable, Optional, Dict
import urllib.parse
import os
import numpy as np
import torch
import torchvision
from .html_table import HTMLTableVisualizer
import logging

logger = logging.getLogger(__name__)
VCV_URL_PREFIX = "http://vcv.stanford.edu"
VCV_FILE_URL_PREFIX = f"{VCV_URL_PREFIX}/cgi-bin/file-explorer/?"

def setup_vi(path=None, exist_ok=True, title='default'):
    from ..common import vi_helper

    if path is None:
        from ..common import vi
        return vi, vi_helper
    if not exist_ok:
        assert not os.path.exists(path), (path, title)
    vi = HTMLTableVisualizer(path, title=title)
    vi.begin_html()
    vi_helper.print_url(vi)
    return vi, vi_helper

def tensor2im(image_tensor: Union[torch.Tensor, List[torch.Tensor], None]):
    """

    Args:
        image_tensor: (list of) shape [..., C, H, W]

    Returns:

    """
    if image_tensor is None:
        return None
    if isinstance(image_tensor, list):
        return [tensor2im(x) for x in image_tensor]  # keep the list structure
    # if image_tensor.dim() == 5 or image_tensor.dim() == 4:
    #     return [tensor2im(image_tensor[idx])
    #             for idx in range(image_tensor.size(0))]

    # n_dim = len(image_tensor.shape)  # (..., c, h, w)
    image_tensor = image_tensor.movedim(-3, -1)

    image_numpy = image_tensor.cpu().float().numpy()
    # assume range (0, 1)
    image_numpy = image_numpy * 255.0
    image_numpy = np.clip(image_numpy, 0, 255)  # (..., h, w, c)

    # to pil
    # output_img = Image.fromarray(np.uint8(image))
    # if width is not None and height is not None:
    #     output_img = output_img.resize((width, height), Image.BICUBIC)
    return image_numpy.astype(np.uint8)

def collect_tensor(tensor: torch.Tensor,
                   process_grid: Callable[[torch.Tensor], np.ndarray] = tensor2im,
                   pad_value=0, padding=0,
                   value_check: bool = True):
    """

    Args:
        tensor: expect range [0, 1]
        pad_value:
        padding:
        process_grid: e.g. tensor2im, tensor2flow
        value_check:

    Returns:

    """
    if value_check and (tensor.min() < -1e-3 or tensor.max() > 1 + 1e-3):
        print("checking min / max, not normalized? ")
        print(tensor.min(), tensor.max())
        # raise

    if len(tensor.shape) == 3:
        tensor = tensor[None]
    elif len(tensor.shape) == 4:  # (n, c, h, w)
        nrow_this = 1
    elif len(tensor.shape) == 5:  # (n, t, c, h, w)
        nrow_this = tensor.shape[1]

    tensor = tensor.flatten(end_dim=-4)
    if tensor.shape[0] == 1:
        # this is a hack since make_grid does not pad bs=1 tensors
        image_grid = torch.ones((tensor.shape[-3], tensor.shape[-2] + 2 * padding, tensor.shape[-1] + 2 * padding),
                                ) * pad_value
        image_grid.narrow(1, padding, tensor.shape[-2]).narrow(
            2, padding, tensor.shape[-1]
        ).copy_(tensor[0])

        if image_grid.shape[0] == 1:
            # output as rgb channels
            image_grid = image_grid.expand(3, image_grid.shape[1], image_grid.shape[2])

    else:
        image_grid = torchvision.utils.make_grid(tensor, nrow=nrow_this, pad_value=pad_value,
                                                 padding=padding)  # (b or b*d, c, h, w) -> (c, h, w)
        # make_grid will always output channel = 3
    image_grid_np = process_grid(image_grid)
    return image_grid_np

def print_viscam_url(path, verbose=True):
    path = os.path.abspath(path)
    # local file
    url = f"file:///{path}"
    if verbose:
        logger.info(f"[URL] {url}")
    return url

def print_vcv_url(path, verbose=True, nrow=4, query_kwargs: Optional[Dict] = None) -> str:
    path = os.path.abspath(path)

    filename = ''
    if os.path.isfile(path):
        path, filename = os.path.split(path)

    query = dict(
        dir=path,
        patterns_show=filename,
        patterns_highlight='',
        w=1500 // nrow, h=1500,
        n=nrow, autoplay=1, showmedia=1,
    )
    if query_kwargs is not None:
        query.update(query_kwargs)
    url_query = urllib.parse.urlencode(query)
    url = f'{VCV_FILE_URL_PREFIX}{url_query}'
    if verbose:
        logger.info(f"[URL] {url}")
    return url


from flask import Flask, render_template
import sqlite3

app = Flask(__name__)

def get_data():
    conn = sqlite3.connect("air_quality_data.db")
    cur = conn.cursor()
    cur.execute("SELECT * FROM measurements")
    data = cur.fetchall()
    conn.close()
    return data

@app.route("/")
def index():
    data = get_data()
    return render_template("index.html", data=data)

if __name__ == "__main__":
    app.run(debug=True, use_reloader=False)


import __builtin__
import etcd
import maps
import mock
from mock import patch
import pytest

from tendrl.commons.flows.exceptions import FlowExecutionFailedError
from tendrl.commons.objects.node.flows.stop_services import \
    StopServices
import tendrl.commons.objects.node_context as node
from tendrl.commons.objects.service import Service
from tendrl.commons import TendrlNS
from tendrl.commons.utils import cmd_utils
from tendrl.commons.utils import etcd_utils
from tendrl.commons.utils import log_utils as logger

def read(*args, **kwargs):
    if args:
        if args[0] == "nodes/Test_node/Os":
            raise etcd.EtcdKeyNotFound
    else:
        return None

def service_success(param):
    return None, '', 0

def service_failed(param):
    return '', 'error', 0

def service_info(*args, **kwargs):
    return {"exists": True, "running": True, "error": []}

def service_error(*args, **kwargs):
    return {"exists": True,
            "running": False,
            "error": ["permission issue"]
            }

def mock_log(*args, **kwargs):
    pass

@patch.object(etcd, "Client")
@patch.object(etcd.Client, "read")
@patch.object(node.NodeContext, '_get_node_id')
@patch.object(etcd_utils, 'read')
def init(patch_etcd_utils_read,
         patch_get_node_id,
         patch_read,
         patch_client):
    patch_get_node_id.return_value = 1
    patch_read.return_value = etcd.Client()
    patch_client.return_value = etcd.Client()
    setattr(__builtin__, "NS", maps.NamedDict())
    setattr(NS, "_int", maps.NamedDict())
    NS._int.etcd_kwargs = {
        'port': 1,
        'host': 2,
        'allow_reconnect': True}
    NS._int.client = etcd.Client(**NS._int.etcd_kwargs)
    NS._int.wclient = etcd.Client(**NS._int.etcd_kwargs)
    NS["config"] = maps.NamedDict()
    NS.config["data"] = maps.NamedDict()
    NS.config.data['tags'] = "test"
    NS.config.data['logging_socket_path'] = "test"
    NS.publisher_id = "node_context"
    NS.config.data['etcd_port'] = 8085
    NS.config.data['etcd_connection'] = "Test Connection"
    patch_etcd_utils_read.return_value = maps.NamedDict(
        value='{"status": "UP",'
              '"pkey": "tendrl-node-test",'
              '"node_id": "test_node_id",'
              '"ipv4_addr": "test_ip",'
              '"tags": "[\\"my_tag\\"]",'
              '"sync_status": "done",'
              '"locked_by": "fd",'
              '"fqdn": "tendrl-node-test",'
              '"last_sync": "date"}')
    tendrlNS = TendrlNS()
    return tendrlNS

def test_constructor():
    init()
    StopServices()

@mock.patch(
    'tendrl.commons.event.Event.__init__',
    mock.Mock(return_value=None)
)
@mock.patch(
    'tendrl.commons.message.Message.__init__',
    mock.Mock(return_value=None)
)
def test_run():
    init()
    obj = StopServices()
    assert obj.parameters is not None
    obj.parameters = maps.NamedDict()
    obj.parameters['job_id'] = "sample_uuid"
    obj.parameters['flow_id'] = "sample_uuid_1"
    obj.parameters["Services[]"] = ["TestService"]
    with patch.object(logger, 'log', mock_log):
        with patch.object(cmd_utils.Command, 'run', service_success):
            with patch.object(Service, 'get_service_info', service_info):
                ret_val = obj.run()
                assert ret_val is True
    with patch.object(logger, 'log', mock_log):
        with patch.object(cmd_utils.Command, 'run', service_failed):
            with patch.object(Service, 'get_service_info', service_error):
                with pytest.raises(FlowExecutionFailedError):
                    obj.run()


"""Task 6: Extend the Region"""
import rasterio
from rasterio.transform import from_origin
from rasterio.merge import merge
from rasterio import plot
import numpy as np

# function to create new raster
# adjusted/based on: Bryce Frank 18-04-18 https://gis.stackexchange.com/questions/279953/numpy-array-to-gtiff-using-rasterio-without-source-raster
def new_raster(min_x, max_y, max_easting, min_northing, out_raster):
    pixel_size = 5
    dif_x_cell = int((max_easting - min_x)/5)
    dif_y_cell = int((max_y - min_northing)/5)
    arr = np.zeros((dif_y_cell, dif_x_cell))  # raster value set to 0 because raster extension should be at sea level
    transform = from_origin(min_x, max_y, pixel_size, pixel_size)
    extension = rasterio.open(out_raster, 'w', driver='GTiff',
                                height=arr.shape[0], width=arr.shape[1],
                                count=1, dtype=str(arr.dtype),
                                crs='+init=epsg:27700',
                                transform=transform)
    extension.write(arr, 1)
    extension.close()

# find extend of new raster
min_easting, min_northing, max_easting, max_northing = 430000, 80000, 465000, 95000 # storing box extent
min_from_edge = 5000
elev_min_easting, elev_min_northing, elev_max_easting, elev_max_northing = min_easting - min_from_edge , min_northing - min_from_edge, max_easting + min_from_edge, max_northing + min_from_edge # coordinates of elevation raster
ext_min_easting, ext_min_northing, ext_max_easting, ext_max_northing = elev_min_easting - min_from_edge, elev_min_northing- min_from_edge, elev_max_easting + min_from_edge, elev_max_northing + min_from_edge # max outer coordinates of extended raster

# create new raster
new_raster(ext_min_easting, elev_min_northing, ext_max_easting, ext_min_northing,'extension_a.tif') # raster a: extent below elevation raster
new_raster(ext_min_easting, elev_max_northing, elev_min_easting, elev_min_northing,'extension_b.tif') # raster b: extent left to elevation raster
new_raster(ext_min_easting, ext_max_northing, ext_max_easting, elev_max_northing, 'extension_c.tif') # raster c: extent above elevation raster
new_raster(elev_max_easting, elev_max_northing, ext_max_easting, elev_min_northing, 'extension_d.tif') # raster d: extent right to elevation raster

# merge new rasters
# source https://rasterio.readthedocs.io/en/latest/api/rasterio.merge.html
extension_a = rasterio.open('extension_a.tif')
extension_b = rasterio.open('extension_b.tif')
extension_c = rasterio.open('extension_c.tif')
extension_d = rasterio.open('extension_d.tif')
elevation = rasterio.open('sz.asc')

extension_raster = [extension_a, extension_b, extension_c, extension_d, elevation] # list of extension rasters

extension_merged, out_trans = merge(extension_raster) # merge extension rasters

rasterio.open('extension_merged.tif', 'w', driver='GTiff',
                                height=extension_merged.shape[0], width=extension_merged.shape[1],
                                count=1, dtype=str(extension_merged.dtype),
                                crs='+init=epsg:27700',
                                transform=out_trans) # write merged extension raster
extension = rasterio.open('extension_merged.tif')
rasterio.plot.show(extension)

# test if user is on the isle of wight
# adjusted from https://automating-gis-processes.github.io/CSC18/lessons/L4/point-in-polygon.html

#island = gpd.GeoDataFrame.from_file('isle_of_wight.shp')

#if (island.contains(user_location)).bool():
#    location = 'on land'
#else:
#    location = 'in the sea'
#    print ('you are not on the island')
#    exit()  # stop application if user input not on land


import uvicorn
if __name__ == '__main__':
    uvicorn.run('nn_web:app', host='127.0.0.1', port=33334, log_level="info")

from os import listdir
from typing import Optional
import torch
from pydantic import BaseModel
from fastapi import FastAPI, File,Form, HTTPException, Response, status
import numpy as np
from PIL import Image
import torch
import timm
from torchvision import transforms
from pathlib import Path
from tqdm import tqdm

IMAGE_PATH="./../../../public/images"
device = "cuda" if torch.cuda.is_available() else "cpu"
model = timm.create_model('beit_base_patch16_224_in22k', pretrained=True)
model.head=torch.nn.Identity()
model.eval()
model.to(device)

import sqlite3
import io
conn = sqlite3.connect('NN_features.db')
conn_orig = sqlite3.connect('NN_features_orig.db')

dim=768
import faiss
sub_index = faiss.IndexFlatL2(dim)
index_id_map = faiss.IndexIDMap2(sub_index)
# import hnswlib
# index = hnswlib.Index(space='l2', dim=dim)
# index.init_index(max_elements=1_000_000, ef_construction=200, M=32)

import pickle
pca_w_file = Path("./pca_w.pkl")
pca = None
if pca_w_file.is_file():
    with open(pca_w_file, 'rb') as pickle_file:
        pca = pickle.load(pickle_file)

_transform=transforms.Compose([
                       transforms.ToTensor(),
                       transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])

def transform(im):
    desired_size = 224
    old_size = im.size  # old_size[0] is in (width, height) format
    ratio = float(desired_size)/max(old_size)
    new_size = tuple([int(x*ratio) for x in old_size])
    im = im.resize(new_size, Image.ANTIALIAS)
    new_im = Image.new("RGB", (desired_size, desired_size))
    new_im.paste(im, ((desired_size-new_size[0])//2, (desired_size-new_size[1])//2))
    return _transform(new_im)

def get_all_data_iterator(arraysize=10000):
    cursor = conn.cursor()
    query = '''
        SELECT id, nn_features
        FROM nn_table
        '''
    cursor.execute(query)
    while True:
        results = cursor.fetchmany(arraysize)
        if not results:
            break
        yield results

def init_index():
    for result in tqdm(get_all_data_iterator(10000)):
        ids = [x[0] for x in result]
        features = [convert_array(x[1]) for x in result]
        # features = Parallel(n_jobs=1)(delayed(convert_array)(feature) for feature in features)
        ids=np.int64(ids)
        features=np.array(features,dtype=np.float32)
        index_id_map.add_with_ids(features,ids)
    print("Index is ready")

def read_img_buffer(image_data):
    img = Image.open(io.BytesIO(image_data))
    img=img.convert('L').convert('RGB') #GREYSCALE
    # if img.mode != 'RGB':
    #     img = img.convert('RGB')
    return img

def get_features(image_buffer):
    image=read_img_buffer(image_buffer)
    image = transform(image).unsqueeze(0).to(device)
    with torch.no_grad():
        feature_vector = model(image).cpu().numpy()[0]
    feature_vector/=np.linalg.norm(feature_vector)
    return feature_vector

def create_table():
    cursor = conn.cursor()
    query = '''
	    CREATE TABLE IF NOT EXISTS nn_table(
	    	id INTEGER NOT NULL UNIQUE PRIMARY KEY,
	    	nn_features BLOB NOT NULL
	    )
	'''
    cursor.execute(query)
    conn.commit()

def check_if_exists_by_id(id):
    cursor = conn.cursor()
    query = '''SELECT EXISTS(SELECT 1 FROM nn_table WHERE id=(?))'''
    cursor.execute(query,(id,))
    all_rows = cursor.fetchone()
    return all_rows[0] == 1

def delete_descriptor_by_id(id):
    cursor = conn.cursor()
    query = '''DELETE FROM nn_table WHERE id=(?)'''
    cursor.execute(query,(id,))
    conn.commit()

def get_all_ids():
    cursor = conn.cursor()
    query = '''SELECT id FROM nn_table'''
    cursor.execute(query)
    all_rows = cursor.fetchall()
    return list(map(lambda el:el[0],all_rows))

def convert_array(text):
    out = io.BytesIO(text)
    out.seek(0)
    return np.load(out)

def adapt_array(arr):
    out = io.BytesIO()
    np.save(out, arr)
    out.seek(0)
    return sqlite3.Binary(out.read())

def add_descriptor(id,nn_features):
    cursor = conn.cursor()
    query = '''INSERT INTO nn_table(id, nn_features) VALUES (?,?)'''
    cursor.execute(query,(id,nn_features))
    conn.commit()

def add_descriptor_orig(id,nn_features):
    cursor = conn_orig.cursor()
    query = '''INSERT INTO nn_table(id, nn_features) VALUES (?,?)'''
    cursor.execute(query,(id,nn_features))
    conn_orig.commit()

def sync_db():
    ids_in_db=set(get_all_ids())
    file_names=listdir(IMAGE_PATH)
    for file_name in file_names:
        file_id=int(file_name[:file_name.index('.')])
        if file_id in ids_in_db:
            ids_in_db.remove(file_id)
    for id in ids_in_db:
        delete_descriptor_by_id(id)   #Fix this
        print(f"deleting {id}")
    print("db synced")

def get_aqe_vector(feature_vector):
    top_n_query=5
    _, I = index_id_map.search(feature_vector, top_n_query)
    alpha=1
    top_features=[]
    for i in range(top_n_query):
        top_features.append(index_id_map.reconstruct(int(list(I[0])[i])).flatten())
    new_feature=[]
    for i in range(dim):
        _sum=0
        for j in range(top_n_query):
            _sum+=top_features[j][i] * np.dot(feature_vector, top_features[j].T)**alpha
        new_feature.append(_sum)

    new_feature=np.array(new_feature)
    new_feature/=np.linalg.norm(new_feature)
    new_feature=new_feature.astype(np.float32).reshape(1,-1)
    return new_feature

def nn_find_similar(feature_vector, k, distance_threshold):
    new_feature_vector=get_aqe_vector(feature_vector)
    print(new_feature_vector)
    if k is not None:
        D, I = index_id_map.search(new_feature_vector, k)
        D = D.flatten()
        I = I.flatten()
    elif distance_threshold is not None:
        _, D, I = index_id_map.range_search(new_feature_vector, distance_threshold)

    res=[{"image_id":int(I[i]),"distance":float(D[i])} for i in range(len(D))]
    res = sorted(res, key=lambda x: x["distance"])
    return res

app = FastAPI()
@app.get("/")
async def read_root():
    return {"Hello": "World"}

@app.post("/calculate_nn_features")
async def calculate_NN_features_handler(image: bytes = File(...),image_id: str = Form(...)):
    try:
        image_id=int(image_id)
        features=get_features(image)
        add_descriptor_orig(image_id,adapt_array(features.astype(np.float32)))
        if pca:
            features=pca.transform(features.reshape(1,-1))[0]
            features/=np.linalg.norm(features)
        features=features.astype(np.float32)
        add_descriptor(image_id,adapt_array(features))
        # index.add_items(features,[image_id])
        index_id_map.add_with_ids(features.reshape(1,-1), np.int64([image_id]))
        return Response(status_code=status.HTTP_200_OK)
    except:
        raise HTTPException(status_code=500, detail="Can't calculate nn features")

class Item_image_id(BaseModel):
    image_id: int

@app.post("/delete_nn_features")
async def delete_nn_features_handler(item:Item_image_id):
    delete_descriptor_by_id(item.image_id)
    res = index_id_map.remove_ids(np.int64([item.image_id]))
    if res == 0: #nothing to delete
        print(f"err: no image with id {item.image_id}")
    return Response(status_code=status.HTTP_200_OK)
    # try:
    #     # index.mark_deleted(item.image_id)
    #     index_id_map.remove_ids(np.int64([item.image_id]))
    # except RuntimeError:
    #     print(f"err: no image with id {item.image_id}")
    # return Response(status_code=status.HTTP_200_OK)

class Item_image_id_k(BaseModel):
    image_id: int
    k: Optional[str] = None
    distance_threshold: Optional[str] = None

@app.post("/nn_get_similar_images_by_id")
async def get_similar_images_by_id_handler(item: Item_image_id_k):
    try:
        k=item.k
        distance_threshold=item.distance_threshold
        if item.k:
            k = int(k)
        if item.distance_threshold:
            distance_threshold = float(distance_threshold)
        if (k is None) == (distance_threshold is None):
            raise HTTPException(status_code=500, detail="both k and distance_threshold present")

        target_features = index_id_map.reconstruct(item.image_id).reshape(1,-1)
        print(target_features)
        #target_features = index.get_items([item.image_id])
        results = nn_find_similar(target_features,k, distance_threshold)
        return results
    except RuntimeError:
        raise HTTPException(status_code=500, detail="Can't get get_similar_images_by_id_handler")

@app.post("/nn_get_similar_images_by_image_buffer")
async def nn_get_similar_images_by_image_buffer_handler(image: bytes = File(...), k: Optional[str] = Form(None), distance_threshold: Optional[str] = Form(None)):
    try:
        if k:
            k = int(k)
        if distance_threshold:
            distance_threshold = float(distance_threshold)
        if (k is None) == (distance_threshold is None):
            raise HTTPException(status_code=500, detail="both k and distance_threshold present")

        target_features=get_features(image)
        if pca:
            target_features=pca.transform(target_features.reshape(1,-1))[0]
            target_features/=np.linalg.norm(target_features)
        target_features=target_features.astype(np.float32).reshape(1,-1)
        results = nn_find_similar(target_features, k, distance_threshold)
        return results
    except RuntimeError:
        raise HTTPException(status_code=500, detail="Can't get nn_get_similar_images_by_image_buffer_handler")

print(__name__)
if __name__ == 'nn_web':
    create_table()
    # sync_db()
    init_index()