import cmd
import os
import sys
from copy import copy
from functools import reduce
from colorama import init, Fore

from ClassCollection import ClassCollection
from Command import Command
from Momento import Momento
from ActionStack import ActionStack
import Interface

import readline

class MontyREPL(cmd.Cmd):
    def __init__(self, completekey='tab', stdin=None, stdout=None):
        self.default_delims = readline.get_completer_delims()
        super().__init__(completekey, stdin, stdout)
        init(autoreset=True)
        self.model = ClassCollection()
        self.saveStates = ActionStack(Momento(Command("",""), self.model))
        self.intro = (Fore.GREEN + '\nMontyPython UML Editor\n' + '='*80 +
            '\nType help [verbose|<command-name>] or ? for help on commands.\n' + Fore.RESET)

        self.prompt = Fore.CYAN + 'monty> ' + Fore.RESET
        dh = 'Type help [verbose|<command_name>] for descriptions'
        self.doc_header = dh

        # Read in Help.txt and extract every help description for every command.
        self.cmd_desc = {}
        with open('Help.txt') as help_doc:
            begin = 0
            lines = help_doc.readlines()
            while lines[begin][0] != '-':
                begin += 1

            lines = [lines[l].strip() for l in range(begin,len(lines))]
            l = 0
            while l < len(lines):
                if len(lines[l]) > 0 and lines[l][0] != '-':
                    command = lines[l]
                    desc = f'{command}\n'
                    l += 1
                    while l < len(lines) and len(lines[l]) > 0:
                        desc += f'    {lines[l]}\n'
                        l += 1
                    self.cmd_desc[command.split()[0]] = desc.rstrip()
                l += 1
        self.root_dir = copy(os.curdir)

    # Interface
    #
    # These are all standalone functions that don't affect the ClassCollection,
    # therefore there is no Command object created for any of these.
    def do_exit(self, args):
        Interface.exit()

    def do_help(self, args):
        if args == 'verbose':
            Interface.help()
        else:
            cmd.Cmd.do_help(self, args)

    def do_clear(self, args):
        os.system('cls' if os.name == 'nt' else 'clear')

    def do_list_relationships(self, args):
        for rel in self.model.relationshipDict:
            r = self.model.relationshipDict[rel]
            print(f'{r.src} --> {r.dst} ({r.typ})')

    def do_list_classes(self, args):
        for c in self.model.classDict:
            self.do_list_class(c)

    # TODO: Edit the seperation lines to match the length of the longest
    #       line of text, with some pre-defined minimum length.
    def do_list_class(self, args):
        c = args.split()[0]
        name = f' {c}'
        fields = []
        methods = []
        for field in self.model.classDict[c].fieldDict:
            fields.append(f' {self.model.classDict[c].fieldDict[field]}')
        for method in self.model.classDict[c].methodDict:
            for m in self.model.classDict[c].methodDict[method]:
                methods.append(f' {m}')

        longest = max(15, len(max([name] + fields + methods, key=lambda s:len(s))))+1
        print_line = lambda l: print(f'|{l}{" "*max(1, longest-len(l))}|')
        print_sep = lambda: print(f'+{"-"*longest}+')

        print_sep()
        print_line(name)
        print_sep()

        if len(fields) > 0:
            for f in fields:
                print_line(f)
            print_sep()
        if len(methods) > 0:
            for m in methods:
                print_line(m)
            print_sep()

    def do_save(self, args):
        if len(args) > 0:
            Interface.saveFile(self.model, os.path.expanduser(args))
        else:
            self.help_save()

    def do_load(self, args):
        if len(args) > 0:
            Interface.loadFile(self.model, os.path.expanduser(args))
            self.saveStates.reset(Momento(Command("",""), self.model))
        else:
            self.help_load()

    def do_undo(self, args):
        self.saveStates.undoPop()
        self.model = self.saveStates.currentObj.state

    def do_redo(self, args):
        self.saveStates.redoPop()
        self.model = self.saveStates.currentObj.state

    # Classes
    def do_add_class(self, args):
        self.execute(self.model.addClass, args)

    def do_delete_class(self, args):
        self.execute(self.model.deleteClass, args)

    def do_rename_class(self, args):
        self.execute(self.model.renameClass, args)

    # Relationships
    def do_add_relationship(self, args):
        self.execute(self.model.addRelationship, args)

    def do_delete_relationship(self, args):
        self.execute(self.model.deleteRelationship, args)

    def do_rename_relationship(self, args):
        self.execute(self.model.renameRelationship, args)

    # Methods
    def do_add_method(self, args):
        args = args.split()
        if len(args) < 3:
            raise RuntimeError('Please include return type')
        params = []
        for i in range(3, len(args), 2):
            try:
                params.append([args[i], args[i + 1]])
            except IndexError:
                raise IndexError('Odd number of arguments for method parameters. '+
                'Method not added')
        args = args[:3] + [params]
        self.execute(self.model.addMethod, args)

    def do_delete_method(self, args):
        args = args.split()
        params = self.handle_overloaded_methods(args[0], args[1])
        self.execute(self.model.deleteMethod, [args[0], args[1], params])

    def do_rename_method(self, args):
        args = args.split()
        params = self.handle_overloaded_methods(args[0], args[1])
        self.execute(self.model.renameMethod, [args[0], args[1], params, args[2]])

    # Fields
    def do_add_field(self, args):
        self.execute(self.model.addField, args)

    def do_delete_field(self, args):
        self.execute(self.model.deleteField, args)

    def do_rename_field(self, args):
        self.execute(self.model.renameField, args)

    # Parameters
    def do_add_parameter(self, args):
        args = args.split()
        params = self.handle_overloaded_methods(args[0], args[1])
        self.execute(self.model.addParameter, [args[0], args[1], params, args[2], args[3]])

    def do_delete_parameter(self, args):
        args = args.split()
        params = self.handle_overloaded_methods(args[0], args[1])
        self.execute(self.model.removeParameter, [args[0], args[1], params, args[2]])

    def do_change_parameters(self, args):
        args = args.split()
        old_params = self.handle_overloaded_methods(args[0], args[1])
        new_params = []
        for i in range(2, len(args), 2):
            new_params.append([args[i], args[i + 1]])
        self.execute(self.model.changeAllParameters, [args[0], args[1], old_params, new_params])

    # help_cmd functions
    def do_verbose(self, args):
        self.do_help('verbose')
    def help_exit(self):
        self.print_cmd_help('exit')
    def help_help(self):
        self.print_cmd_help('help')
    def help_list_relationships(self):
        self.print_cmd_help('list_relationships')
    def help_list_classes(self):
        self.print_cmd_help('list_classes')
    def help_list_class(self):
        self.print_cmd_help('list_class')
    def help_clear(self):
        self.print_cmd_help('clear')

    def help_save(self):
        self.print_cmd_help('save')
    def help_load(self):
        self.print_cmd_help('load')
    def help_undo(self):
        self.print_cmd_help('undo')
    def help_redo(self):
        self.print_cmd_help('redo')

    def help_add_class(self):
        self.print_cmd_help('add_class')
    def help_delete_class(self):
        self.print_cmd_help('delete_class')
    def help_rename_class(self):
        self.print_cmd_help('rename_class')

    def help_add_relationship(self):
        self.print_cmd_help('add_relationship')
    def help_delete_relationship(self):
        self.print_cmd_help('delete_relationship')
    def help_rename_relationship(self):
        self.print_cmd_help('rename_relationship')

    def help_add_method(self):
        self.print_cmd_help('add_method')
    def help_delete_method(self):
        self.print_cmd_help('delete_method')
    def help_rename_method(self):
        self.print_cmd_help('rename_method')

    def help_add_field(self):
        self.print_cmd_help('add_field')
    def help_delete_field(self):
        self.print_cmd_help('delete_field')
    def help_rename_field(self):
        self.print_cmd_help('rename_field')

    def help_add_parameter(self):
        self.print_cmd_help('add_parameters')
    def help_delete_parameter(self):
        self.print_cmd_help('delete_parameters')
    def help_change_parameters(self):
        self.print_cmd_help('change_parameters')

    def help_verbose(self):
        print('verbose\nAlias for help verbose')

    # Custom tab completion functions
    # To be able to autocomplete a new function
    #   1. Create a dict of int -> list of words that can be autocompleted
    #       based on the index of the argument (starting at 1)
    #   2. Pass all but begidx and endidx to self.arg_complete
    #
    # arg_completions is a dictionary of lists based on the index of the
    # current argument (including the command name, so starting at 1,
    # similar to how sys.stdin works)
    def complete_list_class(self, text, line, begidx, endidx):
        arg_completions = {
            1: list(self.model.classDict.keys())
        }
        return self.arg_complete(text, line, arg_completions)

    def complete_list_relationships(self, text, line, begidx, endidx):
        arg_completions = {
            1: list(self.model.classDict.keys()),
            2: list(self.model.classDict.keys())
        }
        return self.arg_complete(text, line, arg_completions)

    def complete_delete_class(self, text, line, begidx, endidx):
        arg_completions = {
            1: list(self.model.classDict.keys())
        }
        return self.arg_complete(text, line, arg_completions)

    def complete_rename_class(self, text, line, begidx, endidx):
        arg_completions = {
            1: list(self.model.classDict.keys()),
            2: list(self.model.classDict.keys())
        }
        return self.arg_complete(text, line, arg_completions)

    def complete_add_relationship(self, text, line, begidx, endidx):
        arg_completions = {
            1: list(self.model.classDict.keys()),
            2: list(self.model.classDict.keys()),
            3: ['aggregation', 'composition', 'inheritance', 'realization']
        }
        return self.arg_complete(text, line, arg_completions)

    def complete_delete_relationship(self, text, line, begidx, endidx):
        arg_completions = {
            1: list(self.model.classDict.keys()),
            2: list(self.model.classDict.keys())
        }
        return self.arg_complete(text, line, arg_completions)

    def complete_rename_relationship(self, text, line, begidx, endidx):
        arg_completions = {
            1: list(self.model.classDict.keys()),
            2: list(self.model.classDict.keys()),
            3: ['aggregation', 'composition', 'inheritance', 'realization']
        }
        return self.arg_complete(text, line, arg_completions)

    def complete_add_method(self, text, line, begidx, endidx):
        arg_completions = {
            1: list(self.model.classDict.keys())
        }
        return self.arg_complete(text, line, arg_completions)

    def complete_delete_method(self, text, line, begidx, endidx):
        curr_args = line.split()
        arg_completions = {
            1: list(self.model.classDict.keys()),
            2: ([] if len(curr_args) < 2 or curr_args[1] not in self.model.classDict
                    else self.model.getAllMethods(curr_args[1]))
        }
        return self.arg_complete(text, line, arg_completions)

    def complete_rename_method(self, text, line, begidx, endidx):
        curr_args = line.split()
        arg_completions = {
            1: list(self.model.classDict.keys()),
            2: ([] if len(curr_args) < 2 or curr_args[1] not in self.model.classDict
                    else self.model.getAllMethods(curr_args[1]))
        }
        return self.arg_complete(text, line, arg_completions)

    def complete_add_field(self, text, line, begidx, endidx):
        arg_completions = {
            1: list(self.model.classDict.keys())
        }
        return self.arg_complete(text, line, arg_completions)

    def complete_delete_field(self, text, line, begidx, endidx):
        curr_args = line.split()
        arg_completions = {
            1: list(self.model.classDict.keys()),
            2: ([] if len(curr_args) < 2 or curr_args[1] not in self.model.classDict
                    else self.model.getFields(curr_args[1]))
        }
        return self.arg_complete(text, line, arg_completions)

    def complete_rename_field(self, text, line, begidx, endidx):
        curr_args = line.split()
        arg_completions = {
            1: list(self.model.classDict.keys()),
            2: ([] if len(curr_args) < 2 or curr_args[1] not in self.model.classDict
                    else self.model.getFields(curr_args[1]))
        }
        return self.arg_complete(text, line, arg_completions)

    def complete_add_parameter(self, text, line, begidx, endidx):
        curr_args = line.split()
        arg_completions = {
            1: list(self.model.classDict.keys()),
            2: ([] if len(curr_args) < 2 or curr_args[1] not in self.model.classDict
                    else self.model.getFields(curr_args[1]))
        }
        return self.arg_complete(text, line, arg_completions)

    def complete_delete_parameter(self, text, line, begidx, endidx):
        curr_args = line.split()

        # Only get method parameters if both the method and class exist.
        method_params = []
        if (len(curr_args) >= 3 and curr_args[1] in self.model.classDict
            and curr_args[2] in self.model.getAllMethods(curr_args[1])):
            for m in self.model.getMethodsByName(curr_args[1], curr_args[2]):
                param_names = [p[1] for p in m.parameters]
                for p in param_names:
                    if p not in method_params:
                        method_params.append(p)

        arg_completions = {
            1: list(self.model.classDict.keys()),
            2: ([] if len(curr_args) < 2 or curr_args[1] not in self.model.classDict
                    else self.model.getAllMethods(curr_args[1])),
            3: method_params
        }
        return self.arg_complete(text, line, arg_completions)

    def complete_change_parameters(self, text, line, begidx, endidx):
        curr_args = line.split()
        arg_completions = {
            1: list(self.model.classDict.keys()),
            2: ([] if len(curr_args) < 2 or curr_args[1] not in self.model.classDict
                    else self.model.getFields(curr_args[1]))
        }
        return self.arg_complete(text, line, arg_completions)

    def complete_save(self, text, line, begidx, endidx):
        try:
            readline.set_completer_delims('~\\/')
            curr_args = line.split()

            if len(curr_args) > 2:
                prev = curr_args[1]
                for rest in curr_args[2:]:
                    curr_args[1] += f' {rest}'
                    prev = rest

            path = ''
            if len(curr_args) > 1:
                if curr_args[1][0] == '~':
                    path = os.path.join(os.path.expanduser(curr_args[1]))
                else:
                    path = os.path.join('.', curr_args[1])
            else:
                path = os.path.join('.', '')

            files_and_dirs = []
            prediction = ''
            if len(curr_args) != 1:
                _, prediction = os.path.split(path)
            if not os.path.exists(path):
                path = os.path.dirname(path)

            if os.path.isdir(path):
                for f in os.listdir(path):
                    if f.startswith(prediction):
                        if os.path.isdir(os.path.join(path, f)):
                            files_and_dirs.append(os.path.join(f, ''))
                        else:
                            files_and_dirs.append(f)
            return files_and_dirs
        except Exception as e:
            return ['**Error, cannot save here', 'please try another directory**']

    def complete_load(self, text, line, begidx, endidx):
        try:
            readline.set_completer_delims('~\\/')
            curr_args = line.split()

            if len(curr_args) > 2:
                prev = curr_args[1]
                for rest in curr_args[2:]:
                    curr_args[1] += f' {rest}'
                    prev = rest

            path = ''
            if len(curr_args) > 1:
                if curr_args[1][0] == '~':
                    path = os.path.join(os.path.expanduser(curr_args[1]))
                else:
                    path = os.path.join('.', curr_args[1])
            else:
                path = os.path.join('.', '')

            files_and_dirs = []
            prediction = ''
            if len(curr_args) != 1:
                _, prediction = os.path.split(path)
            if not os.path.exists(path):
                path = os.path.dirname(path)

            if os.path.isdir(path):
                for f in os.listdir(path):
                    if f.startswith(prediction):
                        if os.path.isdir(os.path.join(path, f)):
                            files_and_dirs.append(os.path.join(f, ''))
                        else:
                            files_and_dirs.append(f)
            return files_and_dirs
        except Exception as e:
            return ['**Error, cannot save here', 'please try another directory**']

    # Helpers
    def print_cmd_help(self, command):
        print(self.cmd_desc[command])

    def postcmd(self, stop, line):
        pass

    def emptyline(self):
        pass

    def execute(self, function, args):
        if isinstance(args, str):
            args = args.split()
        command = Command(function, *args)
        command.execute()

        #Prior version had a list of commands to check against
        #Which A) didn't work as intended and B) would be redundant
        #as excluded functions already weren't added.
        self.saveStates.add(Momento(command, self.model))

    def onecmd(self, line):
        readline.set_completer_delims(self.default_delims)
        try:
            return super().onecmd(line)
        except Exception as e:
            if '()' not in str(e):
                print(e)
            else:
                print('Wrong number of arguments')
            self.print_cmd_help(line.split()[0])
            return False

    def handle_overloaded_methods(self, className, methodName):
        methods = self.model.getMethodsByName(className, methodName)
        for method, idx in zip(methods, range(1, len(methods) + 1)):
            print(f'{idx}. {method}')
        num = int(input(Fore.CYAN + 'Method number: ')) - 1
        return methods[num].parameters

    # arg_complete -> list(str)
    #
    # text: word being matched against
    # line: current line of input
    # arg_completions: dictionary of lists that contain valid words to
    #   autocomplete against based on the index of the current arg
    def arg_complete(self, text, line, arg_completions):
        idx = len(line.split())
        if text != '':
            idx -= 1
        if idx > len(arg_completions):
            return []
        return [arg for arg in arg_completions[idx] if arg.startswith(text)]

if __name__ == '__main__':
    MontyREPL().cmdloop()


#!/home/einfochips/Desktop/YOLO/evn/bin/python3

import torch
import numpy as np
import cv2
import rospy
from sensor_msgs.msg import Image
from geometry_msgs.msg import Point
from std_msgs.msg import Int16
from cv_bridge import CvBridge,CvBridgeError
import time
import math
from tesingm.srv import servicetest
from tesingm.srv import servicetestRequest
from tesingm.srv import servicetestResponse
import threading

model = torch.hub.load('ultralytics/yolov5','custom',path='/home/einfochips/Desktop/YOLO/yolov5/runs/train/exp4/weights/last.pt')
# model.conf = 0.65
model.classes = (0,1,2,3,5,6,7)
a = model.classes
steps_name = {
   0: "ScreenOff",
   1: "ScreenOn",
   2: "Zero",
   3: "Settings",
   5: "Bluetooth",
   6: "BtnOff",
   7: "BtnOn",
}

steps_count = (0,1,2,3,5,6,7)
steps = 0
nn_new_x= None
new_y = None
old_coord = [0.0, -163.0]
new_steps = 0
c_x = None
c_y = None

def camera_to_world(cam_mtx, r, t, img_points):
    inv_k = np.asmatrix(cam_mtx).I
    r_mat = np.zeros((3, 3), dtype=np.float64)
    cv2.Rodrigues(r, r_mat)
    # invR * T
    inv_r = np.asmatrix(r_mat).I  # 3*3
    transPlaneToCam = np.dot(inv_r, np.asmatrix(t))  # 3*3 dot 3*1 = 3*1
    world_pt = []
    coords = np.zeros((3, 1), dtype=np.float64)
    for img_pt in img_points:
        coords[0][0] = img_pt[0][0]
        coords[1][0] = img_pt[0][1]
        coords[2][0] = 1.0
        worldPtCam = np.dot(inv_k, coords)  # 3*3 dot 3*1 = 3*1
        # [x,y,1] * invR
        worldPtPlane = np.dot(inv_r, worldPtCam)  # 3*3 dot 3*1 = 3*1
        # zc
        scale = transPlaneToCam[2][0] / worldPtPlane[2][0]
        # zc * [x,y,1] * invR
        scale_worldPtPlane = np.multiply(scale, worldPtPlane)
        # [X,Y,Z]=zc*[x,y,1]*invR - invR*T213596990823
        worldPtPlaneReproject = np.asmatrix(scale_worldPtPlane) - np.asmatrix(transPlaneToCam)  # 3*1 dot 1*3 = 3*3
        pt = np.zeros((3, 1), dtype=np.float64)
        pt[0][0] = worldPtPlaneReproject[0][0]
        pt[1][0] = worldPtPlaneReproject[1][0]
        pt[2][0] = 0
        world_pt.append(pt.T.tolist())
    return world_pt

def detect(image):

    global steps_count, steps, nn_new_x, new_y, results

    bridge = CvBridge()
    open=bridge.imgmsg_to_cv2(image)
    col=cv2.cvtColor(open,cv2.COLOR_BGR2RGB)
    results = model(col)

    cv2.imshow('YOLO',np.squeeze(results.render()))
    cv2.waitKey(1)
    # a = torch.tensor(results.xyxy[0])[0][0].item()
    # print(a[0][0].item())
    # print(int(torch.tensor(results.xyxy[0])[0][5].item()))

    try:
        if int(torch.tensor(results.xyxy[0])[0][5].item()) == steps_count[steps] and len(torch.tensor(results.xyxy[0])) == 1:
            if a[steps] == steps_count[steps]:
                # cv2.circle(img=col,center=(((int(torch.tensor(results.xyxy[0])[0][0].item())+int(torch.tensor(results.xyxy[0])[0][2].item()))//2),((int(torch.tensor(results.xyxy[0])[0][1].item())+int(torch.tensor(results.xyxy[0])[0][3].item()))//2)),radius=3,color=(0,255,0),thickness=-1)
                #cv2.imshow('YOLO',np.squeeze(results.render()))
                # rate = rospy.Rate(0.5)
                c_x = ((int(torch.tensor(results.xyxy[0])[0][0].item())+int(torch.tensor(results.xyxy[0])[0][2].item()))//2)
                c_y = ((int(torch.tensor(results.xyxy[0])[0][1].item())+int(torch.tensor(results.xyxy[0])[0][3].item()))//2)
                # print("center points of object == :", c_x,c_y)
                #zp = int(torch.tensor(results.xyxy[0])[0][5].item())
                cur_x, cur_y, cur_z = 0, -163,212
                K=np.array([[434.2196960449219, 0.0, 266.6663412990292],[0.0, 491.9661865234375,238.71625361069528],[0.0, 0.0, 1.0]],dtype=np.float64).reshape(3,3)
                R=np.array([[-0.050112674474709636], [-3.0323416167631896],[0.012589481083534122]],dtype=np.float64).reshape(3,1)
                T=np.array([[16.512429040858382], [51.675548988183714], [202.46864073880207]],dtype=np.float64).reshape(3,1)

                x, y, _ = camera_to_world(K,R,T, np.array([c_x, c_y]).reshape((1, 1, 2)))[0][0]

                t = math.sqrt(x * x + y * y + 120 * 120) / 120

                new_x, new_y = cur_x + x, cur_y + y + 15
                nn_new_x = new_x - 25
                # print("continuous xandy for:",steps_count[steps]  ,nn_new_x, new_y)
                    #jetmax.set_position((nn_new_x, new_y, 80), t)
                    #rospy.sleep(t)
                    #jetmax.set_position((new_x , new_y, 70), 0.3)
                    #rospy.sleep(t + 0.6)

            else:
                pass
        elif int(torch.tensor(results.xyxy[0])[0][5].item()) == steps_count[steps] and len(torch.tensor(results.xyxy[0])) > 1:
                c_x = ((int(torch.tensor(results.xyxy[0])[0][0].item())+int(torch.tensor(results.xyxy[0])[0][2].item()))//2)
                c_y = ((int(torch.tensor(results.xyxy[0])[0][1].item())+int(torch.tensor(results.xyxy[0])[0][3].item()))//2)
                # print("center points of object 0>1 :", c_x,c_y)
                #zp = int(torch.tensor(results.xyxy[0])[0][5].item())
                cur_x, cur_y, cur_z = 0, -163,212
                K=np.array([[434.2196960449219, 0.0, 266.6663412990292],[0.0, 491.9661865234375,238.71625361069528],[0.0, 0.0, 1.0]],dtype=np.float64).reshape(3,3)
                R=np.array([[-0.050112674474709636], [-3.0323416167631896],[0.012589481083534122]],dtype=np.float64).reshape(3,1)
                T=np.array([[16.512429040858382], [51.675548988183714], [202.46864073880207]],dtype=np.float64).reshape(3,1)

                x, y, _ = camera_to_world(K,R,T, np.array([c_x, c_y]).reshape((1, 1, 2)))[0][0]

                t = math.sqrt(x * x + y * y + 120 * 120) / 120

                new_x, new_y = cur_x + x, cur_y + y + 15
                nn_new_x = new_x - 25
                # print("continuous xandy for:",steps_count[steps]  ,nn_new_x, new_y)

        elif int(torch.tensor(results.xyxy[0])[1][5].item()) == steps_count[steps] and len(torch.tensor(results.xyxy[0])) > 1:
                c_x = ((int(torch.tensor(results.xyxy[0])[1][0].item())+int(torch.tensor(results.xyxy[0])[1][2].item()))//2)
                c_y = ((int(torch.tensor(results.xyxy[0])[1][1].item())+int(torch.tensor(results.xyxy[0])[1][3].item()))//2)
                # print("center points of object 1>1 :", c_x,c_y)
                #zp = int(torch.tensor(results.xyxy[0])[0][5].item())
                cur_x, cur_y, cur_z = 0, -163,212
                K=np.array([[434.2196960449219, 0.0, 266.6663412990292],[0.0, 491.9661865234375,238.71625361069528],[0.0, 0.0, 1.0]],dtype=np.float64).reshape(3,3)
                R=np.array([[-0.050112674474709636], [-3.0323416167631896],[0.012589481083534122]],dtype=np.float64).reshape(3,1)
                T=np.array([[16.512429040858382], [51.675548988183714], [202.46864073880207]],dtype=np.float64).reshape(3,1)

                x, y, _ = camera_to_world(K,R,T, np.array([c_x, c_y]).reshape((1, 1, 2)))[0][0]

                t = math.sqrt(x * x + y * y + 120 * 120) / 120

                new_x, new_y = cur_x + x, cur_y + y + 15
                nn_new_x = new_x - 25
                # print("continuous xandy for:",steps_count[steps]  ,nn_new_x, new_y)

        else:
            # nn_new_x= None
            # new_y = None
            # c_x = None
            # c_y = None
            pass
    except IndexError:
            cv2.imshow('YOLO',col)
            cv2.waitKey(1)
    #cv2.imshow('YOLO',np.squeeze(results.render()))
    cv2.imshow('YOLO',col)

    cv2.waitKey(1)

def requesting(msg):
    global old_coord, new_steps, results

    # if nn_new_x == old_coord[0] and new_y == old_coord[1] and new_steps == steps and len(torch.tensor(results.xyxy[0])) >= 1:
    #     print("old coord:", old_coord)
    #     time.sleep(0.5)
    #     client = client_obj((str(nn_new_x) +" "+ str(new_y)))
    #     return client.reply
    # else:
    #     new_steps = steps
    #     old_coord = [nn_new_x,new_y]
    #     new_steps = steps
    #     print("changing coord:", old_coord)

    try:
        time.sleep(0.5)
        print(str(nn_new_x) +" "+ str(new_y))
        client = client_obj((str(nn_new_x) +" "+ str(new_y)))
        return client.reply
    except:
        print("Service call failed")

def main():

    cam_sub = rospy.Subscriber("/usb_cam/image_rect_color",Image,detect)
    try:
        rospy.spin()
    except KeyboardInterrupt:
        cv2.destroyAllWindows()

if __name__=="__main__":
    rospy.init_node("opencv")
    client_obj = rospy.ServiceProxy('test_service', servicetest)
    lock = threading.RLock()
    t1 = threading.Thread(target = main)
    t1.start()
    #print("moving")
    #time.sleep(2)
    while True:
        time.sleep(3)
        # print("Global Steps:-", steps)
        # print("global x&y:",nn_new_x, new_y)
        # print("midpoints x&y:",c_x,c_y)
        rospy.wait_for_service('test_service')
        # if nn_new_x == None:
        #     print("pass")
        # else:
        #     #time.sleep(1)
            # print("latest coord:", nn_new_x, new_y)

        with lock:
            c = str(nn_new_x)+ str(new_y)
            print("c",c)
            s = requesting(c)
            # print("Reply:",s)
            if s == "Done":
                try:
                    while True:
                        if int(torch.tensor(results.xyxy[0])[0][5].item()) == steps_count[steps]:
                            if steps>6:
                                break
                            steps += 1
                            # print("Global Steps after incrementing:-", steps)
                            # print("next to detect:" , steps_name[steps])
                            break
                        else:
                            # print("Global Steps remained same:-", steps)
                            break
                except:
                    pass
            elif s == "None":
                pass


import requests
import time

class APIResource:
    """Class that is used to represent something that will be returned from the api

    Attributes
    ----------
    api : LordOfTheRings
        The instance of the LordOfTheRings class that this resource will use to make requests
    endpoint : str
        The endpoint associated with this API resource
    id : str, optional
        The id of the resource you wish to get from the api.

    """
    def __init__(self, api, endpoint, id=None):
        """Initialize an APIResource instance.

        Parameters
        ----------
        api : LordOfTheRings
            The instance of the LordOfTheRings class that this resource will use to make requests.
        endpoint : str
            The endpoint associated with this API resource.
        id : str, optional
            The unique identifier of this resource (default is None).

        """
        self.api = api
        self.endpoint = endpoint
        self.id = id

    def get_all(self):
        """Fetch all resources of this type

        Returns
        -------
        ChainableMethods
            An instance of ChainableMethods to allow for option chaining like filtering, sorting, or pagination.

        """
        return ChainableMethods(self)

    def get(self):
        """Fetch a single resource by the id of the resource

        Returns
        -------
        dict
            The JSON response with the data from the resource that was queried

        Raises
        ------
        Exception
            If the ID of the resource has not been set.

        """
        if self.id:
            return self.api.request('GET', f'{self.endpoint}/{self.id}')
        else:
            raise Exception('Resource ID not provided')

class ChainableMethods:
    """Class to allow chainable methods on a resource.

    Attributes
    ----------
    resource : APIResource
        The resource that will have method chaining.
    params : dict
        A dictionary of parameters for the API request.

    """
    def __init__(self, resource):
        """Initialize a ChainableMethods instance.

        Parameters
        ----------
        resource : APIResource
            The resource on which the methods should be chained.

        """
        self.resource = resource
        self.params = {}

    def limit(self, limit):
        """Set the limit parameter for the request, limitting the amount of records returned by the api

        Parameters
        ----------
        limit : int
            The maximum number of results to return.

        Returns
        -------
        ChainableMethods
            The current instance to allow method chaining.

        """
        self.params['limit'] = limit
        return self

    def page(self, page):
        """Set the page parameter for the request, specifying which page of results should be returned

        Parameters
        ----------
        page : int
            The page of results to return.

        Returns
        -------
        ChainableMethods
            The current instance to allow method chaining.

        """
        self.params['page'] = page
        return self

    def offset(self, offset):
        """Set the offset parameter for the request.

        Parameters
        ----------
        offset : int
            The number of results to skip before starting to fetch.

        Returns
        -------
        ChainableMethods
            The current instance to allow method chaining.

        """
        self.params['offset'] = offset
        return self

    def filter_by(self, **filters):
        """Set filter parameters for the request.

        Parameters
        ----------
        **filters : dict
            The filters to apply to the request.

        Returns
        -------
        ChainableMethods
            The current instance to allow method chaining.

        """
        for key, value in filters.items():
            if isinstance(value, str):
                # match
                self.params[key] = value
            elif isinstance(value, tuple):
                if value[0] == 'not':
                    # negate match
                    self.params[f"{key}!"] = value[1]
                elif value[0] in ('<', '>'):
                    # less than, greater than or equal to
                    self.params[f"{key}{value[0]}"] = value[1]
            elif isinstance(value, list):
                if all(isinstance(v, str) for v in value):
                    # include
                    self.params[key] = ','.join(value)
                elif all(isinstance(v, tuple) and v[0] == 'not' for v in value):
                    # exclude
                    self.params[f"{key}!"] = ','.join(v[1] for v in value)
        return self

    def sort_by(self, sort_by):
        """Set the sort parameter for the request.

        Parameters
        ----------
        sort_by : str
            The field by which to sort the results. For example, name:asc

        Returns
        -------
        ChainableMethods
            The current instance to allow method chaining.

        """
        self.params['sort'] = sort_by
        return self

    def fetch(self):
        """Make the API request and return the response.

        Returns
        -------
        dict
            The JSON response from the server.

        """
        return self.resource.api.request('GET', self.resource.endpoint, params=self.params)

class Movie(APIResource):
    """A class representing a movie resource.

    Attributes
    ----------
    quotes : Quotable
        A Quotable instance composed with the movie, allowing the user to get quotes for a particular movie
    """
    def __init__(self, api, id=None):
        """Initialize a Movie instance.

        Parameters
        ----------
        api : LordOfTheRings
            The main API instance.
        id : str, optional
            The ID of the movie.
        """
        endpoint = 'movie'
        super().__init__(api, endpoint, id)
        self.quotes = Quotable(api, endpoint, self.id)

class Quote(APIResource):
    """A class representing a quote resource."""
    def __init__(self, api, id=None):
        """Initialize a Quote instance.

        Parameters
        ----------
        api : LordOfTheRings
            The main API instance.
        id : str, optional
            The ID of the quote.
        """
        endpoint = 'quote'
        super().__init__(api, endpoint, id)

class Quotable(APIResource):
    """A class representing a quotable resource, that can have quotes in the form of {resource}/{id}/quote"""
    def __init__(self, api, parent_endpoint, parent_id):
        """Initialize a Quotable instance.

        Parameters
        ----------
        api : LordOfTheRings
            The main API instance.
        parent_endpoint : str
            The endpoint of the parent resource.
        parent_id : str
            The ID of the parent resource.
        """
        super().__init__(api, f'{parent_endpoint}/{parent_id}/quote')

class LordOfTheRings:
    """The client for interfacing with the Lord of The Rings API.

    Attributes
    ----------
    base_url : str
        The base URL for the API.
    auth_token : str
        The authentication token for the API.
    session : requests.Session
        The session for making HTTP requests.
    max_retries : int
        The maximum number of retries when a request fails.
    failure_delay : int
        The delay (in seconds) between retries.
    """
    def __init__(self, auth_token, max_retries=3, failure_delay=1):
        """Initialize a LordOfTheRings client instance.

        Parameters
        ----------
        auth_token : str
            The authentication token for the API.
        max_retries : int, optional
            The maximum number of retries when a request fails.
        failure_delay : int, optional
            The delay (in seconds) between retries.
        """
        self.base_url = 'https://the-one-api.dev/v2/'
        self.auth_token = auth_token
        self.session = requests.Session()
        self.max_retries = max_retries
        self.failure_delay = failure_delay
        if auth_token:
            self.session.headers.update({'Authorization': f'Bearer {auth_token}'})

    def request(self, method, endpoint, **kwargs):
        """Send a request to the API.

        Parameters
        ----------
        method : str
            The HTTP method ('GET', 'POST', etc.).
        endpoint : str
            The API endpoint (excluding the base URL).
        **kwargs
            Any other parameters to pass to requests.Session.request.

        Returns
        -------
        dict
            The JSON response from the API.

        Raises
        ------
        Exception
            If the request fails after max_retries.
        """
        url = f'{self.base_url.rstrip("/")}/{endpoint.lstrip("/")}'
        for _ in range(self.max_retries):
            response = self.session.request(method, url, **kwargs)
            if response.status_code == 429:  # Rate limit error
                print("Rate limit reached. Retrying...")
                time.sleep(self.failure_delay)
                continue
            elif response.status_code >= 500: # Server error
                print("Server error. Retrying...")
                time.sleep(self.failure_delay)
                continue
            elif response.status_code >= 400: # Client error
                raise Exception(f'Request failed with status {response.status_code}')
            else:
                return response.json()
        raise Exception('Max retries exceeded')

    def movies(self, id=None):
        """Get a movie resource.

        Parameters
        ----------
        id : str, optional
            The ID of the movie.

        Returns
        -------
        Movie
            A Movie instance.
        """
        return Movie(self, id)

    def quotes(self, id=None):
        """Get a quote resource.

        Parameters
        ----------
        id : str, optional
            The ID of the quote.

        Returns
        -------
        Quote
            A Quote instance.
        """
        return Quote(self, id)


import tqdm
import networkx as nx
import logging
import os

from .dynamic_graph import DynamicGraph

logger = logging.getLogger(__name__)

GRAPH_NAME = 'email'
USER_RELATION_NAME = 'sent'
ATTR_RELATION_NAME = 'dept'

def get_user_name(id, dept_id):
    return "dept{}:user{}".format(dept_id, id)

def get_dept_name(id):
    return "dept{}".format(id)

def load_users_relationship(graph, file_path, dept_id, entity2dept, t2entity):
    with open(file_path, 'r') as f:
        lines = f.readlines()
        lines = list(map(lambda line: line.rstrip().split(' '), lines))

        # entity_ids = set()
        time_insts = set()
        for u, v, t in lines:
            u_name = get_user_name(u, dept_id)
            v_name = get_user_name(v, dept_id)
            time_insts.add(t)
            dept_name = get_dept_name(dept_id)

            graph.add_relationship_edge(u_name, USER_RELATION_NAME, v_name, t)

            entity2dept[u_name] = dept_name
            entity2dept[v_name] = dept_name

            entities = t2entity.get(t, set())
            entities.add(u_name)
            entities.add(v_name)
            t2entity[t] = entities

            # u_key = "{}_{}".format(u_name, t)
            # v_key = "{}_{}".format(v_name, t)
            # if u_key in entity_ids:
            #     count_before = graph.get_num_edges(t)

            # graph.add_attribute_edge(u_name, ATTR_RELATION_NAME, dept_name, t)

            # if u_key in entity_ids:
            #     count_after = graph.get_num_edges(t)

            #     assert count_before == count_after, "before: {} - after:{}".format(count_before, count_after)

            # if v_key in entity_ids:
            #     count_v_before=graph.get_num_edges(t)

            # graph.add_attribute_edge(v_name, ATTR_RELATION_NAME, dept_name, t)

            # if v_key in entity_ids:
            #     count_v_after = graph.get_num_edges(t)
            #     assert count_v_before == count_v_after, "before: {} - after: {}".format(count_v_before, count_v_after)

            # entity_ids.add(u_key)
            # entity_ids.add(v_key)

def add_attribute_relationships(graph, entity2dept, t2entity):
    time_ints = graph.time_instances

    for t, entities in tqdm.tqdm(t2entity.items()):
        for entity in entities:
            dept_name = entity2dept[entity]
            graph.add_attribute_edge(entity, ATTR_RELATION_NAME, dept_name, t)
    # for entity, dept_name in tqdm.tqdm(entity2dept.items()):
    #     for t in time_ints:
    #         graph.add_attribute_edge(entity, dept_name, dept_name, t)

class EmailTempGraph(DynamicGraph):
    def __init__(self):
        super().__init__()

    @staticmethod
    def from_raw_file(data_dir, args):
        # load attributes
        # load relationships
        graph = EmailTempGraph()
        entity2dept = {}
        t2entity = {}

        user_relationship_path = os.path.join(data_dir, 'email-Eu-core-temporal.txt')
        load_users_relationship(graph, user_relationship_path, 0, entity2dept, t2entity)

        logger.debug("{} entities2dept: of dept 0".format(len(entity2dept)))
        logger.debug("{} t2entity: of dept 0".format(len(t2entity)))

        logger.info("adding users relationships")
        for dept_id in range(1,5):
            user_relationship_path = os.path.join(data_dir, 'email-Eu-core-temporal-Dept{}.txt'.format(dept_id))
            load_users_relationship(graph, user_relationship_path, dept_id, entity2dept, t2entity)

        logger.info("adding attribute relationships")
        add_attribute_relationships(graph, entity2dept, t2entity)

        return graph


# IMPORTS --------------------------------------------------------------

# Standard imports
import pdb
import logging

import sys
from datetime import datetime as dt

# Third party imports
pass

# local application imports
from ccat import config
from ccat.model.database.bucket import Bucket
from ccat.model.exchange.order import Order
from ccat.controller.strategy.reversal import Strategy

# SETTINGS -------------------------------------------------------------

# Get arguments from the cronjob/terminal.
# Default to default settings if sys.argv does not have any arguments
try:
    '''
    Cronjob:
    0 0 * * * python3 bot.py 1 6 500 10

    Run bot.py with the arguments below every day at midnight:
    sys.argv[1] = 1     > market_id = 1
    sys.argv[2] = 6     > timeframe_id = 6
    sys.argv[3] = 500   > count = 500
    sys.argv[4] = 500   > stake_size = 500
    '''
    market_id = int(sys.argv[1])
    timeframe_id = int(sys.argv[2])
    count = int(sys.argv[3])
    stake_size = int(sys.argv[4])

# If no ill-formed arguments are passed, a test order is created
# TODO: Remove this hack and pass args to sys.argv instead
except:
    market_id = 1
    timeframe_id = 6
    count = 500
    stake_size = 1

# Strategy settings
kwargs = {
    'len_ma_top': 24, #24
    'len_ma_bottom': 24,
    'prefix':'reversal'}

# EXECUTE STRATEGY -----------------------------------------------------

# Get a bucket instance with the right market and timeframe id's
bucket = Bucket(market_id = market_id, timeframe_id = timeframe_id)

# Update table with the candles for the timeframe
bucket.update(count=count)

# Get the dataframe with the bucket data for the latest count candles
df_bucket = bucket.read_latest(count=count)

# Create strategy instance using the strategy settings
strategy = Strategy(df_bucket = df_bucket, **kwargs)

# Get signal
signal = strategy.signal()

# LOGGING --------------------------------------------------------------

logging.basicConfig(
    filename=config.lf_bot,
    level=logging.INFO)

def do_log(order):
    info = (
        '{:3} | '
        'utc: {:<20} | '
        'signal: {:2} | '
        'stake: {:6} | '
        'status: {:<9} | '
        'timeframe_id: {:1} | '
        'market_id: {:3} | ').format(
            '',
            dt.strftime(dt.utcnow(), "%Y.%m.%d %H:%M:%S"),
            signal,
            stake_size,
            order.order_status,
            timeframe_id,
            market_id)
    logging.info(info)

# EXECUTE ORDER --------------------------------------------------------

def execute(signal):

    if signal != 0:
        order = Order(market_id = market_id)

        if signal == 1:
            order = order.submit(side = 'buy', amount = stake_size)
            do_log(order)
            return order

        elif signal == -1:
            order = order.submit(side = 'sell', amount = stake_size)
            do_log(order)
            return order

    else:
        order = type('Order', (object,), {})()
        order.order_status = "-"
        do_log(order)
        return order

# MAIN -----------------------------------------------------------------

if __name__ == '__main__':

    execute(signal)


from bs4 import BeautifulSoup
import requests
import re

CAC40_URL = "https://companiesmarketcap.com/cac-40/largest-companies-by-market-cap/"
AMERICAN_LARGEST_COMPANIES = (
    "https://companiesmarketcap.com/usa/largest-companies-in-the-usa-by-market-cap/"
)

def get_cac40_symbols() -> list[str]:
    soup = BeautifulSoup(requests.get(CAC40_URL).content, "lxml")
    raw_codes = soup.find_all("div", class_="company-code")
    return list(map(lambda input: str(re.sub("<[^>]*>", "", str(input))), raw_codes))

def get_largest_symbols_by_country_normal(n: int = 500) -> list[str]:
    n_per_page = 100
    assert (
        n >= n_per_page and n <= 3571
    ), "Error, provide a valid number of symbol, <= 3571 and >= 100."

    raw_codes = []
    pages = (n // n_per_page) + 1

    for i in range(pages):
        soup = BeautifulSoup(
            requests.get(f"{AMERICAN_LARGEST_COMPANIES}?page={i+1}").content, "lxml"
        )
        raw_codes.extend(soup.find_all("div", class_="company-code"))
    return list(map(lambda input: str(re.sub("<[^>]*>", "", str(input))), raw_codes))[
        :n
    ]


import statsmodels.api as sm
import logging
from housing_model import HousingModel

from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_absolute_error

import pandas as pd

# this should go to a config file with all the other (future) hyper-parameters
random_state = 237

class OLS(HousingModel):
    def __init__(self):
        super().__init__()

    def predict(self, x, x_test, cv=False):
        """
        IModel's predict functions OLS implementation.

        :param x: training data set
        :param x_test: test data set
        :param cv: True if prediction should be made with cross-validation
        """

        logging.info("Performing OLS prediction...")
        if cv:
            logging.warning("Cross validation not implemented for OLS.")

        # cut off target variable and id from the data
        y = pd.DataFrame(x['SalePrice'], columns=['SalePrice'])
        x.drop(['SalePrice'], axis=1, inplace=True)

        test_ids = x_test.Id
        x_test.drop(['Id'], axis=1, inplace=True)

        # standardize target variable
        mu = y.mean()
        sigma = y.std()

        y = (y - mu) / sigma

        # splitting the data into training and validation sets
        x_train, x_valid, y_train, y_valid = train_test_split(
            x, y, train_size=0.9, test_size=0.1, random_state=random_state
        )

        x_train = sm.add_constant(x_train)
        x_valid = sm.add_constant(x_valid)
        x_test = sm.add_constant(x_test)

        # OLS
        model = sm.OLS(list(y_train.SalePrice), x_train)
        results = model.fit()

        # make predictions for validation data and transform
        y_valid = self.inverse_transform(y_valid, mu, sigma)
        prediction = self.inverse_transform(pd.DataFrame(results.predict(x_valid), columns=['SalePrice']), mu, sigma)

        logging.info(f"OLS validation MAE: {mean_absolute_error(y_valid, prediction)}")

        test_pred = self.inverse_transform(pd.DataFrame(results.predict(x_test), columns=['SalePrice']), mu, sigma)
        test_pred = pd.DataFrame({'Id': test_ids, 'SalePrice': test_pred})
        test_pred.to_csv('submissions\\submission_OLS.csv', index=False)


from matplotlib.dates import AutoDateFormatter, AutoDateLocator
import numpy as np
import pandas as pd
from scipy.integrate import odeint
from csv import reader
from csv import writer
import scipy
from scipy.integrate import solve_ivp
from scipy.optimize import minimize
import matplotlib.pyplot as plt
from datetime import timedelta, datetime
import argparse
import sys
import os
import json
import ssl
import urllib.request
import statsmodels.api as sm

import pandasql as psql

from patsy import dmatrices

import matplotlib as mpl
mpl.rc('figure', max_open_warning = 0)

def load_data(path,sourcefile):
        df = pd.read_csv(path+"/"+sourcefile)
        return df

def generate_dataset(data, r_combined):
    tti = 0
    states = data['state'].drop_duplicates()
    data2 = data.copy()
    data3 = pd.DataFrame()
    for s in states:
        rc = r_combined[r_combined['state']==s]
        #print(rc)
        dat = data2[data2['state']==s]

        if len(dat)>0 and len(rc)>0:
            print(s)
            dat = dat.sort_values(by=['dateval'])
            #dat = dat.reset_index()
            beta = rc['lag_confirmed'].values[0]
            N = dat['population'].values[0]
            dat['susceptible'] = np.where(dat['holdout']==0, N+0.0,0.0)
            alpha = dat['R_0'].values[0]*beta
            dat['alpha'] = alpha
            if tti==0:
               data3 = dat
            else:
                data3 = data3.append(dat,ignore_index=True )
            tti = tti +1
    print(data3['state'].drop_duplicates())
    return data3

def test_covid(dataset,location_name):

    data = dataset[dataset['confirmed']>0]
    data = psql.sqldf("""
    select {loc},d_temp,period_group,did1,
    avg(ifnull(recovered,0)) as recovered,
    avg(ifnull(death,0)) as death,
    avg(ifnull(confirmed,0)) as positive from data
    group by 1,2,3,4,5
    """.format(loc=location_name))

    Y = data['positive']
    Y = (data['positive']-np.average(data['positive']))/np.std(data['positive'])
    X = data[['d_temp','period_group','did1']]

    formula = """positive ~ d_temp + period_group + did1 """
    response, predictors = dmatrices(formula, data, return_type='dataframe')
    po_results = sm.GLM(response, predictors, family=sm.families.Poisson()).fit()
    print(po_results.summary())

    Y = data['recovered']
    Y = (data['recovered']-np.average(data['recovered']))/np.std(data['recovered'])
    formula = """recovered ~ d_temp + period_group + did1 """
    response, predictors = dmatrices(formula, data, return_type='dataframe')
    po_results = sm.GLM(response, predictors, family=sm.families.Poisson()).fit()
    print(po_results.summary())

    Y = data['death']
    Y = (data['death']-np.average(data['death']))/np.std(data['death'])

    formula = """death ~ d_temp + period_group + did1 """
    response, predictors = dmatrices(formula, data, return_type='dataframe')
    po_results = sm.GLM(response, predictors, family=sm.families.Poisson()).fit()
    print(po_results.summary())

def test_gov_covid(dataset,location_name):

    data = dataset[dataset['positive']>0]
    Y = data['positive']
    Y = (data['positive']-np.average(data['positive']))/np.std(data['positive'])
    X = data[['d_temp','period_group','did1']]
    formula = """positive ~ d_temp  + period_group+CS+ED+GP+NEBC+OTH+SAH + did1 """
    response, predictors = dmatrices(formula, data, return_type='dataframe')
    po_results = sm.GLM(response, predictors, family=sm.families.Poisson()).fit()
    print(po_results.summary())
    Y = data['recovered']
    Y = (data['recovered']-np.average(data['recovered']))/np.std(data['recovered'])
    formula = """recovered ~ d_temp  + period_group+CS+ED+GP+NEBC+OTH+SAH + did1 """
    response, predictors = dmatrices(formula, data, return_type='dataframe')
    po_results = sm.GLM(response, predictors, family=sm.families.Poisson()).fit()
    print(po_results.summary())
    Y = data['death']
    Y = (data['death']-np.average(data['death']))/np.std(data['death'])
    formula = """death ~ d_temp  + period_group+CS+ED+GP+NEBC+OTH+SAH + did1 """
    response, predictors = dmatrices(formula, data, return_type='dataframe')
    po_results = sm.GLM(response, predictors, family=sm.families.Poisson()).fit()
    print(po_results.summary())

def solve_one(state,df,cfr,alpha,beta,c_x,c_names,N,horizon):
    def mysysfunc(h,s):
        z1 = 0.0
        z2 = 0.0
        tt = int(s)
        if tt>horizon and horizon<=len(df['dateval']):
            tt=horizon-1
            z1 = 0.0
            z2 = 0.0
        try:
            for i in range(len(c_x)):
                z1 = z1 + df[c_names[i]].values[tt]*c_x[i]

        except:
            z1 = z1

        dsdt = - (alpha/N) * h[0]*h[1]
        didt = (alpha/N)*h[0]*h[1] - beta*h[1]  -z1
        drdt = z1+beta*h[1]

        dhdt = [dsdt,didt,drdt]
        return dhdt

    t = [i for i in range(horizon)]
    # ############SETUP AND RUN THE SIMULATION=============
    h0 = [df['population'].values[0],df['confirmed'].values[0],df['removed'].values[0]]

    y = odeint(mysysfunc,h0,t) # integrate
    y = np.where(y<0,0,y)

    return y

def runDynamicSimulator(data1,coefsdfR,sir_names,xnamesr,horizon1):
    ###DEFINE SIMULATION CONSTANTS#######################################################
	# horizon = len(date_list)  # days . - forecast days
    # data1 starts from end of historical data date minus 7 days (from training)
    data = data1.copy()
    data = data.fillna(0)

    coefsR = coefsdfR.copy()

    coefsR = coefsR.fillna(0.0)

    its = 0
    for state in data['state'].drop_duplicates():

        if 1==1:
        #try:

            df = data[data['state']==state]
            print(df[0:10])
            if len(df['dateval'])>0:
                df = df.sort_values(by=['dateval'])
                df = df.loc[df['confirmed'].ne(0).idxmax():]
                df['pred_confirmed'] = 0.0
                df['pred_susceptible'] = 0.0
                df['pred_removed'] = 0.0
                #df = df.reset_index()
                dd = df[df['confirmed']>0]
                N = np.max(df['population'])
                print("Simulating state= "+state)
                horizon = len(df['dateval'])

                cfr = coefsR[coefsR['state']==state]
                alpha = df['alpha'].values[0]
                beta =  coefsR['lag_confirmed'].values[0]
                horizon_days = [i for i in range(horizon)]
                x1 = ['Intercept','gov_action','TAVG']
                c_xr = np.zeros(len(x1))

                df['pred_susceptible']= N #df['susceptible']
                for t in horizon_days:
                    if t==0:
                        df['pred_confirmed'].values[t] =df['confirmed'].values[t]
                        df['pred_susceptible'].values[t] = N #df['susceptible'].values[t]
                        df['pred_removed'].values[t] = df['removed'].values[t]

                    if t>0 and t<horizon:
                        z1 = 0.0
                        for c in range(len(x1)):
                            z1 = z1 + cfr[x1[c]].values[0]*df[x1[c]].values[t]
                        df['pred_confirmed'].values[t] = df['pred_confirmed'].values[t-1]+(alpha)*df['pred_susceptible'].values[t-1]*df['pred_confirmed'].values[t-1] - beta*df['pred_confirmed'].values[t-1] - z1

                        df['pred_removed'].values[t] = df['pred_removed'].values[t-1]+beta*df['pred_confirmed'].values[t-1] + z1

                        df['pred_susceptible'].values[t] = df['pred_susceptible'].values[t-1] - (alpha)*df['pred_susceptible'].values[t-1]*df['pred_confirmed'].values[t-1]

            if its ==0:
                out_df = df
            else:
                out_df = out_df.append(df)

            # # plot results
            plt.figure(1)
            plt.plot(out_df['dateval'],out_df['susceptible'],'b-')
            plt.plot(out_df['dateval'],out_df['confirmed'],'r--')
            plt.plot(out_df['dateval'],out_df['removed'],'g--')

            plt.xticks(rotation=45)
            plt.xlabel('Time')
            plt.ylabel('Populations')
            plt.legend(['Suceptibes','Confirmed','Removed'])
            plt.tight_layout()
            plt.savefig(os.path.join('/Users/aokossi/Documents/public service resarch/output/covid_plot_'+state+'.png'))
            plt.clf()
            its = its +1

        else:
        #except:
            print("skipped state for lack of government data "+state)

    return out_df

def runSimulator(data1,coefsdfR,sir_names,xnamesr,horizon1,date_gov_adjust, print_graph):
    	###DEFINE SIMULATION CONSTANTS#######################################################
	# horizon = len(date_list)  # days . - forecast days
    # data1 starts from end of historical data date minus 7 days (from training)
    # print(data1.columns)
    if date_gov_adjust > 0:
        data1.loc[data1.date>date_gov_adjust, 'gov_action'] = 0
    data = data1.copy()
    data = data.fillna(0)

    coefsR = coefsdfR.copy()

    coefsR = coefsR.fillna(0.0)

    df = pd.DataFrame()
    out_df = pd.DataFrame()
    its = 0
    for state in coefsR['state'].drop_duplicates():
        if 1==1:
        #try:
            cr = coefsR[coefsR['state']==state]
            df = data[data['state']==cr['state'].values[0]]
            if len(df)>0 and len(cr)>0:
            #if len(df['dateval'])>0:
                df = df.sort_values(by=['dateval'])
                df = df.reset_index()
                df = df.loc[df['confirmed'].ne(0).idxmax():]

                dd = df[df['confirmed']>0]
                N = np.max(df['population'])
                print("Simulating state= "+state)
                horizon = len(df['dateval']) # np.where(len(df['dateval'])<horizon1,len(df['dateval']),horizon1)

                cfr = cr
                alpha = df['alpha'].values[0]
                #t = np.linspace(0, horizon)  #[0:horizon] # times to report solution . - time in day sequence
                horizon_days = [i for i in range(horizon)]
                x1 = ['Intercept','gov_action','TAVG']
                c_xr = np.zeros(len(x1))
                for c in range(len(x1)):
                    c_xr[c] = cfr[x1[c]].values[0]
                c_xd = np.zeros(len(x1))

                # print(cfr)
                # print(c_xr)
                # print(c_xd)

                y = solve_one(state=state,df=df,cfr=cfr,alpha=alpha,beta=cfr['lag_confirmed'].values[0],c_x = c_xr,c_names = x1,N=N,horizon=horizon)

                print("solution for "+state)
                df['pred_susceptible'] = y[0:horizon,0]
                df['pred_confirmed'] = y[0:horizon,1]
                df['pred_removed'] = y[0:horizon,2]

                if print_graph == True:
                    # plot results
                    plt.figure(1)
                    plt.figure(figsize=(15,10))
                    xtick_locator = AutoDateLocator()
                    xtick_formatter = AutoDateFormatter(xtick_locator)

                    date_list = pd.to_datetime(df['dateval'])

                    ax = plt.axes()
                    ax.xaxis.set_major_locator(xtick_locator)
                    ax.xaxis.set_major_formatter(xtick_formatter)
                    plt.plot(date_list,y[:horizon,0],'b-')
                    plt.plot(date_list,y[:horizon,1],'r--')
                    plt.plot(date_list,y[:horizon,2],'g--')
                    plt.xlabel('Time')
                    plt.ylabel('Populations')
                    plt.legend(['Suceptibes','Confirmed','removed'])
                    plt.title('Prediction at '+state)
                    plt.savefig(os.path.join('output/covid_plot/covid_plot_'+state+'_'+str(date_gov_adjust)+'.png'))
                    plt.clf()
                    plt.close()
                out_df = out_df.append(df,ignore_index=True )

            its = its +1

        else:
        #except:
            print("skipped state for lack of government data "+state)

    return out_df

def causal_simulation(path,start_date,f_start_date,datafile="dataset_full.csv",govpolicyfile="gov_dates_mandates.csv", num_date_omit=0, print_graph=True):
    data = pd.read_csv(path+"/"+datafile)

    start_dt = datetime.strptime(start_date, '%m/%d/%y').strftime('%Y-%m-%d')
    print(start_dt)

    dateval = pd.date_range(start_dt, periods=horizon+180).tolist()
    dates = pd.DataFrame({'dateval': dateval})
    dates['dateval'] = dates['dateval'].apply(lambda x: datetime.strptime(str(x),'%Y-%m-%d %H:%M:%S').strftime('%Y-%m-%d') )
    data['dateval'] = data['date'].apply(lambda x: datetime.strptime(str(x),'%Y%m%d' ).strftime('%Y-%m-%d') )
    if num_date_omit > 0:
        temp_start_date = datetime.strptime(f_start_date, '%m/%d/%y') - timedelta(days=30)
        temp_start_date = int(temp_start_date.strftime('%Y%m%d'))
        print(temp_start_date)

        temp = data.loc[(data['confirmed'].isna())&(data['date']>temp_start_date), ['location_name','confirmed','dateval']].sort_values(by='dateval').reset_index(drop=True)
        temp_start_date = temp.loc[0, 'dateval']
        temp_start_training_date = (datetime.strptime(temp_start_date, '%Y-%m-%d') - timedelta(days=num_date_omit)).strftime('%Y-%m-%d')
        print(temp_start_training_date)
        temp_start_simulation_date = (datetime.strptime(temp_start_date, '%Y-%m-%d') - timedelta(days=num_date_omit-1)).strftime('%Y-%m-%d')

    data['state'] = data['province_state']

    zx = 0

    data = psql.sqldf("""
    select province_state , country_region as country, date, confirmed,
       recovered, death, population, TAVG/10 as TAVG,
     a1.location_name, a1.dateval, country_region ||'-'||state as state,
    case when TAVG<=0 then 1 else 0 end as is_freezing,
    case when TAVG>0 and TAVG/10<20 then 1 else 0 end as is_cold,
    case when TAVG>=20 and TAVG/10<35 then 1 else 0 end as is_warm,
    case when TAVG>=35 then 1 else 0 end as is_hot,
    case when TAVG>=20 then 1 else 0 end as temp_th,

      case when julianday(a1.dateval)>julianday('2020-03-20') then 1 else 0 end
       as gov_action
    from data a1
    """).drop_duplicates()

    data['Intercept'] = 1.0
    data = data[(data['dateval']>=start_dt)]
    data['holdout'] = np.where((data['dateval']>=datetime.strptime(f_start_date, '%m/%d/%y').strftime('%Y-%m-%d')),1,0)

    print(data)

    data_save = data.copy()

    # data smoothing to correct irregular data issues: like dropped cumulative values
    data1 = pd.DataFrame()

    z = 0

    for state in data['state'].drop_duplicates():
            dat = data[(data['state']==state)].sort_values(by=['dateval'])
            if len(dat['dateval'])>1:
                dat = dat.fillna(0)
                dat = dat.loc[dat['confirmed'].ne(0.0).idxmax():]
                rho_data = dat[dat['confirmed']>0].sort_values(by=['dateval'])
                rho_data = rho_data[0:30]

                zz1 = 0.0
                zz2 = 0.0
                zz3 = 0.0
                for s in range(len(rho_data['dateval'])):
                    if (s>0):
                        if rho_data['confirmed'].values[s]-rho_data['confirmed'].values[s-1]>0:
                            zz1 = zz1 + rho_data['confirmed'].values[s]-rho_data['confirmed'].values[s-1]
                        if rho_data['recovered'].values[s]-rho_data['recovered'].values[s-1]>0:
                            zz2 = zz2 + rho_data['recovered'].values[s]-rho_data['recovered'].values[s-1]
                        if rho_data['death'].values[s]-rho_data['death'].values[s-1]>0:
                            zz3  = zz3 + rho_data['death'].values[s]-rho_data['death'].values[s-1]
                rho = 0.0
                if (zz2+zz3) >0.0:
                    rho = (zz1+zz2+zz3)/(zz2+zz3)
                print("R_0 for "+state +" : "+str(rho))
                dat['lag_confirmed'] = 0.0
                dat['lag_recovered'] = 0.0
                dat['lag_death'] = 0.0
                dat['lag_removed'] = 0.0

                dat['d_recovered'] = 0.0
                dat['d_death'] = 0.0
                dat['d_removed'] = 0.0

                N = dat['population'].values[0]
                dd = dat[dat['confirmed']>1]

                dat['R_0'] = rho
                z1 = 0
                tt = 1
                dat['removed'] = dat['death'] + dat['recovered']

                for t in range(len(dat)):

                    if t>0 and t<=len(dat):
                        dat['lag_confirmed'].values[t] = dat['confirmed'].values[t-1]
                        dat['lag_removed'].values[t] = dat['removed'].values[t-1]
                        dat['lag_recovered'].values[t] = dat['recovered'].values[t-1]
                        dat['lag_death'].values[t] = dat['death'].values[t-1]
                        dat['d_recovered'].values[t] =  dat['recovered'].values[t] - dat['recovered'].values[t-1]
                        dat['d_death'].values[t] = dat['death'].values[t]-dat['death'].values[t-1]
                        dat['d_removed'].values[t] = dat['removed'].values[t]-dat['removed'].values[t-1]

                data1 = data1.append(dat,ignore_index=True)
                z = z +1

    data = data1.copy()
    rb = np.mean(data[data['R_0']>0]['R_0'])
    data['R_0'] = np.where(data['R_0']==0,rb,data['R_0'])
    data = data.fillna(0)
    data.to_csv("output/simulation_output/input_data.csv")
    if num_date_omit > 0:
        data_train = data[(data['removed']>0) & ((data['holdout']==0) | (data['dateval']<=temp_start_training_date))][['dateval','Intercept','state','TAVG','gov_action','is_freezing','is_cold','is_warm','is_hot','lag_confirmed','lag_death','lag_recovered','d_death','d_recovered','d_removed','removed']]
        print(temp_start_training_date)
        print(max(data_train['dateval']))
    else:
        data_train = data[(data['removed']>0) & (data['holdout']==0)][['dateval','Intercept','state','TAVG','gov_action','is_freezing','is_cold','is_warm','is_hot','lag_confirmed','lag_death','lag_recovered','d_death','d_recovered','d_removed','removed']]

    endog =data_train['d_removed']
    exog = data_train[[ 'Intercept','gov_action','TAVG','lag_confirmed']]
    model = sm.MixedLM(endog, exog, exog_re=exog[[ 'Intercept','lag_confirmed']],  groups=data_train["state"])
    po_results = model.fit()
    print(po_results.summary())

    # Get Coefficient

    k = []
    v1 = []
    v2 = []
    v3 = []
    d = po_results.random_effects

    for i in d:
        my_str = ''.join((ch if ch in '0123456789.-' else ' ') for ch in str(d[i]))
        listOfNumbers = [float(i) for i in my_str.split()]
        v1 = v1 +[str(d[i]).split(" ")[0] ]
        l = str(d[i]).split(" ")
        if str(listOfNumbers[0]).strip()=='':
            v2 = v2 +[0.00 ]
            v3 = v3 + [0.0]
        else:
            v2 = v2 +[listOfNumbers[0] ]
            v3 = v3 +[listOfNumbers[1] ]
        k = k + [i]

    r_combined = pd.DataFrame({'state':k,'coef_name':v1,'coef_value':v2,'re_lag_confirmed':v3})

    r_combined['fe_Intercept'] = po_results.fe_params['Intercept']

    r_combined['Intercept'] = r_combined['fe_Intercept']+r_combined['coef_value']

    r_combined['lag_confirmed'] = po_results.fe_params['lag_confirmed'] +r_combined["re_lag_confirmed"]
    r_combined['gov_action'] = po_results.fe_params['gov_action']
    r_combined['TAVG'] = po_results.fe_params['TAVG']

    r_combined.fillna(0.0)
    r_combined.to_csv("output/simulation_output/recover_coefs.csv")

    mean_beta = np.mean(r_combined[r_combined['lag_confirmed']>0]['lag_confirmed'])
    r_combined['lag_confirmed'] = np.where(r_combined['lag_confirmed']<0,mean_beta,r_combined['lag_confirmed'])

    # Get Prediction and Bias

    t_dat = generate_dataset(data, r_combined)
    if num_date_omit > 0:
        t_dat = t_dat[(t_dat['removed']>0) & ((t_dat['holdout']==0) | (t_dat['dateval']<=temp_start_training_date))]
    else:
        t_dat = t_dat[(t_dat['removed']>0) & (t_dat['holdout']==0)]

    pred_on_train = runSimulator(data1=t_dat,
    coefsdfR=r_combined,
    sir_names=['susceptible','confirmed','death','removed'],
    xnamesr=['Intercept','gov_action','TAVG','lag_confirmed'],
    horizon1=60, date_gov_adjust=0, print_graph=print_graph)

    # == Adjust Prediction of Removed with Bias

    pred_on_train['bias_removed'] = pred_on_train['pred_removed'] - pred_on_train['removed']
    # pred_on_train.to_csv('output/simulation_output/pred_on_train.csv')
    mean_bias = pred_on_train.groupby('location_name')['bias_removed'].mean().reset_index()
    mean_bias.to_csv('output/simulation_output/bias.csv')

    loc_list = set(pred_on_train['location_name'])
    for loc in loc_list:
        bias = mean_bias.loc[mean_bias['location_name']==loc, 'bias_removed'].iloc[0]
        if bias > 0:
            pred_on_train.loc[pred_on_train['location_name']==loc, 'pred_removed'] = pred_on_train.loc[pred_on_train['location_name']==loc, 'pred_removed']-bias
        elif bias < 0:
            pred_on_train.loc[pred_on_train['location_name']==loc, 'pred_removed'] = pred_on_train.loc[pred_on_train['location_name']==loc, 'pred_removed']+bias

        loc_name = "".join(c for c in loc if c.isalnum())
        temp_for_plot = pred_on_train.loc[pred_on_train['location_name']==loc, ['dateval', 'removed', 'pred_removed']]
        plt.figure(figsize=(12,12))
        fig = temp_for_plot.plot(x='dateval', y=['removed', 'pred_removed'], rot=45, ax=plt.gca()).get_figure()
        fig.savefig(os.path.join('output/covid_plot/actual_pred/covid_plot_compare_'+loc_name+'.png'))
        fig.clf()

    pred_on_train['pred_removed'] = np.where(pred_on_train['pred_removed']<0, 0, pred_on_train['pred_removed'])

    pred_on_train.to_csv('output/simulation_output/adjusted_pred_on_train.csv')

    print(pred_on_train)

    # Adjust R-Combined with Bias

    mean_bias2 = mean_bias.copy()
    r_combined2 = r_combined.copy()
    mean_bias2['location_name'] = mean_bias2['location_name'].str.replace('[^a-zA-Z]', '')
    r_combined2['state'] = r_combined2['state'].str.replace('[^a-zA-Z]', '')
    loc_list = set(mean_bias2['location_name'])
    for loc in loc_list:
        bias = mean_bias2.loc[mean_bias2['location_name']==loc, 'bias_removed'].iloc[0]
        if bias > 0:
            r_combined2.loc[r_combined2['state']==loc, 'Intercept'] = r_combined2.loc[r_combined2['state']==loc, 'Intercept']-bias
        elif bias < 0:
            r_combined2.loc[r_combined2['state']==loc, 'Intercept'] = r_combined2.loc[r_combined2['state']==loc, 'Intercept']+bias

    r_combined = r_combined2

    tti = 0

    states = data['state'].drop_duplicates().str.replace('[^a-zA-Z]', '')
    data2 = data.copy()
    data2['state'] = data2['state'].str.replace('[^a-zA-Z]', '')
    data3 = pd.DataFrame()
    for s in states:
        rc = r_combined[r_combined['state']==s]
        #print(rc)
        dat = data2[data2['state']==s]

        if len(dat)>0 and len(rc)>0:
            print(s)
            dat = dat.sort_values(by=['dateval'])
            #dat = dat.reset_index()
            beta = rc['lag_confirmed'].values[0]
            N = dat['population'].values[0]
            dat['susceptible'] = np.where(dat['holdout']==0, N+0.0,0.0)

            alpha = dat['R_0'].values[0]*beta
            dat['alpha'] = alpha

            if tti==0:
               data3 = dat
            else:
                data3 = data3.append(dat,ignore_index=True )
            tti = tti +1

    print(data3)
    print(data3['state'].drop_duplicates())

    # data3.to_csv(path+"/before_sim_data_test.csv")
    #runDynamicSimulator
    #runSimulator(data1,coefsdfR,sir_names,xnamesr,horizon1)
    date_start_sim = 20200510

    if num_date_omit > 0:
        sim_data = data3[(data3['holdout']==1)&(data3['dateval']>=temp_start_simulation_date)]
        print(temp_start_simulation_date)
        print(min(sim_data['dateval']))
    else:
        sim_data = data3[(data3['holdout']==1)]

    sim_data_output_after = runSimulator(data1=sim_data,
    coefsdfR=r_combined,
    sir_names=['susceptible','confirmed','death','removed'],
    xnamesr=['Intercept','gov_action','TAVG','lag_confirmed'],
    horizon1=60, date_gov_adjust=date_start_sim, print_graph=print_graph)
    sim_data_output_after.to_csv("output/simulation_output/simulations_after_adjust_at_"+str(date_start_sim)+"_omitlastD_"+str(num_date_omit)+".csv")

    sim_data_output_before = runSimulator(data1=sim_data,
    coefsdfR=r_combined,
    sir_names=['susceptible','confirmed','death','removed'],
    xnamesr=['Intercept','gov_action','TAVG','lag_confirmed'],
    horizon1=60, date_gov_adjust=0, print_graph=print_graph)
    sim_data_output_before.to_csv("output/simulation_output/simulations_before_adjust_at_"+str(date_start_sim)+"_omitlastD_"+str(num_date_omit)+".csv")

    sim_data_compare = sim_data_output_after.merge(sim_data_output_before, on=['index', 'province_state', 'country','date','dateval','location_name'], suffixes=('_after', '_before'))
    sim_data_compare['diff_susceptible'] = sim_data_compare['pred_susceptible_after'] - sim_data_compare['pred_susceptible_before']
    sim_data_compare['diff_confirmed'] = sim_data_compare['pred_confirmed_after'] - sim_data_compare['pred_confirmed_before']
    sim_data_compare['diff_removed'] = sim_data_compare['pred_removed_after'] - sim_data_compare['pred_removed_before']

    sim_data_compare = sim_data_compare.loc[:, ['province_state', 'country','date','dateval','location_name','pred_susceptible_after', 'pred_confirmed_after', 'pred_removed_after', 'pred_susceptible_before', 'pred_confirmed_before', 'pred_removed_before', 'diff_susceptible', 'diff_confirmed', 'diff_removed']]
    sim_data_compare.to_csv("output/simulation_output/simulations_compare"+str(date_start_sim)+"_omitlastD_"+str(num_date_omit)+".csv")

    if print_graph == True:
        for location in sim_data_compare['location_name'].drop_duplicates():
            dat = sim_data_compare[(sim_data_compare['location_name']==location)].sort_values(by=['date'])
            # plot results
            plt.figure(1)
            plt.figure(figsize=(15,10))
            xtick_locator = AutoDateLocator()
            xtick_formatter = AutoDateFormatter(xtick_locator)
            date_list = pd.to_datetime(dat['dateval'])
            ax = plt.axes()
            ax.xaxis.set_major_locator(xtick_locator)
            ax.xaxis.set_major_formatter(xtick_formatter)
            plt.plot(date_list,dat['diff_susceptible'],'b-')
            plt.plot(date_list,dat['diff_confirmed'],'r--')
            plt.plot(date_list,dat['diff_removed'],'g--')
            plt.xlabel('Time')
            plt.ylabel('Populations')
            plt.title('Compare Before/After Gov. Intervention Adjust at '+''.join(e for e in location if e.isalnum())+' : '+str(date_start_sim))
            plt.legend(['Diff Suceptibes','Diff Confirmed','Diff Removed'])
            plt.savefig(os.path.join('output/covid_plot/covid_plot_compare_'+"omitlastD_"+str(num_date_omit)+''.join(e for e in location if e.isalnum())+'_'+str(date_start_sim)+'.png'))
            plt.clf()
            plt.close()

if __name__=="__main__":

    path="data/simulation_data"
    datasource="dataset_full.csv"
    pop_source="pop_dataset.csv"
    location_name="country_region"
    start_date="2/22/20"
    horizon = 30

    causal_simulation(path=path,start_date=start_date,f_start_date="4/20/20", datafile="dataset_full.csv",govpolicyfile="gov_dates_mandates.csv", print_graph=True, num_date_omit=7)

    exit(0)


import boto3
from boto3.dynamodb.conditions import Key
from cStringIO import StringIO
import json
import decimal
import logging

LOG = logging.getLogger()
LOG.setLevel(logging.DEBUG)
BUCKET = "testing.submityoursalary.com"
PENDING_KEY = "pending-reviews/pending-reviews.json"
REVIEW_KEY = "reviews/reviews.json"
SUBMISSIONS_TABLE_NAME = 'submityoursalary-submissions'

def value_convert(v):
    if isinstance(v, decimal.Decimal):
        if v % 1 > 0:
            return float(v)
        else:
            return int(v)
    return v

def delete_item(key_id):
    dynamo_submissions = b.resource('dynamodb').Table(SUBMISSIONS_TABLE_NAME)
    dynamo_submissions.delete_item(Key={'id': key_id})

b = boto3.session.Session(profile_name='submityoursalary', region_name='us-east-1')
s = b.resource('dynamodb').Table(SUBMISSIONS_TABLE_NAME)
s3 = b.resource('s3')

FIELDS = ["id", "emoji", "company", "create_time", "position", "review"]

pending = Key('approve').lt(2)
approved = Key('approve').gte(2)
rejected = Key('reject').gte(2)

# rejected
r = s.scan(FilterExpression=rejected)
if 'Items' in r:
    for i in r['Items']:
        LOG.info("Deleting {}".format(i['id']))
        print("Deleting {}".format(i['id']))
        delete_item(i['id'])

# pending
p = s.scan(FilterExpression=pending)
if 'Items' in p:
    pending = [{k: value_convert(d[k]) for k in FIELDS} for d in p['Items']]
    pending_contents = StringIO(json.dumps(pending, indent=4, sort_keys=True))
    bucket = s3.Bucket(BUCKET)
    bucket.put_object(ACL='public-read', Bucket=BUCKET, Key=PENDING_KEY, Body=pending_contents.read())

# approved
a = s.scan(FilterExpression=approved)
if 'Items' in a:
    approved = [{k: value_convert(d[k]) for k in FIELDS} for d in a['Items']]
    approved_contents = StringIO(json.dumps(approved, indent=4, sort_keys=True))
    bucket = s3.Bucket(BUCKET)
    bucket.put_object(ACL='public-read', Bucket=BUCKET, Key=REVIEW_KEY, Body=approved_contents.read())


"""
This module is responsible for attributing changes in a git diff (which operates on raw text, and has no understanding of the source code)
into Java code hierarchies that contain them - packages, classes and methods. This is done by parsing
the source files changed within a diff(see `java_parser` module) and matching each hunk of changed text
to the nearest Java elements that contain them, based on their byte offsets in the file.

"""
from __future__ import annotations
from typing import Any, Optional, List, Iterable, Tuple, NamedTuple, Set
import pygit2
import logging
from .java_decl_parser import JavaParser, JavaHierarchy
from .java_change import JavaChange, ChangeType, JavaIdentifier, PosToJavaUnitMatcher
import subprocess

log = logging.getLogger("java_change_detector")

class GitHunkOffsets(NamedTuple):
    """ This class contains the byte offsets of the text that was added or deleted
        in a Git diff

        (The pygit2 hunk object provides information about line number ranges, we
        prefer to deal with byte offset ranges since Java statements aren't necessarily
        line oriented)
    """

    # (inclusive, exclusive)

    deleted_byte_range: Optional[Tuple[int, int]]
    added_byte_range: Optional[Tuple[int, int]]

    @staticmethod
    def from_pygit_hunk(hunk) -> GitHunkOffsets:
        deleted_start_pos, deleted_end_pos = -1, -1
        added_start_pos, added_end_pos = -1, -1

        for line in hunk.lines:
            line_start_pos = line.content_offset
            line_end_pos = line_start_pos + len(line.raw_content)
            if line.origin == "+":
                if added_start_pos < 0:
                    added_start_pos = line_start_pos
                added_end_pos = line_end_pos

            elif line.origin == "-":
                if deleted_start_pos < 0:
                    deleted_start_pos = line_start_pos
                deleted_end_pos = line_end_pos

        deleted_byte_range = (
            deleted_start_pos, deleted_end_pos) if deleted_start_pos >= 0 else None
        added_byte_range = (
            added_start_pos, added_end_pos) if added_start_pos >= 0 else None

        return GitHunkOffsets(deleted_byte_range=deleted_byte_range,
                              added_byte_range=added_byte_range)

    def display(self, old: Optional[bytes], new: Optional[bytes]) -> str:
        """ Displays the hunk(given the raw contents of the old and new files it
            was produced from) """
        st = []
        if old is not None and self.deleted_byte_range is not None:
            old_from, old_to = self.deleted_byte_range
            old_st = str(old[old_from: old_to], 'latin-1')
            st.append(f"-\t{repr(old_st)}")
        if new is not None and self.added_byte_range is not None:
            new_from, new_to = self.added_byte_range
            new_st = str(new[new_from: new_to], 'latin-1')
            st.append(f"+\t{repr(new_st)}")
        return "\n".join(st)
class JavaChangeDetector:
    """ Used to determine changes to a Java project via git """

    def __init__(self, repo: pygit2.Repository):
        self.__repo = repo
        self.__parser = JavaParser()

    def identify_changes_new_file(self, patch) -> Iterable[JavaChange]:

        new_file = patch.delta.new_file
        new_file_contents: bytes = self.__repo[new_file.id].data

        log.debug(f"Identifying changes in a new file {new_file.path}")

        hierarchy = self.__parser.parse_java_bytes(new_file_contents)
        for hierch in hierarchy.iterate_dfs_preorder():
            yield JavaChange(identifier=JavaIdentifier.from_bottommost_hierarchy(hierch),
                             change_type=ChangeType.ADD)

    def identify_changes_deleted_file(self, patch) -> Iterable[JavaChange]:
        deleted_file = patch.delta.old_file
        deleted_file_contents: bytes = self.__repo[deleted_file.id].data

        log.debug(f"Identifying changes in a deleted file {deleted_file.path}")

        hierarchy = self.__parser.parse_java_bytes(deleted_file_contents)
        for hierch in hierarchy.iterate_dfs_preorder():
            yield JavaChange(identifier=JavaIdentifier.from_bottommost_hierarchy(hierch),
                             change_type=ChangeType.DELETE)

    def identify_changes_modified_file(self, patch) -> Iterable[JavaChange]:
        old_file = patch.delta.old_file
        new_file = patch.delta.new_file
        old_content: bytes = self.__repo[old_file.id].data
        new_content: bytes = self.__repo[new_file.id].data

        log.debug(f"Identifying changes in a modified file {old_file.path}")
        old_hierch = self.__parser.parse_java_bytes(old_content)
        new_hierch = self.__parser.parse_java_bytes(new_content)
        old_hierch_matcher = PosToJavaUnitMatcher(old_hierch)
        new_hierch_matcher = PosToJavaUnitMatcher(new_hierch)
        deletes: Set[JavaIdentifier] = set()
        adds: Set[JavaIdentifier] = set()
        for hunk in patch.hunks:
            hunk_offsets = GitHunkOffsets.from_pygit_hunk(hunk)
            # print("Displaying hunk", hunk_offsets.display(old_content, new_content), sep="\n")
            if hunk_offsets.deleted_byte_range is not None:
                deletes.update(map(JavaIdentifier.from_bottommost_hierarchy, old_hierch_matcher.find_range(
                    hunk_offsets.deleted_byte_range[0],
                    hunk_offsets.deleted_byte_range[1]
                )))
            if hunk_offsets.added_byte_range is not None:
                adds.update(map(JavaIdentifier.from_bottommost_hierarchy, new_hierch_matcher.find_range(
                    hunk_offsets.added_byte_range[0],
                    hunk_offsets.added_byte_range[1]
                )))
        updates = adds & deletes
        for add in adds:
            if add not in updates:
                yield JavaChange(identifier=add, change_type=ChangeType.ADD)
        for delete in deletes:
            if delete not in updates:
                yield JavaChange(identifier=delete, change_type=ChangeType.DELETE)
        for update in updates:
                yield JavaChange(identifier=update, change_type=ChangeType.MODIFY)

    def identify_changes(self, patch) -> Iterable[JavaChange]:
        """ Given a git repository and patch(object that represents a difference in a file between two commits, possibly
            with renaming), determines the Java hierarchies that were changed"""
        delta = patch.delta

        status_char = delta.status_char()
        if status_char == "A":
            return self.identify_changes_new_file(patch)
        elif status_char == "D":
            return self.identify_changes_deleted_file(patch)
        elif status_char == "M":
            return self.identify_changes_modified_file(patch)
        else:
            log.warning(f"Unsupported git delta status character '{status_char}', skipping")
            return []


import requests
import json, os, re, time
import urllib.request

url = "https://graphql.anilist.co/"

# To get access token please visit: https://anilist.co/api/v2/oauth/authorize?client_id=2861&response_type=token
accessToken = json.loads(open(os.path.join('', 'api.json')).read())
if accessToken in "REPLACE WITH ACCESS TOKEN":
    print("Please add your access token.")
else:
    #user specified page number
    startpage = int(input('Enter start page number -->'))
    stoppage = int(input('Enter page number to stop at -->'))
    print('         ------STARTING------')
    while startpage < stoppage:

        no = startpage

        #POST request data
        payload = "{\"query\":\"query ($p: Int) {\\n  Page(page: $p) {\\n    users(sort: ID_DESC) {\\n      id\\n      avatar{\\n        large\\n      }\\n      \\n    }\\n  }\\n}\",\"variables\":{\"p\":" + str(no) + "}}"
        headers = {
            'content-type': "application/json",
                'authorization': "Bearer " + accessToken
         }

        #POST request parameters.
        response = requests.request("POST", url, data=payload, headers=headers)

        #Output - query data
        qdata = response.text
        #remove backslashes.
        qdata = qdata.replace('\\','')
        #remove square brackets.
        qdata = qdata.replace('[', '')
        qdata = qdata.replace(']', '')
        #Remove default image profiles.
        qdata = qdata.replace('"https://s4.anilist.co/file/anilistcdn/user/avatar/large/default.png"', 'null')
        #remove "large":"
        qdata = qdata.replace('"large":"','')
        #remove quotation marks
        qdata = qdata.replace('"','')

        #DO NOT CASEFOLD. IMAGE URL IS CASE SENSITIVE
        #Turn qdata into an array at each },{
        rdata = qdata.split('},{')

        #print(rdata)

        #loop 50 times with +1 each cycle, because max entries per page is 50.
        nq = 1
        while nq < 49:
            #set image as rdata
            image = rdata[nq]
            idnum = rdata[nq].split(',')
            match = re.search("{(.+?)}", image,flags=re.IGNORECASE)
            result = match.group(1)

            if result == 'large:null':
                nq = nq+1
            else:
                idnum = idnum[0].replace('id:','')
                page = urllib.request.urlopen("http://localhost:5555/api/classify?url=" + result).read().decode('utf-8')
                jsonify = json.loads(page)
                get_predictions = jsonify.get('predictions',{})
                pred1 = get_predictions[0]
                pred2 = get_predictions[1]
                nsfw = ["hentai", "sexy", "porn"]
                possible_class1 = pred1.get('class',{})
                possible_class2 = pred2.get('class',{})
                predict1 = pred1.get('prob',{}) * 100
                predict2 = pred2.get('prob',{}) * 100

                if possible_class1 in nsfw:
                    print('https://anilist.co/user/' + idnum)
                    print("1st guess: " + possible_class1.upper() + ": " + str(predict1) + "%")
                else:
                    pass

                if possible_class2 in 'hentai':
                    if int(predict2) > 10:
                        print('https://anilist.co/user/' + idnum)
                        print("2nd guess: " + possible_class2.upper() + ": " + str(predict2) + "%")
                    else:
                        pass
                else:
                    pass
                nq = nq+1
        time.sleep(1)
        startpage = startpage+1
        if startpage == stoppage:
            print('         ------FINISHED------')
        else:
            print('         Page: ' + str(startpage))


import tensorflow as tf
from typing import List, Optional
from tensorflow.keras.layers import Layer
from .DownConvolution import DownConvolution
from ...mesh.Mesh import Mesh
from ..feature import features_valid

class Encoder(Layer):
    """This combines several DownConvolution layers."""

    down_convolutions: List[DownConvolution]

    def __init__(
        self,
        out_channels: List[int],
        convolutions_per_sequence: int,
        num_sequences_per_down_convolution: int,
        leaky_relu_alpha: float,
        pool_targets: List[Optional[int]],
    ):
        Layer.__init__(self)

        assert len(out_channels) == len(pool_targets)
        for i in range(1, len(pool_targets)):
            assert pool_targets[i] is None or pool_targets[i] < pool_targets[i - 1]

        self.down_convolutions = [
            DownConvolution(
                out_channels[i],
                convolutions_per_sequence,
                num_sequences_per_down_convolution,
                leaky_relu_alpha,
                pool_targets[i],
            )
            for i in range(len(pool_targets))
        ]

    def call(self, mesh: Mesh, features: tf.Tensor) -> tf.Tensor:
        assert features_valid(mesh, features)

        snapshots = []
        for down_convolution in self.down_convolutions:
            features, snapshot = down_convolution(mesh, features)
            snapshots.append(snapshot)

        return features, snapshots


#!/usr/bin/python3

#flask stuff
from flask import Blueprint, render_template

#return output in json
import json

#custom-form class
from forms import RemoteJobsForm

#custom config definitions
from config import Config

import localenv

from localutils import getUsername

from jobs_scraper import show_job_list, get_job_list, search_job_list

#ui-form route
remotejobsui = Blueprint('remotejobsui', __name__)

#variables
httpRCok=200
httpRCbad=400
httpRCsvr=500

devopsContact=localenv.CONTACT

try:
  buyMeCoffee = str(localenv.BUY_ME_COFFEE_URL).split("/")[-1]
except:
  buyMeCoffee=None

@remotejobsui.route('/', methods=['GET', 'POST'])
def uiform():
    username = getUsername()

    form = RemoteJobsForm()
    formOutput = {}

    job_details = localenv.JOB_DETAILS
    if form.validate_on_submit():
        keyword_sub = form.searchkeyword.data
        #formOutput = get_job_list(keyword_sub, job_details) #redo job-scraping
        formOutput = search_job_list(keyword_sub) #search from sqlite-db

        return render_template('remotejobs.html', title='Remote Jobs', user=username, form=form, output=formOutput, appcontact=devopsContact, buyMeCoffee=buyMeCoffee)
    else:
        #formOutput = get_job_list(None, job_details)
        formOutput = show_job_list() #search from sqlite-db

        return render_template('remotejobs.html', title='Remote Jobs', user=username, form=form, output=formOutput, appcontact=devopsContact, buyMeCoffee=buyMeCoffee)

    return render_template('remotejobs.html', title='Remote Jobs', user=username, form=form, output=formOutput, appcontact=devopsContact, buyMeCoffee=buyMeCoffee)


from datetime import datetime

from django.http import HttpResponse
from django.shortcuts import render, redirect

COMPANY_MAIL = "the_boss@company.com"

# The simplest possible view
def show_contact_info(request):
    contact_info = "Sofia, bul. Lomsko Shose, Ivan Ivanov"
    return HttpResponse(contact_info)

class ShirtSizes:
    @staticmethod
    def get_available_shirt_sizes():
        return ["XS", "S", "M", "L", "XL", "XXL"]
        # return ["XS", ]

# View with context
def show_about(request):
    context = {
        "introduction": "We are the best online shop for shirts!\n Men and women.\nTake a look! ",
        "email_address": "ivan.ivanov@company.com",
        "name": {
            "first": "Ivan",
            "last": "Ivanov",
        },
        "shirt_sizes": ShirtSizes(),
    }
    return render(request, "about.html", context=context)

def index(request):
    context = {
        "current_time": datetime.now(),
        # "user": "Gosho",
        "user": "",
    }
    return render(request=request, template_name="index.html", context=context)

def redirect_to_home(request):
    return redirect(to="index")

class Article:
    starting_id = 0

    def __init__(self, headline, tease, full_text, local_path):
        self.headline = headline
        self.tease = tease
        self.full_text = full_text
        self.local_path = local_path
        self.article_id = Article.starting_id
        Article.starting_id += 1

    def get_absolute_url(self):
        return "https://www.shirt_company.com" + self.local_path

articles = [
    Article(
        headline="How to find the size that matches you?",
        tease="It has never been easier to find a perfectly matching shirt. Find how in this article.",
        full_text="{full text of the article}",
        local_path="article/1/",
    ),
    Article(
        headline="The trendiest colors of this summer",
        tease="Can you guess the winner out of purple, blue or orange?",
        full_text="{full text of the article}",
        local_path="article/2/",
    ),
]

def show_articles(request):
    context = {
        "articles": articles,
    }
    return render(request, "articles.html", context)

def show_single_article_by_id(request, article_id):
    article = [a for a in articles if a.article_id == article_id][0]
    context = {
        "article": article,
    }
    return render(request, "article.html", context)

def checkout_basket(request):
    context = {
        "basket": ["milk", "coffee", "bread", ],
        # "basket": [],
        "bill": 134.1214401239,
    }

    return render(request, "checkout.html", context)


import numpy as np
import matplotlib.pyplot as plt

# 目标函数:y=x^2
def func(x):
    return np.square(x)

# 目标函数一阶导数:dy/dx=2*x
def dfunc(x):
    return 2 * x

# Gradient Descent
def GD(x_start, df, epochs, lr):
    """
    梯度下降法。给定起始点与目标函数的一阶导函数，求在epochs次迭代中x的更新值
    :param x_start: x的起始点
    :param df: 目标函数的一阶导函数
    :param epochs: 迭代周期
    :param lr: 学习率
    :return: x在每次迭代后的位置（包括起始点），长度为epochs+1
    """
    xs = np.zeros(epochs+1)
    x = x_start
    xs[0] = x
    for i in range(epochs):
        dx = df(x)
        # v表示x要改变的幅度
        v = - dx * lr
        x += v
        xs[i+1] = x
    return xs

def demo1_GD_lr():
    # 函数图像
    line_x = np.linspace(-5, 5, 100)
    line_y = func(line_x)
    plt.figure('Gradient Desent: Learning Rate')

    x_start = -5
    epochs = 5

    lr = [0.1, 0.3, 0.9]

    color = ['r', 'g', 'y']
    size = np.ones(epochs+1) * 10
    size[-1] = 70
    for i in range(len(lr)):
        x = GD(x_start, dfunc, epochs, lr=lr[i])
        plt.subplot(1, 3, i+1)
        plt.plot(line_x, line_y, c='b')
        plt.plot(x, func(x), c=color[i], label='lr={}'.format(lr[i]))
        plt.scatter(x, func(x), c=color[i])
        plt.legend()
    plt.show()
demo1_GD_lr()

2022-2022大二第二学期/最优化/梯度下降法/附录1代码.py