import cv2
import numpy as np

# 步骤1: 加载图像并分割为红绿蓝通道
image = cv2.imread('1.jpg')
blue, green, red = cv2.split(image)

# 步骤2: 显示原始绿通道图像
cv2.imshow('Original Green Channel', green)

# 步骤3: 克隆绿通道图像两次
clone1 = green.copy()
clone2 = green.copy()

# 步骤4: 计算绿通道的最小值和最大值
min_value = np.min(green)
max_value = np.max(green)

# 步骤5: 将clone1的所有元素赋值为(thresh/2)
thresh = (max_value - min_value) / 2.0
clone1[:] = thresh

# 步骤6: 将clone2的所有元素赋值为0
clone2[:] = 0

# 步骤7: 使用cv2.compare比较图像
comparison_result = cv2.compare(green, clone1, cv2.CMP_GE)

# 步骤8: 使用cv2.subtract进行减法操作
processed_green = cv2.subtract(green, thresh / 2, green, comparison_result)

# 显示处理后的绿通道图像
cv2.imshow('Processed Green Channel', processed_green)

# 等待用户按下任意键，然后关闭所有窗口
cv2.waitKey(0)
cv2.destroyAllWindows()


import logging
import os

import youtube_dl
from telegram import ChatAction
from telegram.error import NetworkError
from telegram.ext import CommandHandler

from utils.decorators import handle_empty_arg, send_recording_action
from utils.utils import send_message_to_admin

logger = logging.getLogger(__name__)

FILENAME = 'Audio_Cuervot'
VLC_LINK = 'https://play.google.com/store/apps/details?id=org.videolan.vlc'

@send_recording_action
@handle_empty_arg(required_params=('args',), error_message='Y la url del video? `/yttomp3 <url>`',
                  parse_mode='markdown')
def youtube_to_mp3(bot, update, args):
    video_url = args[0]

    def ext(f):
        return f'bestaudio[ext={f}]'

    def format_extensions(extens):
        return '/'.join(map(ext, extens))

    try:
        extensions = ('mp3', '3gp', 'aac', 'wav', 'flac', 'm4a')
        ydl_opts = {
            'format': format_extensions(extensions),
            'outtmpl': f'{FILENAME}.%(ext)s',
        }
        logger.info(f'Starting download of {video_url}..')
        with youtube_dl.YoutubeDL(ydl_opts) as ydl:
            ydl.download([video_url])
        logger.info(f'Video downloaded.')

    except Exception:
        logger.error('Audio not downloaded.', exc_info=True)
        update.message.reply_text(f'No audio found for {video_url}')
        return

    # Download was successful, now we must open the audio file
    try:
        logger.info('Reading audio file from local storage')
        file, filename = get_audio_file(extensions)
        update.message.reply_text(f'âœ… Archivo descargado. Enviando...')
        bot.send_chat_action(chat_id=update.message.chat_id, action=ChatAction.UPLOAD_AUDIO)
        logger.info(f'Filename: {filename}')
        if file:
            logger.info('Sending file to user')
            update.message.reply_document(document=file)
            update.message.reply_text(f'ðŸ’¬ Tip: PodÃ©s usar [VLC]({VLC_LINK}) para reproducir el audio ðŸŽ¶',
                                      parse_mode='markdown', disable_web_page_preview=True)
            file.close()
            logger.info('File sent successfully')
    except NetworkError:
        logger.error('A network error occurred.', exc_info=True)
        update.message.reply_text(text='ðŸš€ Hay problemas de conexiÃ³n en estos momentos. IntentÃ¡ mas tarde..')
        send_message_to_admin(bot, f'Error mandando {video_url}, {filename}')

    except Exception:
        msg = 'Error uploading file to telegram servers'
        logger.exception(msg), send_message_to_admin(bot, msg)
        update.message.reply_text(text=msg)

    else:
        if filename:
            try:
                # Remove the file we just sent, as the name is hardcoded.
                logger.info(f"Removing file '{filename}'")
                os.remove(filename)
                logger.info('File removed')
            except FileNotFoundError:
                msg = f'Error removing audio file. File not found {filename}'
                logger.error(msg), send_message_to_admin(bot, msg)
            except Exception:
                msg = f"UnknownError removing audio file. '{filename}'"
                logger.error(msg), send_message_to_admin(bot, msg)

def get_audio_file(exts):
    """Youtube-dl does not return file data on success, so we must guess the file extension"""
    possible_filenames = (f"{FILENAME}.{ext}" for ext in exts)
    for filename in possible_filenames:
        try:
            return open(filename, 'rb'), filename
        except FileNotFoundError:
            continue

    return None, None

yt_handler = CommandHandler('yttomp3', youtube_to_mp3, pass_args=True)
yt_handler_alt = CommandHandler('y', youtube_to_mp3, pass_args=True)


import glob
import time
from evdev import InputDevice, categorize, ecodes
from pishowerutils import logger

class UsbRfidReader:
    """
    Class for reading out RFID transponders via USB HID readers

    Opens USB HID devices like simple RFID readers for EM4100 and
    Mifare NFC transponders and reads the received transponder
    serial number.

    Supported are Linux systems and any card/code readers registered
    as HID devices under /dev/input/event[0..n].

    WARNING:
    File read permissions to the devices are needed - check your
    user's groups! (e.g. for membership of group 'input')
    """

    __inputDevice = None
    __inputDevicePath = ''
    __deviceOpenRetries = 0
    __minCodeLength = 8
    __scancodes = {
        # Scancode: ASCIICode
        0: None, 1: u'ESC', 2: u'1', 3: u'2', 4: u'3', 5: u'4', 6: u'5',
        7: u'6', 8: u'7', 9: u'8', 10: u'9', 11: u'0', 12: u'-', 13: u'=',
        14: u'BKSP', 15: u'TAB', 16: u'Q', 17: u'W', 18: u'E', 19: u'R',
        20: u'T', 21: u'Y', 22: u'U', 23: u'I', 24: u'O', 25: u'P', 26: u'[',
        27: u']', 28: u'CRLF', 29: u'LCTRL', 30: u'A', 31: u'S', 32: u'D',
        33: u'F', 34: u'G', 35: u'H', 36: u'J', 37: u'K', 38: u'L', 39: u';',
        40: u'"', 41: u'`', 42: u'LSHFT', 43: u'\\', 44: u'Z', 45: u'X',
        46: u'C', 47: u'V', 48: u'B', 49: u'N', 50: u'M', 51: u',', 52: u'.',
        53: u'/', 54: u'RSHFT', 56: u'LALT', 100: u'RALT'
    }

    def __init__(self, devicePath='', minCodeLength=8):
        """
        Init the input device object

        Parameter 'devicePath' for input event device like
        '/dev/input/event0' is optional. If empty all available
        input devices are tried.
        """
        self.__minCodeLength = minCodeLength
        self.__inputDevicePath = devicePath
        self.open()

    def __del__(self):
        """ Destroy class and handle for input device """
        self.close()

    def open(self):
        """
        Open an input event device for reading

        Tries to open the device path given to the constructor
        or search for all available input event devices.
        """
        device_paths = []
        if len(self.__inputDevicePath):
            device_paths = ["{0}".format(self.__inputDevicePath)]
        else:
            device_paths = glob.glob('/dev/input/event*')  # search all available devices
        for path in device_paths:
            try:
                self.close()
                self.__inputDevice = InputDevice(path)
                self.__deviceOpenRetries = 0
                self.__inputDevicePath = path
                logger.debug('Opened RFID reader device \'{0}\''.format(path))
                time.sleep(1.0)  # prevent timing issues
                return
            except:
                if self.__deviceOpenRetries < 1:
                    logger.error('Unable to open USB RFID reader \'{0}\'!'.format(path))
                self.__inputDevice = None
                self.__deviceOpenRetries += 1

    def close(self):
        """ Close the input device and wait 1 second """
        try:
            if self.__inputDevice is not None:
                self.__inputDevice.close()
                self.__inputDevice = None
                logger.debug('RFID input device closed')
                time.sleep(1.0)  # prevent timing issues
        except:
            logger.warning('Error while closing RFID input device!')

    def is_open(self):
        """ Returns True, if connection to RFID reader is open """
        return self.__inputDevice is not None

    def readline(self):
        """ Try reading a code from the input device and return it as string """
        if self.__inputDevice is None:
            self.open()
            if self.__inputDevice is None:
                return None
        line = ''
        try:
            while True:
                event = self.__inputDevice.read_one()
                if event is None and line == '':
                    # There are blank events in between characters, so we don't want
                    # to break if we've started reading them
                    break  # start a new read.
                if event is None or event.type != ecodes.EV_KEY:
                    continue
                data = categorize(event)
                # catch only keyup, and not Enter
                if data.keystate == 0 and data.scancode != 42:
                    if data.scancode == 28:
                        # looking return key to be pressed
                        if len(line) < self.__minCodeLength:
                            logger.warning('ignoring to small code: {0}'.format(line))
                            break
                        else:
                            logger.debug('code read: \'{0}\''.format(line))
                            return line
                    else:
                        line += self.__scancodes[data.scancode]
        except:
            logger.error('Parsing input stream failed!')
            self.close()
        return None

# command line execution
if __name__ == '__main__':
    rfidReader = UsbRfidReader('/dev/input/event2', 10)
    while True:
        try:
            code = rfidReader.readline()
            if code is not None:
                logger.debug('found a code: \'{0}\''.format(code))
            time.sleep(0.5)
        except KeyboardInterrupt:
            logger.debug('Key pressed - finishing now...')
            break
        except:
            logger.error('Unknown error received. Reading aborted!')
            break


"""
Implementation of Linear Programming IRL by Ng and Russell, 2000
(c) 2018 Aaron Snoswell
"""

import math
import copy
import numpy as np

from cvxopt import matrix, solvers

def lp_irl(T, gamma, l1, *, Rmax=1.0):
    """
    Implements Linear Programming IRL by NG and Russell, 2000

    Given a transition matrix T(s, a, s') encoding a stationary, deterministic
    policy and a discount factor gamma finds a reward vector R(s) for which
    the policy is optimal.

    This method uses the Linear Programming IRL algorithm by Ng and Russell,
    2000 (http://ai.stanford.edu/~ang/papers/icml00-irl.pdf). See
    https://www.inf.ed.ac.uk/teaching/courses/rl/slides17/8_IRL.pdf for a more
    accessible overview.

    @param T - A sorted transition matrix T(s, a, s') encoding a stationary
        deterministic policy. The structure of T must be that the 0th action
        T[:, 0, :] corresponds to the expert policy, and T[:, i, :], i != 0
        corresponds to the ith non-expert action at each state
    @param gamma - The expert's discount factor
    @param l1 - L1 regularization weight for LP optimisation objective
        function
    @param Rmax - Maximum reward value

    @return A reward vector for which the given policy is optimal
    @return A result object from the optimiser

    TODO: Adjust L1 norm constraint generation to allow negative rewards in
    the final vector.
    """

    # Measure size of state and action sets
    n = T.shape[0]
    k = T.shape[1]

    # Compute the discounted transition matrix term
    T_disc_inv = np.linalg.inv(np.identity(n) - gamma * T[:, 0, :])

    # Formulate the linear programming problem constraints
    # NB: The general form for adding a constraint looks like this
    # c, A_ub, b_ub = f(c, A_ub, b_ub)

    # Prepare LP constraint matrices
    c = np.zeros(shape=[1, n], dtype=float)
    A_ub = np.zeros(shape=[0, n], dtype=float)
    b_ub = np.zeros(shape=[0, 1])

    def add_optimal_policy_constraints(c, A_ub, b_ub):
        """
        Add constraints to ensure the expert policy is optimal
        This will add (k-1) * n extra constraints
        """
        for i in range(k - 1):
            constraint_rows = -1 * (T[:, 0, :] - T[:, i, :]) @ T_disc_inv
            A_ub = np.vstack((A_ub, constraint_rows))
            b_ub = np.vstack((b_ub, np.zeros(shape=[constraint_rows.shape[0], 1])))
        return c, A_ub, b_ub

    def add_costly_single_step_constraints(c, A_ub, b_ub):
        """
        Augment the optimisation objective to add the costly-single-step
        degeneracy heuristic
        This will add n extra optimisation variables and (k-1) * n extra
        constraints
        NB: Assumes the true optimisation variables are first in the objective
        function
        """

        # Expand the c vector add new terms for the min{} operator
        c = np.hstack((c, -1 * np.ones(shape=[1, n])))
        css_offset = c.shape[1] - n
        A_ub = np.hstack((A_ub, np.zeros(shape=[A_ub.shape[0], n])))

        # Add min{} operator constraints
        for i in range(k - 1):
            # Generate the costly single step constraint terms
            constraint_rows = -1 * (T[:, 0, :] - T[:, i, :]) @ T_disc_inv

            # constraint_rows is nxn - we need to add the min{} terms though
            min_operator_entries = np.identity(n)

            # And we have to make sure we put the min{} operator entries in
            # the correct place in the A_ub matrix
            num_padding_cols = css_offset - n
            padding_entries = np.zeros(shape=[constraint_rows.shape[0], num_padding_cols])
            constraint_rows = np.hstack((constraint_rows, padding_entries, min_operator_entries))

            # Finally, add the new constraints
            A_ub = np.vstack((A_ub, constraint_rows))
            b_ub = np.vstack((b_ub, np.zeros(shape=[constraint_rows.shape[0], 1])))

        return c, A_ub, b_ub

    def add_l1norm_constraints(c, A_ub, b_ub, l1):
        """
        Augment the optimisation objective to add an l1 norm regularisation
        term z += l1 * ||R||_1
        This will add n extra optimisation variables and 2n extra constraints
        NB: Assumes the true optimisation variables are first in the objective
        function
        """

        # We add an extra variable for each each true optimisation variable
        c = np.hstack((c, l1 * np.ones(shape=[1, n])))
        l1_offset = c.shape[1] - n

        # Don't forget to resize the A_ub matrix to match
        A_ub = np.hstack((A_ub, np.zeros(shape=[A_ub.shape[0], n])))

        # Now we add 2 new constraints for each true optimisation variable to
        # enforce the absolute value terms in the l1 norm
        for i in range(n):

            # An absolute value |x1| can be enforced via constraints
            # -x1 <= 0             (i.e., x1 must be positive or 0)
            #  x1 + -xe1 <= 0
            # Where xe1 is the replacement for |x1| in the objective
            #
            # TODO ajs 04/Apr/2018 This enforces that R must be positive or 0,
            # but I was under the impression that it was also possible to
            # enforce an abs operator without this requirement - e.g. see
            # http://lpsolve.sourceforge.net/5.1/absolute.htm
            constraint_row_1 = [0] * A_ub.shape[1]
            constraint_row_1[i] = -1
            A_ub = np.vstack((A_ub, constraint_row_1))
            b_ub = np.vstack((b_ub, [[0]]))

            constraint_row_2 = [0] * A_ub.shape[1]
            constraint_row_2[i] = 1
            constraint_row_2[l1_offset + i] = -1
            A_ub = np.vstack((A_ub, constraint_row_2))
            b_ub = np.vstack((b_ub, [[0]]))

        return c, A_ub, b_ub

    def add_rmax_constraints(c, A_ub, b_ub, Rmax):
        """
        Add constraints for a maximum R value Rmax
        This will add n extra constraints
        """
        for i in range(n):
            constraint_row = [0] * A_ub.shape[1]
            constraint_row[i] = 1
            A_ub = np.vstack((A_ub, constraint_row))
            b_ub = np.vstack((b_ub, Rmax))
        return c, A_ub, b_ub

    # Compose LP optimisation problem
    c, A_ub, b_ub = add_optimal_policy_constraints(c, A_ub, b_ub)
    c, A_ub, b_ub = add_costly_single_step_constraints(c, A_ub, b_ub)
    c, A_ub, b_ub = add_rmax_constraints(c, A_ub, b_ub, Rmax)
    c, A_ub, b_ub = add_l1norm_constraints(c, A_ub, b_ub, l1)

    # Solve for a solution

    # NB: cvxopt.solvers.lp expects a 1d c vector
    from cvxopt import matrix, solvers
    res = solvers.lp(matrix(c[0, :]), matrix(A_ub), matrix(b_ub))

    def normalize(vals):
        """
        Helper function to normalize a vector to the range (0, 1)
        """
        min_val = np.min(vals)
        max_val = np.max(vals)
        return (vals - min_val) / (max_val - min_val)

    # Extract the true optimisation variables and re-scale
    rewards = Rmax * normalize(res['x'][0:n]).T

    return rewards, res

if __name__ == "__main__":

    def build_sorted_transition_matrix(S, A, T, pi):
        """
        Given a vector of states S, a vector of actions A, a transition matrix
        T(s-a, s') and a policy dictionary, builds a sorted transition matrix
        T(s, a, s'), where the 0th action T(:, 0, :) corresponds to the expert
        policy, and the ith action T(:, i, :), i!=0 corresponds to the ith non
        -expert action at each state
        """

        # Build the compact form Transition matrix
        n = len(S)
        k = len(A)

        # Helper function to get a transition probability
        si = lambda s: S.tolist().index(s)
        ai = lambda a: A.tolist().index(a)
        trans = lambda s1, a, s2: T[si(s1) * k + ai(a), si(s2)]

        Tfull = np.zeros(shape=[n, k, n])
        for s_from in S:
            # Build vector of actions, sorted with the expert one first
            expert_action = pi[s_from]
            sorted_actions = np.append([expert_action], np.delete(A, ai(expert_action)))
            for a in sorted_actions:
                for s_to in S:
                    Tfull[si(s_from), ai(a), si(s_to)] = trans(s_from, a, s_to)

        return Tfull

    # Sample problem for lp_irl
    T = build_sorted_transition_matrix(
        np.array(["s0", "s1", "s2"]),
        np.array(["b", "o"]),
        np.array([[0,    0.4, 0.6 ],
                  [0,    0,   1   ],
                  [0,    0,   1   ],
                  [0,    0,   1   ],
                  [1,    0,   0   ],
                  [1,    0,   0   ]]),
        {
            "s0": "b",
            "s1": "o",
            "s2": "o"
        }
    )

    # Try LP IRL
    print(T)
    rewards, _ = lp_irl(T, 0.9, l1=10)
    print(rewards)


from PyQt5 import QtWidgets, QtCore, QtGui
from PyQt5.QtWidgets import *
from PyQt5.QtCore import pyqtSignal
from PyQt5.QtGui import QFont, QPixmap
import sys
import edit_section
import freeway_window
import home as primary
import section_vector

class Vehicle(QLabel):
    """
    Vehicle object is a QLabel that is clickable
    All vehicles shown on the map in GUI are of type Vehicle
    """
    def __init__(self,lane,lead,gap,model,r,g,b,parent=None):
        super(Vehicle, self).__init__(parent)
        self.parent_window = parent
        self.lane = lane
        self.lead = lead
        self.gap = gap
        self.model = model
        self.color_r = r
        self.color_g = g
        self.color_b = b
        self.position = -1
        self.initUI()

    def initUI(self):

        #set dimensions and style
        self.length = primary.height/22
        self.width = primary.width/65
        self.setMinimumHeight(primary.height/22)
        self.setMinimumWidth(primary.width/65)
        self.setMaximumHeight(primary.height/22)
        self.setMaximumWidth(primary.width/65)
        self.setAlignment(QtCore.Qt.AlignCenter)
        self.setFrameStyle(1)

        #calculate luminance to have border and text the correct color
        luminance = 0.2126 * int(self.color_r) + 0.7152 * int(self.color_g) + 0.0722 * int(self.color_b)
        text_color = "white"
        if luminance > 128:
            text_color = "black"

        #set font and style
        self.setFont(QFont("Arial",12))
        self.setStyleSheet("background:rgb({},{},{}); color:{};".format(self.color_r,self.color_g,self.color_b,text_color))

    #allow clickable
    clicked=pyqtSignal()
    def mouseReleaseEvent(self, ev):
        self.clicked.emit()

    def change_color(self,r,g,b):
        """
        connected: none
        function: Changes the vehicles color to the specified rgb values. This function is used in multiple files to change vehicle color
        """
        luminance = 0.2126 * int(r) + 0.7152 * int(g) + 0.0722 * int(b)
        text_color = "white"
        if luminance > 128:
            text_color = "black"
        self.setStyleSheet("background:rgb({},{},{}); color:{};".format(r,g,b,text_color))

def main():
    primary.main()

if __name__ == "__main__":
    main()


from PySide2 import QtWidgets, QtGui, QtCore
from pygears.conf import Inject, inject
from .node import NodeItem
from .pipe import Pipe
from .main_window import Shortcut
from functools import partial

snipe_shortcuts = []

def sniper():
    Sniper()

class SnipeCodeItem(QtWidgets.QGraphicsTextItem):
    def __init__(self, text, parent=None):
        super().__init__(text, parent=parent)
        f = self.font()
        f.setPointSize(12)
        f.setBold(True)
        self.setFont(f)

        # self.font().setPointSize(16)
        # self.font().setBold(True)
        # # self.setFont(self.font())
        # self.setFont(QtGui.QFont("Source Code Pro", 16))

    def paint(self, painter, option, widget):
        painter.setBrush(QtCore.Qt.white)
        painter.drawRect(self.boundingRect())
        super().paint(painter, option, widget)

class Sniper:
    @inject
    def __init__(self,
                 main=Inject('gearbox/main/inst')):

        Shortcut('graph', QtCore.Qt.Key_F, self.snipe_select)
        Shortcut('graph', QtCore.Qt.CTRL + QtCore.Qt.Key_F,
                 self.snipe_select_nodes)
        Shortcut('graph', QtCore.Qt.SHIFT + QtCore.Qt.Key_F,
                 self.snipe_select_pipes)

        self.main = main

    @inject
    def snipe_cancel(self, graph=Inject('gearbox/graph')):
        self.main.key_cancel.disconnect(self.snipe_cancel)
        for text, s in snipe_shortcuts:
            s.setParent(None)
            graph.scene().removeItem(text)

        self.main.change_domain('graph')
        snipe_shortcuts.clear()

    @inject
    def snipe_shot(self, pipe, graph=Inject('gearbox/graph')):
        graph.select(pipe)
        self.snipe_cancel()

    @inject
    def snipe_shot_prefix(self, prefix, graph=Inject('gearbox/graph')):
        import pdb
        pdb.set_trace()
        for text, s in snipe_shortcuts:
            if text.toPlainText().startswith(prefix):
                continue

            s.setParent(None)
            graph.scene().removeItem(text)

    def snipe_shot_prefix_test(self):
        print("Ambiguous")

    def get_visible_objs(self, node, objtype):
        if (objtype is None) or (objtype is NodeItem):
            for n in node._nodes:
                yield n
                if not n.collapsed:
                    yield from self.get_visible_objs(n, objtype)

        if (objtype is None) or (objtype is Pipe):
            for p in node.pipes:
                yield p

    def snipe_select_nodes(self):
        self.snipe_select(objtype=NodeItem)

    def snipe_select_pipes(self):
        self.snipe_select(objtype=Pipe)

    @inject
    def snipe_select(self, objtype=None, graph=Inject('gearbox/graph')):
        self.main.key_cancel.connect(self.snipe_cancel)

        nodes = graph.selected_nodes()

        nodes = [
            node.parent if node.collapsed else node for node in nodes
        ]

        if not nodes:
            pipes = graph.selected_pipes()
            if not pipes:
                nodes = [graph.top]
            else:
                nodes = [pipe.parent for pipe in pipes]

        keys = [6]

        for n in nodes:
            if n.collapsed:
                continue

            for i, obj in enumerate(self.get_visible_objs(n, objtype=objtype)):
                key_codes = [key + ord('A') for key in reversed(keys)]
                snipe_text = ''.join([chr(key).upper() for key in key_codes])
                text = SnipeCodeItem(snipe_text)

                text.setZValue(100)
                graph.scene().addItem(text)

                if isinstance(obj, NodeItem):
                    text.setPos(
                        obj.mapToScene((obj.boundingRect().bottomLeft() + obj.
                                        boundingRect().bottomRight()) / 2) -
                        QtCore.QPointF(text.boundingRect().width() / 2,
                                       text.boundingRect().height()))
                else:
                    text.setPos(
                        obj.mapToScene(obj.path().pointAtPercent(0.5)) -
                        QtCore.QPointF(text.boundingRect().width(),
                                       text.boundingRect().height()) / 2)

                self.main.change_domain('_snipe')
                shortcut = QtWidgets.QShortcut(
                    QtGui.QKeySequence(*key_codes), self.main)

                shortcut.activated.connect(partial(self.snipe_shot, obj))

                # if len(keys) > 0:
                #     shortcut.activatedAmbiguously.connect(
                #         self.snipe_shot_prefix_test)
                # shortcut.activatedAmbiguously.connect(
                #     partial(
                #         self.snipe_shot_prefix, prefix=snipe_text[:-1]))

                snipe_shortcuts.append((text, shortcut))

                for pos in range(len(keys) + 1):
                    if pos == len(keys):
                        keys.insert(0, 0)
                        break

                    keys[pos] += 1

                    if keys[pos] < 25:
                        break
                    else:
                        keys[pos] = 0


from fastapi import FastAPI
from repository import scrappy_data
from modules import CityModel

app = FastAPI()

@app.get('/')
def index():
    return {'log':'weather[API]'}

@app.post('/get_city_weather')
def get_city_weather(city: str) -> CityModel:
    data = scrappy_data(city=city)
    city = CityModel(nome = data['location'], date=data['time'], statu=data['info'], temperatur=data['weather'])
    return city


import os
import webbrowser
from functools import cache

import imageio
from reportlab.graphics import renderPM
from svglib.svglib import svg2rlg
from utils import Log
from utils.xmlx import _

from hbd.core import bbox_utils
from hbd.draw.draw_line import DrawLine
from hbd.draw.draw_node import DrawNode

log = Log(__name__)

class Draw(DrawNode, DrawLine):
    def __init__(self, config, styler):
        self.config = config
        self.styler = styler

    @cache
    def get_t(self):
        return bbox_utils.get_t(self.styler, self.config.loc_list)

    def draw_nodes(self):
        t = self.get_t()
        nodes = []
        for label, (x, y) in self.config.node_idx.items():
            nodes.append(self.draw_node(label, x, y, t))
        return nodes

    def draw_lines(self):
        t = self.get_t()
        lines = []
        for line in self.config.line_idx.values():
            lines.append(self.draw_line(line, t))
        return lines

    def draw_rect_border(self):
        return _('rect', None, self.styler.rect_border)

    def draw_title(self):
        self.config.title
        return _('text', self.config.title, self.styler.text_title)

    def draw_subtitle(self):
        return _('text', self.config.subtitle, self.styler.text_subtitle)

    def draw_footer_text(self):
        footer_text = self.config.footer_text
        font_size = min(
            self.styler.text_footer_text['font_size'],
            self.styler.svg['width'] / len(footer_text),
        )
        return _(
            'text',
            footer_text,
            self.styler.text_footer_text | dict(font_size=font_size),
        )

    def draw_watermark(self):
        return _(
            'text',
            '@nuuuwan',
            self.styler.text_watermark,
        )

    def draw(self, png_path, do_open=True):
        svg = _(
            'svg',
            [
                self.draw_watermark(),
                self.draw_title(),
                self.draw_subtitle(),
                self.draw_footer_text(),
            ]
            + self.draw_lines()
            + self.draw_nodes(),
            self.styler.svg,
        )
        svg_path = png_path[:-3] + 'svg'
        svg.store(svg_path)
        log.debug(f'Saved {svg_path}')

        png_path = Draw.convert_svg_to_png(svg_path)
        if do_open:
            webbrowser.open(os.path.abspath(png_path))
        return png_path

    @staticmethod
    def convert_svg_to_png(svg_path):
        png_path = svg_path[:-3] + 'png'

        drawing = svg2rlg(svg_path)
        renderPM.drawToFile(drawing, png_path, fmt="PNG")
        log.info(f'Saved {png_path}')

        return png_path

    @staticmethod
    def build_animated_gif(png_path_list, gif_path):
        png_path_list.sort()
        last_png_path = png_path_list[-1]
        for i in range(0, 5):
            png_path_list.append(last_png_path)

        images = []
        for png_path in png_path_list:
            images.append(imageio.imread(png_path))
        DURATION = 55.70 / 24
        imageio.mimwrite(gif_path, images, duration=DURATION)
        log.info(f'Built {gif_path} (from {len(png_path_list)} png files)')
        webbrowser.open(os.path.abspath(gif_path))


# -*- coding: utf-8 -*-

import pandas as pd
import os
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
# from keras.callbacks import Callback
from sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score

import tokenization

import tensorflow as tf
import tensorflow_hub as hub
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.utils import shuffle
from sklearn.metrics import f1_score

# Importing argparse for command line arguments.
import argparse

# Parsing command line arguments.
parser = argparse.ArgumentParser(description='Train the proposed model. An example on how to run the script is as follows: \n python main.py --model=comemnet-bilstm')
parser.add_argument('-m','--model',default='comemnet-bilstm',  help="Can be either 'comemnet-lstm', 'comemnet-bilstm' or 'erin'.\n Example: model=comemnet-bilstm (default=comemnet-bilstm)")
parser.add_argument('-d','--demo', default=False, help="Boolean. Whether to create a gradio demo (default=False).")
parser.add_argument('-p', '--pretrained', default=False, help='Boolean. Whether to use pretrained model. (default=False). If pretrained=True, please download the google drive files mentioned in the README.')

args = vars(parser.parse_args())
print("You selected the following parameters for the script to run.")
print(args)

if args['demo']:
    import gradio as gr

# print("Using TensorFlow version: ",tf.__version__)

#checking for GPU device, otherwise model will be trained on CPU
device_name = tf.test.gpu_device_name()
print(device_name)
if device_name != '/device:GPU:0':
  print('GPU device not found, falling back to CPU. Warning, code will run slowly on CPU.')
print('Found GPU at: {}'.format(device_name))

#data is initially parsed from JSON files
#now preprocesssing the parsed data for model training
print("Loading and preprocessing data")
data_path = 'Processed Data/Input_500_29_relation.tsv'

train_data = pd.read_csv(data_path, encoding='utf-8', sep = '\t')

#filling nan values with empty strings
train_data.fillna("", inplace = True)

# Shuffle data so that there is a higher chance of the train and test data being from the same distribution.
train_data = shuffle(train_data, random_state = 1)

# # Now read the rows, convert them into strings and then only keep the unique ones.
sentences_and_labels =  np.array([[' '.join(map(str, row[:-1].tolist())).strip(), row[-1]] for row in train_data.iloc[:,:].values])

sentences = sentences_and_labels[:, 0]

#using label encoder to map relations to integer lablels
label = preprocessing.LabelEncoder()
y = label.fit_transform(train_data['relation'])
label_mappings = integer_mapping = {i: l for i, l in enumerate(label.classes_)}

def get_labels(y_pred):
    """
    This method generates integer labels from one-hot representations;
    Returns integer label values

    y_pred -- specifies one-hot labels generated by our model
    """
    y_pred_label = np.zeros((len(y_pred), 1))
    print(y_pred_label.shape)
    for index in range(len(y_pred)):
        y_pred_label[index] = np.argmax(y_pred[index])
    return y_pred_label

#if pretrained is set to False, then initialize a new model and train it
if args['pretrained'] == False:
    print("Retraining model from scratch.")
    print("Loading bert from TensorFlow-Hub, this may take a while... Please be patient.")
    m_url = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2'
    bert_layer = hub.KerasLayer(m_url, trainable=False)

    vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
    do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
    tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)

    def bert_encode(texts, tokenizer, max_len=512):
        all_tokens = []
        all_masks = []
        all_segments = []

        for text in texts:
            text = tokenizer.tokenize(text)

            text = text[:max_len-2]
            input_sequence = ["[CLS]"] + text + ["[SEP]"]
            pad_len = max_len-len(input_sequence)

            tokens = tokenizer.convert_tokens_to_ids(input_sequence) + [0] * pad_len
            pad_masks = [1] * len(input_sequence) + [0] * pad_len
            segment_ids = [0] * max_len

            all_tokens.append(tokens)
            all_masks.append(pad_masks)
            all_segments.append(segment_ids)

        return np.array(all_tokens), np.array(all_masks), np.array(all_segments)

    def build_model_erin(bert_layer, max_len=512):
        """
        This method reimplements Macdonald and Barbosa's model;
        Use the reported parameters from their paper to ensure fair comparison;
        Returns model object

        bert_layer -- specifies the pretrained BERT layer for generating embedding vectors
        max_len --  specifies maximum length of input tokens for BERT layer
        """
        input_word_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
        input_mask = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
        segment_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")

        pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])

        #print(tf.shape(sequence_output))
        clf_output = sequence_output[:, :, :]
        #print(tf.shape(clf_output))

        #build model with one LSTM layer, followed by fully connected and softmax layers
        lay = tf.keras.layers.LSTM(1, return_sequences=True)(clf_output)
        lay = tf.keras.layers.Flatten()(lay)
        out = tf.keras.layers.Dense(29, activation='softmax')(lay)

        model = tf.keras.models.Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
        #utilize RMSProp optimizer as reported by Macdonald and Barbosa
        model.compile(tf.keras.optimizers.RMSprop(lr=0.001), loss='sparse_categorical_crossentropy', metrics=['accuracy'])

        return model

    def build_model_comemnet_lstm(bert_layer, max_len=512):
        """
        This method implements our baseline model;
        baseline model comprises of a CNN layer and an LSTM layer;
        Returns model object

        bert_layer -- specifies the pretrained BERT layer for generating embedding vectors
        max_len --  specifies maximum length of input tokens for BERT layer
        """
        input_word_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
        input_mask = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
        segment_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")

        pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
        # np.savez_compressed(f"bert_sequence_op_seed_{seed}.npz", sequence_output)

        #print(tf.shape(sequence_output))
        clf_output = sequence_output[:, :, :]
        #print(tf.shape(clf_output))

        #build model with one CNN layer with 8 filters and one LSTM layer with 8 hidden units
        lay = tf.keras.layers.Conv1D(filters=8, kernel_size=5, strides=1, padding="same", activation="relu")(clf_output)
        lay = tf.keras.layers.MaxPooling1D(2, 2)(lay)
        lay = tf.keras.layers.LSTM(8, return_sequences=True, dropout=0.2)(lay)
        lay = tf.keras.layers.Flatten()(lay)
        out = tf.keras.layers.Dense(29, activation='softmax')(lay)

        model = tf.keras.models.Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
        #use sparse categorical crossentropy as output labels are defined as integer values
        model.compile(tf.keras.optimizers.Adam(lr=2e-5), loss='sparse_categorical_crossentropy', metrics=['accuracy'])

        return model

    def build_model_comemnet_bilstm(bert_layer, max_len=512):
        """
        This method is an improvement over our baseline;
        improved model includes a CNN and BiLSTM network;
        Returns model object

        bert_layer -- specifies the pretrained BERT layer for generating embedding vectors
        max_len --  specifies maximum length of input tokens for BERT layer
        """
        input_word_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
        input_mask = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
        segment_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")

        pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
        # np.savez_compressed(f"bert_sequence_op_seed_{seed}.npz", sequence_output)

        # print(tf.shape(sequence_output))
        clf_output = sequence_output[:, :, :]
        # print(tf.shape(clf_output))

        #improve our baseline model by adding BiLSTM layer for modeling dependencies in both directions
        lay = tf.keras.layers.Conv1D(filters=8, kernel_size=5, strides=1, padding="same", activation="relu")(clf_output)
        lay = tf.keras.layers.MaxPooling1D(2, 2)(lay)
        #implement dropout to prevent overfitting and ensure generalizability
        lay = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(8, return_sequences=True, dropout=0.2))(lay)
        lay = tf.keras.layers.Flatten()(lay)
        out = tf.keras.layers.Dense(29, activation='softmax')(lay)

        model = tf.keras.models.Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
        model.compile(tf.keras.optimizers.Adam(lr=2e-5), loss='sparse_categorical_crossentropy', metrics=['accuracy'])

        return model

    """### Obtaining Train, test splits.
    ###### In the train splits, we will have a separate validation split.
    """

    #specifying checkpoint directory so trained model is saved after each epoch
    print("Checkpoint will only be saved for the last epoch.")
    checkpoint_path = "training_relations/cp.ckpt"
    checkpoint_dir = os.path.dirname(checkpoint_path)

    """# Do not run the following cell if using checkpointed files."""

    """
    Perform evaluation using Monte-carlo CV;
    model was tested on 5 different seeds and all metrics were averaged across seeds for reporting final results;
    model hyperparameters was tuned using 5 fold CV on development set (20% of training set)
    """

    with tf.device(device_name):
        splits = 5 # For five fold cross-validation.
        seeds = [i for i in range(splits)]  # Fix the seed value for reproducibility.
        # seeds = [0]

        val_dict = {}
        test_dict = {}
        test_accs = []
        test_f1s = []

        # First get random train-test splits. Doesn't include validation, which will be obtained from the train set.
        for seed in seeds:
            print(f"Training model for seed {seed}.")
            x_t, x_test, y_t, y_test = train_test_split(sentences, y, random_state=seed, test_size=0.2)   # Global training and test sets.

            # Now get validation sets from each training set.
            # kf = KFold(n_splits=5, shuffle=False) # Setting shuffle=False because shuffled dataset already before.
            # fold_count = 0

            # for train_index, val_index in kf.split(x_t):
            #     #print(x_t.shape)
            #     #print(y_t.shape)
            #     x_train, x_val = x_t[train_index], x_t[val_index]   # Training and validation features.
            #     y_train, y_val = y_t[train_index], y_t[val_index]   # Training and validation labels.

            #     #encode train data
            #     max_len = 80
            #     train_input = bert_encode(x_train, tokenizer, max_len=max_len)
            #     train_labels = y_train
            #     x_val = bert_encode(x_val, tokenizer, max_len=max_len)

            #     model = build_model(bert_layer, max_len=max_len)
            #     model.summary()
            #     #checkpoint = tf.keras.callbacks.ModelCheckpoint('model.h5', monitor='val_accuracy', save_best_only=True, verbose=1)
            #     checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path, save_weights_only=True, verbose=1)
            #     earlystopping = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=5, verbose=1)
            #     train_sh = model.fit(
            #     train_input, train_labels,
            #     #validation_split=0.2,
            #     validation_data=(x_val, y_val),
            #     epochs=2,
            #     callbacks=[checkpoint, earlystopping],
            #     batch_size=16,
            #     verbose=1)

            #     # Validation sets can be used for hyperparamter tuning.
            #     val_dict[str(seed) + str(fold_count)] = train_sh.history
            #     fold_count += 1

            #encode whole train data
            #specify max length parameters for our models
            max_len = 80
            #Macdonald and Barbosa considered 50 max length for their proposed model
            #we preserve their reported hyperparameters for replicating their results
            erin_max_len = 50

            print("Encoding input through BERT encoder.")
            if args['model'] == 'erin':
                train_input = bert_encode(x_t, tokenizer, max_len=erin_max_len)
                train_labels = y_t
            else:
                train_input = bert_encode(x_t, tokenizer, max_len=max_len)
                train_labels = y_t

            #train model on whole train data
            if args['model'] == "comemnet-bilstm":
                model = build_model_comemnet_bilstm(bert_layer, max_len=max_len)
            elif args['model'] =='comemnet-lstm':
                model = build_model_comemnet_lstm(bert_layer, max_len=max_len)
            elif args['model'] == 'erin':
                model = build_model_erin(bert_layer, max_len=erin_max_len)

            model.summary()

            #checkpoint = tf.keras.callbacks.ModelCheckpoint('model.h5', monitor='val_accuracy', save_best_only=True, verbose=1)
            checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path, save_weights_only=True, verbose=1)
            earlystopping = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=5, verbose=1)
            train_sh = model.fit(
            train_input, train_labels,
            validation_split=0,
            epochs=6,
            callbacks=[checkpoint, earlystopping],
            batch_size=16,
            verbose=1)

            #encode test data
            if args['model'] == 'erin':
                test_input = bert_encode(x_test, tokenizer, max_len=erin_max_len)
            else:
                test_input = bert_encode(x_test, tokenizer, max_len=max_len)

            # Evaluate the model on the test data using `evaluate`
            print("Evaluating on test data for ", seed)
            results = model.evaluate(test_input, y_test, batch_size=16)

            #calculate F1-score
            y_pred = model.predict(test_input, verbose=1)
            y_pred_label = get_labels(y_pred)
            f1_value = f1_score(y_test, y_pred_label, average='macro')
            results.append(f1_value)
            test_f1s.append(f1_value)
            test_accs.append(results[1])
            print(f"Test loss, Test acc, F1-score for seed {seed}: ", results)

            test_dict[seed] = results
        print("Average accuracy: ", np.mean(test_accs))
        print("Average F1 score: ", np.mean(test_f1s))
        print("Results for all seed values: ", test_dict)

"""## The following section is for loading the checkpoint weights and then obtaining the confusion matrix."""

if args['pretrained']:
    print("Currently confusion matrix is only supported for CoMeMNet-BiLSTM")
    cms = []
    # #code for loading weights from checkpoint
    # with tf.device(device_name):
    #     results = []
    #     splits = 5 # For five fold cross-validation.
    #     #seeds = [i for i in range(splits)]  # Fix the seed value for reproducibility.
    #     seeds = [4]#0, 1, 2, 3, 4]
    #
    #     val_dict = {}
    #     test_dict = {}
    #
    #     # First get random train-test splits. Doesn't include validation, which will be obtained from the train set.
    #     for seed in seeds:
    #         x_t, x_test, y_t, y_test = train_test_split(sentences, y, random_state=seed, test_size=0.2)   # Global training and test sets.
    #
    #         # Evaluate the new model
    #         max_len = 80
    #         if args['model'] == "comemnet-bilstm":
    #             model = build_model_comemnet_bilstm(bert_layer, max_len=max_len)
    #         elif args['model'] == 'comemnet-lstm':
    #             model = build_model_comemnet_lstm(bert_layer, max_len=max_len)
    #         elif args['model'] == 'erin':
    #             model = build_model_erin(bert_layer, max_len=max_len)
    #         model.summary()
    #         # Loads the weights for new model
    #         checkpoint_path = f"/content/drive/Shareddrives/CMPUT 656 Data and Results/Result 29 Relations/Seed {seed}/training_relations/cp.ckpt"
    #         training_relations_path = f"/content/drive/Shareddrives/CMPUT 656 Data and Results/Result 29 Relations/Seed {seed}/training_relations/"
    #         # /content/drive/Shareddrives/CMPUT 656 Data and Results/Result 29 Relations/Seed 0/training_relations/cp.ckpt.data-00000-of-00001
    #
    #
    #         model.load_weights(checkpoint_path)
    #
    #         #encode test data
    #         test_input = bert_encode(x_test, tokenizer, max_len=max_len)
    #
    #         # Evaluate the model on the test data using `evaluate`
    #         print("Evaluate on test data for ", seed)
    #         # results = new_model.evaluate(test_input, y_test, batch_size=16)
    #
    #         #calculate F1-score
    #         y_pred = model.predict(test_input, batch_size=16, verbose=1)
    #         y_pred_label = get_labels(y_pred)
    #         f1_value = f1_score(y_test, y_pred_label, average='macro')
    #         np.savez_compressed(training_relations_path +  "predictions.npz", y_pred_label)
    #         cm = confusion_matrix(y_pred_label, y_test, labels=[i for i in label_mappings.keys()])
    #         cms.append(cm)
    #
    #         results.append(f1_value)
    #         print("Test loss, Test acc, F1-score:", results)

    for k, v in label_mappings.items():
        print(k, v)

    #mapping each relations to unique integer labels
    reduced_label_mappings = {
        0: 'None',
        1: 'award=nominee',
        2: 'author-works_written',
        3: 'book-genre',
        4: 'company-industry',
        5: 'person-graduate',
        6: 'actor-character',
        7: 'director-film',
        8: 'film-country',
        9: 'film-genre',
        10: 'film-language',
        11: 'film-music',
        12: 'film-production_company',
        13: 'actor-film',
        14: 'producer-film',
        15: 'writer-film',
        16: 'political_party-politician',
        17: 'location-contains',
        18: 'musician-album',
        19: 'musician-origin',
        20: 'person-place_of_death',
        21: 'person-nationality',
        22: 'person-parents',
        23: 'person-place_of_birth',
        24: 'person-profession',
        25: 'person-religion',
        26: 'person-spouse',
        27: 'football_position-player',
        28: 'sports_team-player'
    }

    """# Read in all the predictions for different seeds and make a confusion matrix by averaging them together.

    NOTE: The confusion matrix is normalized.
    """

    cms = []
    seeds = [0, 1, 2, 3, 4]
    for seed in seeds:
        training_relations_path = f"Result 29 Relations/Seed {seed}/training_relations/"
        x_t, x_test, y_t, y_test = train_test_split(sentences, y, random_state=seed, test_size=0.2)
        del x_t
        del x_test
        del y_t
        predictions = np.load(training_relations_path + "predictions.npz", allow_pickle=True)['arr_0'].tolist()
        cms.append(confusion_matrix(predictions, y_test, labels=[i for i in label_mappings.keys()]))

    plt.clf()
    averaged_cms = np.mean(cms, axis=0)
    averaged_cms = averaged_cms.astype('float') / averaged_cms.sum(axis=1)[:, np.newaxis]
    cms_df = pd.DataFrame(averaged_cms, index = [value for value in reduced_label_mappings.values()],
                          columns=[value for value in reduced_label_mappings.values()])
    fig, ax = plt.subplots(figsize=(15, 10))
    heat = sns.heatmap(cms_df, vmin=0.0, vmax=1.0, cbar_kws={'label': 'Normalized value'})
    heat.figure.axes[-1].yaxis.label.set_size(20)
    yticks = [i.upper() for i in cms_df.index]
    xticks = [i.upper() for i in cms_df.columns]
    plt.yticks(plt.yticks()[0], labels=yticks, rotation=0)
    plt.xticks(plt.xticks()[0], labels=xticks, rotation=270)
    plt.title("Confusion matrix for all 29 relations - CoMemNet-BiLSTM", fontsize=20)
    plt.show()

"""# Gradio app to demo the model."""

if args['demo']:
    print("Loading BERT encoder, this might take some time... Please be patient.")
    m_url = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2'
    bert_layer = hub.KerasLayer(m_url, trainable=False)

    def get_labels(y_pred):
        y_pred_label = np.zeros((len(y_pred),1))
        print(y_pred_label.shape)
        for index in range(len(y_pred)):
            y_pred_label[index] = np.argmax(y_pred[index])
        return y_pred_label

    vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
    do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
    tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)

    def bert_encode(texts, tokenizer, max_len=512):
        all_tokens = []
        all_masks = []
        all_segments = []

        for text in texts:
            text = tokenizer.tokenize(text)

            text = text[:max_len-2]
            input_sequence = ["[CLS]"] + text + ["[SEP]"]
            pad_len = max_len-len(input_sequence)

            tokens = tokenizer.convert_tokens_to_ids(input_sequence) + [0] * pad_len
            pad_masks = [1] * len(input_sequence) + [0] * pad_len
            segment_ids = [0] * max_len

            all_tokens.append(tokens)
            all_masks.append(pad_masks)
            all_segments.append(segment_ids)

        return np.array(all_tokens), np.array(all_masks), np.array(all_segments)

    def build_model(bert_layer, max_len=512):
        input_word_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
        input_mask = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
        segment_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")

        pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])

        # print(tf.shape(sequence_output))
        clf_output = sequence_output[:, :, :]
        # print(tf.shape(clf_output))

        lay = tf.keras.layers.Conv1D(filters=8, kernel_size=5, strides=1, padding="same", activation="relu")(clf_output)
        lay = tf.keras.layers.MaxPooling1D(2, 2)(lay)
        lay = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(8, return_sequences=True, dropout=0.2))(lay)
        lay = tf.keras.layers.Flatten()(lay)
        out = tf.keras.layers.Dense(29, activation='softmax')(lay)

        model = tf.keras.models.Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
        model.compile(tf.keras.optimizers.Adam(lr=2e-5), loss='sparse_categorical_crossentropy', metrics=['accuracy'])

        return model

    reduced_label_mappings = {
        0: 'None',
        1: 'award-nominee',
        2: 'author-works_written',
        3: 'book-genre',
        4: 'company-industry',
        5: 'person-graduate',
        6: 'actor-character',
        7: 'director-film',
        8: 'film-country',
        9: 'film-genre',
        10: 'film-language',
        11: 'film-music',
        12: 'film-production_company',
        13: 'actor-film',
        14: 'producer-film',
        15: 'writer-film',
        16: 'political_party-politician',
        17: 'location-contains',
        18: 'musician-album',
        19: 'musician-origin',
        20: 'person-place_of_death',
        21: 'person-nationality',
        22: 'person-parents',
        23: 'person-place_of_birth',
        24: 'person-profession',
        25: 'person-religion',
        26: 'person-spouse',
        27: 'football_position-player',
        28: 'sports_team-player'
    }

    seed = 0
    print(f"Using model checkpoint file for seed {seed}")
    checkpoint_path = f"Result 29 Relations/Seed {seed}/training_relations/cp.ckpt"
    max_len = 80

    new_model = build_model(bert_layer, max_len=max_len)
    new_model.load_weights(checkpoint_path)

    def predict_relation(sentence):
        print("Encoding input through BERT encoder.")
        test_input = bert_encode(np.array([sentence]), tokenizer, max_len=max_len)
        y_pred = new_model.predict(test_input, batch_size=16, verbose=1)
        y_pred_label = get_labels(y_pred)
        relations = [rel for rel in reduced_label_mappings.values()]
        probabilities = y_pred.tolist()[0]
        result_dict = {}
        for k, v in zip(relations, probabilities):
            result_dict[k] = v
        return result_dict

    iface = gr.Interface(
        predict_relation,
        inputs="text",
        outputs="label",
        interpretation="default",
        title="CoMemNet - Relation Extractor", description="NOTE: Model is trained on Wikipedia table data and not continuous text."
    )

    iface.launch(debug=False)


# -*- coding: utf-8 -*-
"""
Created on Mon May 27 15:16:45 2019
@author: weiho
"""
import os
'''
#Uses GPU 0
os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
os.environ['CUDA_VISIBLE_DEVICES'] = '1'
'''
import csv
import keras
from keras.applications import VGG16, ResNet50, InceptionV3
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.models import Sequential, load_model, Model
from keras.preprocessing.image import ImageDataGenerator
from keras.layers import Input, Dense, Activation, Flatten, Dropout, BatchNormalization
from keras.layers import Conv2D, MaxPooling2D
from keras.utils import multi_gpu_model
from keras import regularizers, optimizers
from keras import backend as K
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import sys
import tensorflow as tf

from generators import categorical_generator, make_batch, test_generator
from aux_func import numImgs, allImgs, get_freq_dict

#model_type = vgg16, resnet, inception
#y_classes = categories used during training
#Trains a model on data from train_df_path, cross-validating with val_df_path
#Saves model at experiment_path/model_save_name.h5, class indices at experiment_path/model_save_name:indices.txt
#Accuracy at experiment_path/accuracy.png, loss at experiment_path/loss.png
def train_categorical(gpus, experiment_path, train_df_path, train_image_path, val_df_path, val_image_path, y_name, y_classes, augment_number,
                      model_type, model_save_name, epoch_number, learning_rate, decay, regularization, batch_size=32, img_height=224, img_width=224):

    checkpoint = ModelCheckpoint(os.path.join(experiment_path, model_save_name+'.h5'), monitor='val_loss', verbose=1, save_best_only=False, save_weights_only=False, mode='auto', period=25)
    callbacks = [checkpoint]

    #Session variables
    traindf = pd.read_csv(train_df_path)
    valdf = pd.read_csv(val_df_path)

    freq_dict = get_freq_dict(traindf, y_name, y_classes)
    num_labels = len(y_classes)

    numTrainImgs = numImgs(traindf, y_name, y_classes)
    numValImgs = numImgs(valdf, y_name, y_classes)

    class_weight = list(map(lambda x: 1/freq_dict[x], y_classes))

    #Generators
    train_generator = make_batch(generator='categorical',
                                 dataframe=traindf,
                                 image_path=train_image_path,
                                 y_name=y_name,
                                 y_classes=y_classes,
                                 batch_size=batch_size,
                                 augment_number=augment_number,
                                 height=img_height,
                                 width=img_width,
                                 forTrain=True
                                 )

    val_generator = make_batch(generator='categorical',
                               dataframe=valdf,
                               image_path=val_image_path,
                               y_name=y_name,
                               y_classes=y_classes,
                               batch_size=batch_size,
                               height=img_height,
                               width=img_width,
                               forTrain=False
                               )

    #Model
    if model_type == 'vgg16':

        net_input = Input(shape=(img_height, img_width, 3))
        conv_base = VGG16(
            weights='imagenet',
            include_top=False,
            input_shape=(img_height, img_width, 3))(net_input)
        conv_base.trainable = False
        flatten1 = Flatten()(conv_base)
        dropout1 = Dropout(0.5)(flatten1)
        dense1 = Dense(
            num_labels,
            activation='softmax',
            kernel_regularizer=regularizers.l2(regularization))(dropout1)
        with tf.device('/cpu:0'):
            model = Model(inputs=net_input, outputs=dense1)

    elif model_type == 'resnet':

        net_input = Input(shape=(img_height, img_width, 3))
        conv_base = ResNet50(
            weights='imagenet',
            include_top=False,
            input_shape=(img_height, img_width, 3))(net_input)
        conv_base.trainable = False
        flatten1 = Flatten()(conv_base)
        dropout1 = Dropout(0.5)(flatten1)
        dense1 = Dense(
            num_labels,
            activation='softmax',
            kernel_regularizer=regularizers.l2(regularization))(dropout1)
        with tf.device('/cpu:0'):
            model = Model(inputs=net_input, outputs=dense1)

    elif model_type == 'inception':

        net_input = Input(shape=(img_height, img_width, 3))
        conv_base = InceptionV3(
            weights='imagenet',
            include_top=False,
            input_shape=(img_height, img_width, 3))(net_input)
        conv_base.trainable = False
        flatten1 = Flatten()(conv_base)
        dropout1 = Dropout(0.5)(flatten1)
        dense1 = Dense(
            num_labels,
            activation='softmax',
            kernel_regularizer=regularizers.l2(regularization))(dropout1)
        with tf.device('/cpu:0'):
            model = Model(inputs=net_input, outputs=dense1)

    else:
        print('\nError in train_categorical: invalid model type\n')

    multi_model = multi_gpu_model(model, gpus=gpus)
    model.summary()

    #Train
    multi_model.compile(optimizers.RMSprop(lr=learning_rate, decay=decay), loss='categorical_crossentropy', metrics=['accuracy'])

    history = multi_model.fit_generator(generator=train_generator,
                                        steps_per_epoch=numTrainImgs//batch_size,
                                        validation_data=val_generator,
                                        validation_steps=numValImgs,
                                        class_weight=class_weight,
                                        epochs=epoch_number,
                                        verbose=1,
                                        callbacks=callbacks
                                        )

    #Save model parameters
    model.save(os.path.join(experiment_path, model_save_name+'.h5'))

    #Save model index meanings
    with open(os.path.join(experiment_path, model_save_name+':indices.txt'), mode='w', encoding='utf-8') as filewrite:
        csv_writer = csv.writer(filewrite, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
        csv_writer.writerow(['class', 'frequency', 'index'])
        for i in range(len(y_classes)):
            csv_writer.writerow([y_classes[i], freq_dict[y_classes[i]], i])

    #Plots
    acc = history.history['acc']
    val_acc = history.history['val_acc']
    loss = history.history['loss']
    val_loss = history.history['val_loss']
    epochs = range(len(acc))

    plt.figure()
    plt.plot(epochs, acc, 'bo', label='Training accuracy')
    plt.plot(epochs, val_acc, 'r*', label='Validation accuracy')
    plt.title('Training and validation accuracy')
    plt.ylabel('Loss')
    plt.xlabel('Epoch')
    plt.legend()
    plt.savefig(os.path.join(experiment_path, 'accuracy.png'))

    plt.figure()
    plt.plot(epochs, loss, 'bo', label='Training loss')
    plt.plot(epochs, val_loss, 'r*', label='Validation loss')
    plt.title('Training and validation loss')
    plt.ylabel('Loss')
    plt.xlabel('Epoch')
    plt.legend()
    plt.savefig(os.path.join(experiment_path, 'loss.png'))

#experiment_path/model_test_name/model_test_name.h5 = model being evaluated, experiment_path/model_test_name/model_test_name:indices.txt = class indices
#Evaluates performance of model_test_name on test_df_path
#Creates test results file at experiment_path/model_test_name/model_test_name:test_results.txt
#Creates summary of test results at experiment_path/model_test_name/model_test_name:test_summary.txt
#Returns accuracies by class
def eval_categorical(experiment_path, test_df_path, test_image_path, y_name, model_test_name, height=224, width=224):

    #Session variables
    testdf = pd.read_csv(test_df_path)
    labelindices = pd.read_csv(os.path.join(experiment_path, model_test_name+':indices.txt'))
    y_classes = list(labelindices.loc[:, 'class'])
    print('Classes in model ' + model_test_name + ': ' + str(y_classes))
    freq_dict = get_freq_dict(testdf, y_name, y_classes)

    #PictureIDs, objectID, categoryNames, and true labels by image
    objpics = np.empty([0, 4], dtype=np.int32)
    for i in range(len(testdf)):
        if testdf.iloc[i].loc[y_name] in y_classes:
            for pic in testdf.iloc[i].loc['picIDs'].split(','):
                objpics = np.append(objpics, [[pic, testdf.iloc[i].loc['objectID'], testdf.iloc[i].loc['categoryName'], testdf.iloc[i].loc[y_name]]], axis=0)
    objpicsdf = pd.DataFrame(objpics, columns=['picID', 'objectID', 'categoryName', 'truth'], index=None)

    #Load model
    model = load_model(os.path.join(experiment_path, model_test_name+'.h5'))

    #Test generator
    eval_generator = test_generator(dataframe=testdf,
                                    image_path=test_image_path,
                                    y_name=y_name,
                                    y_classes=y_classes,
                                    include_NA=False,
                                    height=height,
                                    width=width
                                    )

    labeledExamples = numImgs(testdf, y_name, y_classes=y_classes)

    testresults = model.predict_generator(generator=eval_generator,
                                          steps=labeledExamples,
                                          verbose=1)

    #Probabilities of each class by image
    guessprobs = pd.DataFrame(testresults, index=None)
    guessprobs.columns = y_classes

    #Highest probability class by image
    guessesdf = np.argmax(testresults, axis=1)
    guessesdf = pd.DataFrame(guessesdf, columns=['pred'], index=None)

    #Test results file
    resultsdf = objpicsdf.join(guessesdf.join(guessprobs))
    resultsdf.to_csv(os.path.join(experiment_path, model_test_name+':test_results.txt'), encoding='utf-8', index=None, na_rep='NA')

    #Top k-accuracies by individual image
    k_accuracies = np.zeros(len(y_classes))

    #Confusion matrix
    #[i, j] = n : label j is the nth most likely label given by the model to images with true label i
    confusion_matrix = np.zeros((len(y_classes), len(y_classes)))

    for i in range(len(resultsdf)):
        pic_truth = resultsdf.iloc[i].loc['truth']
        truthindex = y_classes.index(pic_truth)
        guessprob = list(guessprobs.iloc[i])
        rankguesses = list(map(lambda x : guessprob.index(x), sorted(guessprob, reverse=True)))
        for guess in range(len(rankguesses)):
            confusion_matrix[truthindex, rankguesses[guess]] += guess+1
        for k in range(1, len(y_classes)+1):
            if y_classes.index(pic_truth) in rankguesses[:k]:
                k_accuracies[k-1] += 1

    k_accuracies /= labeledExamples
    for i in range(confusion_matrix.shape[0]):
        confusion_matrix[i] /= freq_dict[y_classes[i]]

    #Write summary of test results
    with open(os.path.join(experiment_path, model_test_name+':test_summary.txt'), 'w') as summary_writefile:
        #k-accuracies per image
        summary_writefile.write('k-accuracies per image: \n')
        for k in range(len(k_accuracies)):
            summary_writefile.write(str(k+1) + '-accuracy: ' + str(k_accuracies[k]) + '\n')
        summary_writefile.write('\n')

        #Confusion matrix
        summary_writefile.write('Confusion matrix: nth most likely label to be assigned\n')
        summary_writefile.write('Classes: ' + str(y_classes) + '\n')
        for i in range(confusion_matrix.shape[0]):
            summary_writefile.write(y_classes[i] + ': ' + str(confusion_matrix[i]) + '\n')
        summary_writefile.write('\n')

    #Return accuracies by class
    return k_accuracies

#experiment_path/model_test_name/model_test_name.h5 = model being evaluated, experiment_path/model_test_name/model_test_name:indices.txt = class indices
#Uses model_test_name to predict classes of images in test_df_path
#Creates test results file at experiment_path/model_test_name/model_test_name:results_save_name.txt
#Returns accuracies by class
def predict_categorical(experiment_path, test_df_path, test_image_path, model_test_name, results_save_name, height=224, width=224):

    #Session variables
    testdf = pd.read_csv(test_df_path)
    labelindices = pd.read_csv(os.path.join(experiment_path, model_test_name+':indices.txt'))
    y_classes = list(labelindices.loc[:, 'class'])
    print('Classes in model ' + model_test_name + ': ' + str(y_classes))

    #PictureIDs and objectID for each image
    objpics = np.empty([0, 2], dtype=np.int32)
    for i in range(len(testdf)):
        for pic in testdf.iloc[i].loc['picIDs'].split(','):
            objpics = np.append(objpics, [[pic, testdf.iloc[i].loc['objectID']]], axis=0)
    objpicsdf = pd.DataFrame(objpics, columns=['picID', 'objectID'], index=None)

    #Load model
    model = load_model(os.path.join(experiment_path, model_test_name+'.h5'))

    #Test generator
    predict_generator = test_generator(dataframe=testdf,
                                    image_path=test_image_path,
                                    y_name=None,
                                    y_classes=None,
                                    include_NA=True,
                                    height=height,
                                    width=width
                                    )

    totalExamples = allImgs(testdf, y_name=None, include_NA=True)

    testresults = model.predict_generator(generator=predict_generator,
                                          steps=totalExamples,
                                          verbose=1)

    #Probabilities of each class by image
    guessprobs = pd.DataFrame(testresults, index=None)
    guessprobs.columns = y_classes

    #Highest probability class by image
    guessesdf = np.argmax(testresults, axis=1)
    guessesdf = pd.DataFrame(guessesdf, columns=['pred'], index=None)

    #Predict results file
    resultsdf = objpicsdf.join(guessesdf.join(guessprobs))
    resultsdf.to_csv(os.path.join(experiment_path, model_test_name+':' + results_save_name + '.txt'), encoding='utf-8', index=None, na_rep='NA')


from enum import Enum
from machination.helpers import accepts
from machination.exceptions import InvalidArgumentValue

class StringifiedEnum(Enum):
    def __str__(self):
        return str(self.value)

class Architecture(StringifiedEnum):
    i386 = "i386"
    x64 = "x64"

    @staticmethod
    @accepts(str)
    def fromString(val):
        vals = {
                "i386" : Architecture.i386,
                "x64" : Architecture.x64
                }
        if val in vals:
            return vals[val]
        else:
            raise InvalidArgumentValue("Unknown architecture")


'''*****************************************************************************
Purpose: To extract insider trading data from sec website
This program extracts insider trading data from the sec website and stores it in
excel file.
Usage:
Enter a ticker (ex: 'AAPL, MSFT') or type 'all' to search thru all the tickers in file:

Here you can enter either single ticker or separate multiple tickers by comma or
type 'all' to go thru all the tickers in fName = 'ticker and cik.csv' (default name)

Enter the starting date (Ex: 2020-MM-DD): # enter date in this format no error checking

Would you like to extract data to excel file (Press enter for no OR enter filename):
Here either enter file name or press enter if you don't want to save the data.
By default the program doesn't display any data on shell, but you can add
print(all_df) or print(symbol_df) to print the data

Error checking:
I have implemented some error checking:
1.) If user enters incorrect ticker name ex: 'appl, msft, amzn, wrongone' in this
case program will skip 'appl' and 'wrongone' it will print out error msg with ticker
name and will continue to get insider data and save it to file for rest of the tickers.

2.) Let's say you are pulling the data for short period of time and there isn't much
info to calculate avg_purch/avg_sale and buy to sell ratio it will append rest
of the data to excel file.

-------------------------------------------------------------------
****************************************************************************'''
import requests
from bs4 import BeautifulSoup
import pandas as pd
import datetime
import csv

fName = 'ticker and cik.csv'    # set the ticker/CIK file name

data_dict, symbols = {}, []
with open(fName, 'r') as file:
    reader = csv.reader(file)
    for row in reader:
        data_dict[row[0]] = row[1]   # creating a dict ex. format: 'msft': '789019'
        symbols.append(row[0])

'''taking user input'''
# ticker
ticker = input("Enter a ticker (ex: 'AAPL, MSFT') or type 'all' to search thru all the tickers in file: ").replace(" ", "")
ticker = ticker.lower()
if ticker == 'all': symbols = symbols
else:
    tickers = ticker.split(",")
    symbols = tickers

# date
start = input("Enter the starting date (Ex: 2020-MM-DD): ")
start2 = datetime.datetime(int(start[:4]),int(start[5:7]),int(start[8:]))
end_date = datetime.date.today()

# extract to excel
extract = input("Would you like to extract data to excel file (Press enter for no OR enter filename): ")
print()

def transaction(url):
    '''gets the transaction report from url
    Parameter:    url: string
                       url for data extraction
    Return: trans_report: soup object
                       transaction report
    '''
    response = requests.get(url)
    web = response.content
    soup = BeautifulSoup(web, 'html.parser')
    trans_report = soup.find('table', {'id':'transaction-report'})
    return trans_report

def insiders(all_data, cik):
    '''gets the insider info of given cik number
    Parameter:    all_data: list
                       empty list to add data
                  cik: string
                       cik number
    Return:       all_data: list
                       list of all the data
                  headers: list
                       list of the headers "Acquistion or Disposition...."
    '''
    num = 0
    url = f'https://www.sec.gov/cgi-bin/own-disp?action=getissuer&CIK={cik}&type=&dateb=&owner=include&start={num}'

    urls, c = [], 0
    urls.append(url)
    for url in urls:
        c2, headers, data = 0, [], []
        report = transaction(url)
        for i in report.children:
            if i != '\n':
                collection = i.get_text().split('\n')
                c2 += 1
                for x in collection:
                    if x!= '':          # takng out empty data
                        if len(headers) != 12:  # 12 headers
                            headers.append(x)
                        else:
                            x = x.replace('$','')
                            # checking date boundary
                            if c == 1 and start > x:     # @ c==1, x is date
                                return  all_data, headers
                            elif (x == 'A' or x == 'D'):    # start of new row
                                if c != 0:
                                    all_data.append(data)
                                data, c = [], 0        # new row: clear data
                                data.append(x)
                                c += 1
                            else:
                                data.append(x)
                                c += 1

        if c2 == 81:
            all_data.append(data)
            num += 1
            url = f'https://www.sec.gov/cgi-bin/own-disp?action=getissuer&CIK={cik}&type=&dateb=&owner=include&start={num*80}'
            urls.append(url)
    return  all_data, headers

def data_df(all_df):
    '''gets the insider info of given cik number
    Parameter:    all_df: list
                       list of all the data frame
    Return:       None
    '''
    done = 0
    for symbol in symbols:
        all_data = []

        # key error handling
        try: cik = data_dict[symbol]
        except Exception as e:
            print(f"The symbol {e} is not valid, rest of the data will be saved to excel file (if available)")
            if symbol == symbols[-1]: break
            else: continue
        all_data, headers = insiders(all_data, cik)

        # adding to data frame
        df = pd.DataFrame.from_records(all_data, columns=headers)
        done += 1
        print(f"Finished extracting {symbol.upper()} insider data from {start} till {end_date}.")
        print(f"Finished: {done}/{len(symbols)} symbols.")

        df['Purchchase'] = pd.to_numeric(df['Acquistion or Disposition'].apply(lambda x: 1 if x == 'A' else 0) * df['Number of Securities Transacted'])
        df['Sale'] = pd.to_numeric(df['Acquistion or Disposition'].apply(lambda x: 1 if x == 'D' else 0) * df['Number of Securities Transacted'])

        name = 'Transaction Type'
        sell = df['Transaction Type'].str.count("S-Sale").sum()
        buy = df['Transaction Type'].str.count("P-Purchase").sum()

        sale = df['Acquistion or Disposition'] == 'D'
        purch = df['Acquistion or Disposition'] == 'A'
        num_purch = len(df[purch])
        num_sale = len(df[sale])
        total_sale = int(df['Sale'].sum(skipna=True))
        total_purch = int(df['Purchchase'].sum(skipna=True))

        # adding data to separate df for excel
        symbol_df = pd.DataFrame({'Symbol': symbol.upper(),
                                           '# of Purchases': num_purch,
                                           '# of Sales': num_sale,
                                           'Total Bought': f'{total_purch:,}',
                                           'Total Sold': f'{total_sale:,}',
                                           'S-Sale count': f'{sell}',
                                           'P-Purchase count': f'{buy}'},
                                            index = [1])

        # handling division by zero error/adding data to separate df for excel
        try:
            avg_sale = int(total_sale/num_sale)
            avg_purch = int(total_purch/num_purch)
            ratio = round(num_purch/num_sale, 2)    # purchase to sell ratio

        except ZeroDivisionError as e:
            print(f"\n{e} error for '{symbol}' there isn't much data available to calculate avg sale/purch & ratio, data will be added to excel without these metrics")

        else:
            symbol_df.insert(3, 'Buy/Sell Ratio', ratio)
            symbol_df.insert(6, 'Avg Shares Bought', avg_purch)
            symbol_df.insert(7, 'Avg Shares Sold', avg_sale)

        symbol_df.set_index('Symbol', inplace=True)
        all_df.append(symbol_df)

def excel(all_df):
    '''adds the data to excel sheet, data is saved in same floder as this file
    Parameter:    all_df: list
                       list of all the data frame
    Return:       None
    '''
    try:
        if len(extract) != 0:
            pd.concat(all_df).to_excel(extract +'.xlsx', index = True)
            print(f"Extracted the data to {extract}.xlsx\n")
    except Exception as e: print(e)

if __name__ == '__main__':
    all_df = []
    data_df(all_df)
    excel(all_df)


import requests
import json
import execjs  # 必须，需要先用pip 安装，用来执行js脚本
from urllib.parse import quote

# 用来判断是否需要打印日志
debug = False

class Py4Js:

    def __init__(self):
        self.ctx = execjs.compile("""
            function TL(a) {
                var k = "";
                var b = 406644;
                var b1 = 3293161072;
                var jd = ".";
                var $b = "+-a^+6";
                var Zb = "+-3^+b+-f";
                for (var e = [], f = 0, g = 0; g  m ? e[f++] = m : (2048 > m ? e[f++] = m >> 6 | 192 : (55296 == (m & 64512) && g + 1 < a.length && 56320 == (a.charCodeAt(g + 1) & 64512) ? (m = 65536 + ((m & 1023) << 10) + (a.charCodeAt(++g) & 1023),
                    e[f++] = m >> 18 | 240,
                    e[f++] = m >> 12 & 63 | 128) : e[f++] = m >> 12 | 224,
                    e[f++] = m >> 6 & 63 | 128),
                    e[f++] = m & 63 | 128)
                }
                a = b;
                for (f = 0; f < e.length; f++) a += e[f],
                a = RL(a, $b);
                a = RL(a, Zb);
                a ^= b1 || 0;
                0 > a && (a = (a & 2147483647) + 2147483648);
                a %= 1E6;
                return a.toString() + jd + (a ^ b)
            };
            function RL(a, b) {
                var t = "a";
                var Yb = "+";
                for (var c = 0; c = t ? d.charCodeAt(0) - 87 : Number(d),
                    d = b.charAt(c + 1) == Yb ? a >>> d: a << d;
                    a = b.charAt(c) == Yb ? a + d & 4294967295 : a ^ d
                }
                return a
            }
        """)

    def get_tk(self, text):
        return self.ctx.call("TL", text)

def build_url(text, tk, tl='zh-CN'):
    """
    需要用转URLEncoder
    :param text:
    :param tk:
    :param tl:
    :return:
    """
    return 'https://translate.google.cn/translate_a/single?client=webapp&sl=auto&tl=' + tl + '&hl=zh-CN&dt=at&dt=bd&dt=ex&dt=ld&dt=md&dt=qca&dt=rw&dt=rm&dt=ss&dt=t&source=btn&ssel=0&tsel=0&kc=0&tk=' \
           + str(tk) + '&q=' + quote(text, encoding='utf-8')

def translate(js, text, tl='zh-CN'):
    """
    tl为要翻译的语言
    de：德语
    ja：日语
    sv：瑞典语
    nl：荷兰语
    ar：阿拉伯语
    ko：韩语
    pt：葡萄牙语
    zh-CN：中文简体
    zh-TW：中文繁体
    """

    header = {
        'authority': 'translate.google.cn',
        'method': 'GET',
        'path': '',
        'scheme': 'https',
        'accept': '*/*',
        'accept-encoding': 'gzip, deflate, br',
        'accept-language': 'zh-CN,zh;q=0.9,ja;q=0.8',
        # 'cookie': '_ga=GA1.3.110668007.1547438795; _gid=GA1.3.791931751.1548053917; 1P_JAR=2019-1-23-1; NID=156=biJbQQ3j2gPAJVBfdgBjWHjpC5m9vPqwJ6n6gxTvY8n1eyM8LY5tkYDRsYvacEnWNtMh3ux0-lUJr439QFquSoqEIByw7al6n_yrHqhFNnb5fKyIWMewmqoOJ2fyNaZWrCwl7MA8P_qqPDM5uRIm9SAc5ybSGZijsjalN8YDkxQ',
        'cookie': '_ga=GA1.3.110668007.1547438795; _gid=GA1.3.1522575542.1548327032; 1P_JAR=2019-1-24-10; NID=156=ELGmtJHel1YG9Q3RxRI4HTgAc3l1n7Y6PAxGwvecTJDJ2ScgW2p-CXdvh88XFb9dTbYEBkoayWb-2vjJbB-Rhf6auRj-M-2QRUKdZG04lt7ybh8GgffGtepoA4oPN9OO9TeAoWDY0HJHDWCUwCpYzlaQK-gKCh5aVC4HVMeoppI',
        # 'cookie': '',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64)  AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36',
        'x-client-data': 'CKi1yQEIhrbJAQijtskBCMG2yQEIqZ3KAQioo8oBCL+nygEI7KfKAQjiqMoBGPmlygE='
    }
    url = build_url(text, js.get_tk(text), tl)
    res = []
    try:
        r = requests.get(url, headers=header)
        result = json.loads(r.text)
        r.encoding = "UTF-8"
        if debug:
            print(r.url)
            print(r.headers)
            print(r.request.headers)
            print(result)

        res = result[0]
        if res is None:
            if result[7] is not None:
                # 如果我们文本输错，提示你是不是要找xxx的话，那么重新把xxx正确的翻译之后返回
                try:
                    correct_text = result[7][0].replace('', ' ').replace('', '')
                    if debug:
                        print(correct_text)
                    correct_url = build_url(correct_text, js.get_tk(correct_text), tl)
                    correct_response = requests.get(correct_url)
                    correct_result = json.loads(correct_response.text)
                    res = correct_result[0]
                except Exception as e:
                    if debug:
                        print(e)
                    res = []

    except Exception as e:
        res = []
        if debug:
            print(url)
            print("翻译" + text + "失败")
            print("错误信息:")
            print(e)
    finally:
        return res

def get_translate(word, tl='zh-CN'):
    js = Py4Js()
    translate_result = translate(js, word, tl)

    if debug:
        print("word== %s, tl== %s" % (word, tl))
        print(translate_result)
    return translate_result

if __name__ == '__main__':
    debug = True
    translate_text = '3.Hear voice prompt \"start configuration mode\". click \"reset successfully\" button\n'
    results = get_translate(translate_text, 'zh-CN')
    translate_result = ""

    if "." in translate_text or "?" in translate_text:
        for result in results:
            if result[0]:
                translate_result += result[0]
    else:
        result_translate = results[0]

    if debug:
        print("translate_result:" + translate_result)


import numpy as np
import scipy.stats as stats
import matplotlib.pyplot as plt

alpha=0.05
x1=np.array([56, 60, 62, 55, 71, 67, 59, 58, 64, 67])
x2=np.array([57, 58, 69, 48, 72, 70, 68, 71, 50, 53])
x3=np.array([57, 67, 49, 48, 47, 55, 66, 51, 54])
d = stats.kruskal(x1,x2,x3)
print('\n>>> Statistics = %.2f \n>>> Р-value = %.3f' % (d))
s, p = stats.kruskal(x1,x2,x3)
if p > alpha:
    print('>>> Cтатистически значимых различий нет, нулевая гипотеза '
          'не отвергается, время на дистанцию одинаковое.')
else:
    print('>>> Cтатистически значимые различия присутствуют, нулевую '
          'гипотезу отвергаем, время на дистанцию не одинаковое')


# -*- coding: utf-8 -*-
"""
Created on Tue Sep  5 17:12:20 2023

@author: jorji
"""

import folium

# Create a Folium map centered on the US
m = folium.Map(location=[37.0902, -95.7129], zoom_start=4)

# Add GeoJSON data with population density information
folium.Choropleth(
    geo_data=geojson_path,
    name='Population Density',
    data=gdf,
    columns=['GEOID', 'Population_Density'],
    key_on='feature.properties.GEOID',
    fill_color='YlOrRd',
    fill_opacity=0.7,
    line_opacity=0.2,
    legend_name='Population Density (people per square kilometer)',
).add_to(m)

# Add tooltips
style_function = lambda x: {'fillColor': 'white', 'color': 'black', 'fillOpacity': 0.1, 'weight': 0.1}
highlight_function = lambda x: {'fillColor': 'blue', 'color': 'blue', 'fillOpacity': 0.50, 'weight': 0.1}
g = folium.GeoJson(
    geojson_path,
    style_function=style_function,
    control=False,
    highlight_function=highlight_function,
    tooltip=folium.features.GeoJsonTooltip(
        fields=['NAME', 'Population_Density', 'POPULATION', 'ALAND'],
        aliases=['County:', 'Population Density:', 'Population:', 'Land Area:'],
        localize=True,
    ),
)
g.add_to(m)

# Add a layer control
folium.LayerControl().add_to(m)

# Save the map as an HTML file
m.save('population_density_map.html')

# Display the map
m