#! /usr/bin/env python

# mcast_clients.py -- P. Demorest, 2015/02
#
# Based on code originally in async_mcast.py by PD and S. Ransom
#
# These classes set up networking, and parse incoming Obs and VCI
# documents into appropriate data structures.

import os
import struct
import logging
import asyncore, socket
import urllib
import vcixml_parser
import obsxml_parser

class McastClient(asyncore.dispatcher):
    """Generic class to receive the multicast XML docs."""

    def __init__(self, group, port, name=""):
        asyncore.dispatcher.__init__(self)
        self.name = name
        self.group = group
        self.port = port
        addrinfo = socket.getaddrinfo(group, None)[0]
        self.create_socket(addrinfo[0], socket.SOCK_DGRAM)
        self.set_reuse_addr()
        self.bind(('',port))
        mreq = socket.inet_pton(addrinfo[0],addrinfo[4][0]) \
                + struct.pack('=I', socket.INADDR_ANY)
        self.socket.setsockopt(socket.IPPROTO_IP,
                socket.IP_ADD_MEMBERSHIP, mreq)
        self.read = None

    def handle_connect(self):
        logging.debug('connect %s group=%s port=%d' % (self.name,
            self.group, self.port))

    def handle_close(self):
        logging.debug('close %s group=%s port=%d' % (self.name,
            self.group, self.port))

    def writeable(self):
        return False

    def handle_read(self):
        self.read = self.recv(100000)
        logging.debug('read ' + self.name + ' ' + self.read)
        try:
            self.parse()
        except Exception as e:
            logging.exception("error handling '%s' message" % self.name)

    def handle_error(self, type, val, trace):
        logging.error('unhandled exception: ' + repr(val))

class ObsClient(McastClient):
    """Receives Observation XML.

    If the controller input is given, the controller.add_obs(obs) method will
    be called for every document received.

    If use_configUrl is true, the VCI will be retrieved from the url given
    in the Observation document, parsed, and controller.add_vci(vci) will
    be called if a controller exists.
    """

    def __init__(self,controller=None,use_configUrl=False):
        McastClient.__init__(self,'239.192.3.2',53001,'obs')
        self.controller = controller
        self.use_configUrl = use_configUrl

    def parse(self):
        obs = obsxml_parser.parseString(self.read)
        logging.info("read obs configId='%s' seq=%d" % (obs.configId,
            obs.seq))
        if self.use_configUrl:
            logging.info("retrieve vci from '%s'" % obs.configUrl)
            try:
                vciread = urllib.urlopen(obs.configUrl).read()
                logging.debug('retrieved vci ' + vciread)
                vci = vcixml_parser.parseString(vciread)
                if self.controller is not None:
                    self.controller.add_vci(vci)
            except Exception as e:
                logging.exception("error retrieving VCI from '%s'"
                        % obs.configUrl)
        if self.controller is not None:
            self.controller.add_obs(obs)

class VCIClient(McastClient):
    """Receives VCI XML.

    If the controller input is given, the controller.add_vci(vci) method will
    be called for every document received.
    """

    def __init__(self,controller=None):
        McastClient.__init__(self,'239.192.3.1',53000,'vci')
        self.controller = controller

    def parse(self):
        vci = vcixml_parser.parseString(self.read)
        if type(vci) == vcixml_parser.subArray:
            logging.info("read vci configId='%s'" % vci.configId)
            if self.controller is not None:
                self.controller.add_vci(vci)
        else:
            logging.info("read vci non-subArray, ignoring")

# This is how these would be used in a program.  Note that no controller
# is passed, so the only action taken here is to print log messages when
# each XML document comes in.
if __name__ == '__main__':
    logging.basicConfig(format="%(asctime)-15s %(levelname)8s %(message)s",
            level=logging.DEBUG)
    vci_client = VCIClient()
    obs_client = ObsClient(use_configUrl=True)
    try:
        asyncore.loop()
    except KeyboardInterrupt:
        # Just exit without the trace barf on control-C
        logging.info('got SIGINT, exiting')


import numpy as np
from anytree import AnyNode,Node, RenderTree
import random
import sys
import math
import pandas as pd

def setup_test(w,l):
    # The weights file is expected to be a csv file containing 2 columns. The first row are columns numbers.
    # Each row after that contained an example number followed by its weights. Rows are ordered by example index.
    weights = np.genfromtxt(w, delimiter=',')

    # The links/tree file is expected to be in the format produced by the skicy linkage method.
    # Specificly, each row contain 5 entries: row number, first cluster to be marged, second cluster to be marged, distance between clusters, number of examples in new cluster.
    # Cluster with index under the size of dataset represent leafs containing the example of the same index. Otherwise, the cluster is described in the row numbered: index - number of examples in dataset +1

    links = np.genfromtxt(l, delimiter=',')

    # normally the weights/links csv comes with line and columns numbering, that should be removed
    weights = weights[1:, 1:]
    links = links[1:, 1:]

    size = np.size(weights, 0)  # number of examples in input data
    nodes = np.empty((size + np.size(links, 0)), dtype=AnyNode)

    # creating the leafs + updated weight
    for a in range(size):
        nodes[a] = AnyNode(id="x" + str(a), weight=weights[a,0], num_examples=1, approx_discrepancy=0, Tdiscrepancy=0, delta=0, num_sampled=0)

    # parameters to calculate the split quality and average of the hierarchical tree
    worst_q = 0
    sum_q = 0
    counter_q = 0
# creating the internal tree nodes
    for j in range(np.size(links, 0)):
        sum_of_weights = nodes[int(links[j, 0])].weight + nodes[int(links[j, 1])].weight
        sum_of_examples = nodes[int(links[j, 0])].num_examples + nodes[int(links[j, 1])].num_examples
        nodes[size + j] = AnyNode(id="v" + str(j), num_examples=sum_of_examples,
                                        num_sampled=0,  messeured = 0 ,  messeuredBx = 0  ,messeuredBy = 0  ,  weight=sum_of_weights,
                                        approx_discrepancy=0, delta=np.inf, Tdiscrepancy=0)

        nodes[int(links[j, 0])].parent = nodes[size + j]
        nodes[int(links[j, 1])].parent = nodes[size + j]
        # calculating true discrepancy
        leafs = get_leaves(nodes[size + j])
        sum_dis = 0
        for k in range(np.size(leafs, 0)):
            sum_dis = sum_dis + abs(leafs[k].weight - (nodes[size + j].weight / nodes[size + j].num_examples))
        nodes[size + j].Tdiscrepancy = sum_dis
        # calculating q
        if sum_dev > 0:
            current_q = max(nodes[int(links[j, 0])].Tdeviation, nodes[int(links[j, 1])].Tdeviation) / nodes[
                size + j].Tdeviation
            sum_q += current_q
            counter_q += 1
            if current_q > worst_q:
                worst_q = current_q

    print("Deviation of root is: " + str(nodes[-1].Tdeviation))
    print("Split quality q of this tree is: " + str(worst_q))
    print("Average split quality q of this tree is: " + str(sum_q / counter_q))
    # returns the root of the tree
    return nodes[-1]

# Returns all examples in the subtree rooted at v
def get_leaves(v):
    if v.is_leaf:
        return [v]
    mid_nodes = list(v.children)
    leaf_nodes = []
    while len(mid_nodes) > 0:
        if not mid_nodes[0].is_leaf:
            mid_nodes = mid_nodes + list(mid_nodes[0].children)
            mid_nodes.pop(0)
        else:
            leaf_nodes = leaf_nodes + [mid_nodes.pop(0)]
    return leaf_nodes

# selects node to be sampled via LUCB method
def select_nodes_LUCB(P):
    m = 0
    h_t = P[0]
    for i in range(np.size(P)):
        if P[i].approx_discrepancy > m:
            m = P[i].approx_discrepancy
            h_t = P[i]
    m = 0
    l_t = P[0]
    for i in range(np.size(P)):
        if P[i].approx_discrepancy + P[i].delta >= m:
            if P[i].id != h_t.id:
                m = P[i].approx_discrepancy + P[i].delta
                l_t = P[i]
    if h_t.id == l_t.id:
        if np.size(P) > 1:
            print("PROBLEM IN LUCB SELECT")
            print(P)
    return [h_t, l_t]

# selects node to be sampled via UCB method
def select_nodes_UCB(P):
    m = 0
    l_t = P[0]
    for i in range(np.size(P)):
        if P[i].approx_discrepancy + P[i].delta >= m:
            if P[i].num_sampled < P[i].num_examples:
                m = P[i].approx_discrepancy + P[i].delta
                l_t = P[i]
    # check for extreme cases
    if m == 0:
        print("Reached discrepancy 0")
        print("Pruning size of " + str(np.size(P,0)))
        # return 4
    return [l_t]

# randomly selects an example in node v to query about its weight
def query_node(v):
    leafs = get_leaves(v)
    x = random.choice(leafs)
    x.num_sampled += 1
    v.num_sampled += 1

    v.messeured += abs(x.weight - v.weight / v.num_examples) - x.weight # used for the discrepancy estimator
    v.messeuredBy += pow(abs(x.weight - v.weight / v.num_examples) - x.weight ,2) # used to calculate empirical bernstein based bound

    # if its the first time we query this example, we update the budget accordingly
    if x.num_sampled >= 2:
        return 0
    else:
        return 1

# update estimator and bounds
def update_approxdis_and_delta_Berstein(v, K):
    if not v.is_leaf:
        leafs = get_leaves(v)
        if len(list(filter(lambda a: a.num_sampled == 0, leafs))) == 0:
            v.approx_discrepancy = v.Tdiscrepancy
            v.delta = 0
        elif v.num_sampled > 1:
            v.approx_discrepancy = v.weight + (v.num_examples / v.num_sampled) * v.messeured
        # Delta from paper
            denominator = pow(v.num_sampled*3.2, 2) * K
            small_delta = 3 * Delta / denominator
        # Calculate V_n(m) as in empirical bernstein - faster way
            v_m = (v.num_sampled * v.messeuredBy - pow(v.messeured,2)) / (v.num_sampled * (v.num_sampled - 1))
        # At times due, to numerical computation issues, v_m may result it being negative which is not possible
            if v_m <= 0:
                global Worse_vm
                if v_m < Worse_vm:
                    Worse_vm = v_m
                v_m = 0

        # minimum between empirical berstein and hoeffding
            h = v.weight * (math.sqrt((2 * np.log(4 / small_delta)) / (v.num_sampled)) + (2 * np.log(4 / small_delta)) / (3 * v.num_sampled))
            b = 2 * (v.num_examples) * math.sqrt(2* v_m * np.log(2 / small_delta) / v.num_sampled) + 28 * np.log(2 / small_delta) / (3* (v.num_sampled - 1))
            v.delta = min(h,b)
        # check for instances the bound is false
            if abs(v.approx_discrepancy - v.Tdiscrepancy) > v.delta:
                print("**** BOUND ERROR *****")
                print(v)
                print("**********************")

# calculate discrepancy with regard to w_p
def calculate_dis(P):
    total_dis = 0
    num_leafs = 0
    for i in range(np.size(P)):
        total_dis = total_dis + P[i].Tdiscrepancy
        if P[i].is_leaf:
            num_leafs += 1
    return total_dis

# calculate discrepancy with regard to w'_p
def calculate_dis_modified(P):
    total_dis=0
    for v in P:
        sum_weighted = 0
        sampled = 0
        for x in get_leaves(v):
            if x.num_sampled >= 1:
                sum_weighted += x.weight
                sampled += 1
        for x in get_leaves(v):
            if x.num_sampled == 0:
                total_dis += abs(x.weight - ((v.weight - sum_weighted)/(v.num_examples - sampled)))
    return total_dis*0.5  # normalized as explained in the paper

def report_results(P):
    print("The pruning size is: " + str(np.size(P)))
    total_dis = 0
    max_dis = 0
    most_dis = P[0]
    for i in range(np.size(P)):
        print(P[i].id, end=" , ")
        print("discrepancy : " + str(P[i].Tdiscrepancy) + " , Number of queries: " + str(P[i].num_sampled))
        total_dis = total_dis + P[i].Tdiscrepancy
        if P[i].Tdiscrepancy > max_dis:
            max_dis = P[i].Tdiscrepancy
            most_dis = P[i]
    max_dis = 0
    ap_most_dis = P[0]
    for i in range(np.size(P)):
        # print(P[i].id, end = " , ")
        if P[i].approx_discrepancy > max_dis:
            max_dis = P[i].approx_discrepancy
            ap_most_dis = P[i]
    print("Most discrepancy by our estimate: " + ap_most_dis.id + " which is " + str(ap_most_dis.id == most_dis.id))

    print("The discrepancy of pruning achieved is: " + str(total_dis))
    print("***********************************************************")

# c - cost of high order weight query of a node. default is 0.
def AWP(root, Pruning, c):
    # splitting root node
    P = list(root.children)
    row = ['AWP']
    budget = 0 # counts the number of weight queries used
    for k in range(np.size(Pruning)): # return the state of algorithm at designated stopping points
        while np.size(P) < Pruning[k]:
            # Selecting a node to be sampled
            # n_to_query = select_nodes_LUCB(P) - different version
            n_to_query = select_nodes_UCB(P)
            for i in range(np.size(n_to_query)):
                budget = budget + query_node(n_to_query[i])
                update_approxdis_and_delta_Berstein(n_to_query[i], Pruning[k])
            # Selecting a node to be split (if one exist)
            l_t = select_nodes_UCB(P)[0]
            val = l_t.approx_discrepancy + l_t.delta
            flag = True
            options = []
            for i in range(np.size(P)):
                if Beta * (P[i].approx_discrepancy - P[i].delta) > val:
                    options = options + [i]
                elif Beta * (l_t.approx_discrepancy - l_t.delta) < P[i].approx_discrepancy + P[i].delta:
                    flag = False
            # chose one among the candidates for split
            if len(options) > 0:
                i = random.choice(options)
                if P[i].is_leaf:
                    print("problem - leaf selected to be split")
                else:
                    children = list(P.pop(i).children)
                    P = P + children
                    budget = budget + c
            elif flag: # split l_t in the appropriate case
                children = list(l_t.children)
                P.remove(l_t)
                P = P + children
                budget = budget + c
        row = row + [calculate_dis_modified(P), budget]
        print("Arrived to pruning of size " +str(len(P)))
        sys.stdout.flush()
        sys.stderr.flush()
    return row

# resets the tree before additional experiments
def reboot_tree(root):
    root.num_sampled = 0
    root.approx_discrepancy = 0
    root.delta = 2

    mid_nodes = list(root.children)
    while len(mid_nodes) > 0:
        if not mid_nodes[0].is_leaf:
            mid_nodes = mid_nodes + list(mid_nodes[0].children)
            mid_nodes[0].num_sampled = 0
            mid_nodes[0].messeured = 0
            mid_nodes[0].messeuredBy = 0
            mid_nodes[0].approx_discrepancy = 0
            mid_nodes[0].delta = np.inf
        else:
            mid_nodes[0].num_sampled = 0
        mid_nodes.pop(0)

def base_line_EMPIRICAL(root, b, s): # b is the budget for example weight queries, s is the final pruning size required
    P = [root]
    leafs = get_leaves(root)
    # querying
    for k in range(b):
        x = random.choice(leafs)
        val = x.weight
        leafs.remove(x)
        x.num_sampled += 1
        while x.id != root.id:
            x = x.parent
            x.num_sampled += 1
            if x.num_examples == x.num_sampled:
                x.approx_discrepancy = x.Tdiscrepancy
                x.delta = 0
            else:
                x.messeured += abs(val - x.weight / x.num_examples)
                x.approx_discrepancy = (x.num_examples / x.num_sampled) * x.messeured
    # split iteratively the most discrepancy by the naive estimator approximation
    for i in range(s - 1):
        most_dis = 0
        val = P[0].approx_discrepancy
        for j in range(np.size(P)):
            if P[j].approx_discrepancy > val:
                most_dis = j
                val = P[j].approx_discrepancy
        children = list(P[most_dis].children)
        P.pop(most_dis)
        P = P + children
    return [ calculate_dis_modified(P)]

def base_line_UNIFORM(root, b, s): # b is the budget for example weight queries, s is the final pruning size required
    P = [root]
    leafs = get_leaves(root)
    # querying
    for k in range(b):
        x = random.choice(leafs)
        val = x.weight
        leafs.remove(x)
        x.num_sampled += 1
        while x.id != root.id:
            x = x.parent
            x.num_sampled += 1
            if x.num_examples == x.num_sampled:
                x.approx_discrepancy = x.Tdiscrepancy
                x.delta = 0
            else:
                x.messeured += abs(val - x.weight / x.num_examples) - val
                x.approx_discrepancy = x.weight + (x.num_examples / x.num_sampled) * x.messeured
    # split iteratively the most discrepancy by our approximation
    for i in range(s - 1):
        most_dis = 0
        val = P[0].approx_discrepancy
        for j in range(np.size(P)):
            if P[j].approx_discrepancy > val:
                most_dis = j
                val = P[j].approx_discrepancy
        children = list(P[most_dis].children)
        P.pop(most_dis)
        P = P + children
    return [ calculate_dis_modified(P)]

def base_line_WEIGHT(root,b, s): # b is the budget for example weight queries, s is the final pruning size required
    P = [root]
    leafs = get_leaves(root)
    # querying
    for k in range(b):
        x = random.choice(leafs)
        val = x.weight
        leafs.remove(x)
        x.num_sampled += 1
        while x.id != root.id:
            x = x.parent
            x.num_sampled += 1
    # split iteratively the most weighted
    for i in range(s - 1):
        most_weighted = 0
        val = P[0].approx_discrepancy
        for j in range(np.size(P)):
            if P[j].weight > val:
                most_weighted = j
                val = P[j].weight
        children = list(P[most_weighted].children)
        P.pop(most_weighted)
        P = P + children
    return [calculate_dis_modified(P)]

# not used eventually in experiments. An oracal that selects the node to split by the maximal improvement in discrepancy
def orcal_localimprovement(root,Pruning):
    # deep copy
    S = []
    for i in range(np.size(Pruning)):
        S = S + [Pruning[i]]
    P = [root]
    dis_list =["Oi"]
    while len(S) > 0:
        best_index = 0
        best_gain = 0
        for i in range(np.size(P)):
            if not P[i].is_leaf:
                gain = P[i].Tdiscrepancy - P[i].children[0].Tdiscrepancy - P[i].children[1].Tdiscrepancy
                if gain > best_gain:
                    best_index = i
                    best_gain = gain

        v = P.pop(best_index)
        P = P + list(v.children)
        if len(P) == S[0]:
            dis_list += [calculate_dis_modified(P),0]
            S.pop(0)

    return dis_list

# Main
print("Started")
sys.stdout.flush()
sys.stderr.flush()
Delta = 0.05
Beta = 4
Worse_vm = 0 # keep track of numerical fix in empirical bernstein calculations
input_string = list(sys.argv)
root = setup_test(input_string[1],input_string[2]) # the root of the input hierarchical tree
print("Experiment setup completed")
sys.stdout.flush()
sys.stderr.flush()

Pruning = [2]
Col = ["Algtype" , str(2) + "_discrepancy_obtained"  , str(2) + "_NumExamples"]
stepsize = int(input_string[4])
for i in range(int(int(input_string[5]) / stepsize)):
    Pruning = Pruning +[(i + 1)*stepsize ]
    Col = Col + [ str((i + 1)*stepsize) + "_discrepancy_obtained"  , str((i + 1)*stepsize ) + "_NumExamples"]

df = pd.DataFrame(columns=Col) # will store the results

index = 0
for i in range(int(input_string[6])): # determine the number of repetitions for experiment
    print("Started the " + str(i+1) + "'th repetition")
    row = AWP(root, Pruning , 0)
    print("Done with AWP")
    sys.stdout.flush()
    sys.stderr.flush()
    df.loc[index] = row
    index += 1
    base_row = ['EMPIRICAL']
    smart_base_row = ['UNIFORM']
    base_weighted = ['WEIGHT']
    # base_Ua_row = ['U_a']
    for t in range(np.size(Pruning)): #running the baselines with the same number of example weight queries as used by AWP for each K size
        reboot_tree(root)
        base_row = base_row + base_line_EMPIRICAL(root,  int(row[(t+1)*2]), Pruning[t])+ [ int(row[(t+1)*2])]
        reboot_tree(root)
        smart_base_row = smart_base_row + base_line_UNIFORM(root, int(row[(t + 1) * 2]), Pruning[t])+ [ int(row[(t + 1) * 2])]
        reboot_tree(root)
        base_weighted = base_weighted + base_line_WEIGHT(root, int(row[(t + 1) * 2]), Pruning[t]) + [int(row[(t + 1) * 2])]

    df.loc[index] = base_row
    df.loc[index+1] = smart_base_row
    df.loc[index+2] = base_weighted
    index += 3
    reboot_tree(root)
    print("Done with repetition number " + str(i))
df.to_csv(input_string[3] )


# coding=utf8

"""
剑指 Offer 32 - II. 从上到下打印二叉树 II
https://leetcode-cn.com/problems/cong-shang-dao-xia-da-yin-er-cha-shu-ii-lcof/
"""
import collections
from typing import List

class TreeNode:
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

class Solution:
    def levelOrder(self, root: TreeNode) -> List[List[int]]:
        if not root:
            return []
        res = []
        q = collections.deque()
        q.append(root)
        while q:
            arr = []
            for i in range(len(q)):
                node = q.popleft()
                arr.append(node.val)
                if node.left:
                    q.append(node.left)
                if node.right:
                    q.append(node.right)
            res.append(arr)
        return res


from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from typing import Optional
from enum import Enum

app = FastAPI(title="FastAPI App")

class Status(Enum):
    TO_DO = "todo"
    IN_PROGRESS = "in progress"
    DONE = "done"

class Task(BaseModel):
    id: int
    title: str
    description: Optional[str]
    status: Status

class TaskModel(BaseModel):
    title: str
    description: Optional[str]
    status: Status

tasks = [Task(id=1, title="title", description=None, status=Status.DONE)]

@app.get("/")
async def get_tasks():
    return {"message": "FastAPI works!"}

@app.get("/tasks", response_model=list[Task])
async def get_tasks():
    return tasks

@app.get("/tasks/{task_id}", response_model=Task)
async def get_item_by_id(task_id: int):
    task = [task for task in tasks if task.id == task_id]
    if task:
        return task[0]
    raise HTTPException(status_code=404, detail="Task not found")

@app.post("/tasks")
async def create_task(task: TaskModel):
    next_id = max(tasks, key=lambda x: x.id).id + 1
    new_task = Task(id=next_id, title=task.title, description=task.description, status=task.status)
    tasks.append(new_task)

@app.put("/tasks/{task_id}")
async def change_task(task_id: int, task: TaskModel):
    update_task = [t for t in tasks if t.id == task_id]
    if not update_task:
        raise HTTPException(status_code=404, detail="Task not found")
    update_task[0].title = task.title
    update_task[0].description = task.description
    update_task[0].status = task.status
    return update_task[0]

@app.delete("/tasks/{task_id}")
async def get_item_by_id(task_id: int):
    task = [task for task in tasks if task.id == task_id]
    if task:
        tasks.remove(task[0])
        return task_id
    raise HTTPException(status_code=404, detail="Task not found")


# -*- coding = 'utf-8' -*-
'''
#@Time: 2022/7/2 17:52
#@Author: TephrocactusHC
#@File: ceo.py
#@Project: NKUSpider
#@Software: PyCharm
'''

import pandas as pd
from bs4 import BeautifulSoup as bs
import requests
import re

urls = ['https://ceo.nankai.edu.cn/szll/xdgxyjs.htm', 'https://ceo.nankai.edu.cn/szll/gdzbmqjyjsyjs.htm',
        'https://ceo.nankai.edu.cn/szll/wdzgcx.htm', 'https://ceo.nankai.edu.cn/szll/dzkxygcx.htm',
        'https://ceo.nankai.edu.cn/szll/dzxxgcx.htm', 'https://ceo.nankai.edu.cn/szll/txgcx.htm',
        'https://ceo.nankai.edu.cn/szll/dzxxsyjxzx.htm']
headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) '
                  'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}
root_url = 'https://ceo.nankai.edu.cn/szll/'

def find_teacher(url):
    response = requests.get(root_url + url, headers=headers)
    response.encoding = 'utf-8'
    result = re.findall('邮箱.*?\s*(.*?)\s*', response.text, re.S)
    return result

for url in urls:
    thelist = []
    table = pd.read_html(url)[0]
    response = requests.get(url, headers=headers)
    response.encoding = 'utf-8'
    page = bs(response.text, 'html.parser')
    teacher_list = page.select('tbody tr td a')
    for teacher in teacher_list:
        email_list = find_teacher(teacher['href'])
        email_list = list(set(email_list))
        thelist.append(email_list)
    table['邮箱'] = thelist
    print(table)
    if url == 'https://ceo.nankai.edu.cn/szll/xdgxyjs.htm':
        table.to_csv('ceo.csv', mode='a', index=False)
    else:
        table.to_csv('ceo.csv', mode='a', index=False, header=False)


import torch
from pygranso.pygransoStruct import pygransoStruct

def user_fn(X_struct,A,d,device,folding_type):
    assert folding_type in [None, "l2"], "unidentified folding type for PyGranso."
    V = X_struct.V
    # objective function
    f = -torch.trace(V.T@A@V)
    # inequality constraint, matrix form
    ci = None
    # equality constraint
    ce = pygransoStruct()
    constr_vec = (V.T@V - torch.eye(d).to(device=device, dtype=torch.double)).reshape(d**2,1)
    # if folding_type == 'l1':
    #     ce.c1 = torch.sum(torch.abs(constr_vec))
    # elif folding_type == 'l2':
    #     ce.c1 = torch.sum(constr_vec**2)**0.5
    # elif folding_type == 'linf':
    #     ce.c1 = torch.amax(torch.abs(constr_vec))
    if folding_type == 'l2':
        ce.c1 = torch.sum(constr_vec**2)**0.5
    else: # unfolded
        ce.c1 = V.T@V - torch.eye(d).to(device=device, dtype=torch.double)

    return [f,ci,ce]

def opts_init(device,pygranso_config,final_obj_list,data_idx,n,d):
    opts = pygransoStruct()
    opts.torch_device = device
    opts.print_frequency = 10
    opts.maxit = pygranso_config["maxit"]
    opts.print_use_orange = False
    opts.print_ascii = True
    opts.quadprog_info_msg  = False
    opts.opt_tol = pygranso_config["opt_tol"]
    opts.viol_eq_tol = pygranso_config["viol_eq_tol"]
    opts.maxclocktime = pygranso_config["maxclocktime"]
    opts.mu0 = pygranso_config["mu0"]
    opts.fvalquit = final_obj_list[data_idx]*pygranso_config["threshold"]
    opts.x0 =  torch.randn((n*d,1)).to(device=device, dtype=torch.double)
    opts.x0 = opts.x0/torch.linalg.norm(opts.x0)
    return opts


import os

from flask.ext.wtf import Form
from wtforms import StringField, TextAreaField, BooleanField, IntegerField, SubmitField
from wtforms.validators import DataRequired, Optional, ValidationError

from .gitapi import GitAPI

def repo_path_validate(form, field):
    repo_path = field.data
    if not os.path.isdir(repo_path):
        raise ValidationError('Absolute path to the repository is not exist')
    API = GitAPI(repo_path)
    if not API.is_repo():
        raise ValidationError('There is not GIT repo in {}'.format(repo_path))

def ignore_pathes_validate(form, field):
    repo_path = form.repo_path.data
    ignore_pathes = field.data.splitlines()
    if not ignore_pathes:
        return
    for path in ignore_pathes:
        if path[0] == '/':
            raise ValidationError('Path %s is not relative' % path)

def min_days_validate(form, field):
    min_days = field.data
    if min_days < 0:
        raise ValidationError('Minimal working days value should be more then zero')

def min_loc_validate(form, field):
    min_loc = field.data
    if min_loc < 0:
        raise ValidationError('Minimal LOC value should be more then zero')

class ConfigForm(Form):
    repo_path = StringField('Absolute path to the repository', [DataRequired('Field is required'), repo_path_validate])
    ignore_pathes = TextAreaField('List of ignored folders and files', [ignore_pathes_validate])
    min_days = IntegerField('Minimal working days', [Optional(), min_days_validate])
    min_loc = IntegerField('Minimal LOC', [Optional(), min_loc_validate])
    anonym = BooleanField('Anonym mode')
    submit = SubmitField('Get Rate')


'''
This module runs the flask webpage, running scheduled
events and accepting and proccesing data entered on
the webpage by the user. It sends data (from files or
processed within the code) to the webpage for users to
see.
'''
import os
import logging
from flask import Flask
from flask import render_template
from flask import request
from flask import Markup
from werkzeug.utils import redirect
from app import read_config_file
from covid_data_handler import *
from covid_news_handling import *

app = Flask(__name__)
#array to store deleted items

config_file=read_config_file()
API_KEY= config_file["API key:"]
LOCATION=config_file["Location:"]
NATION=config_file["Nation:"]

logging.basicConfig(filename='sys.log',level=logging.DEBUG)

if os.path.isfile('updates.json'):
    with open('updates.json', 'r', encoding="UTF-8") as updates_file:
        update_data=json.loads(updates_file.read())
else:
    update_data=[]

if os.path.isfile('repeated_updates.json'):
    with open('repeated_updates.json', 'r', encoding="UTF-8") as repeated_updates_file:
        list_of_repeated_updates=json.loads(repeated_updates_file.read())
else:
    list_of_repeated_updates=[]

for update in update_data:
    if update["type"]=="covid":
        addition_to_update=schedule_covid_updates(update["content"],update["title"])
        update["event"]=addition_to_update["event"]
    elif update["type"]=="news":
        addition_to_update= update_news(update["title"],update["content"])
        update["event"]=addition_to_update["event"]

for update in list_of_repeated_updates:
    if update["type"]=="covid":
        addition_to_update=repeat_scheduled_covid_update(update["content"],update["title"],schedule_covid_updates)
        update["events"]=addition_to_update["events"]
    elif update["type"]=="news":
        addition_to_update= repeat_scheduled_news_update(update["content"],update["title"],update_news)
        update["events"]=addition_to_update["events"]

@app.route('/index')
def main_page():
    '''
    Loads and processes all data and sends it to the webpage.
    Also runs the schedule for the data updates and manages user
    inputs and edits data structures accordingly
    Returns:
        render_template(...): A HTML template called index.html
        rendered with all necessary data for display on the website
    '''
    API_schedule_covid.run(blocking=False)
    API_schedule_news.run(blocking=False)

    title=request.args.get("notif")
    if title: #remove news widgets
        remove_news_article(title)
        return redirect("/index")

    #reading data from files t o send to webpage
    with open('news_data.json', 'r', encoding="UTF-8") as news_file:
        news=json.loads(news_file.read())

    #constructing a list to send news data to web page
    update_name=request.args.get("two")
    news_data=[]
    #inserting links to article source sites
    for item in news:
        news_url=Markup(''+" Read full article here" + "")
        content=Markup(item["description"])+news_url
        news_data.append({"title":item["title"],"content":content})
        if len(news_data)>=8: # limits article widgets
            break
    #reading the covid data  to send to website
    #data is read from a file to avoid polling the API
    if os.path.isfile('national_covid_data.json'):
        with open('national_covid_data.json', 'r', encoding="UTF-8") as covid_file:
            nation_data=json.loads(covid_file.read())
    else:
        logging.info("can't find national_covid_data.json ")

    if os.path.isfile('covid_data.json'):
        with open('covid_data.json', 'r', encoding="UTF-8") as covid_file:
            local_data=json.loads(covid_file.read())
    else:
        logging.info("can't find covid_data.json ")

    #re adds a repeated scheduled event so it can be displayed as a widget
    queue_covid=API_schedule_covid.queue
    queue_news=API_schedule_news.queue

    for item in queue_covid:
        names=item[3]
        if item[2] == schedule_covid_updates:
            #if its a repeated covid update
            try:
                chronology=names[0]
                update_title=names[1]
                event=item
                update_dict={"event":event,"title":update_title,"content":chronology,"type":"covid"}
                if update_dict not in update_data:
                    logging.info('re-added a covid repeated update to the list of updates')
                    update_data.append(update_dict)
            except:
                logging.info('re-adding a covid repeated update to the list of updates failed')

    for item in queue_news:
        names=item[3]
        if item[2] == update_news:
            #if its a repeated news update
            try:
                chronology=names[1]
                update_title=names[0]
                event=item
                update_dict={"event":event,"title":update_title,"content":chronology,"type":"news"}
                if update_dict not in update_data:
                    logging.info('re-added a news repeated update to the list of updates')
                    update_data.append(update_dict)
            except:
                logging.info('re-adding a covid repeated update to the list of updates failed')

    #removing events

    for entry in update_data:
        if entry["event"] not in API_schedule_news.queue and entry["event"] not in API_schedule_covid.queue:
            update_data.remove(entry)
            logging.info('removed an update that had occured')

    cancel_covid_update,cancel_news_update=False,False

    remove_event=request.args.get("update_item") #removes event from event list +schedule
    if remove_event:
        for entry in update_data:
            if entry["title"]==remove_event:
                #cancelling event from schedule
                try:
                    API_schedule_covid.cancel(entry["event"])
                    cancel_covid_update=entry
                    logging.info("removed "+entry["title"]+(" from update schedule"))
                except:
                    try:
                        API_schedule_news.cancel(entry["event"])
                        cancel_news_update=entry
                        logging.info("removed "+entry["title"]+(" from update schedule"))
                    except:
                        logging.info("removing "+entry["title"]+(" from update schedule failed"))
        if cancel_covid_update:
            update_data.remove(cancel_covid_update)
            #remove from widget list
            logging.info("removed a covid entry from update_data")
        if cancel_news_update:
            update_data.remove(cancel_news_update)
            #remove from widget list
            logging.info("removed a news entry from update_data")
        for entry in list_of_repeated_updates: #REMOVES repeated updates from both queues
            if entry["title"]==remove_event:

                try:
                    API_schedule_covid.cancel(entry["events"][0])
                    API_schedule_covid.cancel(entry["events"][1])
                    #events field has 2 events- this cancels both
                    #one event is to schedule an update
                    #the other is to schedule the reschedule function
                    logging.info("removed repeated event from the covid schedule")
                except ValueError:
                    logging.info(" failed to remove repeated event from the covid API schedule")

                try:
                    API_schedule_news.cancel(entry["events"][0])
                    API_schedule_news.cancel(entry["events"][1])
                    logging.info("removed repeated event from the news API schedule")
                except ValueError:
                    logging.info("failed to remove repeated event from the news API schedule")
        return redirect("/index")

    if len(update_data)==0 and len(list_of_repeated_updates)!=0:
        #in case there schould be no updates, but repeated
        #updates haven't been cancelled yet
        logging.info("wiped list_of_repeated_updates and cancelled all events")
        list_of_repeated_updates.clear()
        #cancels all items in the schedules
        for item in API_schedule_news.queue:
            API_schedule_news.cancel(item)

        for item in API_schedule_covid.queue:
            API_schedule_covid.cancel(item)

    #schedules events
    if update_name:
        valid_name=True
        #this ensures the same title can't be used
        for item in update_data:
            if update_name == item["title"]:
                valid_name=False
        if valid_name:
            time_input = request.args.get('update')
            if time_input:
                repeat_update=request.args.get("repeat")
                update_covid_data=request.args.get("covid-data")
                update_news_data=request.args.get("news")

                if update_covid_data:
                    #adding to the list of update events
                    event_dict=schedule_covid_updates(time_input,update_name)
                    logging.info("scheduled a covid update")
                    event_dict["content"]=time_input
                    update_data.append(event_dict)
                    if repeat_update:
                        list_of_repeated_updates.append(repeat_scheduled_covid_update(time_input,
                        update_name,schedule_covid_updates))
                        logging.info("scheduled a repeated covid update")

                if update_news_data:
                    #adding to the list of update events
                    event_dict=update_news(update_name,time_input)
                    logging.info("scheduled a news update")
                    event_dict["content"]=time_input
                    update_data.append(event_dict)
                    if repeat_update:
                        list_of_repeated_updates.append(repeat_scheduled_news_update(time_input,
                        update_name,update_news))
                        logging.info("scheduled a repeated news update")
        return redirect("/index")

    update_list_for_web=[]
    #converting into one list - removes excess widgets caused by 2 schedulers
    #also converting to json dumpable lists
    for item in update_data:
        if item == update_data[0]:
            update_list_for_web.append({"title":item["title"],
            "type":item["type"],"content":item["content"]})
        for value in update_list_for_web:
            if value["title"] != item["title"]:
                update_list_for_web.append({"title":item["title"],
            "type":item["type"],"content":item["content"]})

    #writing updates to file so they can be used if the code
    #stops running
    repeated_updates_to_file=[]
    for item in list_of_repeated_updates:
        repeated_updates_to_file.append({"title":item["title"],
            "type":item["type"],"content":item["content"]})

    with open('updates.json', 'w', encoding="UTF-8") as updates_file:
        json.dump(update_list_for_web, updates_file, indent=6)

    with open('repeated_updates.json', 'w', encoding="UTF-8") as repeated_updates_file:
        json.dump(repeated_updates_to_file, repeated_updates_file, indent=6)

    #processing data
    last7days_cases,current_hospital_cases,total_deaths= process_covid_API_data(local_data)
    last7days_cases_nation, current_hospital_cases_nation, total_deaths_nation=process_covid_API_data(nation_data)
    current_hospital_cases_nation="Current Hospital Cases: "+str(current_hospital_cases_nation)
    total_deaths_nation="Current Total Deaths: "+str(total_deaths_nation)
    logging.info("processed API data")
    #sending data to webpage
    return render_template("index.html", location=LOCATION, local_7day_infections=last7days_cases,
    nation_location=NATION,national_7day_infections=last7days_cases_nation,
    hospital_cases=current_hospital_cases_nation,deaths_total=total_deaths_nation,
    news_articles=news_data, updates=update_list_for_web, image="boon.png")

if __name__=="__main__":
    app.run()


from django.contrib import admin
from django.urls import path, include, re_path
from django.views.generic.base import TemplateView
from allauth.account.views import confirm_email
from rest_framework import permissions
from drf_yasg.views import get_schema_view
from drf_yasg import openapi
from django.conf.urls.static import static
from django.conf import settings

urlpatterns = [
    # path("", include("home.urls")),
    path("accounts/", include("allauth.urls")),
    path("modules/", include("modules.urls")),
    # path("api/v1/", include("home.api.v1.urls")),
    # path("api/v1/", include("dashboard.api.v1.urls")),
    # path("api/v1/", include("business_site.api.v1.urls")),
    # path("api/v1/", include("site_admin.api.v1.urls")),
    # path("api/v1/", include("push_notification.api.v1.urls")),
    path("admin/", admin.site.urls),
    path("users/", include("users.urls", namespace="users")),
    path("rest-auth/", include("rest_auth.urls")),
    # Override email confirm to use allauth's HTML view instead of rest_auth's API view
    path("rest-auth/registration/account-confirm-email/<str:key>/", confirm_email),
    path("rest-auth/registration/", include("rest_auth.registration.urls")),
] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
admin.site.site_header = "zero_cheating"
admin.site.site_title = "zero_cheating Admin Portal"
admin.site.index_title = "zero_cheating Admin"

# swagger
api_info = openapi.Info(
    title="zero_cheating API",
    default_version="v1",
    description="API documentation for zero_cheating App",
)

schema_view = get_schema_view(
    api_info,
    public=True,
    permission_classes=(permissions.IsAuthenticated,),
)

urlpatterns += [
    path("api-docs/", schema_view.with_ui("swagger", cache_timeout=0), name="api_docs")
]

urlpatterns += [
    path("", TemplateView.as_view(template_name='index.html')),
    path("firebase-messaging-sw.js", TemplateView.as_view(template_name="firebase-messaging-sw.js",
        content_type='application/javascript'))
]
urlpatterns += [re_path(r"^(?:.*)/?$",
                TemplateView.as_view(template_name='index.html'))]


"""download_handler module."""
import requests
from page_loader.file_handler import save_to_file
from page_loader.logging import get_logger
from progress.bar import Bar

logger = get_logger(__name__)

def fetch_resource(url: str) -> bytes:
    """Fetch resource from URL.

    Args:
        url (str): URL of resource.

    Returns:
        (bytes): Resource content.

    Raises:
        requests.exceptions.RequestException: If request error status.
    """
    try:
        response = requests.get(url)
        response.raise_for_status()
    except requests.exceptions.RequestException as req_exception:
        logger.error("Can't download resource at URL {0}\n{1}".format(
            url,
            req_exception,
        ))
        logger.debug(req_exception, exc_info=True)
        raise
    return response.content

def download_assets(output_path: str, assets: list) -> None:
    """Download assets and save to files.

    Args:
        output_path (str): Path where to save downloaded assets.
        assets (list): List of assets to download.
    """
    logger.info('Assets downloading started.')
    with Bar('Downloading assets', max=len(assets)) as assets_download_bar:
        for asset in assets:
            logger.debug('Downloading resource {0}'.format(asset.asset_url))
            try:
                asset_content = fetch_resource(asset.asset_url)
            except requests.exceptions.RequestException:
                logger.info('Skipping asset {0}'.format(asset.asset_url))
                continue
            save_to_file(
                content_to_save=asset_content,
                output_path=output_path,
                file_name=asset.asset_file_name,
            )
            assets_download_bar.next()
    logger.info('Assets downloading finished.')


# Solution 1: 내 풀이
class Solution:
    def isValid(self, s: str) -> bool:
        stack = []
        for char in s:
            if char in "([{":
                stack.append(char)
                continue
            elif not stack:     # stack이 비었는데 닫는 괄호인 경우
                return False
            if char == ')' and stack[-1] == '(':
                stack.pop()
            elif char == ']' and stack[-1] == '[':
                stack.pop()
            elif char == '}' and stack[-1] == '{':
                stack.pop()
            else:   # stack에 일치하는 짝이 없는 경우
                return False

        if stack:
            return False
        else:   # stack에 문자가 남은 경우
            return True

# Solution 2: 정돈된 코드(풀이 참고)
from collections import defaultdict

class Solution2:
    def isValid(self, s: str) -> bool:
        paren_dict = defaultdict(str)
        paren_dict = {')':'(', '}':'{', ']':'['}
        stack = []

        for char in s:
            if char in paren_dict: # 닫는 괄호일 때
                if not stack or stack.pop() != paren_dict[char]:
                    return False
            else:
                stack.append(char)
        return not stack


#AIM: to find number of genes in each primary and secondary class of pathways in the HPA background list

# NOTE(peter) this caches requests made with the 'requests' library in a local database
import requests
import requests_cache

requests_cache.install_cache('stored_geneid_requests')

#extract list of kegg gene ids from 'HPA_ENSG2KEGG'

path_to_HPA_kegg_ids = 'HPA_kegg_ids.txt'
HPA_kegg_ids = open(path_to_HPA_kegg_ids, 'w')#write to file

HPA_kegg_list = []
path_to_HPA_kegg = 'HPA_ENSG2KEGG.txt'
HPA_kegg_file = open(path_to_HPA_kegg, 'rU')#read file
for line in HPA_kegg_file:
    # split creates a list out of the line
    # [0] is gene Ensemble gene identifier (ENSG), [1] is KEGG ID
    stripped_line = line.rstrip()
    split_line = stripped_line.split()
    kegg_id = split_line[1]
    strip_kegg_id = kegg_id.rstrip(';') #strip off the ';' from any with more than 1 kegg ID
    if strip_kegg_id != '-': #remove ENSG not converted to KEGG ids
        HPA_kegg_list.append(strip_kegg_id)
HPA_kegg_list = HPA_kegg_list[1:] #line[0] is header
for item in HPA_kegg_list[1:]:
    item_to_write = item+'\n'
    HPA_kegg_ids.write(item_to_write)
HPA_kegg_ids.close()

#create list of pathways in HPA background
#using the KEGG API to find the pathways each gene belongs to:
#http://rest.kegg.jp/link/pathway/'gene ID eg hsa:10458'
#remove duplicates -> create HPA_unique_pathway_list

import urllib.request #allows python to access url
path_to_HPA_kegg_pathways = 'HPA_kegg_pathways.txt'
HPA_kegg_pathways = open(path_to_HPA_kegg_pathways, 'w')#write to file

for gene in HPA_kegg_list:
    kegg_gene = str(gene)
    url = "http://rest.kegg.jp/link/pathway/" + kegg_gene
    url_read = requests.get(url)   #read url output
    item_to_write = url_read.text
    #see http://stackoverflow.com/questions/540342/python-3-0-urllib-parse-error-type-str-doesnt-support-the-buffer-api
    if 'hsa' in item_to_write:#remove empty lines
        HPA_kegg_pathways.write(item_to_write)
HPA_kegg_pathways.close()

#make set to be unique pathways
HPA_pathway_set = set()
path_to_HPA_pathways = 'HPA_kegg_pathways.txt'
HPA_pathways = open(path_to_HPA_pathways, 'rU')
for line in HPA_pathways:
    # split creates a list out of the line
    # [0] is KEGG gene ID, [1] is KEGG pathway ID
    stripped_line = line.rstrip() #remove '\n'
    split_line = stripped_line.split('\t') #tab separated
    pathway = split_line[1]
    pathway_i = pathway.lstrip('path')#remove prefix to pathway id
    pathway_id = pathway_i.lstrip(':')
    HPA_pathway_set.add(pathway_id)
HPA_unique_pathway_list = list(HPA_pathway_set)
print('HPA_pathway_set:', len(HPA_pathway_set))
print('HPA_unique_pathway_list:', len(HPA_unique_pathway_list))

#for each pathway in the HPA_unique_pathway_list, find the number of genes in pathway
#using API query
#make dictionary in json file where: {pathway1: [gene1, gene2], pathway:[gene,gene2,gene3]...}

pathway_gene_dict = {}
import json
with open('HPA_pathways_genes.json', 'w') as outfile:#don't need to file.close() when use with
    for entry in HPA_unique_pathway_list:
        pathway = str(entry)
        url = "http://togows.dbcls.jp/entry/pathway/"+pathway+"/genes.json"
        url_read = requests.get(url)  #read url output
        item_str = url_read.text
        item_strip = item_str.lstrip("[\n  {\n").rstrip(" }\n]")
        item_list = item_strip.split(',\n')
        #genes are separated by ',\n'
        #entry in list looks like:
        #'    "23533": "PIK3R5; phosphoinositide-3-kinase, regulatory subunit 5 [KO:K02649]"'
        gene_id_list = []
        for entry in item_list:
            entry_list = entry.split(':') #to separate gene id number from description
            gene = entry_list[0]
            gene_str = gene.strip().rstrip('"').lstrip('"') #strip away blank space on rhs and ""
            if len(gene_str) >1: #don't add 'hsa:' to empty entries
                gene_id = 'hsa:'+gene_str #so we will have the right query format for gene id in next step eg 'hsa:23533'
                gene_id_list.append(gene_id)
            if len(gene_id_list) >=1: #remove empty pathways
                pathway_gene_dict[pathway] = gene_id_list
    dict_as_json = json.dumps(pathway_gene_dict) #dumps turns list into a json string
    outfile.write(dict_as_json)

#find number of HPA background list pathways in primary and secondary class
#first create file with columns: KEGG_Primary_class, KEGG_secondary_class, pathway
path_to_analysis = 'class_analysis.txt'
pathway_analysis = open(path_to_analysis, 'w')#write

import json
with open('HPA_pathways_genes.json') as f:
    pathway_dict = json.load(f) #load (not loads, which is for string) does opposite of dumps
    for pathway_ID in pathway_gene_dict:
        #find pathway class using TOGOWS API
        class_url = 'http://togows.org/entry/kegg-pathway/'+pathway_ID+'/classes.json'
        class_url_response = requests.get(class_url)  #read url output
        class_url_str = class_url_response.text
        class_json = json.loads(class_url_str) #json looks like: [['Organismal Systems', 'Endocrine system']]
        class_list = class_json[0] #take list within list
        primary_class = class_list[0]
        secondary_class = class_list[1]
        genes = pathway_gene_dict[pathway_ID]
        len_genes = str(len(genes))
        item_to_write = primary_class+'\t'+secondary_class+'\t'+pathway_ID+'\n'
        pathway_analysis.write(item_to_write)
    pathway_analysis.close()

#second, add up all the primary class pathways
pathway_analysis_read = open(path_to_analysis, 'rU')

#make cumulative dictionary; {class1:gene_number, etc}
primary_dict = {}
secondary_dict = {}
for line in pathway_analysis_read:
    stripped_line = line.rstrip()
    split_line = stripped_line.split('\t')
    primary_class = split_line[0]
    secondary_class = split_line[1]
    primary_dict[primary_class] = primary_dict.get(primary_class, 0) + 1
    secondary_dict[secondary_class] = secondary_dict.get(secondary_class, 0) + 1
print('Primary class:')
for entry in primary_dict:
    print(entry+'\t'+str(primary_dict[entry]))
print('Secondary class:')
for entry in secondary_dict:
    print(entry+'\t'+str(secondary_dict[entry]))

#make dictionaries for other data sets in 'class_analysis_chosen.py'


import numpy as np
import pandas as pd
import os
import pickle
# from sklearn.externals import joblib
import matplotlib.pyplot as plt
from collections import Counter
from sklearn.model_selection import train_test_split
from datetime import timedelta
import scipy as sp
from SolarForecasting.ModulesProcessing import collect_data, clean_data
from SolarForecasting.ModulesLearning import preprocessing as preprocess
from SolarForecasting.ModulesLearning import postprocessing as postprocess
from SolarForecasting.ModulesLearning.ModuleLSTM import train as tranformers
from SolarForecasting.ModulesLearning.ModulesCNN import train as cnn
from SolarForecasting.ModulesLearning.ModulesCNN.Model import crps_score
from matplotlib import cm

pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)

# city
city = 'Sioux_Falls_SD'#'Boulder_CO'#'Goodwin_Creek_MS'#'Desert_Rock_NV''Penn_State_PA'

# lead time
lead_times = [24*7] #7days in advance

# season
seasons =['year']
res = '1hour' #15min

# file locations
path_local = "/Users/saumya/Desktop/SolarProject/"
path_cluster = "/pl/active/machinelearning/Solar_forecasting_project/"
path_project = path_local
path = path_project+"Data/"
folder_saving = path_project + city+"/Models/"
folder_plots = path_project + city+"/Plots/"
clearsky_file_path = path+'clear-sky/'+city+'_1min_original.csv'

# scan all the features
features = ['year','month','day','hour','min','zen','dw_solar','uw_solar','direct_n','diffuse','dw_ir','dw_casetemp','dw_dometemp','uw_ir','uw_casetemp','uw_dometemp','uvb','par','netsolar','netir','totalnet','temp','rh','windspd','winddir','pressure']

## ## selected features for the study
final_features = ['year','month','day','hour','zen','dw_solar','uw_solar','direct_n','dw_ir','uw_ir','temp','rh','windspd','winddir','pressure', 'clear_ghi']
ensmeble_col_list = ["ensemble_" + str(ensemble_num) for ensemble_num in range(51)]

# target or Y
target_feature = 'clearness_index'

# start and end month+year
startyear = 2016
endyear = 2018
startmonth = 1
endmonth = 12

testyear = 2018

# hyperparameters
n_timesteps = 24#48 #1day (can be 12hrs or 48hrs for a few models)
n_features = 16 #+51 #+ 1
quantile = False #False

#hyperparameters for LSTM/Transformer/CNNs
n_layers = 1 #2 #3
num_heads = 2 #4
d_model = 96#64 #128

hidden_size= 50
batch_size = 16 #32 #16 #16

epochs = 300 #250
lr = 1e-4 #1e-5 #1e-4

alphas = np.arange(0.05, 1.0, 0.05) #range of alpha (for quantile forecast)
q50 = 9  #median index in 'alphas'

def get_data():

    ## collect raw data
    years = [2015, 2016, 2017, 2018] #[2005, 2006, 2007, 2008, 2009]
    object = collect_data.SurfradDataCollector(years, [city], path)

    object.download_data()

    ## cleanse data to get processed version
    for year in years:
        object = clean_data.SurfradDataCleaner(city, year, path)
        object.process(path_to_column_names='ModulesProcessing/column_names.pkl')

def include_previous_features(X, col_to_indices_mapping):
    '''
    Features at time t includes features from previous n_timesteps
    '''
    # y_list = []
    # previous_time_periods = list(range(1, n_timesteps+1))
    # # indices_except_ensmebles = sorted([feature_ind for feature, feature_ind in col_to_indices_mapping.items() if not feature.startswith('ensemble')])
    # # print(indices_except_ensmebles)
    # # Xsub = X[:,indices_except_ensmebles]
    #
    # for l in previous_time_periods:
    #     # print("rolling by: ", l)
    #     X_train_shifted = np.roll(X, l, axis=0)
    #
    #     y_list.append(X_train_shifted)
    #     # y_list.append(dw_solar_rolled)
    # y_list = y_list[::-1]
    #
    # previous_time_periods_columns = np.column_stack(y_list)
    # X = np.column_stack([previous_time_periods_columns, X])
    #
    # print("X shape after adding prev features: ", X.shape)
    # return X

    X = np.expand_dims(X, axis=2)
    # indices_except_ensmebles = sorted(
    #     [feature_ind for feature, feature_ind in col_to_indices_mapping.items() if not feature.startswith('ensemble')])
    # # print(X.shape)
    X_with_prev_timesteps = []
    data_size = X.shape[0]
    for i in range(n_timesteps,data_size):
        prev_list =[]
        for j in range(n_timesteps,0,-1):
            prev = X[i-j,:,:]
            # prev_sub = prev[indices_except_ensmebles,:]
            # print(prev.shape)
            prev_list.append(prev)

        curr = X[i,:,:]
        prev_list.append(curr)
        temp = np.concatenate(prev_list, axis=1)

        # print(temp.shape)
        X_with_prev_timesteps.append(temp)

    # print(len(X_with_prev_timesteps))

    X = np.array(X_with_prev_timesteps)
    # print(X.shape)
    return X

def include_nwp_features(df):

    missing_hour_dates = {2016:[263,265], 2017:[283], 2018:[144]}

    fnwp = path + "NWP/" + city + "/"

    #Alreadt sorted
    # df = dfraw.sort_values(by=['year','month','day','hour'])
    # print("inside nwp")
    # print(dfraw.equals(df))

    for ensemble in ensmeble_col_list:
        df[ensemble] = ""

    nwp_new_columns = []
    for year in [2016,2017,2018]:
        print(len(df[df['year'] == year]))
        year_list = []
        fnwp_yr = fnwp+"nwp_"+str(year)+".npy"
        nwp_yr = np.load(fnwp_yr)
        nwp_yr = nwp_yr/(168*3600)
        days = nwp_yr.shape[0]
        for day in range(days):
            if day not in missing_hour_dates[year]:
                nwp_day = nwp_yr[day]
                print(nwp_day.shape)
                x_1_12 = np.tile(nwp_day[0], (12,1))
                x_13_24 = np.tile(nwp_day[1], (12,1))
                x_day = np.concatenate((x_1_12, x_13_24), axis=0)
                print(x_day.shape)
                year_list.append(x_day)

        year_nwp = np.concatenate(year_list)
        print(year_nwp)
        print(year_nwp.shape)
        nwp_new_columns.append(year_nwp)

    nwp_new_columns = np.concatenate(nwp_new_columns)
    print(np.max(nwp_new_columns), np.min(nwp_new_columns))

    df.loc[:,ensmeble_col_list] = nwp_new_columns

    return df

def get_crps_for_ngboost(model, X, y):
    '''
    CRPS is the evaluation metric to be used for probabilsitic forecasting
    '''

    params = model.pred_dist(X)._params
    loc = params[0]
    scale = np.exp(params[1])
    Z = (y - loc) / scale

    score = scale * (
            Z * (2 * sp.stats.norm.cdf(Z) - 1)
            + 2 * sp.stats.norm.pdf(Z)
            - 1 / np.sqrt(np.pi)
    )

    return np.average(score)

def get_crps_for_ngboost_scaled(model, X, y,X_before_normalized,index_clearghi,lead):
    '''
    CRPS is the evaluation metric to be used for probabilsitic forecasting
    '''

    params = model.pred_dist(X)._params
    loc = params[0]
    scale = np.exp(params[1])

    clearsky = X_before_normalized[:, index_clearghi]
    y = np.roll(y, lead)
    loc = np.roll(loc, lead)
    scale = np.roll(scale, lead)

    print(clearsky.shape, y.shape, loc.shape, scale.shape)
    y = np.multiply(y, clearsky)
    loc = np.multiply(loc, clearsky)
    scale = np.multiply(scale, clearsky)

    Z = (y - loc) / scale

    score = scale * (
            Z * (2 * sp.stats.norm.cdf(Z) - 1)
            + 2 * sp.stats.norm.pdf(Z)
            - 1 / np.sqrt(np.pi)
    )
    print(score.shape)
    score = score[2 * lead:]

    return np.average(score)

def get_CH_PeEN_baseline_crps(X_test, X_train, col_to_indices_mapping, y_test):
    index_hour = col_to_indices_mapping['hour']
    index_clearness = col_to_indices_mapping['clearness_index_input']

    current_hour_list = X_test[:, index_hour]

    CH_PeEN_baseline = []
    CH_PeEN_point_baseline = []

    # X_train_only_last_timestep = X_train[:, :, -1]

    for i in range(len(X_test)):
        current_hour = current_hour_list[i]
        # print(current_hour,current_month)
        X_train_for_hour = X_train[X_train[:,index_hour]==current_hour]

        hour_dist = X_train_for_hour[:, index_clearness]#X_train_for_hour[:, index_ghi]/X_train_for_hour[:, index_clearghi]

        CH_PeEN_point_baseline.append(np.average(hour_dist))

        quantiles = np.quantile(hour_dist, alphas)
        CH_PeEN_baseline.append(quantiles)

    CH_PeEN_baseline_quantiles = np.asarray(CH_PeEN_baseline)
    CH_PeEN_point_baseline = np.reshape(CH_PeEN_point_baseline, (len(CH_PeEN_point_baseline), 1))

    clearsky = y_test[:, 1].reshape(y_test.shape[0], 1)
    true = y_test[:, 0].reshape(y_test.shape[0], 1)  # np.roll(y_test, lead)
    y_test = np.multiply(true, clearsky)

    CH_PeEN_baseline_quantiles = np.multiply(CH_PeEN_baseline_quantiles, clearsky)
    CH_PeEN_point_baseline = np.multiply(CH_PeEN_point_baseline, clearsky)

    crps_CH_PeEN = crps_score(CH_PeEN_baseline_quantiles, y_test, alphas)

    return CH_PeEN_point_baseline, CH_PeEN_baseline_quantiles, crps_CH_PeEN

def get_climatology_baseline_crps(X_test, X_train, col_to_indices_mapping, y_test):

    index_hour = col_to_indices_mapping['hour']
    index_ghi = col_to_indices_mapping['dw_solar']

    current_hour_list = X_test[:, index_hour]
    print(current_hour_list.shape)

    climatology_baseline = []
    climatology_point_baseline = []

    # X_train_only_last_timestep = X_train[:,:,-1]
    # print(X_train_only_last_timestep.shape)

    for i in range(len(X_test)):
        current_hour = current_hour_list[i]
        # print(current_hour,current_month)
        X_train_for_hour = X_train[X_train[:, index_hour] == current_hour]
        hour_dist = X_train_for_hour[:, index_ghi]

        climatology_point_baseline.append(np.average(hour_dist))

        quantiles = np.quantile(hour_dist, alphas)
        climatology_baseline.append(quantiles)

    climatology_baseline_quantiles = np.asarray(climatology_baseline)
    climatology_point_baseline = np.reshape(climatology_point_baseline, (len(climatology_point_baseline), 1))

    clearsky = y_test[:, 1].reshape(y_test.shape[0], 1)
    true = y_test[:, 0].reshape(y_test.shape[0], 1)  # np.roll(y_test, lead)
    y_test = np.multiply(true, clearsky)

    crps_climatology = crps_score(climatology_baseline_quantiles, y_test, alphas)

    return climatology_point_baseline, climatology_baseline_quantiles, crps_climatology

# def reliability_diagrams(alphas, outputs, outputs_ch_pn, outputs_climatology, target, folder_saving):
#
#
#     outputs_tcn_attn = np.load(
#         "/Users/saumya/Desktop/SolarProject/Boulder_CO/Models/year/ML_models_2018/cnn/1hour/final_tcn_week_ahead_1days_lag_small_kernel_1hr_res_attn_quantile/tcn_attention.npy")
#     outputs_transformers = np.load(
#         "/Users/saumya/Desktop/SolarProject/Boulder_CO/Models/year/ML_models_2018/cnn/1hour/final_transformers_2day_2heads_less_dmodel_lag_week_ahead_1hr_res_quantile/transformer.npy")
#     print(outputs_tcn_attn.shape, outputs_transformers.shape)
#
#     ## had to add the following for boulder, since timesteps were different for tcn/transformers
#     # target = target[24:, :]
#     total_obs = len(target)
#     # outputs = outputs[24:,:]
#     # outputs_tcn_attn = outputs_tcn_attn[24:,:]
#     # outputs_ch_pn = outputs_ch_pn[24:,:]
#     # outputs_climatology = outputs_climatology[24:,:]
#     # print(outputs_tcn_attn.shape, outputs_transformers.shape)
#
#
#     obs_proportion_list = []
#     obs_proportion_list_chpn = []
#     obs_proportion_list_climatology = []
#     obs_proportion_list_tcn_attn =[]
#     obs_proportion_list_transformers = []
#     for i, alpha in zip(range(len(alphas)), alphas):
#         output = outputs[:, i].reshape((-1, 1))
#         output_ch_pn = outputs_ch_pn[:, i].reshape((-1, 1))
#         output_climatology = outputs_climatology[:, i].reshape((-1, 1))
#         output_transformers = outputs_transformers[:, i].reshape((-1, 1))
#         output_tcn_attn = outputs_tcn_attn[:, i].reshape((-1, 1))
#
#         obs_proportion_list.append(np.sum(target<=output)/total_obs)
#         obs_proportion_list_chpn.append(np.sum(target <= output_ch_pn) / total_obs)
#         obs_proportion_list_climatology.append(np.sum(target <= output_climatology) / total_obs)
#         obs_proportion_list_tcn_attn.append(np.sum(target <= output_tcn_attn) / total_obs)
#         obs_proportion_list_transformers.append(np.sum(target <= output_transformers) / total_obs)
#
#     print(obs_proportion_list == obs_proportion_list_climatology)
#
#     fig, ax = plt.subplots()
#     # ax.plot(alphas, alphas)
#     ax.plot(alphas, obs_proportion_list, marker='o', color ='g',label="TCN")
#     ax.plot(alphas, obs_proportion_list_tcn_attn, marker='d', color='y', label="TCN_with_attention")
#     ax.plot(alphas, obs_proportion_list_transformers, marker='h', color='m', label="Transformers")
#     ax.plot(alphas, obs_proportion_list_chpn, marker="s", color ='b',label="CH_PeEN")
#     ax.plot(alphas, obs_proportion_list_climatology, marker="*",color ='k', label="Climatology")
#     ax.set_xlabel('Nominal proportion')
#     ax.set_ylabel('Observed proportion')
#     ax.legend(loc='upper left')
#
#
#     plt.savefig(folder_saving + "reliability_with_all_plots")
#     plt.close()
#
#
# def sharpness_diagrams(alphas, outputs, outputs_ch_pn, outputs_climatology,folder_saving):
#     outputs_tcn_attn = np.load(
#         "/Users/saumya/Desktop/SolarProject/Boulder_CO/Models/year/ML_models_2018/cnn/1hour/final_tcn_week_ahead_1days_lag_small_kernel_1hr_res_attn_quantile/tcn_attention.npy")
#     outputs_transformers = np.load(
#         "/Users/saumya/Desktop/SolarProject/Boulder_CO/Models/year/ML_models_2018/cnn/1hour/final_transformers_2day_2heads_less_dmodel_lag_week_ahead_1hr_res_quantile/transformer.npy")
#
#
#     print(outputs_tcn_attn.shape, outputs_transformers.shape)
#
#
#     intervals = np.array([0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9])
#     sharpness = []
#     sharpness_ch_pn = []
#     sharpness_climatology = []
#     sharpness_tcn_attn = []
#     sharpness_transformers = []
#     for i in intervals:
#
#         h = 0.5+(i/2)
#         l = 0.5-(i/2)
#
#         h = np.where(np.around(alphas,2) == np.around(h,2))[0][0]
#         l = np.where(np.around(alphas,2) == np.around(l,2))[0][0]
#
#         sharpness.append(np.mean(outputs[:,h] - outputs[:,l]))
#         sharpness_ch_pn.append(np.mean(outputs_ch_pn[:, h] - outputs_ch_pn[:, l]))
#         sharpness_climatology.append(np.mean(outputs_climatology[:, h] - outputs_climatology[:, l]))
#         sharpness_tcn_attn.append(np.mean(outputs_tcn_attn[:, h] - outputs_tcn_attn[:, l]))
#         sharpness_transformers.append(np.mean(outputs_transformers[:, h] - outputs_transformers[:, l]))
#
#
#     print(sharpness==sharpness_climatology)
#
#     fig, ax = plt.subplots()
#     # plt.plot(alphas, alphas)
#     ax.plot(intervals, sharpness, marker='o', color ='g',label = "TCN")
#     ax.plot(intervals, sharpness_tcn_attn, marker='d', color='y', label="TCN_with_attention")
#     ax.plot(intervals, sharpness_transformers, marker='h', color='m', label="Transformers")
#     ax.plot(intervals, sharpness_ch_pn, marker="s", color ='b',label="CH_PeEN")
#     ax.plot(intervals, sharpness_climatology, marker="*", color ='k',label=" Hourly Climatology")
#     ax.set_xlabel('intervals')
#     ax.set_ylabel('Average width')
#     ax.legend(loc = 'upper left')
#
#     plt.savefig(folder_saving + "sharpness_with_all_plots")
#     plt.close()
#
#
# def probabilistic_prediction_plots(X_test,outputs,y_test, col_to_indices_mapping,folder_saving):
#     # outputs_tcn_attn = np.load(
#     #     "/Users/saumya/Desktop/SolarProject/Boulder_CO/Models/year/ML_models_2018/cnn/1hour/final_tcn_week_ahead_1days_lag_small_kernel_1hr_res_attn_quantile/tcn_attention.npy")
#     # outputs_tcn = np.load(
#     #     "/Users/saumya/Desktop/SolarProject/Boulder_CO/Models/year/ML_models_2018/cnn/1hour/final_tcn_week_ahead_1days_lag_small_kernel_1hr_res_quantile/tcn.npy")
#     # outputs_lstm = np.load(
#     #     "/Users/saumya/Desktop/SolarProject/Boulder_CO/Models/year/ML_models_2018/cnn/1hour/final_lstm_50_week_ahead_2day_lag_1hr_res_quantile/lstm.npy")
#     #
#     # sub_outputs_tcn_attn = outputs_tcn_attn[7*24:8*24,:]
#     # sub_outputs_tcn = outputs_tcn[7*24:8*24,:]
#     # sub_outputs_lstm = outputs_lstm[6 * 24:7 * 24, :]
#     index_hour = -n_features + col_to_indices_mapping['hour']
#     index_month = -n_features + col_to_indices_mapping['month']
#     print(X_test[X_test[:,index_month]==3][:,index_hour])
#     print(X_test[:,index_month]==3)
#     index_ghi = -n_features + col_to_indices_mapping['dw_solar']
#     true = y_test[X_test[:,index_month]==3][:10*3,:] #X_test[X_test[:,index_month]==3][:10*5,index_ghi]
#     sub_outputs = outputs[X_test[:,index_month]==3][:10*3,:]
#     true  = np.reshape(true,-1)
#     print(sub_outputs.shape, true.shape)
#
#     true_full_day=[]
#     for i in range(3):
#         true_before = [0 for _ in range(13)]
#         print(true_before+true[10*i:10*(i+1)].tolist()+[0])
#         true_full_day.extend(true_before+true[10*i:10*(i+1)].tolist()+[0])
#
#
#
#
#     x_full_day = list(range(len(true_full_day)))
#     # x_only_day = x_full_day[13:23] + x_full_day[24+13:24+23] + x_full_day[48+13:48+23]
#     # x_only_day = list(range(13,23)) + list(range(24+13,24+23)) + list(range(48+13,48+23))
#     # print(x_full_day,x_only_day)
#
#
#     # x = range(len(true))
#     fig, ax = plt.subplots(1)
#     ax.plot(x_full_day, true_full_day, lw=1, label='true', color='g')
#     ax.plot(x_full_day[13:23], sub_outputs[:10,9], lw=1, label='predicted median', color='blue')
#     ax.plot(x_full_day[24+13:24+23], sub_outputs[10:20, 9], lw=1, color='blue')
#     ax.plot(x_full_day[48+13:48+23], sub_outputs[20:30, 9], lw=1, color='blue')
#     # ax.fill_between(x, sub_outputs[:,0],sub_outputs[:,-1], facecolor='gray', alpha=0.5,
#     #                 label='prediction')
#
#     total_quantiles = sub_outputs.shape[1]
#     for i in range(int(total_quantiles/2)):
#         # label = ""
#         print(i,total_quantiles-i-1)
#         print(alphas[i], alphas[total_quantiles - i - 1])
#         c = plt.cm.Blues(0.2 + .6 * (float(i) / total_quantiles * 2))
#         ax.fill_between(x_full_day[13:23], sub_outputs[:10,i], sub_outputs[:10,total_quantiles-i-1], color=c)
#         ax.fill_between(x_full_day[24+13:24+23], sub_outputs[10:20, i], sub_outputs[10:20, total_quantiles - i - 1], color=c)
#         ax.fill_between(x_full_day[48+13:48+23], sub_outputs[20:30, i], sub_outputs[20:30, total_quantiles - i - 1], color=c)
#
#     # plt.legend(framealpha=1)
#     # plt.show()
#
#     # ax.fill_between(x_test[6 * 24:7 * 24], sub_outputs_lstm[:, 0], sub_outputs_lstm[:, -1], facecolor='green', alpha=0.5,
#     #                 label='prediction lstm')
#     # ax.fill_between(x_test[6 * 24:7 * 24], sub_outputs_tcn[:, 0], sub_outputs_tcn[:, -1], facecolor='yellow', alpha=0.5,
#     #                 label='prediction tcn ')
#     # ax.fill_between(x_test[6 * 24:7 * 24], sub_outputs_tcn_attn[:, 0], sub_outputs_tcn_attn[:, -1], facecolor='red', alpha=0.5,
#     #                 label='prediction tcn+attention')
#     ax.set_xticks([])
#     ax.legend(loc='upper left')
#     ax.set_xlabel('Time: 3 days in March 2018')
#     ax.set_ylabel('Irradiance')
#
#     plt.savefig(folder_saving + "probabilistic_prediction_best_plots_include_night")
#     plt.close()

def main():

    processed_file_path = path + 'processed/' + city + "/"
    # df.to_pickle(processed_file_path + "final_data_At_" + res + "_resolution_2016-2018_updated.pkl")
    df = pd.read_pickle(processed_file_path + "final_data_At_" + res + "_resolution_2016-2018_updated.pkl") ##this files inlcudes the day times - I'll be dropping them later!!

    # final_features.extend(ensmeble_col_list)#["ensemble_0"]) #ensmeble_col_list)
    ## name of the regression model (this helps to distinguish the models within a "CNN"/"LSTM"/"Transformer" super-folder)
    # reg = "baselines"
    reg = "tcn_no_nwp_week_ahead_24_seq_lag_small_kernel_1hr_res"
    # reg = "final_transformers_no_nwp_d64_week_ahead_24_lag_1hr_res_2layers_quantile"
    for season_flag in seasons:
        ## ML_models_2018 is the folder to save results on testyear 2018
        os.makedirs(
            folder_saving + season_flag + "/final_ML_models_"+str(testyear)+"/cnn/"+str(res)+"/"+reg+"/",
            exist_ok=True)
        f = open(folder_saving + season_flag + "/final_ML_models_"+str(testyear)+"/cnn/"+str(res)+"/"+reg+"/results.txt", 'a')

        for lead in lead_times:
            # create dataset with lead
            df_lead = preprocess.create_lead_dataset(df, lead, final_features, target_feature)
            df_lead = df_lead[:len(df_lead)-lead]
            print(df_lead.describe())
            df_lead_2018 = df_lead[df_lead.year == 2018]
            print("len 2018 df lead", len(df_lead_2018))
            ## dropping rows with 0 clear_ghi and taking only daytimes
            df_lead = df_lead[(df_lead['clear_ghi'] > 0) & (df_lead['clear_ghi_target'] > 0) ] #0]
            # print(len(df_lead))
            df_lead = df_lead[(df_lead['zen'] < 85) & (df_lead['zen_target'] < 85)]
            df_lead.reset_index(drop=True, inplace=True)
            print("after removing data points with 0 clear_ghi and selecting daytimes", len(df_lead))

            ## get the yearly/seasonal data
            df, test_startdate, test_enddate = preprocess.get_yearly_or_season_data(df_lead, season_flag, testyear)
            print("\n\n after getting seasonal data (test_startdate; test_enddate)", test_startdate, test_enddate)
            print(df.tail)

            # dividing into training and test set
            df_train, df_heldout = preprocess.train_test_spilt(df, season_flag, testyear)
            print("\n\n after dividing_training_test")
            print("train_set\n", len(df_train))
            print("test_set\n", len(df_heldout))

            if len(df_train) > 0 and len(df_heldout) > 0:
                # extract the X_train, y_train, X_test, y_test
                X_train, y_train, X_heldout, y_heldout, col_to_indices_mapping = preprocess.get_train_test_data(
                    df_train, df_heldout, final_features, target_feature)
                print("\n\n train and test df shapes ")
                print(X_train.shape, y_train.shape, X_heldout.shape, y_heldout.shape)

                # #including features from prev imestamps
                X_train = include_previous_features(X_train, col_to_indices_mapping)
                X_heldout = include_previous_features(X_heldout, col_to_indices_mapping)

                # X_train = X_train[n_timesteps:,:]
                # X_heldout = X_heldout[n_timesteps:, :]
                y_train = y_train[n_timesteps:, :]
                y_heldout = y_heldout[n_timesteps:, :]

                print("Final train size: ", X_train.shape, y_train.shape)
                print("Final heldout size: ", X_heldout.shape, y_heldout.shape)

                ## dividing the X_train data into train(70%)/valid(30%) the heldout data is kept hidden
                X_train, X_valid, y_train, y_valid = train_test_split(
                    X_train, y_train, test_size=0.3, random_state=42)#42

                X_test_before_normalized = X_heldout.copy()
                X_train_before_normalized = X_train.copy()

                ## normalizing the heldout with the X_train used for training
                X_train, X_valid, X_test = preprocess.standardize_from_train(X_train, X_valid, X_heldout)

                print("valid size", X_valid.shape, y_valid.shape)
                print("heldout size:", X_test.shape, y_heldout.shape)

                y_test = y_heldout
                y_true = y_test
                y_pred = None

                f.write("\n" + city + " at Lead " + str(lead) + " and " + season_flag + " Season"+"\n")
                #
                # with open(folder_saving + season_flag + "/ML_models_"+str(testyear)+"/cnn/"+str(res)+"/"+reg+"//model_at_lead_" + str(lead) + ".pkl", 'rb') as file:
                #     model = pickle.load(file)

                ## If doing probabilsitic prediction, just use the test function to scale the ys and crps score too
                ## In case of deterministic use the postprocessing modules

                y_pred, y_valid_pred, valid_crps, test_crps_scaled = cnn.test_DCNN_with_attention(quantile, X_valid,y_valid,
                                                                                X_test, y_test,
                                                                                n_timesteps + 1, n_features,
                                                                                folder_saving + season_flag + "/final_ML_models_" + str(
                                                                                testyear) + "/cnn/" + str(
                                                                                res) + "/" + reg + "/",
                                                                                "dcnn_lag_for_lead_" + str(
                                                                                lead),  X_test_before_normalized, lead)  # "multi_horizon_dcnn", n_outputs=n_output_steps)

                # y_pred, y_valid_pred, valid_crps, test_crps_scaled  = tranformers.test_LSTM(quantile, X_valid, y_valid,X_test, y_test, n_timesteps+1, n_features,hidden_size,alphas, q50,
                #                                folder_saving + season_flag + "/final_ML_models_"+str(testyear)+"/cnn/"+str(res)+"/"+reg+"/","model_lag_for_lead_" + str(lead),X_test_before_normalized, lead)#"multi_horizon_dcnn", n_outputs=n_output_steps)

                # y_pred, y_valid_pred, valid_crps, test_crps_scaled = tranformers.test_transformer(quantile, X_valid, y_valid,
                #                                                                           X_test, y_test, n_timesteps + 1,
                #                                                                            n_features, n_layers,
                #                                                                            num_heads, d_model, alphas,
                #                                                                            q50,
                #                                                                            folder_saving + season_flag + "/final_ML_models_" + str(
                #                                                                                testyear) + "/cnn/" + str(
                #                                                                                res) + "/" + reg + "/",
                #                                                                           "dcnn_lag_for_lead_" + str(
                #                                                                                lead),X_test_before_normalized, lead)  # "multi_horizon_dcnn", n_outputs=n_output_steps)
                #
                #

                print("\n" + city + " at Lead " + str(lead) + " and " + season_flag + " Season")

            #     print("#####TEST#################")
            #     # postprocessing on test and model's RMSE
                y_true, y_pred = postprocess.postprocessing_target(y_pred, y_true)

                rmse_our, mae_our, mb_our, sd_our, r2_our = postprocess.evaluation_metrics(y_true, y_pred)
                #
                print("Performance of our model (rmse, mae, mb, sd, r2): \n\n", round(rmse_our, 2), round(mae_our, 2),
                      round(mb_our, 2), round(sd_our, 2), round(r2_our, 2))

            #     #
                # # baseliens and plots
                # index_clearness = col_to_indices_mapping['clearness_index_input']
                # y_sp = postprocess.smart_persistence_model(X_test_before_normalized, y_test, index_clearness, lead)
                #
                #
                # rmse_sp, mae_sp, mb_sp, sd_sp, r2_sp = postprocess.evaluation_metrics(y_true, y_sp)
                # print("Performance of smart persistence model (rmse, mae, mb, sd, r2): \n\n", round(rmse_sp, 2),
                #       round(mae_sp, 2),
                #       round(mb_sp, 2), round(sd_sp, 2), round(r2_sp, 2))
                #
                #
                #
                # y_CH_PeEN, CH_PeEN_baseline_quantiles, crps_CH_PeEN = get_CH_PeEN_baseline_crps(X_test_before_normalized, X_train_before_normalized, col_to_indices_mapping, y_test)
                # y_climatology, climatology_probabilistic_baseline, crps_climatology = get_climatology_baseline_crps(X_test_before_normalized, X_train_before_normalized, col_to_indices_mapping, y_test)
                #
                # rmse_clm, mae_clm, mb_clm, sd_clm, r2_clm = postprocess.evaluation_metrics(y_true, y_climatology)
                # print("Performance of hourly climatology baseline model (rmse, mae, mb, sd, r2): \n\n",
                #       round(rmse_clm, 2),
                #       round(mae_clm, 2),
                #       round(mb_clm, 2), round(sd_clm, 2), round(r2_clm, 2))
                #
                # rmse_chp, mae_chp, mb_chp, sd_chp, r2_chp = postprocess.evaluation_metrics(y_true, y_CH_PeEN)
                # print("Performance of CH_PeEN baseline model (rmse, mae, mb, sd, r2): \n\n", round(rmse_chp, 2),
                #       round(mae_chp, 2),
                #       round(mb_chp, 2), round(sd_chp, 2), round(r2_chp, 2))
                #
                #     # reliability_diagrams(np.arange(0.05, 1.0, 0.05), y_pred, CH_PeEN_baseline_quantiles, climatology_probabilistic_baseline, y_test,
                # #                          folder_saving + season_flag + "/ML_models_" + str(
                # #                              testyear) + "/cnn/" + str(
                # #                              res) + "/")
                # #
                # #     sharpness_diagrams(np.arange(0.05, 1.0, 0.05),  y_pred, CH_PeEN_baseline_quantiles, climatology_probabilistic_baseline,
                # #                        folder_saving + season_flag + "/ML_models_" + str(
                # #                            testyear) + "/cnn/" + str(
                # #                            res) + "/")
                #
                # # probabilistic_prediction_plots(X_test_before_normalized,y_pred, y_test, col_to_indices_mapping, folder_saving + season_flag + "/ML_models_" + str(
                # #                                                                              testyear) + "/cnn/" + str(
                # #                                                                             res) + "/" )
                # # # #
                # print('CRPS CH_PeEN scaled score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
                #     round(crps_CH_PeEN, 2)) + '\n')
                # f.write('CRPS CH_PeEN scaled score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
                #     round(crps_CH_PeEN, 2)) + '\n')
                #
                # print('CRPS hourly Climatology scaled score on heldout data for year 2018 for lead' + str(
                #     lead) + '=' + str(
                #     round(crps_climatology, 2)) + '\n')
                # f.write('CRPS hourly Climatology scaled score on heldout data for year 2018 for lead' + str(
                #     lead) + '=' + str(
                #     round(crps_climatology, 2)) + '\n')
                #

                # valid_crps = get_crps_for_ngboost(model, X_valid, y_valid)
                # test_crps = get_crps_for_ngboost(model, X_test, y_test)
            #     # print("before crps",y_test.shape)
            #     test_crps_scaled = get_crps_for_ngboost_scaled(model, X_test, y_test, X_test_before_normalized, index_clearghi, lead)
            #
            #     print('CRPS score on valid data for lead' + str(lead) + '=' + str(
            #         round(valid_crps, 2)) + '\n')
            #
            #     print('CRPS scaled score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
            #         round(test_crps_scaled, 2)) + '\n')
            #     f.write('CRPS scaled score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
            #         round(test_crps_scaled, 2)) + '\n')
                # #
            #
            #
            #     ## saving predictions with and without night times (makes sense with deterministic forecasts, otherwise it the 50th percentile predictions)
                test_with_predictions_array = X_test_before_normalized[:,[col_to_indices_mapping['month'], col_to_indices_mapping['day'], col_to_indices_mapping['hour']],-1]
                # test_with_predictions_array['true_GHI'] = y_test
                test_with_predictions_array = pd.DataFrame(test_with_predictions_array, columns=['month', 'day', 'hour'])

                test_with_predictions_array['predictions_GHI'] = y_pred

                test_with_predictions_array_with_night_times = df_lead_2018[
                    ['year', 'month', 'day', 'hour', 'clearness_index']]

                test_with_predictions_array_with_night_times = test_with_predictions_array_with_night_times.merge(
                    test_with_predictions_array, on=['month', 'day', 'hour'], how='left')

                print(test_with_predictions_array_with_night_times.describe())

                print(np.nanmax(test_with_predictions_array_with_night_times.predictions_GHI.values))
                np.save(folder_saving + season_flag + "/final_ML_models_" + str(testyear) + "/cnn/" + str(
                    res) + "/" + reg + "/" + 'model_predictions_with_all_info_including_night_times.npy',
                        test_with_predictions_array_with_night_times)

            else:
                print("not enough data for the season: ", season_flag, "and lead: ", lead)

        f.close()

if __name__=='__main__':
    main()

# In case you want to find skill score
#     # calculate the skill score of our model over persistence model
#     skill_sp = postprocess.skill_score(rmse_our, rmse_sp)
#     print("\nSkill rmse of our model over smart persistence: ", round(skill_sp, 2))
#
#     skill_np = postprocess.skill_score(rmse_our, rmse_np)
#     print("\nSkill rmse of our model over normal persistence: ", round(skill_np, 2))
#
#     skill_clm = postprocess.skill_score(rmse_our, rmse_clm)
#     print("\nSkill rmse of our model over hourly climatology baseline: ", round(skill_clm, 2))
#
#     skill_chp = postprocess.skill_score(rmse_our, rmse_chp)
#     print("\nSkill rmse of our model over CH_PeEN baseline: ", round(skill_chp, 2))
#
#     f.write('skill rmse score on heldout data for year 2018 for lead' + str(lead) + '=' + str(round(skill_sp, 2)) + '\n')
#     f.write('skill wrt hourly climatology rmse score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
#         round(skill_clm, 2)) + '\n')
#     f.write(
#         'skill wrt CH_PeEN rmse score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
#             round(skill_chp, 2)) + '\n')
#
#     skill_sp = postprocess.skill_score(mae_our, mae_sp)
#     print("\nSkill mae of our model over smart persistence: ", round(skill_sp, 2))
#
#     skill_np = postprocess.skill_score(mae_our, mae_np)
#     print("\nSkill mae of our model over normal persistence: ", round(skill_np, 2))
#
#     skill_clm = postprocess.skill_score(mae_our, mae_clm)
#     print("\nSkill mae of our model over climatology baseline: ", round(skill_clm, 2))
#
#     skill_chp = postprocess.skill_score(mae_our, mae_chp)
#     print("\nSkill mae of our model over climatology baseline: ", round(skill_chp, 2))
#
#     f.write('skill mae score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
#         round(skill_sp, 2)) + '\n')
#     f.write(
#         'skill wrt hourly climatology mae score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
#             round(skill_clm, 2)) + '\n')
#     f.write(
#         'skill wrt CH_PeEN mae score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
#             round(skill_chp, 2)) + '\n')


from tools.date_parser import get_utc_from_timestamp
from tracking.courier import Courier
from tracking.requests_handler import RequestsHandler

class Asendia(Courier):
    name = "Asendia"
    fromto = f"CN{Courier.r_arrow}FR"
    headers = {"Content-Type": "application/json", "Accept-Language": "fr"}
    url = "https://tracking.asendia.com/alliot/items/references"

    def get_url_for_browser(self, idship):
        return f"https://tracking.asendia.com/tracking/{idship}"

    @RequestsHandler(max_retry=1)
    def get_content(self, idship, request):
        return request.request_json(
            "POST",
            self.url,
            json={"criteria": [idship], "shipped": False},
            headers=self.headers,
        )

    def parse_content(self, content):
        events = []

        timeline = content[0]["events"]
        for event in timeline:
            label = event["translatedLabelBC"]
            location = event["location"]["name"]

            if label and location:
                country = event["location"]["countryCode"]
                if country not in location:
                    location = ", ".join((location, country))

                date = get_utc_from_timestamp(event["date"] / 1000)

                events.append(dict(date=date, status=location, label=label))

        return events, {}


from django_filters.rest_framework import DjangoFilterBackend
from django_filters import FilterSet
from django_filters import rest_framework as filters

from rest_framework.parsers import JSONParser
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status
from rest_framework import generics
from rest_framework import mixins
from rest_framework.decorators import action
from rest_framework.authentication import (
    SessionAuthentication,
    BasicAuthentication,
    TokenAuthentication,
)
from rest_framework.permissions import IsAuthenticated, IsAdminUser
from rest_framework import viewsets
from rest_framework.decorators import action

from poll.forms import PollForm, ChoiceForm
from poll.models import *
from poll.serializers import (
    QuestionSerializer,
    ChoiceSerializer,
    QuestionSearchSerializer,
)

class QuestionSearchViewSet(mixins.ListModelMixin, viewsets.GenericViewSet):
    serializer_class = QuestionSearchSerializer

    def get_queryset(self):
        result = QuestionDocument.search()

class PollFilter(FilterSet):
    tags = filters.CharFilter(method="filter_by_tags")

    class Meta:
        model = Question
        fields = ["tags"]

    def filter_by_tags(self, queryset, name, value):
        tag_names = value.strip().split(",")
        tags = Tag.objects.filter(name__in=tag_names)
        return queryset.filter(tags__in=tags).distinct()

class PollViewSet(viewsets.ModelViewSet):
    serializer_class = QuestionSerializer
    queryset = Question.objects.all()
    lookup_field = "id"
    filter_backends = (DjangoFilterBackend,)
    filter_class = PollFilter
    authentication_classes = (TokenAuthentication,)

    @action(detail=True, methods=["GET"])
    def choices(self, request, id=None):
        question = self.get_object()
        choices = Choice.objects.filter(question=question)
        serializer = ChoiceSerializer(choices, many=True)
        return Response(serializer.data, status=200)

    @action(detail=True, methods=["POST"])
    def choice(self, request, id=None):
        question = self.get_object()
        data = request.data
        data["question"] = question.id
        serializer = ChoiceSerializer(data=data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=201)
        return Response(serializer.erros, status=400)

class PollListView(
    generics.GenericAPIView,
    mixins.ListModelMixin,
    mixins.CreateModelMixin,
    mixins.RetrieveModelMixin,
    mixins.UpdateModelMixin,
    mixins.DestroyModelMixin,
):
    serializer_class = QuestionSerializer
    queryset = Question.objects.all()
    lookup_field = "id"
    authentication_classes = [
        TokenAuthentication,
        SessionAuthentication,
        BasicAuthentication,
    ]
    permission_classes = [IsAuthenticated, IsAdminUser]

    def get(self, request, id=None):
        if id:
            return self.retrieve(request, id)
        else:
            return self.list(request)

    def post(self, request):
        return self.create(request)

    def perform_create(self, serializer):
        serializer.save(created_by=self.request.user)

    def put(self, request, id=None):
        return self.update(request, id)

    def perform_update(self, serializer):
        print(self.request.user)
        serializer.save(created_by=self.request.user)

    def delete(self, request, id=None):
        return self.destroy(request, id)

class PollAPIView(APIView):
    def get(self, request):
        questions = Question.objects.all()
        serailizer = QuestionSerializer(questions, many=True)
        return Response(serailizer.data, status=200)

    def post(self, request):
        data = request.data
        serializer = QuestionSerializer(data=data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=201)
        return Response(serializer.erros, status=400)

class PollDetailView(APIView):
    def get_object(self, id):
        try:
            return Question.objects.get(id=id)
        except Question.DoesNotExist as e:
            return Response({"error": "Given question object not found."}, status=404)

    def get(self, request, id=None):
        instance = self.get_object(id)
        serailizer = QuestionSerializer(instance)
        return Response(serailizer.data)

    def put(self, request, id=None):
        data = request.data
        instance = self.get_object(id)
        serializer = QuestionSerializer(instance, data=data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=200)
        return Response(serializer.erros, status=400)

    def delete(self, request, id=None):
        instance = self.get_object(id)
        instance.delete()
        return HttpResponse(status=204)

@csrf_exempt
def poll(request):
    if request.method == "GET":
        questions = Question.objects.all()
        serailizer = QuestionSerializer(questions, many=True)
        return JsonResponse(serailizer.data, safe=False)

    elif request.method == "POST":
        json_parser = JSONParser()
        data = json_parser.parse(request)
        serializer = QuestionSerializer(data=data)
        if serializer.is_valid():
            serializer.save()
            return JsonResponse(serializer.data, status=201)
        return JsonResponse(serializer.errors, status=400)

@csrf_exempt
def poll_details(request, id):
    try:
        instance = Question.objects.get(id=id)
    except Question.DoesNotExist as e:
        return JsonResponse({"error": "Given question object not found."}, status=404)

    if request.method == "GET":
        serailizer = QuestionSerializer(instance)
        return JsonResponse(serailizer.data)

    elif request.method == "PUT":
        json_parser = JSONParser()
        data = json_parser.parse(request)
        serializer = QuestionSerializer(instance, data=data)
        if serializer.is_valid():
            serializer.save()
            return JsonResponse(serializer.data, status=200)
        return JsonResponse(serializer.erros, status=400)

    elif request.method == "DELETE":
        instance.delete()
        return HttpResponse(status=204)


import curses

class PromptWindow:
    def __init__(self,stdscr=None,title=None):
        self.stdscr = stdscr
        self.title = title
        self.box_text = ""
        self.cursor_index = 0
        self.screen_h = 0
        self.screen_w = 0
        self.win_h = 0
        self.win_w = 0
        self.win_x = 0
        self.win_y = 0
    def set_win_dimensions(self, height, width ):
        self.screen_h, self.screen_w = self.stdscr.getmaxyx()
        self.win_h = height
        self.win_w = width
        self.win_y = (self.screen_h // 2) - (height // 2)
        self.win_x = (self.screen_w // 2) - (width // 2)

    def display(self, stdscr, height= 3, width = 20):
        self.stdscr = stdscr
        self.set_win_dimensions(height,width)
        win = curses.newwin( self.win_h,
                             self.win_w,
                             self.win_y,
                             self.win_x)

        win.box()
        win.addstr(0, (( self.win_w//2) - len(self.title)//2), self.title )

        running = True;

        while running:
            win.clear()
            win.box()
            win.addstr(0, (( self.win_w//2) - len(self.title)//2), self.title )

            if (len(self.box_text) < (self.win_w - 3)):
                win.addstr(1,1, self.box_text)
            else:
                win.addstr(1,1, self.box_text[self.cursor_index - (self.win_w-3):self.cursor_index])

            if (len(self.box_text) < (self.win_w - 3)):
                win.move(1,self.cursor_index+1)
            stdscr.refresh()
            win.refresh()
            key = stdscr.getkey()
            if (key == "KEY_BACKSPACE"): #backspace
                if (self.cursor_index == (len(self.box_text)-1)):
                    self.box_text = self.box_text[:-1]
                else:
                    self.box_text =self.box_text[:self.cursor_index-1]+\
                            self.box_text[self.cursor_index:]
                    if self.cursor_index > 0:
                        self.cursor_index -=1
            elif (key == "\n"): #enter
                running = False
            elif (key == "KEY_LEFT"): #left
                if self.cursor_index >= 0:
                    self.cursor_index -=1
            elif (key == "KEY_RIGHT"): #right
                if self.cursor_index < len(self.box_text):
                    self.cursor_index +=1
            else:
                if (self.cursor_index == (len(self.box_text)-1)):
                    self.box_text+= (key)
                else:
                    self.box_text =self.box_text[:self.cursor_index]+\
                            key + self.box_text[self.cursor_index:]

                self.cursor_index +=1

            #win.refresh()
            #stdscr.refresh()
        return self.box_text

        win.refresh()
        stdscr.refresh()

if __name__ == "__main__":
    my_aw = PromptWindow(title="enter rss feed link")
    print(curses.wrapper(my_aw.display, width=50))


from django.urls import path, include
from .views import ClientViewSet, ContractViewSet, EventViewSet

from rest_framework import routers

router = routers.DefaultRouter()
router.register(r'clients', ClientViewSet)
router.register(r'contracts', ContractViewSet)
router.register(r'events', EventViewSet)

urlpatterns = [
    path('', include(router.urls)),
]