#!/usr/bin/env python3
"""
flowdronetello.py

FlowDroNet implementation for the DJI/Ryze Tello

Written by Moritz Sperling
Based on the work of A. Loquercio et al., 2018 (https://github.com/uzh-rpg/rpg_public_dronet)
and P. Ferriere (https://github.com/philferriere/tfoptflow)
and D. Fuentes (https://github.com/damiafuentes/DJITelloPy)

Licensed under the MIT License (see LICENSE for details)
"""

import os
import sys
import cv2
import json
import time
import pygame
import djitellopy
import numpy as np
from copy import deepcopy
from pygame.locals import *
localpath = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, localpath + '/../workflow/util/')
from img_utils import flow_to_img, pred_as_bar, pred_as_indicator
sys.path.insert(0, './tfoptflow/tfoptflow/')
from model_flowdronet import ModelFlowDroNet, _DEFAULT_FLOWDRONET_OPTS

#
#        8888888          d8b 888    d8b          888 d8b                   888    d8b
#          888            Y8P 888    Y8P          888 Y8P                   888    Y8P
#          888                888                 888                       888
#          888   88888b.  888 888888 888  8888b.  888 888 .d8888b   8888b.  888888 888  .d88b.  88888b.
#          888   888 "88b 888 888    888     "88b 888 888 88K          "88b 888    888 d88""88b 888 "88b
#          888   888  888 888 888    888 .d888888 888 888 "Y8888b. .d888888 888    888 888  888 888  888
#          888   888  888 888 Y88b.  888 888  888 888 888      X88 888  888 Y88b.  888 Y88..88P 888  888
#        8888888 888  888 888  "Y888 888 "Y888888 888 888  88888P' "Y888888  "Y888 888  "Y88P"  888  888
#

class FrontEnd(object):
    """ Maintains the Tello display and moves it through the keyboard keys.
        Press escape key to quit.
        The controls are:
            - Tab: Takeoff
            - Shift: Land
            - Space: Emergency shutdown
            - Backspace: Shutdown
            - WSAD: Forward, backward, left and right.
            - Q and E: Counter clockwise and clockwise rotations
            - R and F: Up and down.
            - P: Switch through controllable parameters
            - + and -: Raise or lower parameter
            - #: Enable / Disable FlowDroNet
            - O: Show / Hide optical flow
            - C: Toggle recording of frames
    """

    def __init__(self):

        # FlowDroNet Configuration
        self.pwcnet_ckpt_path = localpath + '/models/pwc-net/pwcnet.ckpt-11000'
        self.dronet_model_path = localpath + '/models/FlowDroNet/model_graph_final.pb'
        self.output_folder = '/recordings/test_0'
        self.logfile = "log.json"
        self.FPS = 10
        self.hud_scale = 1.5
        self.starting_speed = 100

        # Configuration of controllable parameters
        # initial values
        self.params = {
            'v_max': 50,
            'r_max': 100,
            'r_scale': 0,
            'prev_len': 3,
            'prev_coll': 0.6,
            'prev_steer': 0.3,
            'reverse_thresh': 0.6,
            'reverse_offset': 0.7,
        }
        # stepsize
        self.params_d = {
            'v_max': 5,
            'r_max': 5,
            'r_scale': 50,
            'prev_len': 1,
            'prev_coll': 0.1,
            'prev_steer': 0.1,
            'reverse_thresh': 0.1,
            'reverse_offset': 0.1,
        }
        # max (min is 0)
        self.params_m = {
            'v_max': 100,
            'r_max': 100,
            'r_scale': 1000,
            'prev_len': 10,
            'prev_coll': 1,
            'prev_steer': 1,
            'reverse_thresh': 1,
            'reverse_offset': 1,
        }

        # Logging
        self.log_info = {
            'cur_v': [],
            'cur_s': [],
            'prd_c': [],
            'prd_s': []
        }

        # Init internal variables (do not change anything below here)
        self.for_back_velocity = 0  # Drone velocities between -100~100
        self.left_right_velocity = 0
        self.up_down_velocity = 0
        self.yaw_velocity = 0
        self.internalSpeed = 100
        self.send_rc_control = False
        self.v_old = [0, ]
        self.s_old = [0, ]
        self.was_dronet = True
        self.last_pred_col = 0
        self.last_pred_ang = 0
        self.last_time = time.time()
        self.is_armed = False
        self.battery_percentage = 0
        self.should_stop = False
        self.record_data = False
        self.show_raw_data = False
        self.frame = None
        self.current_parameter = 0
        self.param_keys = list(self.params.keys())
        self.log_count = 0

        # Configure the pwc-net model for inference, starting with the default options
        self.nn_opts = deepcopy(_DEFAULT_FLOWDRONET_OPTS)
        self.nn_opts['verbose'] = False
        self.nn_opts['batch_size'] = 1
        self.nn_opts['ckpt_path'] = self.pwcnet_ckpt_path
        self.nn_opts['dronet_mode'] = 'rgb'
        self.nn_opts['dronet_model_path'] = self.dronet_model_path
        self.target_size = (int(self.nn_opts['y_shape'][1]), int(self.nn_opts['y_shape'][0]))

        # Create output folder if not exists
        if not os.path.exists(os.path.join(self.output_folder, str(self.log_count).zfill(2))):
            os.makedirs(os.path.join(self.output_folder, str(self.log_count).zfill(2)))

        # Init pygame with display
        pygame.init()
        pygame.display.init()
        pygame.display.set_caption("FlowDroNeTello")
        self.screen = pygame.display.set_mode([int(self.hud_scale * self.target_size[0]) + 44,
                                               int(self.hud_scale * self.target_size[1]) + 36])

        # Show bootscreen
        loadimg = cv2.imread(localpath + "/../misc/images/startuplogo.png")
        loadimg = np.fliplr(loadimg)
        loadimg = np.rot90(loadimg)
        loadimg = pygame.surfarray.make_surface(loadimg)
        self.screen.fill([0, 0, 0])
        self.screen.blit(loadimg, (0, 0))
        pygame.display.update()
        pygame.time.set_timer(USEREVENT + 1, int(1. / self.FPS * 1000))

        # Instantiate the model in inference mode and display the model configuration
        self.nn = ModelFlowDroNet(mode='test', options=self.nn_opts)
        self.nn.print_config()

        # Init Tello object that interacts with the Tello drone
        self.tello = djitellopy.Tello()

    #
    #      888b     d888          d8b               888
    #      8888b   d8888          Y8P               888
    #      88888b.d88888                            888
    #      888Y88888P888  8888b.  888 88888b.       888      .d88b.   .d88b.  88888b.
    #      888 Y888P 888     "88b 888 888 "88b      888     d88""88b d88""88b 888 "88b
    #      888  Y8P  888 .d888888 888 888  888      888     888  888 888  888 888  888
    #      888   "   888 888  888 888 888  888      888     Y88..88P Y88..88P 888 d88P
    #      888       888 "Y888888 888 888  888      88888888 "Y88P"   "Y88P"  88888P"
    #                                                                         888
    #                                                                         888
    #                                                                         888
    #

    def run(self):

        if not self.tello.connect():
            print("Tello not connected")
            return

        if not self.tello.set_speed(self.internalSpeed):
            print("Speed as low as possible")
            return

        # In case streaming is on. This happens when we quit this program without the escape key.
        if not self.tello.streamoff():
            print("Could not stop video stream")
            return

        if not self.tello.streamon():
            print("Could not start video stream")
            return

        # start framereader and read initial frame
        frame_read = self.tello.get_frame_read()
        img = cv2.resize(frame_read.frame, self.target_size)
        prv = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

        # Main Loop
        while not self.should_stop:
            # sometimes read battery status
            if np.random.random() < 0.05:
                self.battery_percentage = self.tello.get_battery()

            # read frame
            img = cv2.resize(frame_read.frame, self.target_size)
            cur = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

            # create img pair
            if self.for_back_velocity > 0:
                img_pair = [prv, cur]
            else:
                img_pair = [cur, prv]

            # get output from flowdronet
            if self.show_raw_data or self.is_armed:
                flow = self.get_dronet_command(img_pair)
            else:
                flow = None
            prv = cur

            # produce hud
            hud = self.update_hud(cur, flow)

            # save frame and hud if recording
            if self.record_data:
                self.save_img(frame_read.frame, 'rec_img')
                self.save_img(hud, 'rec_hud')

            # handle input from dronet or user
            for event in pygame.event.get():
                if event.type == USEREVENT + 1:
                    self.send_input()
                elif event.type == QUIT:
                    self.should_stop = True
                elif event.type == KEYDOWN:
                    if (event.key == K_ESCAPE) or (event.key == K_BACKSPACE):
                        self.should_stop = True
                    else:
                        self.keydown(event.key)
                elif event.type == KEYUP:
                    self.keyup(event.key)

                # shutdown stream
                if frame_read.stopped:
                    frame_read.stop()
                    break

            # wait a little
            time.sleep(1 / self.FPS)

        # save log
        if self.record_data:
            with open(os.path.join(self.output_folder, str(self.log_count).zfill(2), self.logfile) , "w") as f:
                json.dump(self.log_info, f)

        # always call before finishing to deallocate resources
        self.tello.end()
        pygame.quit()
        exit(0)

    def get_dronet_command(self, img_pair):

        # inference
        flow, steer_coll = self.nn.flowdronet_predict((img_pair,))
        theta, p_t = steer_coll[0], steer_coll[1]
        self.last_pred_col = p_t
        self.last_pred_ang = theta

        # calculate resulting velocity:
        if p_t < self.params['reverse_thresh']:
            # if the probability to crash is low, then go forward with v_max * (1 - prob) but also
            # take a percentage of the previous average speed into account:
            # v_new = (1 - a) * v_old +  a * (1 - p_t) * v_max
            mean_velocity = np.mean(self.v_old)
            prev_velocity = (1 - self.params['prev_coll']) * mean_velocity if not np.isnan(mean_velocity) else 0
            velocity = prev_velocity + self.params['prev_coll'] * (1 - p_t) * self.params['v_max']
        else:
            # if the probability to crash is high, instantly move backwards to keep flow alive.
            # v_new = (1 - d - p_t) * v_max
            velocity = (1 - self.params['reverse_offset'] - p_t) * self.params['v_max']
            self.v_old.clear()

        # theta_new = (1 - b) * s_old + b * theta * scale with range[-r_max .. r_max]
        steering = np.clip((1 - self.params['prev_steer']) * np.mean(self.s_old)
                           + self.params['prev_steer'] * theta * self.params['r_scale'],
                       -self.params['r_max'], self.params['r_max']) * np.clip(p_t * 2, 0.2, 1)

        # set forward and yaw velocity if dronet is armed
        if self.is_armed:
            # append current velocity and steering angle for averaging
            self.v_old.append(velocity)
            self.s_old.append(steering)

            # set actual velocities
            self.for_back_velocity = int(velocity)
            self.yaw_velocity = int(steering)

            # delete oldest values
            if len(self.v_old) > self.params['prev_len']:
                self.v_old.pop(0)
                self.s_old.pop(0)

        if self.record_data:
            self.log_info['cur_v'].append(int(velocity))
            self.log_info['cur_s'].append(int(steering))
            self.log_info['prd_c'].append(float(p_t))
            self.log_info['prd_s'].append(float(theta))

        return flow

    #
    #      8888888                            888         888b     d888          888    888                    888
    #        888                              888         8888b   d8888          888    888                    888
    #        888                              888         88888b.d88888          888    888                    888
    #        888   88888b.  88888b.  888  888 888888      888Y88888P888  .d88b.  888888 88888b.   .d88b.   .d88888 .d8888b
    #        888   888 "88b 888 "88b 888  888 888         888 Y888P 888 d8P  Y8b 888    888 "88b d88""88b d88" 888 88K
    #        888   888  888 888  888 888  888 888         888  Y8P  888 88888888 888    888  888 888  888 888  888 "Y8888b.
    #        888   888  888 888 d88P Y88b 888 Y88b.       888   "   888 Y8b.     Y88b.  888  888 Y88..88P Y88b 888      X88
    #      8888888 888  888 88888P"   "Y88888  "Y888      888       888  "Y8888   "Y888 888  888  "Y88P"   "Y88888  88888P'
    #                       888
    #                       888
    #                       888
    #

    def keydown(self, key):
        """ Update velocities based on key pressed
        Arguments:
            key: pygame key
        """
        if key == pygame.K_w:  # set forward velocity
            self.is_armed = False
            self.for_back_velocity = self.params['v_max']
        elif key == pygame.K_s:  # set backward velocity
            self.is_armed = False
            self.for_back_velocity = -self.params['v_max']
        elif key == pygame.K_a:  # set left velocity
            self.is_armed = False
            self.left_right_velocity = -self.params['v_max']
        elif key == pygame.K_d:  # set right velocity
            self.is_armed = False
            self.left_right_velocity = self.params['v_max']
        elif key == pygame.K_r:  # set up velocity
            self.is_armed = False
            self.up_down_velocity = self.params['v_max']
        elif key == pygame.K_f:  # set down velocity
            self.is_armed = False
            self.up_down_velocity = -self.params['v_max']
        elif key == pygame.K_e:  # set yaw clockwise velocity
            self.is_armed = False
            self.yaw_velocity = self.params['r_max']
        elif key == pygame.K_q:  # set yaw counter clockwise velocity
            self.is_armed = False
            self.yaw_velocity = -self.params['r_max']
        elif key == pygame.K_TAB:  # takeoff
            self.tello.takeoff()
            self.send_rc_control = True
        elif key == pygame.K_LSHIFT:  # land
            self.is_armed = False
            self.tello.land()
            self.send_rc_control = False
        elif key == pygame.K_SPACE:  # emergency shutdown
            self.is_armed = False
            self.tello.emergency()
            self.send_rc_control = False
            self.should_stop = True
        elif key == pygame.K_HASH:  # arm/disarm dronet
            self.is_armed = not self.is_armed

            # instantly start moving to actually have optical flow and not just noise
            self.for_back_velocity = self.starting_speed if self.is_armed else 0
            self.yaw_velocity = 0
        elif key == pygame.K_p:  # switch through parameters
            if self.current_parameter < len(self.params) - 1:
                self.current_parameter = self.current_parameter + 1
            else:
                self.current_parameter = 0
        elif key == pygame.K_PLUS:  # raise current parameter
            what = self.param_keys[self.current_parameter]
            if self.params[what] < self.params_m[what] - 0.01:
                self.params[what] = self.params[what] + self.params_d[what]
        elif key == pygame.K_MINUS:  # lower current parameter
            what = self.param_keys[self.current_parameter]
            if self.params[what] > 0.01:
                self.params[what] = self.params[what] - self.params_d[what]
        elif key == pygame.K_o:  # show/hide optical flow and predictbars
            self.show_raw_data = not self.show_raw_data
            if self.show_raw_data:
                self.screen = pygame.display.set_mode([int(self.hud_scale * self.target_size[0] + 76),
                                                       int(self.hud_scale * 2 * self.target_size[1] + 76)])
            else:
                self.screen = pygame.display.set_mode([int(self.hud_scale * self.target_size[0] + 44),
                                                       int(self.hud_scale * self.target_size[1] + 36)])
        elif key == pygame.K_c:  # toggle recording of data
            self.record_data = not self.record_data

            if self.record_data:
                if not os.path.exists(os.path.join(self.output_folder, str(self.log_count).zfill(2))):
                    os.makedirs(os.path.join(self.output_folder, str(self.log_count).zfill(2)))
                    print(os.path.join(self.output_folder, str(self.log_count).zfill(2)))
                else:
                    with open(os.path.join(self.output_folder, str(self.log_count).zfill(2), self.logfile), "w") as f:
                        json.dump(self.log_info, f)
                    self.log_count = self.log_count + 1
                    self.log_info['cur_v'].clear()
                    self.log_info['cur_s'].clear()
                    self.log_info['prd_c'].clear()
                    self.log_info['prd_s'].clear()

    def keyup(self, key):
        """ Update velocities based on key released
        Arguments:
            key: pygame key
        """
        if key == pygame.K_w or key == pygame.K_s:  # set zero forward/backward velocity
            self.for_back_velocity = 0
        elif key == pygame.K_a or key == pygame.K_d:  # set zero left/right velocity
            self.left_right_velocity = 0
        elif key == pygame.K_r or key == pygame.K_f:  # set zero up/down velocity
            self.up_down_velocity = 0
        elif key == pygame.K_q or key == pygame.K_e:  # set zero yaw velocity
            self.yaw_velocity = 0

    def send_input(self):
        """ Update routine. Send velocities to Tello."""
        print("V: " + str(self.for_back_velocity) + "; Y: " + str(self.yaw_velocity))
        if self.send_rc_control:
            self.tello.send_rc_control(self.left_right_velocity, self.for_back_velocity, self.up_down_velocity,
                                       self.yaw_velocity)

    #
    #        888    888          888                                888b     d888          888    888                    888
    #        888    888          888                                8888b   d8888          888    888                    888
    #        888    888          888                                88888b.d88888          888    888                    888
    #        8888888888  .d88b.  888 88888b.   .d88b.  888d888      888Y88888P888  .d88b.  888888 88888b.   .d88b.   .d88888 .d8888b
    #        888    888 d8P  Y8b 888 888 "88b d8P  Y8b 888P"        888 Y888P 888 d8P  Y8b 888    888 "88b d88""88b d88" 888 88K
    #        888    888 88888888 888 888  888 88888888 888          888  Y8P  888 88888888 888    888  888 888  888 888  888 "Y8888b.
    #        888    888 Y8b.     888 888 d88P Y8b.     888          888   "   888 Y8b.     Y88b.  888  888 Y88..88P Y88b 888      X88
    #        888    888  "Y8888  888 88888P"   "Y8888  888          888       888  "Y8888   "Y888 888  888  "Y88P"   "Y88888  88888P'
    #                                888
    #                                888
    #                                888
    #

    def update_hud(self, frame, flow):
        """Draw drone info and print on frame"""

        # resize to desired scale
        new_size = (int(self.target_size[0] * self.hud_scale), int(self.target_size[1] * self.hud_scale))
        img = cv2.resize(frame, new_size)

        # show blinking red dot when recording
        if self.record_data and round(time.time()) % 2 == 0:
            cv2.putText(img, "Recording", (img.shape[1] - 187, 38),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 0, 0), lineType=30)
            if round(time.time()) % 2 == 0:
                cv2.circle(img, (new_size[0] - 30, 30), 15, (255, 0, 0), -1)

        # show battery percentage
        #cv2.putText(img, "Battery: {:d} %".format(int(self.battery_percentage)), (img.shape[1] - 187, img.shape[0] - 12),
        #            cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 0, 0), lineType=30)

        # display info on screen
        if self.is_armed:
            stats = ["FlowDroNet active."]
            if self.was_dronet:
                stats.append("Predictions:")
                stats.append("[C: {:4.3f}] [SA: {:4.3f}]".format(float(self.last_pred_col), float(self.last_pred_ang)))
                stats.append("Commands:")
                stats.append("[V: {:03d}] [SA: {:03d}]".format(int(self.for_back_velocity), int(self.yaw_velocity)))
            else:
                stats.append("Command overwritten ...")
        elif self.show_raw_data:
            stats = ["FlowDroNet disarmed.", "Predictions:",
                     "[C: {:4.3f}] [SA: {:4.3f}]".format(float(self.last_pred_col), float(self.last_pred_ang))]
        else:
            stats = ["FlowDroNet disabled."]

        stats.append(self.param_keys[self.current_parameter]
                     + ": {:4.1f}".format(self.params[self.param_keys[self.current_parameter]]))
        for idx, stat in enumerate(stats):
            text = stat.lstrip()
            cv2.putText(img, text, (10, 30 + (idx * 30)),
                        cv2.FONT_HERSHEY_SIMPLEX,
                        0.8, (255, 0, 0), lineType=30)

        # display yaw velocity as indicator in the middle
        hspacer = np.ones((12, img.shape[1], 3), dtype=np.uint8) * 255
        yawbar = pred_as_indicator(self.yaw_velocity / 100., (20, img.shape[1]), "Yaw Velocity")
        yawbar = cv2.cvtColor(yawbar, cv2.COLOR_BGR2RGB)

        # show flow and predictions if desired
        if (self.show_raw_data or self.is_armed) and flow is not None:
            steerbar = pred_as_indicator(self.last_pred_ang, (20, img.shape[1]), "Steering Pred.")
            steerbar = cv2.cvtColor(steerbar, cv2.COLOR_BGR2RGB)
            flow_img = cv2.resize(flow_to_img(flow), new_size) if flow.shape[2] == 2 else cv2.resize(flow, new_size)
            self.frame = np.vstack((img, hspacer, yawbar, hspacer, steerbar, hspacer, flow_img))
        else:
            self.frame = np.vstack((img, hspacer, yawbar, hspacer))

        # display velocity as bar on the side
        vspacer = np.ones((self.frame.shape[0], 12, 3), dtype=np.uint8) * 255
        velobar = pred_as_indicator(self.for_back_velocity / 100, (20, self.frame.shape[0]), "Velocity",
                                    mode="vertical")
        velobar = cv2.cvtColor(velobar, cv2.COLOR_BGR2RGB)

        # show predictions if desired
        if (self.show_raw_data or self.is_armed) and flow is not None:
            vspacer = np.ones((self.frame.shape[0], 12, 3), dtype=np.uint8) * 255
            collbar = pred_as_bar(self.last_pred_col, (self.frame.shape[0], 20), "Collision Prob.")
            collbar = cv2.cvtColor(collbar, cv2.COLOR_BGR2RGB)
            output = np.hstack((vspacer, self.frame, vspacer, velobar, vspacer, collbar))
        else:
            output = np.hstack((vspacer, self.frame, vspacer, velobar))

        # update output
        self.frame = np.fliplr(output)
        self.frame = np.rot90(self.frame)
        self.frame = pygame.surfarray.make_surface(self.frame)
        self.screen.fill([0, 0, 0])
        self.screen.blit(self.frame, (0, 0))
        pygame.display.update()

        return cv2.cvtColor(output, cv2.COLOR_BGR2RGB)

    def save_img(self, img, string):
        """ Save frame to disk with timestamp. """
        folder = os.path.join(self.output_folder, str(self.log_count).zfill(2))
        fname = os.path.join(folder, time.strftime(string + "_%Y%m%d_%H%M%S_0.png", time.gmtime()))
        i = 0
        while os.path.exists(fname):
            fname = fname.replace('_' + str(i) + '.', '_' + str(i + 1) + '.', )
            i = i + 1
        cv2.imwrite(fname, img)

def main():
    frontend = FrontEnd()

    # run frontend
    frontend.run()

if __name__ == '__main__':
    main()


import datetime
from functools import reduce
from itertools import groupby
from operator import itemgetter

import pandas as pd
from IPython.display import display_html, display_svg

def matrix_svg(value_counts, ranges=(3, 5, 10), display=True):

    MONTHS = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep',
              'Oct', 'Nov', 'Dec')

    def get_color(count):
        if count == 0:
            return '#eee'
        elif count < ranges[0]:
            return '#c6e48b'
        elif count < ranges[1]:
            return '#7bc96f'
        elif count < ranges[2]:
            return '#239a3b'
        else:
            return '#196127'

    def build_svg_day(weekday, date, month, count, x):
        fmt = ('<rect class="day" width="10" height="10" x="{x}" y="{y}" ' +
               'fill="{color}" data-count="{count}" data-date="{date}"></rect>')
        y = weekday * 12
        color = get_color(count)
        return fmt.format(x=x, y=y, color=color, count=count, date=date)

    def build_svg_week(week, days):
        svg = '<g transform="translate({x}, 0)">'.format(x=week * 13)
        x = 13 - week
        for day in days:

            svg += build_svg_day(*day, x)
        return svg + '</g>'

    def build_svg_month(x, month):
        return '<text x="{x}" y="{y}" class="month">{month}</text>'.format(
            x=x, y=-10, month=month)

    def build_svg(dates, width=674, height=104):

        def list_date_tuples(start, end, counter):
            today = datetime.date.today()

            def _init_date_tuple(date):
                return (date.weekday(), date, MONTHS[date.month - 1],
                        counter(date))

            return [_init_date_tuple(d)
                    for d in (today - datetime.timedelta(days=start + x)
                    for x in reversed(range(end)))]

        def build_counter(xs, ys):
            def _inner(x):
                t = pd.Timestamp(x)
                return ys[xs.index(t)] if t in xs else 0
            return _inner

        # build the counter from `dates` -- this allows us to count how many
        # tickets are linked to a given `date`
        counter = build_counter(*list(zip(*dates)))

        # build a weeekday, date, month tuple list
        days = list_date_tuples(0, 365, counter)

        # ensure we start with a 1st day of the week -- a.k.a. Monday
        days = list_date_tuples(365, days[0][0], counter) + days

        def split(acc, curr):
            xs, i = acc
            if curr[0] == 0:
                return xs + [[curr]], i + 1
            xs[i] = xs[i] + [curr]
            return xs, i

        weeks = list(reduce(split, days, ([[]], 0))[0])

        def reducer(y, x):
            if len(y) > 0:
                prev = y[-1]
                if prev[1] == x[1]:
                    return y + [(prev[0] + 1, x[1])]
            return y + [x]

        months = [max(x[1]) for x in groupby(reduce(reducer, [(1, x[2])
                  for x in days if x[0] == 0], []), itemgetter(1))]

        svg = ('<svg width="{width}" height="{height}" ' +
               'class="js-calendar-graph-svg">').format(width=width,
                                                        height=height)
        svg += ''''''
        svg += '<g transform="translate(16, 20)">'
        for i, week in enumerate(weeks):
            svg += build_svg_week(i, week)
        x = 12 + (months[0][0] * 12 if months[0][0] < 3 else 0)
        for i, month in enumerate(months):
            n, name = month
            if n > 2:
                svg += build_svg_month(x, name)
                x += n * 12

        svg += '''
        <text text-anchor="start" class="wday" dx="-14" dy="8"
            style="display: none;">Mon</text>
        <text text-anchor="start" class="wday" dx="-14" dy="20">Tue</text>
        <text text-anchor="start" class="wday" dx="-14" dy="32"
            style="display: none;">Wed</text>
        <text text-anchor="start" class="wday" dx="-14" dy="44">Thu</text>
        <text text-anchor="start" class="wday" dx="-14" dy="57"
            style="display: none;">Fri</text>
        <text text-anchor="start" class="wday" dx="-14" dy="69">Sat</text>
        <text text-anchor="start" class="wday" dx="-14" dy="81"
            style="display: none;">Sun</text>'''
        svg += '</g>'
        svg += '</svg>'
        return svg

    result = build_svg(value_counts.items())

    return display_svg(result, raw=True) if display else result

def matrix_chart(value_counts, title, ranges=(3, 5, 10)):

    style = '''
    '''
    html = '''

{svg}

{title}

                Less

                More

'''
    return display_html(
        style + html.format(
            title=title, svg=matrix_svg(value_counts, ranges), display=False),
        raw=True)


from datetime import datetime, timedelta
from collections import defaultdict
from pprint import pprint

AVERAGE_WINDOW = 7
NUM_SAMPLES = 7

def increasing(samples):
    last = 0
    for i, d in sorted(samples.items(), reverse=True):
        if d < (last * 1.1):
            return False
        last = d
    return True

def make_deltas(filename):
    fh = open(filename, 'r')
    ref_data = defaultdict(lambda: defaultdict(int))
    for line in fh:
        lemma, sample, hits = line.split('\t')
        ref_data[lemma][int(sample)] = int(hits)
    fh.close()

    for lemma, samples in ref_data.items():
        diffs = []
        for sample in range(1, NUM_SAMPLES):
            try:
                diffs.append(samples[sample-1] - samples[sample])
            except:
                pass
        if filter(lambda d: d<100, diffs):
            continue

        if not increasing(samples):
            continue
        print [lemma, diffs, samples]
        #print [lemma, diffs]

    #pprint(ref_data)

if __name__ == '__main__':
    make_deltas('samples/windows.tsv')


import cv2
import numpy as np
from sklearn import mixture
import sys
from time import time
start = time()

# Enter key parameters for the preprocessing
global height, width
dimen_limit = 200
img = cv2.imread(sys.argv[1])
height, width, _ = img.shape
large = max(height,width)

for i in range(1,100):
    if large/i < dimen_limit:
        break

# img = cv2.equalizeHist(img)
# img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY).reshape(height,width,1)
# img = np.concatenate((img,img,img),axis=2)
# img = cv2.cvtColor(img, cv2.COLOR_BGR2YUV)
# img = cv2.medianBlur(img,5)

def DS(img): # Downsampling

    img_small = cv2.resize(img,None,fx=1/i, fy=1/i, interpolation = cv2.INTER_CUBIC)
    dimen_1, dimen_2, _ = img_small.shape
#     img_flat = cv2.Canny(img_small.copy(),10,10).flatten().reshape(-1,1) # Edge filter
    img_flat = np.concatenate((img_small[:,:,0].flatten().reshape(-1,1),
                               img_small[:,:,1].flatten().reshape(-1,1),
                               img_small[:,:,2].flatten().reshape(-1,1)),axis=1)

    return img_flat, dimen_1, dimen_2

def GMM(img_flat, dimen_1, dimen_2): # GMM clustering

    gmm_tmp = mixture.GaussianMixture(n_components=2, covariance_type='full', max_iter=500)
    gmm_tmp.fit(img_flat)
    tmp = np.argmax(gmm_tmp.predict_proba(img_flat),axis=1).reshape(-1,1)
    tmp0 = tmp.reshape(dimen_1, dimen_2).astype('int16')

    return tmp0

img_flat, dimen_1, dimen_2 = DS(img)
tmp0 = GMM(img_flat, dimen_1, dimen_2)

# from matplotlib import pyplot as plt
# plt.imshow(cv2.resize(np.ceil(img_gmm*255), (width,height)), cmap="hot")
# plt.show()

np.savetxt("0.csv", tmp0, delimiter=",",fmt='%1d')

print('Time used',time()-start)


import pandas as pd
import numpy as np
from fxdayu_data.data.decorators import value_wrapper
import itertools

def match(iterable, *iters):
    for ex in map(itertools.product, iterable, *iters):
        for e in ex:
            yield e

def multi_from_frame(frame):
    names = []
    items = []
    for name, item in frame.iteritems():
        names.append(name)
        items.append(item.values)

    return pd.MultiIndex.from_arrays(items, names=names)

multi_match = value_wrapper(list, pd.MultiIndex.from_tuples)(match)

def frame_2_multi_series(frame, names=None):
    if isinstance(frame, pd.DataFrame):
        value = np.concatenate(frame.values)
        index = pd.MultiIndex.from_product((frame.index, frame.columns), names=names)
        return pd.Series(value, index)

    else:
        raise TypeError('type of frame should be pandas.DataFrame')

def frames_2_multi_frame(names=None, **frames):
    name, frame = frames.popitem()
    index = pd.MultiIndex((frame.index, frame.columns), names=names)
    dct = {name: np.concatenate(frame.values)}
    shape = frame.shape
    for name_, frame_ in frames.items():
        if frame_.shape == shape:
            dct[name_] = np.concatenate(frame_.values)
        else:
            raise ValueError(
                'shape not match: shape of %s is %s but shape of %s is %s' % (name_, frame_.shape, name, shape)
            )

    return pd.DataFrame(dct, index)

def tsf_2_multi_frame(time_frame, columns=None, names=None):
    first = []
    second = []
    values = []
    for time, frame in time_frame:
        if isinstance(frame, pd.DataFrame):
            first.append([time])
            second.append(frame.index)
            values.append(frame.values)
        else:
            raise TypeError('type of frame in time_frame should be pandas.DataFrame')

    index = multi_match(first, second)
    if names:
        index = index.set_names(names)
    frame = np.concatenate(values)
    return pd.DataFrame(frame, index, columns)

def make_multi_frame(data, **kwargs):
    return pd.DataFrame(
        {name: np.concatenate(func(data).values) for name, func in kwargs.items()},
        pd.MultiIndex.from_product([data.index, data.columns])
    )

def panel_2_multi_frame(panel, names=None):
    if isinstance(panel, pd.Panel):
        return pd.DataFrame(
            {minor: np.concatenate(panel.minor_xs(minor).values) for minor in panel.minor_axis},
            pd.MultiIndex.from_product((panel.major_axis, panel.items), names=names)
        )
    else:
        raise (TypeError("Type of panel should be pandas.Panel"))

def roll_panel(panel, window=1, axis=1):
    if isinstance(panel, pd.Panel):
        slicer = [slice(None), slice(None), slice(None)]
        index = getattr(panel, ['items', 'major_axis', 'minor_axis'][axis])
        for i in range(window, len(index)):
            slicer[axis] = slice(i-window, i)
            yield index[i-1], panel.iloc[tuple(slicer)]
    else:
        raise (TypeError("Type of panel should be pandas.Panel"))


import logging
from optparse import make_option

from django.core.management.base import BaseCommand

from amo.utils import chunked

from mkt.developers.tasks import refresh_iarc_ratings

log = logging.getLogger('z.task')

class Command(BaseCommand):
    """
    Refresh old or corrupt IARC ratings by re-fetching the certificate.
    """
    option_list = BaseCommand.option_list + (
        make_option('--apps',
                    help='Webapp ids to process. Use commas to separate '
                         'multiple ids.'),
    )
    help = __doc__

    def handle(self, *args, **kw):
        from mkt.webapps.models import Webapp

        # Get apps.
        apps = Webapp.objects.filter(iarc_info__isnull=False)
        ids = kw.get('apps')
        if ids:
            apps = apps.filter(
                id__in=(int(id.strip()) for id in ids.split(',')))

        for chunk in chunked(apps.values_list('id', flat=True), 100):
            refresh_iarc_ratings.delay(chunk)


import boto3, json

def lambda_handler(event, context):

    client = boto3.client('codebuild')

    response = client.start_build(
        projectName="paulhastings-staging",
        artifactsOverride={
            'type': 'CODEPIPELINE'
        }
    )
    return "Paul Hastings AWS Staging build triggered"

    print(response)


import asyncio
from dataclasses import dataclass
import logging
import operator
import traceback
import random
from typing import (
    Dict,
    Iterable,
    Optional,
    Sequence,
    Tuple,
    cast,
)

from cancel_token import (
    CancelToken,
)

from eth_utils import ValidationError, to_tuple
from eth_utils.toolz import first

from eth.exceptions import (
    BlockNotFound,
)

from eth2.beacon.helpers import (
    compute_start_slot_of_epoch,
)
from eth2.beacon.chains.base import (
    BaseBeaconChain,
)
from eth2.beacon.types.attestations import (
    Attestation,
)
from eth2.beacon.types.blocks import (
    BaseBeaconBlock,
)
from eth2.beacon.typing import (
    Epoch,
    Slot,
    HashTreeRoot,
    Version,
    SigningRoot,
)
from eth2.beacon.constants import (
    ZERO_SIGNING_ROOT,
)

from libp2p import (
    initialize_default_swarm,
)
from libp2p.typing import TProtocol

from libp2p.crypto.keys import (
    KeyPair,
)
from libp2p.host.basic_host import (
    BasicHost,
)
from libp2p.network.network_interface import (
    INetwork,
)
from libp2p.security.secio.transport import ID as SecIOID
from libp2p.security.secio.transport import Transport as SecIOTransport
from libp2p.network.stream.net_stream_interface import (
    INetStream,
)
from libp2p.peer.id import (
    ID,
)
from libp2p.peer.peerinfo import (
    PeerInfo,
)
from libp2p.peer.peerstore import (
    PeerStore,
)
from libp2p.pubsub.pubsub import (
    Pubsub,
)
from libp2p.pubsub.gossipsub import (
    GossipSub,
)
from libp2p.security.base_transport import BaseSecureTransport
from libp2p.stream_muxer.abc import IMuxedConn
from libp2p.stream_muxer.mplex.exceptions import MplexStreamEOF, MplexStreamReset
from libp2p.stream_muxer.mplex.mplex import MPLEX_PROTOCOL_ID, Mplex

from multiaddr import (
    Multiaddr,
    protocols,
)

import ssz

from p2p.service import (
    BaseService,
)

from .configs import (
    GOSSIPSUB_PROTOCOL_ID,
    GoodbyeReasonCode,
    GossipsubParams,
    PUBSUB_TOPIC_BEACON_BLOCK,
    PUBSUB_TOPIC_BEACON_ATTESTATION,
    REQ_RESP_BEACON_BLOCKS,
    REQ_RESP_GOODBYE,
    REQ_RESP_HELLO,
    REQ_RESP_RECENT_BEACON_BLOCKS,
    ResponseCode,
)
from .exceptions import (
    HandshakeFailure,
    ReadMessageFailure,
    RequestFailure,
    WriteMessageFailure,
)
from .messages import (
    Goodbye,
    HelloRequest,
    BeaconBlocksRequest,
    BeaconBlocksResponse,
    RecentBeaconBlocksRequest,
    RecentBeaconBlocksResponse,
)
from .topic_validators import (
    get_beacon_attestation_validator,
    get_beacon_block_validator,
)
from .utils import (
    make_rpc_v1_ssz_protocol_id,
    make_tcp_ip_maddr,
    read_req,
    read_resp,
    write_req,
    write_resp,
)

logger = logging.getLogger('trinity.protocol.bcc_libp2p')

REQ_RESP_HELLO_SSZ = make_rpc_v1_ssz_protocol_id(REQ_RESP_HELLO)
REQ_RESP_GOODBYE_SSZ = make_rpc_v1_ssz_protocol_id(REQ_RESP_GOODBYE)
REQ_RESP_BEACON_BLOCKS_SSZ = make_rpc_v1_ssz_protocol_id(REQ_RESP_BEACON_BLOCKS)
REQ_RESP_RECENT_BEACON_BLOCKS_SSZ = make_rpc_v1_ssz_protocol_id(
    REQ_RESP_RECENT_BEACON_BLOCKS
)

@dataclass
class Peer:

    node: "Node"
    _id: ID
    fork_version: Version  # noqa: E701
    finalized_root: SigningRoot
    finalized_epoch: Epoch
    head_root: HashTreeRoot
    head_slot: Slot

    @classmethod
    def from_hello_request(
        cls, node: "Node", peer_id: ID, request: HelloRequest
    ) -> "Peer":
        return cls(
            node=node,
            _id=peer_id,
            fork_version=request.fork_version,
            finalized_root=request.finalized_root,
            finalized_epoch=request.finalized_epoch,
            head_root=request.head_root,
            head_slot=request.head_slot,
        )

    async def request_beacon_blocks(
        self, start_slot: Slot, count: int, step: int = 1
    ) -> Tuple[BaseBeaconBlock, ...]:
        return await self.node.request_beacon_blocks(
            self._id,
            head_block_root=self.head_root,
            start_slot=start_slot,
            count=count,
            step=step,
        )

    async def request_recent_beacon_blocks(
        self, block_roots: Sequence[HashTreeRoot]
    ) -> Tuple[BaseBeaconBlock, ...]:
        return await self.node.request_recent_beacon_blocks(self._id, block_roots)

class PeerPool:
    peers: Dict[ID, Peer]

    def __init__(self) -> None:
        self.peers = {}

    def add(self, peer: Peer) -> None:
        self.peers[peer._id] = peer

    def remove(self, peer_id: ID) -> None:
        del self.peers[peer_id]

    def __contains__(self, peer_id: ID) -> bool:
        return peer_id in self.peers.keys()

    def __len__(self) -> int:
        return len(self.peers)

    def get_best(self, field: str) -> Peer:
        sorted_peers = sorted(
            self.peers.values(), key=operator.attrgetter(field), reverse=True
        )
        return first(sorted_peers)

    def get_best_head_slot_peer(self) -> Peer:
        return self.get_best("head_slot")

DIAL_RETRY_COUNT = 10

class Node(BaseService):

    _is_started: bool = False

    key_pair: KeyPair
    listen_ip: str
    listen_port: int
    host: BasicHost
    pubsub: Pubsub
    bootstrap_nodes: Tuple[Multiaddr, ...]
    preferred_nodes: Tuple[Multiaddr, ...]
    chain: BaseBeaconChain

    handshaked_peers: PeerPool = None

    def __init__(
            self,
            key_pair: KeyPair,
            listen_ip: str,
            listen_port: int,
            chain: BaseBeaconChain,
            security_protocol_ops: Dict[TProtocol, BaseSecureTransport] = None,
            muxer_protocol_ops: Dict[TProtocol, IMuxedConn] = None,
            gossipsub_params: Optional[GossipsubParams] = None,
            cancel_token: CancelToken = None,
            bootstrap_nodes: Tuple[Multiaddr, ...] = (),
            preferred_nodes: Tuple[Multiaddr, ...] = ()) -> None:
        super().__init__(cancel_token)
        self.listen_ip = listen_ip
        self.listen_port = listen_port
        self.key_pair = key_pair
        self.bootstrap_nodes = bootstrap_nodes
        self.preferred_nodes = preferred_nodes
        # TODO: Add key and peer_id to the peerstore
        if security_protocol_ops is None:
            security_protocol_ops = {
                SecIOID: SecIOTransport(key_pair)
            }
        if muxer_protocol_ops is None:
            muxer_protocol_ops = {MPLEX_PROTOCOL_ID: Mplex}
        network: INetwork = initialize_default_swarm(
            key_pair=key_pair,
            transport_opt=[self.listen_maddr],
            muxer_opt=muxer_protocol_ops,
            sec_opt=security_protocol_ops,
            peerstore_opt=None,  # let the function initialize it
            disc_opt=None,  # no routing required here
        )
        self.host = BasicHost(network=network, router=None)

        if gossipsub_params is None:
            gossipsub_params = GossipsubParams()
        gossipsub_router = GossipSub(
            protocols=[GOSSIPSUB_PROTOCOL_ID],
            degree=gossipsub_params.DEGREE,
            degree_low=gossipsub_params.DEGREE_LOW,
            degree_high=gossipsub_params.DEGREE_HIGH,
            time_to_live=gossipsub_params.FANOUT_TTL,
            gossip_window=gossipsub_params.GOSSIP_WINDOW,
            gossip_history=gossipsub_params.GOSSIP_HISTORY,
            heartbeat_interval=gossipsub_params.HEARTBEAT_INTERVAL,
        )
        self.pubsub = Pubsub(
            host=self.host,
            router=gossipsub_router,
            my_id=self.peer_id,
        )

        self.chain = chain

        self.handshaked_peers = PeerPool()

        self.run_task(self.start())

    @property
    def is_started(self) -> bool:
        return self._is_started

    async def _run(self) -> None:
        self.logger.info("libp2p node %s is up", self.listen_maddr)
        await self.cancellation()

    async def start(self) -> None:
        # host
        self._register_rpc_handlers()
        # TODO: Register notifees
        await self.host.get_network().listen(self.listen_maddr)
        await self.connect_preferred_nodes()
        # TODO: Connect bootstrap nodes?

        # pubsub
        await self.pubsub.subscribe(PUBSUB_TOPIC_BEACON_BLOCK)
        await self.pubsub.subscribe(PUBSUB_TOPIC_BEACON_ATTESTATION)
        self._setup_topic_validators()

        self._is_started = True

    def _setup_topic_validators(self) -> None:
        self.pubsub.set_topic_validator(
            PUBSUB_TOPIC_BEACON_BLOCK,
            get_beacon_block_validator(self.chain),
            False,
        )
        self.pubsub.set_topic_validator(
            PUBSUB_TOPIC_BEACON_ATTESTATION,
            get_beacon_attestation_validator(self.chain),
            False,
        )

    async def dial_peer(self, ip: str, port: int, peer_id: ID) -> None:
        """
        Dial the peer ``peer_id`` through the IPv4 protocol
        """
        await self.host.connect(
            PeerInfo(
                peer_id=peer_id,
                addrs=[make_tcp_ip_maddr(ip, port)],
            )
        )

    async def dial_peer_with_retries(self, ip: str, port: int, peer_id: ID) -> None:
        """
        Dial the peer ``peer_id`` through the IPv4 protocol
        """
        for i in range(DIAL_RETRY_COUNT):
            try:
                # exponential backoff...
                await asyncio.sleep(2**i + random.random())
                await self.dial_peer(ip, port, peer_id)
                return
            except ConnectionRefusedError:
                logger.debug(
                    "could not connect to peer %s at %s:%d;"
                    " retrying attempt %d of %d...",
                    peer_id,
                    ip,
                    port,
                    i,
                    DIAL_RETRY_COUNT,
                )
                continue
        raise ConnectionRefusedError

    async def dial_peer_maddr(self, maddr: Multiaddr) -> None:
        """
        Parse `maddr`, get the ip:port and PeerID, and call `dial_peer` with the parameters.
        """
        try:
            ip = maddr.value_for_protocol(protocols.P_IP4)
            port = maddr.value_for_protocol(protocols.P_TCP)
            peer_id = ID.from_base58(maddr.value_for_protocol(protocols.P_P2P))
            await self.dial_peer_with_retries(ip=ip, port=port, peer_id=peer_id)
        except Exception:
            traceback.print_exc()
            raise

    async def connect_preferred_nodes(self) -> None:
        results = await asyncio.gather(
            *(self.dial_peer_maddr(node_maddr)
              for node_maddr in self.preferred_nodes),
            return_exceptions=True,
        )
        for result in results:
            if isinstance(result, Exception):
                logger.warning("could not connect to %s", result)

    async def disconnect_peer(self, peer_id: ID) -> None:
        if peer_id in self.handshaked_peers:
            self.logger.debug("Disconnect from %s", peer_id)
            self.handshaked_peers.remove(peer_id)
            await self.host.disconnect(peer_id)
        else:
            self.logger.debug("Already disconnected from %s", peer_id)

    async def broadcast_beacon_block(self, block: BaseBeaconBlock) -> None:
        await self._broadcast_data(PUBSUB_TOPIC_BEACON_BLOCK, ssz.encode(block))

    async def broadcast_attestation(self, attestation: Attestation) -> None:
        await self._broadcast_data(PUBSUB_TOPIC_BEACON_ATTESTATION, ssz.encode(attestation))

    async def _broadcast_data(self, topic: str, data: bytes) -> None:
        await self.pubsub.publish(topic, data)

    @property
    def peer_id(self) -> ID:
        return self.host.get_id()

    @property
    def listen_maddr(self) -> Multiaddr:
        return make_tcp_ip_maddr(self.listen_ip, self.listen_port)

    @property
    def listen_maddr_with_peer_id(self) -> Multiaddr:
        return self.listen_maddr.encapsulate(Multiaddr(f"/p2p/{self.peer_id.to_base58()}"))

    @property
    def peer_store(self) -> PeerStore:
        return self.host.get_network().peerstore

    async def close(self) -> None:
        # FIXME: Add `tear_down` to `Swarm` in the upstream
        network = self.host.get_network()
        for listener in network.listeners.values():
            listener.server.close()
            await listener.server.wait_closed()
        # TODO: Add `close` in `Pubsub`

    def _register_rpc_handlers(self) -> None:
        self.host.set_stream_handler(REQ_RESP_HELLO_SSZ, self._handle_hello)
        self.host.set_stream_handler(REQ_RESP_GOODBYE_SSZ, self._handle_goodbye)
        self.host.set_stream_handler(REQ_RESP_BEACON_BLOCKS_SSZ, self._handle_beacon_blocks)
        self.host.set_stream_handler(
            REQ_RESP_RECENT_BEACON_BLOCKS_SSZ,
            self._handle_recent_beacon_blocks,
        )

    #
    # RPC Handlers
    #

    # TODO: Add a wrapper or decorator to handle the exceptions in handlers,
    #   to close the streams safely. Probably starting from: if the function
    #   returns successfully, then close the stream. Otherwise, reset the stream.

    # TODO: Handle the reputation of peers. Deduct their scores and even disconnect when they
    #   behave.

    # TODO: Register notifee to the `Network` to
    #   - Record peers' joining time.
    #   - Disconnect peers when they fail to join in a certain amount of time.

    async def _validate_hello_req(self, hello_other_side: HelloRequest) -> None:
        state_machine = self.chain.get_state_machine()
        state = self.chain.get_head_state()
        config = state_machine.config
        if hello_other_side.fork_version != state.fork.current_version:
            raise ValidationError(
                "`fork_version` mismatches: "
                f"hello_other_side.fork_version={hello_other_side.fork_version}, "
                f"state.fork.current_version={state.fork.current_version}"
            )

        # Can not validate the checkpoint with `finalized_epoch` higher than ours
        if hello_other_side.finalized_epoch > state.finalized_checkpoint.epoch:
            return

        # Get the finalized root at `hello_other_side.finalized_epoch`
        # Edge case where nothing is finalized yet
        if (
            hello_other_side.finalized_epoch == 0 and
            hello_other_side.finalized_root == ZERO_SIGNING_ROOT
        ):
            return

        finalized_epoch_start_slot = compute_start_slot_of_epoch(
            hello_other_side.finalized_epoch,
            config.SLOTS_PER_EPOCH,
        )
        finalized_root = self.chain.get_canonical_block_root(
            finalized_epoch_start_slot)

        if hello_other_side.finalized_root != finalized_root:
            raise ValidationError(
                "`finalized_root` mismatches: "
                f"hello_other_side.finalized_root={hello_other_side.finalized_root}, "
                f"hello_other_side.finalized_epoch={hello_other_side.finalized_epoch}, "
                f"our `finalized_root` at the same `finalized_epoch`={finalized_root}"
            )

    def _make_hello_packet(self) -> HelloRequest:
        state = self.chain.get_head_state()
        head = self.chain.get_canonical_head()
        finalized_checkpoint = state.finalized_checkpoint
        return HelloRequest(
            fork_version=state.fork.current_version,
            finalized_root=finalized_checkpoint.root,
            finalized_epoch=finalized_checkpoint.epoch,
            head_root=head.hash_tree_root,
            head_slot=head.slot,
        )

    def _compare_chain_tip_and_finalized_epoch(self,
                                               peer_finalized_epoch: Epoch,
                                               peer_head_slot: Slot) -> None:
        checkpoint = self.chain.get_head_state().finalized_checkpoint
        head_block = self.chain.get_canonical_head()
        peer_has_higher_finalized_epoch = peer_finalized_epoch > checkpoint.epoch
        peer_has_equal_finalized_epoch = peer_finalized_epoch == checkpoint.epoch
        peer_has_higher_head_slot = peer_head_slot > head_block.slot
        if (
            peer_has_higher_finalized_epoch or
            (peer_has_equal_finalized_epoch and peer_has_higher_head_slot)
        ):
            # TODO: kickoff syncing process with this peer
            self.logger.debug("Peer's chain is ahead of us, start syncing with the peer.")
            pass

    async def _handle_hello(self, stream: INetStream) -> None:
        # TODO: Find out when we should respond the `ResponseCode`
        #   other than `ResponseCode.SUCCESS`.

        peer_id = stream.mplex_conn.peer_id

        self.logger.debug("Waiting for hello from the other side")
        try:
            hello_other_side = await read_req(stream, HelloRequest)
            has_error = False
        except (ReadMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
            has_error = True
            if isinstance(error, ReadMessageFailure):
                await stream.reset()
            elif isinstance(error, MplexStreamEOF):
                await stream.close()
        finally:
            if has_error:
                await self.disconnect_peer(peer_id)
                return
        self.logger.debug("Received the hello message %s", hello_other_side)

        try:
            await self._validate_hello_req(hello_other_side)
        except ValidationError as error:
            self.logger.info(
                "Handshake failed: hello message %s is invalid: %s",
                hello_other_side,
                str(error)
            )
            await stream.reset()
            await self.say_goodbye(peer_id, GoodbyeReasonCode.IRRELEVANT_NETWORK)
            await self.disconnect_peer(peer_id)
            return

        hello_mine = self._make_hello_packet()

        self.logger.debug("Sending our hello message %s", hello_mine)
        try:
            await write_resp(stream, hello_mine, ResponseCode.SUCCESS)
            has_error = False
        except (WriteMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
            has_error = True
            if isinstance(error, WriteMessageFailure):
                await stream.reset()
            elif isinstance(error, MplexStreamEOF):
                await stream.close()
        finally:
            if has_error:
                self.logger.info(
                    "Handshake failed: failed to write message %s",
                    hello_mine,
                )
                await self.disconnect_peer(peer_id)
                return

        if peer_id not in self.handshaked_peers:
            peer = Peer.from_hello_request(self, peer_id, hello_other_side)
            self.handshaked_peers.add(peer)
            self.logger.debug(
                "Handshake from %s is finished. Added to the `handshake_peers`",
                peer_id,
            )

        # Check if we are behind the peer
        self._compare_chain_tip_and_finalized_epoch(
            hello_other_side.finalized_epoch,
            hello_other_side.head_slot,
        )

        await stream.close()

    async def say_hello(self, peer_id: ID) -> None:
        hello_mine = self._make_hello_packet()

        self.logger.debug(
            "Opening new stream to peer=%s with protocols=%s",
            peer_id,
            [REQ_RESP_HELLO_SSZ],
        )
        stream = await self.host.new_stream(peer_id, [REQ_RESP_HELLO_SSZ])
        self.logger.debug("Sending our hello message %s", hello_mine)
        try:
            await write_req(stream, hello_mine)
            has_error = False
        except (WriteMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
            has_error = True
            if isinstance(error, WriteMessageFailure):
                await stream.reset()
            elif isinstance(error, MplexStreamEOF):
                await stream.close()
        finally:
            if has_error:
                await self.disconnect_peer(peer_id)
                error_msg = f"fail to write request={hello_mine}"
                self.logger.info("Handshake failed: %s", error_msg)
                raise HandshakeFailure(error_msg)

        self.logger.debug("Waiting for hello from the other side")
        try:
            resp_code, hello_other_side = await read_resp(stream, HelloRequest)
            has_error = False
        except (ReadMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
            has_error = True
            if isinstance(error, ReadMessageFailure):
                await stream.reset()
            elif isinstance(error, MplexStreamEOF):
                await stream.close()
        finally:
            if has_error:
                await self.disconnect_peer(peer_id)
                self.logger.info("Handshake failed: fail to read the response")
                raise HandshakeFailure("fail to read the response")

        self.logger.debug(
            "Received the hello message %s, resp_code=%s",
            hello_other_side,
            resp_code,
        )

        # TODO: Handle the case when `resp_code` is not success.
        if resp_code != ResponseCode.SUCCESS:
            # TODO: Do something according to the `ResponseCode`
            error_msg = (
                "resp_code != ResponseCode.SUCCESS, "
                f"resp_code={resp_code}, error_msg={hello_other_side}"
            )
            self.logger.info("Handshake failed: %s", error_msg)
            await stream.reset()
            await self.disconnect_peer(peer_id)
            raise HandshakeFailure(error_msg)

        hello_other_side = cast(HelloRequest, hello_other_side)
        try:
            await self._validate_hello_req(hello_other_side)
        except ValidationError as error:
            error_msg = f"hello message {hello_other_side} is invalid: {str(error)}"
            self.logger.info(
                "Handshake failed: %s. Disconnecting %s",
                error_msg,
                peer_id,
            )
            await stream.reset()
            await self.say_goodbye(peer_id, GoodbyeReasonCode.IRRELEVANT_NETWORK)
            await self.disconnect_peer(peer_id)
            raise HandshakeFailure(error_msg) from error

        if peer_id not in self.handshaked_peers:
            peer = Peer.from_hello_request(self, peer_id, hello_other_side)
            self.handshaked_peers.add(peer)
            self.logger.debug(
                "Handshake to peer=%s is finished. Added to the `handshake_peers`",
                peer_id,
            )

        # Check if we are behind the peer
        self._compare_chain_tip_and_finalized_epoch(
            hello_other_side.finalized_epoch,
            hello_other_side.head_slot,
        )

        await stream.close()

    async def _handle_goodbye(self, stream: INetStream) -> None:
        peer_id = stream.mplex_conn.peer_id
        self.logger.debug("Waiting for goodbye from %s", peer_id)
        try:
            goodbye = await read_req(stream, Goodbye)
            has_error = False
        except (ReadMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
            has_error = True
            if isinstance(error, ReadMessageFailure):
                await stream.reset()
            elif isinstance(error, MplexStreamEOF):
                await stream.close()

        self.logger.debug("Received the goodbye message %s", goodbye)

        if not has_error:
            await stream.close()
        await self.disconnect_peer(peer_id)

    async def say_goodbye(self, peer_id: ID, reason: GoodbyeReasonCode) -> None:
        goodbye = Goodbye(reason)
        self.logger.debug(
            "Opening new stream to peer=%s with protocols=%s",
            peer_id,
            [REQ_RESP_GOODBYE_SSZ],
        )
        stream = await self.host.new_stream(peer_id, [REQ_RESP_GOODBYE_SSZ])
        self.logger.debug("Sending our goodbye message %s", goodbye)
        try:
            await write_req(stream, goodbye)
            has_error = False
        except (WriteMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
            has_error = True
            if isinstance(error, WriteMessageFailure):
                await stream.reset()
            elif isinstance(error, MplexStreamEOF):
                await stream.close()

        if not has_error:
            await stream.close()
        await self.disconnect_peer(peer_id)

    @to_tuple
    def _get_blocks_from_canonical_chain_by_slot(
        self,
        slot_of_requested_blocks: Sequence[Slot],
    ) -> Iterable[BaseBeaconBlock]:
        # If peer's head block is on our canonical chain,
        # start getting the requested blocks by slots.
        for slot in slot_of_requested_blocks:
            try:
                block = self.chain.get_canonical_block_by_slot(slot)
            except BlockNotFound:
                pass
            else:
                yield block

    @to_tuple
    def _get_blocks_from_fork_chain_by_root(
        self,
        start_slot: Slot,
        peer_head_block: BaseBeaconBlock,
        slot_of_requested_blocks: Sequence[Slot],
    ) -> Iterable[BaseBeaconBlock]:
        # Peer's head block is on a fork chain,
        # start getting the requested blocks by
        # traversing the history from the head.

        # `slot_of_requested_blocks` starts with earliest slot
        # and end with most recent slot, so we start traversing
        # from the most recent slot.
        cur_index = len(slot_of_requested_blocks) - 1
        block = peer_head_block
        if block.slot == slot_of_requested_blocks[cur_index]:
            yield block
            cur_index -= 1
        while block.slot > start_slot and cur_index >= 0:
            try:
                block = self.chain.get_block_by_root(block.parent_root)
            except (BlockNotFound, ValidationError):
                # This should not happen as we only persist block if its
                # ancestors are also in the database.
                break
            else:
                while block.slot < slot_of_requested_blocks[cur_index]:
                    if cur_index > 0:
                        cur_index -= 1
                    else:
                        break
                if block.slot == slot_of_requested_blocks[cur_index]:
                    yield block

    def _validate_start_slot(self, start_slot: Slot) -> None:
        config = self.chain.get_state_machine().config
        state = self.chain.get_head_state()
        finalized_epoch_start_slot = compute_start_slot_of_epoch(
            epoch=state.finalized_checkpoint.epoch,
            slots_per_epoch=config.SLOTS_PER_EPOCH,
        )
        if start_slot < finalized_epoch_start_slot:
            raise ValidationError(
                f"`start_slot`({start_slot}) lower than our"
                f" latest finalized slot({finalized_epoch_start_slot})"
            )

    def _get_requested_beacon_blocks(
        self,
        beacon_blocks_request: BeaconBlocksRequest,
        requested_head_block: BaseBeaconBlock,
    ) -> Tuple[BaseBeaconBlock, ...]:
        slot_of_requested_blocks = tuple(
            beacon_blocks_request.start_slot + i * beacon_blocks_request.step
            for i in range(beacon_blocks_request.count)
        )
        self.logger.info("slot_of_requested_blocks: %s", slot_of_requested_blocks)
        slot_of_requested_blocks = tuple(
            filter(lambda slot: slot <= requested_head_block.slot, slot_of_requested_blocks)
        )

        if len(slot_of_requested_blocks) == 0:
            return tuple()

        # We have the peer's head block in our database,
        # next check if the head block is on our canonical chain.
        try:
            canonical_block_at_slot = self.chain.get_canonical_block_by_slot(
                requested_head_block.slot
            )
            block_match = canonical_block_at_slot == requested_head_block
        except BlockNotFound:
            self.logger.debug(
                (
                    "The requested head block is not on our canonical chain  "
                    "requested_head_block: %s  canonical_block_at_slot: %s"
                ),
                requested_head_block,
                canonical_block_at_slot,
            )
            block_match = False
        finally:
            if block_match:
                # Peer's head block is on our canonical chain
                return self._get_blocks_from_canonical_chain_by_slot(
                    slot_of_requested_blocks
                )
            else:
                # Peer's head block is not on our canonical chain
                # Validate `start_slot` is greater than our latest finalized slot
                self._validate_start_slot(beacon_blocks_request.start_slot)
                return self._get_blocks_from_fork_chain_by_root(
                    beacon_blocks_request.start_slot,
                    requested_head_block,
                    slot_of_requested_blocks,
                )

    async def _handle_beacon_blocks(self, stream: INetStream) -> None:
        peer_id = stream.mplex_conn.peer_id
        if peer_id not in self.handshaked_peers:
            self.logger.info(
                "Processing beacon blocks request failed: not handshaked with peer=%s yet",
                peer_id,
            )
            await stream.reset()
            return

        self.logger.debug("Waiting for beacon blocks request from the other side")
        try:
            beacon_blocks_request = await read_req(stream, BeaconBlocksRequest)
            has_error = False
        except (ReadMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
            has_error = True
            if isinstance(error, ReadMessageFailure):
                await stream.reset()
            elif isinstance(error, MplexStreamEOF):
                await stream.close()
        finally:
            if has_error:
                return
        self.logger.debug("Received the beacon blocks request message %s", beacon_blocks_request)

        try:
            requested_head_block = self.chain.get_block_by_hash_tree_root(
                beacon_blocks_request.head_block_root
            )
        except (BlockNotFound, ValidationError) as error:
            self.logger.info("Sending empty blocks, reason: %s", error)
            # We don't have the chain data peer is requesting
            requested_beacon_blocks: Tuple[BaseBeaconBlock, ...] = tuple()
        else:
            # Check if slot of specified head block is greater than specified start slot
            if requested_head_block.slot < beacon_blocks_request.start_slot:
                reason = (
                    f"Invalid request: head block slot({requested_head_block.slot})"
                    f" lower than `start_slot`({beacon_blocks_request.start_slot})"
                )
                try:
                    await write_resp(stream, reason, ResponseCode.INVALID_REQUEST)
                    has_error = False
                except (WriteMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
                    has_error = True
                    if isinstance(error, WriteMessageFailure):
                        await stream.reset()
                    elif isinstance(error, MplexStreamEOF):
                        await stream.close()
                finally:
                    if has_error:
                        self.logger.info(
                            "Processing beacon blocks request failed: failed to write message %s",
                            reason,
                        )
                        return
                await stream.close()
                return
            else:
                try:
                    requested_beacon_blocks = self._get_requested_beacon_blocks(
                        beacon_blocks_request, requested_head_block
                    )
                except ValidationError as val_error:
                    reason = "Invalid request: " + str(val_error)
                    try:
                        await write_resp(stream, reason, ResponseCode.INVALID_REQUEST)
                        has_error = False
                    except (WriteMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
                        has_error = True
                        if isinstance(error, WriteMessageFailure):
                            await stream.reset()
                        elif isinstance(error, MplexStreamEOF):
                            await stream.close()
                    finally:
                        if has_error:
                            self.logger.info(
                                "Processing beacon blocks request failed: "
                                "failed to write message %s",
                                reason,
                            )
                            return
                    await stream.close()
                    return
        # TODO: Should it be a successful response if peer is requesting
        # blocks on a fork we don't have data for?
        beacon_blocks_response = BeaconBlocksResponse(blocks=requested_beacon_blocks)
        self.logger.debug("Sending beacon blocks response %s", beacon_blocks_response)
        try:
            await write_resp(stream, beacon_blocks_response, ResponseCode.SUCCESS)
            has_error = False
        except (WriteMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
            has_error = True
            if isinstance(error, WriteMessageFailure):
                await stream.reset()
            elif isinstance(error, MplexStreamEOF):
                await stream.close()
        finally:
            if has_error:
                self.logger.info(
                    "Processing beacon blocks request failed: failed to write message %s",
                    beacon_blocks_response,
                )
                return

        self.logger.debug(
            "Processing beacon blocks request from %s is finished",
            peer_id,
        )
        await stream.close()

    async def request_beacon_blocks(self,
                                    peer_id: ID,
                                    head_block_root: HashTreeRoot,
                                    start_slot: Slot,
                                    count: int,
                                    step: int) -> Tuple[BaseBeaconBlock, ...]:
        if peer_id not in self.handshaked_peers:
            error_msg = f"not handshaked with peer={peer_id} yet"
            self.logger.info("Request beacon block failed: %s", error_msg)
            raise RequestFailure(error_msg)

        beacon_blocks_request = BeaconBlocksRequest(
            head_block_root=head_block_root,
            start_slot=start_slot,
            count=count,
            step=step,
        )

        self.logger.debug(
            "Opening new stream to peer=%s with protocols=%s",
            peer_id,
            [REQ_RESP_BEACON_BLOCKS_SSZ],
        )
        stream = await self.host.new_stream(peer_id, [REQ_RESP_BEACON_BLOCKS_SSZ])
        self.logger.debug("Sending beacon blocks request %s", beacon_blocks_request)
        try:
            await write_req(stream, beacon_blocks_request)
            has_error = False
        except (WriteMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
            has_error = True
            if isinstance(error, WriteMessageFailure):
                await stream.reset()
            elif isinstance(error, MplexStreamEOF):
                await stream.close()
        finally:
            if has_error:
                error_msg = f"fail to write request={beacon_blocks_request}"
                self.logger.info("Request beacon blocks failed: %s", error_msg)
                raise RequestFailure(error_msg)

        self.logger.debug("Waiting for beacon blocks response")
        try:
            resp_code, beacon_blocks_response = await read_resp(stream, BeaconBlocksResponse)
            has_error = False
        except (ReadMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
            has_error = True
            if isinstance(error, ReadMessageFailure):
                await stream.reset()
            elif isinstance(error, MplexStreamEOF):
                await stream.close()
        finally:
            if has_error:
                self.logger.info("Request beacon blocks failed: fail to read the response")
                raise RequestFailure("fail to read the response")

        self.logger.debug(
            "Received beacon blocks response %s, resp_code=%s",
            beacon_blocks_response,
            resp_code,
        )

        if resp_code != ResponseCode.SUCCESS:
            error_msg = (
                "resp_code != ResponseCode.SUCCESS, "
                f"resp_code={resp_code}, error_msg={beacon_blocks_response}"
            )
            self.logger.info("Request beacon blocks failed: %s", error_msg)
            await stream.reset()
            raise RequestFailure(error_msg)

        await stream.close()

        beacon_blocks_response = cast(BeaconBlocksResponse, beacon_blocks_response)
        return beacon_blocks_response.blocks

    async def _handle_recent_beacon_blocks(self, stream: INetStream) -> None:
        peer_id = stream.mplex_conn.peer_id
        if peer_id not in self.handshaked_peers:
            self.logger.info(
                "Processing recent beacon blocks request failed: not handshaked with peer=%s yet",
                peer_id,
            )
            await stream.reset()
            return

        self.logger.debug("Waiting for recent beacon blocks request from the other side")
        try:
            recent_beacon_blocks_request = await read_req(stream, RecentBeaconBlocksRequest)
            has_error = False
        except (ReadMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
            has_error = True
            if isinstance(error, ReadMessageFailure):
                await stream.reset()
            elif isinstance(error, MplexStreamEOF):
                await stream.close()
        finally:
            if has_error:
                return
        self.logger.debug(
            "Received the recent beacon blocks request message %s",
            recent_beacon_blocks_request,
        )

        recent_beacon_blocks = []
        for block_root in recent_beacon_blocks_request.block_roots:
            try:
                block = self.chain.get_block_by_hash_tree_root(block_root)
            except (BlockNotFound, ValidationError):
                pass
            else:
                recent_beacon_blocks.append(block)

        recent_beacon_blocks_response = RecentBeaconBlocksResponse(blocks=recent_beacon_blocks)
        self.logger.debug("Sending recent beacon blocks response %s", recent_beacon_blocks_response)
        try:
            await write_resp(stream, recent_beacon_blocks_response, ResponseCode.SUCCESS)
            has_error = False
        except (WriteMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
            has_error = True
            if isinstance(error, WriteMessageFailure):
                await stream.reset()
            elif isinstance(error, MplexStreamEOF):
                await stream.close()
        finally:
            if has_error:
                self.logger.info(
                    "Processing recent beacon blocks request failed: failed to write message %s",
                    recent_beacon_blocks_response,
                )
                return

        self.logger.debug(
            "Processing recent beacon blocks request from %s is finished",
            peer_id,
        )
        await stream.close()

    async def request_recent_beacon_blocks(
            self,
            peer_id: ID,
            block_roots: Sequence[HashTreeRoot]) -> Tuple[BaseBeaconBlock, ...]:
        if peer_id not in self.handshaked_peers:
            error_msg = f"not handshaked with peer={peer_id} yet"
            self.logger.info("Request recent beacon block failed: %s", error_msg)
            raise RequestFailure(error_msg)

        recent_beacon_blocks_request = RecentBeaconBlocksRequest(block_roots=block_roots)

        self.logger.debug(
            "Opening new stream to peer=%s with protocols=%s",
            peer_id,
            [REQ_RESP_RECENT_BEACON_BLOCKS_SSZ],
        )
        stream = await self.host.new_stream(peer_id, [REQ_RESP_RECENT_BEACON_BLOCKS_SSZ])
        self.logger.debug("Sending recent beacon blocks request %s", recent_beacon_blocks_request)
        try:
            await write_req(stream, recent_beacon_blocks_request)
            has_error = False
        except (WriteMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
            has_error = True
            if isinstance(error, WriteMessageFailure):
                await stream.reset()
            elif isinstance(error, MplexStreamEOF):
                await stream.close()
        finally:
            if has_error:
                error_msg = f"fail to write request={recent_beacon_blocks_request}"
                self.logger.info("Request recent beacon blocks failed: %s", error_msg)
                raise RequestFailure(error_msg)

        self.logger.debug("Waiting for recent beacon blocks response")
        try:
            resp_code, recent_beacon_blocks_response = await read_resp(
                stream,
                RecentBeaconBlocksResponse,
            )
            has_error = False
        except (ReadMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
            has_error = True
            if isinstance(error, ReadMessageFailure):
                await stream.reset()
            elif isinstance(error, MplexStreamEOF):
                await stream.close()
        finally:
            if has_error:
                self.logger.info("Request recent beacon blocks failed: fail to read the response")
                raise RequestFailure("fail to read the response")

        self.logger.debug(
            "Received recent beacon blocks response %s, resp_code=%s",
            recent_beacon_blocks_response,
            resp_code,
        )

        if resp_code != ResponseCode.SUCCESS:
            error_msg = (
                "resp_code != ResponseCode.SUCCESS, "
                f"resp_code={resp_code}, error_msg={recent_beacon_blocks_response}"
            )
            self.logger.info("Request recent beacon blocks failed: %s", error_msg)
            await stream.reset()
            raise RequestFailure(error_msg)

        await stream.close()

        recent_beacon_blocks_response = cast(
            RecentBeaconBlocksResponse,
            recent_beacon_blocks_response,
        )
        return recent_beacon_blocks_response.blocks


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import json
import os
import sys
from django.http import HttpResponse
from django.shortcuts import render
import base64
from urllib import parse
# Create your views here.
from django.shortcuts import render, redirect
#from models import InterationRe,IndirectRe,Node,UserInfo

sys.path.append(os.path.dirname(os.path.abspath(__file__)))

print("********************", os.getcwd())
from agraph.server import agraph, genRelations,uri2str,uri2name
from datamap.tree import *
from datamap.entry import Execute
from pythonfun import fundef,Cls,ClsNameSpace,ModuleLoad
default_scope_func = {'fundef':fundef,'Cls':Cls, 'ClsNameSpace':ClsNameSpace,'ModuleLoad':ModuleLoad}

import logging
import logging.config
import time

log_filename = "logging.log"
logging.basicConfig(level=logging.DEBUG,
    format='[%(asctime)s] %(levelname)s [%(funcName)s: %(filename)s, %(lineno)d] %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S',
    filemode='a',
	filename = log_filename)

def auth(func):
	def inner(request):
		if request.user.is_authenticated:
			userinfo = {
				'username':request.user.username
			}
			return func(request, userinfo)
		else:
			return  render(request, 'registration/login.html', context={'next': request.path})
	return inner
@auth
def index(request, userinfo):
	return render(request, 'jsmind/index.html', context = {'userinfo':userinfo})

str2cmd = lambda strs:'for expr in [%s]:exec(expr)'%strs

	#return {param['name']:param['root'],param['children']:[roots.append(parent) or insertChild(parent)  for parent in parents if not parent in childrens and not parent in roots]}
	#insertChild = lambda s: not s in parents and  {param['name']:s} or {param['name']:s, param['children']:[pairs.add(parents[i], childrens[i]) or insertChild(childrens[i])  for i in range(0,len(parents)) if parents[i] == s and not (parents[i], childrens[i]) in pairs]}
	#return {param['name']:param['root'],param['children']:[roots.append(parent) or insertChild(parent)  for parent in parents if not parent in childrens and not parent in roots]}

@auth
def repoClass(request, userinfo):
	catalog = request.GET.get('catalog', "")
	repo = request.GET.get('repo',"owl")
	lang = request.GET.get('lang', "zh")
	relation = request.GET.get('rel', "rdfs:subClassOf")
	cmdlist = ['query = """SELECT ?s ?o ?sName ?oName WHERE { \
				?s  rdfs:subClassOf  ?o; \
					rdfs:label ?sName FILTER (lang(?sName) = "zh"). \
					?o rdfs:label ?oName FILTER (lang(?oName) = "zh")"""' , 'agraph.repoOpen(catalog, repo)',
					'elementmap = lambda elem : (uri2str(elem, "sName"), uri2str(elem, "oName"),uri2name(elem,"s"), uri2name(elem,"o"))',
					'elementgen = lambda elem : elem[2][0] and elem[3][0] and parents.append(elem[3][0] + \':"%s"\'%elem[1]) or childrens.append(elem[2][0] + \':"%s"\'%elem[0])',
					'parents ,childrens,roots = ([],[],[])',
					'genRelations(elementmap, elementgen, query_lang)',
					'len(parents) or genRelations(elementmap, elementgen, query)',
					'data = pairs2Tree(parents, childrens)',
					'viewstyle = "height:100%"']
	#print [catalog, repo, lang, relation]
	#agraph.repoOpen(catalog, repo)

	#query_lang = """
	#	SELECT ?s ?o ?sName ?oName WHERE {
	#		?s  %s  ?o;
	#			rdfs:label ?sName FILTER (lang(?sName) = "%s").
	#		?o rdfs:label ?oName FILTER (lang(?oName) = "%s")
	#	}"""%(relation, lang, lang)
	#
	#query ="""
	#	SELECT ?s ?o ?sName ?oName WHERE {
	#		?s  %s  ?o;
	#			rdfs:label ?sName.
	#		?o rdfs:label ?oName
	#	}"""%relation

	elementmap = lambda elem : (uri2str(elem, "sName"), uri2str(elem, "oName"),uri2name(elem,"s"), uri2name(elem,"o"))
	elementgen = lambda elem : elem[2][0] and elem[3][0] and parents.append(elem[3][0] + ':"%s"'%elem[1]) or childrens.append(elem[2][0] + ':"%s"'%elem[0])
	parents ,childrens,roots = ([],[],[])
	genRelations(elementmap, elementgen, query_lang)
	len(parents) or genRelations(elementmap, elementgen, query)

	data = pairs2Tree(parents, childrens)
	logging.info(data)
	viewstyle = "height:100%"

	return

@auth
def repoConn(request, userinfo):

	isjson = request.GET.get('isjson', False)
	catalog = request.GET.get('catalog', "")
	repo = request.GET.get('repo','sys')
	query = request.GET.get('query',"""
							SELECT ?s ?p ?o {
								?s  ?p  ?o;}
							""")
	print [catalog, repo, query]
	agraph.repoOpen(catalog, repo)

	parents ,childrens= ([],[])
	elementmap = lambda elem : (uri2name(elem,"s"), uri2name(elem,"o"))
	elementgen = lambda elem : elem[0][1] and elem[1][1] and (parents.append(elem[0]) or childrens.append(elem[1]))
	genRelations(elementmap, elementgen, query)
	logging.info(parents)
	logging.info(childrens)
	data = pairs2TreeWithCatalog(parents, childrens)
	if isjson:
		return HttpResponse(json.dumps(data), content_type="application/json")
	else:
		viewstyle = "height:100%"
		return render(request, 'jsmind/grapdep.html', context = {'viewstyle':viewstyle, 'data':json.dumps(data), 'userinfo':userinfo})

@auth
def viz(request, userinfo):
	viewstyle = "height:100%"
	isjson = request.GET.get('isjson', False)
	catalog = request.GET.get('catalog', "")
	repo = request.GET.get('repo','sys')
	query = request.GET.get('query',"""
							SELECT ?s ?p ?o {
								?s  ?p  ?o;}
							""")
	print [catalog, repo, query]
	agraph.repoOpen(catalog, repo)

	parents ,childrens= ([],[])
	elementmap = lambda elem : (uri2name(elem,"s"), uri2name(elem,"o"))
	elementgen = lambda elem : elem[0][1] and elem[1][1] and (parents.append(elem[0]) or childrens.append(elem[1]))
	genRelations(elementmap, elementgen, query)
	logging.info(parents)
	logging.info(childrens)
	data = treeWithCatalog2dot(parents, childrens)

	if isjson:
		return HttpResponse(data, content_type="application/json")
	else:
		viewstyle = "height:100%"
		return render(request, 'jsmind/viz.html', context = {'viewstyle':viewstyle, 'data':data, 'userinfo':userinfo})
@auth
def showclass(request, userinfo):
	return render(request, 'jsmind/showclass/showobj.html')

@auth
def cal(request, userinfo):
	exe = Execute([('Public', 'D:/pythonLib/public.owl'),('Module','D:/pythonLib/module.owl')])
	cal = request.POST.getlist('cals[]',[])
	scope = {'request':request, 'userinfo':userinfo, 'user':request.user}
	scope.update(default_scope_func)

	if not len(cal):
		cal_bs64 = request.POST.get('cals',request.GET.get('cals',""))
		cal_str = str(base64.b64decode(cal_bs64),encoding = "utf-8")
		print(cal_str)
		cal = json.loads(cal_str)
	print('................',cal,'................')
	exe.execute("\n".join(cal), scope)
	print(scope['data'])
	if 'data_type' in scope:
		print(scope['data_type'])
		return HttpResponse(scope['data'], content_type=scope['data_type'])#"application/json"
	else:
		return scope['data']

@auth
def call(request, userinfo):
	scope = {'request':request, 'userinfo':userinfo, 'user':request.user}
	scope.update(default_scope_func)

	path = parse.unquote(request.POST.get('path',request.GET.get('path',"")))
	args_base64 = parse.unquote(request.POST.get('args',request.GET.get('args',"")))
	exe = Execute([('Public', 'D:/pythonLib/public.owl'),('Module','D:/pythonLib/module.owl'),('callmodule', path)])
	args = str(base64.b64decode(args_base64),encoding = "utf-8")
	print('---------------', path,args,'---------------')
	exe.execute('self.ontos["callmodule"].'+args, scope)
	print(scope['data'])
	if 'data_type' in scope:
		print(scope['data_type'])
		return HttpResponse(scope['data'], content_type=scope['data_type'])#"application/json"
	else:
		return scope['data']

@auth
def edit(request, userinfo):

	catalog = request.GET.get('catalog', "")
	repo = request.GET.get('repo',"owl")
	lang = request.GET.get('lang', "zh")
	relation = request.GET.get('rel', "?p")

	agraph.repoOpen(catalog, repo)
	query_lang = """
	SELECT ?s ?o ?sName ?oName WHERE {
		?s  %s  ?o;
			rdfs:label ?sName FILTER (lang(?sName) = "%s").
		?o rdfs:label ?oName FILTER (lang(?oName) = "%s")
	}"""%(relation, lang, lang)

	#query ="""
	#	SELECT ?s ?o ?sName ?oName WHERE {
	#		?s  %s  ?o;
	#			rdfs:label ?sName.
	#		?o rdfs:label ?oName
	#	}"""%relation

	query ="""
		SELECT ?s ?o  WHERE {
			?s  %s  ?o;
		}"""%relation

	elementmap = lambda elem : (uri2name(elem,"s"), uri2name(elem,"o"))
	elementgen = lambda elem : elem[0][1] and elem[1][1] and parents.append(elem[0][0] + ':"%s"'%elem[0][1]) or childrens.append(elem[1][0] + ':"%s"'%elem[1][1])

	parents ,childrens,roots = ([],[],[])
	#genRelations(elementmap, elementgen, query_lang)
	len(parents) or genRelations(elementmap, elementgen, query)

	logging.info(parents)
	logging.info(childrens)
	print("get data",parents)

	data = pairs2Tree(parents, childrens, name="topic")

	temp = {}
	#extendChild = lambda data, i :  temp.update(data['children'][i]) or data['children'].__setitem__(i, {}) or data['children'][i].update(temp)
	data = json.loads(json.dumps(data)) #to split the same node to more.

	updateid = lambda data,id : data.update({'id':"%s%s"%(data['topic'],id)}) or (data.has_key('children') and [updateid(data['children'][i], data['id']) for i in xrange(len(data['children']))])
	updateid(data,'0')

	#logging.info(data)
	return render(request, 'jsmind/jsmind.html', context = {'userinfo':userinfo,'data':json.dumps(data)})

@auth
def getcenter(request, auth):
	if request.is_ajax():
		rsp = {'errorcode': 100, 'detail': 'Get success'}
		id = request.POST.get('type_id')
		return HttpResponse(json.dumps(rsp), content_type="application/json")
	else:
		return HttpResponse("error")

#create a relationship
@auth
def rel_create(request, userinfo):
	if request.is_ajax():
		rsp = {'errorcode': 100, 'detail': 'Get success'}
		request.POST.get('start')
		request.POST.get('end')

		return HttpResponse(json.dumps(rsp), content_type="application/json")
	else:
		return HttpResponse("error")

#delete a relationship  should build a forget mechanism don't delte the data immedaitally
@auth
def rel_del(request, userinfo):
	if request.is_ajax():
		rsp = {'errorcode': 100, 'detail': 'Get success'}
		id = request.POST.get('type_id')
		return HttpResponse(json.dumps(rsp), content_type="application/json")
	else:
		return HttpResponse("error")

@auth
def node_create(request, userinfo):
	if request.is_ajax():
		rsp = {'errorcode': 100, 'detail': 'Get success'}
		id = request.POST.get('type_id')
		return HttpResponse(json.dumps(rsp), content_type="application/json")
	else:
		return HttpResponse("error")

@auth
def node_delete(request, userinfo):
	if request.is_ajax():
		rsp = {'errorcode': 100, 'detail': 'Get success'}
		id = request.POST.get('type_id')
		return HttpResponse(json.dumps(rsp), content_type="application/json")
	else:
		return HttpResponse("error")


from pyspark.sql import SparkSession
from pyspark.sql.functions import explode
from pprint import pprint

if __name__ == "__main__":

    session = SparkSession.builder.appName("Payload").getOrCreate()

    dataFrameReader = session.read

    responses = dataFrameReader \
        .option("header", "true") \
        .option("inferSchema", value = True) \
        .json("payload/payload500.json")

    print("=== Print out schema ===")
    responses.printSchema()
    pprint(responses.columns)
    df = responses.select(explode('events'))
    df.show()
    #pprint(df.collect())
    #responses.show()

    session.stop()


import tushare as ts
import pandas as pd

def Multiple_stocks(tickers):
    def data(ticker):
        stocks = ts.get_k_data(ticker,start='2019-07-01',end='2019-08-18')
        stocks.set_index('date', inplace=True)
        stocks.index = pd.to_datetime(stocks.index)
        return stocks

    datas = map(data, tickers)

    return pd.concat(datas, keys=tickers, names=['Ticker', 'trade_date'])

tickers = ['600030','000679']
all_stocks = Multiple_stocks(tickers)
print(all_stocks.head(50))