""" Crawler """

from abc import ABCMeta, abstractmethod
import uuid
from datetime import datetime
import re
from copy import deepcopy
from typing import Iterator, Callable
import requests
from urllib.parse import urlparse
from bs4 import BeautifulSoup
import graph
import indexer as idx

class CrawlerPayload():
    __slots__ = 'link_id', 'url', 'retrieved_at', \
        'raw_content', 'nofollow_urls', 'urls', 'title', 'text_content'

    def __init__(self,
                 link_id: str = str(uuid.uuid4()),
                 url: str = '',
                 retrieved_at: datetime = datetime.min,
                 raw_content: str = '',                  # set by LinkFetcher
                 nofollow_urls: list[str] = [],          # set by LinkExtractor
                 urls: list[str] = [],                   # set by LinkExtractor
                 title: str = '',                        # set by TextExtractor
                 text_content: str = ''):                # set by TextExtractor
        self.link_id = link_id
        self.url = url
        self.retrieved_at = retrieved_at
        self.raw_content = raw_content
        self.nofollow_urls = nofollow_urls
        self.urls = urls
        self.title = title
        self.text_content = text_content

    def __repr__(self) -> str:
        return f'\n\
            link_id:\t{self.link_id}\n\
            url:\t{self.url}\n\
            retrieved_at:\t{self.retrieved_at}\n\
            raw_content:\t{self.raw_content}\n\
            nofollow_urls:\t{self.nofollow_urls}\n\
            urls:\t{self.urls}\n\
            title:\t{self.title}\n\
            text_content: \t{self.text_content}\n'

class Processor(metaclass=ABCMeta):
    """ Process payloads as partof pipeline stage """
    @ abstractmethod
    def process(self, payload: CrawlerPayload) -> CrawlerPayload:
        """ Process payload and send to next stage or raise """

class LinkFetcher(Processor):
    def process(self, payload: CrawlerPayload) -> CrawlerPayload:
        # decide on error object passing, but keep simple
        # either no handling at all here or try and returning the except

        crawler_payload: CrawlerPayload = payload

        if regex_nonhtml.search(crawler_payload.url):
            return

        result = requests.get(url=crawler_payload.url)
        crawler_payload.raw_content = result.text
        return crawler_payload

regex_nonhtml = re.compile('(?i)\.(?:jpg|jpeg|png|gif|ico|css|js)$')
regex_basehref = re.compile('(?i)<base.*?href\s*?=\s*?"(.*?)\s*?"')
regex_allhrefs = re.compile('(?i)<a.*?href\s*?=\s*?"\s*?(.*?)\s*?".*?>')
regex_nofollowhrefs = re.compile('(?i)rel\s*?=\s*?"?nofollow"?')
regex_title = re.compile('(?i)<title.*?>(.*?)</title>')

class LinkExtractor(Processor):
    def process(self, payload: CrawlerPayload) -> CrawlerPayload:
        allhrefs = regex_allhrefs.findall(payload.raw_content)
        payload.urls = [url for url in allhrefs if self._is_absolute(url)]
        return payload

    def _is_absolute(self, urlstring: str) -> bool:
        parsed = urlparse(urlstring)
        return parsed.scheme and parsed.netloc

class ContentExtractor(Processor):
    def process(self, payload: CrawlerPayload) -> CrawlerPayload:
        parser: BeautifulSoup = BeautifulSoup(
            markup=payload.raw_content, features='html.parser')
        # Extract title
        title = parser.find_all('title')
        if title:
            payload.title = title[0].get_text()

        # Remove all script and style elements
        for script in parser(['script', 'style']):
            script.extract()
        text = parser.get_text()
        lines = (line.strip() for line in text.splitlines())
        chunks = (phrase.strip()
                  for line in lines for phrase in line.split('  '))
        text = '\n'.join(chunk for chunk in chunks if chunk)
        payload.text_content = text

        return payload

class GraphUpdater(Processor):
    def __init__(self, graph: graph.Graph):
        self.graph = graph

    def process(self, payload: CrawlerPayload) -> CrawlerPayload:
        """Store result in graph"""
        src = self._handle_src(payload=payload)
        removal_treshold = datetime.utcnow()

        if not src:
            return
        if payload.nofollow_urls and not self._handle_nofollow_links(payload=payload):
            return
        if not self._process_edge(payload=payload, src=src):
            return
        self.graph.remove_stale_edges(src.link_id, removal_treshold)
        return payload

    def _handle_src(self, payload: CrawlerPayload):
        link_src = graph.Link(
            link_id=payload.link_id,
            url=payload.url,
            retrieved_at=datetime.utcnow()
        )
        return self.graph.upsert_link(link_src)

    def _handle_nofollow_links(self, payload: CrawlerPayload):
        """Upsert discovered no-follow links without creating an edge."""
        upserted = []
        for nofollow_url in payload.nofollow_urls:
            link_dst = graph.Link(url=nofollow_url)
            upserted.append(self.graph.upsert_link(link_dst))
        return upserted

    def _process_edge(self, payload: CrawlerPayload, src: graph.Link):
        edges = []
        for dst_url in payload.urls:
            dst = self.graph.upsert_link(link=graph.Link(url=dst_url))
            if not dst:
                return
            edge = graph.Edge(src=src.link_id, dst=dst.link_id)
            edges.append(self.graph.upsert_edge(edge=edge))
        return edges

class TextIndexer(Processor):
    def __init__(self, indexer: idx.Indexer):
        self.indexer = indexer

    def process(self, payload: CrawlerPayload) -> CrawlerPayload:
        document = idx.Document(
            link_id=payload.link_id,
            url=payload.url,
            title=payload.title,
            content=payload.text_content,
            indexed_at=datetime.utcnow()
        )
        self.indexer.upsert_document_index(document=document)
        return payload

class Crawler():
    def __init__(self, graph: graph.Graph, indexer: idx.Indexer):
        self.stages = [
            LinkFetcher(),
            LinkExtractor(),
            ContentExtractor(),
            GraphUpdater(graph=graph),
            TextIndexer(indexer=indexer)
        ]

    def run(self, links_iter: Iterator[graph.Link]):
        outputs = []
        for link in links_iter:
            payload_in = CrawlerPayload(link_id=link.link_id,
                                        url=link.url,
                                        retrieved_at=link.retrieved_at)
            payload_out = self.crawl(payload_in)
            outputs.append(payload_out)
        return outputs

    def crawl(self, payload: CrawlerPayload) -> CrawlerPayload:
        inout = payload
        for stage in self.stages:
            inout = stage.process(inout)
        return inout


#!/usr/bin/python3
# -*- coding: utf-8 -*-
#
#   Author          : Viacheslav Zamaraev
#   email           : zamaraev@gmail.com
#   Script Name     : 02_csv2xlsx.py
#   Created         : 25th September 2019
#   Last Modified	: 25th September 2019
#   Version		    : 1.0
#   PIP             : pip install pandas
#   RESULT          : Excel File
# Modifications	: 1.1 -
#               : 1.2 -
#
# Description   : This script will conver csv file to Excel file

import os.path
from datetime import datetime
from sys import platform as _platform
import os.path

try:
    import pandas as pd
except:
    print("we need pands. try: pip install pandas")

#some global configurations
import cfg

def get_output_directory():
    dir_out = str(os.getcwd())
    # Linux platform
    if _platform == "linux" or _platform == "linux2" or _platform == "darwin":
        dir_out = cfg.folder_out_linux
        if os.path.exists(dir_out) and os.path.isdir(dir_out):
            return dir_out
    if _platform == "win32" or _platform == "win64":  # Windows or Windows 64-bit
        dir_out = cfg.folder_out_win
        if os.path.exists(dir_out) and os.path.isdir(dir_out):
            return dir_out
    else:
        dir_out = str(os.getcwd())
        print('Output directories from config wrong: ' + cfg.folder_out_win + ' or ' + cfg.folder_out_linux + ' Using current directory: ' + dir_out)
    print('Using Output directory: ' + dir_out)
    return dir_out

def csv2xls(filename=''):

    if (os.path.exists(filename) and os.path.isfile(filename)):
        file_excel = filename.split('.')[0] + '.xlsx'
        df_new = pd.read_csv(filename, sep=cfg.csv_delimiter)
        writer = pd.ExcelWriter(file_excel)
        df_new.to_excel(writer, index=False)
        writer.save()
    else:
        print('ERROR! can\'t read a file OR file does not exist. File: ' + filename)

# ---------------- do main --------------------------------
def main():
    time1 = datetime.now()
    print('Starting at :' + str(time1))

    file_csv = str(os.path.join(get_output_directory(), cfg.file_csv))
    csv2xls(file_csv)

    time2 = datetime.now()
    print('Finishing at :' + str(time2))
    print('Total time : ' + str(time2 - time1))
    print('DONE !!!!')

if __name__ == '__main__':
    main()


#using the eglRendererPlugin (hardware OpenGL acceleration)
#using EGL on Linux and default OpenGL window on Win32.

#make sure to compile pybullet with PYBULLET_USE_NUMPY enabled
#otherwise use testrender.py (slower but compatible without numpy)
#you can also use GUI mode, for faster OpenGL rendering (instead of TinyRender CPU)

import numpy as np
import cv2
import pybullet as p
import time
import pkgutil
import os
from threading import Thread
import vtk
import vtktools
# import multiprocessing

# https://stackoverflow.com/questions/19448078/python-opencv-access-webcam-maximum-resolution
def set_res(cap, x,y):
    cap.set(cv2.CAP_PROP_FRAME_WIDTH, int(x))
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, int(y))
    return int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)),int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))

class Scene(object):
    def __init__(self, height, width, window_name, fov=60):
        camPos = [0,0,0]
        camUpVector = [0,-1,0]
        camTargetPos = [0,0,1]
        nearPlane = 0.01
        farPlane = 100

        self.window_name = window_name
        self.height = height
        self.width = width

        ##############    VTK stuff   ##############

        # Create a rendering window and renderer
        self.ren = vtk.vtkRenderer()
        self.renWin = vtk.vtkRenderWindow()
        self.renWin.SetOffScreenRendering(1)
        self.renWin.SetAlphaBitPlanes(1)  # Enable usage of alpha channel
        self.renWin.AddRenderer(self.ren)

        self.renWin.SetSize(width, height)

        self.camera = self.ren.GetActiveCamera()
        self.camera.SetPosition(camPos)
        self.camera.SetViewUp(camUpVector)
        self.camera.SetFocalPoint(camTargetPos)
        self.camera.SetViewAngle(fov)

        self.w2if = vtk.vtkWindowToImageFilter()
        self.w2if.SetInput(self.renWin)
        self.w2if.SetInputBufferTypeToRGBA()

        ############## pybullet stuff ##############

        p.connect(p.DIRECT)
        # conid = p.connect(p.SHARED_MEMORY)
        # if (conid<0):
        #     p.connect(p.GUI)
        p.resetSimulation()
        p.setGravity(0,0,-10)
        # p.setPhysicsEngineParameter(numSolverIterations=5)
        # p.setPhysicsEngineParameter(fixedTimeStep=1./240.)
        # p.setPhysicsEngineParameter(numSubSteps=10)

        self.viewMatrix = p.computeViewMatrix(camPos, camTargetPos, camUpVector)
        aspect = width / height;
        self.projectionMatrix = p.computeProjectionMatrixFOV(fov, aspect, nearPlane, farPlane);

        egl = pkgutil.get_loader('eglRenderer')
        if (egl):
            p.loadPlugin(egl.get_filename(), "_eglRendererPlugin")
        else:
            p.loadPlugin("eglRendererPlugin")

        self.play = True
        self.physics_thread = Thread(target=self.update_physics, args=())
        self.physics_thread.daemon = True
        print("READY")

    def update_physics(self):
        stop = time.time()
        t = 0
        while True:
            try:
                    if cv2.getWindowProperty(self.window_name,1) == -1 :
                            raise Exception
            except:
                    time.sleep(0.1)
                    continue
            if not self.play:
                    return
            t = (t + time.time() - stop) / 2
            if t < 1/60:
                time.sleep(1/60 - t)
                t = 1/60
            p.setTimeStep(t)
            p.stepSimulation()
            stop = time.time()

    def start(self):
        self.physics_thread.start()

    def stop(self):
        self.play = False
        p.resetSimulation()

    def render(self, vtk_render=True):
        if not vtk_render:
            img_arr = p.getCameraImage(self.width, self.height, self.viewMatrix,
                                       self.projectionMatrix, shadow=1,
                                       lightDirection=[1,1,1], renderer=p.ER_BULLET_HARDWARE_OPENGL)
            w=img_arr[0] #width of the image, in pixels
            h=img_arr[1] #height of the image, in pixels
            rgb=img_arr[2] #color data RGBA
            dep=img_arr[3] #depth data
            np_img_arr = np.reshape(rgb, (h, w, 4))
            return np_img_arr*(1./255.)

        self.renWin.Render()
        self.w2if.Modified()
        self.w2if.Update()
        return vtktools.vtkImageToNumpy(self.w2if.GetOutput())

def main():

    pixelWidth, pixelHeight = 1920 // 2, 1080 // 2

    window_name = 'frame'

    scn = Scene(pixelHeight, pixelWidth, window_name)

    # VTK STUFF
    sphere = vtk.vtkSphereSource()
    sphere.SetRadius(0.01 / 2)
    mapper = vtk.vtkPolyDataMapper()
    if vtk.VTK_MAJOR_VERSION <= 5:
            mapper.SetInput(sphere.GetOutput())
    else:
            mapper.SetInputConnection(sphere.GetOutputPort())

    actors = {}
    poses = {}

    dataDir = '/home/biomed/Art/bullet3/data'

    #p.loadPlugin("eglRendererPlugin")
    p.loadURDF(os.path.join(dataDir, "plane.urdf"),[0,0,.4])
    # p.loadURDF(os.path.join(dataDir, "r2d2.urdf"))

    p.configureDebugVisualizer(p.COV_ENABLE_RENDERING,0)
    ob = {}
    count = 0
    for i in range (5):
        for j in range (5):
            for k in range (5):
                ob[count] = p.loadURDF(os.path.join(dataDir, "sphere_1cm.urdf"),[0.02*i + np.random.random_sample(1) * 0.02,0.02*j + np.random.random_sample(1) * 0.01,0.02*k],useMaximalCoordinates=True)
                actors[count] = vtk.vtkActor()
                actors[count].SetMapper(mapper)
                scn.ren.AddActor(actors[count])
                count = count + 1

    main_start = time.time()
    cap = cv2.VideoCapture(0)
    res = set_res(cap, 1280//2, 960//2)

    p.configureDebugVisualizer(p.COV_ENABLE_RENDERING,1)

    count = 0
    scn.start()

    while True:
        ret, frame = cap.read()

        for k in actors.keys():
            pos, rot = p.getBasePositionAndOrientation(ob[k])
            actors[k].SetPosition(pos)

        cv2.imshow(window_name,scn.render(count%2))
        count = count + 1

        if cv2.waitKey(1) & 0xFF == ord('q'):
            play = False
            break
        stop = time.time()

    scn.stop()
    cap.release()
    cv2.destroyAllWindows()

    p.resetSimulation()

if __name__ == '__main__':
    main()


from pyspark.sql import SparkSession
from pyspark.sql.functions import count, collect_set, col

if __name__ == '__main__':
    spark = SparkSession.builder.appName("learning").master("local").getOrCreate()
    df = spark.read.format('csv')\
                   .option('sep', ';')\
                   .option('header', 'true')\
                   .load('user.csv')

    df.registerTempTable("user")

    spark.sql("""
        select name, avg(age) as avg_age
        from   user
        group by name
    """).show()

    spark.sql("""
            select name, avg(age) as avg_age
            from   user
            group by name grouping sets((name))
        """).show()

    spark.sql("""
                select name, apellido, avg(age) as avg_age
                from   user
                group by name, apellido grouping sets((name, apellido), (apellido), ())
            """).show()

    #Los valores en el grouping set deben estar en el groupby!


"""
inspired by http://www.di.ufpb.br/lucidio/Btrees.pdf and https://algs4.cs.princeton.edu/62btree/
"""

from random import shuffle, randint, choice
from typing import TypeVar, Generic, List
from bisect import bisect_right

import time

KeyType = TypeVar('KeyType', int, str)
T = TypeVar('T')

# Number of keys per node/page
DEGREE = 16

def main():

    run_function_tests()

    run_performance_test()

def run_function_tests():
    """
        This method tests if btree performs correctly
        by running same random commands(insert,delete,find)
        on btree and naivetree indexes. And comparing results.
        (we assume that naivetree works correctly since it just wrapper on python 'dict')
    """
    print("\t### running functional testing ###")
    btree = BTree()
    naive = NaiveTreeIndex()

    # you can tune this variables
    iterations = 100000
    maxint = 150000

    inserts, deletions, finds = 0, 0, 0

    incorrect_finds = 0
    incorrect_deletions = 0

    for i in range(iterations):
        action = randint(1, 3)
        if action == 1: # add
            k, v = randint(1, maxint), randint(1, maxint)
            naive.add_key_value(SmartKey(k,v))
            btree.add_key_value(SmartKey(k, v))
            inserts += 1
        elif (action == 2 and len(naive.tree.keys()) > 0): # find
            key = SmartKey(choice(list(naive.tree.keys())))
            btree_res = btree.find_key(key)
            naive_res = naive.find(key)
            if btree_res != naive_res:
                incorrect_finds += 1
            finds += 1
        elif len(naive.tree.keys()) > 0: # delete
            key = SmartKey(choice(list(naive.tree.keys())))
            btree.remove(key)
            naive.remove(key)
            if btree.find_key(key) != None:
                incorrect_deletions += 1
            deletions += 1

    print('  ', iterations, 'iterations proceeded')
    print('  ', incorrect_finds, 'incorrect finds from', finds)
    print('  ', incorrect_deletions, 'incorrect deletions from', deletions)
    if incorrect_finds + incorrect_deletions == 0:
        print(" ✔ Functional tests passed successfully")
    print("\t### functional testing finished ###")

def run_performance_test():

    print("\t$$$ performance test started $$$")

    # you can tune this variables
    n = 10000
    maxint = 10000

    insert_time = []
    search_time = []
    delete_time = []

    nums_k_v = [(i, randint(1, maxint)) for i in range(n)]
    shuffle(nums_k_v)

    btree = BTree()
    naiveL = NaiveListIndex()

    # measure insert
    st = time.time()
    for k,v in nums_k_v:
        naiveL.add_key_value(SmartKey(k, v))
    et = time.time();
    insert_time.append(et - st)

    st = time.time()
    for k,v in nums_k_v:
        btree.add_key_value(SmartKey(k, v))

    et = time.time();
    insert_time.append(et - st)

    cp = nums_k_v[:]
    shuffle(cp)

    # measure search
    st = time.time()
    for k, _ in cp:
        naiveL.find_key(SmartKey(k))

    et = time.time();
    search_time.append(et - st)

    st = time.time()
    for k, _ in cp:
        btree.find_key(SmartKey(k))

    et = time.time();
    search_time.append(et - st)

    # measure deletion

    st = time.time()
    for k, _ in cp:
        naiveL.remove(SmartKey(k))

    et = time.time();
    delete_time.append(et - st)

    st = time.time()
    for k, _ in cp:
        btree.remove(SmartKey(k))

    et = time.time();
    delete_time.append(et - st)

    # print results
    print(f"btree has x{insert_time[0]/insert_time[1]} times faster insertion ")
    print(f"btree has x{delete_time[0]/delete_time[1]} times faster deletion ")
    print(f"btree has x{search_time[0]/search_time[1]} times faster search ")
    print("\t$$$ performance test finished $$$")

class SmartKey(Generic[KeyType,T]):
    """

    """
    def __init__(self, key: KeyType, value: T = None) -> None:
        self.key: KeyType = key
        self.value: List[T] = value

    def __str__(self):
        return f"[{self.key}] => {self.value}"

    def __lt__(self, other: 'SmartKey') -> bool:
        return self.key < other.key

    def __eq__(self, other: 'SmartKey') -> bool:
        return self.key == other.key

class Page:
    """
        Btree node
    """
    def __init__(self, leaf: bool) -> None:
        self.leaf = leaf
        self.keys: List[SmartKey] = []
        self.children: List[Page] = []

    def add_key_value(self, key: KeyType, value: T) -> None:

        wrapped_key = SmartKey(key, [value]) # store list of values in leaf nodes

        if len(self.keys) == 0:
            self.keys.append(wrapped_key)
        else:
            left_upper_index = bisect_right(self.keys, wrapped_key)
            if (left_upper_index > 0 and self.keys[left_upper_index - 1].key == wrapped_key.key):
                self.keys[left_upper_index - 1].value.extend(wrapped_key.value)
            else:
                self.keys.insert(left_upper_index, wrapped_key)

    def is_external(self) -> bool:
        return self.leaf

    def find(self, key: SmartKey) -> List[T]:
        """Try to find key in page"""
        left_upper_index = bisect_right(self.keys, key)
        if left_upper_index == 0:
            if len(self.children) == 0:
                return None
            if (key > self.keys[0]):
                return self.children[1].find(key)
            else:
                return self.children[0].find(key)

        prevKey = self.keys[left_upper_index - 1]
        if prevKey.key == key.key:
            return self.keys[left_upper_index - 1].value

        if self.is_external():
            # since there is no other way to find key left
            return None
        else:
            if (key > self.keys[left_upper_index - 1]):
                return self.children[left_upper_index ].find(key)
            else:
                return self.children[left_upper_index - 1].find(key)

    def next(self, key: SmartKey) -> 'Page':
        left_upper_index = bisect_right(self.keys, key)
        return self.children[left_upper_index ]

    def is_full(self) -> bool:
        return len(self.keys) == DEGREE * 2 - 1

    def split(self) -> (SmartKey, 'Page'):
        """
            move the highest-ranking half of the keys in the page to a new page
        """
        # should create with same leaf
        r_key = self.keys[DEGREE - 1]
        new_page_keys = self.keys[DEGREE:]     # last half
        self.keys = self.keys[:DEGREE - 1]    # first half
        new_page_children = self.children[DEGREE:]
        self.children = self.children[:DEGREE]

        new_page = Page(self.leaf)
        new_page.keys = new_page_keys
        new_page.children = new_page_children

        return r_key, new_page

    def split_and_add_page(self, ch: 'Page') -> None:
        ind = self.children.index(ch)

        k, z = ch.split()
        self.children.insert(ind + 1, z)
        self.keys.insert(ind, k)

    def find_predecessor(self) -> SmartKey:
        if self.is_external():
            return self.keys[-1]
        return self.children[-1].find_predecessor()

    def find_successor(self) -> SmartKey:
        if self.is_external():
            return self.keys[0]
        return self.children[0].find_successor()

    def merge(self, ind: int, left: 'Page', right: 'Page') -> 'Page':
        new_page = Page(left.is_external() and right.is_external())
        # copy keys
        new_page.keys = left.keys
        new_page.keys.append(self.keys[ind])
        new_page.keys.extend(right.keys)
        # copy children
        new_page.children = left.children
        new_page.children.extend(right.children)

        self.children.pop(ind + 1)
        self.keys.pop(ind)
        self.children[ind] = new_page
        return new_page

    def borrow_from_left(self, kind: int, ind: int):
        left = self.children[ind - 1]
        cur = self.children[ind]
        # mirgate key
        cur.keys.insert(0, self.keys[kind])
        self.keys[kind] = left.keys[-1]
        left.keys.pop(len(left.keys) - 1)
        # migrate child
        if not cur.is_external():
            cur.children.insert(0, left.children[-1])
            left.children.pop(len(left.children) - 1)

    def borrow_from_right(self, ind: int):
        right = self.children[ind + 1]
        cur = self.children[ind]
        # migrate key
        cur.keys.append(self.keys[ind])
        self.keys[ind] = right.keys[0]
        right.keys.pop(0)
        # migrate child
        if not cur.is_external():
            cur.children.append(right.children[0])
            right.children.pop(0)

    def remove(self, key: KeyType) -> 'Page':
        left_upper_index = bisect_right(self.keys, SmartKey(key))
        if self.is_external(): # case 1
            if (left_upper_index > 0 and self.keys[left_upper_index - 1].key == key):
                self.keys.pop(left_upper_index - 1)
            return self
        ind = max(left_upper_index - 1, 0)
        if self.keys[ind].key == key: # case 2
            # wow we found key in page
            pred_ch = self.children[ind]
            succ_ch = self.children[ind + 1]
            if len(pred_ch.keys) >= DEGREE: # case A
                k = pred_ch.find_predecessor()
                self.keys[ind] = k
                pred_ch.remove(k.key)
            elif len(succ_ch.keys) >= DEGREE: # case B
                k = succ_ch.find_successor()
                self.keys[ind] = k
                succ_ch.remove(k.key)
            else: # case C
                merged = self.merge(ind, pred_ch, succ_ch)
                merged.remove(key)
                if len(self.keys) == 0:
                        return merged

        else: # case 3
            needed_ch_ind = ind
            if key > self.keys[ind].key:
                needed_ch_ind += 1
            ch = self.children[needed_ch_ind]
            if len(ch.keys) == DEGREE - 1: # case A
                # try to grub from right sibling or merge 2 siblings in worst case
                if (needed_ch_ind + 1 < len(self.children) and len(self.children[needed_ch_ind + 1].keys) > DEGREE - 1):
                    self.borrow_from_right(needed_ch_ind)
                elif (needed_ch_ind > 0 and len(self.children[needed_ch_ind - 1].keys) > DEGREE - 1):
                    self.borrow_from_left(ind, needed_ch_ind)
                else: # case B
                    merged = self.merge(ind, self.children[ind], self.children[ind + 1])
                    merged.remove(key)
                    if len(self.keys) == 0:
                        return merged
                # try one more time, now it should get to another branch
                self.remove(key)
            else:
                ch.remove(key)

        return self

class BTree:

    def __init__(self) -> None:
        """
            sentinel key should be less then any other keys
        """
        self.root = Page(True)

    def add_key_value(self, key: SmartKey) -> None:
        self._put(self.root, key)
        if self.root.is_full():
            left = self.root
            k, right = self.root.split()

            self.root = Page(False)
            self.root.keys = [k]
            self.root.children = [left, right]

    def find_key(self, key: SmartKey) -> SmartKey:
        return self.root.find(key)

    def remove(self, key: SmartKey) -> None:
        self.root = self.root.remove(key.key)

    def print_tree(self, deep: int = 0) -> None:
        """Print an level-order representation."""
        this_level = [self.root]
        while this_level:
            next_level = []
            output = ""
            for node in this_level:
                if node.children:
                    next_level.extend(node.children)
                output += str(list(map(lambda f: f.key, node.keys))) + " "
            print(output)
            this_level = next_level

    def _put(self, page: Page, key: SmartKey) -> None:

        if (page.is_external() or key in page.keys):
            page.add_key_value(key.key, key.value)
            return

        nxt = page.next(key)
        self._put(nxt, key)
        if nxt.is_full():
            page.split_and_add_page(nxt)

class NaiveTreeIndex:
    """
        Tree for functional testing. It based on python 'dict'.
    """
    def __init__(self, *args, **kwargs) -> None:
        self.tree: dict = {}

    def add_key_value(self, key: SmartKey):
        if key.key in self.tree:
            self.tree[key.key].append(key.value)
        else:
            self.tree[key.key] = [key.value]

    def find(self, key: SmartKey) -> T:
        return  self.tree[key.key] if key.key in self.tree else None

    def remove(self, key: SmartKey) -> None:
        if key.key in self.tree:
            self.tree.pop(key.key)

class NaiveListIndex:
    """
        Index for comparing performance of btree. Works on python lists
    """

    def __init__(self, *args, **kwargs):
        self.list = []

    def add_key_value(self, key: SmartKey):
        for k, v in self.list:
            if k == key.key:
                v.append(key.value)
                return
        self.list.append((key.key, [key.value]))

    def find_key(self, key:SmartKey):
        for k, v in self.list:
            if k == key.key:
                return self.list[k]

        return None

    def remove(self, key: SmartKey):
        for k, v in self.list:
            if k == key.key:
                self.list.remove((k, v))

if __name__ == "__main__":
    main()


import eventlet.green.zmq as zmq

class Sender:

    def __init__(self):
        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.PUB)
        self.port = self.socket.bind_to_random_port("tcp://*")

    def send(self, message):
        self.socket.send_string(message)

class Receiver:

    def __init__(self, port=5555):
        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.SUB)
        self.socket.connect("tcp://127.0.0.1:" + str(port))
        self.socket.setsockopt(zmq.SUBSCRIBE, b"")  # b"" allows us to receive all

    def receive(self):
        return self.socket.recv()

if __name__ == '__main__':
    r = Receiver()
    print("receiving..")
    print(r.receive())


from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import six
import logging

import filestore.api as fsa
from .utils import fs_setup, fs_teardown
import numpy as np

from .utils import SynHandlerMod, SynHandlerEcho
import uuid
import pytest
logger = logging.getLogger(__name__)

def setup_module(module):
    fs_setup()

def teardown_module(module):
    fs_teardown()

def test_get_handler_global():
    h_cache = fsa._FS_SINGLETON._handler_cache
    mock_base = dict(spec='syn-mod',
                     resource_path='',
                     resource_kwargs={'shape': (5, 7)})

    res = fsa.insert_resource(**mock_base)
    cache_key = (str(res['id']), SynHandlerMod.__name__)
    with fsa.handler_context({'syn-mod': SynHandlerMod}):

        handle = fsa.get_spec_handler(res['id'])

        assert isinstance(handle, SynHandlerMod)
        assert cache_key in h_cache

    assert cache_key not in h_cache

def test_overwrite_global():
    h_cache = fsa._FS_SINGLETON._handler_cache
    mock_base = dict(spec='syn-mod',
                     resource_path='',
                     resource_kwargs={'shape': (5, 7)})

    res = fsa.insert_resource(**mock_base)

    cache_key = (str(res['id']), SynHandlerMod.__name__)
    with fsa.handler_context({'syn-mod': SynHandlerMod}):
        fsa.get_spec_handler(res['id'])
        assert cache_key in h_cache
        fsa.register_handler('syn-mod', SynHandlerEcho, overwrite=True)
        assert cache_key not in h_cache

def test_context():
    with SynHandlerMod('', (4, 2)) as hand:
        for j in range(1, 5):
            assert np.all(hand(j) < j)

def test_register_fail():
    with fsa.handler_context({'syn-mod': SynHandlerMod}):
        # shouldn't raise, it is a no-op as it is regiristering
        # the same class with the same name
        fsa.register_handler('syn-mod', SynHandlerMod)
        # should raise as it is trying to change the registered class
        with pytest.raises(RuntimeError):
            fsa.register_handler('syn-mod', SynHandlerEcho)

def test_context_manager_replace():
    with fsa.handler_context({'syn-mod': SynHandlerMod}):
        assert fsa._FS_SINGLETON.handler_reg['syn-mod'] is SynHandlerMod
        with fsa.handler_context({'syn-mod': SynHandlerEcho}):
            assert fsa._FS_SINGLETON.handler_reg['syn-mod'] is SynHandlerEcho
        assert fsa._FS_SINGLETON.handler_reg['syn-mod'] is SynHandlerMod
    assert 'syn-mod' not in fsa._FS_SINGLETON.handler_reg

def test_deregister():
    test_reg = fsa._FS_SINGLETON.handler_reg
    test_spec_name = str(uuid.uuid4())
    fsa.register_handler(test_spec_name, SynHandlerMod)
    assert test_reg[test_spec_name] is SynHandlerMod
    fsa.deregister_handler(test_spec_name)
    assert test_spec_name not in test_reg


from tensorflow.keras.layers import Conv2D, Dense, Input, add, Activation, AveragePooling2D, GlobalAveragePooling2D, Lambda, \
    concatenate
from tensorflow.keras.initializers import he_normal
from tensorflow.keras.layers import Concatenate
from tensorflow.keras.callbacks import LearningRateScheduler, TensorBoard, ModelCheckpoint
from tensorflow.keras import regularizers
import tensorflow as tf
import os
import numpy as np
from matplotlib import pyplot as plt
from tensorflow.keras.layers import Conv2D, BatchNormalization, Activation, MaxPool2D, Dropout, Flatten, Dense
from tensorflow.keras import Model
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import ReduceLROnPlateau

growth_rate = 12  # k
depth = 40     # L
compression = 0.5

img_rows, img_cols = 32, 32
img_channels = 3
num_classes = 10
batch_size = 32  # 64 or 32 or other
epochs = 300
iterations = 1563
weight_decay = 1e-4

mean = [125.307, 122.95, 113.865]
std = [62.9932, 62.0887, 66.7048]

def scheduler(epoch):
    if epoch < 150:
        return 0.1
    if epoch < 225:
        return 0.01
    return 0.001

def densenet(img_input, classes_num):
    def conv(x, out_filters, k_size):
        return Conv2D(filters=out_filters,
                      kernel_size=k_size,
                      strides=(1, 1),
                      padding='same',
                      kernel_initializer='he_normal',
                      kernel_regularizer=regularizers.l2(weight_decay),
                      use_bias=False)(x)

    def dense_layer(x):
        return Dense(units=classes_num,
                     activation='softmax',
                     kernel_initializer='he_normal',
                     kernel_regularizer=regularizers.l2(weight_decay))(x)

    def bn_relu(x):
        x = BatchNormalization(momentum=0.9, epsilon=1e-5)(x)
        x = Activation('relu')(x)
        return x

    def bottleneck(x):
        channels = growth_rate * 4
        x = bn_relu(x)
        x = conv(x, channels, (1, 1))
        x = bn_relu(x)
        x = conv(x, growth_rate, (3, 3))
        return x

    def single(x):
        x = bn_relu(x)
        x = conv(x, growth_rate, (3, 3))
        return x

    def transition(x, inchannels):
        outchannels = int(inchannels * compression)
        x = bn_relu(x)
        x = conv(x, outchannels, (1, 1))
        x = AveragePooling2D((2, 2), strides=(2, 2))(x)
        return x, outchannels

    def dense_block(x, blocks, nchannels):
        concat = x
        for i in range(blocks):
            x = bottleneck(concat)
            concat = concatenate([x, concat], axis=-1)
            nchannels += growth_rate
        return concat, nchannels

    nblocks = (depth - 4) // 6
    nchannels = growth_rate * 2

    x = conv(img_input, nchannels, (3, 3))
    x, nchannels = dense_block(x, nblocks, nchannels)
    x, nchannels = transition(x, nchannels)
    x, nchannels = dense_block(x, nblocks, nchannels)
    x, nchannels = transition(x, nchannels)
    x, nchannels = dense_block(x, nblocks, nchannels)
    x = bn_relu(x)
    x = GlobalAveragePooling2D()(x)
    x = dense_layer(x)
    return x

if __name__ == '__main__':

    # load data

    np.set_printoptions(threshold=np.inf)

    cifar10 = tf.keras.datasets.cifar10
    (x_train, y_train), (x_test, y_test) = cifar10.load_data()
    x_train, x_test = x_train / 255.0, x_test / 255.0

    # 数据增强
    datagen_train = ImageDataGenerator(
        rotation_range=20, zoom_range=0.15,
        width_shift_range=0.2, height_shift_range=0.2, shear_range=0.15,
        horizontal_flip=True, fill_mode="nearest")

    datagen_train.fit(x_train)

    # build network
    img_input = Input(shape=(img_rows, img_cols, img_channels))
    output = densenet(img_input, num_classes)
    model = Model(img_input, output)

    # model.load_weights('ckpt.h5')

    print(model.summary())

    # set callback
    tb_cb = TensorBoard(log_dir='./densenet/', histogram_freq=0)
    change_lr = LearningRateScheduler(scheduler)
    ckpt = ModelCheckpoint('./ckpt.h5', save_best_only=False, mode='auto', period=10)
    cbks = [change_lr, tb_cb, ckpt]

    model.compile(optimizer='adam',
                  loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False),
                  metrics=['sparse_categorical_accuracy'])

    checkpoint_save_path = "./checkpoint/DesNet40.ckpt"
    if os.path.exists(checkpoint_save_path + '.index'):
        print('-------------load the model-----------------')
        model.load_weights(checkpoint_save_path)

    cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_save_path,
                                                     save_weights_only=True,
                                                     save_best_only=True)

    reduce_lr = ReduceLROnPlateau(monitor='val_loss', patience=10, mode='auto')

    history = model.fit_generator(datagen_train.flow(x_train, y_train, batch_size=32), epochs=epochs,
                                  steps_per_epoch=iterations,
                                  validation_data=(x_test, y_test), validation_freq=1,
                                  callbacks=[cp_callback,reduce_lr])
    model.summary()

    # print(model.trainable_variables)
    file = open('./weights.txt', 'w')
    for v in model.trainable_variables:
        file.write(str(v.name) + '\n')
        file.write(str(v.shape) + '\n')
        file.write(str(v.numpy()) + '\n')
    file.close()

    ###############################################    show   ###############################################

    # 显示训练集和验证集的acc和loss曲线
    acc = history.history['sparse_categorical_accuracy']
    val_acc = history.history['val_sparse_categorical_accuracy']
    loss = history.history['loss']
    val_loss = history.history['val_loss']

    plt.subplot(1, 2, 1)
    plt.plot(acc, label='Training Accuracy')
    plt.plot(val_acc, label='Validation Accuracy')
    plt.title('Training and Validation Accuracy')
    plt.legend()

    plt.subplot(1, 2, 2)

    plt.plot(loss, label='Training Loss')
    plt.plot(val_loss, label='Validation Loss')
    plt.title('Training and Validation Loss')
    plt.legend()
    plt.show()


#-----scapy | networkx | matplotlib needed via sudo pip install scapy | networkx | matplotlib
#-----input ploth.py [filepath]
#-----example : python3 ploth.py filepath[ip_link_12_4_19_1.txt]
#-----
import networkx as nx
import matplotlib
import matplotlib.pyplot as plt
import sys
import threading
import time
positive_ip = {}#{ index1:ip1 , index2:ip2 , ... }
ip_degree = {}#{ (index1,ip1) : degree1 , (index2,ip2) : degree2 , ... }
degree_ip = {}#{ degree1 : [number1,[ip1,ip2,ip3,...]] , degree2 : [number2,[...]] , ...}
ip_ip = []#[ (index1,index2) , (index1,index3) , ...]
ip_net = []#[ (ip1,ip2) , (ip1,ip3) , (ip1,net1) , ... ] net : 192.168:[number of ip beneath the subnet]
iplinkpath = 'ip_link_11_11_22_47.txt'

G = nx.Graph()

def read_ip_link(ip_link_path):
    global ip_ip
    NET_ALLOWANCE = 3
    ip_link = []
    with open(ip_link_path,'r') as f_r:
        while True:
            str1 = f_r.readline()
            if str1 == '':
                break
            elif str1[:5] == 'total':
                print(str1.replace('\n',''))
                continue
            str1 = str1.replace('=>',',').replace(' ','').replace('\n','')
            line = str1.split(',')
            ip_link.append(line)
    #throw out ***.***.***.*** start
    for i in range(len(ip_link)):
        while True:
            if '***.***.***.***' in ip_link[i]:
                flag_finish = False
                line = ip_link[i]
                len_line = len(line)
                index = line.index('***.***.***.***')
                if index == 0:
                    print('***.***.***.*** at head of routine\n')
                    break
                cur = line[index]
                pre = line[index-1]
                num_unknown = 1
                post = line[index+num_unknown]
                #print('pre : ',line,' | line : '+str(i))
                isBreak = False
                while index + num_unknown < len_line:
                    if post == cur:
                        if index + num_unknown == len_line -1:
                            isBreak = True
                            break
                        else:
                            num_unknown += 1
                            post = line[index + num_unknown]
                    else:
                        break
                if isBreak:
                    print('***.***.***.*** at end of routine\n')
                    break

                for j in range(i+1,len(ip_link)):
                    line_j = ip_link[j]
                    if post in line_j and pre in line_j and line_j.index(post) - line_j.index(pre) == num_unknown + 1:
                        index_j = line_j.index(pre) + 1
                        flag_finish = False
                        for k in range(num_unknown):
                            if line_j[index_j+k] != '***.***.***.***':
                                ip_link[i][index+k]=line_j[index_j+k]
                                flag_finish = True
                        if flag_finish == True:
                            print('post :',line_j,'| line : '+str(j))
                            print('repair :',ip_link[i],'| line : '+str(i))
                            break

                if flag_finish == False:
                    print('unsolved : ',line,' | line : '+str(i))
                    break
            else:
                break
    #print(ip_link)
    #throw out ***.***.***.*** done
    #ip_link list => ip_ip list
    #ip_ip = []
    for line in ip_link:
        pre = 0
        for i in range(1,len(line)):
            tup = (line[pre],line[i])
            tup_r = (line[i],line[pre])
            if tup in ip_ip or tup_r in ip_ip:
                pass
            else:
                ip_ip.append(tup)
            pre = i
    #finish ip_ip
    #ip_ip list => ip_degree dict
    ip_degree = {}#{ ip1 : degree1 , ip2 : degree2 , ... }
    for tup_t in ip_ip:
        A = tup_t[0]
        B = tup_t[1]
        if A in ip_degree:
            ip_degree[A] += 1
        else:
            ip_degree[A] = 1
        if B in ip_degree:
            ip_degree[B] += 1
        else:
            ip_degree[B] = 1
    #finish ip_degree
    #ip_degree dict => degree_1 list
    degree_1 = []
    for key in ip_degree.keys():
        if ip_degree[key] == 1:
            degree_1.append(key)
    ip_degree.clear()
    #finish degree_1
    #ip_ip list&degree_1 list => ip_end dict
    ip_end = {}#{ ip1 : [end1,end2,...], ip2 : [end1,end2,...]}
    ip_ip_del = []
    for tup_t in ip_ip:
        A = tup_t[0]
        B = tup_t[1]
        flag = 0# 0 unknown/A&B!degree_1-> | 1 A->degree_1 | 2 B->degree_1 | 3 A&B->degree_1
        if A in degree_1:
            flag = 1
        if B in degree_1:
            if flag == 1:
                flag = 3
            else:
                flag = 2
        if flag == 0:
            pass
        elif flag == 1:
            if B in ip_end:
                ip_end[B].append(A)
            else:
                ip_end[B] = [A]
            ip_ip_del.append(tup_t)
        elif flag == 2:
            if A in ip_end:
                ip_end[A].append(B)
            else:
                ip_end[A] = [B]
            ip_ip_del.append(tup_t)
        else:
            print(tup_t,' each has 1 degree')
    degree_1.clear()
    for del_o in ip_ip_del:
        ip_ip.remove(del_o)
    #finish ip_end
    #ip_ip list & ip_end dict => ip_ip list
    for key in ip_end.keys():
        net = {}#{'a.b':[subnet8bit,len,num],...}
        #11111111.11111111.11111111.00000000 00000000:0 10000000:128 11000000:192 11100000:224 11110000:240 11111000:248 11111100:252 11111110:254 11111111:255
        ip_list = ip_end[key]
        if len(ip_list) <= NET_ALLOWANCE:
            for ip_ in ip_list:
                ip_ip.append((key,ip_))
            continue
        for ip_ in  ip_list:
            index1 = ip_.index('.')
            index2 = ip_[index1+1:].index('.') + index1 + 1
            index3 = ip_[index2+1:].index('.') + index2 + 1
            subnet_num = int(ip_[index2+1:index3])
            subnet = bin(256+subnet_num)[3:]
            tup = [ip_[:index2],subnet,8]#[subnet01,subnet2,len0~8]
            if tup[0] in net:#{'a.b':[subnet8bit,len,num],...}
                if net[tup[0]][1] == 0:
                    net[tup[0]][2] += 1
                    continue
                length = 0
                for i in range(8):
                    if tup[1][i] == net[tup[0]][0][i]:
                        length += 1
                    else:
                        net[tup[0]][0]=net[tup[0]][0][:i] + '0' * (8-i)
                net[tup[0]][1] = length
                net[tup[0]][2] += 1
            else:
                net[tup[0]] = [tup[1],tup[2],1]
        for key_ in net.keys():
            value = net[key_]
            num = int(value[0],2)
            subnet = key_+'.'+str(num)+'.0/'+str(value[1]+16)+':'+str(value[2])
            print(value[0],'=>',num,':',subnet)
            ip_ip.append((key,subnet))
    filepath = 'ip_net'+ip_link_path[ip_link_path.index('_'):]
    with open(filepath,'w') as f_w:
        for key in ip_end.keys():
            f_w.write('node : '+key+'\n')
            f_w.write('end : ')
            loop = 0
            MAX_LOOP = 10
            for ip_ in ip_end[key]:
                tmp = ''
                if loop == 0:
                    tmp += '\n   | '
                loop += 1
                loop = loop % MAX_LOOP
                tmp += ip_ + ' => '
                f_w.write(tmp)
            f_w.write('\n')

def import_graph():
    global iplinkpath
    global ip_ip
    global G

    pngpath = iplinkpath
    G.add_edges_from(ip_ip)

    colors_node = []
    colors_edge = []
    for node in G.nodes():
        colors_node.append('#0000FF')
    for edge in G.edges():
        colors_edge.append('#FFC0CC')

    nx.draw(G,node_size=10,node_color = colors_node,edge_color = colors_edge,alpha = 1,pos = nx.spring_layout(G),with_labels = True,font_size = 12)
    fig = matplotlib.pyplot.gcf()
    fig.set_size_inches(18.5, 10.5)
    index = pngpath.index('.')
    pngpath = pngpath[:index+1]
    pngpath += 'png'
    print('save image to '+pngpath)
    fig.savefig(pngpath,dpi=300)
    plt.savefig('last.png',dpi=300)
    plt.show()

if __name__ == '__main__':
    if len(sys.argv) >= 2:
        iplinkpath = sys.argv[1]
    read_ip_link(iplinkpath)
    import_graph()


import logging
import os
import pickle

from google.appengine.api import taskqueue
from google.appengine.ext import webapp

from aeta import config

# The queue to put deferred tasks in if no queue is specified.
_DEFAULT_QUEUE = 'default'

class DeferredCall(object):
  """A deferred function call.

  The DeferredCall can be created, pickled, unpickled, then called.

  Attributes:
    func: The function to call.
    countdown: How many seconds to wait before calling the function.
    args: Arguments to pass to func.
    kwargs: Keyword arguments to pass to func.
  """

  def __init__(self, func, *args, **kwargs):
    """Creates a deferred function call.

    Args:
      func: The function to call.
      _countdown: How many seconds to wait before running the task.
      *args: Arguments to pass to func.
      **kwargs: Keyword arguments to pass to func.

    Raises:
      ValueError: func is not callable
    """
    if not hasattr(func, '__call__'):
      raise ValueError('"func" must be callable but is %s' % type(func))
    self.func = func
    self.args = args
    self.countdown = kwargs.pop('_countdown', 0)
    self.kwargs = kwargs

  def run(self):
    """Makes the function call that was deferred."""
    self.func(*self.args, **self.kwargs)

def defer_multi(calls, queue=_DEFAULT_QUEUE):
  """Defers multiple function calls.

  Args:
    calls: A list of DeferredCall objects.
    queue: The queue to run the tasks in.
  """
  url = config.get_config().url_path_deferred
  # Go in batches of MAX_TASKS_PER_ADD to avoid the limit.
  for batch_i in range(0, len(calls), taskqueue.MAX_TASKS_PER_ADD):
    tasks = []
    for call in calls[batch_i : batch_i + taskqueue.MAX_TASKS_PER_ADD]:
      tasks.append(taskqueue.Task(url=url, countdown=call.countdown,
                                  payload=pickle.dumps(call)))
    taskqueue.Queue(queue).add(tasks)

def defer(func, *args, **kwargs):
  """Emulates deferred.defer.

  This is mostly identical to deferred.defer but fewer features are supported.

  Args:
    func: The function to call.
    _queue: The name of the queue to use for this task.
    _countdown: How many seconds to wait before running the task.
    *args: Arguments to pass to func.
    **kwargs: Keyword arguments to pass to func.

  Raises:
    ValueError: func is not callable
  """
  queue = kwargs.pop('_queue', _DEFAULT_QUEUE)
  # DeferredCall() will handle the _countdown keyword argument.
  defer_multi([DeferredCall(func, *args, **kwargs)], queue=queue)

class DeferredHandler(webapp.RequestHandler):
  """Handles requests from the task queue to run deferred tasks."""

  def post(self):
    # Additional protection is required because this handler can execute
    # arbitrary code.  Even if the user sets "protected: false" in aeta.yaml,
    # it should not be possible to call this handler outside the task queue.
    if not os.environ.get('HTTP_X_APPENGINE_TASKNAME'):
      self.response.set_status(403)
      self.response.out.write("aeta's task handler can only be called from "
                              'the task queue.')
      return
    payload = self.request.body
    try:
      call = pickle.loads(payload)
    except Exception:
      # prevent task from being retried
      logging.exception('[aeta] Failed during deferred unpickling.')
      return
    try:
      call.run()
    except Exception:
      logging.exception('[aeta] Failed during deferred task; retrying.')
      raise


import numpy as np
import matplotlib.pyplot as plt
from scipy.interpolate import spline
from scipy.interpolate import interp1d
from scipy import interpolate
#keeping the weight constant wt=10 - sparse + low weight
Alpha = []
BU = []
TD = []
RB = []
i = 1
# j = 1
f = open("table3.txt","r")

f1 = f.readlines()
for line in f1:
    if i > 28:
        temp = line.split()
        Alpha.append(float(temp[0]))
        RB.append(float(temp[1]))
        BU.append(float(temp[2]))
        TD.append(float(temp[3]))
    i += 1

print (Alpha)
print (RB)
print (TD)
print (BU)
# plt.plot(Alpha, BU, 'r-', Alpha, TD, 'g-', Alpha, RB, 'b-')
# plt.plot(Alpha, BU, 'r-', Alpha, TD, 'g-', Alpha, RB, 'b-')
plt.plot(Alpha,BU,label="Bottom up Splay tree",color="red")
plt.plot(Alpha,TD,label="Top down Splay tree",color="green")
plt.plot(Alpha,RB,label="Red Black Tree",color="cyan")
plt.xlim(0,1)
plt.xlabel('Skew Fector')
plt.ylabel('CPU time')
plt.title('CPU times versus skewness for 100,000 searches for an activity ratio of 80 per cent updates and 20 per cent search')
plt.legend()
plt.show()


from utils.Component import Component
from utils.Dependency import Dependency

import os.path

class ProFTPd(Component):

    def __init__(self, config):
        super(ProFTPd, self).__init__('ProFTPd', os.path.dirname(__file__), config)

        self.download_url = 'ftp://ftp.proftpd.org/distrib/source/proftpd-%s.tar.gz' % self.version

        self.dependencies = [
            Dependency('ZLib', cFlags=['-I${INCLUDE_PATH}'], ldFlags=['-L${LIB_PATH}']),
            Dependency('Ncurses', cFlags=['-I${INCLUDE_PATH}'], ldFlags=['-L${LIB_PATH}']),
            Dependency('OpenSSL', cFlags=['-I${INCLUDE_PATH}'], ldFlags=['-L${LIB_PATH}']),
            Dependency('MySQL', cFlags=['-I${INCLUDE_PATH}'], ldFlags=['-L${LIB_PATH}'])
        ]

    def configureFlags(self):
        flags = super(ProFTPd, self).configureFlags()

        flags.extend([
            '--with-modules=mod_sql:mod_sql_mysql:mod_tls'
        ])

        return  flags


# -*- coding: utf-8 -*-

import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.layers import Dense, Activation, Flatten, Dropout, BatchNormalization, Conv2D, MaxPooling2D
from tensorflow.keras import utils
from tensorflow.keras.datasets import cifar10
from tensorflow.keras import regularizers
from tensorflow.keras.callbacks import LearningRateScheduler
import numpy as np

def lr_schedule(epoch):

    return 0.001

(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')

#z-score
mean =  np.mean(x_train,axis=(0,1,2,3))
std_div = np.std(x_train,axis=(0,1,2,3))

xm=x_train-mean
xtm=x_test-mean
x_train = (xm*1.0)/(std_div)
x_test = (xtm*1.0)/(std_div)

num_classes = 10

y_test = tensorflow.keras.utils.to_categorical(y_test,10)
y_train = tensorflow.keras.utils.to_categorical(y_train,10)

weight_decay = 0.0001

nn = tf.keras.models.Sequential()

nn.add(Conv2D(32, (5,5), padding='same', kernel_regularizer=regularizers.l1(weight_decay), input_shape=x_train.shape[1:]))
nn.add(Activation('relu'))
nn.add(BatchNormalization())
nn.add(Conv2D(32, (5,5), padding='same', kernel_regularizer=regularizers.l1(weight_decay)))
nn.add(Activation('relu'))
nn.add(BatchNormalization())
nn.add(MaxPooling2D(pool_size=(2,2)))
nn.add(Dropout(0.2))

nn.add(Conv2D(64, (5,5), padding='same', kernel_regularizer=regularizers.l1(weight_decay), input_shape=x_train.shape[1:]))
nn.add(Activation('relu'))
nn.add(BatchNormalization())
nn.add(Conv2D(64, (5,5), padding='same', kernel_regularizer=regularizers.l1(weight_decay)))
nn.add(Activation('relu'))
nn.add(BatchNormalization())
nn.add(MaxPooling2D(pool_size=(2,2)))
nn.add(Dropout(0.3))

nn.add(Conv2D(128, (5,5), padding='same', kernel_regularizer=regularizers.l1(weight_decay)))
nn.add(Activation('relu'))
nn.add(BatchNormalization())
nn.add(Conv2D(128, (5,5), padding='same', kernel_regularizer=regularizers.l1(weight_decay)))
nn.add(Activation('relu'))
nn.add(BatchNormalization())
nn.add(MaxPooling2D(pool_size=(2,2)))
nn.add(Dropout(0.4))

nn.add(Flatten())

nn.add(Dense(num_classes, activation='softmax'))

nn.summary()

#data augmentation
datagen = ImageDataGenerator(rotation_range=10,
    width_shift_range=0.15,fill_mode="wrap",zca_epsilon=0.000001,
    height_shift_range=0.15,
    horizontal_flip=True,
    )
datagen.fit(x_train)

batch_size = 32

opt_rms = tf.keras.optimizers.RMSprop(lr=0.001,decay=0.000001)
nn.compile(loss='categorical_crossentropy', optimizer=opt_rms, metrics=['accuracy'])
nn.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),\
                    steps_per_epoch=x_train.shape[0] // batch_size,epochs=82,\
                    verbose=1,validation_data=(x_test,y_test),callbacks=[LearningRateScheduler(lr_schedule)])

#testing
scores = nn.evaluate(x_test, y_test, batch_size=128, verbose=1)
print('\nTest result: %.3f loss: %.3f' % (scores[1]*100,scores[0]))

# Confusion matrix result
import matplotlib.pyplot as plt
from sklearn.metrics import classification_report, confusion_matrix
Y_pred = model.predict(x_test, verbose=2)
y_pred = np.argmax(Y_pred, axis=1)

for ix in range(10):
    print(ix, confusion_matrix(np.argmax(y_test,axis=1),y_pred)[ix].sum())
cm = confusion_matrix(np.argmax(y_test,axis=1),y_pred)
print(cm)

# Visualizing of confusion matrix
import seaborn as sn
import pandas  as pd

df_cm = pd.DataFrame(cm, range(10),range(10))
plt.figure(figsize = (10,7))
sn.set(font_scale=1.4)#for label size
sn.heatmap(df_cm, annot=True,annot_kws={"size": 12})# font size
plt.show()


# encoding: utf-8

import hmac
import json
import time
import base64
import collections

from website import settings

# Written by @jmcarp originally
def order_recursive(data):
    """Recursively sort keys of input data and all its nested dictionaries.
    Used to ensure consistent ordering of JSON payloads.
    """
    if isinstance(data, dict):
        return collections.OrderedDict(
            sorted(
                (
                    (key, order_recursive(value))
                    for key, value in data.items()
                ),
                key=lambda item: item[0]
            )
        )
    if isinstance(data, list):
        return [
            order_recursive(value)
            for value in data
        ]
    return data

def serialize_payload(payload):
    ordered = order_recursive(payload)
    return base64.b64encode(json.dumps(ordered).encode())

def unserialize_payload(message):
    payload = json.loads(base64.b64decode(message))
    return order_recursive(payload)

class Signer(object):

    def __init__(self, secret, digest):
        assert callable(digest)
        self.secret = secret
        self.digest = digest

    def sign_message(self, message):
        return hmac.new(
            key=self.secret.encode(),
            digestmod=self.digest,
            msg=message,
        ).hexdigest()

    def sign_payload(self, payload):
        message = serialize_payload(payload)
        signature = self.sign_message(message)
        return message.decode(), signature

    def verify_message(self, signature, message):
        expected = self.sign_message(message)
        return signature == expected

    def verify_payload(self, signature, payload):
        _, expected = self.sign_payload(payload)
        return signature == expected

def sign_data(signer, data, ttl=100):
    target = {'time': int(time.time() + ttl)}
    target.update(data)
    payload, signature = signer.sign_payload(target)
    payload = payload.decode() if isinstance(payload, bytes) else payload
    return {
        'payload': payload,
        'signature': signature,
    }

default_signer = Signer(settings.DEFAULT_HMAC_SECRET, settings.DEFAULT_HMAC_ALGORITHM)


import argparse
import os.path
import random

import mlflow.keras
import numpy as np
import tensorflow as tf
import yaml
from numpy.random import seed
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
from sklearn.model_selection import train_test_split

from src.data.dataloader import load_data
from src.models.singleoutput.model import create_model
from src.models.singleoutput.preprocessing import scale

seed(1)
tf.random.set_seed(1)
tf.config.experimental.enable_op_determinism()
random.seed(2)

def eval_metrics(actual, pred):
    acc = round(accuracy_score(actual, pred, normalize=True) * 100, 2)
    precision = round(precision_score(actual, pred, average='macro') * 100, 2)
    recall = round(recall_score(actual, pred, average='macro') * 100, 2)
    f1 = round(f1_score(actual, pred, average='macro') * 100, 2)
    return acc, precision, recall, f1

def train(data_path, n_layers, n_units, activation_function, loss, metric, epochs, batch_size):
    # Log parameters:
    mlflow.log_param("n_layers", n_layers)
    mlflow.log_param("units", n_units)
    mlflow.log_param("activation_function", activation_function)
    mlflow.log_param("loss", loss)
    mlflow.log_param("metric", metric)
    mlflow.log_param("epochs", epochs)
    mlflow.log_param("batch_size", batch_size)

    # Read dataset:
    x_train, y_train = load_data(os.path.join(data_path, 'train.csv'))
    # Scaling:
    x_train = scale(x_train)
    # Split dataset:
    x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.3, random_state=42)

    # Create model:
    model = create_model((x_train.shape[1],), n_layers=n_layers, n_units=n_units,
                         activation=activation_function)

    # Train:
    class LogMetricsCallback(tf.keras.callbacks.Callback):
        def on_epoch_end(self, epoch, logs=None):
            mlflow.log_metric('training_loss', logs['loss'], epoch)
            mlflow.log_metric(f'training_{metric}', logs[metric], epoch)
            mlflow.log_metric('val_loss', logs['val_loss'], epoch)
            mlflow.log_metric(f'val_{metric}', logs[f'val_{metric}'], epoch)

    model.compile(loss=loss, optimizer='adam', metrics=[metric])
    history = model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size, verbose=1,
                        validation_data=(x_val, y_val), callbacks=[LogMetricsCallback()])

    # Evaluation
    x_test, y_test = load_data(os.path.join(data_path, 'test.csv'))
    # Scaling:
    x_test = scale(x_test)

    # evaluate and log:
    test_loss, test_metric = model.evaluate(x_test, y_test, verbose=2)
    mlflow.log_metric('test_loss', test_loss)
    mlflow.log_metric('test_loss', test_loss)

    # Other metric evaluation:
    y_pred = np.rint(np.clip(model.predict(x_test), 0, 9))

    acc, precision, recall, f1 = eval_metrics(y_test, y_pred)
    mlflow.log_metric('test_acc', acc)
    mlflow.log_metric('test_precision', precision)
    mlflow.log_metric('test_recall', recall)
    mlflow.log_metric('test_f1', f1)

    # Log model:
    mlflow.tensorflow.log_model(model, 'models')

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Train a Keras model with single output for MNIST classification")
    parser.add_argument("--config-file", "-c", type=str, default='configs/singleoutput_0.yaml')
    args = parser.parse_args()

    # Load the configuration file:
    with open(args.config_file, 'r') as file:
        config = yaml.safe_load(file)

    # Get parameters:
    data_path = config['data']['dataset_path']

    # MLflow setup
    # mlflow.set_tracking_uri(config['mlflow']['mlruns_path'])
    mlflow.set_experiment(config['mlflow']['experiment_name'])
    with mlflow.start_run():
        train(data_path, config['model']['num_layers'], config['model']['num_units'],
              config['model']['activation_function'], config['training']['loss_function'], config['training']['metric'],
              config['training']['num_epochs'],
              config['training']['batch_size'])


import numpy as np

from yt.funcs import mylog

def link_galaxies_and_halos(obj):
    """Link galaxies and halos to one another.

    This function creates the links between galaxy-->halo and
    halo-->galaxy objects.  Is run during creation, and loading in of
    each CAESAR file.

    Parameters
    ----------
    obj : :class:`main.CAESAR`
        Object containing halos and galaxies lists.

    """
    if not obj._has_galaxies:
        return

    mylog.info('Linking galaxies and halos')

    # halos
    for halo in obj.halos:
        halo.galaxies = []
        for galaxy_index in halo.galaxy_index_list:
            halo.galaxies.append(obj.galaxies[galaxy_index])

    # galaxies
    for galaxy in obj.galaxies:
        if galaxy.parent_halo_index > -1:
            galaxy.halo = obj.halos[galaxy.parent_halo_index]
        else:
            galaxy.halo = None

def link_clouds_and_galaxies(obj):
    """Link clouds and galaxies to one another.

    This function creates the links between cloud-->galaxy and
    galaxy-->cloud objects.  Is run during creation, and loading in of
    each CAESAR file.

    Parameters
    ----------
    obj : :class:`main.CAESAR`
        Object containing halos and galaxies lists.

    """

    if not obj._has_clouds:
        return

    mylog.info('Linking clouds and galaxies')

    #galaxies
    for galaxy in obj.galaxies:
        galaxy.clouds = []
        for cloud_index in galaxy.cloud_index_list:
            galaxy.clouds.append(obj.clouds[cloud_index])

    for cloud in obj.clouds:
        if cloud.parent_galaxy_index > -1:
            cloud.galaxy = obj.galaxies[cloud.parent_galaxy_index]
        else:
            cloud.galaxy = None

def create_sublists(obj):
    """Create sublists of objects.

    Will create the sublists:
        - central_galaxies
        - satellite_galaxies
        - unassigned_galaxies (those without a halo)

    Parameters
    ----------
    obj : :class:`main.CAESAR`
        Object containing halos and galaxies lists already linked.

    """
    if not obj._has_galaxies:
        return

    mylog.info('Creating sublists')

    obj.central_galaxies   = []
    obj.satellite_galaxies = []

    # assign halo sub lists
    for halo in obj.halos:
        halo.central_galaxy = -1
        for galaxy in halo.galaxies:
            if galaxy.central:
                halo.central_galaxy = galaxy.GroupID
#             else:
#                 halo.satellite_galaxies.append(galaxy)

    # assign galaxy sub lists
    for galaxy in obj.galaxies:
        if galaxy.central and galaxy.halo is not None:
#             galaxy.satellites = galaxy.halo.satellite_galaxies
            obj.central_galaxies.append(galaxy)
        elif galaxy.halo is not None:
#             galaxy.satellites = []
            obj.satellite_galaxies.append(galaxy)
        else:
            if not hasattr(obj, 'unassigned_galaxies'):
                obj.unassigned_galaxies = []
            obj.unassigned_galaxies.append(galaxy)


from random import sample
import numpy as np
import io
import os
import imageio
import matplotlib.pyplot as plt
from IPython.display import display
import torch
import torchvision
from ipywidgets import widgets, HBox, VBox, Layout, Box
from torch.autograd import Variable

def save_pred_gifs(pred_batch, nsamples=5, text=None, batch_first=False, show=False):

    if batch_first:
        pred_batch = pred_batch.permute(1, 0, 2, 3, 4)

    pred_batch = pred_batch.cpu().numpy() * 255.0
    pred_batch = pred_batch.squeeze(2)

    result_path = os.path.join(os.getcwd(), "results/")
    if not os.path.exists(result_path):
        os.mkdir(result_path)

    for i in range(nsamples):
        video = pred_batch[:, i, :]
        imageio.mimsave(os.path.join(
            result_path, f"predicted_frames{i+1}_{text}.gif"), video.astype(np.uint8), "GIF", fps=5)

    if show:
        pred = []
        for i in range(nsamples):
            pred.append(widgets.Image(value=open(os.path.join(
                result_path, f"predicted_frames{i+1}_{text}.gif"), 'rb').read()))

        print("------------predicted Frames (predicted 10 frames of the sequence)----------------")
        display(HBox(pred))

def save_grid_batch(real_batch, pred_batch=None, nsamples=5, text=None, batch_first=False, show=False):
    '''
    sample_batch: batch of sequences shape:(seq_len, batch_size, nc, h, w)
    nsamples: #sequences to display
    '''
    if batch_first:
        real_batch = real_batch.permute(1, 0, 2, 3, 4)

    real_batch = real_batch.cpu().numpy() * 255.0

    if pred_batch is not None:
        if batch_first:
            pred_batch = pred_batch.permute(1, 0, 2, 3, 4)

        pred_batch = pred_batch.cpu().numpy() * 255.0
    seq_len = real_batch.shape[0]

    result_path = os.path.join(os.getcwd(), "results")
    if not os.path.exists(result_path):
        os.mkdir(result_path)

    if pred_batch is not None:
        fig, ax = plt.subplots(2*nsamples, seq_len)
        fig.set_size_inches(18, 10)
        for i in range(nsamples):
            video = pred_batch[:, i, :]
            real_video = real_batch[:, i, :]
            for j in range(seq_len):
                ax[2*i, j].imshow(real_video[j][0], cmap="gray")
                ax[2*i, j].axis("off")

                if j >= 10:
                    ax[2*i+1, j].imshow(video[j][0], cmap="gray")
                    ax[2*i+1, j].axis("off")
                else:
                    h, w = video[j][0].shape
                    whiteFrame = 255 * np.ones((h, w), np.uint8)
                    ax[2*i+1, j].imshow(whiteFrame, cmap="gray")
                    ax[2*i+1, j].axis("off")
        plt.tight_layout()
    else:
        fig, ax = plt.subplots(nsamples, seq_len)
        fig.set_size_inches(18, 5)
        for i in range(nsamples):
            video = real_batch[:, i, :]
            for j in range(seq_len):
                ax[i, j].imshow(video[j][0], cmap="gray")
                ax[i, j].axis("off")
        plt.tight_layout()
    if show:
        plt.savefig(os.path.join(result_path, f"{text}.png"))
        plt.show()
        plt.close(fig)
    else:
        plt.savefig(os.path.join(result_path, f"{text}.png"))
        plt.close(fig)

def show_grid(real_batch, pred_batch=None, nsamples=5, batch_first=False, pred_frames=10):
    '''
    sample_batch: batch of sequences shape:(seq_len, batch_size, nc, h, w)
    nsamples: #sequences to display
    '''

    assert pred_batch is not None
    sl, b, c, h, w = pred_batch.shape
    if batch_first:

        real_batch = real_batch.permute(1, 0, 2, 3, 4)
        pred_batch = pred_batch.permute(1, 0, 2, 3, 4)

    pred_batch[:pred_frames] = torch.zeros(pred_frames, b, c, h, w)

    show_imgs = []
    for i in range(nsamples):
        show_imgs.append(real_batch[:, i, :])
        show_imgs.append(pred_batch[:, i, :])

    show_imgs = torch.stack(show_imgs).reshape(-1, c, h, w)

    grid = torchvision.utils.make_grid(show_imgs, nrow=sl, padding=4)

    return grid

def add_border(video, color, pad=4):
    paddded_video = []
    video = video.unsqueeze(1)

    for i in range(video.shape[0]):
        x = video[i]
        w = x.size()[1]
        nc = x.size()[0]
        px = Variable(torch.zeros(3, w+2*pad, w+2*pad))
        if color == 'red':
            px[0] =0.7
        elif color == 'green':
            px[1] = 0.7
        if nc == 1:
            for c in range(3):
                px[c, pad:w+pad, pad:w+pad] = x
        else:
            px[:, pad:w+pad, pad:w+pad] = x

        paddded_video.append(px)

    paddded_video = torch.stack(paddded_video)

    return paddded_video.squeeze(1).permute(0,2,3,1)

def save_gif_batch(real_batch, pred_batch=None,  nsamples=5, text=None, batch_first=False, show=False):
    '''
    sample_batch: batch of sequences shape:(seq_len, batch_size, nc, h, w)
    nsamples: #sequences to display
    '''
    if batch_first:
        real_batch = real_batch.permute(1, 0, 2, 3, 4)
    # Reverse process before displaying
    real_batch = real_batch.cpu().numpy()
    real_batch = real_batch.squeeze(2)

    if pred_batch is not None:
        if batch_first:
            pred_batch = pred_batch.permute(1, 0, 2, 3, 4)

        pred_batch = pred_batch.cpu().numpy()
        pred_batch = pred_batch.squeeze(2)

    result_path = os.path.join(os.getcwd(), "results")
    if not os.path.exists(result_path):
        os.mkdir(result_path)

    for i in range(nsamples):
        all = []
        video = real_batch[:, i, :]
        video = add_border(torch.from_numpy(video),color="green").cpu().numpy() * 255.0
        imageio.mimsave(os.path.join(result_path, f"past_frames{i+1}_{text}.gif"), video[:10].astype(np.uint8), "GIF", fps=5)
        imageio.mimsave(os.path.join(result_path, f"future_frames{i+1}_{text}.gif"), video[10:].astype(np.uint8), "GIF", fps=5)
        all.append(video[:10])
        if pred_batch is not None:
            video = pred_batch[:, i, :]
            video = add_border(torch.from_numpy(video),color="red").cpu().numpy() * 255.0
            all.append(video[10:])
            imageio.mimsave(os.path.join(result_path, f"predicted_frames{i+1}_{text}.gif"), video[10:].astype(np.uint8), "GIF", fps=5)
            all = np.concatenate(all)
            imageio.mimsave(os.path.join(result_path, f"all_frames{i+1}_{text}.gif"), all.astype(np.uint8), "GIF", fps=5)

    if show:
        pred = []
        past = []
        future = []
        for i in range(nsamples):
            past.append(widgets.Image(value=open(os.path.join(result_path, f"past_frames{i+1}_{text}.gif"), 'rb').read()))
            future.append(widgets.Image(value=open(os.path.join(result_path, f"future_frames{i+1}_{text}.gif"), 'rb').read()))
            if pred_batch is not None:
                pred.append(widgets.Image(value=open(os.path.join(
                    result_path, f"predicted_frames{i+1}_{text}.gif"), 'rb').read()))

        print("------------Context Frames (First 10 frames of the sequence)----------------")
        display(HBox(past))

        print("------------Future Ground Truth Frames (Next 10 frames of the sequence)----------------")
        display(HBox(future))

        if pred_batch is not None:
            print("------------Predicted Frames----------------")
            display(HBox(pred))


from datetime import datetime as dt

class Measurement:
    ATTRIBUTES = [
        "machine_name",
        "sensor_type",
        "date_measurement",
        "start_timestamp",
        "end_timestamp",
        "realvalue",
        "unit"
    ]
    CSV_HEADER = ';'.join(['"' + i + '"' for i in ATTRIBUTES]) + '\n'
    def __init__(self, csv):
        attributes = csv.split(';')
        for i, attr in enumerate(attributes):
            self[Measurement.ATTRIBUTES[i]] = attr

    def to_csv(self, attributes = ATTRIBUTES):
        return ';'.join([self[attr] for attr in attributes])

    def __getitem__(self, key):
        if type(key) is str:
            if key == "machine_name":
                return self.machine_name
            elif key == "sensor_type":
                return self.sensor_type
            elif key == "date_measurement":
                return dt.fromordinal(self.date_measurement).strftime("%Y-%m-%d")
            elif key == "start_timestamp":
                return dt.fromtimestamp(self.start_timestamp).strftime('"%Y-%m-%d %H:%M:%S.%f"')
            elif key == "end_timestamp":
                return dt.fromtimestamp(self.end_timestamp).strftime('"%Y-%m-%d %H:%M:%S.%f"')
            elif key == "realvalue":
                return str(self.realvalue)
            elif key == "unit":
                return self.unit
            else:
                raise KeyError()

    def __setitem__(self, key, value):
        if type(key) is str:
            if key == "machine_name":
                self.machine_name = value
            elif key == "sensor_type":
                self.sensor_type = value
            elif key == "date_measurement":
                if type(value) == int:
                    self.date_measurement = value
                elif type(value) == str:
                    self.date_measurement = dt.strptime(value, "%Y-%m-%d").toordinal()
            elif key == "start_timestamp":
                if type(value) == int:
                    self.start_timestamp = value
                elif type(value) == str:
                    if '.' in value:
                        self.start_timestamp = dt.strptime(value, '"%Y-%m-%d %H:%M:%S.%f"').timestamp()
                    else:
                        self.start_timestamp = dt.strptime(value, '"%Y-%m-%d %H:%M:%S"').timestamp()
            elif key == "end_timestamp":
                if type(value) == int:
                    self.end_timestamp = value
                elif type(value) == str:
                    if '.' in value:
                        self.end_timestamp = dt.strptime(value, '"%Y-%m-%d %H:%M:%S.%f"').timestamp()
                    else:
                        self.end_timestamp = dt.strptime(value, '"%Y-%m-%d %H:%M:%S"').timestamp()
            elif key == "realvalue":
                self.realvalue = float(value)
            elif key == "unit":
                self.unit = value
            else:
                raise KeyError()

# FL02;lifting_gear_a_max;2019-07-10;"2019-07-10 07:13:29.647";"2019-07-10 07:13:29.647";8609.316;mg (milli-g)


# --- Import libraries --- #
import comet as cm
import numpy as np
import xarray as xr
import pickle

# --- Read in data --- #

# Set directories and file names
dailyDir   = '/Users/meganfowler/Documents/NCAR/Analysis/Coupling_initial/data/day/'
middleName = '_day_CESM2_amip_r10i1p1f1_gn_'
varNames   = ['hfls','hfss','mrso','mrsos','tas','hurs','ps']

# Set time period portion of fileNames
timeName_flx  = ['19700101-19791231','19800101-19891231',
                 '19900101-19991231','20000101-20091231','20100101-20150101']

# Read in test file to get lat/lon
fileName = dailyDir+varNames[0]+middleName+timeName_flx[0]+'.nc'
testDF   = xr.open_dataset(fileName, decode_times=True)

# Get lat and lon
lat = testDF.lat.values
lon = testDF.lon.values

# Read in surface properties (T, RH, P)

for iT in range(len(timeName_flx)):

    # Sfc Temp
    tFile          = dailyDir+varNames[4]+middleName+timeName_flx[iT]+'.nc' # File name
    TsfcDF         = xr.open_dataset(tFile,decode_times=True)
    TsfcDF['time'] = TsfcDF.indexes['time'].to_datetimeindex()

    # Sfc RH
    rhFile       = dailyDir+varNames[5]+middleName+timeName_flx[iT]+'.nc' # File name
    rhDF         = xr.open_dataset(rhFile,decode_times=True)
    rhDF['time'] = rhDF.indexes['time'].to_datetimeindex() # Convert from cf time (non-standard calendar) to datetime

    # Sfc pressure
    psFile       = dailyDir+varNames[6]+'_CFday_CESM2_amip_r10i1p1f1_gn_'+timeName_flx[iT]+'.nc' # File name
    psDF         = xr.open_dataset(psFile,decode_times=True)
    psDF['time'] = psDF.indexes['time'].to_datetimeindex() # Convert from cf time (non-standard calendar) to datetime

    if iT==0:
        Tsfc_full  = TsfcDF
        RHsfc_full = rhDF
        Psfc_full  = psDF
    else:
        Tsfc_full  = xr.concat([Tsfc_full,  TsfcDF], dim="time")
        RHsfc_full = xr.concat([RHsfc_full, rhDF],   dim="time")
        Psfc_full  = xr.concat([Psfc_full,  psDF],   dim="time")

    print('Done with ', timeName_flx[iT])

# --- Now compute dewpoint temperature and pressure of the LCL --- #

# Convert temperature to ˚C
T_degC = Tsfc_full.tas.values - 273.15

# Get RH as fraction, not %
RH_frac = RHsfc_full.hurs.values / 100.0

# Convert surface pressure to hPa
PS_hPa = Psfc_full.ps.values / 100.0

# Compute dew point and P_LCL
Td   = np.full([np.shape(T_degC)[0], len(lat), len(lon)], np.nan)  #Saved to be in Kelvin
Plcl = np.full([np.shape(T_degC)[0], len(lat), len(lon)], np.nan)  #Saved to be in hPa

# Define constants for dew point calculation
c1 = 610.94   # [Pa]
a1 = 7.625
b1 = 243.04   # [degC]

for iT in range(np.shape(T_degC)[0]):

    # Compute dewpoint temperature:
    numerator   = b1 * (np.log(RH_frac[iT,:,:]) + ( (a1*T_degC[iT,:,:])/(b1 + T_degC[iT,:,:]) ) )
    denominator = a1 - np.log(RH_frac[iT,:,:]) - ( (a1*T_degC[iT,:,:])/(b1 + T_degC[iT,:,:]) )

    Td[iT,:,:] = (numerator/denominator) + 273.15   #Convert to K here too

    # Compute pressure level of LCL:
    inner        = ((Tsfc_full.tas[iT,:,:] - Td[iT,:,:]) / 223.15) + 1  # Part inside ()
#    Plcl[iT,:,:] = PS_hPa[iT,:,:]*(inner**(-3.5))
    Plcl[iT,:,:] = PS_hPa[iT,:,:] - (PS_hPa[iT,:,:]*(inner**(-3.5)))

    if (iT % 1000)==0:
        print('Done with ', (iT/23726)*100, ' % of days')

 # Add check to set Plcl to zero if it's negative (below the surface pressure)
Plcl[Plcl<0] = np.nan

# Looking at Betts (2004), it looks like Plcl is actually the mean depth to cloud base; not just its pressure level
#   So to get the depth of the layer from sfc to cloud bottom in hPa, need to take Plcl - PS.
# Plcl = PS_hPa - Plcl

# Save the array with residaul S.T. and S.M. z-scores in it
saveDir  = '/Users/meganfowler/Documents/NCAR/Analysis/Coupling_initial/Coupling_CAM6CLM5/processed_data/'
saveFile = 'LCL-pressure-HeightAboveGround_1970-2014.p'

pickle.dump( Plcl, open( saveDir+saveFile, "wb" ), protocol=4 )

print('Finished computing LCL-pressure level successfully and saved pickle file.')


import pandas as pd
import numpy as np
import json
import os
import helper
from matplotlib import pyplot as plt

def calc_percentiles(df):
    """Calculate percentiles of pRRx/pRRx% parameters in AF and SR, return dict.

    Args:
        df (pd.DataFrame): DF with values of pRRx/pRRx% parameters

    Returns:
        (dict): Dictionary with values of percentiles in AF and SR
    """
    params = helper.get_params(df)
    # Parameter types (perc, ms)
    groups = [group for group in params if len(params[group]) > 0]
    perc_dict = {g: dict() for g in groups}
    for group in groups:
        features = params[group]
        perc_dict[group] = {f: {'AF': {}, 'SR': {}} for f in features}
        # Calculate percentiles (0-100)
        for i, feature in enumerate(features):
            for is_af, rhythm, in ((0, 'SR'), (1, 'AF')):
                val = df[df['is_af'] == is_af][feature].values
                perc = list(np.arange(101)/100)
                q = list(np.quantile(val, perc))
                perc_dict[group][feature][rhythm] = {
                    'centiles': perc,
                    'values': q}
    return perc_dict

def plot_distr(perc_dict, db, x_sec, fig_dir):
    """Plot medians, and 10-90 and 25-75 percentile ranges
    of pRRx/pRRx% parameters in AF and SR

    Args:
        perc_dict (dict): Dictionary with parameter names and percentiles in AF and SR
        db (str): Acronym of the database
        x_sec (int): Length of RR sequence [s]
        fig_dir (str): Write directory for images
    """
    for group in perc_dict.keys():
        ax = helper.default_fig()
        title = f'Quartiles, 10-th and 90-th percentile of {group}'
        if group == 'pRRx':
            xlabel = 'Threshold x [ms]'
        else:
            xlabel = 'Threshold x [%]'
        features = list(perc_dict[group].keys())
        x_thr = helper.get_x_thr(features)
        for rhythm, color in (('SR', 'tab:blue'), ('AF', 'tab:orange')):
            # Values of percentiles
            val = np.array([perc_dict[group][f][rhythm]['values'] for f in features])
            # Median line
            p50 = val[:, 50]
            helper.plot_line(
                ax, xval=x_thr, yval=p50, label=rhythm,
                xlim=(0, np.ceil(x_thr[-1]/5)*5),
                ylim=(0, 100), xlabel=xlabel, ylabel='[%]',
                title=title, color=color, marker='o', markersize=2,
                linewidth=2)
            # Bands: 25-75 and 10-90 percentile ranges
            for perc, lw, alpha in [(25, 1, 0.6), (10, 1, 0.3)]:
                p_low = val[:, perc]
                p_high = val[:, 100-perc]
                ax.plot(x_thr, p_low, '-o', markersize=1, color=color,
                        alpha=alpha, linewidth=lw)
                ax.plot(x_thr, p_high, '-o', markersize=1, color=color,
                        alpha=alpha, linewidth=lw)
                ax.fill_between(x_thr, p_low, p_high, alpha=alpha, color=color)
        ax.legend()
        fname = f"distr_{group}_{db}_vs_threshold_{x_sec}s.png"
        helper.save_fig(fig_dir, fname)
        plt.show()

if __name__ == '__main__':
    prrx_dir = '../data/processed'
    fig_dir = '../reports/images/distr'
    x_sec = 60
    for db in ['ltafdb', 'afdb']:
        df = helper.read_prrx(prrx_dir, db, x_sec)
        perc_dict = calc_percentiles(df)
        if not os.path.exists(fig_dir):
            os.makedirs(os.path.join(fig_dir))
        plot_distr(perc_dict, db, x_sec, fig_dir)


import gensim as gs
import numpy as np
import os
import csv
import time
import random
from scipy.io import mmread

# read configuration
exec(open("config.py").read())

if 'data' not in globals():
    data = mmread(dataset)

# by ppl as document word 2 vec on each like entity
d = data.tocsr()

wseq = range(0,d.shape[1])
wseq = ['W{}'.format(i) for i in range(len(wseq))]
wseq = np.array(wseq)

print("nrow: %d, ncol: %d" % (d.shape[0],d.shape[1]))

def get_hot_idx(binary_arr):
    return np.asarray(np.where(binary_arr==1)[0])

def get_word_seq(idx,wseq):
    # if np.max(idx) > len(wseq):
    #     assert 'index %d is out of dimension' % np.max(idx)
    # else:
    return wseq[idx]

def form_corpus(d,wseq):
    corpus = []
    for i in range(0,d.shape[0]):
        b = np.asarray(d[i,:].todense())[0]
        idx = get_hot_idx(b)
        doc = get_word_seq(idx,wseq)
        corpus.append(doc)
    return corpus

class iterate_corpus(object):
    def __init__(self,mtx,wseq):
        # mtx is a sparse matrix
        # wseq is a pre-defined word sequence, like "W0","W1","W2"
        self.mtx = mtx
        self.wseq = wseq

    def __iter__(self):
        for i in range(0,self.mtx.shape[0]):
            b = np.asarray(self.mtx[i,:].todense())[0]
            idx = get_hot_idx(b)
            doc = get_word_seq(idx,self.wseq)
            yield gs.models.doc2vec.TaggedDocument(doc,[i])

# for every data size to build a model
for b in range(len(cuts)):
    corpus = iterate_corpus(d[:cuts[b],:],wseq) # iterable corpus used for training
    input_like = form_corpus(d[:cuts[b],:],wseq) # used for output

    print("===============")
    print("data size = "+str(cuts[b]))
    print("00_PVDBOW starting...")

    start = time.time()

    model_pvdbow = gs.models.doc2vec.Doc2Vec(size=fsize,min_count=1,dm=0,window=8,workers=6)
    model_pvdbow.build_vocab(corpus)
    for _ in range(ite):
        model_pvdbow.train(corpus)
    model_pvdbow.save("00_PVDBOW/model_d"+str(fsize)+"_size"+str(cuts[b])+".doc2vec")

    end = time.time()
    print("take "+str(end-start)+" seconds")

    list_doc2vec = []
    for i in range(len(input_like)):
        this = model_pvdbow.infer_vector(input_like[i])
        list_doc2vec.append(this)
    f = open("00_PVDBOW_n"+str(least)+"_d"+str(fsize)+"_size"+str(cuts[b])+".csv","w")
    w = csv.writer(f)
    w.writerows(list_doc2vec)
    f.close()

    print("01_PDB starting...")

    start = time.time()

    model_pdb = gs.models.doc2vec.Doc2Vec(size=fsize, min_count=1, window=8, workers=6)
    model_pdb.build_vocab(corpus)
    for _ in range(ite):
        model_pdb.train(corpus)
    model_pdb.save("01_PDB/model_d"+str(fsize)+"_size"+str(cuts[b])+".doc2vec")

    end = time.time()
    print("take "+str(end-start)+" seconds")

    list_doc2vec = []
    for i in range(len(input_like)):
        this = model_pdb.infer_vector(input_like[i])
        list_doc2vec.append(this)
    f = open("01_PDB_n"+str(least)+"_d"+str(fsize)+"_size"+str(cuts[b])+".csv","w")
    w = csv.writer(f)
    w.writerows(list_doc2vec)
    f.close()
print("doc2vec embedding model finished.")


# QBApp/boomiAPIParseResults.py
"""
Created on Wed Jan 8 10:16:40 2020

Functions to parse all the boomi atomspnere API calls, retrieving always the fields:
    * count: (Integer).
    * error: (Bolean).
    * results: List of tuples of (id, name, other fields), usually sort by name (index 1)
    * additional info (List).
    * query token (String) used when Boomi Atomsphere API paginates a query (more than 100 rows on th results).

@author: jaimeHMol
"""
# TODO: REFACTOR: Implement the parse and error handling as methods of every object
#                 created on the requests.
# TODO: Improve performance of the recursion implemented for the pagination
# TODO: Unify XML Name Spaces

from . import boomiAPIBuildRequests

import xml.etree.ElementTree as ET
import logging

XMLNAMESPACES = {"xsi": "http://www.w3.org/2001/XMLSchema-instance",
                 "bns": "http://api.platform.boomi.com/"}

def environmentList(environmentsResponse):
    """ Parsing the list of environments received on the response input obtained from Boomi Atompshere API.
    """
    count = 0            # Mandatory
    error = False        # Mandatory
    results = []         # Mandatory
    additional = []      # Optional
    queryToken = ''      # Optional

    environmentsResponseXML = ET.fromstring(environmentsResponse)

    if environmentsResponseXML.tag != "error":

        count = environmentsResponseXML.get("numberOfResults")
        logging.info("*** [Info] total number of results (environments): " + count)

        i = 0
        for child in environmentsResponseXML:
            if child.tag == "{http://api.platform.boomi.com/}result":
                i = i + 1
                results.append((i, child.get("name")))
            elif child.tag == "{http://api.platform.boomi.com/}queryToken":
                queryToken = child.text

        results.sort(key = lambda tup: tup[1])
        results.insert(0,("","Choose..."))

    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + environmentsResponseXML[0].text)

    return count, error, results, additional, queryToken

def atomList(atomsResponse):
    """ Parsing the list of atoms received on the response input obtained from Boomi Atompshere API.
    """
    count = 0       # Mandatory
    error = False   # Mandatory
    results = []    # Mandatory
    additional = [] # Optional
    queryToken = '' # Optional

    atomsResponseXML = ET.fromstring(atomsResponse)

    # logging.info(atomsResponseXML.tag)
    if atomsResponseXML.tag != "error":

        count = atomsResponseXML.get('numberOfResults')
        logging.info("*** [Info] total number of results (atoms): " + count)

        i = 0
        for child in atomsResponseXML:
            if child.tag == "{http://api.platform.boomi.com/}result":
                i = i + 1
                results.append((child.get("id"), child.get("name")))
            elif child.tag == "{http://api.platform.boomi.com/}queryToken":
                queryToken = child.text

        results.sort(key = lambda tup: tup[1])
        logging.info("result: " + str(results))
        results.insert(0,("","Choose..."))

    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + atomsResponseXML[0].text)

    return count, error, results, additional, queryToken

def processList(request, processesResponse):
    """ Parsing the list of processes received on the response input obtained from Boomi Atompshere API.
    """
    count = 0            # Mandatory
    error = False        # Mandatory
    results = []         # Mandatory
    additional = []      # Optional
    queryToken = ''      # Optional
    accResults = []

    processesResponseXML = ET.fromstring(processesResponse)

    if processesResponseXML.tag != "error":

        count = processesResponseXML.get("numberOfResults")
        logging.info("*** [Info] total number of results (processes): " + count)

        i = 0
        for child in processesResponseXML:
            if child.tag == "{http://api.platform.boomi.com/}result": #HINT: Be carefull with the name spaces
                i = i + 1
                results.append((child.get("id"), child.get("name")))
            elif child.tag == "{http://api.platform.boomi.com/}queryToken":
                queryToken = child.text

        # Recursive block to retrieve all the results (using the queryToken)
        # and the pagination used by the Boomi Atomsphere API
        accResults = accResults + results
        if queryToken != '':
            # accResults.append(results)
            processesResponse = boomiAPIBuildRequests.queryPagination(request, "Process", queryToken)
            _, _, newResult, _, _ = processList(request, processesResponse)

            return count, error, accResults + newResult, additional, queryToken
        else:

            accResults.sort(key = lambda tup: tup[1])
            return count, error, accResults, additional, queryToken

        # logging.info("result: " + str(results))
    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + processesResponseXML[0].text)

    return count, error, results, additional, queryToken

def scheduleList(scheduleResponse):
    """ Parsing the list of schedules received on the response input obtained from Boomi Atompshere API.
    """
    count = 0            # Mandatory
    error = False        # Mandatory
    results = []         # Mandatory
    additional = []      # Optional
    queryToken = ''      # Optional

    schedulesResponseXML = ET.fromstring(scheduleResponse)
    if schedulesResponseXML.tag != "error":

        count = schedulesResponseXML.get("numberOfResults")

        i = 0
        for child in schedulesResponseXML:
            if child.tag == "{http://api.platform.boomi.com/}result":
                i = i + 1
                if child.find("bns:Schedule", XMLNAMESPACES) is not None:
                    cadence = child.find("bns:Schedule", XMLNAMESPACES).get("minutes")
                else:
                    cadence = "NA"
                results.append((child.get("atomId"), child.get("processId"), cadence, child.get("id")))
            elif child.tag == "{http://api.platform.boomi.com/}queryToken":
                queryToken = child.text

        results.sort(key = lambda tup: tup[1])

    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + schedulesResponseXML[0].text)

    return count, error, results, additional, queryToken

def scheduleStatusGet(scheduleGetResponse):
    """ Parsing the schedule status received on the response input obtained from Boomi Atompshere API.
    """
    error = False        # Mandatory
    result = ''          # Mandatory

    scheduleGetResponseXML = ET.fromstring(scheduleGetResponse)
    if scheduleGetResponseXML.tag != "error":
        if scheduleGetResponseXML.get("enabled") == "true":
            result = "Yes"
        elif scheduleGetResponseXML.get("enabled") == "false":
            result = "No"
    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + scheduleGetResponseXML[0].text)

    return error, result

def executionRecordList(executionResponse):
    """ Parsing the list of executions received on the response input obtained from Boomi Atompshere API.
    """
    count = 0             # Mandatory
    error = False         # Mandatory
    results = []           # Mandatory
    additional = []       # Optional
    queryToken = ''       # Optional

    executionsResponseXML = ET.fromstring(executionResponse)
    if executionsResponseXML.tag != "error":

        count = executionsResponseXML.get("numberOfResults")
        logging.info("*** [Info] total number of results (executions): " + count)

    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + executionsResponseXML[0].text)
    pass

def deploymentList(deploymentResponse):
    """ Parsing the list of deployments received on the response input obtained from Boomi Atompshere API.
    """
    count = 0            # Mandatory
    error = False        # Mandatory
    results = []          # Mandatory
    additional = []      # Optional
    queryToken = ''      # Optional

    deploymentResponseXML = ET.fromstring(deploymentResponse)
    if deploymentResponseXML.tag != "error":

        count = deploymentResponseXML.get("numberOfResults")

        i = 0
        for child in deploymentResponseXML:
            if child.tag == "{http://api.platform.boomi.com/}result":
                i = i + 1
                if child.find("bns:processId", XMLNAMESPACES) is not None:
                    procId = child.find("bns:processId", XMLNAMESPACES).text
                if child.find("bns:environmentId", XMLNAMESPACES) is not None:
                    environmentId = child.find("bns:environmentId", XMLNAMESPACES).text

                results.append((procId, environmentId))
            elif child.tag == "{http://api.platform.boomi.com/}queryToken":
                queryToken = child.text

        results.sort(key = lambda tup: tup[1])
        # logging.info("deploymentResult: " + str(results))

    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + deploymentResponseXML[0].text)

    return count, error, results, additional, queryToken

def environmentAtomList(environmentAtomsResponse):
    """ Parsing the list of atoms and their related environments received on the response input obtained from Boomi Atompshere API.
    """
    count = 0       # Mandatory
    error = False   # Mandatory
    results = {}    # Mandatory
    additional = [] # Optional
    queryToken = '' # Optional

    environmentAtomsResponseXML = ET.fromstring(environmentAtomsResponse)

    # logging.info(atomsResponseXML.tag)
    if environmentAtomsResponseXML.tag != "error":

        count = environmentAtomsResponseXML.get('numberOfResults')
        logging.info("*** [Info] total number of results (environment-atoms): " + count)

        i = 0
        for child in environmentAtomsResponseXML:
            if child.tag == "{http://api.platform.boomi.com/}result":
                i = i + 1
                #Devolver un diccionario
                v_atomId, v_environmentId = child.get("atomId"), child.get("environmentId")
                results[v_atomId] = v_environmentId

            elif child.tag == "{http://api.platform.boomi.com/}queryToken":
                queryToken = child.text

        # results.sort(key = lambda tup: tup[1])

    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + environmentAtomsResponseXML[0].text)

    return count, error, results, additional, queryToken

def ProcessEnvironmentAttachment(processEnvironmentAttachmentResponse):
    """ Parsing the Process-Environment attachment information received on the response input obtained from Boomi Atompshere API.
    """
    count = 0       # Mandatory
    error = False   # Mandatory
    results = ''    # Mandatory
    additional = [] # Optional
    queryToken = '' # Optional

    processEnvironmentAttachmentResponseXML = ET.fromstring(processEnvironmentAttachmentResponse)

    # logging.info(atomsResponseXML.tag)
    if processEnvironmentAttachmentResponseXML.tag != "error":

        count = processEnvironmentAttachmentResponseXML.get('numberOfResults')

        i = 0
        for child in processEnvironmentAttachmentResponseXML:
            if child.tag == "{http://api.platform.boomi.com/}result":
                i = i + 1
                results = child.get("id")

            elif child.tag == "{http://api.platform.boomi.com/}queryToken":
                queryToken = child.text

        # results.sort(key = lambda tup: tup[1])
    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + processEnvironmentAttachmentResponseXML[0].text)

    return count, error, results, additional, queryToken