import random
import pyasge
from game.pathfinding import resolve
from game.catdata import CatType
from enum import IntEnum

class NodeType(IntEnum):
    SELECTOR = 1
    SEQUENCE = 2
    DECORATOR = 3
    LEAF = 4
    ROOT = 5

class BehaviourTreeRodent:
    def __init__(self, data, rodent, cats):
        # Last branch with the path to door
        self.branch4 = SequenceNode(
            [DecoratorNode(OnDoorTile(data, rodent), data), GoToDoor(data, rodent)], data)
        # Branch if cat is in agro range
        self.branch3 = SequenceNode(
            [InRangeRodent(cats, rodent, rodent.rodent_data.sight_range),
             GoToCat(data, rodent, cats)], data)
        # Branch if cat is in attack range
        self.branch2 = SequenceNode(
            [InRangeRodent(cats, rodent, rodent.rodent_data.attack_range),
             GoToCat(data, rodent, cats), AttackCat(cats, rodent, rodent.rodent_data)], data)
        # Branch if rodent is low and needs to go to spawn
        self.branch1 = SequenceNode(
            [LowHP(rodent.rodent_data, rodent.rodent_data.low_hp_range), DecoratorNode(OnSpawnTile(data, rodent), data),
             GoToSpawn(data, rodent)], data)

        self.base_selector = SelectorNode(
            [self.branch1, self.branch2, self.branch3,
             self.branch4, GoToCat(data, rodent, cats)], data)

        self.root = RootNode(self.base_selector)

    def update(self, game_time: pyasge.GameTime):
        self.root.update(game_time)

class BehaviourTreeCat:
    def __init__(self, data, cat, rodents, cat_data):
        # If RODENTS are in CAT range
        self.range_node = InRangeCat(rodents, cat, cat_data)
        # Shoot projectile at closest RODENT
        self.shoot_node = ShootProjectile(rodents, cat, cat_data)

        self.branch1 = SequenceNode([self.range_node, self.shoot_node], data)
        self.base_selector = SelectorNode([self.branch1], data)
        self.root = RootNode(self.base_selector)

    def update(self, game_time: pyasge.GameTime):
        self.root.update(game_time)

class BehaviourTreeResourceCat:
    def __init__(self, data, cat):
        # Generate yarn on a timer
        self.generate_node = GenerateYarn(data, cat)

        self.branch1 = SequenceNode([self.generate_node], data)
        self.base_selector = SelectorNode([self.branch1], data)
        self.root = RootNode(self.base_selector)

    def update(self, game_time: pyasge.GameTime):
        self.root.update(game_time)

class BehaviourTreeSlowCat:
    def __init__(self, data, cat, rodents, cat_data):
        # Slow RODENTS in range
        self.slow_node = SlowRodent(rodents, cat, cat_data)

        self.branch1 = SequenceNode([self.slow_node], data)
        self.base_selector = SelectorNode([self.branch1], data)
        self.root = RootNode(self.base_selector)

    def update(self, game_time: pyasge.GameTime):
        self.root.update(game_time)

class BehaviourTreeBoostCat:
    def __init__(self, data, cat, cats, cat_data):
        # Boost CATS in range
        self.boost_node = BoostCat(cats, cat, cat_data)

        self.branch1 = SequenceNode([self.boost_node], data)
        self.base_selector = SelectorNode([self.branch1], data)
        self.root = RootNode(self.base_selector)

    def update(self, game_time: pyasge.GameTime):
        self.root.update(game_time)

class RootNode:
    def __init__(self, child):
        # Root node, returns what is child returns
        self.type = NodeType.ROOT
        self.child = child

    def update(self, game_time: pyasge.GameTime):
        return self.child.update(game_time)

class SequenceNode:
    def __init__(self, nodes, data):
        # Sequence node, goes through children until one fails or all are gone through
        # Returns true if all children succeeded, returns false if one child fails
        self.data = data
        self.type = NodeType.SEQUENCE
        self.children = nodes

    def update(self, game_time: pyasge.GameTime):
        for child in self.children:
            update = child.update(game_time)
            if not update:
                return update
        return True

class SelectorNode:
    def __init__(self, nodes, data):
        # Selector node, goes through children until one succeeds or all are gone through
        # Returns false if all children fail, returns true if one child succeeds
        self.data = data
        self.type = NodeType.SELECTOR
        self.children = nodes

    def update(self, game_time: pyasge.GameTime):
        for child in self.children:
            update = child.update(game_time)
            if update:
                return update
        return False

class DecoratorNode:
    def __init__(self, child, data):
        # Decorator node, returns opposite of child
        self.data = data
        self.type = NodeType.DECORATOR
        self.child = child

    def update(self, game_time: pyasge.GameTime):
        return not self.child.update(game_time)

class InRangeCat:
    def __init__(self, targets, actor, data):
        # If there is a target is in range, return true [CHANGING RANGE, USED FOR CATS]
        self.targets = targets
        self.actor = actor
        self.data = data

    def update(self, game_time: pyasge.GameTime):
        att_range = self.data.attack_range * pow(self.data.stat_inc_per_level, self.data.upgrade_level-1)
        if self.data.boost_buff:
            att_range = att_range * self.data.boost_percent
        for target in self.targets:
            distance = pyasge.Point2D.distance(target.sprite.midpoint, self.actor.sprite.midpoint)
            if distance <= att_range and not target.rodent_data.invisible:
                return True
        return False

class InRangeRodent:
    def __init__(self, targets, actor, range_fixed):
        # If there is a target in range, return true [FIXED RANGE, USED FOR RODENTS]
        self.targets = targets
        self.actor = actor
        self.range = range_fixed

    def update(self, game_time: pyasge.GameTime):
        for target in self.targets:
            distance = pyasge.Point2D.distance(target.sprite.midpoint, self.actor.sprite.midpoint)
            if distance <= self.range and target.cat_data.cat_type == CatType.BLOCKER:
                self.actor.rodent_data.cat_in_range = True
                return True
        self.actor.rodent_data.cat_in_range = False
        return False

class ShootProjectile:
    def __init__(self, targets, actor, cat_data):
        # Shoot projectile at closest target
        self.targets = targets
        self.actor = actor
        self.timer = 0
        self.cat_data = cat_data

    def update(self, game_time: pyasge.GameTime):
        max_reload_timer = self.cat_data.reload_time / pow(self.cat_data.stat_inc_per_level, self.cat_data.upgrade_level-1)
        if self.cat_data.boost_buff:
            max_reload_timer = max_reload_timer / self.cat_data.boost_percent
        # If entity has had enough time between shots
        if self.timer >= max_reload_timer:
            # Find the closest target
            closest_target = None
            closest_dist = 1000
            for target in self.targets:
                distance = pyasge.Point2D.distance(target.sprite.midpoint, self.actor.sprite.midpoint)
                if distance < closest_dist and not target.rodent_data.invisible:
                    closest_dist = distance
                    closest_target = target
            # Shoot at closest target
            self.actor.cat_data.shooting = True
            self.actor.cat_data.targeted_rodent = closest_target
            self.timer = 0
        else:
            self.timer += game_time.frame_time
        return True

class LowHP:
    def __init__(self, data, low_hp):
        # Returns true if target is low hp
        self.data = data
        self.low_hp = low_hp

    def update(self, game_time: pyasge.GameTime):
        if self.data.fleeing:
            return True
        if self.data.hp <= self.low_hp:
            self.data.fleeing = True
            return True
        return False

class OnSpawnTile:
    def __init__(self, game_data, rodent):
        self.game_data = game_data
        self.rodent = rodent

    def update(self, game_time: pyasge.GameTime):
        # If rodent pos in range of game_data.spawn_tiles, de spawn the rodent and return true
        rodent_tile = self.game_data.game_map.tile(self.rodent.sprite.midpoint)
        for spawn in self.game_data.game_map.spawn_points:
            sp_point = pyasge.Point2D(spawn[0], spawn[1])
            spawn_tile = self.game_data.game_map.tile(sp_point)
            if spawn_tile == rodent_tile:
                self.rodent.rodent_data.on_spawn_tile = True
                return True
        return False

class OnDoorTile:
    def __init__(self, game_data, rodent):
        self.game_data = game_data
        self.rodent = rodent

    def update(self, game_time: pyasge.GameTime):
        # If rodent pos in range of game_data.home_points, de spawn the rodent and return true
        rodent_tile = self.game_data.game_map.tile(self.rodent.sprite.midpoint)
        for point in self.game_data.game_map.home_points:
            sp_point = pyasge.Point2D(point[0], point[1])
            spawn_tile = self.game_data.game_map.tile(sp_point)
            if spawn_tile == rodent_tile:
                self.rodent.rodent_data.on_home_tile = True
                return True
        return False

class GoToSpawn:
    def __init__(self, game_data, rodent):
        self.game_data = game_data
        self.rodent = rodent

    def update(self, game_time: pyasge.GameTime):
        # Path find to spawn
        if not self.rodent.rodent_data.update_path:
            return True
        self.rodent.rodent_data.update_path = False
        quickest_path = []
        quickest_cost = 0
        for spawn in self.game_data.game_map.spawn_points:
            path = resolve(pyasge.Point2D(spawn[0], spawn[1]), self.game_data,
                           self.rodent.sprite.midpoint, self.rodent.rodent_data.can_hidden_path)
            cost = path.pop(-1)
            if len(quickest_path) == 0 or cost < quickest_cost:
                quickest_path = path
                quickest_cost = cost
        self.rodent.destination = quickest_path
        return True

class GoToDoor:
    def __init__(self, game_data, rodent):
        self.game_data = game_data
        self.rodent = rodent

    def update(self, game_time: pyasge.GameTime):
        # Path find to door
        if not self.rodent.rodent_data.update_path:
            return True
        self.rodent.rodent_data.update_path = False
        quickest_path = []
        quickest_cost = 0
        for home_point in self.game_data.game_map.home_points:
            path = resolve(pyasge.Point2D(home_point[0], home_point[1]), self.game_data,
                           self.rodent.sprite.midpoint, self.rodent.rodent_data.can_hidden_path)
            cost = path.pop(-1)
            if len(quickest_path) == 0 or cost < quickest_cost:
                quickest_path = path
                quickest_cost = cost
        self.rodent.destination = quickest_path
        return True

class AttackCat:
    def __init__(self, targets, actor, rodent_data):
        # Attacks closest target [USED FOR RODENTS]
        self.targets = targets
        self.actor = actor
        self.timer = 0
        self.rodent_data = rodent_data

    def update(self, game_time: pyasge.GameTime):
        max_reload_timer = self.rodent_data.time_between_hits
        # If entity has had enough time between hits
        if self.timer >= max_reload_timer:
            # Find closest target
            closest_target = None
            closest_dist = 1000
            for target in self.targets:
                distance = pyasge.Point2D.distance(target.sprite.midpoint, self.actor.sprite.midpoint)
                if distance < closest_dist and target.cat_data.cat_type == CatType.BLOCKER:
                    closest_dist = distance
                    closest_target = target
            # Attack closest target
            self.actor.rodent_data.attacking = True
            self.actor.rodent_data.targeted_cat = closest_target
            self.timer = 0
        else:
            self.timer += game_time.frame_time
        #print("Attacking cat")
        return True

class GoToCat:
    def __init__(self, game_data, actor, targets):
        self.game_data = game_data
        self.actor = actor
        self.targets = targets

    def update(self, game_time: pyasge.GameTime):
        # Find closest target
        if not self.actor.rodent_data.update_path:
            return True
        self.actor.rodent_data.update_path = False
        closest_target = None
        closest_dist = 1000
        for target in self.targets:
            distance = pyasge.Point2D.distance(target.sprite.midpoint, self.actor.sprite.midpoint)
            if distance < closest_dist and target.cat_data.cat_type == CatType.BLOCKER:
                closest_dist = distance
                closest_target = target
        self.actor.rodent_data.targeted_cat = closest_target
        # Path find to target cat
        path = resolve(pyasge.Point2D(closest_target.sprite.midpoint.x, closest_target.sprite.midpoint.y),
                       self.game_data, self.actor.sprite.midpoint, self.actor.rodent_data.can_hidden_path)
        path.pop(-1)
        self.actor.destination = path
        return True

class GenerateYarn:
    def __init__(self, game_data, cat):
        self.game_data = game_data
        self.cat = cat
        self.timer = 0

    def update(self, game_time: pyasge.GameTime):
        # Inc Yarn
        if self.timer >= self.cat.cat_data.reload_time:
            self.game_data.yarn += self.cat.cat_data.yarn_per_resource
            self.timer = 0
        else:
            self.timer += game_time.frame_time

        return True

class SlowRodent:
    def __init__(self, targets, actor, data):
        # Slow all targets in range
        self.targets = targets
        self.actor = actor
        self.data = data

    def update(self, game_time: pyasge.GameTime):
        # Determines if target is in range
        att_range = self.data.attack_range * pow(self.data.stat_inc_per_level, self.data.upgrade_level-1)
        for target in self.targets:
            distance = pyasge.Point2D.distance(target.sprite.midpoint, self.actor.sprite.midpoint)
            # De buffs them if in range
            if distance <= att_range:
                target.rodent_data.slow_de_buff = True
            else:
                target.rodent_data.slow_de_buff = target.rodent_data.slow_de_buff
        return True

class BoostCat:
    def __init__(self, targets, actor, data):
        # Boost all targets in range
        self.targets = targets
        self.actor = actor
        self.data = data

    def update(self, game_time: pyasge.GameTime):
        # Determines if in range
        att_range = self.data.attack_range * pow(self.data.stat_inc_per_level, self.data.upgrade_level-1)
        for target in self.targets:
            distance = pyasge.Point2D.distance(target.sprite.midpoint, self.actor.sprite.midpoint)
            # Buffs them if in range
            if distance <= att_range:
                target.cat_data.boost_buff = True
            else:
                target.cat_data.boost_buff = False
        return True


import matplotlib.image as mpimg
import os
from os import walk
import matplotlib.pyplot as plt
import numpy as np
import cv2
import pickle
import glob
import time
from sklearn.svm import LinearSVC
from sklearn.preprocessing import StandardScaler
from multiprocessing import cpu_count
from functools import partial
from multiprocessing import Pool

#from skimage.feature import hog
from extract_feature_functions import *
from lane_processing_pipeline import *
# NOTE: the next import is only valid for scikit-learn version <= 0.17
# for scikit-learn >= 0.18 use:
# from sklearn.model_selection import train_test_split
from sklearn.model_selection import train_test_split

# Define a function to extract features from a single image window
# This function is very similar to extract_features()
# just for a single image rather than list of images
def single_img_features(img, color_space='RGB', spatial_size=(32, 32),
                        hist_bins=32, orient=9,
                        pix_per_cell=8, cell_per_block=2, hog_channel=0,
                        spatial_feat=True, hist_feat=True, hog_feat=True):
    #1) Define an empty list to receive features
    img_features = []
    #2) Apply color conversion if other than 'RGB'
    if color_space != 'RGB':
        if color_space == 'HSV':
            feature_image = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)
        elif color_space == 'LUV':
            feature_image = cv2.cvtColor(img, cv2.COLOR_RGB2LUV)
        elif color_space == 'HLS':
            feature_image = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
        elif color_space == 'YUV':
            feature_image = cv2.cvtColor(img, cv2.COLOR_RGB2YUV)
        elif color_space == 'YCrCb':
            feature_image = cv2.cvtColor(img, cv2.COLOR_RGB2YCrCb)
    else: feature_image = np.copy(img)
    #3) Compute spatial features if flag is set
    if spatial_feat == True:
        spatial_features = bin_spatial(feature_image, size=spatial_size)
        #4) Append features to list
        img_features.append(spatial_features)
    #5) Compute histogram features if flag is set
    if hist_feat == True:
        hist_features = color_hist(feature_image, nbins=hist_bins)
        #6) Append features to list
        img_features.append(hist_features)
    #7) Compute HOG features if flag is set
    if hog_feat == True:
        if hog_channel == 'ALL':
            hog_features = []
            for channel in range(feature_image.shape[2]):
                hog_features.extend(get_hog_features(feature_image[:,:,channel],
                                    orient, pix_per_cell, cell_per_block,
                                    vis=False, feature_vec=True))
        else:
            hog_features = get_hog_features(feature_image[:,:,hog_channel], orient,
                        pix_per_cell, cell_per_block, vis=False, feature_vec=True)
        #8) Append features to list
        img_features.append(hog_features)

    #9) Return concatenated array of features
    return np.concatenate(img_features)

# Define a function you will pass an image
# and the list of windows to be searched (output of slide_windows())
def search_windows(windows, img, clf, scaler, color_space='RGB',
                    spatial_size=(32, 32), hist_bins=32,
                    hist_range=(0, 256), orient=9,
                    pix_per_cell=8, cell_per_block=2,
                    hog_channel=0, spatial_feat=True,
                    hist_feat=True, hog_feat=True):

    #1) Create an empty list to receive positive detection windows
    on_windows = []
    test_img = []

    #2) Iterate over all windows in the list
    t_feature = 0
    t_prediction = 0
    features_count = 0
    for window in windows:
        #3) Extract the test window from original image
        test_img = cv2.resize(img[window[0][1]:window[1][1], window[0][0]:window[1][0]], (64, 64))
        #4) Extract features for that window using single_img_features()
        t1 = time.time()
        features = single_img_features(test_img, color_space=color_space,
                            spatial_size=spatial_size, hist_bins=hist_bins,
                            orient=orient, pix_per_cell=pix_per_cell,
                            cell_per_block=cell_per_block,
                            hog_channel=hog_channel, spatial_feat=spatial_feat,
                            hist_feat=hist_feat, hog_feat=hog_feat)
        t2 = time.time()
        t_feature += t2-t1

        #5) Scale extracted features to be fed to classifier
        test_features = scaler.transform(np.array(features).reshape(1, -1))
        #6) Predict using your classifier
        t1 = time.time()
        prediction = clf.predict(test_features)
        t2 = time.time()
        t_prediction += t2-t1

        #7) If positive (prediction == 1) then save the window
        if prediction == 1:
            on_windows.append(window)

#     print('Total Feature extraction time {:.5f}'.format(t_feature))
#     print('Total Prediction time {:.5f}'.format(t_prediction))
    #8) Return windows for positive detections
    return on_windows

def get_image_path(path, nImgs=None):
    image_path = []

    imgCount = 0
    for (dirpath, _, _) in walk(path):
        images = glob.glob(dirpath + '/*.png')
        for img in images:
            image_path.append(img)
            if imgCount > nImgs:
                break
            imgCount += 1
    return image_path

# Read in cars and notcars
def train_load_svc(params):
    # Get location of all vehicle images
    nTrain_img = 8000
    path ="../training_data/vehicles"
    cars = get_image_path(path, nTrain_img)
    vehicle_labels = [1] * len(cars)
    print('Number of vehicle images read: {}'.format(len(cars)))

    # Get location of all non vehicle images
    path ="../training_data/non-vehicles"
    notcars = get_image_path(path, nTrain_img)
    non_vehicle_labels = [0] * len(notcars)
    print('Number of non-vehicle images read: {}'.format(len(notcars)))

    ### Set parameters
    color_space = params['color_space']
    orient = params['orient']
    pix_per_cell = params['pix_per_cell']
    cell_per_block = params['cell_per_block']
    hog_channel = params['hog_channel']
    spatial_size = params['spatial_size']
    hist_bins = params['hist_bins']
    spatial_feat = params['spatial_feat']
    hist_feat = params['hist_feat']
    hog_feat = params['hog_feat']
    y_start_stop = params['y_start_stop']

#     #plot hog feature
#     sample_img = mpimg.imread(cars[52])
#     sample_img_cspace = cv2.cvtColor(sample_img, cv2.COLOR_RGB2HLS)
#
#     _, vis_img_ch0 = get_hog_features(sample_img_cspace[:,:,0],
#                                             orient, pix_per_cell, cell_per_block,
#                                             vis=True, feature_vec=True)
#     _, vis_img_ch1 = get_hog_features(sample_img_cspace[:,:,1],
#                                             orient, pix_per_cell, cell_per_block,
#                                             vis=True, feature_vec=True)
#     _, vis_img_ch2 = get_hog_features(sample_img_cspace[:,:,2],
#                                             orient, pix_per_cell, cell_per_block,
#                                             vis=True, feature_vec=True)
#     plt.subplot(231)
#     plt.imshow(sample_img)
#     plt.subplot(232)
#     plt.imshow(sample_img_cspace)
#     plt.subplot(234)
#     plt.imshow(vis_img_ch0)
#     plt.subplot(235)
#     plt.imshow(vis_img_ch1)
#     plt.subplot(236)
#     plt.imshow(vis_img_ch2)
#     plt.show

    car_features = extract_features(cars, color_space=color_space,
                            spatial_size=spatial_size, hist_bins=hist_bins,
                            orient=orient, pix_per_cell=pix_per_cell,
                            cell_per_block=cell_per_block,
                            hog_channel=hog_channel, spatial_feat=spatial_feat,
                            hist_feat=hist_feat, hog_feat=hog_feat)
    notcar_features = extract_features(notcars, color_space=color_space,
                            spatial_size=spatial_size, hist_bins=hist_bins,
                            orient=orient, pix_per_cell=pix_per_cell,
                            cell_per_block=cell_per_block,
                            hog_channel=hog_channel, spatial_feat=spatial_feat,
                            hist_feat=hist_feat, hog_feat=hog_feat)

    # Create an array stack of feature vectors
    X = np.vstack((car_features, notcar_features)).astype(np.float64)

    # Define the labels vector
    y = np.hstack((np.ones(len(car_features)), np.zeros(len(notcar_features))))
    # Split up data into randomized training and test sets
    rand_state = np.random.randint(0, 100)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=rand_state)

    # Fit a per-column scaler
    X_scaler = StandardScaler().fit(X_train)
    # Apply the scaler to X
    X_train = X_scaler.transform(X_train)
    X_test = X_scaler.transform(X_test)

    print('Using:',orient,'orientations',pix_per_cell,
        'pixels per cell and', cell_per_block,'cells per block')
    print('Feature vector length:', len(X_train[0]))
    # Use a linear SVC
    svc = LinearSVC(C=10)
    # Check the training time for the SVC
    t=time.time()
    svc.fit(X_train, y_train)
    t2 = time.time()
    print(round(t2-t, 2), 'Seconds to train SVC...')
    # Check the score of the SVC
    print('Test Accuracy of SVC = ', round(svc.score(X_test, y_test), 4))
    # Check the prediction time for a single sample
    t=time.time()

    # pickle the model
    print('Pickling SVM model ...')
    save_quant = {'clf': svc, 'X_scaler':X_scaler}
    with open('../car_classifier.p', 'wb') as handle:
        pickle.dump(save_quant, handle)

    return svc, X_scaler

def find_car_in_frame(image, svc, X_scaler, params):

    ### Set parameters
    color_space = params['color_space']
    orient = params['orient']
    pix_per_cell = params['pix_per_cell']
    cell_per_block = params['cell_per_block']
    hog_channel = params['hog_channel']
    spatial_size = params['spatial_size']
    hist_bins = params['hist_bins']
    spatial_feat = params['spatial_feat']
    hist_feat = params['hist_feat']
    hog_feat = params['hog_feat']
    y_start_stop = params['y_start_stop']

    draw_image = np.copy(image)

    # Uncomment the following line if you extracted training
    # data from .png images (scaled 0 to 1 by mpimg) and the
    # image you are searching is a .jpg (scaled 0 to 255)
    draw_image = draw_image.astype(np.float32)/255

#     x_start_stop = [[350, 850], [200, 1100], [0, 1280], [0, 1280]]
#     y_start_stop = [[375, 575], [300, 600], [300, 650], [300, 650]]
#     windows = slide_window(draw_image, x_start_stop=x_start_stop, y_start_stop=y_start_stop,
#                         scale=[0.5, 1.0, 1.55, 2.5],
#                         xy_window=(64, 64), xy_overlap=(0.5, 0.5))
#     window_img = draw_boxes(draw_image, windows, color=(255, 0, 0), thick=2)

#     x_start_stop = [[200, 1000]] #0.75
#     y_start_stop = [[375, 500]]
    x_start_stop = [[100, 1100], [200, 1200], [100, 1280]]
    y_start_stop = [[375, 600], [400, 650], [400, 650]]
    windows = slide_window(image, x_start_stop=x_start_stop, y_start_stop=y_start_stop,
                        scale=[0.75, 1.5, 2.5],
                        xy_window=(64, 64), xy_overlap=(0.5, 0.5))
    window_img = draw_boxes(draw_image, windows, color=(0, 0, 255), thick=2)

    t1=time.time()
    hot_windows = search_windows(windows, draw_image, svc, X_scaler, color_space=color_space,
                            spatial_size=spatial_size, hist_bins=hist_bins,
                            orient=orient, pix_per_cell=pix_per_cell,
                            cell_per_block=cell_per_block,
                            hog_channel=hog_channel, spatial_feat=spatial_feat,
                            hist_feat=hist_feat, hog_feat=hog_feat)
    t2=time.time()
    print('{} windows to search for cars...'.format(len(windows) ))
    print('{:2f} seconds to find cars.'.format(t2-t1))
#     window_img = draw_boxes(draw_image, hot_windows, color=(0, 0, 255), thick=3)

    heat_img, labels = add_heat(np.zeros(image.shape[0:2]), hot_windows, threshold=3)

#     marked_image, bbox = draw_labeled_bboxes(window_img, labels)
    marked_image, bbox = draw_labeled_bboxes(image, labels)

#     plt.figure(figsize=(12, 2))
#     plt.subplot(1,3,1)
#     plt.imshow(window_img)
#     plt.subplot(1,3,2)
#     plt.imshow(heat_img, cmap='hot')
#     plt.subplot(1,3,3)
#     plt.imshow(marked_image)
#     plt.show()

    return marked_image, heat_img, bbox

if __name__ == '__main__':

        ### TODO: Tweak these parameters and see how the results change.
    params = {}
    params['color_space'] = 'HLS' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
    params['orient'] = 11  # HOG orientations
    params['pix_per_cell'] = 8 # HOG pixels per cell
    params['cell_per_block'] = 2 # HOG cells per block
    params['hog_channel'] = 'ALL' # Can be 0, 1, 2, or "ALL"
    params['spatial_size'] = (16, 16) # Spatial binning dimensions
    params['hist_bins'] = 8    # Number of histogram bins
    params['spatial_feat'] = True # Spatial features on or off
    params['hist_feat'] = True # Histogram features on or off
    params['hog_feat'] = True # HOG features on or off
    params['y_start_stop'] = [400, 670] # Min and max in y to search in slide_window()

    if os.path.isfile('../car_classifier.p'):
        print('Loading trained model ../car_classifier.p')
        print('To train a new model please delete this file')
        load_quant = pickle.load(open('../car_classifier.p', 'rb'))
        svc = load_quant['clf']
        X_scaler = load_quant['X_scaler']
    else:
        svc, X_scaler = train_load_svc(params)

    img = mpimg.imread('../test_images/bbox-example-image.jpg')

    marked_image, heat_img, bbox = find_car_in_frame(img, svc, X_scaler, params)

    plt.figure()
    plt.imshow(marked_image)
    plt.show()

#     # calculate perspective transform
#     M, M_inv = calcPerspectiveTransform(pltFlg=False)
#     # read camera calibration
#     mtx, dist = load_cam_calibration()
#
#     # detect lane location and mark them
#     imgUndistort, left_fit, right_fit, lane_curv, lane_offset, calc_check, combined_binary, unwarped_marked = \
#         laneMarker(img, M, M_inv, mtx, dist, 0, np.array([0, 0, 0]), np.array([0, 0, 0]))
#
#     composite_img = annotate_output_figure(imgUndistort, left_fit, right_fit, combined_binary, unwarped_marked, M_inv)
#     plt.imshow(composite_img)
# #     plt.savefig(r'../output_images/' + filename)
#     plt.show()


import pytesseract
import sys
import argparse
import requests
from bs4 import BeautifulSoup
import datetime
import random
import os
try:
    import Image
except ImportError:
    from PIL import Image
from subprocess import check_output

def resolve(path):
    print("Resampling the Image")
    samples = [100,200,300,400,500,600]
    chave = str(random.getrandbits(128))+'.jpeg'
    ranking = {}
    for sample in samples:
        check_output(['convert', path, '-resample', str(sample), chave])
        valor = pytesseract.image_to_string(Image.open(chave), lang='eng', config='--oem 3').replace(' ','')
        if(valor not in ranking):
            ranking[valor] = 0
        ranking[valor] = ranking[valor] + 1
    valores = sorted(ranking.items(), key = lambda kv:(kv[1], kv[0]), reverse=True)
    print(valores)
    #os.remove(chave)
    return valores[0][0]

if __name__=="__main__":
    '''argparser = argparse.ArgumentParser()
    argparser.add_argument('path',help = 'Captcha file path')
    argparser.add_argument('sample',help = 'Size image')
    '''
    s = requests.session()
    url = "http://www.isbn.bn.br/website/consulta/cadastro"
    h = {
        'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'pt-BR,pt;q=0.8,en-US;q=0.5,en;q=0.3',
        'Accept-Encoding': 'gzip, deflate',
        'Referer': 'http://www.isbn.bn.br/website/consulta/cadastro/filtrar',
        'Connection': 'keep-alive'
    }
    req = s.get(url)
    jsessionid = s.cookies.get_dict()['JSESSIONID']

    h = {
        'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0',
        'Accept': 'image/webp,*/*',
        'Accept-Language': 'pt-BR,pt;q=0.8,en-US;q=0.5,en;q=0.3',
        'Accept-Encoding': 'gzip, deflate',
        'Referer': 'http://www.isbn.bn.br/website/consulta/cadastro/filtrar',
        'Connection': 'keep-alive',
        'Cookie': 'JSESSIONID='+jsessionid
    }
    tentativa = 0
    while(tentativa < 10):
        data = datetime_object = datetime.datetime.now()
        tempo = int(datetime.datetime.timestamp(data)*1000)
        caminho = 'http://www.isbn.bn.br/website/jcaptcha?'+str(tempo)
        r = s.get(caminho, headers=h)
        open('imagem.jpeg', 'wb').write(r.content)

        #args = argparser.parse_args()
        path = 'imagem.jpeg'
        print('Resolving Captcha')
        captcha_text = resolve(path).replace(' ','')
        print('Extracted Text',captcha_text)

        h = {
            'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'pt-BR,pt;q=0.8,en-US;q=0.5,en;q=0.3',
            'Accept-Encoding': 'gzip, deflate',
            'Referer': 'http://www.isbn.bn.br/website/consulta/cadastro/filtrar',
            'Connection': 'keep-alive',
            'Cookie': 'JSESSIONID='+jsessionid
        }
        url = 'http://www.isbn.bn.br/website/consulta/cadastro/filtrar'
        isbn = '9788576051268'
        #isbn = '9788573289206'
        req = s.post(url, data={'campo':'1','valor':isbn,'imagemCaptcha':captcha_text},headers=h)
        soup = BeautifulSoup(req.text, 'html.parser')
        book = soup.find(name='span',attrs={"id":'imagemCaptcha.errors'})
        if(book == None):
            tentativa = 10
            div = soup.find(name='div',attrs={"class":'conteudo'})
            divs = div.findChildren("div", recursive=False)
            for d in divs:
                if(d.findChildren(name='strong', recursive=False)):
                    texto = d.text.strip().replace('Participações ','').replace('ISBN ','').replace('Título ','').replace('Edição ','').replace('Tipo de Suporte ','').replace('Páginas ','').replace('Editor(a) ','').replace('Editor(a) ','').strip()
                    if('Participações' in texto):
                        print(texto)
                        print(texto.split('\n'))
                        linhas = texto.split('\n')
                        autores = ''
                        for linha in linhas:
                            if('( Autor)' in linha):
                                autor = linha.replace('\r','').replace('( Autor)','').strip()
                                autores = autores + ';' + autor
                        print(autores[1:])
        tentativa = tentativa + 1

    print(book)


from collections import namedtuple

Category = namedtuple("Category", ["id", "name", "parent"])

class Expense(object):

    def __init__(self, id, user_id, year, month, day, week, description, category, cost):
        self.id = id
        self.user_id = user_id
        self.year = year
        self.month = month
        self.day = day
        self.week = week
        self.description = description
        self.category = category
        self.cost = cost

    def asList(self):
        return [
            self.id,
            "%d-%d-%d" % (self.year, self.month, self.day),
            self.description,
            self.cost,
            self.category.parent,
            self.category.name
        ]


from flask import Blueprint, jsonify

api = Blueprint("api",__name__)

data = [
    {
        "id" : 1,
        "name" : "stone"

    },
    {
        "id" : 2,
        "name" : "fish"
    },
    {
        "id" : 3,
        "name" : "vegetables"
    },
    {
        "id" : 4,
        "name" : "beans"
    },
    {
        "id" : 5,
        "name" : "air-purifier"
    }
]

@api.route("/blogs/")
def blog_list():
    return jsonify([
        {
            "name":"anies",
            "position":20
        },

        {
            "name":"fioiyobong",
            "age":2
        },
        {
            "school":"aptech",
            "address":"worldwide"
        },
        {
            "gender":"female",
            "blood-type":"AB+"
        }
    ])

@api.get("/blogs/<int:id>")
def blog_retrieve(id):
    # fun function that filter a list of dictionary
    return "blog with id "+str(id)


import pytest
from pytest import CaptureFixture

from data_structures.binary_tree import AVLTree

class TestBalancedTree:

  @pytest.fixture
  def avl_tree(self) -> AVLTree:
    avl_tree = AVLTree()
    avl_tree.insert(16)
    avl_tree.insert(8)
    avl_tree.insert(20)
    avl_tree.insert(4)
    avl_tree.insert(12)
    avl_tree.insert(10)
    avl_tree.insert(18)
    return avl_tree

  @pytest.fixture
  def avl_tree2(self) -> AVLTree:
    avl_tree = AVLTree()
    avl_tree.insert(16)
    avl_tree.insert(8)
    avl_tree.insert(20)
    avl_tree.insert(4)
    avl_tree.insert(12)
    avl_tree.insert(10)
    avl_tree.insert(18)
    return avl_tree

  @pytest.fixture
  def avl_tree3(self) -> AVLTree:
    avl_tree = AVLTree()
    avl_tree.insert(15)
    avl_tree.insert(8)
    avl_tree.insert(20)
    avl_tree.insert(4)
    avl_tree.insert(12)
    avl_tree.insert(10)
    avl_tree.insert(18)
    return avl_tree

  @pytest.fixture
  def new_tree(self) -> AVLTree:
    return AVLTree()

  def format_print(self, avl_tree_values: list[int]):
    strings: list[str] = []
    for value in avl_tree_values:
      strings.append(str(value))
      strings.append("\n")

    return "".join(strings)

  def test_avl_tree_insert(self, avl_tree: AVLTree):
    avl_tree.insert(1)

  def test_avl_tree_bfs(self, avl_tree: AVLTree, new_tree: AVLTree,
                        capsys: CaptureFixture[str]):
    avl_tree.bfs()
    captured = capsys.readouterr()
    # bfs = [16, 8, 20, 4, 12, 18, 10]
    bfs = [12, 8, 18, 4, 10, 16, 20]
    assert captured.out == self.format_print(bfs)

  def test_avl_tree_pre_order_dfs(self, avl_tree: AVLTree,
                                  capsys: CaptureFixture[str]):
    avl_tree.pre_order_dfs()
    captured = capsys.readouterr()
    # pre_order_dfs = [16, 8, 4, 12, 10, 20, 18]
    pre_order_dfs = [12, 8, 4, 10, 18, 16, 20]
    assert captured.out == self.format_print(pre_order_dfs)

  def test_avl_tree_in_order_dfs(self, avl_tree: AVLTree,
                                 capsys: CaptureFixture[str]):
    avl_tree.in_order_dfs()
    captured = capsys.readouterr()
    # in_order_dfs = [4, 8, 10, 12, 16, 18, 20]
    in_order_dfs = [4, 8, 10, 12, 16, 18, 20]
    assert captured.out == self.format_print(in_order_dfs)

  def test_avl_tree_post_order_dfs(self, avl_tree: AVLTree,
                                   capsys: CaptureFixture[str]):
    avl_tree.post_order_dfs()
    captured = capsys.readouterr()
    # post_order_dfs = [4, 10, 12, 8, 18, 20, 16]
    post_order_dfs = [4, 10, 8, 16, 20, 18, 12]
    assert captured.out == self.format_print(post_order_dfs)

  def test_empty_avl_tree_traversal(self, new_tree: AVLTree,
                                    capsys: CaptureFixture[str]):
    new_tree.bfs()
    new_tree.pre_order_dfs()
    new_tree.in_order_dfs()
    new_tree.post_order_dfs()
    captured = capsys.readouterr()
    assert captured.out == ""

  def test_avl_tree_equality(self, avl_tree: AVLTree, avl_tree2: AVLTree):
    assert avl_tree == avl_tree2

  def test_avl_tree_inequality(self, avl_tree: AVLTree, avl_tree3: AVLTree):
    assert avl_tree != avl_tree3

  def test_avl_tree_leaf(self, avl_tree: AVLTree):
    node = avl_tree.root
    while node and node.left:
      assert avl_tree.is_leaf(node) == False
      node = node.left
    assert avl_tree.is_leaf(node)

  def test_avl_tree_height(self, new_tree: AVLTree):

    height = -1
    assert new_tree.height() == height
    for i in range(16):
      new_tree.insert(i)
      if i in [0, 1, 3, 7, 15]:
        height += 1
        assert new_tree.height() == height

    node = new_tree.root
    assert new_tree.subtree_height_difference(node) == 1
    while node:
      node = node.left
    assert new_tree.subtree_height_difference(node) == 0

  def test_min_value(self, new_tree: AVLTree):
    assert new_tree.min_value() == float("inf")
    for i in range(16, -1, -1):
      new_tree.insert(i)
      assert new_tree.min_value() == i

  def test_find_value(self, new_tree: AVLTree):
    for i in range(16):
      assert new_tree.find(i) == False
      new_tree.insert(i)
      assert new_tree.find(i)

  def test_avl_tree_string_and_repr(self, new_tree: AVLTree):
    new_tree.insert(2)
    new_tree.insert(1)
    new_tree.insert(3)
    new_tree.insert(4)

    assert f"{new_tree.root}" == "2"
    assert f"{new_tree.root!r}" == "AVLNode(value = 2, left = AVLNode(value = 1), right = AVLNode(value = 3, right = AVLNode(value = 4)))"

if __name__ == "__main__":
  pytest.main([__file__])


import requests

def find_asn(env) :
    f = open(f'/archive/_env/{env}', 'r')

    for _asn in f.readlines() :
        try :
            asn = _asn.strip()
            url_prefix = f"https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS{asn}"
            url_path = f"https://stat.ripe.net/data/as-path-length/data.json?resource=AS{asn}&sort_by=geo"

            data = requests.get( url_prefix )
            prefix_num = len(data.json()['data']['prefixes'])

            data = requests.get( url_path )
            avg = t = 0
            for ele in data.json()['data']['stats'] :
                avg += float(ele['stripped']['avg'])
                t += 1

            length = avg/t
            result = open(f'/result/asn-info-{env}.txt', "a+")
            result.write( f"{asn} {prefix_num} {length}\n")
            result.close()

        except Exception as e:
            missing = open(f'/result/missing.txt', 'a+')
            missing.write(f"{asn} {str(e)}\n")
            missing.close()

    f.close()


import bpy

##################################################################

path_tox3d='C:/Users/xxx/Documents/blender/'
path_toimage='C:/Users/xxx/Documents/blender/'
filename_ofx3d='projectname_'
filename_ofimage='newimage_'
startframe_ofx3d=0
endframe_ofx3d=10

##################################################################
#---------------------------------------------------------------#
##  Delete everything from scene before importing              ##

#bpy.ops.object.select_all(action='TOGGLE')
#bpy.ops.object.delete()

#---------------------------------------------------------------#
##  Add a text                                                  #
#bpy.ops.object.text_add()
#bpy.ops.object.editmode_toggle()
#bpy.ops.font.delete()
#bpy.ops.font.text_insert(text="Frame: ")
#bpy.ops.object.editmode_toggle()
#bpy.ops.object.select_all(action='TOGGLE')
#bpy.data.objects['Text'].name="Frametext"

#---------------------------------------------------------------#
#bpy.context.scene.frame_set(currentframe)
bpy.ops.object.add()
bpy.data.objects['Empty'].select=True
bpy.data.objects['Empty'].name="Aux_Empty"
#bpy.ops.anim.keyframe_insert(type='Location', confirm_success=True)
bpy.context.active_object.location =(0.0, 0.0, 0.0)
bpy.data.objects['Aux_Empty'].select=False

for currentframe in range(startframe_ofx3d,endframe_ofx3d):
    bpy.context.scene.frame_set(currentframe)
    #bpy.ops.object.add()
    bpy.data.objects['Aux_Empty'].select=True
    bpy.context.scene.objects.active = bpy.data.objects['Aux_Empty']
    #bpy.data.objects['Empty'].name="MyEmpty"
    obj = bpy.context.object
    obj.location[2] = 0.0
    #obj.keyframe_insert()
    obj.keyframe_insert(data_path='location')
    #bpy.context.active_object.location =(0.0, 0.0, 0.0)
    bpy.data.objects['Aux_Empty'].select=False

    #---------------------------------------------------------------#
    ## import the x3d file
    bpy.ops.import_scene.x3d(filepath=path_tox3d+filename_ofx3d+str(currentframe)+'.x3d', filter_glob="*.x3d;*.wrl", axis_forward='Z', axis_up='Y')
    #---------------------------------------------------------------#
    ## add the frame number to the text
    #bpy.data.objects["Frametext"].select=True
    #bpy.ops.object.editmode_toggle()
    #bpy.ops.font.delete()
    #bpy.ops.font.text_insert(text="Frame: "+str(currentframe))
    #bpy.ops.object.editmode_toggle()
    #bpy.data.objects["Frametext"].select=False

    #---------------------------------------------------------------#
    ## rename the geometry to "imported geometry"
    bpy.data.objects['ShapeIndexedFaceSet'].name="imported_geometry"
    bpy.data.objects['imported_geometry'].select=True
    bpy.context.scene.objects.active = bpy.data.objects['imported_geometry']
    bpy.context.active_object.material_slots[0].material.use_vertex_color_paint=True
    #bpy.ops.anim.keyframe_insert(type='Location', confirm_success=True)
    bpy.data.objects['imported_geometry'].select=False
    ## rename the default lamp to imported_lamp and delete right away
    bpy.data.objects['TODO'].name="imported_lamp"
    bpy.data.objects['imported_lamp'].select=True
    bpy.ops.object.delete()
    ## rename the default camera to imported_camera and delete right away
    bpy.data.objects['Viewpoint'].name="imported_camera"
    bpy.data.objects['imported_camera'].select=True
    bpy.ops.object.delete()

    #bpy.context.active_object.location = position

    #for object_name in candidate_list:
    #      bpy.data.objects[object_name].select = True
    #     bpy.context.active_object.location = position
    #  bpy.ops.anim.keyframe_insert(type='Location', confirm_success=True)

    bpy.data.scenes['Scene'].render.resolution_percentage=100

    bpy.ops.render.render(write_still=True)

    bpy.data.images['Render Result'].file_format='PNG'
    bpy.data.images['Render Result'].save_render(filepath=path_toimage+filename_ofimage+str(currentframe)+'.png')
    bpy.data.objects['imported_geometry'].select=True
    bpy.ops.object.delete()

bpy.data.objects['Aux_Empty'].select=True
bpy.ops.object.delete()


from datetime import datetime

from django.conf import settings

import pytest
from freezegun import freeze_time

from olympia import amo
from olympia.amo.tests import addon_factory, user_factory, version_factory
from olympia.constants.promoted import LINE, NOT_PROMOTED, NOTABLE
from olympia.promoted.models import PromotedAddon
from olympia.reviewers.models import UsageTier
from olympia.versions.utils import get_staggered_review_due_date_generator
from olympia.zadmin.models import set_config

from ..tasks import NOTABLE_TIER_SLUG, add_high_adu_extensions_to_notable

@pytest.mark.django_db
def test_add_high_adu_extensions_to_notable_tier_absent_or_no_threshold():
    user_factory(pk=settings.TASK_USER_ID)
    set_config(amo.EXTRA_REVIEW_TARGET_PER_DAY_CONFIG_KEY, 999)

    extension_with_high_adu = addon_factory(
        average_daily_users=42, file_kw={'is_signed': True}
    )

    add_high_adu_extensions_to_notable()

    assert (
        extension_with_high_adu.reload().promoted_group(currently_approved=False)
        == NOT_PROMOTED
    )

    UsageTier.objects.create(slug=NOTABLE_TIER_SLUG, lower_adu_threshold=None)

    add_high_adu_extensions_to_notable()

    assert (
        extension_with_high_adu.reload().promoted_group(currently_approved=False)
        == NOT_PROMOTED
    )

@pytest.mark.django_db
def test_add_high_adu_extensions_to_notable():
    user_factory(pk=settings.TASK_USER_ID)
    # Arbitrary_lower_adu_threshold
    lower_adu_threshold = 1234
    UsageTier.objects.create(
        slug=NOTABLE_TIER_SLUG, lower_adu_threshold=lower_adu_threshold
    )
    # arbitrary target per day
    target_per_day = 12
    set_config(amo.EXTRA_REVIEW_TARGET_PER_DAY_CONFIG_KEY, target_per_day)

    extension_with_low_adu = addon_factory(
        average_daily_users=lower_adu_threshold - 1, file_kw={'is_signed': True}
    )
    extension_with_high_adu = addon_factory(
        average_daily_users=lower_adu_threshold, file_kw={'is_signed': True}
    )
    ignored_theme = addon_factory(
        average_daily_users=lower_adu_threshold + 1, type=amo.ADDON_STATICTHEME
    )
    already_promoted = addon_factory(
        average_daily_users=lower_adu_threshold + 1, file_kw={'is_signed': True}
    )
    PromotedAddon.objects.create(addon=already_promoted, group_id=LINE.id)
    promoted_record_exists = addon_factory(
        average_daily_users=lower_adu_threshold + 1, file_kw={'is_signed': True}
    )
    PromotedAddon.objects.create(addon=promoted_record_exists, group_id=NOT_PROMOTED.id)
    unlisted_only_extension = addon_factory(
        average_daily_users=lower_adu_threshold + 1,
        version_kw={'channel': amo.CHANNEL_UNLISTED},
        file_kw={'is_signed': True},
    )
    mixed_extension = addon_factory(
        average_daily_users=lower_adu_threshold + 1, file_kw={'is_signed': True}
    )
    mixed_extension_listed_version = mixed_extension.current_version
    mixed_extension_listed_version.delete()
    mixed_extension_unlisted_version = version_factory(
        addon=mixed_extension, channel=amo.CHANNEL_UNLISTED, file_kw={'is_signed': True}
    )
    deleted_extension = addon_factory(
        average_daily_users=lower_adu_threshold + 1, file_kw={'is_signed': True}
    )
    deleted_extension_version = deleted_extension.current_version
    deleted_extension.delete()

    with freeze_time():
        now = datetime.now()
        add_high_adu_extensions_to_notable()

    assert (
        extension_with_low_adu.reload().promoted_group(currently_approved=False)
        == NOT_PROMOTED
    )
    assert (
        extension_with_high_adu.reload().promoted_group(currently_approved=False)
        == NOTABLE
    )
    assert (
        ignored_theme.reload().promoted_group(currently_approved=False) == NOT_PROMOTED
    )
    already_promoted.reload().promotedaddon.reload()
    assert already_promoted.promoted_group(currently_approved=False) == LINE
    promoted_record_exists.reload().promotedaddon.reload()
    assert promoted_record_exists.promoted_group(currently_approved=False) == NOTABLE
    assert unlisted_only_extension.promoted_group(currently_approved=False) == NOTABLE
    assert mixed_extension.promoted_group(currently_approved=False) == NOTABLE
    assert deleted_extension.promoted_group(currently_approved=False) == NOTABLE

    generator = get_staggered_review_due_date_generator(starting=now)

    assert extension_with_high_adu.current_version.needshumanreview_set.filter(
        is_active=True
    ).exists()
    assert extension_with_high_adu.current_version.due_date == next(generator)
    assert promoted_record_exists.current_version.needshumanreview_set.filter(
        is_active=True
    ).exists()
    assert promoted_record_exists.current_version.due_date == next(generator)
    unlisted_latest_version = unlisted_only_extension.find_latest_version(channel=None)
    assert unlisted_latest_version.needshumanreview_set.filter(is_active=True).exists()
    assert unlisted_latest_version.due_date == next(generator)
    assert mixed_extension_unlisted_version.needshumanreview_set.filter(
        is_active=True
    ).exists()
    assert mixed_extension_unlisted_version.reload().due_date == next(generator)
    assert mixed_extension_listed_version.needshumanreview_set.filter(
        is_active=True
    ).exists()
    # same as due due is per addon
    assert (
        mixed_extension_listed_version.reload().due_date
        == mixed_extension_unlisted_version.due_date
    )
    assert deleted_extension_version.needshumanreview_set.filter(
        is_active=True
    ).exists()
    assert deleted_extension_version.reload().due_date == next(generator)


"""
template for generating data to fool learners (c) 2016 Tucker Balch
"""
import pandas as pd
import numpy as np
import math
import matplotlib.pyplot as plt

# this function should return a dataset (X and Y) that will work
# better for linear regresstion than random trees
def best4LinReg(seed=1489683273):
    np.random.seed(seed)
    X = np.random.standard_normal(size=(1000, 2))
    Z = X.sum(axis=1)
    size = X.shape[0]
    Y = Z + np.random.normal(size=size)
    # 1X = np.mgrid[-5:5:0.5,-5:5:0.5].reshape(2,-1).T
    # Y = X[:,0]*X[:,1] + np.random.normal(size = X.shape[0])
    return X, Y

def best4RT(seed=1489683273):
    np.random.seed(seed)
    mu, sigma = 10, 140
    # X1 = np.random.normal(mu, sigma, size=(500,20))
    # mu, sigma = 0, 120
    # X2 = np.random.normal(mu, sigma, size = (500, 20))
    # X = np.concatenate((X1,X2), axis=0)
    # X = np.random.rand(1000,20)
    #X1 = np.random.uniform(0, 10, 1000)
    X1 = np.random.normal(25, 25, 5000)
    X2 = np.zeros(X1.shape[0])

    for i in range(len(X1)):
        if X1[i] > 25:
            X2[i] = np.random.normal(-100, 5, 1)
        else:
            X2[i] = np.random.normal(100, 5,  1)

    X = np.column_stack((X1, X2))
    Y = np.zeros(X1.shape[0])

    for i in range(len(X1)):
        if X1[i] > 25:
            if X2[i] > -100:
                Y[i] = np.random.normal(10, 1, 1)
            else:
                Y[i] = np.random.normal(5, 1, 1)
        else:
            if X2[i]> -100:
                Y[i] = np.random.normal(5, 1, 1)
            else:
                Y[i] = np.random.normal(10, 1, 1)

    # Y = np.random.rand(X.shape[0])

    # Y= np.random.standard_normal(X1.shape[0] + X2.shape[0])
    # Y = np.random.normal(mu, sigma, X1.shape[0] + X2.shape[0])
    # X = np.random.normal(size = (50, 2))
    # Y = 0.8 * X[:,0] + 5.0 * X[:,1]
    return X, Y

if __name__ == "__main__":
    X1, Y1 = best4LinReg(seed=5)
    X2, Y2 = best4RT(seed=5)
    plt.plot(X1, Y1)
    plt.title('Best4LinReg')
    # plt.show()

    plt.plot(X2, Y2)
    plt.title('Best4RandomTree')
    plt.show()
    # print "they call me Tim."


import logging
import os
from typing import Callable, Dict, List, Tuple
from unittest.mock import Mock, patch

from prometheus_client.core import REGISTRY

from pelorus import AbstractPelorusExporter, utils

def get_number_of_logs(
    log_record_tuples: List[Tuple[str, int, str]], level: int
) -> int:
    return len([record for record in log_record_tuples if record[1] == level])

def get_number_of_error_logs(log_record_tuples: List[Tuple[str, int, str]]) -> int:
    return get_number_of_logs(log_record_tuples, level=logging.ERROR)

def get_number_of_info_logs(log_record_tuples: List[Tuple[str, int, str]]) -> int:
    return get_number_of_logs(log_record_tuples, level=logging.INFO)

def run_prometheus_register(collector: AbstractPelorusExporter) -> None:
    try:
        REGISTRY.register(collector)
    finally:
        REGISTRY.unregister(collector)

class MockExporter:
    def __init__(
        self, set_up: Callable[[], AbstractPelorusExporter], mock_kube_client=Mock()
    ) -> None:
        self.set_up = set_up
        self.mock_kube_client = mock_kube_client

    def run_app(self, arguments: Dict[str, str]) -> AbstractPelorusExporter:
        """Run set up of exporter app with desired environment variables."""
        try:
            collector = None
            logging.getLogger().disabled = False
            for key, value in arguments.items():
                os.environ[key] = value
            with patch.object(utils, "get_k8s_client") as mock_kube_client:
                mock_kube_client.return_value.resources.get.side_effect = (
                    self.mock_kube_client
                )
                collector = self.set_up(prod=False)
            return collector
        finally:
            for key in arguments:
                del os.environ[key]
            if collector:
                REGISTRY.unregister(collector)
            logging.getLogger().disabled = True


import glob
import os
import sys

import process_labels
from PIL import Image
import numpy as np
import concurrent.futures

from pathlib import Path

# convert labels in PRETTY colours (or PRETTY_FILMIC...) to greyscale 8 bits for mmseg

def to_greyscale_labels(png_file, out_folder):

    print (png_file)
    output_path = os.path.join(out_folder, os.path.basename(Path(png_file).name) )
    if os.path.exists(output_path) and os.path.getsize(output_path) > 0:
        return # already done, skip

    pretty = Image.open(png_file, "r")
    label = np.asarray ( pretty )[:,:,0:3]

    tol = 10

    pretty_map = process_labels.colours_for_mode(process_labels.PRETTY)
    output = np.zeros(pretty.size, dtype=int)

    for i, label_name in enumerate (process_labels.LABEL_SEQ_NO_DOOR):

        colour = np.array ( pretty_map[label_name] )
        equality = np.logical_and ( np.greater(label, colour-tol), np.less(label, colour+tol) )
        class_map = np.all(equality, axis=-1)
#        print (f"{label_name} - {colour} :: {class_map.sum()}")
        output = output * (1- class_map) # zero out any previous labels
        output = output + class_map * i  # set greyscale label

    print ("saving to %s"%output_path)
    Image.fromarray(np.uint8(output)).save( output_path )

def to_color_labels(png_file, out_folder):

    print(png_file)
    output_path = os.path.join(out_folder, os.path.basename(Path(png_file).name))
    if os.path.exists(output_path) and os.path.getsize(output_path) > 0:
        return  # already done, skip

    grey = Image.open(png_file, "r")
    label = np.asarray(grey)#[:, :, 0:3]
    color_seg = np.zeros((grey.size[0], grey.size[1], 3), dtype=np.uint8)

    for i, l_name in enumerate ( process_labels.LABEL_SEQ_NO_DOOR ):
        color = process_labels.colours_for_mode(process_labels.PRETTY)[l_name]
        color_seg[i == label, :] = color

    print("saving to %s" % output_path)
    Image.fromarray(color_seg).save(output_path)
    # Image.fromarray(np.uint8(color_seg)).save(output_path)

if __name__ == "__main__":

    _pool = concurrent.futures.ThreadPoolExecutor(max_workers=16)

    labels = []

    labels.extend(glob.glob(os.path.join( sys.argv[1], "labels", "*.png")))

    if True: # rgb to greyscale
        out_dir = os.path.join(sys.argv[1], "labels_8bit")
        fn = to_greyscale_labels
    else: # greyscale to rgb
        fn = to_color_labels
        out_dir = os.path.join(sys.argv[1], "labels_pretty")

    os.makedirs(out_dir, exist_ok=True)

    for lab in labels:
        fn (lab, out_dir )
        _pool.submit ( fn, lab, out_dir, )


import openai
import tiktoken
import numpy as np
import os
import time
import logging
import re

import utils.langchain_helpers.simple_prompt

from utils import openai_helpers
from utils import redis_helpers
from utils import helpers

from langchain.prompts.chat import (
    ChatPromptTemplate,
    HumanMessagePromptTemplate,
    MessagesPlaceholder,
    SystemMessagePromptTemplate,
)

from utils.env_vars import *

system_message = "The assistant is a super helpful assistant that plays the role of a linguistic professor and has ultra high attention to details."

instruction = """From the above Question and Current Conversation, output search keywords to use in a search engine to get an answer for the Question. If the Question is not related to the Current Conversation, then do not use the Current Conversation when generating the Search Keywords.
Search Keywords:"""

body = """
Current Conversation:
{history}

Question: {question}
"""

context_prompt = """
<|im_start|>
{system_message}
<|im_end|>
<|im_start|>user

Current Conversation:
{history}

Question: {question}

{instruction}
<|im_end|>
<|im_start|>assistant
"""

class OldSchoolSearch():

    def search(self, query, history, pre_context, filter_param=None,  enable_unified_search=False,
                lc_agent = None, enable_cognitive_search=False, evaluate_step=True,
                topK=NUM_TOP_MATCHES, stream = False, verbose = False):

        redis_conn = redis_helpers.get_new_conn()

        completion_model = CHOSEN_COMP_MODEL
        embedding_model = CHOSEN_EMB_MODEL
        completion_enc = openai_helpers.get_encoder(completion_model)
        embedding_enc = openai_helpers.get_encoder(embedding_model)

        if verbose: print("Old Query: ", query)
        gen = openai_helpers.get_generation(completion_model)

        if history != '':

            if (gen == 4) or (gen == 3.5):
                messages = [
                    SystemMessagePromptTemplate.from_template(system_message).format(),
                    HumanMessagePromptTemplate.from_template(body).format(history=history, question=query),
                    HumanMessagePromptTemplate.from_template(instruction).format(),
                ]
                messages = openai_helpers.convert_messages_to_roles(messages)
                query = openai_helpers.contact_openai(messages)
            else:
                prompt = context_prompt.format(system_message=system_message,
                                                history=history,
                                                question=query,
                                                instruction=instruction)
                query = openai_helpers.contact_openai(prompt)

        if (gen == 4) or (gen == 3.5):
            p = ''
            for m in utils.langchain_helpers.simple_prompt.get_simple_prompt('', '', '', ''): p += m['content']
            empty_prompt_length = len(completion_enc.encode(p))
        else:
            empty_prompt_length = len(completion_enc.encode(utils.langchain_helpers.simple_prompt.get_simple_prompt('', '', '', '')))

        if verbose: print("New Query: ", query)

        max_comp_model_tokens = openai_helpers.get_model_max_tokens(completion_model)
        max_emb_model_tokens = openai_helpers.get_model_max_tokens(embedding_model)

        if lc_agent.enable_unified_search:
            context = lc_agent.unified_search(query)
        elif enable_cognitive_search:
            context = lc_agent.agent_cog_search(query)
        # elif lc_agent.use_bing:
        #     context = lc_agent.agent_bing_search(query)
        else:
            context = lc_agent.agent_redis_search(query)

        query   = completion_enc.decode(completion_enc.encode(query)[:MAX_QUERY_TOKENS])
        history = completion_enc.decode(completion_enc.encode(history)[:MAX_HISTORY_TOKENS])
        pre_context = completion_enc.decode(completion_enc.encode(pre_context)[:PRE_CONTEXT])

        context_length      = len(completion_enc.encode(context))
        query_length        = len(completion_enc.encode(query))
        history_length      = len(completion_enc.encode(history))
        pre_context_length  = len(completion_enc.encode(pre_context))

        max_context_len = max_comp_model_tokens - query_length - MAX_OUTPUT_TOKENS - empty_prompt_length - history_length - pre_context_length - 1

        context = completion_enc.decode(completion_enc.encode(context)[:max_context_len])

        prompt = utils.langchain_helpers.simple_prompt.get_simple_prompt(context, query, history, pre_context)

        if verbose:
            print("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$")
            print(prompt)
            print("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$")

        if verbose: print("OSS OAI Call")
        answer = openai_helpers.contact_openai(prompt, completion_model, MAX_OUTPUT_TOKENS, stream=stream, verbose=verbose)

        return answer


import pandas as pd

import os
import pickle

from nltk import bigrams
from nltk.stem.lancaster import LancasterStemmer
from nltk import pos_tag

"""
Implementation Notes
This script finds the names of the award winners for a particular ceremony.
It does so by extracting tweets with the action verb "wins" to limit the
scope of the tweets that have to be searched through to those that are most
relevant to the context. Afterwards, bigrams are generated from the text of
these tweets. The bigrams contain tuples like ('best', 'actor') or ('Emma', 'Stone').
These bigrams are then passed to a Parts-of-Speeech tagger. The POS tagger will
tag bigrams with names of individuals with the 'PERSON' tag and generate a
new tuple like ('Emma', 'Person'). The tagged bigrams are then filtered to exclude
any that don't contain 'PERSON' as the tag for both words in the bigram and
the extracted words are treated as the names of the winner and printed out.

Note that this implementation does take a while to run. The bulk of the compute
time is spent tagging the bigrams using the Stanford POS Tagger. To elleviate this,
after the initial run the list of named bigrams is pickled to a file.
"""

PREPROCESSED_DATA_FILE = 'data_preprocessed.p'
PREPROCESSED_DF = pd.read_pickle(PREPROCESSED_DATA_FILE)
LAST_NAMES = open("surnames.txt").read().splitlines()
FIRST_NAMES = open("firstnames.txt").read().splitlines()
FIRST_NAMES = list(map(str.strip, FIRST_NAMES))

def filter_dataframe():
    """
    Extract the tweets that contain verbs relevant to winning.
    """
    df_remove_null = PREPROCESSED_DF.dropna()
    return df_remove_null[df_remove_null['tokens'].str.contains("congratulations")]

def extract_names(bigrams):
    """
    Tag each of the bigram tuples with the appropriate Part of Speech.
    """
    named_bigrams = []
    NUM_BIGRAMS = len(bigrams)
    stemmer = LancasterStemmer()
    for index, bigram in enumerate(bigrams):
        if bigram[0].upper() in FIRST_NAMES:
            person = " ".join(bigrams[index])
            # If a winner is already on the list, don't add them
            if person not in named_bigrams:
                named_bigrams.append(person.title())

    return named_bigrams

def get_bigrams(tweets):
    """
    Generate bigrams for each tweet.
    """
    split_tweets = [tweet.split(" ") for tweet in tweets]
    tweet_bigrams = []
    for tweet in split_tweets:
        tweet_bigrams.extend(list(bigrams(tweet)))
    return list(set(tweet_bigrams))

def find_award_winners():
    """
    Main function that runs through the process of filtering the dataframe,
    extracting bigrams, tagging the bigrams using POS, and then filtering the
    tags that are labelled 'PERSON'.
    """
    df = filter_dataframe()
    bigrams = get_bigrams(df['tokens'].tolist())

    named_bigrams = extract_names(bigrams)

    print("Award Winners")
    # Remove duplicate names in winners list
    for index, name in enumerate(named_bigrams):
        print(index + 1, name)

if __name__ == '__main__':
    find_award_winners()


"""Tasks done in background by Rq.

This file must NOT be imported by app.py, or by any import thereof, because it will cause
a cyclic import.

The functions within are referred with qualified "Python paths"
(eg., "flask_app.modeling.tasks.do_classifier_related_task"). Rq supports that.
"""
import logging
import typing as T

from flask import url_for

import flask_app
from flask_app import emails
from flask_app.database import models
from flask_app.modeling.classifier import ClassifierModel
from flask_app.modeling.lda import Corpus
from flask_app.modeling.lda import LDAModeler
from flask_app.modeling.queue_manager import ClassifierPredictionTaskArgs
from flask_app.modeling.queue_manager import ClassifierTrainingTaskArgs
from flask_app.modeling.queue_manager import TopicModelTrainingTaskArgs
from flask_app.settings import Settings

logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

@flask_app.app.needs_app_context
def do_classifier_related_task(
    task_args: T.Union[ClassifierTrainingTaskArgs, ClassifierPredictionTaskArgs],
) -> None:
    if task_args["task_type"] == "prediction":
        test_set = models.TestSet.get(models.TestSet.id_ == task_args["test_set_id"])
        assert test_set.inference_began
        assert not test_set.inference_completed

        try:
            classifier_model = ClassifierModel(
                labels=task_args["labels"],
                model_path=task_args["model_path"],
                cache_dir=task_args["cache_dir"],
            )

            classifier_model.predict_and_save_predictions(
                test_set_path=task_args["test_file"],
                content_column=Settings.CONTENT_COL,
                predicted_column=Settings.PREDICTED_LABEL_COL,
                output_file_path=task_args["test_output_file"],
            )
        except BaseException as e:
            logger.critical(f"Error while doing prediction task: {e}")
            test_set.error_encountered = True
        else:
            test_set.inference_completed = True
            emailer = emails.Emailer()
            emailer.send_email(
                email_template_name="classifier_inference_finished",
                to_email=test_set.notify_at_email,
                classifier_name=test_set.classifier.name,
                predictions_url=url_for(
                    "ClassifiersTestSetsPredictions",
                    classifier_id=test_set.classifier.classifier_id,
                    test_set_id=test_set.id_,
                    file_type=Settings.DEFAULT_FILE_FORMAT.strip("."),
                    _method="GET",
                ),
            )
        finally:
            test_set.save()

    elif task_args["task_type"] == "training":
        assert task_args["task_type"] == "training"
        clsf = models.Classifier.get(
            models.Classifier.classifier_id == task_args["classifier_id"]
        )
        assert clsf.train_set is not None
        assert clsf.dev_set is not None

        try:
            classifier_model = ClassifierModel(
                labels=task_args["labels"],
                num_train_epochs=task_args["num_train_epochs"],
                model_path=task_args["model_path"],
                train_file=task_args["train_file"],
                dev_file=task_args["dev_file"],
                cache_dir=task_args["cache_dir"],
                output_dir=task_args["output_dir"],
            )
            metrics = classifier_model.train_and_evaluate()
        except BaseException as e:
            logger.critical(f"Error while doing classifier training task: {e}")
            clsf.train_set.error_encountered = True
            clsf.dev_set.error_encountered = True
        else:
            clsf.train_set.training_or_inference_completed = True
            clsf.dev_set.training_or_inference_completed = True
            clsf.dev_set.metrics = models.ClassifierMetrics(**metrics)
            clsf.dev_set.metrics.save()
            emailer = emails.Emailer()
            emailer.send_email(
                email_template_name="classifier_training_finished",
                to_email=clsf.notify_at_email,
                classifier_name=clsf.name,
                metrics=T.cast(T.Dict[str, float], metrics),
            )

        finally:
            clsf.dev_set.save()
            clsf.train_set.save()
            clsf.save()

@flask_app.app.needs_app_context
def do_topic_model_related_task(task_args: TopicModelTrainingTaskArgs) -> None:
    topic_mdl = models.TopicModel.get(
        models.TopicModel.id_ == task_args["topic_model_id"]
    )
    assert topic_mdl.lda_set is not None
    try:
        corpus = Corpus(
            file_name=task_args["training_file"],
            content_column_name=Settings.CONTENT_COL,
            id_column_name=Settings.ID_COL,
        )
        lda_modeler = LDAModeler(
            corpus,
            iterations=task_args["iterations"],
            mallet_bin_directory=task_args["mallet_bin_directory"],
        )
    except BaseException as e:
        logger.critical(f"Error while doing lda training task: {e}")
        topic_mdl.lda_set.error_encountered = True
    else:
        metrics = lda_modeler.model_topics_to_spreadsheet(
            num_topics=topic_mdl.num_topics,
            default_topic_names=topic_mdl.topic_names,
            fname_keywords=task_args["fname_keywords"],
            fname_topics_by_doc=task_args["fname_topics_by_doc"],
        )
        topic_mdl.lda_set.metrics = models.TopicModelMetrics.create(**metrics)
        topic_mdl.lda_set.lda_completed = True
        emailer = emails.Emailer()
        emailer.send_email(
            email_template_name="topic_model_training_finished",
            to_email=topic_mdl.notify_at_email,
            topic_model_name=topic_mdl.name,
            topic_model_preview_url=f"http://{Settings.SERVER_NAME}/topicModelPreviews.html?topic_model_id={topic_mdl.id_}",
            metrics=T.cast(T.Dict[str, T.Union[int, float]], metrics),
        )

    finally:
        topic_mdl.lda_set.save()


import inspect, os
import xml.etree.ElementTree as ET

routeName = "blackdiamond"
filePrefix =  os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) + "/" + routeName # name of input and output file
smallCut = 15 # number of seconds between each reading is small output file
fileTracked = filePrefix + ".tcx"
fullCSV=filePrefix+"Full.csv"
smallCSV=filePrefix+"Small.csv"
fullJS=filePrefix+"Full.js"
smallJS=filePrefix+"Small.js"
newLine = ""

def run():
    removeFile(fullCSV)
    removeFile(smallCSV)
    removeFile(fullJS)
    removeFile(smallJS)

    startJS(fullJS)
    startJS(smallJS)

    tree = ET.parse(fileTracked)
    root = tree.getroot()
    i = 0
    for child in root[0][0][1][5]:
        addToCSV(child, fullCSV)
        addToJS(convertToGoogleMapsJSON(child), fullJS)
        if ((i % smallCut) == 0):
            addToCSV(child, smallCSV)
            addToJS(convertToGoogleMapsJSON(child), smallJS)
        if (i == 0):
            global newLine
            newLine = ",\n"
        i += 1

    endJS(fullJS)
    endJS(smallJS)

def removeFile(fileName):
    try:
        os.remove(fileName)
    except OSError:
        pass

def addToCSV(point, fileVar):
    with open(fileVar, "a") as myFile:
        myFile.write(convertTrackpointToCSV(point))

def addToJS(string, fileVar):
    with open(fileVar, "a") as myFile:
        myFile.write(string)

def startJS(fileVar):
    addToJS(("var path" + routeName + "Coodinates = [\n"), fileVar)

def endJS(fileVar):
    addToJS("\n];", fileVar)

def convertTrackpointToCSV(point):
    return (point[0].text + ", " + point[1][0].text + ", " + point[1][1].text + "\n")

def convertToGoogleMapsJSON(point):
    print (newLine + "{lat: " + point[1][0].text + ", lng: " + point[1][1].text + "}")
    return newLine + "{lat: " + point[1][0].text + ", lng: " + point[1][1].text + "}"

run()


"""
Plot bulk fluxes as a time series, including all chemical tracers
if present.

"""
import matplotlib.pyplot as plt
import numpy as np
import pickle

from lo_tools import Lfun, zfun
from lo_tools import plotting_functions as pfun
import tef_fun
import flux_fun

Ldir = Lfun.Lstart()

in_dir00 = Ldir['LOo'] / 'extract'
gtagex = Lfun.choose_item(in_dir00)
in_dir0 = in_dir00 / gtagex / 'tef'
ext_name = Lfun.choose_item(in_dir0, tag='bulk', exclude_tag='bulk_plots')
in_dir = in_dir0 / ext_name

sect_list = [item.name for item in in_dir.glob('*.p')]

out_dir = in_dir0 / ext_name.replace('bulk', 'bulk_plots')
Lfun.make_dir(out_dir, clean=True)

# =================================
# get the DataFrame of all sections
gridname=gtagex.split('_')[0]
sect_df = tef_fun.get_sect_df(gridname)

testing = False

if testing:
    from importlib import reload
    reload(flux_fun)

if testing:
    sect_list = ['jdf1']
else:
    sect_list = list(sect_df.index)

# PLOTTING
fs = 12
plt.close('all')
pfun.start_plot(fs=fs, figsize=(21,10))

for sect_name in sect_list:

    tef_df, in_sign, dir_str, sdir = flux_fun.get_two_layer(in_dir, sect_name, gridname)
    cols = list(tef_df.columns)

    # Check if we have chemical variables, or just salt
    vn_list = tef_fun.vn_list
    do_chem = True
    for vn in vn_list:
        if vn+'_in' not in cols:
            do_chem = False

    if do_chem:
        vn_list = vn_list = ['Q', 'salt', 'temp', 'NO3', 'phytoplankton', 'detritus', 'oxygen', 'TIC', 'alkalinity']
        vn_list_long = vn_list + ['zooplankton', 'Ldetritus']
        NR = 3; NC = 3 # rows and columns for subplots
    else:
        vn_list = ['Q', 'salt']
        vn_list_long = vn_list
        NR = 2; NC = 1

    # filter more in time (requires that Q be the first item in vn_list)
    nhan = 1 # length of Hanning window in days (use 1 for no filtering)
    for vn in vn_list_long:
        if vn == 'Q':
            tef_df['Q_in'] = zfun.lowpass(tef_df['Qin'].to_numpy(), n=nhan)
            tef_df['Q_out'] = zfun.lowpass(tef_df['Qout'].to_numpy(), n=nhan)
        else:
            tef_df[vn+'_in'] = zfun.lowpass((tef_df[vn+'_in']*tef_df['Qin']).to_numpy(), n=nhan)/tef_df['Q_in']
            tef_df[vn+'_out'] = zfun.lowpass((tef_df[vn+'_out']*tef_df['Qout']).to_numpy(), n=nhan)/tef_df['Q_out']

    # adjust units
    tef_df['Q_in'] = tef_df['Q_in']/1000
    tef_df['Q_out'] = tef_df['Q_out']/1000

    # labels and colors
    ylab_dict = {'Q': r'Transport $[10^{3}\ m^{3}s^{-1}]$',
                'salt': r'Salinity',
                'temp': r'Pot. Temp. $[^{o}C]$',
                'NO3': r'Nitrate $[\mu mol\ N\ L^{-1}]$',
                'phytoplankton': r'phytoplankton $[\mu mol\ N\ L^{-1}]$',
                'detritus': r'detritus $[\mu mol\ N\ L^{-1}]$',
                'oxygen': r'oxygen $[\mu mol\ O_{2}\ L^{-1}]$',
                'TIC': r'TIC $[\mu mol\ C\ L^{-1}]$',
                'alkalinity': r'Alkalinity $[\mu equiv\ L^{-1}]$'}
    import string
    abc = list(string.ascii_lowercase)
    incolor = 'r'
    outcolor = 'b'
    lw = 2

    fig, axes = plt.subplots(nrows=NR, ncols=NC, squeeze=False)

    ii = 0
    for vn in vn_list:
        ir, ic = zfun.get_irc(ii, NC)
        ax = axes[ir, ic]
        tef_df[[vn+'_in',vn+'_out']].plot(ax=ax, legend=False, color=[incolor, outcolor], grid=True, lw=lw)
        ax.set_ylabel(ylab_dict[vn])
        ax.text(.03, .95, '('+abc[ii]+')', va='top', weight='bold', transform=ax.transAxes,
            bbox=dict(facecolor='w', edgecolor='None', alpha=0.5))
        if ir < NR-1:
            ax.set_xticklabels([])
        if vn == 'Q':
            ax.set_title(gtagex + ' : ' + sect_name + ' : Positive is ' + dir_str)
            ax.text(.97, .95, 'Inflow', ha='right', va='top', weight='bold', c=incolor,
                transform=ax.transAxes, size=1.2*fs,
                bbox=dict(facecolor='w', edgecolor='None', alpha=0.5))
            ax.text(.97, .05, 'Outflow', ha='right', va='bottom', weight='bold', c=outcolor,
                transform=ax.transAxes, size=1.2*fs,
                bbox=dict(facecolor='w', edgecolor='None', alpha=0.5))
        if vn == 'phytoplankton':
            tef_df[['zooplankton_in','zooplankton_out']].plot(ax=ax, legend=False, color=[incolor, outcolor],
                grid=True, alpha=.5, lw=lw)
            ax.text(.97, .95, 'Transparent = zooplankton', ha='right', va='top', weight='bold', c='k',
                transform=ax.transAxes, size=fs, alpha=.5,
                bbox=dict(facecolor='w', edgecolor='None', alpha=0.5))
        if vn == 'detritus':
            tef_df[['Ldetritus_in','Ldetritus_out']].plot(ax=ax, legend=False, color=[incolor, outcolor],
                grid=True, alpha=.5, lw=lw)
            ax.text(.97, .95, 'Transparent = Ldetritus', ha='right', va='top', weight='bold', c='k',
                transform=ax.transAxes, size=fs, alpha=.5,
                bbox=dict(facecolor='w', edgecolor='None', alpha=0.5))
        ii += 1

    fig.tight_layout()

    if testing:
        plt.show()
    else:
        plt.savefig(out_dir / (sect_name + '.png'))
        plt.close()

pfun.end_plot()


# ======== IMPORTS =====================================================================================================================
import sys, getopt, cherrypy, json, os, argparse
import importlib.util
from cherrypy.lib import sessions
import logging

# logging.basicConfig(filename="main_logfile.log", filemode="w", format="%(asctime)s %(message)s", level=logging.INFO)
logging.basicConfig(format="%(asctime)s %(message)s", level=logging.INFO)

# ======== GLOBAL VARS =================================================================================================================
SETTINGS_FILE = None
ARGUMENTS = []

# ======== GLOBAL FUNCTIONS ============================================================================================================
def CORS():
    # This sets the response headers , anything you want in the headers to be returned must be done here.
    server_settings = SETTINGS_FILE.get("cherrypy", None) # Load settings for cherrypy
    # cherrypy.response.headers["Access-Control-Allow-Origin"] = "*"
    # The sessions and session validation is not working correctly right now with the custom hosting etc. Need to fix this.
    cherrypy.response.headers["Access-Control-Allow-Origin"] = ARGUMENTS.client_host if ARGUMENTS.client_host else server_settings.get("client_host", "http://127.0.0.1:4200")
    cherrypy.response.headers["Access-Control-Allow-Headers"] = "Content-Type"
    cherrypy.response.headers["Access-Control-Allow-Credentials"] = "true"

class server(object):
    def __init__(self):
        self.module_dict = {}
        self.build_modules()

    def build_modules(self):
        for module in os.listdir(os.getcwd()+"/modules"):
            if module[-3:] == ".py":
                self.module_dict[module[:-3]] = importlib.util.spec_from_file_location(module[:-3], os.getcwd()+"/modules"+"/"+str(module))

    @cherrypy.expose
    def default(self, *args, **kwargs):
        """
        Function: Default function that parses and calls the correct module with the function and parameters. Then parses it and sends the result back to the
                  url call from where it came.
        """
        # Get Module That should be called from the url example http://127.0.0.1/test/getClient -> test will be the module.
        module_ = args[0]
        valid_module = True if self.module_dict.get(module_, None) else False # Check if the module exists

        # Get Function that should be called from the url example http://127.0.0.1/test/getClient -> getClient will be the function.
        func_ = args[1]

        # Get Parameters that was sent with the url request.
        params_ = json.loads(kwargs["parameters"])

        if valid_module:
            try:
                params_["session"] = cherrypy.session.get("user", None)

                # logging.info to know what is happening
                logging.info("*************************** CALLING ****************************")
                logging.info("MODULE: %s; \nFUNCTION: %s; \nPARAMETERS: %s;"%(module_, func_, str(params_)))
                logging.info("****************************************************************")

                # Call the module with the function
                data = self.callModuleFunc(module_, func_, params_)
                result = True
                msg = "Success"
            except Exception as ex:
                logging.info("******************** EXCEPTION OCCURED *************************")
                logging.info("MODULE: %s; \nFUNCTION: %s; \nPARAMETERS: %s; \nEXCEPTION: %s;"%(module_, func_, str(params_), str(ex)))
                logging.info("****************************************************************")
                msg = str(ex) if str(ex) != "'%s' object has no attribute '%s'"%(module_, func_) else "Function %s does not exist on module %s"%(func_, module_)
                data = None
                result = False
        else:
            logging.info("******************** EXCEPTION OCCURED *************************")
            logging.info("EXCEPTION: Module %s was not found"%(module_))
            logging.info("****************************************************************")
            msg = "MODULE " + module_ + " WAS NOT FOUND"
            data = None
            result = False
        ret = {
            "result": result,
            "msg": msg,
            "data": data
        }
        # Can also do the following instead of def CORS() function call in tools.CORS.on
        # cherrypy.response.headers["Access-Control-Allow-Origin"] = "*"
        # cherrypy.response.headers["Access-Control-Allow-Headers"] = "Content-Type"
        return json.dumps(ret)

    def getUser(self):
        if cherrypy.session.get("user", None):
            return cherrypy.session["user"]
        else:
            raise Exception("NOT LOGGED IN")

    def callModule(self, module_name):
        _module_ = importlib.util.module_from_spec(self.module_dict[module_name])
        self.module_dict[module_name].loader.exec_module(_module_)
        return getattr(_module_, module_name)(self)

    def callModuleFunc(self, module_name, func_name, params):
        return getattr(self.callModule(module_name), func_name)(**params)

# ======== INIT FUNCTION =================================================================================================================

if __name__ == "__main__":
    # You can run the server on a specific host and port on startup
    # You can also let the db init run in the init mode.
    # In order to this use the following format : python3 main.py --port=8080 --host=0.0.0.0 --init=True

    parser = argparse.ArgumentParser()
    parser.add_argument("--server_port", type=int)
    parser.add_argument("--server_host")
    parser.add_argument("--client_host")
    parser.add_argument("--init", type=bool)
    ARGUMENTS = parser.parse_args()
    if ARGUMENTS.server_port:
        logging.info("---------- Starting with PORT: %s ---------"%(ARGUMENTS.server_port))
    if ARGUMENTS.server_host:
        logging.info("---------- Starting with HOST: %s ---------"%(ARGUMENTS.server_host))
    # Load settings out of the settings file
    try:
        with open("settings.json") as settings_file:
            SETTINGS_FILE = json.load(settings_file)
    except:
        raise Exception("NO SETTINGS JSON FILE FOUND")

    server_settings = SETTINGS_FILE.get("cherrypy", None) # Load settings for cherrypy

    # If the settings file or the
    if server_settings is None:
        raise Exception("CHERRYPY SETTINGS ARE EMPTY")

    cherrypy.tools.CORS = cherrypy.Tool("before_handler", CORS) # This MUST run before every request sent

    #Update cherrypy with the settings from the settings.json file
    cherrypy.config.update({
        "server.socket_host": ARGUMENTS.server_host if ARGUMENTS.server_host else server_settings["host"], # If the host arugunemnt is present rather use that argument otherwise use the host in the settings file
        "server.socket_port": ARGUMENTS.server_port if ARGUMENTS.server_port else server_settings["port"], # If the port arugunemnt is present rather use that argument otherwise use the port in the settings file
    })
    cherrypy.quickstart(server(), "", config={
        "/": {
            "tools.sessions.on": True,
            "tools.sessions.name": "session_id",
            "tools.sessions.locking": "explicit",
            "tools.sessions.timeout": 600,
            "tools.sessions.storage_type": "ram",
            "tools.CORS.on": True
            }
        })


import base64
import datetime
import io

import dash
from dash.dependencies import Input, Output, State
import dash_core_components as dcc
import dash_html_components as html
import dash_table
import plotly.express as px
import plotly.graph_objects as go
import seismic as sei

import pandas as pd

df1 = pd.DataFrame()

def parse_contents(contents, filename):
    print(filename)
    content_type, content_string = contents.split(",")

    decoded = base64.b64decode(content_string)
    try:
        global df1
        df1 = pd.read_csv(io.StringIO(decoded.decode("utf-8")), sep="\t")
        df1["label"] = 1

    except Exception as e:
        print(e)
        return html.Div(["There was an error processing this file."])

    print("chargé")
    result1 = dcc.RangeSlider(
        id="slid",
        min=df1["sec"].min(),
        max=df1["sec"].max(),
        step=100000,
        value=[df1["sec"].min(), df1["sec"].max()],
    )
    print("time")

    result2 = dcc.Dropdown(
        id="drop",
        options=[{"label": i, "value": i} for i in df1.label.unique()],
        multi=True,
    )
    print("drop")
    result3 = html.Button("extract", id="export", style={"color": "#FF0000"})
    print("ok11")
    return (result1, result2, result3)

    #     dcc.RangeSlider(
    #         id="slid",
    #         min=df1["sec"].min(),
    #         max=df1["sec"].max(),
    #         step=1000,
    #         value=[df1["sec"].min(), df1["sec"].max()],
    #     )
    # )

def grphcreate(valmin, valmax, values):
    a = [
        {
            "data": [
                dict(
                    x=df1[
                        (df1["sec"] > valmin)
                        & (df1["sec"] < valmax)
                        & (df1["label"].isin(values))
                        & (df1["type"].isin(["mainshock", "correlated sismicity"]))
                    ]["p0"],
                    y=df1[
                        (df1["sec"] > valmin)
                        & (df1["sec"] < valmax)
                        & (df1["label"].isin(values))
                        & (df1["type"].isin(["mainshock", "correlated sismicity"]))
                    ]["p1"],
                    mode="markers",
                    hovertext=df1[
                        (df1["sec"] > valmin)
                        & (df1["sec"] < valmax)
                        & (df1["label"].isin(values))
                        & (df1["type"].isin(["mainshock", "correlated sismicity"]))
                    ]["label"],
                    hoverinfo="text",
                    opacity=0.7,
                    marker={
                        "color": df1[
                            (df1["sec"] > valmin)
                            & (df1["sec"] < valmax)
                            & (df1["label"].isin(values))
                            & (df1["type"].isin(["mainshock", "correlated sismicity"]))
                        ]["label"],
                        "size": 8,
                        "line": {"width": 0.5, "color": "white"},
                    },
                )
            ],
            "layout": dict(
                xaxis={"title": "latitude"},
                yaxis={"title": "longitude"},
                legend={"x": 0, "y": 1},
                hovermode="closest",
                margin={"l": 45, "r": 10},
            ),
        },
        {
            "data": [
                dict(
                    x=df1[
                        (df1["sec"] > valmin)
                        & (df1["sec"] < valmax)
                        & (df1["label"].isin(values))
                        & (df1["type"].isin(["mainshock", "correlated sismicity"]))
                    ]["sec"],
                    y=df1[
                        (df1["sec"] > valmin)
                        & (df1["sec"] < valmax)
                        & (df1["label"].isin(values))
                        & (df1["type"].isin(["mainshock", "correlated sismicity"]))
                    ]["mag"],
                    mode="markers",
                    hovertext=df1[
                        (df1["sec"] > valmin)
                        & (df1["sec"] < valmax)
                        & (df1["label"].isin(values))
                        & (df1["type"].isin(["mainshock", "correlated sismicity"]))
                    ]["label"],
                    hoverinfo="text",
                    opacity=0.7,
                    marker={
                        "color": df1[
                            (df1["sec"] > valmin)
                            & (df1["sec"] < valmax)
                            & (df1["label"].isin(values))
                            & (df1["type"].isin(["mainshock", "correlated sismicity"]))
                        ]["label"],
                        "size": 8,
                        "line": {"width": 0.5, "color": "white"},
                    },
                )
            ],
            "layout": dict(
                xaxis={"title": "temps"},
                yaxis={"title": "magnitude"},
                legend={"x": 0, "y": 1},
                hovermode="closest",
                margin={"l": 25, "r": 25},
            ),
        },
    ]

    return a

def clustering(delta_d, delta_t, min_clust):
    global df1
    df1 = sei.seismic_clust(df1, delta_d, delta_t, min_clust)
    print("ok")
    back1 = df1[df1["type"] == "background"]
    cor = df1[df1["type"] == "correlated sismicity"]
    main1 = df1[df1["type"] == "mainshock"]
    # sei.GraphInterEventTime2(main1.sec, back1.sec)

# sei.GraphInterEventTime2(main.sec, back.sec)
path = "./seq/"

def export(clusters):
    sei.get_seq(df1, clusters, path)

def graphMain():
    return [
        {
            "data": [
                {
                    "x": df1[(df1["type"].isin(["mainshock"]))]["sec"],
                    "y": df1[(df1["type"].isin(["mainshock"]))]["mag"],
                    "mode": "markers",
                    "hovertext": df1[(df1["type"].isin(["mainshock"]))]["label"],
                    "hoverinfo": "text",
                    "opacity": 0.7,
                    "name": "mainshock",
                    "mode": "markers",
                },
                {
                    "x": df1[(df1["type"].isin(["background"]))]["sec"],
                    "y": df1[(df1["type"].isin(["background"]))]["mag"],
                    "mode": "markers",
                    "hovertext": "background",
                    "hoverinfo": "text",
                    "opacity": 0.7,
                    "name": "background",
                },
            ],
            "layout": dict(
                xaxis={"title": "latitude"},
                yaxis={"title": "longitude"},
                legend={"x": 0, "y": 1},
                hovermode="closest",
                margin={"l": 25, "r": 25},
            ),
        },
    ]