import roslib; roslib.load_manifest('path_planning_analysis')
import rospy
import numpy
import scipy.spatial
from path_planning_analysis.translator import *
from geometry_msgs.msg import Pose, TransformStamped, Point32
from geometry_msgs.msg import Polygon as RosPolygon
from tf import Transformer
from tf.transformations import quaternion_from_euler
from math import sqrt
import Polygon #https://github.com/jraedler/Polygon2/

def distance(x0, y0, x1, y1):
    return sqrt(pow(x1-x0,2)+pow(y1-y0,2))

def distance_to_line(pX, pY, x0, y0, x1, y1):
    A = pX - x0
    B = pY - y0
    C = x1 - x0
    D = y1 - y0

    dot = A * C + B * D
    len_sq = C * C + D * D
    param = dot / len_sq

    if param < 0:
        xx = x0
        yy = y0
    elif param > 1:
        xx = x1
        yy = y1
    else:
        xx = x0 + param * C
        yy = y0 + param * D

    return distance(pX, pY, xx, yy)

def min_distance(px, py, polygon):
    min_dist = float("inf")
    for i, pt in enumerate(polygon.points):
        # check the distance from the point to the first vertex
        vertex_dist = distance(px, py, pt.x, pt.y)

        # check the distance from the point to the edge
        if i < len(polygon.points)-1:
            nextpt = polygon.points[i+1]
        else:
            nextpt = polygon.points[0]

        edge_dist = distance_to_line(px, py, pt.x, pt.y, nextpt.x, nextpt.y)

        min_dist = min(min_dist, min(vertex_dist, edge_dist))
    return min_dist

def get_convex_hull(vecs):
    vecs = numpy.array(vecs)
    hull = scipy.spatial.Delaunay(vecs).convex_hull
    ps = set()
    for x, y in hull:
        ps.add(x)
        ps.add(y)
    ps = numpy.array(list(ps))
    center = vecs[ps].mean(axis=0)
    A = vecs[ps] - center
    h = vecs[ps[numpy.argsort(numpy.arctan2(A[:,1], A[:,0]))]]
    return h

def line_intersection_helper(a1,a2,b1,b2):
    denom =      (b2.y-b1.y)*(a2.x-a1.x) - (b2.x-b1.x)*(a2.y-a1.y)
    if denom==0:
        return None
    numerator1 = (b2.x-b1.x)*(a1.y-b1.y) - (b2.y-b1.y)*(a1.x-b1.x)
    numerator2 = (a2.x-a1.x)*(a1.y-b1.y) - (a2.y-a1.y)*(a1.x-b1.x)
    return numerator1/denom, numerator2/denom

def line_intersection_helper2(a1,a2,f):
    x = a1.x + f * (a2.x - a1.x)
    y = a1.y + f * (a2.y - a1.y)
    return x,y

def line_intersection(a1,a2,b1,b2, segments=False):
    intersection = line_intersection_helper(a1,a2,b1,b2)
    if not intersection:
        return None

    ua, ub = intersection
    if segments:
        if ua < 0 or ua > 1 or ub < 0 or ub > 1:
            return None

    return line_intersection_helper2(a1, a2, ua)

def point_in_poly(point, poly):
    inside = False

    for i, pt1 in enumerate(poly.points):
        pt2 = poly.points[i-1]
        if point.y > min(pt1.y,pt2.y):
            if point.y <= max(pt1.y,pt2.y):
                if point.x <= max(pt1.x,pt2.x):
                    if pt1.y != pt2.y:
                        xints = (point.y-pt1.y)*(pt2.x-pt1.x)/(pt2.y-pt1.y)+pt1.x
                    if pt1.x == pt2.x or point.x <= xints:
                        inside = not inside
    return inside

def poly_to_ros(p):
    newpoly = RosPolygon()
    newpoly.points = [Point32(x,y,0) for x,y in p]
    return newpoly

def ros_to_poly(p):
    return Polygon.Polygon( [(pt.x,pt.y) for pt in p.points])

def polygon_intersection(poly1, poly2):
    q = ros_to_poly(poly1)
    t = ros_to_poly(poly2)
    intersection = q & t
    if len(intersection)==0:
        return None
    return poly_to_ros(intersection[0])

POM = [1, -1]
F = ' ab'

def get_vertices(pose, size, shape_type):
    tf = Transformer(True, rospy.Duration(10.0))
    m = TransformStamped()
    m.header.frame_id = 'object'
    m.transform.rotation.w = 1

    keys = []
    if shape_type=='box':
        for mx in POM:
            for my in POM:
                for mz in POM:
                    key = 'frame%s%s%s'%(F[mx], F[my], F[mz])
                    m.child_frame_id = key
                    keys.append(key)
                    m.transform.translation.x = size[0]/2.0*mx
                    m.transform.translation.y = size[1]/2.0*my
                    m.transform.translation.z = size[2]/2.0*mz
                    tf.setTransform(m)
    else:
        return None

    m.transform.translation.x = pose.position.x
    m.transform.translation.y = pose.position.y
    m.transform.translation.z = pose.position.z
    m.transform.rotation.x = pose.orientation.x
    m.transform.rotation.y = pose.orientation.y
    m.transform.rotation.z = pose.orientation.z
    m.transform.rotation.w = pose.orientation.w
    m.header.frame_id = 'map'
    m.child_frame_id = 'object'
    tf.setTransform(m)

    vertices = []
    for key in keys:
        (off,rpy) = tf.lookupTransform('map', key, rospy.Time(0))
        vertices.append(off)
    return vertices

def get_polygon(pose, size, shape_type):
    vertices = get_vertices(pose, size, shape_type)
    projected = [(a[0], a[1]) for a in vertices]
    points = get_convex_hull(projected)
    p = RosPolygon()
    for x,y in points:
        p.points.append(Point32(float(x),float(y),0))
    return p

class ObjectField:
    def __init__(self, scenario, name_map, simulation_states, t0):
        self.static_objects = {}
        self.dynamic_objects = {}

        for name, obj in scenario.iteritems():
            t = obj.type
            size = obj.size
            new_obj = {'type': t, 'size': size, 'person': obj.is_person}
            oname = name_map.get(name, name)
            if obj.movement is None:
                p = get_pose_from_scenario(name, obj)
                new_obj['pose'] = p
                get_polygon(p, size, t)
                self.static_objects[name] = new_obj
            else:
                poses = []
                for t, state in simulation_states:
                    if not oname in state.name:
                        continue
                    p = get_pose_from_state(oname, state)
                    poses.append((t-t0, p))
                new_obj['poses'] = sorted(poses)
                new_obj['polygons'] = [None] * len(poses)
                self.dynamic_objects[name] = new_obj

    def get_polygons(self, t0, person=False):
        polygons = {}
        for name, obj in self.static_objects.iteritems():
            if person and not obj['person']:
                continue
            if 'polygon' not in obj:
                obj['polygon'] = get_polygon(obj['pose'], obj['size'], obj['type'])
            polygons[name] = obj['polygon']

        for name, obj in self.dynamic_objects.iteritems():
            i0 = None
            for i, (t, pose) in enumerate(obj['poses']):
                if i==0 or t < t0:
                    continue
                dt0 = abs(t-t0)
                dt1 = abs(obj['poses'][i-1][0]-t0)
                if dt0 < dt1:
                    i0 = i
                else:
                    i0 = i - 1
                break
            if obj['polygons'][i0] is None:
                obj['polygons'][i0] = get_polygon(obj['poses'][i0][1], obj['size'], obj['type'])

            polygons[name] = obj['polygons'][i0]
        return polygons

    def nearest_distance_helper(self, px, py, polygons):
        min_dist = float('inf')
        for polygon in polygons:
            d = min_distance(px, py, polygon)
            min_dist = min(min_dist, d)
        return min_dist

    def get_nearest_polygon_distance(self, px, py, t, person=False):
        polygons = self.get_polygons(t, person)
        return self.nearest_distance_helper(px, py, polygons.values())

    def get_nearest_distance_in_polygon(self, px, py, t, mask, person=False):
        masked_polygons = []
        for name, poly in self.get_polygons(t, person).iteritems():
            new_poly = polygon_intersection(mask, poly)
            if new_poly:
                masked_polygons.append(new_poly)
        return self.nearest_distance_helper(px, py, masked_polygons)


"""
minimal example of using the backtester
"""

# imports
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import yfinance as yf
from typing import Tuple
import qis as qis

# package
from optimalportfolios.optimization.config import PortfolioObjective
from optimalportfolios.optimization.engine import compute_rolling_optimal_weights

# 1. we define the investment universe and allocation by asset classes
def fetch_universe_data() -> Tuple[pd.DataFrame, pd.DataFrame, pd.Series]:
    """
    fetch universe data for the portfolio construction:
    1. dividend and split adjusted end of day prices: price data may start / end at different dates
    2. benchmark prices which is used for portfolio reporting and benchmarking
    3. universe group data for portfolio reporting and risk attribution for large universes
    this function is using yfinance to fetch the price data
    """
    universe_data = dict(SPY='Equities',
                         QQQ='Equities',
                         EEM='Equities',
                         TLT='Bonds',
                         IEF='Bonds',
                         LQD='Credit',
                         HYG='HighYield',
                         GLD='Gold')
    tickers = list(universe_data.keys())
    group_data = pd.Series(universe_data)
    prices = yf.download(tickers, start=None, end=None, ignore_tz=True)['Adj Close']
    prices = prices[tickers]  # arrange as given
    prices = prices.asfreq('B', method='ffill')  # refill at B frequency
    benchmark_prices = prices[['SPY', 'TLT']]
    return prices, benchmark_prices, group_data

# 2. get universe data
prices, benchmark_prices, group_data = fetch_universe_data()
prices = prices.loc['2003':, :]  # use price data from 2003

# 3.a. define optimisation setup
portfolio_objective = PortfolioObjective.MAX_DIVERSIFICATION  # define portfolio objective
min_weights = {x: 0.0 for x in prices.columns} # all weights >= 0
max_weights = {x: 1.0 for x in prices.columns}  # all weights <= 1
rebalancing_freq = 'Q'  # weights rebalancing frequency
returns_freq = 'W-WED'  # use weekly returns
span = 52  # span of number of returns_freq-returns for covariance estimation = 12y
is_long_only = True  # all weights >= 0

# 3.b. compute rolling portfolio weights rebalanced every quarter
weights = compute_rolling_optimal_weights(prices=prices,
                                          portfolio_objective=portfolio_objective,
                                          min_weights=min_weights,
                                          max_weights=max_weights,
                                          rebalancing_freq=rebalancing_freq,
                                          is_long_only=is_long_only,
                                          span=span)

# 4. given portfolio weights, construct the performance of the portfolio
funding_rate = None  # on positive / negative cash balances
rebalancing_costs = 0.0010  # rebalancing costs per volume = 10bp
portfolio_data = qis.backtest_model_portfolio(prices=prices,
                                              weights=weights,
                                              is_rebalanced_at_first_date=True,
                                              ticker='MaxDiversification',
                                              funding_rate=funding_rate,
                                              rebalancing_costs=rebalancing_costs,
                                              is_output_portfolio_data=True)

# 5. using portfolio_data run the reporting with strategy factsheet
# for group-based reporting set_group_data
portfolio_data.set_group_data(group_data=group_data, group_order=list(group_data.unique()))
# set time period for portfolio reporting
time_period = qis.TimePeriod('01Jan2006', '31Aug2023')
fig = qis.generate_strategy_factsheet(portfolio_data=portfolio_data,
                                      benchmark_prices=benchmark_prices,
                                      time_period=time_period,
                                      **qis.fetch_default_report_kwargs(time_period=time_period))
# save report to pdf and png
qis.save_figs_to_pdf(figs=[fig],
                     file_name=f"{portfolio_data.nav.name}_portfolio_factsheet",
                     orientation='landscape',
                     local_path="C://Users//Artur//OneDrive//analytics//outputs")
qis.save_fig(fig=fig, file_name=f"example_portfolio_factsheet", local_path=f"figures/")

# 6. can create customised reporting using portfolio_data custom reporting
def run_customised_reporting(portfolio_data) -> plt.Figure:
    with sns.axes_style("darkgrid"):
        fig, axs = plt.subplots(3, 1, figsize=(12, 12), tight_layout=True)
    kwargs = dict(x_date_freq='A', framealpha=0.8)
    portfolio_data.plot_nav(ax=axs[0], **kwargs)
    portfolio_data.plot_weights(ncol=len(prices.columns)//3,
                                legend_stats=qis.LegendStats.AVG_LAST,
                                title='Portfolio weights',
                                bbox_to_anchor=None,
                                ax=axs[1],
                                **kwargs)
    portfolio_data.plot_returns_scatter(benchmark_price=benchmark_prices.iloc[:, 0],
                                        ax=axs[2],
                                        **kwargs)
    return fig

# run customised report
fig = run_customised_reporting(portfolio_data)
# save png
qis.save_fig(fig=fig, file_name=f"example_customised_report", local_path=f"figures/")

plt.show()


## Modules
import iio
import numpy as np
import os
from skimage.metrics import structural_similarity
import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F

##Functions

def build_variance_map(u1, u2, u3, u4, u5, u6, u7, u8, s1, s2, s3, s4, s5, s6, s7, s8):
    variance_map = s1*torch.tensor(u1) + s2*torch.tensor(u2) + s3*torch.tensor(u3) + s4*torch.tensor(u4) + s5*torch.tensor(u5) + s6*torch.tensor(u6) + s7*torch.tensor(u7) + s8*torch.tensor(u8)
    variance_map = torch.unsqueeze(variance_map, 0)
    variance_map = torch.unsqueeze(variance_map, 0)
    variance_map = variance_map.cuda()
    return variance_map

check_string = lambda string : all(c in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'] for c in string)

def create_parameter(value):
    param = Variable(torch.tensor([value], requires_grad = True).cuda())
    param.requires_grad = True
    param = torch.nn.Parameter(param)
    return param

def find_brightness(u):
    brightness = give_brightness(u)
    u1 = (brightness < 0.125 )
    u2 = (brightness >= 0.125) * (brightness < 0.25 )
    u3 = (brightness >= 0.25 ) * (brightness < 0.375)
    u4 = (brightness >= 0.375) * (brightness < 0.5  )
    u5 = (brightness >= 0.5  ) * (brightness < 0.625)
    u6 = (brightness >= 0.625) * (brightness < 0.75 )
    u7 = (brightness >= 0.75 ) * (brightness < 0.875)
    u8 = (brightness >= 0.875)
    return(u1, u2, u3, u4, u5, u6, u7, u8)

def gives_flow(path_flow, H, W):
    flow = read_flow(path_flow)[:H, :W]
    flow = np.expand_dims(flow, 0)
    flow = Variable(torch.Tensor(flow))
    flow = flow.permute(0, 3, 1, 2)
    return flow
give_brightness = lambda x : torch.mean(x, 0) #average on the three channels

def gives_masks(path_collition, path_warping_error_based, H, W):
    if check_string(path_collition):
        mask1 = np.ones((H, W, 1))
        exclusive_mask1 = np.ones((H, W, 1))
    else:
        mask0 = iio.read(path_collition)[:H, :W]
        mask1 = 1. * (mask0 < 1.5)*(mask0 > 0.5)
        mask1 = 1. - (mask1 > 0)
        exclusive_mask1 = 1. * (mask0 >= 1.5)
        exclusive_mask1 = 1. - exclusive_mask1
    if check_string(path_warping_error_based):
        mask0 = np.ones((H, W, 1))
        exclusive_mask_warping_res = np.ones((H, W, 1))
    else:
        mask_warping_res = iio.read(path_warping_error_based)[:H, :W]
        mask0 = 1. * (mask_warping_res < 1.5)*(mask_warping_res > 0.5)
        exclusive_mask_warping_res = 1. * (mask_warping_res >= 1.5)
        exclusive_mask_warping_res = 1. - exclusive_mask_warping_res

    exclusive_mask = exclusive_mask1 * exclusive_mask_warping_res
    mask = mask1 * mask0

    mask = mask.transpose(2,0,1)
    exclusive_mask = exclusive_mask.transpose(2,0,1)
    mask = np.expand_dims(mask, 0)
    mask = torch.Tensor(mask).view(1,1,H,W).repeat(1,3,1,1).cuda()
    exclusive_mask = np.expand_dims(exclusive_mask, 0)
    exclusive_mask = torch.Tensor(exclusive_mask).view(1,1,H,W).repeat(1,3,1,1).cuda()

    return mask, exclusive_mask

learning_rate = lambda x : 10**-5 * np.exp(-5*x)

def psnr(img1, img2, peak=1):
    '''
    Computes the PSNR 'metric' between two images assumed to be in the range [0,1]
    '''
    x = ((np.array(img1).squeeze() - np.array(img2).squeeze()).flatten())
    return (10*np.log10(peak**2 / np.mean(x**2)))

def read_flow(file):
    assert type(file) is str, "file is not str %r" % str(file)
    assert os.path.isfile(file)is True, "file does not exist %r" % str(file)
    assert file[-4:] == '.flo', "file ending is not .flo %r" % file[:-4]
    f = open(file, 'rb')
    flo_number = np.fromfile(f, np.float32, count = 1)[0]
    assert flo_number == 202021.25, "Flow number %r incorrect. Invalid .flo file" % flo_number
    w = np.fromfile(f, np.int32, count = 1)
    h = np.fromfile(f, np.int32, count = 1)
    data = np.fromfile(f, np.float32, count = 2*w[0]*h[0])
    flow = np.resize(data, (int(h), int(w), 2))
    f.close()
    return flow

def reads_image(path, H, W):
    image = iio.read(path)[:H, :W]
    image = image / 255.
    image = image.transpose(2,0,1)
    image = torch.Tensor(image).cuda()
    image = Variable(image)
    return image

def save_image(path, img):
    img = image.detach().cpu().numpy.squeeze()
    img = img.transpose(1,2,0)
    img = (img*255).astype(np.uint8)
    iio.write(path, img)

ssim = lambda x, y : structural_similarity(x.astype(float),y.astype(float),multichannel=True, data_range=255)

def temp_denoise(model, noisyframe, sigma_noise):
    '''Encapsulates call to denoising model and handles padding.
        Expects noisyframe to be normalized in [0., 1.]
    '''
    # make size a multiple of four (we have two scales in the denoiser)
    sh_im = noisyframe.size()
    expanded_h = sh_im[-2]%4
    if expanded_h:
        expanded_h = 4-expanded_h
    expanded_w = sh_im[-1]%4
    if expanded_w:
        expanded_w = 4-expanded_w
    padexp = (0, expanded_w, 0, expanded_h)
    noisyframe = F.pad(input=noisyframe, pad=padexp, mode='reflect')
    sigma_noise = F.pad(input=sigma_noise, pad=padexp, mode='reflect')

    # denoise
    out = model(noisyframe, sigma_noise)

    if expanded_h:
        out = out[:, :, :-expanded_h, :]
    if expanded_w:
        out = out[:, :, :, :-expanded_w]

    return out

def temp_denoise_8_sigmas(model, noisyframe, sigma_noise1, sigma_noise2, sigma_noise3):
    '''Encapsulates call to denoising model and handles padding.
        Expects noisyframe to be normalized in [0., 1.]
    '''
    # make size a multiple of four (we have two scales in the denoiser)
    sh_im = noisyframe.size()
    expanded_h = sh_im[-2]%4
    if expanded_h:
        expanded_h = 4-expanded_h
    expanded_w = sh_im[-1]%4
    if expanded_w:
        expanded_w = 4-expanded_w
    padexp = (0, expanded_w, 0, expanded_h)
    noisyframe = F.pad(input=noisyframe, pad=padexp, mode='reflect')
    sigma_noise1 = F.pad(input=sigma_noise1, pad=padexp, mode='reflect')
    sigma_noise2 = F.pad(input=sigma_noise2, pad=padexp, mode='reflect')
    sigma_noise3 = F.pad(input=sigma_noise3, pad=padexp, mode='reflect')

    # denoise
    out = model(noisyframe, sigma_noise1, sigma_noise2, sigma_noise3)

    if expanded_h:
        out = out[:, :, :-expanded_h, :]
    if expanded_w:
        out = out[:, :, :, :-expanded_w]

    return out

def tensor_to_image(tensor):
    image = torch.clamp(tensor, 0, 1)
    image = np.array(image.cpu())
    image = np.squeeze(image)
    image = image.transpose(1,2,0)
    return image


import os
import torch
import numpy as np
import visdom
import time
from torch.utils.data import DataLoader
import matplotlib.pyplot as plt

from MTAGN_utils.train_utils import MyDataset, set_seed
from MTAGN_utils.plot_utils import plot_confusion_matrix
from data_generator.CWdata import CW_0, CW_1, CW_2, CW_3
from data_generator.SQdata import SQ_39, SQ_29, SQ_19
from data_generator.EBdata import EB
from models.jlcnn import JLCNN
from models.mt1dcnn import MT1DCNN
from models.mtagn import MTAGN

# ==================== train and test =====================
class multi_task_trainer:
    def __init__(self, model_):
        self.model = model_

    def train(self, train_loader_, valid_loader_, valid_set_, optimizer_, scheduler_, epochs=100):
        # ----------- model initial ----------
        loss_fuc = torch.nn.CrossEntropyLoss()

        train_batch = len(train_loader_)
        valid_batch = len(valid_loader_)

        print('--------------------Training--------------------')
        counter = 1
        T = 2
        avg_cost = np.zeros([epochs, 6])  # 0\1\2 train loss; 3\4\5 valid loss
        lambda_weight = np.ones([2, epochs])
        weight = np.zeros((100, 2))
        for epoch in range(epochs):
            train_loss_1 = 0.0
            train_loss_2 = 0.0
            train_loss = 0.0
            train_acc_1 = 0
            train_acc_2 = 0

            cost = np.zeros(6, dtype=np.float32)

            if epoch == 0 or epoch == 1:
                lambda_weight[:, epoch] = 1.0
            else:
                w_1 = avg_cost[epoch - 1, 0] / avg_cost[epoch - 2, 0]
                w_2 = avg_cost[epoch - 1, 1] / avg_cost[epoch - 2, 1]
                lambda_weight[0, epoch] = 2 * np.exp(w_1 / T) / (np.exp(w_1 / T) + np.exp(w_2 / T))
                lambda_weight[1, epoch] = 2 * np.exp(w_2 / T) / (np.exp(w_1 / T) + np.exp(w_2 / T))

            self.model.train()
            for batch_idx, data in enumerate(train_loader_):
                optimizer_.zero_grad()
                inputs, label1, label2 = data
                inputs, label1, label2 = inputs.to(device), label1.to(device), label2.to(device)

                # forward + backward + update
                output1, output2 = self.model(inputs)
                tr_loss = [loss_fuc(output1, label1),
                           loss_fuc(output2, label2)]
                loss = sum(lambda_weight[i, epoch] * tr_loss[i] for i in range(2))

                train_loss_1 += tr_loss[0].item()
                train_loss_2 += tr_loss[1].item()
                train_loss += loss.item()

                loss.backward()
                optimizer_.step()

                cost[0] = tr_loss[0].item()
                cost[1] = tr_loss[1].item()
                cost[2] = loss.item()

                avg_cost[epoch, :3] += cost[:3] / train_batch

                train_acc_1 += np.sum(np.argmax(output1.cpu().data.numpy(), axis=1) == label1.cpu().numpy())
                train_acc_2 += np.sum(np.argmax(output2.cpu().data.numpy(), axis=1) == label2.cpu().numpy())
            train_acc_1 = (train_acc_1 / train_set.__len__()) * 100
            train_acc_2 = (train_acc_2 / train_set.__len__()) * 100

            # valid
            self.model.eval()
            with torch.no_grad():
                valid_loss_1 = 0.0
                valid_loss_2 = 0.0
                valid_loss = 0.0
                valid_acc_1 = 0
                valid_acc_2 = 0
                if epoch + 1 == epochs:
                    pred_label1 = np.array([], dtype=np.int64)
                    pred_label2 = np.array([], dtype=np.int64)
                for data in valid_loader_:
                    inputs, label1, label2 = data
                    inputs, label1, label2 = inputs.to(device), label1.to(device), label2.to(device)
                    output1, output2 = self.model(inputs)
                    val_loss = [loss_fuc(output1, label1),
                                loss_fuc(output2, label2)]
                    loss = sum(lambda_weight[i, epoch] * val_loss[i] for i in range(2))

                    valid_loss_1 += val_loss[0].item()
                    valid_loss_2 += val_loss[1].item()
                    valid_loss += loss.item()

                    cost[3] = val_loss[0].item()
                    cost[4] = val_loss[1].item()
                    cost[5] = loss.item()

                    avg_cost[epoch, 3:6] += cost[3:6] / valid_batch

                    valid_acc_1 += np.sum(np.argmax(output1.cpu().data.numpy(), axis=1) == label1.cpu().numpy())
                    valid_acc_2 += np.sum(np.argmax(output2.cpu().data.numpy(), axis=1) == label2.cpu().numpy())

                    if epoch + 1 == epochs:
                        pred_label1 = np.append(pred_label1, torch.max(output1, dim=1)[1].cpu().numpy().astype('int64'))
                        pred_label2 = np.append(pred_label2, torch.max(output2, dim=1)[1].cpu().numpy().astype('int64'))

                valid_acc_1 = 100 * valid_acc_1 / valid_set.__len__()
                valid_acc_2 = 100 * valid_acc_2 / valid_set.__len__()

                if epoch + 1 == epochs:
                    plot_confusion_matrix(valid_set_.labels1.numpy(), pred_label1, norm=False, task=1)
                    plot_confusion_matrix(valid_set_.labels2.numpy(), pred_label2, norm=False, task=2)
                    plt.show()
            # vis.line(Y=[[avg_cost[epoch, 0]*train_batch, avg_cost[epoch, 1]*train_batch, avg_cost[epoch, 2]*train_batch, avg_cost[epoch, 3]*train_batch,
            #              avg_cost[epoch, 4]*train_batch, avg_cost[epoch, 5]*train_batch]], X=[counter],
            #          update=None if counter == 0 else 'append', win='loss',
            #          opts=dict(legend=['train_loss_1', 'train_loss_2', 'train_loss', 'valid_loss_1', 'valid_loss_2',
            #                            'valid_loss'], title='Loss', ))
            vis.line(Y=[
                [avg_cost[epoch, 0] * train_batch, avg_cost[epoch, 1] * train_batch, avg_cost[epoch, 2] * train_batch]],
                X=[counter], update=None if counter == 0 else 'append', win='loss',
                opts=dict(legend=['train_loss_1', 'train_loss_2', 'train_loss', 'valid_loss_1', 'valid_loss_2',
                                  'valid_loss'], title='Loss', ))
            vis.line(Y=[[train_acc_1, train_acc_2, valid_acc_1, valid_acc_2]], X=[counter],
                     update=None if counter == 0 else 'append', win='accuracy',
                     opts=dict(legend=['train_acc_1', 'train_acc_2', 'valid_acc_1', 'valid_acc_2'], title='Accuracy'))
            vis.line(Y=[[lambda_weight[0, epoch], lambda_weight[1, epoch]]], X=[counter],
                     update=None if counter == 0 else 'append', win='weight',
                     opts=dict(legend=['weight1', 'weight2'], title='Weight'))
            counter += 1

            scheduler_.step()
            print(
                'epoch: [{}/{}] | Loss: {:.5f} | FTI_acc_tr: {:.2f}% | FSI_acc_tr: {:.2f}% | w1:{:.3f} w2:{:.3f}'.format(
                    epoch + 1, epochs, train_loss, train_acc_1, train_acc_2, lambda_weight[0, epoch],
                    lambda_weight[1, epoch]))
            weight[epoch, 0], weight[epoch, 1] = lambda_weight[0, epoch], lambda_weight[1, epoch]
        print('Finish training!')
        loss_out = avg_cost[:, :3] * train_batch
        print(loss_out.shape)
        np.savetxt('EB-train_loss-DWA.csv', loss_out, delimiter=',')
        print(weight.shape)
        print(weight)
        np.savetxt('lambda-weight-DWA.csv', weight, delimiter=',')
        order_save = input('Save model?(Y/N): ')
        if order_save == 'Y' or order_save == 'y':
            self.save(filename=model_dir, model_name_pkl=model_name)

    def test(self, test_set_, test_loader_):
        self.load(model_path)
        self.model.eval()

        print('-------------------- Testing --------------------')
        acc_1 = 0
        acc_2 = 0
        pred_label1 = np.array([], dtype=np.int64)
        pred_label2 = np.array([], dtype=np.int64)
        with torch.no_grad():
            t0 = time.time()
            for data in test_loader_:
                inputs, label1, label2 = data
                inputs = inputs.to(device)
                output1, output2 = self.model(inputs)

                acc_1 += np.sum(torch.max(output1, dim=1)[1].cpu().numpy() == label1.numpy())
                acc_2 += np.sum(torch.max(output2, dim=1)[1].cpu().numpy() == label2.numpy())

                pred_label1 = np.append(pred_label1, torch.max(output1, dim=1)[1].cpu().numpy().astype('int64'))
                pred_label2 = np.append(pred_label2, torch.max(output2, dim=1)[1].cpu().numpy().astype('int64'))
                t1 = time.time()

            FTI_acc = 100 * acc_1 / test_set_.__len__()
            FSI_acc = 100 * acc_2 / test_set_.__len__()
            te_time = t1 - t0

            # print('pred label1:')
            # for i in range(10):
            #     print(pred_label1[150 * i:150 * (i + 1)])
            # print('pred label2:')
            # for i in range(7):
            #     print(pred_label2[150 * i:150 * (i + 1)])
            print('Accuracy on test_dataset:')
            print(f'FTI-task: {FTI_acc:.2f}%   [{acc_1}/{test_set_.__len__()}]')
            print(f'FSI-task: {FSI_acc:.2f}%   [{acc_2}/{test_set_.__len__()}]')
            print(f'Test time: {te_time:.4f}')
            plot_confusion_matrix(test_set_.labels1.numpy(), pred_label1, norm=False, task=1)
            plot_confusion_matrix(test_set_.labels2.numpy(), pred_label2, norm=False, task=2)
            plt.show()

    def save(self, filename, model_name_pkl):
        if os.path.exists(filename):
            filename = os.path.join(filename, model_name_pkl)
        torch.save(self.model.state_dict(), filename)
        print(f'This model is saved at: {filename}')

    def load(self, filename):
        state = torch.load(filename)
        self.model.load_state_dict(state)
        print('Load model successfully from [%s]' % filename)

if __name__ == '__main__':
    # ==================== Hyper parameters =====================
    EPOCHS = 100
    BATCH_SIZE = 50
    LR = 0.001
    # set_seed(2021)

    # define model, vis, optimiser
    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
    vis = visdom.Visdom(env='dwa')
    model = MTAGN().to(device)
    optimizer = torch.optim.Adam(model.parameters(), lr=LR)
    scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=1)

    train_X, train_Y1, train_Y2, valid_X, valid_Y1, valid_Y2, test_X, test_Y1, test_Y2 = CW_0()
    # ----------- train data ----------
    train_set = MyDataset(train_X, train_Y1, train_Y2)
    train_loader = DataLoader(train_set, shuffle=True, batch_size=BATCH_SIZE)
    print('train_set')
    print("train_X:", train_X.shape)
    print("train_Y1:", train_Y1.shape)
    print("train_Y2:", train_Y2.shape)
    # ----------- valid data ----------
    valid_set = MyDataset(valid_X, valid_Y1, valid_Y2)
    valid_loader = DataLoader(valid_set, shuffle=False, batch_size=BATCH_SIZE)
    print('valid_set')
    print("valid_X:", valid_X.shape)
    print("valid_Y1:", valid_Y1.shape)
    print("valid_Y2:", valid_Y2.shape)
    # ----------- data ----------
    test_set = MyDataset(test_X, test_Y1, test_Y2)
    test_loader = DataLoader(test_set, shuffle=False, batch_size=BATCH_SIZE)
    print("test_X:", test_X.shape)
    print("test_Y1:", test_Y1.shape)
    print("test_Y2:", test_Y2.shape)

    trainer = multi_task_trainer(model)

    # 模型名字及保存地址
    model_dir = r'F:\PycharmProject\MTNet\ablation experiments'
    model_name = 'mtan_eca6_dwa-cw0-100-3.pkl'
    # model_name = 'test.pkl'
    model_path = os.path.join(model_dir, model_name)
    print("Model path: ", model_dir)
    if not os.path.exists(model_dir):
        print(f'Root dir {model_dir} does not exit.')
        exit()
    else:
        print('File exist? ', os.path.exists(model_dir))
    # order_tr = input('Train or not?(Y/N): ')
    # if order_tr == 'Y' or order_tr == 'y':
    trainer.train(train_loader, valid_loader, valid_set, optimizer, scheduler, EPOCHS)

    order_te = input("Test or not?(Y/N): ")
    if order_te == "Y" or order_te == "y":
        trainer.test(test_set, test_loader)


#
# usage: python k36.py {file name}
#

import sys
import collections
from k30 import load_mecab

def frequency_ranking(data):
    cd = collections.Counter([m['base'] for s in data for m in s])
    return [[k, v] for k,v in cd.most_common()]

if __name__ == '__main__':
    fn = sys.argv[1]
    data = load_mecab(fn)
    for v in frequency_ranking(data):
        print(v[0], v[1])


# coding: utf-8

# # LDA demo
# ### source: https://gist.github.com/georgehc/d2353feef7e09b4b53fc087d44f75954

# In[72]:

import nltk
import json
import pyLDAvis
import pyLDAvis.gensim  # don't skip this
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')
import random

# In[2]:

content = []
for line in open('JACS.json', 'r'):
    content.append(json.loads(line))

# ## To load eupmc.json

# In[39]:

content = []
with open('eupmc.json') as json_data:
    content = json.load(json_data)

# In[3]:

Total = []
for c in content:
    ##using both title and content
    total = c['Title']
    Total.append(total)

#print(Total)

# In[4]:

vocab_size = 1000
from sklearn.feature_extraction.text import CountVectorizer

# document frequency (df) means number of documents a word appears in
tf_vectorizer = CountVectorizer(max_df=0.95,
                                min_df=2,
                                max_features=vocab_size,
                                stop_words='english')
tf = tf_vectorizer.fit_transform(Total)

# In[5]:

num_topics = 20

from sklearn.decomposition import LatentDirichletAllocation
lda = LatentDirichletAllocation(n_components=num_topics, learning_method='online', random_state=95865)
lda.fit(tf)

# In[6]:

import numpy as np
topic_word_distributions = np.array([topic_word_pseudocounts / np.sum(topic_word_pseudocounts)
                                     for topic_word_pseudocounts in lda.components_])

# In[7]:

num_top_words = 10

print('Displaying the top %d words per topic and their probabilities within the topic...' % num_top_words)
print()

import numpy as np
for topic_idx in range(num_topics):
    print('[Topic %d]' % topic_idx)
    sort_indices = np.argsort(topic_word_distributions[topic_idx])[::-1]
    for rank in range(num_top_words):
        word_idx = sort_indices[rank]
        print('%s: %f' % (tf_vectorizer.get_feature_names()[word_idx], topic_word_distributions[topic_idx, word_idx]))
    print()

# # Another Way of doing LDA
# ### Source: https://rstudio-pubs-static.s3.amazonaws.com/79360_850b2a69980c4488b1db95987a24867a.html

# In[8]:

from nltk.corpus import stopwords
nltk.download('stopwords')
nltk.download('wordnet')
from nltk.stem.wordnet import WordNetLemmatizer
import string
stop = set(stopwords.words('english'))
exclude = set(string.punctuation)
lemma = WordNetLemmatizer()
def clean(doc):
    stop_free = " ".join([i for i in doc.lower().split() if i not in stop])
    punc_free = ''.join(ch for ch in stop_free if ch not in exclude)
    normalized = " ".join(lemma.lemmatize(word) for word in punc_free.split())
    return normalized

doc_clean = [clean(entry).split() for entry in Total]

#print(doc_clean)

# In[79]:

import gensim
from gensim import corpora,models

random.shuffle(Total)

training = Total[:round(len(Total)*0.6)]
test = Total[round(len(Total)*0.6):]

doc_clean_train = [clean(entry).split() for entry in training]
doc_clean_test = [clean(entry).split() for entry in test]
# Creating the term dictionary of our courpus, where every unique term is assigned an index.
dictionary_tr = corpora.Dictionary(doc_clean_train)
dictionary_te = corpora.Dictionary(doc_clean_test)
# Converting list of documents (corpus) into Document Term Matrix using dictionary prepared above.
doc_term_matrix_te = [dictionary_te.doc2bow(doc) for doc in doc_clean_test]

#mystring = mystring..decode(‘utf-8’)

# In[83]:

tfidf = models.TfidfModel(doc_term_matrix_tr)
corpus_tfidf = tfidf[doc_term_matrix_tr]
corpus_tfidf_te = tfidf[doc_term_matrix_te]

# In[81]:

lda_model_tfidf = gensim.models.LdaMulticore(corpus_tfidf, num_topics=5, id2word=dictionary, passes=2, workers=4)

# In[82]:

Lda = gensim.models.ldamodel.LdaModel
ldamodel = Lda(doc_term_matrix_tr, num_topics=5, id2word = dictionary_tr, passes=50)

# In[46]:

for idx, topic in lda_model_tfidf.print_topics(-1):
    print('Topic: {} \nWords: {}'.format(idx+1, topic))

# In[84]:

lda_model_tfidf.log_perplexity(corpus_tfidf_te)

# In[85]:

ldamodel.log_perplexity(doc_term_matrix_te)

# In[59]:

#Dynamic visualization of model results
pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(lda_model_tfidf, corpus_tfidf, dictionary)
vis

# In[93]:

##pick optimal number of topics
s = []
for i in range(1,40):
    lda_model_tfidf = gensim.models.LdaMulticore(corpus_tfidf, num_topics=i, id2word=dictionary_tr, passes=2, workers=4)
    score = lda_model_tfidf.log_perplexity(doc_term_matrix_te)
    s.append(score)

# In[94]:

#If the coherence score seems to keep increasing, it may make better sense to pick the model
#that gave the lowest perplexity before flattening out. This is exactly the case here.
x = range(1,40)
plt.plot(x, s)
plt.xlabel("Num Topics")
plt.ylabel("Perplexity")
plt.legend(("Perplexity"), loc='best')
plt.show()
#Looks like 34 is optimal here?


import os
import json
import shutil
import pyttsx3
import winreg
import Management.Paths.DataPath as fp
# from Fun.Body.Hand import printData, takeInput
# from Fun.Brain.Community import writeLog
from Setup.Structure import defaultConfig, defaultIntents

def updateCongif(key: str, value: str):
    with open(fp.SOFTWARE_CONFIG_FILE, "r+") as f:
        data = json.load(f)
        data[key] = value
        f.seek(0)
        json.dump(data, f, indent=4)
        f.truncate()

def moveFile(filepath: str, dest: str):
    # writeLog(f"moving {filepath} to {dest}")
    shutil.move(filepath, dest)
    # writeLog(f"moved {filepath} to {dest}")

def copyFile(file: str, dest: str):
    # writeLog(f"Copying {file} at {dest}")
    try:
        shutil.copyfile(file, dest)
        # writeLog(f"Copied {file} successfully at {dest}")
    except:
        # writeLog(f"Failed to copy {file} File at {dest}")
        pass

def setVoice(ind: int):
    engine = pyttsx3.init()
    voices = engine.getProperty('voices')
    # writeLog("Setting up voice index")
    if ind > len(voices) or ind < 0:
        return 0
    return ind

def verifyConfig():
    if not os.path.exists(fp.SOFTWARE_CONFIG_FILE):
        # writeLog(f"Creating config file at {fp.SOFTWARE_CONFIG_FILE}")
        with open(fp.SOFTWARE_CONFIG_FILE, 'w') as configF:
                json.dump(defaultConfig(), configF)
                configF.close()
        # writeLog(f"Created config file successfully at {fp.SOFTWARE_CONFIG_FILE}")
    # writeLog("Checking config file for any remaining settings")
    with open(fp.SOFTWARE_CONFIG_FILE, "r+") as f:
        config_data = json.load(f)
        try:
            config_data["voice"]["index"] = setVoice(config_data["voice"]["index"])
        except:
            config_data["voice"]["index"] = 0
        try:
            config_data["googleAI"] = config_data["googleAI"]
        except:
            config_data["googleAI"] = ''
        if config_data['googleAI'] == '':
            # writeLog("Storing googleAI api")
            # printData("Enter your GoogleAI api key. If you dont set it then you can't use full features like chatting or any research. If you dont know how to get it then kindly visit forum at xyz. To skip this step enter '0'")
            api = takeInput("Enter your GoogleAI api key.")
            if (api == 0) or (api == "0"):
                config_data['googleAI'] = ""
            else:
                config_data['googleAI'] = str(api)
            f.seek(0)
            json.dump(config_data, f, indent=4)
            f.truncate()
            f.close()
            # writeLog("Storing googleAI api successful")
    # writeLog("Checked config file for any remaining settings successful")
    copyFile(fp.SOFTWARE_CONFIG_FILE, fp.LOCALDATA_CONFIG_FILE)

def verifyIntents():
    if not os.path.isfile(fp.LOCALDATA_INTENTS_FILE):
        if not os.path.isfile(fp.SOFTWARE_INTENTS_FILE):
            with open(fp.SOFTWARE_INTENTS_FILE, 'w') as ff:
                ff.close()
        with open(fp.SOFTWARE_INTENTS_FILE, 'r+') as fff:
            json.dump(defaultIntents(), fff, indent=4)
            fff.close()
        copyFile(fp.SOFTWARE_INTENTS_FILE, fp.LOCALDATA_INTENTS_FILE)

def getInstalledApps():
    # writeLog("Fetching installed apps")
    installed_apps = []
    try:
        with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall") as key:
            index = 0
            while True:
                try:
                    subkey_name = winreg.EnumKey(key, index)
                    with winreg.OpenKey(key, subkey_name) as subkey:
                        try:
                            app_name = winreg.QueryValueEx(subkey, "DisplayName")[0]
                            install_location = winreg.QueryValueEx(subkey, "InstallLocation")[0]
                            if install_location and not app_name.lower().endswith('.exe'):
                                installed_apps.append(app_name)
                        except FileNotFoundError as r:
                            # writeLog("No DisplayName or InstallLocation found for subkey")
                            pass
                        except Exception as e:
                            # writeLog(f"Error reading registry values: {e}")
                            pass
                    index += 1
                except FileNotFoundError:
                    # No more subkeys to enumerate
                    break
                except Exception as ee:
                    # writeLog(f"Error enumerating subkeys: {ee}")
                    break
    except Exception as eee:
        # writeLog("Unable to access Windows registry")
        pass

    if len(installed_apps) > 0:
        with open(fp.APPS_FILE, 'w') as json_file:
            json.dump(installed_apps, json_file, indent=4)
            # writeLog("Fetched installed apps successfully")
    else:
        # writeLog("No installed apps fetched")
        pass

def basicSetup():
    logPth = [fp.LOG_FILE]
    if not os.path.isdir(fp.LOCALFOLDER):
        os.mkdir(fp.LOCALFOLDER)
    if not os.path.isdir(fp.LOG_FOLDER):
        os.mkdir(fp.LOG_FOLDER)
    for log in logPth:
        if not os.path.isfile(log):
            with open(log, 'w') as f:
                f.close()

def handleStructure(paths: list):
    for folder in paths:
        if not os.path.isdir(folder):
            # writeLog(f"Creating folder {folder}")
            os.mkdir(folder)

def dirSetup():
    dirs = [
        fp.LOCALFOLDER,
        fp.LOCAL_COMMUNITY_FOLDER,
        fp.DATASET_FOLDER,
        fp.DATABASE_FOLDER,
        fp.CHAT_DATA_FOLDER,
        fp.TRAINED_DATA_FOLDER,
        fp.APPS_FOLDER
    ]
    handleStructure(dirs)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import _init_paths

from tqdm import tqdm
import os
import sys
import cv2
import json
import copy
import numpy as np
import warnings

from opts import opts
from detector import Detector
from tools import tbd_utils, visualization

warnings.filterwarnings("ignore")

image_ext = ['jpg', 'jpeg', 'png', 'webp']
video_ext = ['mp4', 'mov', 'avi', 'mkv']
time_stats = ['tot', 'load', 'pre', 'net', 'dec', 'post', 'merge', 'display']

def demo(opt):
    os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str
    opt.debug = max(opt.debug, 1)
    detector = Detector(opt)

    if opt.demo == 'webcam' or \
            opt.demo[opt.demo.rfind('.') + 1:].lower() in video_ext:
        is_video = True

        cam = cv2.VideoCapture(0 if opt.demo == 'webcam' else opt.demo)
    else:
        is_video = False

        if os.path.isdir(opt.demo):
            image_names = []
            ls = os.listdir(opt.demo)
            for file_name in sorted(ls):
                ext = file_name[file_name.rfind('.') + 1:].lower()
                if ext in image_ext:
                    image_names.append(os.path.join(opt.demo, file_name))
        else:
            image_names = [opt.demo]

    out = None
    out_name = opt.demo[opt.demo.rfind('/') + 1:]
    if opt.save_video:
        if not os.path.exists('../results'):
            os.mkdir('../results')

        fourcc = cv2.VideoWriter_fourcc('I', '4', '2', '0')
        save_video_dir = '../results/{}/videos/'.format(
            opt.exp_id + '_' + out_name)
        if not os.path.exists(save_video_dir):
            os.makedirs(save_video_dir, exist_ok=True)
        save_seq_path = os.path.join(save_video_dir, out_name)

        out = cv2.VideoWriter(save_seq_path+".avi", fourcc, opt.save_framerate, (
            opt.input_w, opt.input_h))

    if opt.debug < 5:
        detector.pause = False
    cnt = 0
    results = {}

    remain_frame = 30
    last_n_frames_bboxes = np.zeros(remain_frame, np.int32)

    save_video_dir = '../results/{}/records/'.format(
        opt.exp_id + '_' + out_name)
    if not os.path.exists(save_video_dir):
        os.makedirs(save_video_dir, exist_ok=True)
    results_mot_file_path = os.path.join(save_video_dir, out_name+".txt")

    while True:
        if is_video:
            _, img = cam.read()
            if img is None:
                save_and_exit(opt, out, results, out_name)
        else:
            if cnt < len(image_names):
                img = cv2.imread(image_names[cnt])
            else:

                avi_to_mp4 = "ffmpeg -i %s.avi %s.mp4" % (
                    save_seq_path, save_seq_path)
                del_avi_file = "rm %s.avi" % (save_seq_path)
                os.system(avi_to_mp4)
                os.system(del_avi_file)
                out.release()
                return

        cnt += 1

        if opt.resize_video:
            img = cv2.resize(img, (opt.input_w, opt.input_h))

        if cnt < opt.skip_first:
            continue

        if not opt.save_video:
            cv2.imshow('input', img)

        ret = detector.run(img)

        time_str = '[{}] frame {} |'.format(opt.demo.split("/")[-1], cnt)
        for stat in time_stats:
            time_str = time_str + '{} {:.3f}s |'.format(stat, ret[stat])
        print(time_str)

        results[cnt] = ret['results']

        save_img_dir = "../results/%s/imgs/" % (opt.exp_id + '_' + out_name)
        if not os.path.exists(save_img_dir):
            os.makedirs(save_img_dir, exist_ok=True)
        if opt.save_video:

            trackers = []
            for tracklet in results[cnt]:
                trackers.append(list(tracklet['bbox']) +
                                list([tracklet['score']]) +
                                list([tracklet['tracking_id']]))

            frame_id = cnt-1
            tbd_utils.SaveRecords(trackers, frame_id, results_mot_file_path)

            last_n_frames_bboxes[frame_id % remain_frame] = len(trackers)
            n_lines = last_n_frames_bboxes.sum()
            img = visualization.DrawTrackers(img, trackers, frame_id,
                                             n_lines, results_mot_file_path)

            out.write(img)

            if not is_video:
                cv2.imwrite(save_img_dir+'{}.jpg'.format(cnt), img)

        if cv2.waitKey(1) == 27:
            save_and_exit(opt, out, results, out_name)
            return
    save_and_exit(opt, out, results)

def save_and_exit(opt, out=None, results=None, out_name=''):
    if opt.save_results and (results is not None):
        save_dir = '../results/{}_results.json'.format(
            opt.exp_id + '_' + out_name)
        print('saving results to', save_dir)
        json.dump(_to_list(copy.deepcopy(results)),
                  open(save_dir, 'w'))
    if opt.save_video and out is not None:
        out.release()
    sys.exit(0)

def _to_list(results):
    for img_id in results:
        for t in range(len(results[img_id])):
            for k in results[img_id][t]:
                if isinstance(results[img_id][t][k], (np.ndarray, np.float32)):
                    results[img_id][t][k] = results[img_id][t][k].tolist()
    return results

if __name__ == '__main__':
    data_root = "/home/caoxiaoyan/MOT_benchmark/BEE20_v2"
    seq_list = os.listdir(data_root)
    bar = tqdm(seq_list)
    for seq_name in bar:
        bar.set_description("Processing:%s" % seq_name)
        if "beev2" in seq_name:
            opt = opts().init()
            opt.demo = os.path.join(data_root, seq_name)
            demo(opt)


import boto3
import sys

def main(stack_name = 'MyFirstVPC',file_path = '../CloudFormation/VPC-NotNAT.yml'):
#def main(stack_name, file_path):
    print("Starting stack creation")
    # open the test file from the current diectory
    template = open(file_path).read()
    # create the boto3 cloudformation client
    cloudformation = boto3.client("cloudformation")
    stacks = cloudformation.describe_stacks(stack_name)
    print(stacks)
    # Validate if stack exists
    waiter = cloudformation.get_waiter('stack_exists')
    print(waiter)
    if 'StackExist' in str(waiter):
        print("StackName '" + stack_name + "' already exist, aborting...")
    else:
        # create the new stack
        cloudformation.create_stack(StackName=stack_name,TemplateBody=template)
        # create the new waiter
        waiter = cloudformation.get_waiter('stack_create_complete')
        # wait until the stack state changes to "CREATE_COMPLETE"
        #    waiter.wait(StackName=stack_name)
        print("StackName '" + stack_name + "' was created sucessfully")

if __name__ == "__main__":
    #stack_name = sys.argv[1]
    #file_path = sys.argv[2]
    #main(stack_name, file_path)
    main()


import random
import os
import colorama
from colorama import Fore, Back, Style

colorama.init(autoreset=True)

def clear_terminal():
    """
    Clear terminal function
    """
    os.system('cls' if os.name == 'nt' else 'clear')

def get_valid_name():
    """
    Welcome message and input player name and store it
    """
    print('***  Welcome to BlackJack ***')
    while True:
        name = input(
            Fore.GREEN
            + Style.BRIGHT
            + "Please enter your name (min 3 characters no spaces): ")
        if name.isalpha() and len(name) >= 3:
            return name
        print(
            Fore.RED
            + Style.BRIGHT
            + "Invalid input. Name must contain at least 3 letters.")

def main():
    """
    Menu options to view instructions or start the game
    """
    while True:
        print("Please choose an option:")
        print("1. View Instructions")
        print("2. Start Game")

        choice = input("Enter the number of your choice: ")

        if choice == "1":
            clear_terminal()
            print("Instructions: \n")
            print("1. The player and dealer are both dealt two cards")
            print("2. The player will get both cards face up")
            print("3. Dealer will have one card hidden")
            print("4. The goal of the game is to get as close"
                  "to 21 without going over")
            print("5. Player will be given the option to Hit"
                  "or Stand during your turn")
            print("6. Once you are satisfied with your score"
                  "and stand then its the dealers turn")
            print("7. The dealer must hit and draw a card on"
                  "16 but stand on 17")
            print("8. If the dealer goes over 21 he loses and you win")
            print("9. If both yourself and the dealer do not"
                  "exceed 21 then the closest to 21 wins")
            print("10. Game can end in a tie if both scores are the same")
            print("11. Scores will be reported at the end of the game \n")

        elif choice == "2":
            clear_terminal()
            play_blackjack()
            break
        else:
            print(
                Fore.RED
                + Style.BRIGHT
                + "Invalid choice. Please enter a valid option (1 or 2).")

def get_card_value(card):
    """
    Apply value to cards
    """
    if card in ['J', 'Q', 'K']:
        return 10
    elif card == 'A':
        return 11
    else:
        return int(card)

def get_hand_value(hand):
    """
    Apply hand value
    """
    value = sum(get_card_value(card) for card in hand)
    num_aces = hand.count('A')
    while value > 21 and num_aces > 0:
        value -= 10
        num_aces -= 1
    return value

def print_hand(hand):
    print(" ".join(hand), "(Value:", get_hand_value(hand), ")")

def play_blackjack():
    """
    Black jack game wins losses list set to 0
    """
    wins = 0
    losses = 0

    while True:
        # Shuffle of deck prior to dealing

        deck = ['2', '3', '4', '5', '6', '7', '8', '9', '10',
                'J', 'Q', 'K', 'A'] * 4
        random.shuffle(deck)

        # Dealing of 2 cards to player and dealer

        player_hand = []
        dealer_hand = []
        player_hand.append(deck.pop())
        dealer_hand.append(deck.pop())
        player_hand.append(deck.pop())
        dealer_hand.append(deck.pop())

        # Final round once player has decided to stand or hit then
        # the remaining dealer card is shown and scores can then
        # be calculated
        print(Fore.GREEN + Style.BRIGHT + f"{username}'s hand:")
        print_hand(player_hand)
        print(
            Fore.MAGENTA
            + Style.BRIGHT
            + "Dealer's hand:")
        print(dealer_hand[0], "X")
        while get_hand_value(player_hand) < 21:
            action = input("Do you want to hit (h) or stand (s)? ").lower()
            if action in ['h', 'hit']:
                player_hand.append(deck.pop())
                print(Fore.GREEN + Style.BRIGHT + f"{username}'s hand:")
                print_hand(player_hand)
            elif action in ['s', 'stand']:
                print(Fore.GREEN + Style.BRIGHT + f"{username} stands.")
                break
            else:
                print(
                    Fore.RED
                    + Style.BRIGHT
                    + "Invalid input. Please enter 'h' or 's'.")

        # If player hand greater than 17 then dealer takes another card
        player_value = get_hand_value(player_hand)
        while get_hand_value(dealer_hand) < 17:
            dealer_hand.append(deck.pop())

        print(Fore.MAGENTA + Style.BRIGHT + "\nDealer's hand:")
        print_hand(dealer_hand)
        # Calculation to determine winner of game
        dealer_value = get_hand_value(dealer_hand)
        if player_value == 21:
            print(
                Fore.GREEN
                + Style.BRIGHT
                + "***Blackjack*** Congratulations, you win!")
            wins += 1
        elif player_value > 21 or (dealer_value <= 21 and
                                   dealer_value > player_value):
            print(Fore.RED + Style.BRIGHT + "Sorry, you lose!")
            losses += 1
        elif dealer_value > 21 or player_value > dealer_value:
            print(Fore.GREEN + Style.BRIGHT + "Congratulations, you win!")
            wins += 1
        else:
            print("It's a tie!")
        # Play Again Request
        play_again = input('Do you want to play again (y/n)? ').lower()
        if play_again != 'y':
            clear_terminal()
            print(
                Fore.YELLOW
                + Style.BRIGHT
                + f"{username}, you won {wins} games and lost {losses} games.")
            print(
                Fore.CYAN
                + Style.BRIGHT
                + f'Thank you for playing {username}')
            break
        else:
            clear_terminal()

if __name__ == "__main__":
    username = get_valid_name()
    print(f"Welcome to your game of BlackJack {username}!")
    main()


# _*_ coding: utf-8 _*_

import os
from codecs import open
from setuptools import setup
from setuptools import find_packages

version_info = {}
current_dir = os.path.abspath(os.path.dirname(__file__))
print(current_dir)
with open(os.path.join(current_dir, 'patan', '__version__.py'), 'r', 'utf-8') as f:
    exec(f.read(), version_info)

setup(
    name='Patan',
    version=version_info['__version__'],
    description='A lightweight web crawling framework',
    project_urls={'Source': 'https://github.com/20perline/patan'},
    url='https://github.com/20perline/patan',
    author='20perline',
    author_email='tlhk2@163.com',
    license='MIT',
    packages=find_packages(),
    include_package_data=True,
    zip_safe=False,
    entry_points={'console_scripts': ['patan = patan.cli:main']},
    install_requires=['aiohttp>=3.6.2', 'glom>=20.5.0', 'click>=6.7', 'aiohttp-sse-client>=0.2.1'],
    python_requires='>=3.7',
)

# local development
# pip install --upgrade .


import re
import requests
import json
from datetime import datetime

try:
    import rb_exceptions
    import soup as soup_helper
except ImportError:  # No implicit package imports in py3.
    from ratebeer import rb_exceptions
    from ratebeer import soup as soup_helper

class Beer(object):
    """The Beer object. Contains information about an individual beer.

    Args:
        url (string): the URL of the beer you're looking for.

    Returns:
        abv (float): percentage alcohol
        brewery (Brewery object): the beer's brewery
        brewed_at (Brewery object): actual brewery if contract brewed
        calories (float): estimated calories for the beer
        description (string): the beer's description
        id (int): id of the beer
        img_url (string): a url to an image of the beer
        mean_rating (float): the mean rating for the beer (out of 5)
        name (string): the full name of the beer (may include the brewery name)
        num_ratings (int): the number of reviews*
        overall_rating (int): the overall rating (out of 100)
        seasonal (string): Summer, Winter, Autumn, Spring, Series, Special, None
        style (string): beer style
        style_url (string): beer style URL
        style_rating (int): rating of the beer within its style (out of 100)
        tags (list of strings): tags given to the beer
        url (string): the beer's url
        weighted_avg (float): the beer rating average, weighted using some unknown
            algorithm (out of 5)

        Any attributes not available will be returned as None

    """
    def __init__(self, url, fetch=None, id=None):
        """Initialize with URL and do not fetch"""
        self.url = url
        self.id = id
        if fetch is None:
            fetch = False
        if fetch:
            self._populate()
        self._has_fetched = fetch

    def __getattr__(self, item):
        """Return the value of the `attr` attribute."""
        if item in self.__dict__.keys():
            return self.__dict__[item]
        elif not self._has_fetched:
            self._populate()
            return getattr(self, item)
        raise AttributeError('{0} has no attribute {1}'.format(type(self), item))

    def __setattr__(self, name, value):
        """Set the `name` attribute to `value."""
        object.__setattr__(self, name, value)

    def __getstate__(self):
        """Provide state information for pickling"""
        result = self.__dict__.copy()
        return result

    def __setstate__(self, statedata):
        """Reset the state after pickling"""
        self.__dict__ = statedata

    def __repr__(self):
        """Unambiguous representation to recreate object"""
        return "<Beer('{0}')>".format(self.url)

    def __str__(self):
        """Provide a nicely formatted representation"""
        return self.name

    def _format(self, value):
        """Sets what is now a blank string or int to None, otherwise returns value"""
        if value == "" or value == 0:
            return None
        else:
            return value

    def _populate(self):
        if not self.id:
            self.id = self.url.split('/')[-2]

        data = [
                 {"operationName":"beer",
                  "variables":{"beerId":self.id},
                  "query":"query beer($beerId: ID!) { \n info: beer(id: $beerId) { \n id \n name \n description \n style { \n id \n name \n glasses { \n id \n name \n __typename \n } \n __typename \n } \n styleScore \n overallScore \n averageRating \n abv \n ibu \n calories \n brewer { \n id \n name \n __typename \n } \n ratingCount \n isRetired \n isUnrateable \n seasonal \n labels \n availability { \n bottle \n tap \n distribution \n __typename \n } \n __typename \n } \n} \n"},
                 # {"operationName":"beerReviews",
                 #  "variables":
                 #   {"beerId":self.id,
                 #    "order":"RECENT",
                 #    "first":10
                 #   },
                 #  "query":"query beerReviews($beerId: ID!, $authorId: ID, $order: ReviewOrder, $after: ID) { \n beerReviewsArr: beerReviews(beerId: $beerId, authorId: $authorId, order: $order, after: $after) { \n items { \n id \n comment \n score \n scores { \n appearance \n aroma \n flavor \n mouthfeel \n overall \n __typename \n } \n author { \n id \n username \n reviewCount \n __typename \n } \n checkin { \n id \n place { \n name \n city \n state { \n name \n __typename \n } \n country { \n name \n __typename \n } \n __typename \n } \n __typename \n } \n createdAt \n updatedAt \n __typename \n } \n totalCount \n last \n __typename \n } \n} \n"},
                 {"operationName":"beerByAlias",
                  "variables":{"aliasId":self.id},
                  "query":"query beerByAlias($aliasId: ID!) {\n beerByAlias(aliasId: $aliasId) {\n id\n name \n overallScore \n __typename \n } \n } \n"},
                 {"operationName":"tagDisplay",
                  "variables":{"beerId":self.id},
                  "query":"query tagDisplay($beerId: ID!, $first: Int) { \n tagDisplayArr: beerTags(beerId: $beerId, first: $first) { \n items { \n id \n urlName: plain \n __typename \n } \n __typename \n } \n} \n"
                 }
                ]

        request = requests.post(
            "https://beta.ratebeer.com/v1/api/graphql/"
           ,data=json.dumps(data)
           ,headers={"content-type": "application/json"}
        )

        try:
            results = json.loads(request.text)
        except:
            raise rb_exceptions.JSONParseException(self.id)

        beer_data = results[0]['data']['info']

        if beer_data == None:
            raise rb_exceptions.PageNotFound(self.id)

        alias_data = results[1]['data']['beerByAlias']

        if alias_data != None:
            raise rb_exceptions.AliasedBeer(self.id, alias_data['id'])

        tag_data = results[2]['data']['tagDisplayArr']['items']

        self.name = beer_data['name']
        self.brewery = Brewery('/brewers/{0}/{1}/'.format(re.sub('[/ ]','-',beer_data['brewer']['name'].lower()),beer_data['brewer']['id']))
        self.brewery.name = beer_data['brewer']['name']
        self.brewed_at = None #no longer supported
        self.overall_rating = self._format(beer_data['overallScore'])
        self.style_rating = self._format(beer_data['styleScore'])
        self.style = beer_data['style']['name']
        self.style_url = "/beerstyles/{0}/{1}/".format(re.sub('/','-',self.style.lower()), beer_data['style']['id'])
        self.img_url = "https://res.cloudinary.com/ratebeer/image/upload/w_152,h_309,c_pad,d_beer_img_default.png,f_auto/beer_{0}".format(self.id)
        self.num_ratings = self._format(beer_data['ratingCount'])
        self.mean_rating = self._format(beer_data['averageRating'])
        self.weighted_avg = None # does not appear to exist anymore
        if(beer_data['seasonal'] != 'UNKNOWN'):
            self.seasonal = beer_data['seasonal']
        else:
            self.seasonal = None
        self.ibu = self._format(beer_data['ibu'])
        self.calories = self._format(beer_data['calories'])
        self.abv = self._format(beer_data['abv'])
        self.retired = beer_data['isRetired']
        self.description = re.sub(r'\x92', '\'', beer_data['description'])
        if tag_data:
            self.tags = [t['urlName'] for t in tag_data]
        else:
            self.tags = None

        self._has_fetched = True

        return self

    def get_reviews(self, review_order="most recent"):
        """Returns reviews for a specific beer.

        Args:
            url (string): The specific url of the beer. Looks like:
                "/beer/deschutes-inversion-ipa/55610/"
            review_order (string): How to sort reviews. Three inputs:
                most recent: Newer reviews appear earlier.
                top raters: RateBeer.com top raters appear earlier.
                highest score: Reviews with the highest overall score appear
                earlier.

        Returns:
            A generator of dictionaries, containing the information about the review.
        """

        if not self._has_fetched:
            self._populate()

        review_order = review_order.lower()
        url_codes = {
            "most recent": 1,
            "top raters": 2,
            "highest score": 3
        }
        url_flag = url_codes.get(review_order)
        if not url_flag:
            raise ValueError("Invalid ``review_order``.")

        page_number = 1
        while True:
            complete_url = u'{0}{1}/{2}/'.format(self.url, url_flag, page_number)
            soup = soup_helper._get_soup(complete_url)
            content = soup.find('div', class_='reviews-container')
            reviews = content.find_all('div', style='padding: 0px 0px 0px 0px;')
            if len(reviews) < 1:
                raise StopIteration

            for review_soup in reviews:
                yield Review(review_soup)

            page_number += 1

class Review(object):
    """
    Args:
        review_soup (soup): the soup of the review

    Returns:
        appearance (int): rating for appearance (out of 5)
        aroma (int): aroma rating (out of 10)
        date (datetime): review date
        overall (int): overall rating (out of 20, for some reason)
        palate (int): palate rating (out of 5)
        rating (float): another overall rating provided in the review. Not sure how this different from overall.
        taste (int): taste rating (out of 10)
        text (string): actual text of the review.
        user_location (string): writer's location
        user_name (string): writer's username
    """

    def __init__(self, review_soup):
        # gets every second entry in a list
        review_title_attr = review_soup.find_all('div')[1].get('title')

        # some ratings may now just contain the x/5.0 rating, with no sub-ratings
        if '' in review_title_attr:
            raw_ratings = re.search(r'(.+?)', review_title_attr).group(1).split('
')
            # strip html and everything else
            for rating_text in raw_ratings:
                parts = rating_text.split(' ')
                # only set a rating if all of the information exists
                if rating_text:
                    label = parts[0]
                    rating_int = int(parts[1][:parts[1].find("/")])
                    setattr(
                        self,
                        label.lower().strip(),
                        rating_int
                    )
        self.rating = float(review_soup.find_all('div')[1].text)

        # get user information
        userinfo = review_soup.next_sibling
        self.text = userinfo.next_sibling.next_sibling.text.strip()
        self.user_name = re.findall(r'(.*?)\xa0\(\d*?\)', userinfo.a.text)[0]
        self.user_location = re.findall(r'-\s(.*?)\s-', userinfo.a.next_sibling)[0]

        # get date it was posted
        date = re.findall(r'-(?:\s.*?\s-)+\s(.*)', userinfo.a.next_sibling)[0]
        self.date = datetime.strptime(date.strip(), '%b %d, %Y').date()

    def __str__(self):
        """Provide a nicely formatted representation"""
        return self.text

class Brewery(object):
    def __init__(self, url, fetch=None):
        """Initialize with URL and do not fetch"""
        self.url = url
        if fetch is None:
            fetch = False
        if fetch:
            self._populate()
        self._has_fetched = fetch

    def __getattr__(self, item):
        """Return the value of the `attr` attribute."""
        if item in self.__dict__.keys():
            return self.__dict__[item]
        elif not self._has_fetched:
            self._populate()
            return getattr(self, item)
        raise AttributeError('{0} has no attribute {1}'.format(type(self), item))

    def __setattr__(self, name, value):
        """Set the `name` attribute to `value."""
        object.__setattr__(self, name, value)

    def __getstate__(self):
        """Provide state information for pickling"""
        result = self.__dict__.copy()
        return result

    def __setstate__(self, statedata):
        """Reset the state after pickling"""
        self.__dict__ = statedata

    def __repr__(self):
        """Unambiguous representation to recreate object"""
        return "<Brewery('{0}')>".format(self.url)

    def __str__(self):
        """Provide a nicely formatted representation"""
        return self.name

    def _populate(self):
        """Returns information about a specific brewery.

        Args:
            url (string): The specific url of the beer. Looks like:
                "/brewers/new-belgium-brewing-company/77/"

        Returns:
            A dictionary of attributes about that brewery."""

        soup = soup_helper._get_soup(self.url)
        s_contents = soup.find_all('div', {'itemtype':'http://schema.org/LocalBusiness'})
        if not s_contents:
            raise rb_exceptions.PageNotFound(self.url)

        self.name = soup.h1.text
        self.type = s_contents[0].find_all('div')[1].text.strip()
        website = s_contents[0].find_all('div',{'class':'media-links'})[0].find_all('a')[0]
        if website:
            self.web = website['href']
        self.telephone = Brewery._find_span(s_contents[0], 'telephone')
        self.street = Brewery._find_span(s_contents[0], 'streetAddress')
        self.city = Brewery._find_span(s_contents[0], 'addressLocality')
        self.state = Brewery._find_span(s_contents[0], 'addressRegion')
        self.country = Brewery._find_span(s_contents[0], 'addressCountry')
        self.postal_code = Brewery._find_span(s_contents[0], 'postalCode')
        self._has_fetched = True

        return self

    @staticmethod
    def _find_span(search_soup, item_prop):
        output = search_soup.find('span', attrs={'itemprop': item_prop})
        output = output.text.strip() if output else None
        return output

    def get_beers(self):
        """Generator that provides Beer objects for the brewery's beers"""
        if not self._has_fetched:
            self._populate()

        _id = self.url.split('/')[-2]
        complete_url = u'/Ratings/Beer/ShowBrewerBeers.asp?BrewerID={0}'.format(_id)
        soup = soup_helper._get_soup(complete_url)
        soup_beer_rows = soup.find('table', id='brewer-beer-table').findAll('tr')

        for row in soup_beer_rows[1:]:
            url = row.a.get('href')
            # Only return rows that are ratable
            if not row.find('a',title="Rate this beer"):
                continue
            # Remove any whitespace characters. Rare, but possible.
            url = re.sub(r"\s+", "", url, flags=re.UNICODE)
            beer = Beer(url)
            beer.name = row.a.text.strip()
            # Add attributes from row
            abv = row.findAll('td')[1].text
            weighted_avg = row.findAll('td')[4].text.strip()
            style_rating = row.findAll('td')[5].text.strip()
            num_ratings = row.findAll('td')[6].text.strip()
            if abv and abv != '-':
                beer.abv = float(abv)
            if weighted_avg:
                beer.weighted_avg = float(weighted_avg)
            if style_rating:
                beer.style_rating = int(style_rating)
            if num_ratings:
                beer.num_ratings = int(num_ratings)
            yield beer


from bs4 import BeautifulSoup
import numpy as np
from sklearn.decomposition import LatentDirichletAllocation as LDA
from sklearn.feature_extraction.text import CountVectorizer
import matplotlib.pyplot as plt
import pyLDAvis
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
import pyLDAvis.sklearn
from nltk.corpus import reuters
from nltk.corpus import stopwords
from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.feature_extraction.text import TfidfVectorizer
from collections import Counter

n_classes = 90
labels = reuters.categories()

def load_data(config={}):
    """
    Load the Reuters dataset.

    Returns
    -------
    Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
    """
    stop_words = stopwords.words("english")
    vectorizer = TfidfVectorizer(stop_words=stop_words)
    mlb = MultiLabelBinarizer()

    documents = reuters.fileids()
    test = [d for d in documents if d.startswith('test/')]
    train = [d for d in documents if d.startswith('training/')]

    docs = {}
    docs['train'] = [reuters.raw(doc_id) for doc_id in train]
    docs['test'] = [reuters.raw(doc_id) for doc_id in test]
    xs = {'train': [], 'test': []}
    xs['train'] = vectorizer.fit_transform(docs['train']).toarray()
    xs['test'] = vectorizer.transform(docs['test']).toarray()
    ys = {'train': [], 'test': []}
    ys['train'] = mlb.fit_transform([reuters.categories(doc_id)
                                     for doc_id in train])
    ys['test'] = mlb.transform([reuters.categories(doc_id)
                                for doc_id in test])
    data = {'x_train': xs['train'], 'y_train': ys['train'],
            'x_test': xs['test'], 'y_test': ys['test'],
            'labels': globals()["labels"]}
    return data

d = load_data()
X=d["x_train"]

lab = d["y_train"]

#changer le max_iter pour de meilleurs performances.
clf = LDA(n_components=n_classes,max_iter=10,topic_word_prior=0.9)
clf.fit(X)

#calcul de pureté du premier cluster
pred_cluster1=[]
for k in range(len(X)):
    pred = np.argmax(clf.transform([X[k]]))
    if(pred==1):
        pred_cluster1.append(k)
lab_clust1=[]
for p in pred_cluster1:
    lab_clust1.append(np.argmax(lab[p]))
c = Counter(lab_clust1)
somme = sum(c.values())
classe_majoritaire = np.max(list(c.values()))
print("Pureté de la classe numero 1 : ")
print(classe_majoritaire/somme)


from dataclasses import dataclass

from odd_collector.adapters.druid.domain.column_type import ColumnType
from odd_collector_sdk.errors import MappingDataError

@dataclass
class ColumnStats:
    name: str
    type_signature: str
    data_type: str
    has_multiple_values: bool
    has_nulls: bool
    size: int
    cardinality: int
    min_value: any
    max_value: any
    error_message: any
    type: ColumnType = ColumnType.dimension

    @staticmethod
    def from_response(column_name: str, record: dict):
        try:
            # Return
            return ColumnStats(
                type_signature=record.get("typeSignature", None),
                data_type=record.get("type", None),
                has_multiple_values=record.get("hasMultipleValues", None),
                has_nulls=record.get("hasNulls", None),
                size=record.get("size", None),
                cardinality=record.get("cardinality", None),
                min_value=record.get("minValue", None),
                max_value=record.get("maxValue", None),
                error_message=record.get("errorMessage", None),
                name=column_name,
            )
        except Exception as e:
            # Throw
            raise MappingDataError(
                "Couldn't transform Druid result to Column model"
            ) from e


import logging
import os
import subprocess
from typing import Dict, Sequence

from .color import Color
from .utils import colorful_print
from .utils import read_config

logger = logging.getLogger(__name__)

class Task:
    short_name: str
    full_name: str
    cmd: Sequence[str] = []
    tasks: Sequence['Task'] = []

class Runner:
    tasks: Dict[str, Task] = {}

    def __init__(self, config_file):
        self.__config_file = config_file
        self.__config_dir = os.path.dirname(config_file)
        self.__config = read_config(config_file)
        logger.debug('config: %s', self.__config)
        self.__config.setdefault('name', '')

        self.__flatten(self.__config, '')
        logger.debug('tasks: %s', self.tasks)

    def run(self, name: str = '') -> int:
        task = self.tasks.get(name) or \
               next((t for t in self.tasks.values() if name == t.short_name), None)

        if not task:
            available = {*self.tasks.keys()}
            raise Exception(f'Cannot find such a task: {name}, available: {available}')

        return self.__run_task(task)

    def __run_task(self, task: Task) -> int:
        name = task.short_name
        colorful_print('start to exec task(' + name + '):', Color.BLUE)

        if not task.tasks:
            cmd = task.cmd
            res = subprocess.run(
                cmd,
                shell=True,
                cwd=self.__config_dir
            )

            if res.returncode:
                colorful_print('some error occurred when execute task(' + name + ')', Color.RED)
            else:
                colorful_print('task(' + name + ') was executed successfully', Color.GREEN)

            return res.returncode
        else:
            for t in task.tasks:
                res = self.__run_task(t)
                if res:
                    return res

    def __flatten(self, task_obj, namespace: str):
        task = Task()
        task.short_name = task_obj['name']
        task.full_name = namespace + task_obj['name']
        task.cmd = task_obj.get('cmd', [])
        self.tasks[task.full_name] = task

        sub_task_obj = task_obj.get('tasks', [])
        if not sub_task_obj:
            return task

        next_ns = task.full_name + '.' if task.full_name else ''
        task.tasks = [self.__flatten(t, next_ns) for t in sub_task_obj]

        return task


import random
from collections import deque

from PIL import Image

def printGrid(grid, n: int, m: int):
    img = Image.new('1', (n, m))
    pixels = img.load()
    for i in range(img.size[0]):
        for j in range(img.size[1]):
            pixels[i, j] = grid[i][j]
    img.save('maze.png')

def makeGrid(width, height):
    newgrid = [[0 for x in range(height)] for y in range(width)]
    for i in range(len(newgrid)):
        for j in range(len(newgrid[i])):
            if i == 0 or j == 0 or i == len(newgrid) - 1 or j == len(newgrid[0]) - 1:
                newgrid[i][j] = 1
    return newgrid

def populateGrid(grid, chance):
    for i in range(len(grid)):  # reminder to test with: for index, value in enumerate(grid)
        for j in range(len(grid[0])):
            if (random.randint(0, 100) <= chance):  # test with list comprehension instead??
                grid[i][j] = 1
    return grid

def automataIteration(grid, minCount, makePillars):
    new_grid = [row[:] for row in grid]
    for i in range(1, len(grid) - 1):
        for j in range(1, len(grid[0]) - 1):
            count = 0
            for k in range(-1, 2):
                for l in range(-1, 2):
                    if grid[i + k][j + l] == 1:
                        count += 1
            if count >= minCount or (count == 0 and makePillars == 1):
                new_grid[i][j] = 1
            else:
                new_grid[i][j] = 0
    return new_grid

def floodFindEmpty(grid, tries, goal):
    times_remade = 0
    percentage = 0

    while times_remade < tries and percentage < goal:
        copy_grid = [row[:] for row in grid]
        open_count = 0
        times_remade += 1
        unvisited = deque([])
        new_grid = [[1 for x in range(len(grid[0]))] for y in range(len(grid))]
        # find a random empty space, hope it's the biggest cave
        randx = random.randint(0, len(grid) - 1)
        randy = random.randint(0, len(grid[0]) - 1)
        while (grid[randx][randy] == 1):
            randx = random.randint(0, len(grid) - 1)
            randy = random.randint(0, len(grid[0]) - 1)
        unvisited.append([randx, randy])
        while len(unvisited) > 0:
            current = unvisited.popleft()
            new_grid[current[0]][current[1]] = 0
            for k in range(-1, 2):
                for l in range(-1, 2):
                    if current[0] + k >= 0 and current[0] + k < len(grid) and current[1] + l >= 0 and current[
                        1] + l < len(grid[0]):  # if we're not out of bounds
                        if copy_grid[current[0] + k][current[1] + l] == 0:  # if it's an empty space
                            copy_grid[current[0] + k][current[1] + l] = 2  # mark visited
                            open_count += 1
                            unvisited.append([current[0] + k, current[1] + l])
        percentage = open_count * 100 / (len(grid) * len(grid[0]))

    return new_grid, percentage

def main():
    #width = int(input("Enter the width: "))
    #height = int(input("Enter the height: "))
    # chance = 100 - int(input("Enter the percentage chance of randomly generating a wall: "))
    # count = int(input("Enter the min count of surrounding walls for the automata rules: "))
    width = 100
    height = 100
    iterations = 2
    chance = 40
    count = 5
    #iterations = int(input("Enter the number of regular iterations: "))
    #pillarIterations = int(input("Enter the number of pillar-generating iterations: "))
    floodTries = 5
    goalPercentage = 30  # above 30% seems to be a good target

    grid = makeGrid(width, height)

    grid = populateGrid(grid, chance)

    for i in range(iterations):
        grid = automataIteration(grid, count, 0)

    grid, percentage = floodFindEmpty(grid, floodTries, goalPercentage)

    # self reminder to try checking map size https://stackoverflow.com/questions/1331471/in-memory-size-of-a-python-structure

    print("")
    printGrid(grid, width, height)

def generate(width: int, height: int, iterations: int):

    chance = 40
    count = 5
    floodTries = 5
    goalPercentage = 30  # above 30% seems to be a good target

    grid = makeGrid(width, height)

    grid = populateGrid(grid, chance)

    for i in range(iterations):
        grid = automataIteration(grid, count, 0)

    grid, percentage = floodFindEmpty(grid, floodTries, goalPercentage)

    printGrid(grid, width, height)

if __name__ == "__main__":
    main()


import datetime

from django.test import TestCase, override_settings
from django.utils import timezone

from tests import factories

@override_settings(CACHES={
    'default': {
        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
    }
})
class BaseApiTest(TestCase):
    @classmethod
    def setUpClass(cls):
        """
        setting up the following scenario
        ---------------------------------------------------
        |PROJ | USERS  |             TIME LOGS             |
        ---------------------------------------------------
            user_admin

        projA
            ├── userA1
            |     ├─────── 2022-02-01 11:00:00 - 2022-02-01 18:00 (7h)
            |     └─────── 2022-03-01 11:00:00 - None
            |
            └── userA2
                  └─────── 2022-02-01 11:00:00 - 2022-03-01 14:00 (3h)

        projB
            └── userB1
                  └─────── 2022-03-01 15:00:00 - None
        --------------
        """
        # create users
        cls.user_admin = factories.AdminFactory()
        cls.userA1 = factories.UserFactory(username='userA1')
        cls.userA2 = factories.UserFactory(username='userA2')
        cls.userB1 = factories.UserFactory(username='userB1')
        super(BaseApiTest, cls).setUpClass()

    @classmethod
    def tearDownClass(cls):
        super(BaseApiTest, cls).tearDownClass()

    def setUp(self):
        # assign users and create projects

        # projects
        self.projA = factories.ProjectFactory(name='projA', creator=self.user_admin)
        self.projB = factories.ProjectFactory(name='projB', creator=self.user_admin)

        # userA1 to projA
        self.userA1_projA = factories.ProjectAssignmentFactory(
            user=self.userA1,
            project=self.projA,
        )
        # userA2 to projA
        self.userA2_projA = factories.ProjectAssignmentFactory(
            user=self.userA2,
            project=self.projA,
        )
        # userB1 to projB
        self.userB1_projB = factories.ProjectAssignmentFactory(
            user=self.userB1,
            project=self.projB
        )

        # logs of userA1 to projA
        self.tl_userA1_projA_7h = factories.TimeLogCompleteFactory7h(
            project_assignment=self.userA1_projA
        )
        self.tl_userA1_projA_3h = factories.TimeLogOnlyStartFactory(
            project_assignment=self.userA1_projA
        )
        # logs of userA2 to projA
        self.tl_userA2_projA_partial = factories.TimeLogCompleteFactory3h(
            project_assignment=self.userA2_projA
        )
        # logs of userB1 to projB
        self.tl_userB1_projB_partial = factories.TimeLogOnlyStartFactory(
            project_assignment=self.userB1_projB,
            start_time=datetime.datetime(
                year=2022,
                month=3,
                day=1,
                hour=15,
                minute=00,
                tzinfo=timezone.utc
            )
        )

        super(BaseApiTest, self).setUp()

    def tearDown(self):
        super(BaseApiTest, self).tearDown()


# -*- coding: utf-8 -*-
"""
Created on Fri Apr 26 00:15:36 2019

@author: Manyue
"""

import numpy as np
import sys
from gym.envs.toy_text import discrete

UP = 0
RIGHT = 1
DOWN = 2
LEFT = 3

GOAL = 4
START = 20
SNAKE1 = 7
SNAKE2 = 17

eps = 0.25

class Robot_vs_snakes_world(discrete.DiscreteEnv):
    def __init__(self):
        self.shape = [5,5]
        nS = np.prod(self.shape)
        nA = 4
        MAX_Y = self.shape[0]
        MAX_X = self.shape[1]

        P = {}
        grid = np.arange(nS).reshape(self.shape)
        it = np.nditer(grid, flags = ['multi_index'])

        while not it.finished:
            s = it.iterindex
            y, x = it.multi_index
            P[s] = {a : [] for a in range(nA)}
            is_done = lambda s: s == GOAL

            if is_done(s):
                reward = 0.0
            elif s == SNAKE1 or s ==SNAKE2:
                reward = -15.0
            else:
                reward = -1.0

            if is_done(s):
                P[s][UP] = [(1.0, s, reward, True)]
                P[s][RIGHT] = [(1.0, s, reward, True)]
                P[s][DOWN] = [(1.0, s, reward, True)]
                P[s][LEFT] = [(1.0, s, reward, True)]
            else:
                ns_up = s if y==0 else s-MAX_X
                ns_right = s if x==(MAX_X - 1) else s+1
                ns_down = s if y==(MAX_Y - 1) else s+MAX_X
                ns_left = s if x==0 else s-1

                P[s][UP] = [(1 - (2*eps), ns_up, reward, is_done(ns_up)),
                 (eps, ns_right, reward, is_done(ns_right)),
                 (eps, ns_left, reward, is_done(ns_left))]

                P[s][RIGHT] = [(1 - (2*eps), ns_right, reward, is_done(ns_right)),
                 (eps, ns_up, reward, is_done(ns_up)),
                 (eps, ns_down, reward, is_done(ns_down))]

                P[s][DOWN] = [(1 - (2*eps), ns_down, reward, is_done(ns_down)),
                 (eps, ns_right, reward, is_done(ns_right)),
                 (eps, ns_left, reward, is_done(ns_left))]

                P[s][LEFT] = [(1 - (2*eps), ns_left, reward, is_done(ns_left)),
                 (eps, ns_up, reward, is_done(ns_up)),
                 (eps, ns_down, reward, is_done(ns_down))]

            it.iternext()
        isd = np.zeros(nS)
        isd[START] = 1.0

        self.P = P
        super(Robot_vs_snakes_world, self).__init__(nS, nA, P, isd)

    def _render(self):
        grid = np.arange(self.nS).reshape(self.shape)
        it = np.nditer(grid, flags = ['multi_index'])
        while not it.finished:
            s = it.iterindex
            y, x = it.multi_index

            if self.s == s:
                output = " R "
            elif s == GOAL:
                output = " G "
            elif s == SNAKE1 or s == SNAKE2:
                output = " S "
            else:
                output = " o "

            if x == 0:
                output = output.lstrip()
            if x == self.shape[1] - 1:
                output = output.rstrip()

            sys.stdout.write(output)

            if x == self.shape[1] - 1:
                sys.stdout.write("\n")

            it.iternext()

        sys.stdout.write("\n")

#env = Robot_vs_snakes_world()
#env.senv.step(DIR)
#env.p[state][action]

#This function is to return the value function, as well as the greedy policy function. Print out
#both the policy and the value function.
def value_iteration(env):

    V = np.zeros(env.nS)
    #..
    theta = 0.0001
    gama = 1
    def oneStep(state, V):
        Q_sa = np.zeros(env.nA)
        for i in range(env.nA):
            for next_S in env.P[state][i]:
                p, s_, r, _ = next_S
                Q_sa[i] += (p*(r+gama *V[s_]))
        return Q_sa

    while True:
        dta = 0
        for a in range(env.nS):
            MY = oneStep(a, V)
            bactval = np.max(MY)
            dta = max(dta,np.abs(bactval-V[a]))
            V[a] = bactval
        if dta < theta:
            break

    policy = np.zeros([env.nS, env.nA])
    for a in range(env.nS):
        MY = oneStep(a,V)
        bact = np.argmax(MY)
        policy[a,bact] = 1.0

    return policy, V

env = Robot_vs_snakes_world()
while env.s != GOAL:
    policy, V = value_iteration(env)
    env.step(np.argmax(policy[env.s]))
    env._render()


## Lenfest Biodiversity -IBA coverage
## coverage by Ecoregion
## coverage within MPAs by Ecoregion

import geopandas as gpd
import pandas as pd

## Ecoregion feature class
base_gdb =  r'V:\lenfest_nmsf_biodiversity\Data\lenfest_nmsf_base_prep.gdb'
ecoregion_base = gpd.read_file(f'{base_gdb}', layer='marine_ecoregions_final')
ecoregion_base["ER Area"] = (ecoregion_base["Shape_Area"] / (1000*1000))
ecoregion_base = ecoregion_base.drop(columns=["geometry","Shape_Length","Shape_Area"])

## CSVs to tables
all_output_csv = r'V:\lenfest_nmsf_biodiversity\Data\IBA\IBA_coverage_ecoregion.csv'
mpa_output_csv = r'V:\lenfest_nmsf_biodiversity\Data\IBA\IBA_coverage_MPA.csv'
fp_mpa_output_csv = r'V:\lenfest_nmsf_biodiversity\Data\IBA\IBA_coverage_FP_MPA.csv'

ecoregion_df = gpd.read_file(all_output_csv)
mpa_df = gpd.read_file(mpa_output_csv)
full_protect_mpa_df = gpd.read_file(fp_mpa_output_csv)

## rename area/percentage columns, drop columns
## drop columns and recalc below - checked that this matches the output from GPD intersect
ecoregion_df = ecoregion_df.apply(pd.to_numeric, errors='ignore')
ecoregion_df["IBA Area in ER"] = (ecoregion_df["iba_intersect_area"] / (1000*1000))
ecoregion_df = ecoregion_df.drop(columns=["field_1","geometry","NAME","Shape_Length_1","Shape_Area_1","Percent of Ecoregion Covered by IBA","iba_intersect_area"])

mpa_df = mpa_df.apply(pd.to_numeric, errors='ignore')
mpa_df["IBA Area in MPA"] = ( mpa_df["mpa_intersect_area"] / (1000*1000))
mpa_df = mpa_df.drop(columns=["field_1","geometry","NAME","Shape_Length_1","Shape_Area_1","mpa_intersect_area"])

full_protect_mpa_df = full_protect_mpa_df.apply(pd.to_numeric, errors='ignore')
full_protect_mpa_df["IBA Area in FP-MPA"] = (full_protect_mpa_df["fp_mpa_intersect_area"] / (1000*1000))
full_protect_mpa_df = full_protect_mpa_df.drop(columns=["field_1","geometry","NAME","Shape_Length_1","Shape_Area_1","fp_mpa_intersect_area"])

## join tables
summary_df = pd.merge(ecoregion_base, ecoregion_df, how="left", left_on="Ecoregion_ID", right_on="Ecoregion_ID")
summary_df = pd.merge(summary_df, mpa_df, how="left", left_on="Ecoregion_ID", right_on="Ecoregion_ID")
summary_df = pd.merge(summary_df, full_protect_mpa_df, how="left", left_on="Ecoregion_ID", right_on="Ecoregion_ID")

## calculate coverage of ecoregion feature in MPA networks
summary_df["PCT of ER Covered by IBA"] = (summary_df["IBA Area in ER"] / summary_df["ER Area"]) * 100
summary_df["PCT of ER IBA in MPA"] = (summary_df["IBA Area in MPA"] / summary_df["IBA Area in ER"]) * 100
summary_df["PCT of ER IBA in FP MPA"] = (summary_df["IBA Area in FP-MPA"] / summary_df["IBA Area in ER"]) * 100

## remove rows with no IBAs
summary_df = summary_df[summary_df['IBA Area in ER'].notna()]

## calculate summary row
summary_df = summary_df.append(
    {'NAME': 'All Ecoregions',
     'Ecoregion_ID': '0',
     'ER Area': summary_df["ER Area"].sum(),
     'IBA Area in ER': summary_df["IBA Area in ER"].sum(),
     'IBA Area in MPA': summary_df["IBA Area in MPA"].sum(),
     'IBA Area in FP-MPA': summary_df["IBA Area in FP-MPA"].sum(),
     'PCT of ER Covered by IBA': summary_df["IBA Area in ER"].sum() / summary_df["ER Area"].sum() * 100,
     'PCT of ER IBA in MPA': summary_df["IBA Area in MPA"].sum() / summary_df["IBA Area in ER"].sum() * 100,
     'PCT of ER IBA in FP MPA': summary_df["IBA Area in FP-MPA"].sum() / summary_df["IBA Area in ER"].sum() * 100
     },
    ignore_index = True)

## clean up table
summary_df.sort_values(by=['IBA Area in ER'], inplace=True, ascending=False)
summary_df.fillna("--", inplace=True)
summary_df.set_index('Ecoregion_ID', inplace=True)
pd.set_option('display.float_format', lambda x: '%.3f' % x)

## output to CSV
summary_df.to_csv(r'V:\lenfest_nmsf_biodiversity\Data\_Scripts\iba_coverage_ecoregion_MPA.csv')


from django.shortcuts import render
from .forms import SubscriberForm

from products.models import *

def landing(request):
    name = "CodingMedved"
    current_day = "03.01.2017"
    form = SubscriberForm(request.POST or None)
    if request.method =='POST' and form.is_valid():

        data = form.cleaned_data
        new_form = form.save()

    return render(request, 'landing.html', locals())

def home(request):
    products_images = ProductImage.objects.filter(is_active=True, is_main=False, product__is_active=True)
    products_images_second = products_images.filter(product__category__id=1)
    products_images_third = products_images.filter(product__category__id=2)
    return render(request, 'home.html', locals())


from contextlib import contextmanager
import logging
import sys
from threading import Lock
from time import sleep
from typing import Any, Iterator, Optional, Tuple, Union

import click
from flask import Flask, jsonify, request
from seleniumwire import webdriver # type: ignore
from selenium.common.exceptions import TimeoutException, InvalidArgumentException, UnexpectedAlertPresentException
from selenium.webdriver.firefox.options import Options
from werkzeug.wrappers import Response

View = Union[Response, str, Tuple[str, int]]

log = logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logging.getLogger('werkzeug').disabled = True
logging.getLogger('seleniumwire.storage').disabled = True
logging.getLogger('seleniumwire.backend').disabled = True
logging.getLogger('seleniumwire.handler').disabled = True
logging.getLogger('seleniumwire.server').disabled = True

# Set up app
app = Flask(__name__)

class DriverPool:
    lock = Lock()
    retries = 4
    retry_wait = .5

    def __init__(self, n_drivers: int, headless: bool = True) -> None:
        options = Options()
        options.headless = headless
        options.set_preference('dom.webnotifications.enabled', False)
        self.drivers = [webdriver.Firefox(options=options) for _ in range(n_drivers)]

    @contextmanager
    def _get_free_driver(self) -> Iterator[Optional[webdriver.Firefox]]:
        for _ in range(self.retries):
            try:
                driver = self.drivers.pop()
                break
            except IndexError:
                sleep(self.retry_wait)
        else:
            driver = None

        try:
            yield driver
        finally:
            if driver is not None:
                self.drivers.append(driver)

    def get(self, url: str, identikey: str) -> str:
        with self._get_free_driver() as driver:
            if driver is None:
                return 'busy'

            # Load for 6 seconds at max. Why 6? Made it up. Everything should be <10ms, unless
            # someone put an infinite redirect in their XSS or something.
            driver.set_page_load_timeout(6)
            try:
                # Set the StudentIdentikey header for all outgoing requests
                def interceptor(request: Any) -> None:
                    request.headers['StudentIdentikey'] = identikey

                driver.request_interceptor = interceptor
                # Get the URL. This might fail if the previous student has an alert popup open, so
                # we catch that error and retry until it works. I don't think this is possible to
                # loop infinitely, but maybe. Hasn't happened yet.
                while True:
                    try:
                        driver.get(url)
                        break
                    except UnexpectedAlertPresentException:
                        pass
            except InvalidArgumentException:
                log.exception('Bad url')
                return 'The URL was invalid'
            except TimeoutException:
                log.exception('Timeout')
                driver.get('')
                return 'timeout'

            return 'success'

drivers: DriverPool = None # type: ignore

@app.route('/')
def index() -> View:
    return 'ok'

@app.route('/visit')
def visit() -> View:
    url = request.json.get('url') # type: ignore
    identikey = request.json.get('identikey') # type: ignore

    if not url or not identikey:
        log.error('Did not get URL or identikey')

        # This should never happen unless there is a bug somewhere
        return jsonify({ 'status': 'internal error (reach out to instructor)' }) # type: ignore

    log.info('{} requested URL: {}'.format(identikey, url))
    status = drivers.get(url, identikey)

    return jsonify({ 'status': status }) # type: ignore

@click.command()
@click.option('--debug', is_flag=True)
@click.option('--port', type=int, default=8082)
def main(debug: bool, port: int) -> None:
    global drivers
    if debug:
        # When running locally, a single browser with the UI enabled is fine
        log.setLevel(logging.DEBUG)
        drivers = DriverPool(1, headless=False)
    else:
        # When running in prod, use 5 headless browsers for concurrency
        drivers = DriverPool(5)

    log.info('Starting support server on http://127.0.0.1:{}'.format(port))

    app.run(threaded=True, debug=debug, port=port)

if __name__ == '__main__':
    main()