import torch
import torch.nn as nn

import torchvision
from torchvision import transforms
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor

import string
import cv2
import json
from PIL import Image
import os
import numpy as np
import random
from tqdm import tqdm
import shutil

EPOCHS = 1000
IMAGE_PATH = 'data'
BEST_MODEL_DIR = 'model'
IMAGE_SIZE = (600, 600)
BATCH_SIZE = 2
LR = 1e-3

if torch.cuda.is_available():
    device = 'cuda'
else:
    device = 'cuda'

def construct_dict(x): return {k: [] for k in range(1, x + 1)}

def images_with_annotations(image_path, annotations_path):
    with open(annotations_path, 'r') as fp:
        content = json.load(fp)

    class_names = {category['id']: category['name']
                   for category in content['categories']}
    class_names[0] = 'background'

    images = {image['id']: image['file_name'] for image in content['images']}

    for key, val in images.items():
        image = Image.open(os.path.join(image_path, val))
        images[key] = image
    img_annots = {k: [] for k in range(1, len(images) + 1)}

    for annot in content['annotations']:
        img_annots[annot['image_id']].append(annot)

    segmentations = {k: [] for k in range(1, len(images) + 1)}
    categories = {k: [] for k in range(1, len(images) + 1)}
    for val in img_annots.values():
        for annot in val:
            categories[annot['image_id']].append(annot['category_id'])
            segmentation = annot['segmentation'][0]
            segmentations[annot['image_id']].append(
                [[segmentation[i], segmentation[i + 1]] for i in range(0, len(segmentation) - 1, 2)])

    return images, segmentations, categories, class_names

def construct_masks(images, segmentations):
    masks = {}
    for key, val in images.items():
        tmp = []
        for segmentation in segmentations[key]:
            mask = np.zeros((val.size[1], val.size[0]), dtype=np.uint8)
            cv2.fillPoly(mask, np.array(
                [segmentation], dtype=np.int32), 255)  # type: ignore
            tmp.append(mask)
        masks[key] = tmp
    return masks

def display_masks_images(images, targets):
    for i in range(len(images)):
        image = np.array(images[i].permute(1, 2, 0).cpu().numpy() * 255, dtype=np.uint8)
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        masks = targets[i]['masks'].cpu().numpy()
        boxes = targets[i]['boxes'].cpu().numpy()
        for mask in masks:
            alpha = 0.5
            image[mask == 1] = (1 - alpha) * image[mask == 1] + alpha * np.array([0, 255, 0])
        for box in boxes:
            x1, y1, x2, y2 = np.array(box, dtype=np.int32)
            cv2.rectangle(image, (x1, y1), (x2, y2), (0, 255, 0), 2)
        cv2.imshow('image', image)
        cv2.waitKey(0)
        cv2.destroyAllWindows()

def resize_images(images, masks, size):
    for key, val in images.items():
        images[key] = val.resize(size)

        tmp = []
        for mask in masks[key]:
            mask = cv2.resize(mask, size)
            tmp.append(mask)
        masks[key] = tmp

        images[key] = cv2.cvtColor(np.array(images[key]), cv2.COLOR_BGR2RGB)
    return images, masks

def augment_data(images, masks, categories, class_names):
    targets = {}

    for k, v in masks.items():
        data = {}
        tmp = []
        li = []
        for mask in v:
            tmp_mask = (mask > 0).astype(np.uint8)
            x, y, w, h = cv2.boundingRect(tmp_mask)
            if h == 0 or w == 0:
                continue

            li.append([x, y, x + w, y + h])
            tmp.append(tmp_mask)
        data['masks'] = torch.tensor(tmp, dtype=torch.uint8)
        data['boxes'] = torch.tensor(li, dtype=torch.float32)
        data['labels'] = torch.tensor(
            [c for c in categories[k]], dtype=torch.int64)
        targets[k] = data

    return images, targets

def _process_image(images, size):
    # resize images if not correct size
    imgs = [cv2.resize(img, size) for img in images]

    # process images
    imgs = torch.stack([torch.tensor(img, dtype=torch.float32)
                       for img in images], 0)
    imgs = imgs.permute(0, 3, 1, 2)
    imgs = imgs / 255.0
    # imgs = imgs - torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)
    # imgs = imgs / torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)
    return imgs

def _augment_training_data(images, targets):
    res_images = []
    res_targets = []
    hflip_transform = transforms.RandomHorizontalFlip(p=0.5)

    for i in range(len(images)):
        image = images[i].permute(1, 2, 0).cpu().numpy()
        masks = targets[i]['masks'].cpu().numpy()

        # random color jitter
        image_pil = Image.fromarray(image.astype(np.uint8))
        image_pil = transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.5)(image_pil)

        # random horizontal flip
        if random.random() > 0.5:
            image_pil = hflip_transform(image_pil)
            image = np.array(image_pil)
            masks = [cv2.flip(mask, 1) for mask in masks]
            boxes = []
            for mask in masks:
                x, y, w, h = cv2.boundingRect(mask)
                if h == 0 or w == 0:
                    continue
                boxes.append([x, y, x + w, y + h])
        else:
            image = np.array(image_pil)
            boxes = targets[i]['boxes'].cpu().numpy()

        res_images.append(torch.tensor(image, dtype=torch.float32).permute(2, 0, 1))
        res_targets.append({'masks': torch.tensor(masks, dtype=torch.uint8), 'boxes': torch.tensor(boxes, dtype=torch.float32), 'labels': targets[i]['labels']})

    return res_images, res_targets

class Data:

    def __init__(self, image_path, size=(600, 600), batch_size=2):
        self.train_images, self.train_targets = self._data_pipeline(
            image_path, os.path.join(image_path, 'train_annotations.json'), size)
        self.test_images, self.test_targets = self._data_pipeline(
            image_path, os.path.join(image_path, 'test_annotations.json'), size, train=False)
        self.batch_size = batch_size

    def _data_pipeline(self, image_path, annotations_path, size, train=True):
        images, segmentations, categories, class_names = images_with_annotations(
            image_path, annotations_path)
        masks = construct_masks(images, segmentations)
        images, masks = resize_images(images, masks, size)
        images, targets = augment_data(images, masks, categories, class_names)
        images, targets = list(images.values()), list(targets.values())

        images = _process_image(images, size)
        # images = list(image.to(device) for image in images)
        # targets = [{k: v.to(device) for k,v in t.items()} for t in targets]
        if train:
            images, targets = _augment_training_data(images, targets)
        self.num_classes = len(class_names)
        return images, targets

    def train_generator(self):
        return Generator(self.train_images, self.train_targets, self.batch_size)

    def test_generator(self):
        return Generator(self.test_images, self.test_targets, self.batch_size)

class Generator:

    def __init__(self, data, targets, batch_size):
        self.data = data
        self.targets = targets

        array = list(range(len(self.data)))
        random.shuffle(array)
        self.data = [self.data[i] for i in array]
        self.targets = [self.targets[i] for i in array]

        self.batch_size = batch_size

    def __len__(self):
        return len(self.data) // self.batch_size

    def __getitem__(self, idx):
        if len(self.data) % self.batch_size != 0 and idx == len(self) - 1:
            return self.data[idx * self.batch_size:], self.targets[idx * self.batch_size:]
        return self.data[idx * self.batch_size: (idx + 1) * self.batch_size], self.targets[idx * self.batch_size: (idx + 1) * self.batch_size]

    def __iter__(self):
        for i in range(len(self)):
            yield self[i]

class Model(nn.Module):

    def __init__(self, num_classes, pretrained=True):
        super(Model, self).__init__()
        self.model = torchvision.models.detection.maskrcnn_resnet50_fpn(
            pretrained=pretrained)
        in_features = self.model.roi_heads.box_predictor.cls_score.in_features  # type: ignore
        self.model.roi_heads.box_predictor = FastRCNNPredictor(
            in_features, num_classes)

    def forward(self, x, y):
        return self.model(x, y)

    def predict(self, image):
        # resize the image
        image = cv2.resize(image, (600, 600))

        # convert image to tensor
        image = torch.tensor(image, dtype=torch.float32)

        # permute image
        image = image.permute(2, 0, 1)

        # normalize image
        image = image / 255.0
        image = image - torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)
        image = image / torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)

        # add batch dimension
        image = image.unsqueeze(0)

        # predict
        self.model.eval()
        with torch.no_grad():
            return self.model(image)

class Trainer:

    def __init__(self, model, optimizer, epochs=EPOCHS):
        self.model = model
        self.optimizer = optimizer
        self.epochs = epochs

    def train(self, epoch, train_loader, verbose=True):
        pbar = tqdm(train_loader, total=len(train_loader))
        for x, y in pbar:
            self.model.train()
            self.optimizer.zero_grad()
            loss_dict = self.model(x, y)
            losses = sum(loss for loss in loss_dict.values())

            losses.backward()  # type: ignore
            self.optimizer.step()
        if verbose:
            print(f'Train: Epoch={epoch} | Loss={losses}')  # type: ignore
        return losses  # type: ignore

    def test(self, epoch, test_loader, verbose=True):
        pbar = tqdm(test_loader, total=len(test_loader))
        total_loss = 0

        for x, y in pbar:
            with torch.no_grad():
                loss_dict = self.model(x, y)
                losses = sum(loss for loss in loss_dict.values())
                total_loss += losses

        if verbose:
            print(f'Test: Epoch={epoch} | Loss={total_loss/len(test_loader)}')
        return total_loss/len(test_loader)

class Helper:

    @staticmethod
    def save_model(model, path):
        os.makedirs(os.path.dirname(path), exist_ok=True)
        torch.save(model.state_dict(), path)

    @staticmethod
    def delete_files_in_path(path):
        if os.path.exists(path):
            for file in os.listdir(path):
                os.remove(os.path.join(path, file))

    @staticmethod
    def get_best_loss():
        if os.path.exists(BEST_MODEL_DIR) and len(os.listdir(BEST_MODEL_DIR)) > 0:
            return float(os.listdir(BEST_MODEL_DIR)[0].split('--loss-')[1].split('.pth')[0])
        else:
            return float('inf')

    @staticmethod
    def early_stop(test_losses, patience=30):
        if len(test_losses) < patience:
            return False
        else:
            return test_losses.index(min(test_losses)) < len(test_losses) - patience - 1

    @staticmethod
    def load_model(model, path):
        model.load_state_dict(torch.load(path))
        return model

    @staticmethod
    def infer(model, image):
        original_size = image.shape[-2:]
        image_copy = image.copy()
        image = cv2.resize(image, IMAGE_SIZE)
        model.eval()
        with torch.no_grad():
            out = model.predict(image)
        # display image
        print(out)
        boxes = out[0]['boxes'].detach().cpu().numpy()
        scores = out[0]['scores'].detach().cpu().numpy()
        labels = out[0]['labels'].detach().cpu().numpy()
        masks = out[0]['masks'].detach().cpu().numpy()
        for box, score, label, mask in zip(boxes, scores, labels, masks):
            if score > 0.5:
                box = box.astype(np.int32)
                mask = cv2.resize(mask, original_size)
                mask = (mask > 0.5).astype(np.uint8)
                color = np.random.randint(0, 255, (1, 3), dtype=np.uint8)
                color = tuple([int(c) for c in color[0]])

                image_copy = Helper.apply_mask(
                    image_copy, mask, color, alpha=0.5)
                image_copy = cv2.rectangle(
                    image_copy, (box[0], box[1]), (box[2], box[3]), (255, 0, 0), 2)
                image_copy = cv2.putText(image_copy, str(
                    label), (box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2)
        cv2.imshow('image', image_copy)
        cv2.waitKey(0)
        cv2.destroyAllWindows()

    @staticmethod
    def apply_mask(image, mask, color, alpha):
        """Apply the given mask to the image.
        """
        for c in range(3):
            image[:, :, c] = np.where(mask == 1,
                                      image[:, :, c] *
                                      (1 - alpha) + alpha * color[c],
                                      image[:, :, c])
        return image

class DataHelper:
    @staticmethod
    def copy_missing_annotation_images(image_dir, annotation_path):
        file1 = json.load(open(annotation_path, 'r'))
        images = file1['images']
        images = [d['file_name'] for d in images]
        local_images = os.listdir(image_dir)
        for image in local_images:
            if image not in images:
                os.makedirs('missing_images', exist_ok=True)
                shutil.copy(os.path.join(image_dir, image),
                            os.path.join('missing_images', image))

    @staticmethod
    def handle_missing_category_ids(path1, id=1):
        file1 = json.load(open(path1, 'r'))

        annotations = file1['annotations']
        for annotation in annotations:
            if 'category_id' not in annotation:
                annotation['category_id'] = id

        return file1

    @staticmethod
    def handle_file_names(dir):
        for file in os.listdir(dir):
            if not file.endswith('.json'):
                random_name = ''.join(random.choices(
                    string.ascii_uppercase + string.digits, k=10))
                file_extension = file.split('.')[-1]
                os.rename(os.path.join(dir, file), os.path.join(
                    dir, random_name + '.' + file_extension))

    @staticmethod
    def combine_annotations(path1, path2):
        file1 = json.load(open(path1, 'r'))
        file2 = json.load(open(path2, 'r'))

        res_info = file1['info']

        images1 = file1['images']
        images2 = file2['images']
        images2_copy = [d.copy() for d in images2]
        initial_file_names = [d['file_name'] for d in images1]
        last_image_id = images1[-1]['id']
        images2 = [d for d in images2 if d['file_name']
                   not in initial_file_names]
        for image in images2:
            image['id'] += last_image_id
        res_images = images1 + images2
        image_conversions = {}
        for image in images2_copy:
            res_image_id = [
                img for img in res_images if img['file_name'] == image['file_name']][0]['id']
            image_conversions[image['id']] = res_image_id

        category1 = file1['categories']
        category2 = file2['categories']
        category2_copy = [d.copy() for d in category2]
        last_category_id = category1[-1]['id']
        initial_category_names = [d['name'] for d in category1]
        category2 = [d for d in category2 if d['name']
                     not in initial_category_names]
        for category in category2:
            category['id'] += last_category_id
        res_categories = category1 + category2
        category_conversions = {}
        for category in category2_copy:
            res_category_id = [
                ctg for ctg in res_categories if ctg['name'] == category['name']][0]['id']
            category_conversions[category['id']] = res_category_id

        annotation1 = file1['annotations']
        annotation2 = file2['annotations']
        last_annotation_id = annotation1[-1]['id']
        for annotation in annotation2:
            annotation['id'] += last_annotation_id
            annotation['image_id'] = image_conversions[annotation['image_id']]
            annotation['category_id'] = category_conversions[annotation['category_id']]
        res_annotations = annotation1 + annotation2

        return {'info': res_info, 'images': res_images, 'categories': res_categories, 'annotations': res_annotations}

if __name__ == '__main__':
    data = Data(image_path=IMAGE_PATH, size=IMAGE_SIZE, batch_size=BATCH_SIZE)
    train_loader = data.train_generator()
    test_loader = data.test_generator()

    model = Model(num_classes=data.num_classes)
    # model = model.to(device)
    optimizer = torch.optim.Adam(model.parameters(), lr=LR)
    trainer = Trainer(model, optimizer)
    test_losses = []
    best_loss = Helper.get_best_loss()

    for epoch in range(EPOCHS):
        trainer.train(epoch, train_loader)
        loss = trainer.test(epoch, test_loader)

        test_losses.append(loss)
        if loss < best_loss:
            Helper.delete_files_in_path(BEST_MODEL_DIR)
            Helper.save_model(
                model, rf'{BEST_MODEL_DIR}/epoch-{epoch+1}--loss-{loss:.3f}.pth')
            best_loss = loss

        if Helper.early_stop(test_losses):
            print(f'Early Stopping at epoch {epoch + 1}')
            break

    # best_model_path = os.path.join(
    #     BEST_MODEL_DIR, os.listdir(BEST_MODEL_DIR)[0])
    # model = Helper.load_model(model, best_model_path)
    # model = torchvision.models.detection.maskrcnn_resnet50_fpn(2)
    # model.eval()
    # image = cv2.imread('test.jpeg')
    # image = cv2.resize(image, (224, 224))
    # image = (torch.tensor(image) / 255.0).permute(2, 0, 1).float()
    # with torch.no_grad():
    #     output = model([image])
    # print(output)
    # Helper.infer(model, cv2.imread('test.jpeg'))


from astropy.table import Table
from helper_functions import luptize_deep_kids
import pandas as pd
import numpy as np
import fitsio
import os
import sys

path = os.path.abspath(sys.path[0])

path_data = f"{path}/../Data/skills_v07D7p1_LF_321_kidsPhotometry_everything_col_flag_shear_m283m283_rot_0.fits"
path_survey_conditions = f"{path}/../Data/noise_selec_combined.csv"
path_save = f"{path}/../Data/kids_training_catalog_lup.pkl"
path_save_small = f"{path}/../Data/kids_training_catalog_lup_small.pkl"

col_kids = [
    "tile_label",
    "axis_ratio_input",
    "Re_input",
    "sersic_n_input",
    "u_input",
    "g_input",
    "r_input",
    "i_input",
    "Z_input",
    "Y_input",
    "J_input",
    "H_input",
    "Ks_input",
]

col_output = [
    "FLUX_GAAP_u",
    "FLUX_GAAP_g",
    "FLUX_GAAP_r",
    "FLUX_GAAP_i",
    "FLUX_GAAP_Z",
    "FLUX_GAAP_Y",
    "FLUX_GAAP_J",
    "FLUX_GAAP_H",
    "FLUX_GAAP_Ks",
    "FLUXERR_GAAP_u",
    "FLUXERR_GAAP_g",
    "FLUXERR_GAAP_r",
    "FLUXERR_GAAP_i",
    "FLUXERR_GAAP_Z",
    "FLUXERR_GAAP_Y",
    "FLUXERR_GAAP_J",
    "FLUXERR_GAAP_H",
    "FLUXERR_GAAP_Ks",
    "FLUX_AUTO",
    "FLUXERR_AUTO",
]

col_survey_cond = [
    "label",
    "InputSeeing_u",
    "InputSeeing_g",
    "InputSeeing_r",
    "InputSeeing_i",
    "InputSeeing_Z",
    "InputSeeing_Y",
    "InputSeeing_J",
    "InputSeeing_H",
    "InputSeeing_Ks",
    "InputBeta_u",
    "InputBeta_g",
    "InputBeta_r",
    "InputBeta_i",
    "InputBeta_Z",
    "InputBeta_Y",
    "InputBeta_J",
    "InputBeta_H",
    "InputBeta_Ks",
    "rmsAW_u",
    "rmsAW_g",
    "rmsAW_r",
    "rmsAW_i",
    "rms_Z",
    "rms_Y",
    "rms_J",
    "rms_H",
    "rms_Ks",
]

kids_data = Table(fitsio.read(path_data))
df_kids_data = pd.DataFrame()
for i, col in enumerate(col_kids+col_output):
    print(i, col)
    df_kids_data[col] = kids_data[col]
df_kids_data = df_kids_data.rename(columns={'tile_label': 'label'})
print(f'Length of deep field catalog: {len(df_kids_data)}')

df_survey_cond_all = pd.read_csv(path_survey_conditions)
df_survey_cond_needed = df_survey_cond_all[col_survey_cond]
print(df_survey_cond_needed)

print('Merging catalogs...')
print('Merging survey catalog and kids on label => df_merged')
df_merged = pd.merge(df_kids_data, df_survey_cond_needed, on='label', how="left")
print(f'Length of merged catalog: {len(df_merged)}')
print(df_merged.isnull().sum())

lst_flux_2_lupt = [
    "FLUX_GAAP_u",
    "FLUX_GAAP_g",
    "FLUX_GAAP_r",
    "FLUX_GAAP_i",
    "FLUX_GAAP_Z",
    "FLUX_GAAP_Y",
    "FLUX_GAAP_J",
    "FLUX_GAAP_H",
    "FLUX_GAAP_Ks"
]

lst_fluxerr_2_lupt = [
    "FLUXERR_GAAP_u",
    "FLUXERR_GAAP_g",
    "FLUXERR_GAAP_r",
    "FLUXERR_GAAP_i",
    "FLUXERR_GAAP_Z",
    "FLUXERR_GAAP_Y",
    "FLUXERR_GAAP_J",
    "FLUXERR_GAAP_H",
    "FLUXERR_GAAP_Ks"
]

lst_bins = [
    "u",
    "g",
    "r",
    "i",
    "Z",
    "Y",
    "J",
    "H",
    "Ks",
]

arr_lup = luptize_deep_kids(np.array(df_merged[lst_flux_2_lupt]), bins=lst_bins)[0]
arr_lup_err = luptize_deep_kids(np.array(df_merged[lst_fluxerr_2_lupt]), bins=lst_bins)[0]

lst_luptize = [
    "luptize_u",
    "luptize_g",
    "luptize_r",
    "luptize_i",
    "luptize_Z",
    "luptize_Y",
    "luptize_J",
    "luptize_H",
    "luptize_Ks",
]

lst_luptize_err = [
    "luptize_err_u",
    "luptize_err_g",
    "luptize_err_r",
    "luptize_err_i",
    "luptize_err_Z",
    "luptize_err_Y",
    "luptize_err_J",
    "luptize_err_H",
    "luptize_err_Ks",
]

for idx_lup, lup in enumerate(lst_luptize):
    df_merged[lup] = arr_lup[:, idx_lup]

for idx_lup_err, lup_err in enumerate(lst_luptize_err):
    df_merged[lup_err] = arr_lup_err[:, idx_lup_err]

df_merged_small = df_merged.sample(n=250000)

df_merged.to_pickle(path_save)
df_merged_small.to_pickle(path_save_small)


from typing import Dict

from kedro.pipeline import Pipeline
from mridle.pipelines.data_engineering import ris, dicom, dispo
from mridle.pipelines.data_science import harvey, feature_engineering, descriptive_viz, random_forest, xgboost, \
    logistic_regression, neural_net, model_comparison, live_data, xgboost_with_live

def register_pipelines() -> Dict[str, Pipeline]:
    """Register the project's pipelines.

    Returns:
        A mapping from a pipeline name to a ``Pipeline`` object.
    """

    ris_pipeline = ris.create_pipeline()
    dicom_pipeline = dicom.create_pipeline()
    dispo_pipeline = dispo.create_pipeline()
    descriptive_viz_pipeline = descriptive_viz.create_pipeline()
    feature_engineering_pipeline = feature_engineering.create_pipeline()
    live_data_pipeline = live_data.create_pipeline()
    harvey_pipeline = harvey.create_pipeline()
    logistic_regression_pipeline = logistic_regression.create_pipeline()
    random_forest_pipeline = random_forest.create_pipeline()
    xgboost_pipeline = xgboost.create_pipeline()
    xgboost_with_live_pipeline = xgboost_with_live.create_pipeline()
    neural_net_pipeline = neural_net.create_pipeline()
    model_comparison_pipeline = model_comparison.create_pipeline()

    return {

        "__default__": ris_pipeline + feature_engineering_pipeline + harvey_pipeline + logistic_regression_pipeline +
        random_forest_pipeline + xgboost_pipeline + model_comparison_pipeline,
        "all": ris_pipeline + dicom_pipeline + dispo_pipeline + descriptive_viz_pipeline +
        feature_engineering_pipeline + harvey_pipeline + logistic_regression_pipeline +
        random_forest_pipeline + xgboost_pipeline + neural_net_pipeline + model_comparison_pipeline,
        "ris": ris_pipeline,
        "dicom": dicom_pipeline,
        "dispo": dispo_pipeline,
        "descriptive_viz": descriptive_viz_pipeline,
        "feature_engineering": feature_engineering_pipeline,
        "live_data": live_data_pipeline,
        "harvey": harvey_pipeline,
        "logistic_regression": logistic_regression_pipeline,
        "random_forest": random_forest_pipeline,
        "xgboost": xgboost_pipeline,
        "xgboost_with_live": xgboost_with_live_pipeline,
        "neural_net": neural_net_pipeline,
        "model_comparison": model_comparison_pipeline,
        "models": harvey_pipeline + logistic_regression_pipeline + random_forest_pipeline + xgboost_pipeline
    }


from flask import make_response, redirect, jsonify, render_template, request, url_for, Response

from jots.webapp import app, jwt

class InvalidUsage(Exception):
  def __init__(self, message, status_code=400, payload=None):
    Exception.__init__(self)
    self.message = message
    self.status_code = status_code
    self.payload = payload

  def to_dict(self):
    rv = dict(self.payload or ())
    rv['message'] = self.message
    return rv

@app.errorhandler(InvalidUsage)
def handle_invalid_usage(error):
  return render_template("error.tmpl",
                         error_message=error.message,
                         error_code=error.status_code,
                         error_data=error.payload), int(error.status_code)

class InvalidAPIUsage(Exception):
  def __init__(self, message, status_code=400, payload=None):
    Exception.__init__(self)
    self.message = message
    self.status_code = status_code
    self.payload = payload

  def to_dict(self):
    rv = dict(self.payload or ())
    rv['message'] = self.message
    return rv

@app.errorhandler(InvalidAPIUsage)
def handle_invalid_api_usage(error):
  response = make_response(jsonify({'errorMessage': error.message,
                                    'errorData': error.payload,
                                    'errorCode': error.status_code}),
                           error.status_code)
  response.headers['Content-Type'] = "application/json"
  return response

@jwt.expired_token_loader
def expired_token_callback(token):
  token_type = token['type']
  if token_type == "access":
    print("access token has expired - goto refresh")
    return redirect(url_for('private_webview_common.refresh_get', request_path=request.path))

  elif token_type == "refresh":
    print("refresh token has expired - goto root/login")
    response = make_response(redirect("/"))
    return response

  else:
    raise InvalidUsage("unknown token type", status_code=403)

@jwt.needs_fresh_token_loader
def fresh_token_loader_callback():
  print("token is not fresh - goto refresh")
  response = make_response(redirect("/token/refresh"))
  return response

@jwt.invalid_token_loader
def invalid_token_callback():
  print("token is invalid - goto root/login")
  response = make_response(redirect("/"))
  return response

@jwt.unauthorized_loader
def missing_token_callback(token):
  print("token is missing - goto root/login")
  response = make_response(redirect("/"))
  return response


import importlib.util
import sys
spec = importlib.util.spec_from_file_location('mangaupdates', 'mangaupdates/__init__.py')
mangaupdates = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = mangaupdates
spec.loader.exec_module(mangaupdates)

from mangaupdates import Series, ListStats
import csv
import pandas as pd
import time
import os
import os.path
import argparse
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry

MAX_RETRIES = 5
CONNECTION_ERROR_DELAY = 90

def make_dataset(series_ids, filename=None, delay=10, list_names=None, mode='n'):

    col_names = ('user_id', 'username', 'score', 'list_name', 'series_id')
    resuming = False
    if filename is not None:
        write_col_names = True
        if os.path.isfile(filename):
            if mode == 'n':
                print(filename, 'exists. Aborting...')
                return
            elif mode == 'a':
                print(filename, 'exists. Rows will be appended.')
                resuming = True
                write_col_names = False

                # get latest sid in final line
                # https://stackoverflow.com/a/54278929
                with open(filename, 'rb') as f:
                    f.seek(-2, os.SEEK_END)
                    while f.read(1) != b'\n':
                        f.seek(-2, os.SEEK_CUR)
                    last_line = f.readline().decode()

                split_line = last_line.split(',')
                last_sid = int(split_line[-1])
                last_list_name = split_line[-2]
                last_sid_index = series_ids.index(last_sid)
                series_ids = series_ids[last_sid_index:]
            elif mode == 'w':
                print(filename, 'exists. Overwriting...')
            else:
                print(f"Error: value for mode ({mode}) should be either 'n', "
                       "'a', or 'w'. Exiting...")
                return
        elif mode == 'n':
            mode = 'w'
        f = open(filename, mode, newline='')
        writer = csv.writer(f)

        if write_col_names:
            writer.writerows([col_names])

    if filename is None:
        rows = []

    if list_names is None:
        list_names = ('read', 'wish', 'unfinished', 'complete', 'hold')
    print('Lists:', list_names)

    sess = requests.Session()
    retries = Retry(total=MAX_RETRIES, backoff_factor=3)
    sess.mount('http://', HTTPAdapter(max_retries=retries))
    loaded = False
    try:
        for i, sid in enumerate(series_ids):
            loaded = False
            lists = ListStats(sid, session=sess)
            print(sid, end='\t\t', flush=True)
            for _ in range(MAX_RETRIES):
                try:
                    lists.populate(list_names=list_names)
                    break
                except requests.exceptions.ConnectionError as e:
                    print(e)
                    print('Retrying...')
                    time.sleep(CONNECTION_ERROR_DELAY)
            else:       # no break
                print('Skipping', sid, '(exceeded MAX_RETRIES)')
                continue

            for key in list_names:
                if resuming and i == 0:
                    if list_names.index(key) <= list_names.index((last_list_name)):
                        print('0 rows (resuming).')
                        continue
                    resuming = False
                    # Since this program only writes to the file every iteration of
                    # a list of a series, we know that on the previous run, the
                    # program probably stopped between (instead of in the middle
                    # of) writing to the file.
                    # Thus, we assume that if the last entry on the file has
                    # some series id `last_sid` and list name `last_list_name`, we
                    # can simply skip all entries before that.
                new_rows = [(val.user_id, val.username, val.rating, key, sid) for val in lists.general_list(key)]
                print(key, f'{len(new_rows)} rows.', sep='\t')
                if filename is None:
                    rows.extend(new_rows)
                else:
                    writer.writerows(new_rows)
            loaded = True
            time.sleep(delay)
    except (KeyboardInterrupt, requests.exceptions.ConnectionError) as e:
        print('\n', e, sep='')
        if loaded:
            print("Stopped after loading", sid)
        else:
            print("Stopped before loading", sid)

    if filename is not None:
        f.close()
        return
    else:
        return rows

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(metavar='INPUT', dest='input',
                        help='csv file containing series ids.')
    parser.add_argument(metavar='OUTPUT', dest='output',
                        help='csv file where the output will be saved.')
    parser.add_argument('--headers', action='store_true',
                        help='there are headers in the input file (overrides --column)')
    parser.add_argument('-n', default=10, dest='N', help='# of series to crawl.')
    parser.add_argument('-m', '--mode', default='n',
                        help="'n': abort if OUTPUT file already exists (default)."
                        "'a': append to OUTPUT file if already exists."
                        "'w': overwrite OUTPUT file (equivalent to --force).")
    parser.add_argument('-f', '--force', action='store_true',
                        help='overwrite the output file if it exists (instead '
                             'of appending to it). overrides --mode.')
    parser.add_argument('--all', action='store_true',
                        help='"read", "wish", and "unfinished" lists will all be '
                             'crawled, otherwise only "read" will be crawled.')
    parser.add_argument('-c', '--column', default=0,
                        help='column (0-indexed) of series id (overriden by --headers')
    parser.add_argument('--resume', action='store_true',
                        help="equivalent to mode='a'. resumes progress if stopped"
                        " previously. overrides --force.")
    parser.add_argument('-d', '--delay', default=10,
                        help='# of seconds of delay between GET requests.')
    parser.add_argument('--listnames', default='rwuch')
    args = parser.parse_args()

    list_names = ['read']
    if args.all:
        list_names.extend(['wish', 'unfinished', 'complete', 'hold'])
    elif set(args.listnames).issubset('rwuch'):
        list_names = []
        if 'r' in args.listnames:
            list_names.append('read')
        if 'w' in args.listnames:
            list_names.append('wish')
        if 'u' in args.listnames:
            list_names.append('unfinished')
        if 'c' in args.listnames:
            list_names.append('complete')
        if 'h' in args.listnames:
            list_names.append('hold')
    else:
        print('--listnames', args.listnames, 'is invalid. Aborting...')
        exit(-1)

    mode = args.mode
    if args.resume:
        mode = 'a'
    elif args.force:
        mode = 'w'

    # Get unique series IDs from file
    series_ids = []
    with open(args.input, 'r', newline='') as csvfile:
        csvreader = csv.reader(csvfile)
        if args.headers:
            header_names = next(csvreader)
            if 'series_id' in header_names:
                header = 'series_id'
            else:
                print('Error: No "series_id" header in', args.INPUT)
                csvfile.close()
                exit(-1)
            sid_index = header_names.index('series_id')
        else:
            sid_index = 0

        for i,row in enumerate(csvreader):
            if i >= int(args.N):
                break
            sid = int(row[sid_index])
            if sid not in series_ids:
                series_ids.append(sid)

    make_dataset(series_ids, filename=args.output, delay=args.delay, mode=mode,
                 list_names=list_names)


import click
from bluebees.client.application.application_data import ApplicationData, app_name_list
from bluebees.client.data_paths import base_dir, app_dir

def validate_name(ctx, param, value):
    if not value:
        raise click.BadParameter('This option is required')
    if not app_name_list() or value not in app_name_list():
        raise click.BadParameter(f'The "{value}" application not exist')
    return value

@click.command()
@click.option('--name', '-n', type=str, default='', required=True,
              help='Specify the name of application', callback=validate_name)
def info(name):
    '''Get description about a application'''

    app_data = ApplicationData.load(base_dir + app_dir + name + '.yml')

    click.echo(click.style('***** Application data *****', fg='cyan'))
    click.echo(click.style(str(app_data), fg='cyan'))


from django.db import models
from .address import Address
from .seller import Seller
from .product import Product
from django.db import transaction, IntegrityError
from typing import Optional
from datetime import datetime

class Stock(models.Model):
    class Meta:
        app_label = 'app'
        db_table = 'stock'

    total = models.IntegerField()

    seller = models.ForeignKey(Seller, on_delete=models.DO_NOTHING)
    product = models.ForeignKey(Product, on_delete=models.DO_NOTHING)
    collect_at_start = models.DateTimeField(blank=True)
    collect_at_end = models.DateTimeField(blank=True)

    def __init__(self, seller, product, total, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.seller = seller
        self.product = product
        self.total = total

    def save(self, *args, **kwargs):
        try:
            super().save()
            return True
        except IntegrityError:
            transaction.set_rollback(True)
            return False

    @classmethod
    def update_stock(cls, filters: dict, total: int, collect_at_start: Optional[datetime] = None,
                     collect_at_end: Optional[datetime] = None):
        stock = cls.objects.first().filter(**filters)
        stock.total = total
        if collect_at_start:
            stock.collect_at_start = collect_at_start
        if collect_at_end:
            stock.collect_at_end = collect_at_end
        stock.save()

    @classmethod
    def find_by_products(cls, product_id: int, origin: tuple):
        stocks = cls.objects.all().filter(cls.product.id == product_id)
        sellers = []
        for stock in stocks:
            sellers.append({
                "seller": stock["seller"],
                "price": stock["product"]["price"],
                "distance": Address.calc_distance((stock["seller"].address.latitude, stock["seller"].address.longitude),
                                                  origin)
            })
        return sorted(sellers, key=lambda k: k['distance'])


from __future__ import division
import logbook

import numpy as np
import pandas as pd
from collections import Counter, OrderedDict, defaultdict

from six import iteritems, itervalues

import zipline.protocol as zp
from . position import positiondict

log = logbook.Logger('Performance')

class PerformancePeriod(object):

    def __init__(
            self,
            starting_cash,
            period_open=None,
            period_close=None,
            keep_transactions=True,
            keep_orders=False,
            serialize_positions=True):

        self.period_open = period_open
        self.period_close = period_close

        self.ending_value = 0.0
        self.period_cash_flow = 0.0
        self.pnl = 0.0
        # sid => position object
        self.positions = positiondict()
        self.ending_cash = starting_cash
        # rollover initializes a number of self's attributes:
        self.rollover()
        self.keep_transactions = keep_transactions
        self.keep_orders = keep_orders

        # Arrays for quick calculations of positions value
        self._position_amounts = pd.Series()
        self._position_last_sale_prices = pd.Series()

        self.calculate_performance()

        # An object to recycle via assigning new values
        # when returning portfolio information.
        # So as not to avoid creating a new object for each event
        self._portfolio_store = zp.Portfolio()
        self._positions_store = zp.Positions()
        self.serialize_positions = serialize_positions

    def rollover(self):
        self.starting_value = self.ending_value
        self.starting_cash = self.ending_cash
        self.period_cash_flow = 0.0
        self.pnl = 0.0
        self.processed_transactions = defaultdict(list)
        self.orders_by_modified = defaultdict(OrderedDict)
        self.orders_by_id = OrderedDict()

    def ensure_position_index(self, sid):
        try:
            self._position_amounts[sid]
            self._position_last_sale_prices[sid]
        except (KeyError, IndexError):
            self._position_amounts = \
                self._position_amounts.append(pd.Series({sid: 0.0}))
            self._position_last_sale_prices = \
                self._position_last_sale_prices.append(pd.Series({sid: 0.0}))

    def add_dividend(self, div):
        # The dividend is received on midnight of the dividend
        # declared date. We calculate the dividends based on the amount of
        # stock owned on midnight of the ex dividend date. However, the cash
        # is not dispersed until the payment date, which is
        # included in the event.
        self.positions[div.sid].add_dividend(div)

    def handle_split(self, split):
        if split.sid in self.positions:
            # Make the position object handle the split. It returns the
            # leftover cash from a fractional share, if there is any.
            position = self.positions[split.sid]
            leftover_cash = position.handle_split(split)
            self._position_amounts[split.sid] = position.amount
            self._position_last_sale_prices[split.sid] = \
                position.last_sale_price

            if leftover_cash > 0:
                self.handle_cash_payment(leftover_cash)

    def update_dividends(self, todays_date):
        """
        Check the payment date and ex date against today's date
        to determine if we are owed a dividend payment or if the
        payment has been disbursed.
        """
        cash_payments = 0.0
        stock_payments = Counter()  # maps sid to number of shares paid
        for sid, pos in iteritems(self.positions):
            cash_payment, stock_payment = pos.update_dividends(todays_date)
            cash_payments += cash_payment
            stock_payments.update(stock_payment)

        for stock, payment in iteritems(stock_payments):
            position = self.positions[stock]
            position.amount += payment
            self.ensure_position_index(stock)
            self._position_amounts[stock] = position.amount
            self._position_last_sale_prices[stock] = \
                position.last_sale_price

        # credit our cash balance with the dividend payments, or
        # if we are short, debit our cash balance with the
        # payments.
        # debit our cumulative cash spent with the dividend
        # payments, or credit our cumulative cash spent if we are
        # short the stock.
        self.handle_cash_payment(cash_payments)

        # recalculate performance, including the dividend
        # payments
        self.calculate_performance()

    def handle_cash_payment(self, payment_amount):
        self.adjust_cash(payment_amount)

    def handle_commission(self, commission):
        # Deduct from our total cash pool.
        self.adjust_cash(-commission.cost)
        # Adjust the cost basis of the stock if we own it
        if commission.sid in self.positions:
            self.positions[commission.sid].\
                adjust_commission_cost_basis(commission)

    def adjust_cash(self, amount):
        self.period_cash_flow += amount

    def calculate_performance(self):
        self.ending_value = self.calculate_positions_value()

        total_at_start = self.starting_cash + self.starting_value
        self.ending_cash = self.starting_cash + self.period_cash_flow
        total_at_end = self.ending_cash + self.ending_value

        self.pnl = total_at_end - total_at_start
        if total_at_start != 0:
            self.returns = self.pnl / total_at_start
        else:
            self.returns = 0.0

    def record_order(self, order):
        if self.keep_orders:
            dt_orders = self.orders_by_modified[order.dt]
            if order.id in dt_orders:
                del dt_orders[order.id]
            dt_orders[order.id] = order
            # to preserve the order of the orders by modified date
            # we delete and add back. (ordered dictionary is sorted by
            # first insertion date).
            if order.id in self.orders_by_id:
                del self.orders_by_id[order.id]
            self.orders_by_id[order.id] = order

    def update_position(self, sid, amount=None, last_sale_price=None,
                        last_sale_date=None, cost_basis=None):
        pos = self.positions[sid]
        self.ensure_position_index(sid)

        if amount is not None:
            pos.amount = amount
            self._position_amounts[sid] = amount
        if last_sale_price is not None:
            pos.last_sale_price = last_sale_price
            self._position_last_sale_prices[sid] = last_sale_price
        if last_sale_date is not None:
            pos.last_sale_date = last_sale_date
        if cost_basis is not None:
            pos.cost_basis = cost_basis

    def execute_transaction(self, txn):
        # Update Position
        # ----------------
        position = self.positions[txn.sid]
        position.update(txn)
        self.ensure_position_index(txn.sid)
        self._position_amounts[txn.sid] = position.amount

        self.period_cash_flow -= txn.price * txn.amount

        if self.keep_transactions:
            self.processed_transactions[txn.dt].append(txn)

    def calculate_positions_value(self):
        return np.dot(self._position_amounts, self._position_last_sale_prices)

    def update_last_sale(self, event):
        if event.sid not in self.positions:
            return

        if event.type != zp.DATASOURCE_TYPE.TRADE:
            return

        if not pd.isnull(event.price):
            # isnan check will keep the last price if its not present
            self.update_position(event.sid, last_sale_price=event.price,
                                 last_sale_date=event.dt)

    def __core_dict(self):
        rval = {
            'ending_value': self.ending_value,
            # this field is renamed to capital_used for backward
            # compatibility.
            'capital_used': self.period_cash_flow,
            'starting_value': self.starting_value,
            'starting_cash': self.starting_cash,
            'ending_cash': self.ending_cash,
            'portfolio_value': self.ending_cash + self.ending_value,
            'pnl': self.pnl,
            'returns': self.returns,
            'period_open': self.period_open,
            'period_close': self.period_close
        }

        return rval

    def to_dict(self, dt=None):
        """
        Creates a dictionary representing the state of this performance
        period. See header comments for a detailed description.

        Kwargs:
            dt (datetime): If present, only return transactions for the dt.
        """
        rval = self.__core_dict()

        if self.serialize_positions:
            positions = self.get_positions_list()
            rval['positions'] = positions

        # we want the key to be absent, not just empty
        if self.keep_transactions:
            if dt:
                # Only include transactions for given dt
                transactions = [x.to_dict()
                                for x in self.processed_transactions[dt]]
            else:
                transactions = \
                    [y.to_dict()
                     for x in itervalues(self.processed_transactions)
                     for y in x]
            rval['transactions'] = transactions

        if self.keep_orders:
            if dt:
                # only include orders modified as of the given dt.
                orders = [x.to_dict()
                          for x in itervalues(self.orders_by_modified[dt])]
            else:
                orders = [x.to_dict() for x in itervalues(self.orders_by_id)]
            rval['orders'] = orders

        return rval

    def as_portfolio(self):
        """
        The purpose of this method is to provide a portfolio
        object to algorithms running inside the same trading
        client. The data needed is captured raw in a
        PerformancePeriod, and in this method we rename some
        fields for usability and remove extraneous fields.
        """
        # Recycles containing objects' Portfolio object
        # which is used for returning values.
        # as_portfolio is called in an inner loop,
        # so repeated object creation becomes too expensive
        portfolio = self._portfolio_store
        # maintaining the old name for the portfolio field for
        # backward compatibility
        portfolio.capital_used = self.period_cash_flow
        portfolio.starting_cash = self.starting_cash
        portfolio.portfolio_value = self.ending_cash + self.ending_value
        portfolio.pnl = self.pnl
        portfolio.returns = self.returns
        portfolio.cash = self.ending_cash
        portfolio.start_date = self.period_open
        portfolio.positions = self.get_positions()
        portfolio.positions_value = self.ending_value
        return portfolio

    def get_positions(self):

        positions = self._positions_store

        for sid, pos in iteritems(self.positions):

            if pos.amount == 0:
                # Clear out the position if it has become empty since the last
                # time get_positions was called.  Catching the KeyError is
                # faster than checking `if sid in positions`, and this can be
                # potentially called in a tight inner loop.
                try:
                    del positions[sid]
                except KeyError:
                    pass
                continue

            # Note that this will create a position if we don't currently have
            # an entry
            position = positions[sid]
            position.amount = pos.amount
            position.cost_basis = pos.cost_basis
            position.last_sale_price = pos.last_sale_price
        return positions

    def get_positions_list(self):
        positions = []
        for sid, pos in iteritems(self.positions):
            if pos.amount != 0:
                positions.append(pos.to_dict())
        return positions


# GUI imports
# from cgitb import text
from email import message
import tkinter as tk
from tkinter import Tk, ttk
from tkinter import font
from PIL import Image, ImageTk

# encryptor / decryptor
import arrows

# Global GUI variables (for showing/hiding inputs)
OTP_selected = False
decrypt_selected = False
encrypt_selected = False
message_out = ""
key_out = ""

def main():
    def toggle_key_input():
        global OTP_selected
        if OTP_selected:
            key_text.grid()
            key_input.grid()
        else:
            key_text.grid_remove()
            key_input.grid_remove()
        OTP_selected = not OTP_selected

    def toggle_OTP_on_decrypt():
        global decrypt_selected
        global encrypt_selected
        global OTP_selected
        if decrypt_selected:
            if OTP_selected:    # incase OTP was selected before selecting decrypt
                key_text.grid_remove()
                key_input.grid_remove()
            otp_text.grid()
            otp_checkbox.grid()
        else:
            encrypt_selected = False

            otp_checkbox.deselect()
            OTP_selected = False

            otp_text.grid_remove()
            otp_checkbox.grid_remove()
            key_text.grid()
            key_input.grid()    # show key input incase otp was selected before selecting decrypt
        decrypt_selected = not decrypt_selected

    def toggle_OTP_on_encrypt():
        global encrypt_selected
        global decrypt_selected
        if decrypt_selected:
            otp_text.grid()
            otp_checkbox.grid()
        decrypt_selected = False
        encrypt_selected = not encrypt_selected

    def enter_onclick():
        global message_out
        global key_out

        result = ()
        key = key_input.get("1.0", tk.END).replace("\n", "")
        message = message_input.get("1.0", tk.END).replace("\n", "")
        message_output.config(text="")
        key_output.config(text="")

        if e_d_selection.get() == 0:
            user_return_message.config(text="Select Encrypt or Decrypt")
        elif not OTP_selected and key == "":
            user_return_message.config(
                text="Enter a key or select \'Generate OTP key\'")
        elif message == "":
            user_return_message.config(
                text="Enter a message to encrypt or decrypt")
        else:
            user_return_message.config(text="")
            if e_d_selection.get() == 1:
                if OTP_selected:
                    result = arrows.encryptor(
                        encrypt_decrypt=1, use_OTP=True, text=message)
                else:
                    result = arrows.encryptor(
                        encrypt_decrypt=1, use_OTP=False, key=key, text=message)
            elif e_d_selection.get() == 2:
                result = arrows.encryptor(
                    encrypt_decrypt=2, key=key, text=message)

            message_out = result[0]
            key_out = result[1]

            message_output.config(text=result[0])
            key_output.config(text=result[1])

    def copy_message():
        global message_out
        root.clipboard_clear()
        root.clipboard_append(message_out)
        root.update()  # text stays after the window is closed

    def copy_key():
        global key_out
        root.clipboard_clear()
        root.clipboard_append(key_out)
        root.update()

    root = tk.Tk()
    # root.configure(background="#C3C3C3")
    canvas = tk.Canvas(root, width=700, height=700)
    canvas.grid(rowspan=30, columnspan=3)

    # logo
    logo = Image.open("img/arrows_logo.jpg")
    logo = ImageTk.PhotoImage(logo)
    logo_label = tk.Label(image=logo)
    logo_label.image = logo
    logo_label.grid(row=0, column=0)

    encrypt_decrypt_text = tk.Label(
        root, text="Encrypt or Decrypt:", font="Raleway")
    encrypt_decrypt_text.grid(row=1, column=0)

    e_d_selection = tk.IntVar()
    e = tk.Checkbutton(root, text="Encrypt", variable=e_d_selection,
                       onvalue=1, command=toggle_OTP_on_encrypt)
    e.grid(row=1, column=1)
    d = tk.Checkbutton(root, text="Decrypt", variable=e_d_selection,
                       onvalue=2, command=toggle_OTP_on_decrypt)
    d.grid(row=1, column=2)

    OTP_selection = tk.IntVar()
    otp_text = tk.Label(root, text="Generate OTP key?", font="Raleway")
    otp_text.grid(row=3, column=0)

    otp_checkbox = tk.Checkbutton(
        root, text='OTP Key', variable=OTP_selection, onvalue=1, command=toggle_key_input)
    otp_checkbox.grid(row=3, column=1)

    key_text = tk.Label(root, text="Enter key:", font="Raleway")
    key_text.grid(row=4, column=0)

    key_input = tk.Text(root, height=3, width=30)
    key_input.grid(row=4, column=1)

    message_text = tk.Label(root, text="Enter message:", font="Raleway")
    message_text.grid(row=5, column=0)

    message_input = tk.Text(root, height=5, width=30)
    message_input.grid(row=5, column=1)

    enter = tk.Button(root, height=2, width=10,
                      text="Enter", command=enter_onclick)
    enter.grid(row=6, column=1)

    user_return_message = tk.Label(root, text="", fg="#ee3377")
    user_return_message.grid(row=7, column=1)

    message_output_text = tk.Label(
        root, text="Message output:", font="Raleway")
    message_output_text.grid(row=8, column=0)

    message_output = tk.Label(root, text="", fg="#ee3377", wraplength=250)
    message_output.grid(row=8, column=1)

    key_output_text = tk.Label(root, text="Key output:", font="Raleway")
    key_output_text.grid(row=9, column=0)

    key_output = tk.Label(root, text="", fg="#ee3377", wraplength=250)
    key_output.grid(row=9, column=1)

    cpy_msg_btn = tk.Button(root, height=1, width=5,
                            text="Copy", command=copy_message)
    cpy_msg_btn.grid(row=8, column=2)

    cpy_key_btn = tk.Button(root, height=1, width=5,
                            text="Copy", command=copy_key)
    cpy_key_btn.grid(row=9, column=2)

    root.columnconfigure(0, minsize=40)
    root.rowconfigure(0, pad=30)
    # root.grid_rowconfigure(2, pad=10)
    # root.grid_rowconfigure(3, pad=10)

    root.mainloop()

if __name__ == "__main__":
    main()


import asyncio
import functools
import json
import os
import subprocess

from git import Repo, Actor
from importlib import resources
from mimetypes import guess_type
from shutil import rmtree
from tornado import web, websocket

async def render(basepath: str):
    """Render the project at the given basepath."""
    install = await asyncio.create_subprocess_exec('poetry', 'install', cwd=basepath)
    await install.wait()
    build = await asyncio.create_subprocess_exec('poetry', 'run', 'make', 'html', cwd=basepath, stdout=subprocess.PIPE)
    await build.wait()
    return (await build.stdout.read()).decode('utf-8')

class FrontendHandler(web.RequestHandler):
    """Handles serving the UI."""

    def initialize(self, basepath: str):
        self.basepath = basepath

    def get(self, path):
        """Return the UI resource at ``path``."""
        if path.startswith('/'):
            path = path[1:]
        base = resources.files('ou_content_author') / 'frontend' / 'public'
        try:
            self.send_resource(base, path.split('/'))
        except FileNotFoundError:
            self.send_resource(base, ('index.html',))

    def send_resource(self, resource, path):
        """Send the resource identified by the ``path``.

        Performs content modification when sending the index.html file to set the correct base path for all resources.
        """
        for part in path:
            resource = resource / part
        try:
            mimetype = guess_type(path[-1])
            if mimetype and mimetype[0]:
                self.set_header('Content-Type', mimetype[0])
            if path == ('index.html',):
                content = resource.read_bytes().decode('utf-8')
                content = content.replace('{app_path}', f'{self.basepath}app/')
                self.write(content.encode('utf-8'))
            else:
                self.write(resource.read_bytes())
        except IsADirectoryError:
            raise FileNotFoundError
        except IndexError:
            raise FileNotFoundError

class ApiHandler(websocket.WebSocketHandler):

    def initialize(self, repository_location: str):
        self._repository_location = repository_location
        self._task = None

    async def on_message(self, message: str):
        data = json.loads(message)
        if 'type' in data:
            if data['type'] == 'clone-repository':
                self.clone_repository(data)
            elif data['type'] == 'delete-repository':
                self.delete_repository(data)
            elif data['type'] == 'checkout-branch':
                self.checkout_branch(data)
            elif data['type'] == 'select-block':
                self.select_block(data)
            elif data['type'] == 'load-file-content':
                await self.load_file_content(data)
            elif data['type'] == 'save-file-content':
                await self.save_file_content(data)
            elif data['type'] == 'commit-changes':
                self.commit_changes(data)
            elif data['type'] == 'discard-changes':
                self.discard_changes(data)
            elif data['type'] == 'add-file':
                self.add_file(data)
            elif data['type'] == 'delete-file':
                self.delete_file(data)

    def send_message(self, message: dict):
        self.write_message(json.dumps(message))

    def clone_repository(self, data: dict):
        """Clone the repository at the URL."""
        if 'url' in data:
            if os.path.exists(self._repository_location):
                rmtree(self._repository_location)
            repo = Repo.clone_from(data['url'], self._repository_location)
            branches = [ref.path.split('/')[-1] for ref in repo.remote().refs if not ref.path.endswith('/HEAD')]
            self.send_message({
                'type': 'repository',
                'branches': branches
            })

    def delete_repository(self, data: dict):
        if os.path.exists(self._repository_location):
            rmtree(self._repository_location)
        self.send_message({
            'type': 'repository-deleted',
        })

    def checkout_branch(self, data: dict):
        """Checkout the branch either locally or from remote"""
        if 'branch' in data:
            repo = Repo(self._repository_location)
            found = False
            for ref in repo.branches:
                if ref.path.endswith(f'/{data["branch"]}'):
                    ref.checkout()
                    found = True
                    break
            if not found:
                for ref in repo.remote().refs:
                    if ref.path.endswith(f'/{data["branch"]}'):
                        repo.git.checkout('-b', data['branch'], ref.path)
                        found = True
            if found:
                blocks = []
                for basepath, dirnames, filenames in os.walk(self._repository_location):
                    for filename in filenames:
                        if filename == 'conf.py':
                            blocks.append(os.path.join(basepath, filename)[len(self._repository_location):])
                self.send_message({
                    'type': 'branch',
                    'blocks': blocks
                })

    def select_block(self, data: dict):
        """Select a given block."""
        if 'block' in data:
            self.scan_block(data['block'])

    async def load_file_content(self, data: dict):
        """Load the content of a file."""
        if 'file' in data and 'block' in data and 'directory' in data['file'] and 'filename' in data['file']:
            filepath = os.path.join(self._repository_location,
                                    os.path.dirname(data['block'])[1:],
                                    data['file']['directory'],
                                    data['file']['filename'])
            if os.path.exists(filepath) and os.path.abspath(filepath).startswith(os.path.abspath(self._repository_location)):
                with open(filepath) as in_f:
                    self.send_message({
                        'type': 'file-content',
                        'content': in_f.read()
                    })
                await self.run_render(os.path.join(self._repository_location, os.path.dirname(data['block'])[1:]),
                                      os.path.dirname(data['block'])[1:],
                                      os.path.join(data['file']['directory'], data['file']['filename']))

    async def save_file_content(self, data: dict):
        """Save the updated file content."""
        if 'file' in data and 'block' in data and 'directory' in data['file'] and 'filename' in data['file'] and 'content' in data:
            filepath = os.path.join(self._repository_location,
                                    os.path.dirname(data['block'])[1:],
                                    data['file']['directory'],
                                    data['file']['filename'])
            if os.path.exists(filepath) and os.path.abspath(filepath).startswith(os.path.abspath(self._repository_location)):
                with open(filepath, 'w') as out_f:
                    out_f.write(data['content'])
                await self.run_render(os.path.join(self._repository_location, os.path.dirname(data['block'])[1:]),
                                      os.path.dirname(data['block'])[1:],
                                      os.path.join(data['file']['directory'], data['file']['filename']))
            self.check_repo_changes()

    def commit_changes(self, data: dict):
        """Commit and push changes."""
        if 'name' in data and data['name'] and 'email' in data and data['email'] and 'message' in data and data['message']:
            repo = Repo(self._repository_location)
            if len(repo.index.diff(None) + repo.index.diff('HEAD')) > 0 or len(repo.untracked_files) > 0:
                repo.git.add('.')
                actor = Actor(data['name'], data['email'])
                repo.index.commit(data['message'], author=actor, committer=actor)
                repo.git.push('--force')
            self.send_message({
                'type': 'changes-committed',
            })
            self.check_repo_changes()

    def discard_changes(self, data: dict):
        """Discard any changes."""
        repo = Repo(self._repository_location)
        repo.git.checkout('--', '.')
        self.send_message({
            'type': 'changes-discarded',
        })
        self.check_repo_changes()

    def add_file(self, data: dict):
        if 'block' in data and 'file' in data and 'directory' in data['file'] and 'filename' in data['file']:
            filepath = os.path.join(self._repository_location,
                                    os.path.dirname(data['block'])[1:],
                                    data['file']['directory'],
                                    data['file']['filename'])
            if not os.path.exists(filepath) and os.path.abspath(filepath).startswith(os.path.abspath(self._repository_location)):
                if not os.path.exists(os.path.dirname(filepath)):
                    os.makedirs(os.path.dirname(filepath), exist_ok=True)
                with open(filepath, 'w') as out_f:
                    if filepath.endswith('.md'):
                        out_f.write('# Page Title')
            self.scan_block(data['block'])
            self.check_repo_changes()

    def delete_file(self, data: dict):
        if 'block' in data and 'file' in data and 'directory' in data['file'] and 'filename' in data['file']:
            filepath = os.path.join(self._repository_location,
                                    os.path.dirname(data['block'])[1:],
                                    data['file']['directory'],
                                    data['file']['filename'])
            if os.path.exists(filepath) and os.path.abspath(filepath).startswith(os.path.abspath(self._repository_location)):
                os.unlink(filepath)
            self.scan_block(data['block'])
            self.check_repo_changes()

    async def run_render(self, basepath: str, block: str, filepath: str):
        """Run the render process and send the required messages."""
        if basepath.endswith('/source'):
            basepath = basepath[:-7]
        self.send_message({
            'type': 'file-rendering',
        })
        if self._task is not None and not self._task.done():
            self._task.cancel()
        self._task = asyncio.create_task(render(basepath))
        self._task.add_done_callback(functools.partial(self.render_complete, block, filepath))

    def render_complete(self, block: str, filepath: str, task: asyncio.Task):
        """Report rendering completed to the client."""
        if not task.cancelled():
            result = task.result()
            if result:
                if '.' in filepath:
                    filepath = f'{filepath[:filepath.rfind(".")]}.html'
                else:
                    filepath = f'{filepath}.html'
                if block.endswith('/source'):
                    block = block[:-7]
                self.send_message({
                    'type': 'file-rendered',
                    'url': f'/{block}/{filepath}',
                    'output': result,
                })

    def check_repo_changes(self):
        """Check if there are changes to commit in the repo."""
        repo = Repo(self._repository_location)
        if len(repo.index.diff(None) + repo.index.diff('HEAD')) > 0 or len(repo.untracked_files) > 0:
            self.send_message({
                'type': 'changes-found'
            })
        else:
            self.send_message({
                'type': 'no-changes-found'
            })

    def scan_block(self, block):
        """Scan the block and send back the """
        blockpath = os.path.dirname(f'{self._repository_location}{block}')
        files = []
        for basepath, dirnames, filenames in os.walk(blockpath):
            for filename in filenames:
                if filename.endswith('.md'):
                    files.append({
                        'directory': basepath[len(blockpath) + 1:],
                        'filename': filename
                    })
        self.send_message({
            'type': 'block',
            'path': block,
            'files': files
        })

class RenderedHandler(web.RequestHandler):
    """Handler for the rendered HTML pages."""

    def initialize(self, repository_location: str):
        """Initialise the handler."""
        self._repository_location = repository_location

    def get(self, path: str):
        """Get the rendered HTML page."""
        if path.startswith('/'):
            path = path[1:]
        path_elements = path.split('/')
        filepath = os.path.abspath(os.path.join(self._repository_location,
                                                path_elements[0],
                                                'build',
                                                'html',
                                                *path_elements[1:]))
        if filepath.startswith(self._repository_location) and os.path.exists(filepath):
            mimetype = guess_type(filepath)
            if mimetype and mimetype[0]:
                self.set_header('Content-Type', mimetype[0])
            with open(filepath, 'rb') as in_f:
                self.write(in_f.read())


from itertools import product # Biblioteca que realiza arranjos de análise combinatória
import time

# Classe que representa o grafo e realiza todas as operações com ele
class Grafo:
    # Método que inicializa a classe
    def __init__(self, matriz):
        self.matriz = matriz
        self.tamanhoCaminho = 0
        self.construirGrafo()
        self.solucionado = False
        self.caminhosEncontrados = []
        self.inicio = 0
        self.fim = 0

    # Método onde percorre a matriz e constrói a lista de vértices e self.arestas
    def construirGrafo(self):
        self.vertices = list(range(len(self.matriz[0])))
        self.arestas = []
        qtdOrdenadas = 0
        qtdNOrdenadas = 0
        i = 0
        while i < len(self.matriz):
            j = 0
            while j < len(self.matriz[i]):
                if((self.matriz[i][j]!=0) and (self.matriz[i][j] == self.matriz[j][i])):
                    self.arestas.append([i,j, self.matriz[i][j], True])
                    qtdNOrdenadas = qtdNOrdenadas + 1
                if((self.matriz[i][j]!=0) and (self.matriz[i][j] != self.matriz[j][i])):
                    self.arestas.append([i,j, self.matriz[i][j], False])
                    qtdOrdenadas = qtdOrdenadas + 1
                j = j + 1
            i = i + 1
        print(self.arestas)
        self.tamanhoCaminho = qtdOrdenadas + int(qtdNOrdenadas/2)

    # Método que inicia o caminhamento no grafo
    def caminharGrafo(self):
        self.inicio = time.time()
        while(self.solucionado == False):
            #self.caminhos = list(product(self.vertices, repeat=self.tamanhoCaminho))
            for caminho in product(self.vertices, repeat=self.tamanhoCaminho):
                caminho = list(caminho)
                if caminho[0] != caminho[len(caminho)-1]:
                    caminho.append(caminho[0])
                self.verifica_aresta_vertice(list(caminho))
            self.tamanhoCaminho = self.tamanhoCaminho + 1
        self.calculaCusto()

    #verifica se aresta é não direcionada e se já foi visitada em outra direção
    def nao_direcionada_visitada(self, caminho, verticeInicial, verticeFinal):
        retorno = False
        if(self.existe_aresta(verticeFinal, verticeInicial)):
            i = 1
            while i < len(caminho):
                if(verticeFinal==caminho[i-1] and verticeInicial==caminho[i]):
                    retorno = True
                i = i + 1
        return retorno

    #verifica se existe essa aresta
    def existe_aresta(self, verticeInicial, verticeFinal):
        for aresta in self.arestas:
            if(aresta[0]==verticeInicial and aresta[1]==verticeFinal):
                return True
        return False

    # verifica se existem self.arestas entre dois vértices
    def verifica_aresta_vertice(self, caminho):
        i = 1
        arestasValidas = True
        while i < len(caminho):
            if self.existe_aresta(caminho[i-1],caminho[i]) == False:
                arestasValidas = False
                break
            i = i + 1
        if(arestasValidas):
            self.verifica_todas_arestas(caminho)

    #verifica se visitou todas as self.arestas
    def verifica_todas_arestas(self, caminho):
        if caminho == [0, 1, 2, 0]:
            print('ok')
        faltaAresta = True
        for aresta in self.arestas:
            i = 1
            faltaAresta = True
            arestaFaltante = None
            while i < len(caminho):
                if(aresta[0]==caminho[i-1] and aresta[1]==caminho[i]):
                    faltaAresta = False
                    arestaFaltante = None
                    break
                arestaFaltante = aresta
                i = i + 1
            if(arestaFaltante!=None):
                if(self.nao_direcionada_visitada(caminho, arestaFaltante[0], arestaFaltante[1]) and arestaFaltante[3]):
                    faltaAresta = False
            if(faltaAresta):
                break
        if(faltaAresta==False):
            self.solucionado = True
            self.caminhosEncontrados.append(caminho)

    # Método que calcula o custo dos caminhos encontrados e imprime o melhor
    def calculaCusto(self):
        melhorCaminho = []
        melhorCusto = 0
        for caminhoEncontrado in self.caminhosEncontrados:
            i = 1
            custoCaminho = 0
            while i < len(caminhoEncontrado):
                custoCaminho = custoCaminho + self.matriz[caminhoEncontrado[i-1]][caminhoEncontrado[i]]
                i = i + 1
            if((melhorCusto==0) or (custoCaminho<melhorCusto)):
                melhorCusto = custoCaminho
                melhorCaminho = caminhoEncontrado
        print('Melhor caminho: '+str(melhorCaminho))
        print('Menor custo: '+str(melhorCusto))
        self.fim = time.time()
        print(self.fim - self.inicio)


# USB VCP example.
# This example shows how to use the USB VCP class to send an image to PC on demand.
#
# WARNING:
# This script should NOT be run from the IDE or command line, it should be saved as main.py
# Note the following commented script shows how to receive the image from the host side.
#
# #!/usr/bin/env python2.7
# import sys, serial, struct
# port = '/dev/ttyACM0'
# sp = serial.Serial(port, baudrate=115200, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE,
#             xonxoff=False, rtscts=False, stopbits=serial.STOPBITS_ONE, timeout=None, dsrdtr=True)
# sp.setDTR(True) # dsrdtr is ignored on Windows.
# sp.write("snap")
# sp.flush()
# size = struct.unpack('<L', sp.read(4))[0]
# img = sp.read(size)
# sp.close()
#
# with open("img.jpg", "w") as f:
#     f.write(img)

import sensor, image, pin, time, ustruct, pyb
from pyb import USB_VCP

sensor.reset()                      # Reset and initialize the sensor.
sensor.set_pixformat(sensor.RGB565) # Set pixel format to RGB565 (or GRAYSCALE)
sensor.set_framesize(sensor.QVGA)   # Set frame size to QVGA (320x240)
sensor.set_auto_exposure(False, 5000)
sensor.set_auto_gain(False)
sensor.set_auto_whitebal(False)
#sensor.set_contrast(   -3 ) #-3 +3
#sensor.set_brightness( +3 ) #-3 +3
sensor.set_saturation( +3 ) #-3 +3
sensor.skip_frames(time = 1000)

DBG = True

pin = Pin('P0', Pin.OUT_OD)
usb = USB_VCP()

imgMs = 0
usbMs = 0
stream = True

while (True):
    ticks = time.ticks()
    img = sensor.snapshot()
    cnt = 0
    for blob in img.find_blobs([(10,100, 50,70, 40,70)], pixels_threshold=120, area_threshold=100, merge=True):
        cnt += 1
        if DBG and stream:
            img.draw_rectangle( blob.rect(), thickness=4)
            img.draw_string( blob.x(), blob.y()-10, '{0:1d}'.format(cnt), scale=4)
            img.draw_string(0,0,  '{0:d}'.format(time.ticks()), scale=4)
            img.draw_string(0,25, '{0:2d} {1:2d}'.format(imgMs, usbMs), scale=4)

    imgMs = time.ticks() - ticks

    # \/ \/ USB CODE \/ \/
    ticks = time.ticks()
    if stream:
        img.copy(x_scale=.25,y_scale=.25,copy_to_fb=True)
        img.compress(88) #90

    cmd = usb.recv(2, timeout=1000) # Change this to match the number of commands received
    if not cmd:
        continue
    if cmd[0] == b's'[0] and stream:
        usb.send(ustruct.pack(">lll", imgMs, usbMs, img.size()))
        usb.send(img)
    else:
        usb.send(ustruct.pack(">lll", imgMs, usbMs, 0))
        if cmd[0] == b's'[0]:
            stream = True
        else:
            stream = False
    usbMs = time.ticks() - ticks

    if cmd[1] == b'r'[0]:
        pyb.LED(1).toggle()
    if cmd[1] == b'g'[0]:
        pyb.LED(2).toggle()
    if cmd[1] == b'b'[0]:
        pyb.LED(3).toggle()
    if cmd[1] == b'w'[0]:
        pin.value(False)
    else:
        pin.value(True)
    # /\ /\ USB CODE /\ /\

    #print('{} {}'.format(imgMs,usbMs))


from dateutil import tz
import datetime
import pytz

import re

pattern = r'^\b(.{4})[:-](.{2})[:-](.{2})\s+(.{2}):(.{2}):(.{2})(Z|([+-])(.{2}):(.{2}))?.*\b$'

class Text2Time:
    def __init__(self, text):
        match = re.search(pattern, text)
        if match:
            groups = match.groups()

            year = int(groups[0].strip())
            month = int(groups[1].strip())
            day = int(groups[2].strip())
            hour = int(groups[3].strip())
            minute = int(groups[4].strip())
            second = int(groups[5].strip())
            if groups[6]:
                if groups[6].strip() == 'Z':
                    timezone_value = 0
                else:
                    timezone_sign = -1 if (groups[7].strip() == '-') else 1
                    timezone_hours = int(groups[8].strip())
                    timezone_minutes = int(groups[9].strip())
                    timezone_value = timezone_sign * (timezone_hours * 60 + timezone_minutes)
            else:
                timezone_value = None
            self.dateTime = Text2Time.create_custom_datetime(
                year, month, day, hour, minute, second, timezone_value)

    @classmethod
    def create_custom_datetime(cls, year, month, day, hour, minute, second, timezone_value=None):
        if timezone_value is None:
            tz_info = tz.tzlocal()
        elif timezone_value == 'Z':
            tz_info = pytz.utc
        else:
            tz_info = tz.tzoffset(None, timezone_value * 60)

        try:
            dt = datetime.datetime(year, month, day, hour, minute, second, tzinfo=tz_info)
            return dt.astimezone(tz.tzlocal())
        except ValueError as e:
            # print(f"Error creating datetime: {e}")
            return None

if __name__ == '__main__':
    time_values = [
        "2014:05:06 14:55:08+05:30",
        "2014:05:06 14:55:08Z",
        "2014:05:06 14:55:08",
        "2014:05:06 14:55: 8",
        "2014:05:06 14: 5: 8"
    ]

    for value in time_values:
        print(f"{value} ==> {Text2Time(value).dateTime}")


from typing import List

class Solution:

    def merge(self, nums1: List[int], m: int, nums2: List[int], n: int) -> None:
        '''合并两个有序数组

        @Note:
            类似归并排序，从后往前
        '''
        i, j, k = m - 1, n - 1, m + n - 1
        if n == 0:
            return
        while True:
            if nums1[i] > nums2[j]:
                nums1[k] = nums1[i]
                k -= 1
                i -= 1
            else:
                nums1[k] = nums2[j]
                j -= 1
                k -= 1
            if i == -1 or j == -1:
                break
        if i == -1:
            nums1[:j + 1] = nums2[:j + 1]  #注意左闭右开

if __name__ == '__main__':
    nums1 = [1, 0]  #[int(x) for x  in input().strip().split(' ')]
    m = 1  #int(input().strip())
    nums2 = [2]  #[int(x) for x  in input().strip().split(' ')]
    n = 1  #int(input().strip())
    solution = Solution()
    solution.merge(nums1, m, nums2, n)
    print(nums1)


# -*- coding: utf-8 -*-
# @Author: Ruban
# @License: Apache Licence
# @File: icdar2013_convert.py

import os
import re
import codecs
import argparse
import numpy as np

parser = argparse.ArgumentParser()
parser.add_argument('--data_dir', type=str, default=r'D:\data\ICDAR2013')

FLAGS = parser.parse_args()

class ICDAR2013Convertor:
    def __init__(self, img_root, txt_root):
        super(ICDAR2013Convertor, self).__init__()
        self.img_root = img_root
        self.txt_root = txt_root

    def convert_to_craft(self):
        img_name_list = os.listdir(self.img_root)
        sample_list = list()
        for img_name in img_name_list:
            txt_name = 'gt_' + img_name[:-len(img_name.split('.')[-1])] + 'txt'
            txt_path = os.path.join(self.txt_root, txt_name)
            img_path = os.path.join(self.img_root, img_name)
            if os.path.exists(txt_path):
                word_boxes = list()
                char_boxes_list = list()
                words = list()
                with codecs.open(txt_path, 'rb', encoding='utf-8') as txt_file:
                    lines = txt_file.read().splitlines()
                    for line in lines:
                        infos = re.split(',? ', line)

                        word = infos[4]
                        word = re.sub('^"', '', word)
                        word = re.sub('"$', '', word)
                        if '\\' in word:
                            print(word)
                        words.append(word)

                        char_boxes_list.append([])

                        left, top, right, bottom = [round(float(p)) for p in infos[:4]]
                        word_box = np.array([[left, top], [right, top], [right, bottom], [left, bottom]])
                        word_boxes.append(word_box)

                sample_list.append([img_path, word_boxes, words, char_boxes_list])

        return sample_list

if __name__ == '__main__':
    import pickle

    image_dir = os.path.join(FLAGS.data_dir, r'Challenge2_Training_Task12_Images')
    txt_dir = os.path.join(FLAGS.data_dir, r'Challenge2_Training_Task1_GT')
    pkl_path = os.path.join(FLAGS.data_dir, r'gt.pkl')

    icdar_2013_convertor = ICDAR2013Convertor(img_root=image_dir,
                                              txt_root=txt_dir)
    craft_sample_list = icdar_2013_convertor.convert_to_craft()
    with open(pkl_path, 'wb') as pkl_file:
        pickle.dump(craft_sample_list, pkl_file)

    image_dir = os.path.join(FLAGS.data_dir, r'Challenge2_Test_Task12_Images')
    txt_dir = os.path.join(FLAGS.data_dir, r'Challenge2_Test_Task1_GT')
    pkl_path = os.path.join(FLAGS.data_dir, r'test_gt.pkl')

    icdar_2013_convertor = ICDAR2013Convertor(img_root=image_dir,
                                              txt_root=txt_dir)
    craft_sample_list = icdar_2013_convertor.convert_to_craft()
    with open(pkl_path, 'wb') as pkl_file:
        pickle.dump(craft_sample_list, pkl_file)


#!/usr/bin/env python
# encoding: utf-8
"""
@author: Mingjun Lei
@file: test_wework.py
@time: 2021/2/26 16:07
@desc: This py file is to test different login scenarios without PO
"""
import json
from time import sleep
import pytest
from selenium import webdriver
from selenium.webdriver.common.by import By

class TestWeWork:

    def setup(self):
        chrome_arg = webdriver.ChromeOptions()
        chrome_arg.debugger_address = '127.0.0.1:9222'
        # self.driver = webdriver.Chrome(options=chrome_arg)
        self.driver = webdriver.Chrome()
        self.driver.implicitly_wait(6)

    def teardown(self):
        self.driver.quit()

    # @pytest.mark.skip
    def test_register(self):
        """
        Sign in from homepage of WechatWork
        """
        self.driver.get("https://work.weixin.qq.com/")  # original website
        self.driver.find_element(By.XPATH, "//*[@class='index_top_operation_loginBtn']").click()  # click on login
        self.driver.find_element(By.XPATH, "//*[@class='login_registerBar_link']").click()  # Company registration
        self.driver.find_element(By.XPATH, "//*[@id='corp_name']").send_keys("test")  # company name
        sleep(2)

    # @pytest.mark.skip
    def test_login_with_debugger(self):
        """
        Reusing chrome browser by port
        """
        self.driver.get("https://work.weixin.qq.com/wework_admin/frame")  # website with login user
        self.driver.find_element(By.XPATH, "//*[@id='menu_contacts']").click()  # change to contact tab
        self.driver.find_element(By.XPATH, "//*[@id='menu_apps']").click()  # change to app management tab
        self.driver.find_element(By.XPATH, "//*[@id='menu_customer']").click()
        self.driver.find_element(By.XPATH, "//*[@id='menu_manageTools']").click()
        self.driver.find_element(By.XPATH, "//*[@id='menu_profile']").click()

    # @pytest.mark.skip
    def test_login_with_cookies(self):
        """
        Access to website with cookies
        """
        # store cookie, login status first with debugger
        # cookies = self.driver.get_cookies()
        # print(cookies)
        # with open("temp.txt", "w", encoding="utf-8") as f:
        #     json.dump(cookies, f)

        # read cookie, could use general chromedriver
        self.driver.get("https://work.weixin.qq.com/wework_admin/frame#index")
        with open("temp.txt", "r", encoding="utf-8") as f:
            cookies = json.load(f)
        print(cookies)
        for cookie in cookies:
            self.driver.add_cookie(cookie)
        self.driver.refresh()
        sleep(2)


import requests
import json
from random import sample, randint
from datetime import datetime, timedelta
import logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(name)s: %(message)s')

if __name__ == "__main__":

    logins = ['alex', 'john', 'mike', 'nadya', 'sasha', 'peter', 'fox', 'cat', 'dog']
    domains = ['@mail.ru', '@yahoo.com', '@hotmail.com', '@yandex.ru', '@gmail.com', '@ya.ru', '@list.ru']
    fnames = ['Nadya', 'Anya', 'Julia', 'Sasha', 'Alex', 'Alexey', 'Dan', 'Andy', 'Ira', 'Nomi', 'Elka']
    lnames = ['Marchewka', 'Polenka', 'Kozalko', 'Sarowsky', 'Ishkov']

    url = 'http://127.0.0.1:8000/api/users/'

    emails = list()
    names = list()

    for login in logins:
        for domain in domains:
            emails.append(login + domain)

    for fname in fnames:
        for lname in lnames:
            names.append(" ".join([fname, lname]))

    for email in emails:
        row = dict()
        row['name'] = sample(names, 1)[0]
        row['email'] = email
        row['is_guest'] = sample([0,1], 1)[0]
        if row['is_guest'] == 1:
            row['registration_date'] = datetime(1970,1,1,0,0,0)
        else:
            row['registration_date'] = datetime.now()
        row['join_date'] = datetime.now()

        r = requests.post(url, data=row)
        logging.info("{0} - {1}".format(str(r.status_code), str(r.reason)))

    logging.info("Generated {0} users".format(len(emails)))


import scipy.spatial.distance
import numpy as np

def distance(data):

  total = 0
  num_obs = data.shape[0]

  for i in range(num_obs):
    for j in range(i+1,num_obs):
      total += scipy.spatial.distance.euclidean(data[i],data[j])

  return total

def correlation(data):

  total = 0

  with np.errstate(divide='ignore',invalid='ignore'):
    corr = np.corrcoef(data)

  # xij = NaN only occurs if either the variable xi or xj had variance 0
  # Then, cov(xi, xj) = 0
  # Therefore, we can set NaN values to 0
  corr[np.isnan(corr)] = 0

  num_vars = corr.shape[0]
  for i in range(num_vars):
    for j in range(i+1,num_vars):
      total += abs(corr[i,j])

  return total

def condition(data):
  return np.linalg.cond(data, p=2)


import argparse
import tempfile
from pathlib import Path

DEFAULT_DIR = Path(tempfile.gettempdir()) / "ya-drone-swarm"
DEFAULT_PROVIDERS = 2

DEFAULT_CPU_QUOTA = 1024
DEFAULT_CPU_SHARES = 0.5  # core percentage
DEFAULT_MEM = 512  # MB

def arg_parser():
    parser = argparse.ArgumentParser(prog="ya-drone-swarm")
    parser.add_argument(
        "-d",
        "--dir",
        type=str,
        default=DEFAULT_DIR,
        help="override working directory",
    )

    subparsers = parser.add_subparsers(dest="cmd", help="command to run")
    parser_up = subparsers.add_parser("up", help="bring up containers")
    parser_env = subparsers.add_parser("env", help="start environment shell")
    subparsers.add_parser("ps", help="show processes")

    _args_up(parser_up)
    _args_env(parser_env)

    return parser

def _args_up(parser):
    def providers(value):
        count = int(value)
        if count <= 0:
            raise ValueError("Number of providers must be greater than 0")
        return count

    def caps(value):
        if value.lower() == "none":
            return 0, 0
        values = value.split(",")
        if len(values) != 2:
            raise ValueError(f"Invalid caps: {value}, expected 'cpu,mem'")
        return float(values[0]), int(values[1])

    default_caps = f"none"

    parser.add_argument(
        "-s",
        "--subnet",
        default="drones",
        type=str,
        help="subnet to use",
    )
    parser.add_argument(
        "-b",
        "--binary",
        default=[],
        action="append",
        nargs=2,
        metavar=("name", "path"),
        help="override binary path",
    )
    parser.add_argument(
        "-p",
        "--providers",
        default=DEFAULT_PROVIDERS,
        type=providers,
        help="number of providers",
    )
    parser.add_argument(
        "--cap-net",
        default=default_caps,
        type=caps,
        help="net resource cap [cpu_percent,mem_MB]",
    )
    parser.add_argument(
        "--cap-req",
        default=default_caps,
        type=caps,
        help="requestor resource cap [cpu_percent,mem_MB]",
    )
    parser.add_argument(
        "--cap-prov",
        default=default_caps,
        type=caps,
        help="provider resource cap [cpu_percent,mem_MB]",
    )
    parser.add_argument(
        "-c", "--central-net", action="store_true", help="use central network"
    )

def _args_env(parser):
    parser.add_argument(
        "-n",
        "--no-init",
        action="store_true",
        help="do not run requestor setup",
    )


from enum import Enum
import os
import re

Test.Summary = 'Exercise HTTP CONNECT Method'
Test.ContinueOnFail = True

class ConnectTest:

    class State(Enum):
        """
        State of process
        """
        INIT = 0
        RUNNING = 1

    def __init__(self):
        self.state = self.State.INIT
        self.__setupOriginServer()
        self.__setupTS()

    def __setupOriginServer(self):
        self.httpbin = Test.MakeHttpBinServer("httpbin")

    def __setupTS(self):
        self.ts = Test.MakeATSProcess("ts")

        self.ts.Disk.records_config.update(
            {
                'proxy.config.diags.debug.enabled': 1,
                'proxy.config.diags.debug.tags': 'http',
                'proxy.config.http.server_ports': f"{self.ts.Variables.port}",
                'proxy.config.http.connect_ports': f"{self.httpbin.Variables.Port}",
            })

        self.ts.Disk.remap_config.AddLines([
            f"map http://foo.com/ http://127.0.0.1:{self.httpbin.Variables.Port}/",
        ])

        self.ts.Disk.logging_yaml.AddLines(
            '''
logging:
  formats:
    - name: common
      format: '%<chi> - %<caun> [%<cqtn>] "%<cqhm> %<pqu> %<cqpv>" %<pssc> %<pscl>'
  logs:
    - filename: access
      format: common
'''.split("\n"))

    def __checkProcessBefore(self, tr):
        if self.state == self.State.RUNNING:
            tr.StillRunningBefore = self.httpbin
            tr.StillRunningBefore = self.ts
        else:
            tr.Processes.Default.StartBefore(self.httpbin, ready=When.PortOpen(self.httpbin.Variables.Port))
            tr.Processes.Default.StartBefore(self.ts)
            self.state = self.State.RUNNING

    def __checkProcessAfter(self, tr):
        assert (self.state == self.State.RUNNING)
        tr.StillRunningAfter = self.httpbin
        tr.StillRunningAfter = self.ts

    def __testCase0(self):
        tr = Test.AddTestRun()
        self.__checkProcessBefore(tr)
        tr.Processes.Default.Command = f"curl -v --fail -s -p -x 127.0.0.1:{self.ts.Variables.port} 'http://foo.com/get'"
        tr.Processes.Default.ReturnCode = 0
        tr.Processes.Default.Streams.stderr = "gold/connect_0_stderr.gold"
        tr.Processes.Default.TimeOut = 3
        self.__checkProcessAfter(tr)

    def __testAccessLog(self):
        """Wait for log file to appear, then wait one extra second to make sure TS is done writing it."""
        Test.Disk.File(os.path.join(self.ts.Variables.LOGDIR, 'access.log'), exists=True, content='gold/connect_access.gold')

        tr = Test.AddTestRun()
        tr.Processes.Default.Command = (
            os.path.join(Test.Variables.AtsTestToolsDir, 'condwait') + ' 60 1 -f ' +
            os.path.join(self.ts.Variables.LOGDIR, 'access.log'))
        tr.Processes.Default.ReturnCode = 0

    def run(self):
        self.__testCase0()
        self.__testAccessLog()

ConnectTest().run()

class ConnectViaPVTest:
    # This test also executes the CONNECT request but using proxy verifier to
    # generate traffic
    connectReplayFile = "replays/connect.replay.yaml"

    def __init__(self):
        self.setupOriginServer()
        self.setupTS()

    def setupOriginServer(self):
        self.server = Test.MakeVerifierServerProcess("connect-verifier-server", self.connectReplayFile)
        # Verify server output
        self.server.Streams.stdout += Testers.ExcludesExpression("uuid: 1", "Verify the CONNECT request doesn't reach the server.")
        self.server.Streams.stdout += Testers.ContainsExpression(
            "GET /get HTTP/1.1\nuuid: 2", reflags=re.MULTILINE, description="Verify the server gets the second request.")

    def setupTS(self):
        self.ts = Test.MakeATSProcess("connect-ts")

        self.ts.Disk.records_config.update(
            {
                'proxy.config.diags.debug.enabled': 1,
                'proxy.config.diags.debug.tags': 'http|iocore_net|rec',
                'proxy.config.http.server_ports': f"{self.ts.Variables.port}",
                'proxy.config.http.connect_ports': f"{self.server.Variables.http_port}",
            })

        self.ts.Disk.remap_config.AddLines([
            f"map / http://127.0.0.1:{self.server.Variables.http_port}/",
        ])
        # Verify ts logs
        self.ts.Disk.traffic_out.Content += Testers.ContainsExpression(
            f"Proxy's Request.*\n.*\nCONNECT 127.0.0.1:{self.server.Variables.http_port} HTTP/1.1",
            reflags=re.MULTILINE,
            description="Verify that ATS recognizes the CONNECT request.")

    def runTraffic(self):
        tr = Test.AddTestRun("Verify correct handling of CONNECT request")
        tr.AddVerifierClientProcess(
            "connect-client", self.connectReplayFile, http_ports=[self.ts.Variables.port], other_args='--thread-limit 1')
        tr.Processes.Default.StartBefore(self.server)
        tr.Processes.Default.StartBefore(self.ts)
        tr.StillRunningAfter = self.server
        tr.StillRunningAfter = self.ts

    def __testMetrics(self):
        tr = Test.AddTestRun("Test metrics")
        tr.Processes.Default.Command = (
            f"{Test.Variables.AtsTestToolsDir}/stdout_wait" + " 'traffic_ctl metric get" +
            " proxy.process.http.total_incoming_connections" + " proxy.process.http.total_client_connections" +
            " proxy.process.http.total_client_connections_ipv4" + " proxy.process.http.total_client_connections_ipv6" +
            " proxy.process.http.total_server_connections" + " proxy.process.http2.total_client_connections" +
            " proxy.process.http.connect_requests" + " proxy.process.tunnel.total_client_connections_blind_tcp" +
            " proxy.process.tunnel.current_client_connections_blind_tcp" +
            " proxy.process.tunnel.total_server_connections_blind_tcp" +
            " proxy.process.tunnel.current_server_connections_blind_tcp" +
            " proxy.process.tunnel.total_client_connections_tls_tunnel" +
            " proxy.process.tunnel.current_client_connections_tls_tunnel" +
            " proxy.process.tunnel.total_client_connections_tls_forward" +
            " proxy.process.tunnel.current_client_connections_tls_forward" +
            " proxy.process.tunnel.total_client_connections_tls_partial_blind" +
            " proxy.process.tunnel.current_client_connections_tls_partial_blind" +
            " proxy.process.tunnel.total_client_connections_tls_http" +
            " proxy.process.tunnel.current_client_connections_tls_http" + " proxy.process.tunnel.total_server_connections_tls" +
            " proxy.process.tunnel.current_server_connections_tls'" + f" {Test.TestDirectory}/gold/metrics.gold")
        # Need to copy over the environment so traffic_ctl knows where to find the unix domain socket
        tr.Processes.Default.Env = self.ts.Env
        tr.Processes.Default.ReturnCode = 0
        tr.StillRunningAfter = self.server
        tr.StillRunningAfter = self.ts

    def run(self):
        self.runTraffic()
        self.__testMetrics()

ConnectViaPVTest().run()

class ConnectViaPVTest2:
    # This test executes a HTTP/2 CONNECT request with Proxy Verifier.
    connectReplayFile = "replays/connect_h2.replay.yaml"

    def __init__(self):
        self.setupOriginServer()
        self.setupTS()

    def setupOriginServer(self):
        self.server = Test.MakeVerifierServerProcess("connect-verifier-server2", self.connectReplayFile)
        # Verify server output
        self.server.Streams.stdout += Testers.ExcludesExpression(
            "test: connect-request", "Verify the CONNECT request doesn't reach the server.")
        self.server.Streams.stdout += Testers.ContainsExpression(
            "GET /get HTTP/1.1\nuuid: 1\ntest: real-request",
            reflags=re.MULTILINE,
            description="Verify the server gets the second(tunneled) request.")

    def setupTS(self):
        self.ts = Test.MakeATSProcess("connect-ts2", enable_tls=True)

        self.ts.Disk.records_config.update(
            {
                'proxy.config.diags.debug.enabled': 1,
                'proxy.config.diags.debug.tags': 'http|hpack',
                'proxy.config.ssl.server.cert.path': f'{self.ts.Variables.SSLDir}',
                'proxy.config.ssl.server.private_key.path': f'{self.ts.Variables.SSLDir}',
                'proxy.config.http.server_ports': f"{self.ts.Variables.ssl_port}:ssl",
                'proxy.config.http.connect_ports': f"{self.server.Variables.http_port}",
            })

        self.ts.addDefaultSSLFiles()
        self.ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key')

        self.ts.Disk.remap_config.AddLines([
            f"map / http://127.0.0.1:{self.server.Variables.http_port}/",
        ])
        # Verify ts logs
        self.ts.Disk.traffic_out.Content += Testers.ContainsExpression(
            f"Proxy's Request.*\n.*\nCONNECT 127.0.0.1:{self.server.Variables.http_port} HTTP/1.1",
            reflags=re.MULTILINE,
            description="Verify that ATS recognizes the CONNECT request.")

    def runTraffic(self):
        tr = Test.AddTestRun("Verify correct handling of CONNECT request on HTTP/2")
        tr.AddVerifierClientProcess(
            "connect-client2", self.connectReplayFile, https_ports=[self.ts.Variables.ssl_port], other_args='--thread-limit 1')
        tr.Processes.Default.StartBefore(self.server)
        tr.Processes.Default.StartBefore(self.ts)
        tr.StillRunningAfter = self.server
        tr.StillRunningAfter = self.ts

    def run(self):
        self.runTraffic()

ConnectViaPVTest2().run()