import json, time, datetime

class Colors:
    """Contains general stdout colors."""
    cyan = '\033[96m'
    red = '\033[91m'
    green = '\033[92m'
    end = '\033[0m'

class IsoBankManager():
    def __init__(self, db_path:str, auth_db:str):
        self.db_path = db_path
        self.auth_db = auth_db
        with open(db_path, 'r') as f:
            global accounts
            accounts = json.load(f)
        with open(auth_db, 'r') as f:
            global auth
            auth = json.load(f)
        print(f"[Framework/Loader] {Colors.green}IsoBank Account Manager initialized.{Colors.end}")

    def save(self):
        with open(self.db_path, 'w+') as f: json.dump(accounts, indent=4)

    def deposit(self, account_id:int, discord_id:int, amount:int):
        if discord_id not in auth[str(account_id)]["discord_ids"]: return "Failed to authorize user"
        accounts["deposited"] += amount
        save()
        return f"Deposited {amount} coins to #{account_id} IsoBank account"

    def withdraw(self, account_id:int, discord_id:int, amount:int):
        if discord_id not in auth[str(account_id)]["discord_ids"]: return "Failed to authorize user"
        accounts["deposited"] -= amount
        save()
        return f"Withdrawn {amount} coins from #{account_id} IsoBank account"


import asyncio
import logging
import threading
from time import sleep
from typing import Optional

from aleph_message.models import ItemHash
from schedule import Job, Scheduler

from aleph.vm.conf import settings
from aleph.vm.orchestrator.models import VmExecution

from .snapshots import CompressedDiskVolumeSnapshot

logger = logging.getLogger(__name__)

def wrap_async_snapshot(execution):
    asyncio.run(do_execution_snapshot(execution))

def run_threaded_snapshot(execution):
    job_thread = threading.Thread(target=wrap_async_snapshot, args=(execution,))
    job_thread.start()

async def do_execution_snapshot(execution: VmExecution) -> CompressedDiskVolumeSnapshot:
    try:
        logger.debug(f"Starting new snapshot for VM {execution.vm_hash}")
        assert execution.vm, "VM execution not set"

        snapshot = await execution.vm.create_snapshot()
        await snapshot.upload()

        logger.debug(f"New snapshots for VM {execution.vm_hash} created in {snapshot.path}")
        return snapshot
    except ValueError:
        msg = "Something failed taking an snapshot"
        raise ValueError(msg)

def infinite_run_scheduler_jobs(scheduler: Scheduler) -> None:
    while True:
        scheduler.run_pending()
        sleep(1)

class SnapshotExecution:
    vm_hash: ItemHash
    execution: VmExecution
    frequency: int
    _scheduler: Scheduler
    _job: Job

    def __init__(
        self,
        scheduler: Scheduler,
        vm_hash: ItemHash,
        execution: VmExecution,
        frequency: int,
    ):
        self.vm_hash = vm_hash
        self.execution = execution
        self.frequency = frequency
        self._scheduler = scheduler

    async def start(self) -> None:
        logger.debug(f"Starting snapshots for VM {self.vm_hash} every {self.frequency} minutes")
        job = self._scheduler.every(self.frequency).minutes.do(run_threaded_snapshot, self.execution)
        self._job = job

    async def stop(self) -> None:
        logger.debug(f"Stopping snapshots for VM {self.vm_hash}")
        self._scheduler.cancel_job(self._job)

class SnapshotManager:
    """
    Manage VM snapshots.
    """

    executions: dict[ItemHash, SnapshotExecution]
    _scheduler: Scheduler

    def __init__(self):
        self.executions = {}
        self._scheduler = Scheduler()

    def run_snapshots(self) -> None:
        job_thread = threading.Thread(
            target=infinite_run_scheduler_jobs,
            args=[self._scheduler],
            daemon=True,
            name="SnapshotManager",
        )
        job_thread.start()

    async def start_for(self, execution: VmExecution, frequency: Optional[int] = None) -> None:
        if not execution.is_instance:
            msg = "Snapshots are not implemented for programs."
            raise NotImplementedError(msg)

        default_frequency = frequency or settings.SNAPSHOT_FREQUENCY

        vm_hash = execution.vm_hash
        snapshot_execution = SnapshotExecution(
            scheduler=self._scheduler,
            vm_hash=vm_hash,
            execution=execution,
            frequency=default_frequency,
        )
        self.executions[vm_hash] = snapshot_execution
        await snapshot_execution.start()

    async def stop_for(self, vm_hash: ItemHash) -> None:
        try:
            snapshot_execution = self.executions.pop(vm_hash)
        except KeyError:
            logger.warning("Could not find snapshot task for instance %s", vm_hash)
            return

        await snapshot_execution.stop()

    async def stop_all(self) -> None:
        await asyncio.gather(*(self.stop_for(vm_hash) for vm_hash, execution in self.executions))


class Yuumi:
    from configLoader import fileLoader
    import numpy as np
    import cv2
    from PIL import ImageGrab
    import pyautogui
    from imageManipulation import imageRec

    cfg = fileLoader("config.cfg")

    cfg.readConfig()

    cfgData = cfg.cfg
    print(cfgData)

    cfg.close()

    imgrec = imageRec()

    # variables:
    # cfg is object
    # cfgDict is dictionary of cfg data

    init = True

    while True:
        img = ImageGrab.grab(cfgData[0]) #BGR not RGB
        img_np = np.array(img)
        mapFrame = cv2.cvtColor(img_np, cv2.COLOR_BGR2RGB)

        hudImg = ImageGrab.grab(cfgData[1])
        hudImg_np = np.array(hudImg)
        hudFrame = cv2.cvtColor(hudImg_np, cv2.COLOR_BGR2RGB)

        champImg = ImageGrab.grab(cfgData[8])
        champImg_np = np.array(champImg)
        champFrame = cv2.cvtColor(champImg_np, cv2.COLOR_BGR2RGB)

        cv2.imshow("f1", mapFrame)
        cv2.imshow("f2", hudFrame)
        cv2.imshow("f3", champFrame)
        #print(pyautogui.position())
        imgrec.getHealthValue(cfgData[9][0], cfgData[9][1], cfgData[9][2], cfgData[9][3])

        if init:
            cv2.moveWindow("f1",cfgData[2],cfgData[3])
            cv2.moveWindow("f2",cfgData[4],cfgData[5])
            cv2.moveWindow("f3",cfgData[6],cfgData[7])
            init = False

        if cv2.waitKey(1) == 27:
            break

    cv2.destroyAllWindows()


import json
from tinydb import TinyDB

class Joueur:
    """
    Classe servant a créer une instance de joueur, et regroupant divers methodes pour la recuperation de data.
    type d'arguments : arguments nommés, str, int
    """

    def __init__(self, nom="", prenom="", date_de_naissance="", sexe="", classement=0):
        self.nom = nom
        self.prenom = prenom
        self.date_de_naissance = date_de_naissance
        self.sexe = sexe
        self.classement = classement

    def serialiser_instance_joueur(self):
        """
        Methode servant à serialiser une instance de joueur.
        type d'arguments : str
        """
        serialized_player = {
            'nom': self.nom,
            'prenom': self.prenom,
            'date de naissance': self.date_de_naissance,
            'sexe': self.sexe,
            'classement': self.classement
        }
        db = TinyDB('data/joueurs.json')
        table_joueurs = db.table('Joueur')
        table_joueurs.insert(serialized_player)

    def joueurs_alpha():
        """
        Methode servant a trier les joueurs par ordre alphabetique.
        return : tulpe nom prenom des joueurs
        rtype : tulpe
        """
        with open('data/joueurs.json', 'r') as joueurs_data:
            data_dict = json.load(joueurs_data)
            nom_list = []
            nom = ''
            prenom = ''
            for joueur in data_dict.values():
                for num_joueur in joueur.values():
                    for cle, valeur in num_joueur.items():
                        if cle == 'nom':
                            nom = valeur
                        if cle == 'prenom':
                            prenom = valeur
                    nom_prenom = nom + ' ' + prenom + '\n'
                    nom_list.append(nom_prenom)
                    nom_list.sort()
            if len(nom_list) < 2:
                return 0
            if len(nom_list) >= 2:
                del nom_list[0]
                return nom_list

    def joueurs_classement():
        """
        Methode servant a trier les joueurs par ordre de classement.
        return : tulpe classement nom prenom des joueurs
        rtype : tulpe
        """
        with open('data/joueurs.json', 'r') as joueurs_data:
            data_dict = json.load(joueurs_data)
            classement_nom_list = []
            nom = ''
            prenom = ''
            classement = ''
            for joueur in data_dict.values():
                for num_joueur in joueur.values():
                    for cle, valeur in num_joueur.items():
                        if cle == 'nom':
                            nom = valeur
                        if cle == 'prenom':
                            prenom = valeur
                        if cle == 'classement':
                            classement = 'Classement : ' + str(valeur)
                    nom_prenom_classement = classement + '\n    ' + nom + ' ' + prenom + '\n'
                    classement_nom_list.append(nom_prenom_classement)
                    classement_nom_list.sort()
            if len(classement_nom_list) < 2:
                return 0
            if len(classement_nom_list) >= 2:
                del classement_nom_list[0]
                return classement_nom_list

    def joueurs_tournoi():
        """
        Methode servant a regroupant les joueurs.
        return : tulpe indice nom prenom classement des joueurs
        rtype : tulpe
        """
        with open('data/joueurs.json', 'r') as joueurs_data:
            data_dict = json.load(joueurs_data)
            classement_nom_list = []
            indice_joueur = 0
            nom = ''
            prenom = ''
            classement = ''
            for joueur in data_dict.values():
                for num_joueur in joueur.values():
                    for cle, valeur in num_joueur.items():
                        if cle == 'nom':
                            nom = valeur
                        if cle == 'prenom':
                            prenom = valeur
                        if cle == 'classement':
                            classement = 'Classement : ' + str(valeur)
                    indice_nom_prenom_classement = 'Indice joueur : ' + str(indice_joueur) + '\n   ' + nom + \
                        ' ' + prenom + '\n   ' + classement + '\n'
                    classement_nom_list.append(indice_nom_prenom_classement)
                    indice_joueur += 1
            if len(classement_nom_list) < 9:
                return 0
            if len(classement_nom_list) >= 9:
                del classement_nom_list[0]
                return classement_nom_list

    def get_joueurs_tournoi(liste_indice_selection, liste_nom_joueurs):
        """
        Methode servant récuperer des joueurs selectionnés par leur indice.
        type d'arguments : int, tulpe
        return : tulpe nom prenom des joueurs
        rtype : tulpe
        """
        liste_nom_joueurs_selection = []
        liste_nom_joueurs_new = []
        for arg in liste_nom_joueurs:
            arg = str(arg).replace("\n", "")
            liste_nom_joueurs_new.append(arg)
        for indice in liste_indice_selection:
            liste_nom_joueurs_selection.append(liste_nom_joueurs_new[indice-1])
        return liste_nom_joueurs_selection

    def get_classement_joueur(joueur_selecion):
        """
        Methode servant a récuperer le classement d'un joueur.
        type d'argument : str
        return : classement
        rtype : int
        """
        with open('data/joueurs.json', 'r') as joueurs_data:
            data_dict = json.load(joueurs_data)
            for joueurs in data_dict.values():
                for num_joueurs in joueurs.values():
                    for cle, valeur in num_joueurs.items():
                        if cle == "nom":
                            nom = valeur
                        if cle == "prenom":
                            prenom = valeur
                    nom_prenom = nom + " " + prenom
                    if nom_prenom == joueur_selecion:
                        for cle, valeur in num_joueurs.items():
                            if cle == "classement":
                                joueur_selecion = valeur
                                return joueur_selecion


from django.shortcuts import render, redirect

from recipes.app.forms import RecipeForm, DeleteRecipeForm
from recipes.app.models import Recipe

def show_home(request):
    recipes = Recipe.objects.all()
    return render(request, 'index.html', {'recipes': recipes})

def create_recipe(request):
    if request.method == 'GET':
        form = RecipeForm()
        return render(request, 'create.html', {'form': form})
    else:
        form = RecipeForm(request.POST)
        if form.is_valid():
            form.save()
            return redirect('home')
        else:
            return render(request, 'create.html', {'form': form})

def edit_recipe(request, pk):
    recipe = Recipe.objects.get(pk=pk)
    if request.method == 'GET':
        form = RecipeForm(instance=recipe)
        return render(request, 'edit.html', {'form': form})
    else:
        form = RecipeForm(request.POST, instance=recipe)
        if form.is_valid():
            form.save()
            return redirect('home')
        else:
            return render(request, 'edit.html', {'form': form})

def delete_recipe(request, pk):
    recipe = Recipe.objects.get(pk=pk)
    if request.method == 'GET':
        form = DeleteRecipeForm(instance=recipe)
        return render(request, 'delete.html', {'form': form})
    else:
        recipe.delete()
        return redirect('home')

def recipe_details(request, pk):
    recipe = Recipe.objects.get(pk=pk)
    ingredients = recipe.ingredients.split(', ')
    return render(request, 'details.html', {'recipe': recipe, 'ingredients': ingredients})


import os
import yaml
import time
import shutil
import torch
import random
import argparse
import numpy as np
from tensorboardX import SummaryWriter
from torch.utils import data
from models import get_model
from loader import get_loader
from optimizers import get_optimizer
from loss import get_loss_function
from utils import save_checkpoint, AverageMeter, accuracy, get_logger

def train(cfg, writer, logger, logdir):

    # Setup seeds
    if cfg['training']['seed'] is not None:
        seed = cfg['training']['seed']
        logger.info("Using seed {}".format(seed))
        torch.manual_seed(cfg.get("seed", cfg['training']['seed']))
        torch.cuda.manual_seed(cfg.get("seed", cfg['training']['seed']))
        np.random.seed(cfg.get("seed", cfg['training']['seed']))
        random.seed(cfg.get("seed", cfg['training']['seed']))

    # Setup device
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    # Setup Dataloader
    t_loader = get_loader(cfg,'train')
    v_loader = get_loader(cfg,'val')

    trainloader = data.DataLoader(
        t_loader,
        batch_size=cfg["training"]["batch_size"],
        num_workers=cfg["training"]["n_workers"],
        shuffle=True,
    )

    valloader = data.DataLoader(
        v_loader,
        batch_size=cfg["training"]["batch_size"],
        num_workers=cfg["training"]["n_workers"]
    )
    n_classes = cfg['data']['num_classes']

    # Setup Model
    model_cfg = {}
    for item in cfg["model"]:
        if cfg["model"][item] is not None:
            model_cfg[item] = cfg["model"][item]
    model = get_model(**model_cfg, num_classes=n_classes, device=device).to(device)
    model = torch.nn.DataParallel(model, device_ids=range(torch.cuda.device_count()))

    # Setup optimizer, lr_scheduler and loss function
    optimizer_cls = get_optimizer(cfg["training"]["optimizer"])
    optimizer_params = {k: v for k, v in cfg["training"]["optimizer"].items() if k != "name"}
    optimizer = optimizer_cls(model.parameters(), **optimizer_params)
    logger.info("Using optimizer {}".format(optimizer))
    loss_fn = get_loss_function(cfg["training"]["loss"])
    logger.info("Using loss {}".format(loss_fn))

    if cfg['training']['scheduler'] == 'cos':
        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=cfg['training']['train_epoch'])
    elif cfg['training']['scheduler'] == 'wide':
        scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, [60, 120, 160], 0.2)

    start_epoch = 0
    best_acc1 = -100.0

    # Resume pre-trained model
    if cfg["training"]["resume"] is not None:
        if os.path.isfile(cfg["training"]["resume"]):
            logger.info(
                "Loading model and optimizer from checkpoint '{}'".format(cfg["training"]["resume"])
            )
            checkpoint = torch.load(cfg["training"]["resume"])
            model.load_state_dict(checkpoint["state_dict"])
            optimizer.load_state_dict(checkpoint["optimizer"])
            best_acc1 = checkpoint["best_acc1"]
            start_epoch = checkpoint["epoch"]
            logger.info(
                "Loaded checkpoint '{}' (iter {})".format(
                    cfg["training"]["resume"], checkpoint["epoch"]
                )
            )
        else:
            logger.info("No checkpoint found at '{}'".format(cfg["training"]["resume"]))

  #================================ Training ==========================================
    for epoch in range(start_epoch, cfg['training']['train_epoch']):

        # Initilize trianing statistics
        batch_time = AverageMeter('Time', ':6.3f')
        data_time = AverageMeter('Data', ':6.3f')
        losses = AverageMeter('Loss', ':.4e')
        top1 = AverageMeter('Acc@1', ':6.2f')
        top5 = AverageMeter('Acc@5', ':6.2f')

        end = time.time()
        model.train()
        for i,(image, target) in enumerate(trainloader):
            data_time.update(time.time() - end)
            image = image.to(device)
            target = target.to(device)
            logit,_,_,_= model(image)
            loss = loss_fn(logit, target)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            # Measure accuracy and record loss
            acc1, acc5 = accuracy(logit, target, topk=(1, 5))
            losses.update(loss.item(), image.size(0))
            top1.update(acc1[0], image.size(0))
            top5.update(acc5[0], image.size(0))

            # Measure elapsed time
            batch_time.update(time.time() - end)
            end = time.time()

            if i % cfg["training"]["print_interval"] == 0:
                output = ('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t'
                        'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                        'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                        'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                        'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                        'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
                    epoch, i, len(trainloader), batch_time=batch_time,
                    data_time=data_time, loss=losses, top1=top1, top5=top5, lr=optimizer.param_groups[-1]['lr']))  # TODO
                print(output)
                logger.info(output + '\n')

        # Write stats to tensorboard
        writer.add_scalar('train/loss', losses.avg, epoch)
        writer.add_scalar('train/acc_top1', top1.avg, epoch)
        writer.add_scalar('train/acc_top5', top5.avg, epoch)
        writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)

        # Update scheduler for learning rate decay
        scheduler.step()
        ##================== Evaluation ============================

        model.eval()
        with torch.no_grad():
            eval_top1 = AverageMeter('Acc@1', ':6.2f')
            eval_top5 = AverageMeter('Acc@5', ':6.2f')
            val_losses = AverageMeter('Loss', ':.4e')

            for i, (image, target) in enumerate(valloader):
                image = image.to(device)
                target = target.to(device)
                logit,_,_,_ = model(image)
                loss = loss_fn(logit, target)

                acc1, acc5 = accuracy(logit, target, topk=(1, 5))
                eval_top1.update(acc1[0], image.size(0))
                eval_top5.update(acc5[0], image.size(0))
                val_losses.update(loss.item(), image.size(0))

                if i % cfg["training"]["print_interval"] == 0:
                    output = ('Test: [{0}][{1}/{2}], lr: {lr:.5f}\t'
                            'Prec@1 {top1.avg:.3f}\t'
                            'Prec@5 {top5.avg:.3f}'.format(
                        epoch, i, len(valloader), top1=eval_top1, top5=eval_top5, lr=optimizer.param_groups[-1]['lr']))  # TODO
                    print(output)

                    output = ('validation Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}'
                            .format(top1=eval_top1, top5=eval_top5))
                    logger.info(output + '\n')

                writer.add_scalar('/loss', val_losses.avg, epoch)
                writer.add_scalar('/acc_top1', eval_top1.avg, epoch)
                writer.add_scalar('/acc_top5', eval_top5.avg, epoch)

            is_best = eval_top1.avg > best_acc1
            best_acc1 = max(eval_top1.avg, best_acc1)

        output_best = 'Best Prec@1: %.3f' % (best_acc1)
        logger.info(output_best + '\n')
        print(output_best)

        if (epoch+1) %200 == 0:
            filename = '{}/ckpt.{}.pth.tar'.format(logdir,epoch)
            torch.save({
                'epoch': epoch,
                'arch': cfg["model"]["arch"],
                'state_dict': model.state_dict(),
                'best_acc1': best_acc1,
                'optimizer' : optimizer.state_dict(),
            }, filename)

        save_checkpoint(logdir, {
            'epoch': epoch,
            'arch': cfg["model"]["arch"],
            'state_dict': model.state_dict(),
            'best_acc1': best_acc1,
            'optimizer' : optimizer.state_dict(),
        }, is_best)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="config")
    parser.add_argument(
        "--config",
        nargs="?",
        type=str,
        default="./configs/resnet18/wide.yaml",
        help="Configuration file to use",
    )

    args = parser.parse_args()

    with open(args.config) as fp:
        cfg = yaml.safe_load(fp)
        logdir = os.path.join("runs",cfg["data"]["name"],cfg["model"]["arch"],cfg['id'])
        if not os.path.exists(logdir):
            os.makedirs(logdir, exist_ok=True)
        print("RUNDIR: {}".format(logdir))
        shutil.copy(args.config, logdir)
        writer = SummaryWriter(logdir)
        logger = get_logger(logdir)
        logger.info("Let the games begin")
        train(cfg, writer, logger, logdir)


import pandas as pd
from helper import Utilities

util = Utilities()

# step1: get original database to be published
day_profile = pd.read_pickle('../dataset/dataframe_all_binary.pkl')

# (optional) subsample the time series in each raw of the database
res = 15
day_profile = day_profile.iloc[0::5, 0::res]

# step2: specify the desired anonymity level
anonymity_level = 5

# util.sanitize_data will privatize the database according to the desired anonymity level
sanitized_profile = util.sanitize_data(day_profile, distance_metric='euclidean',
                                       anonymity_level=anonymity_level,rep_mode ='mean')


import os, sys, time, math, random

import numpy as np

import numpy.ma as ma

import geopandas as gpd
import rasterio as rio

import rasterio.features as rfeat
import rasterio.windows as rwin

import shapely.geometry as shp

from lib.annotators.Annotator import Annotator, anno_kwargs_key

"""
New candidate selection system to work with single datasets at a time.

Annotating in complete lines instead of greedy coordinate point selection.

"""

cma_kwargs_key = {
    'annotator': ("(Annotator [None])", "Annotator to configure parent parameters by. If no annotator is passed, will use annotator default."),
    'pairs': ("(int [10])", "Number of coordinate pairs to be generated for each candidate grouping."),
    'off_dist': ("(int, meters [1])", "Base distance  to offset candidate coordinate pairs.")
}

cma_kwargs_key.update(("Annotator - {}".format(key), item) for key, item in anno_kwargs_key.items())

class Dynamic_Preloading_Annotator(Annotator):
    """
    Uses a predicted class map represented by a rasterio.DatasetReader raster
    to improve shapely.geometry.Linestring annotations.

    Each set of candidate coordinates held in a shapely.geometry.MultiPoint,
    each Linestring's total candidates represented by a gpd.GeoDataFrame.

    Candidate Generation and weighting configuration stored in class attributes.

    Attributes:
        pairs (int):        Number of coordinate pairs to be generated
                                on either side of source annotation.
        off_dist (int):     Offset distance for each candidate point.
        outer_pairs (int):  Number of coordinate pairs to be generated
                                with multiplicative increment.
        outer_multiplier (float):   Multiplier used to increment outer pairs.
        minimum_weight (float):     Minimum weight for candidate update
        coordinate_precision (float):  Decimal coordinate_precision used to compare coordinates.
        verbosity (0,1):    Determines verbosity for printing indexes.
        use_origin_default (bool [True]):   If True, always use origin as default candidate.
                                            If False, use previous index as default candidate

        random_select_chance (float [0.02]):    Chance to use random candidate selection for each candidate group.
        random_select_range (int [4]):          Distance from top coordinate point that a random candidate may be selected.

    """
    def __init__(self, **kwargs):
        """
        Initialize class map annotator.

        Notes:
            If annotator is passed in keys, update parameters from that annotator's configuration. Otherwise expects the passed parameters.

        """

        # Initialize annotator
        if 'annotator' in kwargs.keys():
            super().__init__(**kwargs['annotator'].__dict__)
        else:
            super().__init__(**kwargs)

        # Default Attributes
        default_attr = dict(
            pairs = 10,                 # pairs to be generated on either side linearly.
            off_dist = 1,               # offset dist incrementable (meters)
            coordinate_precision = 1e-4,# Float precision for coordinate comparison
            weight_buffer = 3,          # Buffer to apply to candidate linestrings when weighting
            L = 0.001,                  # weight for line length consideration when weighting canididates
            min_p = 1e-06,              # minimum valid probability
            normalize_full = False,     # Optionally normalize length over K^2 candidates
            kwargs_key = cma_kwargs_key
        )

        # Allowed Attributes
        allowed_attr = list(default_attr.keys())

        # Update with passed arguments
        default_attr.update(kwargs)

        # Update kwargs
        self.__dict__.update((k,v) for k,v in default_attr.items() if k in allowed_attr)

    ### Visualization and Information

    def get_candidate_range(self):
        """ Returns the maximum range that a candidate may be generated. Ceiling function applied to final value. (Accurate) """

        dist = 0

        for i in range(self.pairs + self.outer_pairs):

            # increment
            dist += self.off_dist

            # if outer_pairs are passed, exponentially apply increase
            if i > self.pairs:
                dist += (self.outer_multiplier*(i-self.pairs))
                dist = np.round(dist, 4)

        return int(np.ceil(dist))

    ### CANDIDATE GENERATION

    def _point_set(self, point, prev_p, next_p):
        """
        Generates a set of candidate points for a single coordinate point.

        Saves offset from source annotation, returning points and respective offset distance as dict.

        Args:
            point (shp.Point):  Target point Designed to generate candidate set.
            prev_p (shp.Point): Previous point in annotation line sequence. Used to determine slope.
            next_p (shp.Point): Next point in annotation line sequence. Used to determine slope.

        Returns:

            shp.MultiPoint: holds all candidate points for the source point.
                            The first point in the multipoint is always the source,
                            points increase in distance from the source after that.
        """

        # Get offset weights
        off_x, off_y = self.get_offsets(prev_p, next_p)

        # Initialize set with source point
        candidates = [point]

        # Initialize offset distance
        dist = 0

        for i in range(self.pairs):

            # increment
            dist += self.off_dist

            # Create coordinate offsets for each axis
            x_dist = off_x * dist
            y_dist = off_y * dist

            # Append points and their respective offset tuples
            candidates.append(shp.Point(((point.x + x_dist), (point.y - y_dist))))
            candidates.append(shp.Point(((point.x - x_dist), (point.y + y_dist))))

        # Filter out geometries that do not lie on raster
        for idx, point in enumerate(candidates):
            if not self.crop_window.contains(point):
                del candidates[idx]

        # VERBOSITY
        if self.verbosity == 1:
            self.coordinate_index += 1
            print("created new coordinate group, index:", self.coordinate_index)

        # May remove, empty sets should never appear if other systems work properly
        if len(candidates) == 0:
            print("WARNING: Empty set of candidate coordinates created.")
            return None

        # Convert list of points to shp.MultiPoint and return
        return shp.MultiPoint(candidates)

    def _generate_points(self, line):
        """ Generates candidate groups over a line from internal configuration.

        Args:
            line (shp.Linestring): Linestring to generate candidate points over.

        Returns:
            list(shp.MultiPoint): list of candidate point groups in set,
                                  including origin and terminating groups.
        """

        if not isinstance(line, shp.LineString):
            print("Annotator Error (generate_points): Invalid source line type, expected: {}       Passed: {}\n\n".format(type(shp.LineString), type(line)))

        origin = shp.Point(line.coords[0])
        target_point = line.interpolate(self.interval)

        # Initialize list of candidate groups with origin candidates.
        candidate_groups = [self._point_set(origin, origin, target_point)]

        i = 0
        while True:
            # prev_p starts as origin, target point starts as index 1
            prev_p, this_p, next_p = line.interpolate(self.interval*i), line.interpolate(self.interval*(i+1)), line.interpolate(self.interval*(i+2))
            i += 1

            candidate_groups.append(self._point_set(this_p, prev_p, next_p))

            # if the current target point has the same coordinates as the endpoint of the line, break
            if self.same_coords(this_p.coords[0], line.coords[-1]):
                break

        # VERBOSITY
        if self.verbosity == 1:
            self.line_index += 1
            print("Coordinate Set generation complete, index:", self.line_index)
            print("Total coordinate sets created:", self.coordinate_index)
            self.coordinate_index = 0

        # filter empty groups
        candidate_groups = [x for x in candidate_groups if x != None]
        return candidate_groups

    ### CANDIDATE SELECTION

    def get_length_range(self, group_a, group_b):
        """ Get the min and max candidate line lengths slowly accross k^2 candidates.

        Args:
            group_a: shp.MultiPoint of target
            group_a: shp.MultiPoint of previous
        Returns:
            min_length (float), max_length (float)
        """

        l_max, l_min = 0,0
        for p_idx, pt in enumerate(group_a):
            lengths = [pt.distance(other_pt) for other_pt in group_b]
            if p_idx > 0:
                lengths += [l_max, l_min]
            l_max = np.amax(lengths)
            l_min = np.amin(lengths)

        return l_min, l_max

    def origin_state(self, origin_group, class_map):
        """ Assigns weights to origin group from internal predicted class map

        Args:
            origin_group (shp.MultiPoint): Group of points to select origin point from.
            class_map (rio.DatasetReader): Predicted class map as raster to read weights from.

        Returns:
            int: optimal starting point's index based on predicted class map weights.
        """

        # Initialize weight list
        weights = []

        origin_points = list(origin_group)

        # Buffer points into polygons for evaluation
        buffed_points = [point.buffer(self.weight_buffer) for point in origin_group]

        for idx, point in enumerate(buffed_points):
            if not self.crop_window.contains(point):
                del origin_points[idx]
                del buffed_points[idx]

        # Window to read candidate group from raster
        group_window = rfeat.geometry_window(class_map, buffed_points)

        # read pixel data from class map using geometry window
        group_data = self.class_map.read(1, window=group_window)

        for idx, candidate_origin in enumerate(buffed_points):

            # Create a boolean mask representing the linestring from the windowed data for this candidate group.
            mask = rfeat.geometry_mask([candidate_origin], out_shape=group_data.shape, transform=rwin.transform(group_window, class_map.transform), all_touched=True)

            # Apply boolean mask to raster's data
            masked_data = ma.array(group_data, mask=mask)

            # Calculate mean weight for segment
            weights.append(masked_data.sum() / ma.compressed(masked_data).shape[0])

        origin_state = {
            'xy': [pt.coords[0] for pt in origin_points],
            'parent_idx': None,
            'weight': weights
        }
        return origin_state

    def get_state_row(self, class_map, target_group, prev_group, weights: list):
        """ Gets a row of state with this set's parent nodes and  their values. """

        candidate_line_sets = []

        # Stores state for candidate group
        state = {
            'xy': [],
            'parent_idx': [],
            'weight': [],
            # NOTE: Future terms.
            # - Shape of line
            # - Direction of shift
            # - Neighboring line segment shapes
            # - - **Similarity in shape between neighboring lines should be highly weighted.
        }

        # Concatenate and buffer all points to create a window containing all needed class map data
        all_points = list(target_group) + list(prev_group)
        group_window = rfeat.geometry_window(class_map, [pt.buffer(self.weight_buffer) for pt in all_points])

        # read pixel data from class map using geometry window
        group_data = class_map.read(1, window=group_window)

        # Get terms for normalization
        if self.normalize_full:
            l_min, l_max = self.get_length_range(target_group, prev_group)

        # Iterate over target group's points
        for anchor_idx, anchor in enumerate(target_group):

            # Reset top index and weight
            top_weight, top_idx = 0,0

            # Create candidate lines connecting this anchor point to every previous point
            candidate_lines = [shp.LineString([anchor, point]).buffer(self.weight_buffer) for point in prev_group.geoms]

            # Get local normalization term
            if not self.normalize_full:
                l_min, l_max = self.get_length_range([anchor], prev_group)

            for idx, (candidate_line, weight_value) in enumerate(zip(candidate_lines, weights)):
                # Create a boolean mask representing the linestring from the windowed data for this candidate group.
                mask = rfeat.geometry_mask([candidate_line], out_shape=group_data.shape, transform=rwin.transform(group_window, class_map.transform), all_touched=True)

                # Apply boolean mask to raster's data
                masked_data = ma.array(group_data, mask=mask)

                # Calculate mean weight for segment
                weight = masked_data.sum() / ma.compressed(masked_data).shape[0]

                # Check weight by threshold
                if weight <= self.min_p:
                    weight = 0.0

                # Calculate line length, normalizing by previously evaluated terms.
                line_length = ((candidate_line.length - l_min) / (l_max - l_min))
                length_value = (self.L * (1 - line_length))
                weight += length_value

                if (idx == 0) or ((weight_value + weight) > top_weight):
                    top_weight = (weight_value + weight)
                    top_idx = idx

            # Update state matrix
            state['xy'].append(anchor.coords[0])
            state['parent_idx'].append(top_idx)
            state['weight'].append(top_weight)

        return state

    def _new_annotation(self, line, class_map=None):
        """
        Create a new annotation from a Linestring and predicted class map
        using internal parameters.

        Args:
            line (shp.LineString): Source annotation to be improved in same CRS map raster.
            class_map (rio.DatasetReader): Predicted class map used to evaluate candidates.
            use_default (bool): Optionally continue annotations with zero-weight candidates by taking the same index point as the previous
        Returns:
            list( (float, float) ): (x, y) coordinate offset tuples from the source annotation
            shp.Linestring: Improved annotation
        """

        if not isinstance(line, shp.LineString):
            raise TypeError("Class_Map_Annotator.new_annotation(): Expected type '{}' for line, recieved: '{}'".format(shp.LineString.__name__, type(line).__name__))

        if (class_map == None) and (self.class_map == None):
            raise RuntimeError("Class_Map_Annotator.new_annotation(): No class map to weight candidates.")

        self.class_map = class_map

        # Generate candidate point groups
        candidate_groups = self._generate_points(line)

        state_matrix = [self.origin_state(candidate_groups[0], class_map=class_map)]

        for i in range(1, len(candidate_groups)):
            state_matrix.append(self.get_state_row(class_map=class_map, target_group=candidate_groups[i], prev_group=candidate_groups[i-1], weights=state_matrix[i-1]['weight']))

        return state_matrix

    def preload_candidates(self, gdf, class_map):
        """
        Generate only the candidate coordinate set for visualization,
        optionally clip to raster.

        Args:
            gdf (gpd.GeoDataFrame): Source annotation to generate candidates from.
            class_map (rio.DatasetReader): Optional raster to clip candidate geometries.
            flatten (bool, default=True): Option to flatten output geometries
                True: Return candidates as a single GeoDataFrame of all candidate points. (Default)
                or
                False: Return candidates as a list of GeoDataFrames (one per geometry in input).
            out_path (filepath): Optional filepath to save candidates. If flatten==False, out_path is ignored.

        Returns:
            gpd.GeoDataFrame: Holds candidate groups as MultiPoints in source CRS. (flatten=True)
            or
            list(gpd.GeoDataFrame): One GeoDataFrame of candidate points for each Linestring in source. (flatten=False)
        """

        self.class_map = class_map

        # set window for cropping geoms and clip geometries
        self.set_crop_window(class_map)
        gdf = gdf.to_crs(class_map.crs)
        gdf = gpd.clip(gdf, mask=self.crop_window, keep_geom_type=True)

        all_line_data = []
        for idx, line in enumerate(gdf.geometry):
            line_data = {
                'windows': [],
                'candidates': [],
            }

            # Get All candidate groups across the line
            candidate_groups = self._generate_points(line)

            # Focus on origin
            target_group = list(candidate_groups[0])
            last_buffed_points = [point.buffer(self.weight_buffer) for point in target_group]

            for d_idx, point in enumerate(last_buffed_points):
                if not self.crop_window.contains(point):
                    del target_group[d_idx]
                    del last_buffed_points[d_idx]

            # Add origin window to this line
            origin_window =rfeat.geometry_window(class_map, last_buffed_points)

            # Get origin masks with simple weight buffer
            origin_masks = [rfeat.geometry_mask([point], out_shape=(origin_window.height, origin_window.width), transform=rwin.transform(origin_window, class_map.transform), all_touched=True) for point in last_buffed_points]

            # update list of candidate dicts with origin data
            line_data['candidates'].append({'geoms': target_group, 'lengths': [], 'masks': origin_masks})
            line_data['windows'].append(origin_window)

            # Iterate over other candidate groups for this line
            for i in range(1, len(candidate_groups)):

                # Clarify group indexes
                prev_group = list(candidate_groups[i-1])
                target_group = list(candidate_groups[i])

                # Initialize lists
                candidate_lengths, candidate_masks = [], []

                # Buffer these points to get a reference candidate window
                buffed_points = [point.buffer(self.weight_buffer) for point in target_group]
                for d_idx, point in enumerate(buffed_points):
                    if not self.crop_window.contains(point):
                        del target_group[d_idx]
                        del buffed_points[d_idx]

                group_window = rfeat.geometry_window(class_map, (buffed_points+last_buffed_points))

                # Iterate over candidate points in each group
                for anchor in target_group:
                    candidate_lines = [shp.LineString([anchor, point]) for point in prev_group]
                    candidate_lengths.append([line.length for line in candidate_lines])

                    candidate_lines = [line.buffer(self.weight_buffer) for line in candidate_lines]
                    candidate_masks.append([rfeat.geometry_mask([line], out_shape=(group_window.height, group_window.width), transform=rwin.transform(group_window, class_map.transform), all_touched=True) for line in candidate_lines])

                line_data['candidates'].append({'geoms': target_group, 'lengths': candidate_lengths, 'masks': candidate_masks})
                line_data['windows'].append(group_window)

                # Reset buffed points
                last_buffed_points = buffed_points

                #print("- Finished Group {} of {} (Line {}).".format(i, len(candidate_groups)-1, idx))

            print("Completed Line {} of {}.\n".format(idx, len(gdf.geometry)-1))
            all_line_data.append(line_data)

        return all_line_data

    def back_prop(self, state_matrix) -> list:
        """ Converts state matrix to updated linestring. """

        # Holds updated shapely points
        new_points = []

        top_weight = 0
        top_idx = 0
        for idx, weight in enumerate(state_matrix[-1]['weight']):
            if weight > top_weight:
                top_weight = weight
                top_idx = idx

        i = 1
        while i <= len(state_matrix):
            new_points.insert(0, shp.Point(state_matrix[-i]['xy'][top_idx]))

            # Final points
            if state_matrix[-i]['parent_idx'] == None: break

            top_idx = state_matrix[-i]['parent_idx'][top_idx]
            i += 1
        return shp.LineString(new_points)

    def origin_state_preloaded(self, origin_data, window, class_map):
        """ Assigns weights to origin group from internal predicted class map

        Args:
            origin_group (shp.MultiPoint): Group of points to select origin point from.
            class_map (rio.DatasetReader): Predicted class map as raster to read weights from.

        Returns:
            int: optimal starting point's index based on predicted class map weights.
        """

        # Initialize weight list
        weights = []

        # read pixel data from class map using geometry window
        group_data = class_map.read(1, window=window)

        for idx, mask in enumerate(origin_data['masks']):

            # Apply boolean mask to raster's data
            masked_data = ma.array(group_data, mask=mask)

            # Calculate mean weight for segment
            weights.append(masked_data.sum() / ma.compressed(masked_data).shape[0])

        origin_state = {
            'xy': [pt.coords[0] for pt in origin_data['geoms']],
            'parent_idx': None,
            'weight': weights
        }
        return origin_state

    def get_state_row_preloaded(self, class_map, group_data, window, weights):
        """ Gets a row of state with this set's parent nodes and  their values. """

        candidate_line_sets = []

        # Stores state for candidate group
        state = {
            'xy': [],
            'parent_idx': [],
            'weight': [],
        }

        group_window = window

        # read pixel data from class map using geometry window
        group_arr = class_map.read(1, window=group_window)

        # Get terms for normalization
        if self.normalize_full:
            all_lens = [l for lengths in group_data['lengths'] for l in lengths]
            l_min, l_max = np.amin(all_lens), np.amax(all_lens)

        # Iterate over target group's points
        for anchor_idx, (anchor, anchor_lengths, anchor_mask_list) in enumerate(zip(group_data['geoms'], group_data['lengths'], group_data['masks'])):

            # Reset top index and weight
            top_weight, top_idx = 0,0

            if not self.normalize_full:
                l_min, l_max = np.amin(anchor_lengths), np.amax(anchor_lengths)

            for idx, (p_length, mask, weight_value) in enumerate(zip(anchor_lengths, anchor_mask_list, weights)):

                # Apply boolean mask to raster's data
                masked_data = ma.array(group_arr, mask=mask)

                # Calculate mean weight for segment
                weight = masked_data.sum() / ma.compressed(masked_data).shape[0]

                # Check weight by threshold
                if weight <= self.min_p:
                    weight = 0.0

                # Calculate line length, normalizing by previously evaluated terms.
                line_length = ((p_length - l_min) / (l_max - l_min))
                length_value = (self.L * (1 - line_length))
                weight += length_value

                if (idx == 0) or ((weight_value + weight) > top_weight):
                    top_weight = (weight_value + weight)
                    top_idx = idx

            # Update state matrix
            state['xy'].append(anchor.coords[0])
            state['parent_idx'].append(top_idx)
            state['weight'].append(top_weight)

        return state

    def new_anno_preloaded(self, line_data: dict, class_map=None):
        """
        Create a new annotation from a Linestring and predicted class map
        using internal parameters.

        Args:
            line (shp.LineString): Source annotation to be improved in same CRS map raster.
            class_map (rio.DatasetReader): Predicted class map used to evaluate candidates.
            use_default (bool): Optionally continue annotations with zero-weight candidates by taking the same index point as the previous
        Returns:
            list( (float, float) ): (x, y) coordinate offset tuples from the source annotation
            shp.Linestring: Improved annotation
        """

        if (class_map == None) and (self.class_map == None):
            raise RuntimeError("Class_Map_Annotator.new_annotation(): No class map to weight candidates.")

        self.class_map = class_map

        state_matrix = []

        for idx, (candidate_data, window) in enumerate(zip(line_data['candidates'], line_data['windows'])):
            if idx == 0:
                # Origin
                state_matrix.append(self.origin_state_preloaded(class_map=class_map, origin_data=candidate_data, window=window))
            else:
                state_matrix.append(self.get_state_row_preloaded(class_map=class_map, group_data=candidate_data, window=window, weights=state_matrix[idx-1]['weight']))

        return self.back_prop(state_matrix)

    def update_gdf_from_preload(self, all_data, class_map, out_path = None):

        lines = []

        for line_data in all_data:
            lines.append(self.new_anno_preloaded(line_data, class_map))

        annotation = gpd.GeoDataFrame(geometry=lines, crs=class_map.crs)

        if out_path:
            annotation.to_file(out_path)

        return annotation

    def get_candidates(self, gdf, class_map=None, flatten=True, out_path=None):
        """
        Generate only the candidate coordinate set for visualization,
        optionally clip to raster.

        Args:
            gdf (gpd.GeoDataFrame): Source annotation to generate candidates from.
            class_map (rio.DatasetReader): Optional raster to clip candidate geometries.
            flatten (bool, default=True): Option to flatten output geometries
                True: Return candidates as a single GeoDataFrame of all candidate points. (Default)
                or
                False: Return candidates as a list of GeoDataFrames (one per geometry in input).
            out_path (filepath): Optional filepath to save candidates. If flatten==False, out_path is ignored.

        Returns:
            gpd.GeoDataFrame: Holds candidate groups as MultiPoints in source CRS. (flatten=True)
            or
            list(gpd.GeoDataFrame): One GeoDataFrame of candidate points for each Linestring in source. (flatten=False)
        """

        if class_map:
            # set window for cropping geoms and clip geometries
            self.set_crop_window(class_map)
            gdf = gpd.clip(gdf, mask=self.crop_window, keep_geom_type=True)

        # Get list of groups of points
        all_sets = [self._generate_points(line) for line in gdf.geometry]

        if flatten:
            # Flatten list, convert to geodataframe
            all_candidates = gpd.GeoDataFrame(geometry=[point_group for line_set in all_sets for point_group in line_set], crs=gdf.crs)

            if out_path:
                if '.shp' not in os.path.splitext(out_path)[1]:
                    print("Warning (update_gdf): Saving candidate point GeoDataFrame to folder instead of file: '{}'.\nTo save as file, include the .shp extension.".format(out_path))

                all_candidates.to_file(out_path)

            return all_candidates
        else:
            # Return list of GeoDataFrames, one for each geometry in source annotation
            return [gpd.GeoDataFrame(geometry=point_set, crs=gdf.crs) for point_set in all_sets]


import datetime

from airflow import models
from airflow.contrib.operators import bigquery_get_data
from airflow.contrib.operators import bigquery_operator

yesterday = datetime.datetime.combine(
    datetime.datetime.today() - datetime.timedelta(1),
    datetime.datetime.min.time())

bq_dataset_name = 'test.users_data_last'

default_dag_args = {
    'start_date': yesterday,
    # To email on failure or retry set 'email' arg to your email and enable
    # emailing here.
    'email_on_failure': False,
    'email_on_retry': False,
    # If a task fails, retry it once after waiting at least 5 minutes
    'retries': 1,
    'retry_delay': datetime.timedelta(minutes=5)
}

# [START bigquery_extracton_test]
with models.DAG(
        'users_data_bigquery_load_test',
        # Continue to run DAG once per day
        schedule_interval=datetime.timedelta(days=1),
        default_args=default_dag_args) as dag:
    # [END bigquery_extracton_test]

    bq_extraction_test = bigquery_operator.BigQueryOperator(
            task_id='bq_extraction_test_query',
            bql="""
            SELECT *
            FROM `test.users_data`
            """,
            use_legacy_sql=False,
            destination_dataset_table=bq_dataset_name)

bq_extraction_test


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from .utils import get_init_file, dictify_params, parse_filenames, openers
from betanegbinfit import distributions as dists
from betanegbinfit.models import ModelWindow
from multiprocessing import Pool, cpu_count
from statsmodels.stats import multitest
from collections import defaultdict
from scipy.stats import chi2, norm
from functools import partial
from copy import deepcopy
from typing import List, Tuple
from scipy.optimize import minimize_scalar
import pandas as pd
import numpy as np
import jax
import dill
import re
import os

class Model():
    def __init__(self, dist: str, left=4, mask_size=0, model_name='window', param_mode='window', r_transform=None,
                 symmetrify=False, bad=1):
        self.dist = dist
        self.left = left
        self.allowed_const = left + 1
        self.mask = np.zeros(mask_size, dtype=bool)
        self.model_name = model_name
        self.param_mode = param_mode
        self.r_transform = r_transform
        self.grad = jax.jit(jax.jacfwd(self.fun, argnums=0))
        self.fim = jax.jit(jax.jacfwd(jax.jacfwd(self.negloglik, argnums=0), argnums=0))
        self.bad = bad
        self.symmetrify = symmetrify

    @partial(jax.jit, static_argnums=(0, ))
    def fun(self, p: float, r: jax.numpy.ndarray, k: jax.numpy.ndarray, data: jax.numpy.ndarray,  w:jax.numpy.ndarray,
            mask: jax.numpy.ndarray):
        left = self.left
        if self.dist == 'NB':
            logl = dists.LeftTruncatedNB.logprob(data, r, p, left, r_transform=self.r_transform)
        elif self.dist == 'MCNB':
            logl = dists.LeftTruncatedMCNB.logprob(data, r, p, left, r_transform=self.r_transform)
        else:
            logl = dists.LeftTruncatedBetaNB.logprob(data, p, k, r, left, r_transform=self.r_transform)
        logl *= w
        return jax.numpy.where(mask, 0.0, logl)

    @partial(jax.jit, static_argnums=(0, ))
    def negloglik(self, p: float, r: jax.numpy.ndarray, k: jax.numpy.ndarray, data: jax.numpy.ndarray,  w:jax.numpy.ndarray,
            mask: jax.numpy.ndarray):
        return -self.fun(p, r, k, data, w, mask).sum()

    def update_mask(self, data, r, k, w):
        mask = self.mask
        n = max(len(r), len(data))
        m = len(mask)
        if n > m:
            mask = np.zeros(n, dtype=bool)
            self.mask = mask
            m = n
        mask[:n] = False
        mask[n:] = True
        c = self.allowed_const
        v = max(0, m - len(data)); data = np.pad(data, (0, v), constant_values=c);
        v = max(0, m - len(w)); w = np.pad(w, (0, v), constant_values=c);
        v = max(0, m - len(r)); r = np.pad(r, (0, v), constant_values=c)
        v = max(0, m - len(k)); k = np.pad(k, (0, v), constant_values=c)
        return data, r, k, w

    def sample(self, n, alt, p, r, k):
        n = n.astype(int)
        alt = np.repeat(alt, n)
        r = np.repeat(r, n)
        k = np.repeat(k, n)
        left = self.left
        if self.dist == 'BetaNB':
            res = dists.LeftTruncatedBetaNB.sample(p, k, r, left, size=(len(r),), r_transform=self.r_transform)
        else:
            res = dists.LeftTruncatedNB.sample(r, p, left, size=(len(r),), r_transform=self.r_transform)
        res = np.stack([res, alt]).T
        res, c = np.unique(res, axis=0, return_counts=True)
        res = np.append(res, c.reshape(-1, 1), axis=1)
        return res

    def minimize_scalar(self, f, xatol=1e-16, steps=10):
        if steps:
            ps = list(np.linspace(0.01, 0.99, steps))
            i = np.nanargmin(list(map(f, ps)))
            ps = [0.0] + ps + [1.0]
            b = ps[i], ps[i + 2]
        else:
            b = (0.0, 1.0)
        return minimize_scalar(f, bounds=b, method='bounded', options={'xatol': xatol, 'maxiter': 200})

    def adjust_r(self, r, k, w=None):
        bad = self.bad
        if w is None or bad == 1:
            return r
        p2 = 1 / (bad + 1)
        p1 = 1 - p2
        left = self.left
        if self.dist == 'NB':
            mean = lambda p: dists.LeftTruncatedNB.mean(r, p, left, r_transform=self.r_transform)
        elif self.dist == 'MCNB':
            mean = lambda p: dists.LeftTruncatedMCNB.mean(r, p, left, r_transform=self.r_transform)
        else:
            mean = lambda p: dists.LeftTruncatedBetaNB.mean(p, k, r, left, r_transform=self.r_transform)
        m1 = mean(p1)
        m2 = mean(p2)
        correction = m1 / (w * m1 + (1.0 - w) * m2)
        return r * correction

    def fit(self, data: np.ndarray, params: dict, compute_var=True, sandwich=True, n_bootstrap=0):
        name = self.model_name
        if self.symmetrify:
            data = ModelWindow.symmetrify_counts(data)
        data, fixed, w = data.T
        n = len(data)
        if self.param_mode == 'window' or self.model_name == 'slices':
            r = np.zeros(n, dtype=float)
            k = np.zeros(n, dtype=float)
            for i in range(n):
                slc = fixed[i]
                ps = get_closest_param(params, slc, name, True)
                r[i] = ps['r']
                if self.dist == 'BetaNB':
                    k[i] = ps.get('k', 6e3)
        else:
            slc = fixed.mean()
            ps = get_closest_param(params, slc, name, False)
            r = ps['mu'] + ps['b'] * fixed ** ps.get('slice_power', 1.0)
            if 'mu_k' in ps:
                k = ps['mu_k'] + ps['b_k'] * np.log(fixed)
            else:
                k = np.zeros(len(r))
            r = self.adjust_r(r, k, ps.get('w', None))
        data, r, k, w = self.update_mask(data, r, k, w)
        mask = self.mask
        f = partial(self.negloglik, r=r, k=k, data=data, w=w, mask=mask)
        grad_w = partial(self.grad, r=r, k=k, data=data, w=w, mask=mask)
        fim = partial(self.fim, r=r, k=k, data=data, w=w, mask=mask)
        res = self.minimize_scalar(f)
        x = res.x
        bs = list()
        for i in range(n_bootstrap):
            r_, k_, w_ =  r[:n], k[:n], w[:n]
            data, _, w_ = self.sample(w_, fixed[:n], x, r_, k_).T
            data, r_, k_, w_ = self.update_mask(data, r_, k_, w_)
            mask = self.mask
            f = partial(self.negloglik, r=r_, k=k_, data=data, w=w_, mask=mask)
            res_ = self.minimize_scalar(f)
            if res_.success:
                bs.append(res_.x)

        res.x = float(res.x)
        correction = res.x - np.mean(bs) if n_bootstrap else 0.0
        res.x = np.clip(res.x + correction, 1e-12, 1.0 - 1e-12)
        lf = float(f(res.x))
        if compute_var:
            if sandwich:
                g = grad_w(res.x)
                v = (g ** 2 / w).sum()
                fim = fim(res.x)
                s = -1 if fim < -1e-9 else 1
                return res, s * float(1 / fim ** 2 * v)
            else:
                return res, float(1 / fim(res.x))
        return res, lf

def get_snvs_for_group(snvs: dict, group: set, min_samples: int = 0, min_cover : int = 0, max_cover : int = np.inf):
    res = dict()
    for k, lt in snvs.items():
        t = list(filter(lambda x: (x[0] in group) and sum(x[1:]) < max_cover, lt[1:]))
        if not t:
            continue
        if len(t) >= min_samples and max(sum(v[1:]) for v in t) >= min_cover:
            res[k] = [lt[0]] + t
    return res

def build_count_table(snvs: dict):
    count = defaultdict(lambda: defaultdict(int))
    for lt in snvs.values():
        lt = lt[1:]
        for t in lt:
            count[t[-1]][tuple(t[1:-1])] += 1
    return count

def count_dict_to_numpy(counts: dict):
    res = dict()
    for bad, counts in counts.items():
        r = list()
        for t in sorted(counts):
            r.append(list(t) + [counts[t]])
        res[bad] = np.array(r, dtype=int)
    return res

def build_count_tables(snvs_a: dict, snvs_b: dict):
    counts_a = build_count_table(snvs_a)
    counts_b = build_count_table(snvs_b)
    counts = deepcopy(counts_a)
    for bad in counts:
        c = counts[bad]
        cb = counts_b[bad]
        for t, n in cb.items():
            c[t] += n
    return map(count_dict_to_numpy, (counts_a, counts_b, counts))

def get_closest_param(params: dict, slc: float, model_name: str, compute_line=False):
    res = dict()
    if model_name == 'line':
        ps = str()
    else:
        if f'mu{round(slc)}' in params:
            ps = round(slc)
        else:
            s = 'mu' if model_name == 'window' else 'r'
            n = len(s)
            ps = int(min(filter(lambda x: x.startswith(s) and x[n].isdigit(), params), key=lambda x: abs(int(x[n:]) - slc))[n:])
    if f'mu{ps}' in params:
        res['mu'] = params[f'mu{ps}']
    if f'b{ps}' in params:
        res['b'] = params[f'b{ps}']
    if f'mu_k{ps}' in params:
        res['mu_k'] = params[f'mu_k{ps}']
    if f'b_k{ps}' in params:
        res['b_k'] = params[f'b_k{ps}']
    if f'r{ps}' in params:
        res['r'] = params[f'k{ps}']
    if f'k{ps}' in params:
        res['k'] = params[f'k{ps}']
    if compute_line:
        if 'mu' in res:
            res['r'] = res['mu'] + res.get('b', 0.0) * slc
        if 'mu_k' in res:
            res['k'] = res['mu_k'] + res.get('b_k', 0.0) * slc
    return res

def lrt_test(counts: Tuple[tuple, np.ndarray, np.ndarray, np.ndarray],
        inst_params: dict, params: dict, skip_failures=False, max_sz=None, bad=1.0,
        param_mode='window', n_bootstrap=0):
    if not hasattr(lrt_test, '_cache'):
        lrt_test._cache = dict()
    snv, counts_a, counts_b, counts = counts
    cache = lrt_test._cache
    key = (inst_params['dist'], inst_params['left'], max_sz if max_sz else 0, inst_params['name'], param_mode,
           inst_params['r_transform'], inst_params['symmetrify'])
    if key not in cache:
        model = Model(*key, bad=bad)
        cache[key] = model
    else:
        model = cache[key]
    res = list()
    for allele in ('ref', 'alt'):
        if allele == 'alt':
            counts_a = counts_a[:, (1, 0, 2)]; counts_b = counts_b[:, (1, 0, 2)]; counts = counts[:, (1, 0, 2)]
        try:
            ab_r, ab_logl = model.fit(counts, params[allele], False, n_bootstrap=n_bootstrap)
            ab_p = ab_r.x
            a_r, a_logl = model.fit(counts_a, params[allele], False, n_bootstrap=n_bootstrap)
            a_p = a_r.x
            b_r, b_logl = model.fit(counts_b, params[allele], False, n_bootstrap=n_bootstrap)
            b_p = b_r.x
            success = a_r.success & b_r.success & ab_r.success
            if not success and skip_failures:
                return None, snv
        except (KeyError, IndexError):
            if skip_failures:
                return None, snv
            res.append((float('nan'), float('nan'), float('nan'), float('nan')))
        lrt = 2 * (ab_logl - (a_logl + b_logl))
        pval = chi2.sf(lrt, 1)
        res.append((pval, ab_p, a_p, b_p))
    n_a, n_b = counts_a[:, -1].sum(), counts_b[:, -1].sum()
    return res, (snv, n_a, n_b)

def calc_var(m, data: np.ndarray, renormalize=None):
    data, weights = data[:, :-1], data[:, -1]
    if renormalize:
        n = weights.sum()
    data, weights, mask = m.update_mask(data, weights)
    var = 1 / m.calc_fim(m.last_result.x, data=data, mask=mask,
                        weights=weights)[0][0][-1, -1]
    if renormalize:
        var *= n / renormalize
    return float(var)

def transform_p(p, var):
    pt = np.log(p) - np.log1p(-p)
    var = var * (1 / p + 1 / (1 - p)) ** 2
    return pt, var

def wald_test(counts: Tuple[tuple, np.ndarray, np.ndarray, np.ndarray],
              inst_params: dict, params: dict, skip_failures=False, max_sz=None, bad=1.0,
              contrasts: Tuple[float, float, float] = (1, -1, 0), logit_transform=False,
              param_mode='window', robust_se=True, n_bootstrap=0):
    if not hasattr(wald_test, '_cache'):
        wald_test._cache = dict()
    snv, counts_a, counts_b, counts = counts
    cache = wald_test._cache
    key = (inst_params['dist'], inst_params['left'], max_sz if max_sz else 0, inst_params['name'], param_mode,
           inst_params['r_transform'], inst_params['symmetrify'])
    if key not in cache:
        model = Model(*key, bad=bad)
        cache[key] = model
    else:
        model = cache[key]
    res = list()
    for allele in ('ref', 'alt'):
        if allele == 'alt':
            counts_a = counts_a[:, (1, 0, 2)]; counts_b = counts_b[:, (1, 0, 2)]; counts = counts[:, (1, 0, 2)]
        try:
            a_r, a_var = model.fit(counts_a, params[allele], sandwich=robust_se, n_bootstrap=n_bootstrap)
            a_var = np.clip(a_var, 0.0, np.inf)
            if (1.0 - a_r.x) ** 2 < a_var:
                a_var = (1.0 - a_r.x) ** 2
            a_p = a_r.x
            b_r, b_var = model.fit(counts_b, params[allele], sandwich=robust_se, n_bootstrap=n_bootstrap)
            b_var = np.clip(b_var, 0.0, np.inf)
            if (1.0 - b_r.x) < b_var ** 0.5:
                b_var = (1.0 - b_r.x) ** 2
            b_p = b_r.x
            correct = (a_var + b_var > 0) & np.isfinite(a_var) & np.isfinite(b_var)
            success = a_r.success & b_r.success & correct
            if not success and skip_failures:
                return None, snv
        except (KeyError, IndexError):
            if skip_failures:
                return None, snv
            res.append((float('nan'), float('nan'), float('nan'), float('nan')))
            continue
        if not correct:
            pval = 1.0
        else:
            a, b, eq = contrasts
            if logit_transform:
                _a_p, _a_var = transform_p(a_p, a_var)
                _b_p, _b_var = transform_p(b_p, b_var)
            else:
                _a_p, _a_var = a_p, a_var
                _b_p, _b_var = b_p, b_var
            stat = abs(a * _a_p + b * _b_p + eq)
            var = a ** 2 * _a_var + b ** 2 * _b_var
            pval = 2 * norm.sf(stat, scale=(var) ** 0.5, loc=0.0)
        res.append((pval, a_p, b_p, a_var ** 0.5, b_var ** 0.5))
    n_a, n_b = counts_a[:, -1].sum(), counts_b[:, -1].sum()
    return res, (snv, n_a, n_b)

def _bad_in(t, bad):
    for t in t[1:]:
        if t[-1] == bad:
            return True
    return False

def differential_test(name: str, group_a: List[str], group_b: List[str], mode='wald', min_samples=2, min_cover=0,
                      max_cover=np.inf, skip_failures=True, group_test=True, alpha=0.05, max_cover_group_test=None,
                      filter_chr=None, filter_id=None, contrasts=(1, -1, 0), subname=None, param_mode='window',
                      logit_transform=False, robust_se=True, n_bootstrap=0, fit: str = None, n_jobs=-1):
    if max_cover is None:
        max_cover = np.inf
    if min_cover is None:
        min_cover = np.inf
    if max_cover_group_test is None:
        max_cover_group_test = max_cover
    if filter_chr is not None:
        filter_chr = re.compile(filter_chr)
    if filter_id is not None:
        filter_id = re.compile(filter_id)
    n_jobs = cpu_count() if n_jobs == -1 else n_jobs
    group_a = parse_filenames(group_a)
    group_b = parse_filenames(group_b)
    init_filename = get_init_file(name)
    compressor = init_filename.split('.')[-1]
    open = openers[compressor]
    with open(init_filename, 'rb') as f:
        snvs = dill.load(f)
        scorefiles = snvs['scorefiles']
        snvs = snvs['snvs']
    group_a = {scorefiles.index(f) for f in group_a}
    group_b = {scorefiles.index(f) for f in group_b}
    assert not (group_a & group_b), 'Groups should not intersect.'
    snvs_a = get_snvs_for_group(snvs, group_a, min_samples=min_samples)
    snvs_b = get_snvs_for_group(snvs, group_b, min_samples=min_samples)
    snvs = set(snvs_a) & set(snvs_b)
    if filter_id:
        snvs = set(filter(lambda x: snvs_a[x][0][1] and filter_id.match(x[1][0][1]), snvs))
    if filter_chr:
        snvs = set(filter(lambda x: filter_chr.match(x[0]), snvs))
    snvs_a = {k: snvs_a[k] for k in snvs}
    snvs_b = {k: snvs_b[k] for k in snvs}
    if fit:
        comp_fit = fit.split('.')[-1]
        open2 = openers[comp_fit]
    else:
        fit = f'{name}.fit.{compressor}'
        open2 = open
    with open2(fit, 'rb') as f:
        fits = dill.load(f)
    if group_test:
        _counts_a, _counts_b, _counts = build_count_tables(snvs_a, snvs_b)
    res = list()
    for bad in fits['ref'].keys():
        params = {'ref': dictify_params(fits['ref'][bad]['params']),
                  'alt': dictify_params(fits['ref'][bad]['params'])}
        inst_params = fits['ref'][1]['inst_params']
        if mode == 'lrt':
            cols = ['ref_pval', 'ref_p', 'ref_p_control', 'ref_p_test',
                    'alt_pval', 'alt_p', 'alt_p_control', 'alt_p_test']
            test_fun = partial(lrt_test, inst_params=inst_params, params=params, skip_failures=False, bad=bad,
                               param_mode=param_mode, n_bootstrap=n_bootstrap)
        else:
            cols = ['ref_pval', 'ref_p_control', 'ref_p_test', 'ref_std_control', 'ref_std_test',
                    'alt_pval', 'alt_p_control', 'alt_p_test', 'alt_std_control', 'alt_std_test']
            test_fun = partial(wald_test, inst_params=inst_params, params=params, skip_failures=False, bad=bad,
                               contrasts=contrasts, param_mode=param_mode, logit_transform=logit_transform,
                               robust_se=robust_se, n_bootstrap=n_bootstrap)
        if group_test:
            counts_a = _counts_a[bad]; counts_b = _counts_b[bad]; counts = _counts[bad]
            counts_a = counts_a[counts_a[:, 0] + counts_a[:, 1] < max_cover_group_test]
            counts_b = counts_b[counts_b[:, 0] + counts_b[:, 1] < max_cover_group_test]
            counts = counts[counts[:, 0] + counts[:, 1] < max_cover_group_test]
            (whole_ref, whole_alt), _ = test_fun(('all', counts_a, counts_b, counts))
            df_whole = pd.DataFrame([list(whole_ref) + list(whole_alt)], columns=cols)

        counts = [(snv, *[c[bad] for c in build_count_tables({snv: snvs_a[snv]}, {snv: snvs_b[snv]})])
                  for snv, it in snvs_a.items() if _bad_in(it, bad)]
        if not counts:
            continue
        max_sz = max(c[-1].shape[0] for c in counts)
        f = partial(test_fun, skip_failures=skip_failures, max_sz=max_sz)
        chunk_size = len(counts) // n_jobs
        with Pool(n_jobs) as p:
            if n_jobs > 1:
                it = p.imap_unordered(f, counts, chunksize=chunk_size)
            else:
                it = map(f, counts)
            for r, t in it:
                if r is None:
                    continue
                res.append([*t] + list(r[0]) + list(r[1]))
    df = pd.DataFrame(res, columns=['ind', 'n_control', 'n_test'] + cols)
    if df.empty:
        df['ref_fdr_pval'] = None
        df['alt_fdr_pval'] = None
    else:
        _, df['ref_fdr_pval'], _, _ = multitest.multipletests(df['ref_pval'], alpha=alpha, method='fdr_bh')
        _, df['alt_fdr_pval'], _, _ = multitest.multipletests(df['alt_pval'], alpha=alpha, method='fdr_bh')

    res = {subname: {'tests': df, 'snvs': (snvs_a, snvs_b)}}
    if group_test:
        res[subname]['whole'] = df_whole
    filename = f'{name}.difftest.{compressor}'
    if os.path.isfile(filename):
        with open(filename, 'rb') as f:
            d = dill.load(f)
            if subname in d:
                del d[subname]
            res.update(d)
    with open(filename, 'wb') as f:
        dill.dump(res, f)
    return res


import time

import tweepy

# Credenciales de acceso
consumer_key = ''
consumer_secret = ''
access_token = ''
access_token_secret = ''

# Autenticarse con Tweepy
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)

# Crear el objeto API
api = tweepy.API(auth)

# Tu tweet a ser publicado
tweet = 'Tweet programado para publicar automáticamente'

# Programar la publicación
while True:
    if time.strftime("%H:%M:%S") == '12:00:00':  # Programar la publicación a las 12:00
        api.update_status(tweet)
        print('Tweet publicado')
    time.sleep(1)


import networkx as nx

from source.view.components.message_box import MessageBox

class OperationsGraph:
    name = None

    def __init__(self):
        self.all = OperationsGraph.__subclasses__()
        self.dict_name_operations_graph: {str: OperationsGraph} = {}
        for operations in self.all:
            self.dict_name_operations_graph[operations.name] = operations

class LineGraph(OperationsGraph):
    name = "Line Graph"

    @staticmethod
    def calculate(graph):
        return nx.line_graph(graph)

class InverseLineGraph(OperationsGraph):
    name = "Inverse Line Graph"

    @staticmethod
    def calculate(graph):
        try:
            return nx.inverse_line_graph(graph)
        except nx.NetworkXError:
            message_box = MessageBox("The drawn graph is not a line graph of any graph")
            message_box.exec()

        return None

class Complement(OperationsGraph):
    name = "Complement"

    @staticmethod
    def calculate(graph):
        return nx.complement(graph)

class CliqueGraph(OperationsGraph):
    name = "Clique Graph"

    @staticmethod
    def calculate(graph):
        return nx.make_max_clique_graph(graph)

operations_graph = OperationsGraph()

dict_name_operations_graph = operations_graph.dict_name_operations_graph


"""Helpers for the embodyserial interface."""

import logging
import struct
import time
from datetime import datetime
from datetime import timezone
from typing import Optional

from embodycodec import attributes
from embodycodec import codec
from embodycodec import types

from embodyserial.embodyserial import EmbodySender
from embodyserial.exceptions import MissingResponseError
from embodyserial.exceptions import NackError

class EmbodySendHelper:
    """Facade to make send/receive more protocol agnostic with simple get/set methods."""

    def __init__(self, sender: EmbodySender, timeout: Optional[int] = 30) -> None:
        self.__sender = sender
        self.__send_timeout = timeout

    def get_current_time(self) -> datetime:
        response_attribute = self.__do_send_get_attribute_request(
            attributes.CurrentTimeAttribute.attribute_id
        )
        return datetime.fromtimestamp(response_attribute.value / 1000, tz=timezone.utc)

    def get_serial_no(self) -> str:
        response_attribute = self.__do_send_get_attribute_request(
            attributes.SerialNoAttribute.attribute_id
        )
        response = response_attribute.formatted_value()
        assert response
        return response

    def get_vendor(self) -> str:
        response_attribute = self.__do_send_get_attribute_request(
            attributes.VendorAttribute.attribute_id
        )
        response = response_attribute.formatted_value()
        assert response
        return response

    def get_model(self) -> str:
        response_attribute = self.__do_send_get_attribute_request(
            attributes.ModelAttribute.attribute_id
        )
        response = response_attribute.formatted_value()
        assert response
        return response

    def get_bluetooth_mac(self) -> str:
        response_attribute = self.__do_send_get_attribute_request(
            attributes.BluetoothMacAttribute.attribute_id
        )
        response = response_attribute.formatted_value()
        assert response
        return response

    def get_battery_level(self) -> int:
        response_attribute = self.__do_send_get_attribute_request(
            attributes.BatteryLevelAttribute.attribute_id
        )
        return response_attribute.value

    def get_heart_rate(self) -> int:
        response_attribute = self.__do_send_get_attribute_request(
            attributes.HeartrateAttribute.attribute_id
        )
        return response_attribute.value

    def get_charge_state(self) -> bool:
        response_attribute = self.__do_send_get_attribute_request(
            attributes.ChargeStateAttribute.attribute_id
        )
        return response_attribute.value

    def get_temperature(self) -> float:
        response_attribute = self.__do_send_get_attribute_request(
            attributes.TemperatureAttribute.attribute_id
        )
        assert isinstance(response_attribute, attributes.TemperatureAttribute)
        return response_attribute.temp_celsius()

    def get_firmware_version(self) -> str:
        response_attribute = self.__do_send_get_attribute_request(
            attributes.FirmwareVersionAttribute.attribute_id
        )
        response = response_attribute.formatted_value()
        assert response
        return response

    def get_on_body_state(self) -> bool:
        response_attribute = self.__do_send_get_attribute_request(
            attributes.BeltOnBodyStateAttribute.attribute_id
        )
        return response_attribute.value

    def get_files(self) -> list[tuple[str, int]]:
        """Get a list of tuples with file name and file size."""
        response = self.__sender.send(msg=codec.ListFiles(), timeout=120)
        if not response:
            raise MissingResponseError
        if isinstance(response, codec.NackResponse):
            raise NackError(response)
        assert isinstance(response, codec.ListFilesResponse)

        files: list[tuple[str, int]] = list()
        if len(response.files) == 0:
            return files
        else:
            for file in response.files:
                files.append((str(file.file_name), file.file_size))
            return files

    def delete_file(self, file_name: str) -> bool:
        response = self.__sender.send(
            msg=codec.DeleteFile(types.File(file_name)), timeout=self.__send_timeout
        )
        if not response:
            raise MissingResponseError()
        if isinstance(response, codec.NackResponse):
            raise NackError(response)
        assert isinstance(response, codec.DeleteFileResponse)
        return True

    def delete_file_with_retries(
        self, file_name: str, retries=3, timeout_seconds_per_retry=0.5
    ) -> bool:
        for retry in range(1, retries + 1):
            try:
                if self.delete_file(file_name):
                    logging.info(f"Deleted file on device: {file_name}")
                    return True
                logging.warn(f"Delete failed for {file_name} (attempt: {retry})")
                time.sleep(timeout_seconds_per_retry)
                continue
            except Exception as e:
                logging.warn(f"Delete failed for {file_name} (attempt: {retry}): {e}")
                time.sleep(timeout_seconds_per_retry)
                continue
        return False

    def set_current_timestamp(self) -> bool:
        return self.set_timestamp(datetime.now(timezone.utc))

    def set_timestamp(self, time: datetime) -> bool:
        attr = attributes.CurrentTimeAttribute(int(time.timestamp() * 1000))
        return self.__do_send_set_attribute_request(attr)

    def set_trace_level(self, level: int) -> bool:
        attr = attributes.TraceLevelAttribute(level)
        return self.__do_send_set_attribute_request(attr)

    def reformat_disk(self) -> bool:
        response = self.__sender.send(
            msg=codec.ReformatDisk(), timeout=self.__send_timeout
        )
        if not response:
            raise MissingResponseError()
        if isinstance(response, codec.NackResponse):
            raise NackError(response)
        assert isinstance(response, codec.ReformatDiskResponse)
        return True

    def reset_device(self) -> bool:
        self.__sender.send_async(
            msg=codec.ExecuteCommand(
                command_id=codec.ExecuteCommand.RESET_DEVICE, value=b""
            )
        )
        return True

    def reboot_device(self) -> bool:
        self.__sender.send_async(
            msg=codec.ExecuteCommand(
                command_id=codec.ExecuteCommand.REBOOT_DEVICE, value=b""
            )
        )
        return True

    def click_button(self, click_count: int, click_duration_ms: int) -> bool:
        self.__sender.send_async(
            msg=codec.ExecuteCommand(
                command_id=types.ExecuteCommandType.PRESS_BUTTON.value,
                value=struct.pack(">BH", click_count, click_duration_ms),
            )
        )
        return True

    def delete_all_files(self) -> bool:
        response = self.__sender.send(
            msg=codec.DeleteAllFiles(), timeout=self.__send_timeout
        )
        if not response:
            raise MissingResponseError()
        if isinstance(response, codec.NackResponse):
            raise NackError(response)
        assert isinstance(response, codec.DeleteAllFilesResponse)
        return True

    def __do_send_get_attribute_request(
        self, attribute_id: int
    ) -> attributes.Attribute:
        response = self.__sender.send(
            msg=codec.GetAttribute(attribute_id), timeout=self.__send_timeout
        )
        if not response:
            raise MissingResponseError()
        if isinstance(response, codec.NackResponse):
            raise NackError(response)
        assert isinstance(response, codec.GetAttributeResponse)
        return response.value

    def __do_send_set_attribute_request(self, attr: attributes.Attribute) -> bool:
        response = self.__sender.send(
            msg=codec.SetAttribute(attribute_id=attr.attribute_id, value=attr),
            timeout=self.__send_timeout,
        )
        if not response:
            raise MissingResponseError()
        if isinstance(response, codec.NackResponse):
            raise NackError(response)
        return isinstance(response, codec.SetAttributeResponse)


import torch
import torch.nn as nn
from models.lebert import LEBertModel
from models.cnn import MaskCNN
from transformers import BertPreTrainedModel, BertModel

class NestLEBertForNer(BertPreTrainedModel):
    def __init__(self, config):
        super(NestLEBertForNer, self).__init__(config)
        self.word_embeddings = nn.Embedding(config.word_vocab_size, config.word_embed_dim)
        self.bert = LEBertModel(config)
        self.biaffine_size = 200
        self.dropout = nn.Dropout(config.hidden_dropout_prob)
        self.inner_dim = int(self.config.hidden_size / self.config.num_attention_heads)
        self.fc = torch.nn.Linear(self.config.hidden_size, self.biaffine_size * self.inner_dim * 2)
        self.down_fc = torch.nn.Linear(self.biaffine_size, self.config.num_labels)

        if config.use_cnn:
            self.cnn_dim=200
            self.cnn_fc = nn.Linear(self.config.num_labels, self.cnn_dim)
            self.cnn_fc2 = nn.Linear(self.cnn_dim, self.config.num_labels)
            self.cnn = MaskCNN(self.cnn_dim, self.cnn_dim, kernel_size=3, depth=3)

        self.init_weights()

    def sinusoidal_position_embedding(self, batch_size, seq_len, output_dim):
        position_ids = torch.arange(0, seq_len, dtype=torch.float).unsqueeze(-1)
        indices = torch.arange(0, output_dim // 2, dtype=torch.float)
        indices = torch.pow(10000, -2 * indices / output_dim)
        embeddings = position_ids * indices
        embeddings = torch.stack([torch.sin(embeddings), torch.cos(embeddings)],
                                 dim=-1)
        embeddings = embeddings.repeat(
            (batch_size, *([1] * len(embeddings.shape))))
        embeddings = torch.reshape(embeddings, (batch_size, seq_len, output_dim))
        embeddings = embeddings.to('cuda')
        return embeddings

    def get_table(self, input_ids, attention_mask, token_type_ids, word_ids, word_mask):
        word_embeddings = self.word_embeddings(word_ids)
        outputs = self.bert(
            input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids,
            word_embeddings=word_embeddings, word_mask=word_mask
        )
        encoded_layers = outputs[0]

        batch_size, seq_len = encoded_layers.size(0), encoded_layers.size(1)
        encoded_layers = self.dropout(encoded_layers)

        outputs = self.fc(encoded_layers)
        outputs = torch.split(outputs, self.inner_dim * 2,
                              dim=-1)
        outputs = torch.stack(outputs, dim=-2)
        qw, kw = outputs[..., :self.inner_dim], outputs[...,
                                                self.inner_dim:]

        pos_emb = self.sinusoidal_position_embedding(batch_size, seq_len,
                                                     self.inner_dim)

        cos_pos = pos_emb[..., None, 1::2].repeat_interleave(2,
                                                             dim=-1)
        sin_pos = pos_emb[..., None, ::2].repeat_interleave(2, dim=-1)
        qw2 = torch.stack([-qw[..., 1::2], qw[..., ::2]], -1)
        qw2 = qw2.reshape(qw.shape)
        qw = qw * cos_pos + qw2 * sin_pos
        kw2 = torch.stack([-kw[..., 1::2], kw[..., ::2]], -1)
        kw2 = kw2.reshape(kw.shape)
        kw = kw * cos_pos + kw2 * sin_pos

        logits = torch.einsum('bmhd,bnhd->bhmn', qw, kw)
        pad_mask = attention_mask.unsqueeze(1).unsqueeze(1).expand(batch_size, self.biaffine_size, seq_len, seq_len)

        if hasattr(self, 'cnn'):
            cnn_logits = logits
            biaffine_logits = logits
            cnn_logits = self.cnn(cnn_logits, pad_mask)
            logits = biaffine_logits + cnn_logits
            logits = logits * pad_mask - (1 - pad_mask) * 1e12

        else:
            logits = logits * pad_mask - (1 - pad_mask) * 1e12

            mask = torch.tril(torch.ones_like(logits), -1)
            logits -= mask * 1e12

        logits = self.down_fc(logits.permute(0, 2, 3, 1)).permute(0, 3, 1, 2)

        return logits / self.inner_dim ** 0.5

    def forward(self, input_ids, attention_mask, token_type_ids, word_ids, word_mask, labels=None):

        logits = self.get_table(input_ids, attention_mask, token_type_ids, word_ids, word_mask)
        outputs = (logits,)
        if labels is not None:
            loss = self.compute_loss(logits, labels)
            outputs = (loss,) + outputs
        return outputs

    def compute_loss(self, pred, labels):
        batch_size, num_labels = labels.shape[:2]
        y_true = labels.reshape(batch_size * num_labels, -1)
        y_pred = pred.reshape(batch_size * num_labels, -1)
        loss = self.multilabel_categorical_crossentropy(y_pred, y_true)
        return loss

    def multilabel_categorical_crossentropy(self, y_pred, y_true):
        y_pred = (1 - 2 * y_true) * y_pred
        y_pred_neg = y_pred - y_true * 1e12
        y_pred_pos = y_pred - (1 - y_true) * 1e12
        zeros = torch.zeros_like(y_pred[..., :1])
        y_pred_neg = torch.cat([y_pred_neg, zeros], dim=-1)
        y_pred_pos = torch.cat([y_pred_pos, zeros], dim=-1)
        neg_loss = torch.logsumexp(y_pred_neg, dim=-1)
        pos_loss = torch.logsumexp(y_pred_pos, dim=-1)
        return (neg_loss + pos_loss).mean()


import os
import sys
import re
from pathlib import Path

file_dir = Path(__file__).absolute().parent
docs_dir = file_dir.parent
project_dir = docs_dir.parent

sys.path.append(str(project_dir))

doc_name = "pygmol_doc"

# update the package version:
with open(project_dir.joinpath("setup.py")) as fp:
    setup_text = fp.read()
# for some reason, I cannot import from setup.py :(
version = re.search(r'version\s?=\s?"(\d{1,2}\.\d{1,2}\.\d{1,2})"', setup_text).group(1)
with open(file_dir.joinpath(f"{doc_name}.tex")) as fp:
    text = fp.read()
updated = re.sub(r"\d{1,2}\.\d{1,2}\.\d{1,2}", version, text)
with open(file_dir.joinpath(f"{doc_name}.tex"), "w") as fp:
    fp.write(updated)

# run the latex commands:
os.chdir(file_dir)
commands = [
    f"pdflatex -synctex=1 -interaction=nonstopmode {doc_name}.tex",
    f"bibtex {doc_name}.aux",
    f"pdflatex -synctex=1 -interaction=nonstopmode {doc_name}.tex",
    f"pdflatex -synctex=1 -interaction=nonstopmode {doc_name}.tex",
]
for cmd in commands:
    os.system(cmd)

# remove the byproducts:
to_keep = set([f"{doc_name}.{ext}" for ext in ["tex", "pdf", "bib"]])
to_keep.add(f"{doc_name}_content.tex")
for file_path in file_dir.glob(f"{doc_name}*.*"):
    if file_path.name not in to_keep:
        file_path.unlink()

# move the final pdf one folder up and rename:
file_dir.joinpath(f"{doc_name}.pdf").rename(docs_dir.joinpath("math.pdf"))


#!/usr/bin/python3
# -*- coding: utf-8 -*-
# @Time    : 2020/7/28 0:22
# @Author  : Leslee

'''
Python的多进程模块提供所在用户间管理共享信息的管理者(Manager).
Manager控制着持有Python对象的服务进程，允许其他进程操作共享对象。
1. 控制着管理共享对象的服务进程；
2. 确保当前某一进程修改了共享对象，所有的进程能拿到额外共享对象的更新。
'''

import multiprocessing

def worker(dictionary, key, item):
    dictionary[key] = item
    print("key = %d value = %d" % (key, item))

if __name__ == '__main__':
    mgr = multiprocessing.Manager()

    dictionary =mgr.dict()
    jobs = [multiprocessing.Process(target=worker, args=(dictionary, i, i*2)) for i in range(10)]

    for j in jobs:
        j.start()
    for j in jobs:
        j.join()
    print("Result:",dictionary)


from grawlix.book import HtmlFiles, HtmlFile, OnlineFile, Book, SingleFile, Metadata, EpubInParts
from grawlix.exceptions import UnsupportedOutputFormat
from .output_format import OutputFormat, Update

import asyncio
from bs4 import BeautifulSoup
import os
from ebooklib import epub
from zipfile import ZipFile
import rich

class Epub(OutputFormat):
    extension = "epub"
    input_types = [SingleFile, HtmlFiles, EpubInParts]

    async def download(self, book: Book, location: str, update: Update) -> None:
        if isinstance(book.data, SingleFile):
            await self._download_single_file(book, location, update)
        elif isinstance(book.data, HtmlFiles):
            await self._download_html_files(book.data, book.metadata, location, update)
        elif isinstance(book.data, EpubInParts):
            await self._download_epub_in_parts(book.data, book.metadata, location, update)
        else:
            raise UnsupportedOutputFormat

    async def _download_html_files(self, html: HtmlFiles, metadata: Metadata, location: str, update: Update) -> None:
        output = epub.EpubBook()
        output.set_title(metadata.title)
        for author in metadata.authors:
            output.add_author(author)
        file_count = len(html.htmlfiles) + 1 # Html files + cover

        async def download_cover(cover_file: OnlineFile):
            cover_filename = f"cover.{cover_file.extension}"
            epub_cover = epub.EpubCover(file_name = cover_filename)
            epub_cover.content = await self._download_file(cover_file)
            output.add_item(epub_cover)
            epub_cover_page = epub.EpubCoverHtml(image_name = cover_filename)
            if update:
                update(1/file_count)
            return epub_cover_page

        async def download_file(index: int, file: HtmlFile):
            response = await self._client.get(
                file.file.url,
                headers = file.file.headers,
                cookies = file.file.cookies,
                follow_redirects=True
            )
            soup = BeautifulSoup(response.text, "lxml")
            selected_element = soup.find(attrs=file.selector)
            epub_file = epub.EpubHtml(
                title = file.title,
                file_name = f"part {index}.html",
                content = str(selected_element)
            )
            if update:
                update(1/file_count)
            return epub_file

        # Download files
        tasks = [
            download_file(index, file)
            for index, file in enumerate(html.htmlfiles)
        ]
        if html.cover:
            tasks.append(download_cover(html.cover))
        epub_files = await asyncio.gather(*tasks)

        # Add files to epub
        for epub_file in epub_files:
            output.add_item(epub_file)
            output.spine.append(epub_file)
            output.toc.append(epub_file)

        # Complete book
        output.add_item(epub.EpubNcx())
        output.add_item(epub.EpubNav())
        epub.write_epub(location, output)

    async def _download_epub_in_parts(self, data: EpubInParts, metadata: Metadata, location: str, update: Update) -> None:
        files = data.files
        file_count = len(files)
        progress = 1/(file_count)
        temporary_file_location = f"{location}.tmp"

        added_files: set[str] = set()
        def get_new_files(zipfile: ZipFile):
            """Returns files in zipfile not already added to file"""
            for filename in zipfile.namelist():
                if filename in added_files or filename.endswith(".opf") or filename.endswith(".ncx"):
                    continue
                yield filename

        output = epub.EpubBook()
        for file in files:
            await self._download_and_write_file(file, temporary_file_location)
            with ZipFile(temporary_file_location, "r") as zipfile:
                for filepath in get_new_files(zipfile):
                    content = zipfile.read(filepath)
                    if filepath.endswith("html"):
                        filename = os.path.basename(filepath)
                        is_in_toc = False
                        title = None
                        for key, value in data.files_in_toc.items():
                            toc_filename = key.split("#")[0]
                            if filename == toc_filename:
                                title = value
                                is_in_toc = True
                                break
                        epub_file = epub.EpubHtml(
                            title = title,
                            file_name = filepath,
                            content = content
                        )
                        output.add_item(epub_file)
                        output.spine.append(epub_file)
                        if is_in_toc:
                            output.toc.append(epub_file)
                    else:
                        epub_file = epub.EpubItem(
                            file_name = filepath,
                            content = content
                        )
                        output.add_item(epub_file)
                    added_files.add(filepath)
            if update:
                update(progress)
        os.remove(temporary_file_location)

        output.add_item(epub.EpubNcx())
        output.add_item(epub.EpubNav())
        epub.write_epub(location, output)
        exit()


from flask import Flask, jsonify, request
from flask_restful import Api, Resource
import numpy as np
import traceback
import json
from model.model import Net
import torch
import cv2

app = Flask(__name__)
api = Api(app)

model = Net()
model.load_state_dict(torch.load('mnist_cnn.pt'))
model.eval()

# convert request_input dict to input accepted by model.
def parse_input(request_input):
    """parse input to make it ready for model input.

    Arguments:
        request_input::file- input received from API call.

    Returns:
        model_ready_input::torch.Tensor- data ready to be fed into model.
    """
    img = request_input.read()
    img = np.frombuffer(img, np.uint8)
    img = cv2.imdecode(img,cv2.IMREAD_COLOR)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    img = cv2.resize(img,(28,28))
    img = img.reshape((1,1,28,28))
    return torch.Tensor(img)

# convert model prediction to dict to return as JSON
def parse_prediction(prediction):
    """parse prediction to prepare a request response.

    Arguments:
        prediction::torch.Tensor- output of model inference

    Returns:
        response::dict- dictionary to return as a JSON response.
    """
    prediction = prediction.argmax(dim=1, keepdim=True)
    return {'class':int(prediction[0][0])}

# class inheriting from Resource prepares it to become an API endpoint
class MakePrediction(Resource):
    @staticmethod
    def post():
        if model:
            try:
                request_input = request.files['file']

                model_input = parse_input(request_input)

                prediction = model(model_input)

                model_output = parse_prediction(prediction)

                return jsonify(model_output)

            except:
                return jsonify({'trace': traceback.format_exc()})
        else:
            return jsonify({'trace': 'No model found'})

api.add_resource(MakePrediction, '/predict') # add endpoint to API

class dummy_post(Resource):
    @staticmethod
    def post():
        if model:
            try:
                request_input = request.get_json()

                return jsonify(request_input)

            except:
                return jsonify({'trace': traceback.format_exc()})
        else:
            return jsonify({'trace': 'No model found'})

api.add_resource(dummy_post, '/dummy_post')

class dummy_get(Resource):
    @staticmethod
    def get():
        if model:
            try:
                return str("dummy get request")

            except:
                return jsonify({'trace': traceback.format_exc()})
        else:
            return jsonify({'trace': 'No model found'})

api.add_resource(dummy_get, '/dummy_get')

if __name__ == '__main__':
    app.run(debug=True,host='0.0.0.0',port=8080)