import sys,torch,torchvision,logging
import matplotlib.pyplot as plt
import numpy as np
from pathlib import Path
from torch.utils.data import Dataset, DataLoader
import torch.nn as nn
#logging.basicConfig(level=logging.DEBUG)
logging.info("torch: {0} ,  torchvision:{1}".format(torch.__version__,torchvision.__version__))
class Learner(object):
    def __init__(self,train_dataset,test_dataset,model):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")
        #torch的Dataset
        self.__train_dataset = None
        self.__test_dataset = None
        #优化器
        self.__optimizer =None
        self.__optim_param={}
        #损失函数
        self.__criterion=None
        self.__crit_param={}
        #训练相关
        self.__train_param={}
        self.__train_dataloader=None
        #测试相关
        self.__test_param={}
        self.__test_dataloader=None
        # 模型
        self.__model=None
        #
        self.__device=torch.device("cpu")
        self.__trained=False
        if (isinstance(train_dataset,Dataset)):
            self.__train_dataset=train_dataset
        else:
            raise(TypeError("dataset必须为pythrch Dataset类型"))
        if (isinstance(test_dataset,Dataset)):
            self.__test_dataset=test_dataset
        else:
            raise(TypeError("dataset必须为pythrch Dataset类型"))
        if (isinstance(model,nn.Module)):
            self.__model=model
        else:
            raise(TypeError("model必须为pytorch Module类型"))
    def setCriterion(self,crit,**param):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")
        #设置损失函数
        #https://pytorch.org/docs/stable/_modules/torch/nn/modules/loss.html
        if issubclass(crit,nn.modules.loss._Loss):
            self.__criterion=crit
            self.__crit_param=param
        else:
            raise(TypeError("损失函数必须是 _Loss 子类"))
            #crit(**param)
    def setOptimizer(self,opt,**param):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")
        #设置优化器
        if issubclass(opt,torch.optim.Optimizer):
            self.__optimizer=opt
            self.__optim_param=param
        else:
            raise(TypeError("优化器必须是 Optimizer 子类"))

    def setTrainParam(self,key,value):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")
         # 设置训练的参数
        self.__train_param[key]=value
    def setTestParam(self,key,value):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")
         # 设置训练的参数
        self.__test_param[key]=value

    def checkDevice(self,device="auto"):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")
        #检查cpu还是gpu
        if device=="auto":
            if torch.cuda.is_available():
                self.__device=torch.device("cuda")
        elif device=="cuda":
                self.__device=torch.device("cuda")
        logging.info("Use Device:{}".format(self.__device))

    def __checkTrainParam(self):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")
        # 检查参数 lr,bs
        check=set(['lr','bs'])
        print(self.__train_param.keys())
        if not (check <= self.__train_param.keys()):
            raise(ValueError("必须要设置lr和bs"))
    def showSummary(self):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")
        #打印汇总
        print("Device:{}".format(self.__device))
        print("Optimizer:{},Param:{}".format(self.__optimizer,self.__optim_param))
        print("Criterion:{},Param:{}".format(self.__criterion,self.__crit_param))
        print("TrainParam:{}".format(self.__train_param))

    def train(self,fetch,epoches=10,device="auto",show_summary=False):

        logging.debug("----"+sys._getframe().f_code.co_name+"----")
        self.__losses=[]
        self.__trained=True
        #检查参数
        self.__checkTrainParam()
        #读取数据,默认不打乱
        shuffle= self.__train_param["shuffle"] if "shuffle" in self.__train_param.keys() else False
        self.__train_dataloader=DataLoader(dataset = self.__train_dataset,
                                            batch_size = self.__train_param['bs'],
                                            shuffle = shuffle)  # 将数据打乱
        #判断gpu
        self.checkDevice(device=device)

        # 显示汇总信息
        if (show_summary):
            self.showSummary()

        # 模型
        self.__model=self.__model.to(self.__device)
        self.__model.train()
        # 优化器
        train_opti=self.__optimizer(self.__model.parameters(),**self.__optim_param)
        # 损失函数
        train_crit=self.__criterion(**self.__crit_param).to(self.__device)
        for epoch in range(epoches):
            #循环
            for i,data in enumerate(self.__train_dataloader):
                logging.info("epoch:{},batch:{}".format(epoch+1,i))
                #批次
                x,y=fetch(data)
                train_opti.zero_grad()
                y_hat= self.__model(x.to(self.__device))
                loss = train_crit(y_hat, y.to(self.__device))
                loss.backward()
                train_opti.step()
                self.__losses.append(loss.cpu().data.item())
            print("epoch:{},avg loss:{}".format(epoch+1,np.mean(self.__losses)))
        print("训练完成")
    def test(self,fetch,metrics):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")

        if  self.__trained==False :
            print("请先运行train")
            return
        bs= self.__test_param["bs"] if "bs" in self.__test_param.keys() else self.__train_param['bs']
        self.__test_dataloader=DataLoader(dataset = self.__test_dataset,
                                            batch_size = bs,
                                            shuffle = False)
        self.__model=self.__model.to(self.__device)
        self.__model.eval()
        correct =total = 0
        for data in self.__test_dataloader:
            inputs,labels=fetch(data)
            outputs = self.__model(inputs.to(self.__device)).cpu()
            predicted = metrics(outputs)
            total+=labels.size(0)
            correct+=(predicted == labels).sum()
        print('准确率: %6.f %%' % (100 * correct / total))
    def saveModel(self,save_path):
        torch.save(self.__model.state_dict(), save_path)
    def loadModel(self,load_path):
        self.__model.load_state_dict(torch.load(load_path))
    def showLossPlt(self):
        plt.xlabel('batch #')
        plt.ylabel('Loss')
        plt.plot(self.__losses)
        plt.show()


import numpy as np
import matplotlib.pyplot as plt

import histogram_module
import dist_module
import match_module

# compute and plot the recall/precision curve
#
# D - square matrix, D(i, j) = distance between model image i, and query image j
#
# note: assume that query and model images are in the same order, i.e. correct answer for i-th query image is the i-th model image

def plot_rpc(D, plot_color):

    recall = []
    precision = []
    num_queries = D.shape[1]

    num_images = D.shape[0]
    assert(num_images == num_queries), 'Distance matrix should be a square matrix'

    labels = np.diag([1]*num_images)

    d = D.reshape(D.size)
    l = labels.reshape(labels.size)

    sortidx = d.argsort()
    d = d[sortidx]
    l = l[sortidx]

    tp = 0
    #... (your code here)

    false_negative = num_images
    false_positive = 0

    for idt in range(len(d)):
        tp += l[idt]
        #... (your code here)
        false_negative -= l[idt]
        false_positive += (1 - l[idt])

        precision.append(tp/(tp+false_positive))
        recall.append(tp/(tp+false_negative))

    plt.plot([1-precision[i] for i in range(len(precision))], recall, plot_color+'-')

def compare_dist_rpc(model_images, query_images, dist_types, hist_type, num_bins, plot_colors):

    assert len(plot_colors) == len(dist_types), 'number of distance types should match the requested plot colors'

    for idx in range( len(dist_types) ):

        [best_match, D] = match_module.find_best_match(model_images, query_images, dist_types[idx], hist_type, num_bins)

        plot_rpc(D, plot_colors[idx])

    plt.axis([0, 1, 0, 1]);
    plt.xlabel('1 - precision');
    plt.ylabel('recall');

    # legend(dist_types, 'Location', 'Best')

    plt.legend( dist_types, loc='best')


import os
import argparse
import tqdm
import torch
import numpy as np
import yaml

from mood.dpa.pg_networks import ProgGrowStageType, NetworkType
from mood.dpa.pg_decoders import DECODER_NETWORKS
from mood.dpa.pg_encoders import ENCODER_NETWORKS
from mood.dpa.data_generators import ProgGrowImageGenerator, MixResolution
from mood.utils.datasets import DatasetType, DATASETS
from mood.utils.transforms import TRANSFORMS
from mood.dpa.optimizer import Optimizer
from mood.dpa.rec_losses import ReconstructionLossType
from mood.dpa.pg_rec_losses import PG_RECONSTRUCTION_LOSSES
from mood.utils.utils import save_yaml

torch.backends.cudnn.deterministic = True

class DeepPerceptualAutoencoder:
    def __init__(self, config):
        self.config = config
        self.verbose = config['verbose']
        self.random_seed = config['random_seed']
        self.finetune_from = config['finetune_from']
        self.mode_3d = config.get('mode_3d', False)

        self.checkpoint_root = config['checkpoint_root']
        self.log_root = config['log_root']
        save_yaml(config, os.path.join(self.checkpoint_root, 'train_config.yaml'))
        save_yaml(config, os.path.join(self.log_root, 'train_config.yaml'))

        self.num_workers = config['num_workers']
        self.max_image_res = config['max_image_res']
        self.initial_image_res = config['initial_image_res']
        self.image_dim = config['image_dim']
        self.latent_res = config['latent_res']
        self.latent_dim = config['latent_dim']

        trns_iter = config['trns_iter']
        stab_iter = config['stab_iter']
        self.iters = {
            ProgGrowStageType.stab: stab_iter,
            ProgGrowStageType.trns: trns_iter
        }

        self.iters_per_res = {
            int(res): iters for res, iters in config.get('iters_per_res', {}).items()
        }
        # check iters_per_res
        error_msg = """"
        Incorrect 'iters_per_res' param. Correct example:
        iters_per_res:
          64:
            trns: 1000
            stab: 2000
        """
        for res, values in self.iters_per_res.items():
            if not isinstance(values, dict):
                raise ValueError(error_msg)
            for stage, val in values.items():
                if stage != 'stab' and stage != 'trns':
                   raise ValueError(error_msg)

        self.log_iter = config['log_iter']
        self.val_iter = config['val_iter']
        self.early_stopping_patience = config['early_stopping_patience']
        self.early_stopping_min_delta = config['early_stopping_min_delta']
        self.image_sample_iter = config['image_sample_iter']

        self.batch_sizes = {int(res): batch_size for res, batch_size in config['batch_sizes'].items()}
        # check batch sizes
        _res = self.initial_image_res
        while _res <= self.max_image_res:
            if _res not in self.batch_sizes.keys():
                raise ValueError(f"The batch size isn't specified for the resolution {_res}")
            _res = int(_res * 2)

        "=========================================== initialize ======================================================="

        if self.verbose:
            print(yaml.dump(self.config, default_flow_style=False))

        if self.random_seed is not None:
            torch.manual_seed(self.random_seed)
            np.random.seed(self.random_seed)

        self.stage = ProgGrowStageType.stab
        self.resolution = self.initial_image_res
        self.progress = 0
        self.n_iter = 0
        self.best_val_iter = 0
        self.best_val_loss = 10e6
        self.batch_size = int(self.batch_sizes[self.resolution])

        from mood.utils.loggers import Logger
        os.makedirs(self.checkpoint_root, exist_ok=True)
        self.logger = Logger(self.log_root)

        "=========================================== create data dpa ================================================"

        dataset_type = config['train_dataset']['dataset_type']
        dataset_kwargs = config['train_dataset']['dataset_kwargs']
        transform_kwargs = config['train_dataset']['transform_kwargs']

        transform = TRANSFORMS[DatasetType[dataset_type]](**transform_kwargs)
        dataset = DATASETS[DatasetType[dataset_type]](
            transform=transform,
            **dataset_kwargs
        )

        self.pg_image_gen = ProgGrowImageGenerator(dataset, self.max_image_res,
                                                   batch_size=self.batch_size,
                                                   num_workers=self.num_workers, inf=True, mode_3d=self.mode_3d)

        dataset_type = config['val_dataset']['dataset_type']
        dataset_kwargs = config['val_dataset']['dataset_kwargs']
        transform_kwargs = config['val_dataset']['transform_kwargs']

        transform = TRANSFORMS[DatasetType[dataset_type]](**transform_kwargs)
        val_dataset = DATASETS[DatasetType[dataset_type]](
            transform=transform,
            **dataset_kwargs
        )
        self.val_pg_image_gen = ProgGrowImageGenerator(val_dataset, self.max_image_res,
                                                       batch_size=self.batch_size,
                                                       num_workers=self.num_workers, inf=False, mode_3d=self.mode_3d)

        "============================================= create networks ================================================"

        mtype = config['enc']['type']
        kwargs = config['enc']['kwargs']
        self.enc_pg_networks = ENCODER_NETWORKS[NetworkType[mtype]](
            max_input_res=self.max_image_res,
            output_res=self.latent_res,
            input_dim=self.image_dim,
            output_dim=self.latent_dim,
            mode_3d=self.mode_3d,
            **kwargs
        )

        mtype = config['dec']['type']
        kwargs = config['dec']['kwargs']
        self.dec_pg_networks = DECODER_NETWORKS[NetworkType[mtype]](
            input_res=self.latent_res,
            max_output_res=self.max_image_res,
            input_dim=self.latent_dim,
            output_dim=self.image_dim,
            mode_3d=self.mode_3d,
            **kwargs
        )

        if self.verbose:
            print("================================== MODELS ON LARGEST RESOLUTION ==================================")
            print("====================== Encoder =============================")
            print(self.enc_pg_networks.get_net(ProgGrowStageType.stab, self.max_image_res))
            print("====================== Decoder  ===========================")
            print(self.dec_pg_networks.get_net(ProgGrowStageType.stab, self.max_image_res))

        self.dec = self.dec_pg_networks.get_net(self.stage, self.resolution).cuda()
        self.enc = self.enc_pg_networks.get_net(self.stage, self.resolution).cuda()

        "=========================================== create reconstruction losses ====================================="

        loss_type = config['image_rec_loss']['loss_type']
        loss_kwargs = config['image_rec_loss']['loss_kwargs']
        self.image_rec_loss = PG_RECONSTRUCTION_LOSSES[ReconstructionLossType[loss_type]](
            max_resolution=self.max_image_res, mode_3d=self.mode_3d, **loss_kwargs)
        self.image_rec_loss.set_stage_resolution(self.stage, self.resolution)

        "=========================================== create optimizers ================================================"

        adam_kwargs = config['adam_kwargs']
        self.optimizer = Optimizer(
            enc_params=self.enc.parameters(),
            dec_params=self.dec.parameters(),
            image_rec_loss=self.image_rec_loss,
            adam_kwargs=adam_kwargs,
        ).cuda()

        "=========================================== data for logging ================================================="

        self.pg_image_gen.set_stage_resolution(ProgGrowStageType.stab, self.max_image_res, batch_size=self.batch_size)
        self.display_x = next(self.pg_image_gen)[:4].cpu()
        self.pg_image_gen.set_stage_resolution(self.stage, self.resolution, self.batch_size)
        if self.val_pg_image_gen is not None:
            self.val_pg_image_gen.set_stage_resolution(self.stage, self.resolution, self.batch_size)
        self.mix_res_module = MixResolution(self.stage, self.resolution, self.max_image_res,  mode_3d=self.mode_3d)

        "=========================================== initialization ==================================================="

        self.updating_models_during_training = \
            [self.enc, self.dec, self.image_rec_loss,
             self.pg_image_gen, self.val_pg_image_gen, self.mix_res_module]

        if self.finetune_from is not None:
            self.load_state(torch.load(self.finetune_from))
        else:
            self._init_stage()

    def _create_new_stage(self):
        del self.dec, self.enc, self.updating_models_during_training
        if self.stage == ProgGrowStageType.stab:
            self.stage = ProgGrowStageType.trns
            self.resolution *= 2
        else:
            self.stage = ProgGrowStageType.stab
        self.progress = 0
        self._init_stage()

    def _init_stage(self):
        self.batch_size = int(self.batch_sizes[self.resolution])

        "=========================== update data generators and global params ========================================="

        self.pg_image_gen.set_stage_resolution(self.stage, resolution=self.resolution, batch_size=self.batch_size)
        self.val_pg_image_gen.set_stage_resolution(self.stage, resolution=self.resolution, batch_size=self.batch_size)
        self.mix_res_module = MixResolution(self.stage, self.resolution, self.max_image_res, mode_3d=self.mode_3d)

        "========================================== update networks ==================================================="

        self.enc = self.enc_pg_networks.get_net(self.stage, self.resolution).cuda()
        self.dec = self.dec_pg_networks.get_net(self.stage, self.resolution).cuda()
        self.image_rec_loss.set_stage_resolution(self.stage, self.resolution)

        self.optimizer.set_new_params(
            dec_params=self.dec.parameters(),
            enc_params=self.enc.parameters(),
            image_rec_loss=self.image_rec_loss)

        "========================================== other updates ==================================================="

        self.updating_models_during_training = \
            [self.enc, self.dec, self.image_rec_loss,
             self.pg_image_gen, self.val_pg_image_gen, self.mix_res_module]

        total_iterations = self.iters[self.stage]
        total_iterations = self.iters_per_res\
            .get(self.resolution, {self.stage.name: total_iterations})\
            .get(self.stage.name, total_iterations)

        self.delta_progress = 1.0 / total_iterations
        self.tqdm_logger = tqdm.tqdm(total=total_iterations)

        if self.verbose:
            print("====================== Encoder =============================")
            print(self.enc)
            print("====================== Decoder ===========================")
            print(self.dec)

    def train(self):
        while not (
                self.resolution == self.max_image_res and self.stage == ProgGrowStageType.stab and self.progress >= 1.0):

            "========================================== updating params ==============================================="

            self.n_iter += 1
            self.tqdm_logger.update(1)
            self.progress = np.clip(self.progress + self.delta_progress, 0, 1.0)

            if self.stage == ProgGrowStageType.trns:
                progress_resolution = self.resolution * self.progress + (self.resolution / 2) * (
                        1 - self.progress)

                for model in self.updating_models_during_training:
                    if model is not None:
                        model.set_progress(self.progress)
            else:
                progress_resolution = self.resolution

            "=========================================== train step ==================================================="

            real_x = next(self.pg_image_gen).cuda()
            rec_x = self.dec(self.enc(real_x))
            loss_info = self.optimizer.compute_loss(real_x, rec_x, update_parameters=True)

            del real_x, rec_x

            "============================================== logging ==================================================="

            if self.tqdm_logger.n % self.log_iter == 0:
                self.logger.add_scalars('train/losses', loss_info, self.n_iter)
                self.logger.add_scalar('train/resolution', progress_resolution, self.n_iter)

            if self.tqdm_logger.n % self.image_sample_iter == 0:
                self._save_image_sample()

            "============================================ checkpoint =================================================="

            if self.tqdm_logger.n % self.val_iter == 0:
                val_loss = self._compute_val_loss()
                self.logger.add_scalar('val/total', val_loss, self.n_iter)

                # early stopping at the last stage
                if self.resolution == self.max_image_res and self.stage == ProgGrowStageType.stab:
                    if val_loss < self.best_val_loss - self.early_stopping_min_delta:
                        self.best_val_loss = val_loss
                        self.best_val_iter = self.n_iter
                        self._do_checkpoint()
                        self._save_ad_model()
                    else:
                        if (self.n_iter - self.best_val_iter) / self.val_iter > self.early_stopping_patience:
                            break

            "========================================== create new stage =============================================="

            if abs(self.progress - 1.0) < 1e-6 and \
                    not(self.resolution == self.max_image_res and self.stage == ProgGrowStageType.stab):
                    self._do_checkpoint()
                    self._save_ad_model()
                    self._create_new_stage()

        # if we did not check last iteration for performance, then check and save
        if self.tqdm_logger.n % self.val_iter != 0:
            val_loss = self._compute_val_loss()
            self.logger.add_scalar('val/total', val_loss, self.n_iter)
            if val_loss < self.best_val_loss:
                self.best_val_iter = self.n_iter
                self._do_checkpoint()
                self._save_ad_model()

    def get_state(self):
        return {
            'config': self.config,
            'stage': self.stage.name,
            'resolution': self.resolution,
            'progress': self.progress,
            'n_iter': self.n_iter,
            'best_val_loss': self.best_val_loss,
            'best_val_iter': self.best_val_iter,
            'display_x': self.display_x.data,
            'optimizer': self.optimizer.state_dict(),
            'dec': self.dec.state_dict(),
            'enc': self.enc.state_dict(),
            'mix_res_module': self.mix_res_module.state_dict(),
        }

    def load_state(self, state):
        self.stage = ProgGrowStageType[state['stage']]
        self.resolution = state['resolution']
        self.progress = state['progress']
        self.n_iter = state['n_iter']
        self.best_val_loss = state['best_val_loss']
        self.best_val_iter = state['best_val_iter']
        self.display_x.data = state['display_x']

        self._init_stage()

        self.dec.load_state_dict(state['dec'])
        self.enc.load_state_dict(state['enc'])

        self.mix_res_module.load_state_dict(state['mix_res_module'])

        self.tqdm_logger.update(int(len(self.tqdm_logger) * self.progress))

    def get_state_anomaly_detection_model(self):
        return {
            'niter': self.tqdm_logger.n,
            'config': self.config,
            'stage': self.stage.name,
            'resolution': self.resolution,
            'progress': self.progress,
            'enc': self.enc.state_dict(),
            'dec': self.dec.state_dict(),
        }

    @staticmethod
    def load_anomaly_detection_model(state_dict):
        config = state_dict['config']
        stage = ProgGrowStageType[state_dict['stage']]
        resolution = state_dict['resolution']
        progress = state_dict['progress']

        mtype = config['enc']['type']
        kwargs = config['enc']['kwargs']
        mode_3d =  config.get('mode_3d', False)

        enc_pg_networks = ENCODER_NETWORKS[NetworkType[mtype]](
            max_input_res=config['max_image_res'],
            output_res=config['latent_res'],
            input_dim=config['image_dim'],
            output_dim=config['latent_dim'],
            mode_3d=mode_3d,
            **kwargs
        )

        mtype = config['dec']['type']
        kwargs = config['dec']['kwargs']
        dec_pg_networks = DECODER_NETWORKS[NetworkType[mtype]](
            input_res=config['latent_res'],
            max_output_res=config['max_image_res'],
            input_dim=config['latent_dim'],
            output_dim=config['image_dim'],
            mode_3d=mode_3d,
            **kwargs
        )

        enc = enc_pg_networks.get_net(stage, resolution)
        dec = dec_pg_networks.get_net(stage, resolution)
        enc.load_state_dict(state_dict['enc'])
        dec.load_state_dict(state_dict['dec'])

        loss_type = config['image_rec_loss']['loss_type']
        loss_kwargs = config['image_rec_loss']['loss_kwargs']
        image_rec_loss = PG_RECONSTRUCTION_LOSSES[ReconstructionLossType[loss_type]](
            config['max_image_res'], mode_3d=mode_3d, **loss_kwargs)
        image_rec_loss.set_stage_resolution(stage, resolution)
        image_rec_loss.set_reduction('none')

        mix_res_module = MixResolution(stage, resolution, config['max_image_res'], mode_3d=mode_3d)

        for model in [enc, dec, image_rec_loss, mix_res_module]:
            model.set_progress(progress)

        n_iter = state_dict['niter']

        enc, dec, image_rec_loss = enc.cuda().eval(), dec.cuda().eval(), image_rec_loss.cuda().eval()
        return enc, dec, image_rec_loss, (stage, resolution, progress, n_iter, mix_res_module)

    @staticmethod
    def predict_anomaly_score(loaded_model, image, reduction):
        enc, dec, image_rec_loss, _ = loaded_model

        with torch.no_grad():
            rec_image = dec(enc(image)).detach()
            image_rec_loss.set_reduction(reduction)
            pred = image_rec_loss(image, rec_image)

        return pred

    def _compute_val_loss(self):
        self.enc.eval()
        self.dec.eval()
        torch.set_grad_enabled(False)

        sum_loss = 0
        count = 0

        for real_x in self.val_pg_image_gen:
            real_x = real_x.cuda()
            rec_x = self.dec(self.enc(real_x))

            loss_info = self.optimizer.compute_loss(real_x, rec_x, update_parameters=False)

            val_loss = sum(loss_info.values())

            batch_size = real_x.shape[0]
            sum_loss += val_loss * batch_size
            count += batch_size

        val_avg_loss = sum_loss / count

        self.enc.train()
        self.dec.train()
        torch.set_grad_enabled(True)

        return val_avg_loss

    def _do_checkpoint(self):
        torch.save(self.get_state(), os.path.join(self.checkpoint_root, 'checkpoint.tar'))

    def _save_ad_model(self):
        if self.stage == ProgGrowStageType.stab:
            ad_model_path = 'anomaly_detection_res_{}_iter_{}.tar'.format(self.resolution, self.n_iter)
            torch.save(self.get_state_anomaly_detection_model(), os.path.join(self.checkpoint_root, ad_model_path))

            torch.save(self.get_state_anomaly_detection_model(), os.path.join(self.checkpoint_root, 'anomaly_detection.tar'))

    def _save_image_sample(self):
        torch.set_grad_enabled(False)
        self.optimizer.eval()

        real_images = self.mix_res_module(self.display_x)
        images = torch.cat([
            real_images,
            self.dec(self.enc(real_images.cuda())).cpu().detach()
        ], 0)

        # if 3d mode, get only central slice
        if self.mode_3d:
            images = images[:, :, int(images.shape[2] // 2)]
        name = 'res_{}_stage_{}_niter{}.png'.format(self.resolution, self.stage.name,
                                                       self.tqdm_logger.n)

        self.logger.save_grid(images, grid_size=self.max_image_res * 4, name=name, nrow=min(4, len(self.display_x)))
        self.logger.save_grid(images, grid_size=self.max_image_res * 4, name='sample.png', nrow=min(4, len(self.display_x)))

        torch.set_grad_enabled(True)
        self.optimizer.train()

def main(config):
    trainer = DeepPerceptualAutoencoder(config)
    trainer.train()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('config', type=str, help='Path to config')
    args = parser.parse_args()

    with open(args.config, 'r') as stream:
        config = yaml.load(stream, Loader=yaml.FullLoader)

    main(config)


import jieba
import random
import argparse
from tqdm import tqdm
# import logging
# import pandas as pd

from confusion_set import PinyinConfusionSet, StrokeConfusionSet, YaclcConfusionSet
from utils import is_chinese_char

class yaclcAug:
    def __init__(self, yaclc_char_prob,
                 yaclc_word_prob,
                 replace_prob,
                 py_prob,
                 jy_prob,
                 sk_prob,
                 vocab_path='confusions/vocab.txt',
                 yaclc_char_file='confusions/confusion_char.txt',
                 yaclc_word_file='confusions/confusion_word.txt',
                 same_py_file='confusions/same_pinyin.txt',
                 simi_py_file='confusions/simi_pinyin.txt',
                 stroke_file='confusions/same_stroke.txt'):

        self.yaclc_word_prob = yaclc_word_prob
        self.yaclc_char_prob = yaclc_char_prob

        self.replace_prob = replace_prob
        self.py_prob_thr = py_prob
        self.jy_prob_thr = jy_prob + self.py_prob_thr
        self.sk_prob_thr = sk_prob + self.jy_prob_thr

        self.chinese_vocab = self.get_chinese_vocab(vocab_path)

        self.yaclc_char_confu = YaclcConfusionSet(yaclc_char_file)
        self.yaclc_word_confu = YaclcConfusionSet(yaclc_word_file)
        self.yaclc_char_keys = self.yaclc_char_confu.confusion.keys()
        self.yaclc_word_keys = self.yaclc_word_confu.confusion.keys()

        self.same_py_confu = PinyinConfusionSet(same_py_file)
        self.simi_py_confu = PinyinConfusionSet(simi_py_file)
        self.sk_confu = StrokeConfusionSet(stroke_file)

    def get_chinese_vocab(self, vocab_path):
        chinese_vocab = set()
        with open(vocab_path, 'r', encoding='utf-8') as f:
            for line in f.readlines():
                line = line.strip()
                if is_chinese_char(line):
                    chinese_vocab.add(line)
        return list(chinese_vocab)

    def word_replace(self, sent_list, word_cut, replace_indices):
        idx = 0
        for word in word_cut:
            if word in self.yaclc_word_keys:
                if random.random() <= self.yaclc_word_prob:
                    new_word = self.yaclc_word_confu.get_confusion_item(word)
                    if new_word is not None:
                        sent_list[idx: idx + len(word)] = list(new_word)
                        replace_indices[idx: idx + len(word)] = [1 for _ in range(len(word))]
            idx += len(word)

    def char_replace(self, sent_list, replace_indices):
        for idx in range(len(sent_list)):
            if replace_indices[idx] == 1:
                continue
            if sent_list[idx] in self.yaclc_char_keys and random.random() <= self.yaclc_char_prob:
                sent_list[idx] = self.yaclc_char_confu.get_confusion_item(sent_list[idx])
                replace_indices[idx] = 1

    def process(self, sent):
        length = len(sent)
        sent_list = list(sent)
        replace_indices = [0 for _ in range(length)]
        word_cut = jieba.lcut(sent)
        self.word_replace(sent_list, word_cut, replace_indices)
        self.char_replace(sent_list, replace_indices)

        for idx in range(length):
            if is_chinese_char(sent_list[idx]):
                if random.random() <= self.replace_prob:
                    prob = random.random()
                    if prob <= self.py_prob_thr:
                        new_char = self.same_py_confu.get_confusion_item(sent_list[idx])
                    elif prob <= self.jy_prob_thr:
                        new_char = self.simi_py_confu.get_confusion_item(sent_list[idx])
                    elif prob <= self.sk_prob_thr:
                        new_char = self.sk_confu.get_confusion_item(sent_list[idx])
                    else:
                        new_char = self.chinese_vocab[random.randint(0, len(self.chinese_vocab) - 1)]
                    if new_char is not None:
                        sent_list[idx] = new_char
                        replace_indices[idx] = 1

        if any(replace_indices):
            return ''.join(sent_list)
        return None

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument('-i', '--input_path',
                        help='Path to the source file',
                        required=True)
    parser.add_argument('-o', '--output_path',
                        help='Path to the target file',
                        required=True)
    args = parser.parse_args()

    aug = yaclcAug(yaclc_char_prob=0.1, yaclc_word_prob=0.01, replace_prob=0.01, py_prob=0.5, jy_prob=0.2, sk_prob=0.3)

    text_list = []
    with open(args.input_path, 'r', encoding='utf-8') as rfd:
        for line in rfd.readlines():
            line = line.strip().split('\t')[1].replace(' ', '')
            text_list.append(line)

    aug_text_list = []
    for text in tqdm(text_list, total=len(text_list)):
        new_text = aug.process(text)
        if new_text:
            aug_text_list.append((new_text, text))

    with open(args.output_path, 'w', encoding='utf-8') as wfd:
        for aug_text in aug_text_list:
            wfd.write(f'{(aug_text[0])}\t{aug_text[1]}\n')


#coding=utf-8
import numpy as np
import matplotlib.pyplot as plt
from GlobalConfig import *

def pictureDayUser():
	n_groups = 31
	index = np.arange(n_groups)
	aDayNews = [0 for i in index]
	aDayDifferrentUsers = [0 for i in index]
	aDayDifferrentNews = [0 for i in index]

	for i in index:
		tempUsers = []
		tempNews = []
		with open(GlobalConfig.dayFileName+str(i+1)) as fr:
			news = fr.readlines()
			aDayNews[i] = len(news)
			for item in news:
				tempUsers.append(item.split("\t")[0])
				tempNews.append(item.split("\t")[1])

			aDayDifferrentUsers[i] = len(set(tempUsers))
			aDayDifferrentNews[i] = len(set(tempNews))
		#print aDayNews[i],aDayDifferrentUsers[i],aDayDifferrentNews
	plt.figure(figsize=(30,9))
	bar_with = 0.28
	opacity = 0.6
	#rects1 = plt.bar(index, aDayNews, bar_with, alpha=opacity, color='b', label="News")
	rects2 = plt.bar(index + bar_with, aDayDifferrentNews, bar_with, alpha=opacity, color='r',\
		label="DiffNews")
	rects3 = plt.bar(index + 2*bar_with, aDayDifferrentUsers, bar_with, alpha=opacity, color='y', \
		label="DiffUsers")

	plt.xlabel("Day")
	plt.ylabel("Numbers")
	plt.title("One day's news and users")
	plt.xticks(index+1.5*bar_with, index+1)
	plt.legend()
	plt.show()

if __name__ == "__main__":
	pictureDayUser()


from __future__ import absolute_import
import abc

import pytest
from django.core.exceptions import ValidationError
from django.utils import timezone
from nose.tools import assert_equal, assert_in, assert_raises

import mock

from framework.auth import Auth

from tests.base import DbTestCase
from osf_tests.factories import UserFactory, CommentFactory, ProjectFactory, PreprintFactory, RegistrationFactory, AuthUserFactory
from osf.models import NotableDomain, SpamStatus
from website import settings, mails

@pytest.mark.django_db
@mock.patch('framework.auth.views.mails.send_mail')
def test_throttled_autoban(mock_mail):
    settings.SPAM_THROTTLE_AUTOBAN = True
    user = AuthUserFactory()
    projects = []
    for _ in range(7):
        proj = ProjectFactory(creator=user)
        proj.flag_spam()
        proj.save()
        projects.append(proj)
    mock_mail.assert_called_with(osf_support_email=settings.OSF_SUPPORT_EMAIL,
            can_change_preferences=False,
            to_addr=user.username,
            user=user,
            mail=mails.SPAM_USER_BANNED)
    user.reload()
    assert user.is_disabled
    for project in projects:
        assert not project.is_public

@pytest.mark.enable_implicit_clean
class TestReportAbuse(DbTestCase):

    def setUp(self):
        super(TestReportAbuse, self).setUp()
        self.comment = CommentFactory()
        self.auth = Auth(user=self.comment.user)

    def test_report_abuse(self):
        user = UserFactory()
        time = timezone.now()
        self.comment.report_abuse(
            user, date=time, category='spam', text='ads', save=True)
        assert_equal(self.comment.spam_status, SpamStatus.FLAGGED)
        equivalent = dict(
            date=time,
            category='spam',
            text='ads',
            retracted=False
        )
        assert_in(user._id, self.comment.reports)
        assert_equal(self.comment.reports[user._id], equivalent)

    def test_report_abuse_own_comment(self):
        with assert_raises(ValueError):
            self.comment.report_abuse(
                self.auth.user,
                category='spam', text='ads',
                save=True
            )
        assert_equal(self.comment.spam_status, SpamStatus.UNKNOWN)

    def test_retract_report(self):
        user = UserFactory()
        time = timezone.now()
        self.comment.report_abuse(
            user, date=time, category='spam', text='ads', save=True
        )
        assert_equal(self.comment.spam_status, SpamStatus.FLAGGED)
        self.comment.retract_report(user, save=True)
        assert_equal(self.comment.spam_status, SpamStatus.UNKNOWN)
        equivalent = {
            'date': time,
            'category': 'spam',
            'text': 'ads',
            'retracted': True
        }
        assert_in(user._id, self.comment.reports)
        assert_equal(self.comment.reports[user._id], equivalent)

    def test_retract_report_not_reporter(self):
        reporter = UserFactory()
        non_reporter = UserFactory()
        self.comment.report_abuse(
            reporter, category='spam', text='ads', save=True
        )
        with assert_raises(ValueError):
            self.comment.retract_report(non_reporter, save=True)
        assert_equal(self.comment.spam_status, SpamStatus.FLAGGED)

    def test_retract_one_report_of_many(self):
        user_1 = UserFactory()
        user_2 = UserFactory()
        time = timezone.now()
        self.comment.report_abuse(
            user_1, date=time, category='spam', text='ads', save=True
        )
        assert_equal(self.comment.spam_status, SpamStatus.FLAGGED)
        self.comment.report_abuse(
            user_2, date=time, category='spam', text='all', save=True
        )
        self.comment.retract_report(user_1, save=True)
        equivalent = {
            'date': time,
            'category': 'spam',
            'text': 'ads',
            'retracted': True
        }
        assert_in(user_1._id, self.comment.reports)
        assert_equal(self.comment.reports[user_1._id], equivalent)
        assert_equal(self.comment.spam_status, SpamStatus.FLAGGED)

    def test_cannot_remove_flag_not_retracted(self):
        user = UserFactory()
        self.comment.report_abuse(
            user, category='spam', text='ads', save=True
        )
        self.comment.remove_flag(save=True)
        assert_equal(self.comment.spam_status, SpamStatus.FLAGGED)

    def test_remove_flag(self):
        self.comment.flag_spam()
        self.comment.save()
        assert_equal(self.comment.spam_status, SpamStatus.FLAGGED)
        self.comment.remove_flag(save=True)
        assert_equal(self.comment.spam_status, SpamStatus.UNKNOWN)

    def test_validate_reports_bad_key(self):
        self.comment.reports[None] = {'category': 'spam', 'text': 'ads'}
        with assert_raises(ValidationError):
            self.comment.save()

    def test_validate_reports_bad_type(self):
        self.comment.reports[self.auth.user._id] = 'not a dict'
        with assert_raises(ValidationError):
            self.comment.save()

    def test_validate_reports_bad_value(self):
        self.comment.reports[self.auth.user._id] = {'foo': 'bar'}
        with assert_raises(ValidationError):
            self.comment.save()

@pytest.mark.django_db
class TestSpamState:
    @pytest.fixture(params=[
        CommentFactory,
        ProjectFactory,
        PreprintFactory,
        RegistrationFactory,
        UserFactory,
    ])
    def spammable_thing(self, request):
        spammable_factory = request.param
        return spammable_factory()

    def test_flag_spam(self, spammable_thing):
        assert not spammable_thing.is_spammy
        assert not spammable_thing.is_spam
        spammable_thing.flag_spam()
        spammable_thing.save()
        assert spammable_thing.is_spammy
        assert not spammable_thing.is_spam

    def test_confirm_ham(self, spammable_thing):
        spammable_thing.confirm_ham(save=True)
        assert spammable_thing.is_ham

    def test_confirm_spam(self, spammable_thing):
        spammable_thing.confirm_spam(save=True)
        assert spammable_thing.is_spam

    @pytest.mark.parametrize('assume_ham', (True, False))
    @pytest.mark.parametrize('spam_status, expected_props', (
            (SpamStatus.UNKNOWN, {
                'is_spam': False,
                'is_spammy': False,
                'is_ham': False,
                'is_hammy': None,  # set in the test body based on assume_ham
            }),
            (SpamStatus.FLAGGED, {
                'is_spam': False,
                'is_spammy': True,
                'is_ham': False,
                'is_hammy': False,
            }),
            (SpamStatus.SPAM, {
                'is_spam': True,
                'is_spammy': True,
                'is_ham': False,
                'is_hammy': False,
            }),
            (SpamStatus.HAM, {
                'is_spam': False,
                'is_spammy': False,
                'is_ham': True,
                'is_hammy': True,
            }),
    ))
    def test_spam_status_properties(self, spammable_thing, assume_ham, spam_status, expected_props):
        if spam_status == SpamStatus.UNKNOWN:
            expected_props['is_hammy'] = assume_ham

        with mock.patch.object(type(spammable_thing), 'is_assumed_ham', new_callable=mock.PropertyMock) as mock_assumed_ham:
            mock_assumed_ham.return_value = assume_ham
            spammable_thing.spam_status = spam_status

            assert spammable_thing.is_spam == expected_props['is_spam']
            assert spammable_thing.is_spammy == expected_props['is_spammy']
            assert spammable_thing.is_ham == expected_props['is_ham']
            assert spammable_thing.is_hammy == expected_props['is_hammy']

@pytest.mark.django_db
class TestSpamCheckEmailDomain:
    @mock.patch('osf.models.spam.SpamMixin.do_check_spam', return_value=False)
    @mock.patch.object(settings, 'SPAM_SERVICES_ENABLED', True)
    @mock.patch.object(settings, 'SPAM_CHECK_PUBLIC_ONLY', False)
    def test_trusted_domain(self, mock_do_check_spam):
        user = UserFactory()
        project = ProjectFactory()

        # spam check should normally call do_check_spam
        assert not mock_do_check_spam.called
        is_spam = project.check_spam(user, saved_fields={'title'}, request_headers={})
        assert not is_spam
        assert mock_do_check_spam.called

        # but what if we trust the user's email domain?
        user_email_address = user.emails.values_list('address', flat=True).first()
        user_email_domain = user_email_address.rpartition('@')[2].lower()
        NotableDomain.objects.create(
            domain=user_email_domain,
            note=NotableDomain.Note.ASSUME_HAM_UNTIL_REPORTED,
        )

        # should not call do_check_spam this time
        mock_do_check_spam.reset_mock()
        assert not mock_do_check_spam.called
        is_spam = project.check_spam(user, saved_fields={'title'}, request_headers={})
        assert not is_spam
        assert not mock_do_check_spam.called


#!/usr/bin/env python
# coding: utf-8

import torch
from torchvision.datasets import ImageNet
from torch.utils.data import DataLoader, Dataset
from torchvision import transforms
import torchvision

import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt
import seaborn as sns

import os
from PIL import Image
import json

from circuit_explorer.utils import load_config
from circuit_explorer.utils import get_layers_from_model

from torchvision import transforms, datasets
from circuit_explorer.data_loading import rank_image_data, single_image_data
from circuit_explorer.receptive_fields import receptive_field, receptive_field_for_unit

from collections import defaultdict
from circuit_explorer.target import layer_saver
from heapq import heappush, heappushpop

from circuit_explorer.target import layer_saver
from circuit_explorer.data_loading import default_unnormalize

import hdbscan
from dataset.Dataset_Utils import ImageNetReceptiveField, ImageNet2

position = [6, 6]

def k_layer_activations_from_dataloader(layers,dataloader,model,num_units=[384],k=300,position=[6,6],batch_size=64, device="cuda:0"):

    device = next(model.parameters()).device
    topk_activations = {}
    layer_num_units = {}

    layer_activations = {}
    if isinstance(layers,str):
        layers = [layers]
    for i in layers:
        num_units = get_layers_from_model(model)[i].out_channels
        layer_num_units[i] = num_units
        for unit in range(num_units):
            topk_activations[f'{i}_{unit}'] = []

    for dl_idx, (data, labels) in enumerate(tqdm(dataloader)):
        ds_idxs = list(range(dl_idx*batch_size, (dl_idx+1)*batch_size))

        images = data.to(device)
        with layer_saver(model, layers) as extractor:
            batch_layer_activations = extractor(images)
            for i in layers:
                for unit in range(layer_num_units[i]):
                    unit_activations = list(batch_layer_activations[i][:, unit, position[0], position[1]].detach().to('cpu').numpy())
                    tuples_to_add = zip(unit_activations, ds_idxs)

                    for tup in tuples_to_add:
                        if len(topk_activations[f'{i}_{unit}']) < k:
                            heappush(topk_activations[f'{i}_{unit}'], tup)
                        else:
                            heappushpop(topk_activations[f'{i}_{unit}'], tup)

    return topk_activations

def cluster(target_layer_activations, layer, unit, model, imagenet_train_dataset, min_cluster_size=50, device="cuda:0"):
    original_activations = target_layer_activations[f'{layer}_{unit}']
    activations, indices = list(zip(*original_activations))

    sorted_idxs = np.flip(np.argsort(activations))
    sorted_activations = np.array(activations)[sorted_idxs]
    sorted_indices = np.array(indices)[sorted_idxs]

    values, idxs = sorted_activations, sorted_indices

    layer_activations = {}
    if isinstance(layer, str):
        layers = [layer]
    for i in layers:
        layer_activations[i] = []

    for i in range(len(idxs)):
        data = imagenet_train_dataset[idxs[i]]
        images = data[0].to(device)

        with layer_saver(model, layers) as extractor:
            batch_layer_activations = extractor(images.unsqueeze(0))
            for i in layers:
                layer_activations[i].append(batch_layer_activations[i].detach().to('cpu'))

    for i in layers:
        layer_activations[i] = torch.cat(layer_activations[i])

    cluster_points = layer_activations[layer][:,:,position[0],position[1]]

    clusterer = hdbscan.HDBSCAN(min_cluster_size=min_cluster_size)
    labels = clusterer.fit_predict(cluster_points.numpy())
    return labels, values, idxs

def get_unit_clustering_statistics(target_layer_activations, layer, model, imagenet_train_dataset, num_filters=256, min_cluster_size=50, device="cuda:0"):
    unit_dict = {}
    for u in tqdm(range(num_filters)):
        labels, _, _ = cluster(target_layer_activations, layer, u, model, imagenet_train_dataset, min_cluster_size=min_cluster_size, device=device)

#         num_0s = len(np.where(labels==0)[0])
#         num_1s = len(np.where(labels==1)[0])
#         num_neg_1s = len(np.where(labels==-1)[0])
        unit_dict[u] = {val:len(np.where(labels==val)[0]) for val in np.unique(labels)}
    return unit_dict

def cluster_and_plot_unit(target_layer_activations, layer, model, imagenet_train_dataset, unit, name, min_cluster_size=50, device="cuda:0"):
    labels, values, idx = cluster(target_layer_activations, layer, unit, model, imagenet_train_dataset, min_cluster_size=min_cluster_size, device=device)

    plt.figure(figsize=(12, 12))
    num_images = 16
    for i in range(num_images):
        plt.subplot(4, 4, i+1)
        if i < int(num_images/2):
            plt.imshow(default_unnormalize(imagenet_train_dataset[random.choice(idxs[np.where(labels == 0)])][0]).numpy().transpose((1, 2, 0)))
        else:
            plt.imshow(default_unnormalize(imagenet_train_dataset[random.choice(idxs[np.where(labels == 1)])][0]).numpy().transpose((1, 2, 0)))
        plt.axis('off')
        plt.tick_params(axis='both', left=False, top=False, right=False, bottom=False, labelleft=False, labeltop=False, labelright=False, labelbottom=False)
        plt.tight_layout()

    plt.savefig(name)

def main(FLAGS):

    device = FLAGS.device

    config_file = FLAGS.config_file
    config = load_config(config_file)

    model = config.model
    model = model.to(device)

    all_layers = get_layers_from_model(model)
    all_layers.keys()

    model.eval().cuda()
    mean = (0.485, 0.456, 0.406)
    std = (0.229, 0.224, 0.225)

    preprocessing = transforms.Compose(
                [
                    transforms.Resize(256),
                    transforms.CenterCrop(224),
                    transforms.ToTensor(),
                    transforms.Normalize(mean, std),
                ]
            )

    position = [6,6]
    # layer = 'features.8'
    # unit = 53

    #dog/metal feature
    layer = FLAGS.layer

    all_recep_field_params = receptive_field(model.features, (3,224,224))
    recep_field = receptive_field_for_unit(all_recep_field_params, layer, position)

    # imagenet2_train_dataset = ImageNet2('image_data/imagenet_2', preprocessing,
    #                                                 recep_field=recep_field)

    # imagenet2_train_dataloader = DataLoader(
    #             imagenet2_train_dataset,
    #             batch_size=128,
    #             num_workers=1,
    #             shuffle=False,
    #             drop_last=False,
    #             pin_memory=True
    #         )

    imagenet_train_dataset = ImageNetReceptiveField(FLAGS.data_path, "train", preprocessing,
                                                    recep_field=recep_field)

    imagenet_train_dataloader = DataLoader(
                imagenet_train_dataset,
                batch_size=FLAGS.batch_size,
                num_workers=8,
                shuffle=False,
                drop_last=False,
                pin_memory=True
            )

    target_layer_activations = k_layer_activations_from_dataloader(layer,imagenet_train_dataloader,
                                                                model, k=FLAGS.k, batch_size=FLAGS.batch_size,
                                                                device=device)

    num_filters = all_layers[layer].out_channels
    unit_dict = get_unit_clustering_statistics(target_layer_activations, layer, model,
                                            imagenet_train_dataset, num_filters=num_filters, min_cluster_size=FLAGS.min_cluster_size,
                                            device=device)

    layer_num = int(layer.split('.')[-1])
    torch.save(target_layer_activations, f'outputs/layer_{layer_num}/layer_{layer_num}_image_net_{FLAGS.k}.pt')
    torch.save(unit_dict, f'outputs/layer_{layer_name}/cluster_unit_stats_k={FLAGS.k}_mcs={FLAGS.min_cluster_size}.pt')

    if FLAGS.plot:
        for unit in units:
            cluster_and_plot_unit(target_layer_activations, layer, model, imagenet_train_dataset, unit,
                                    f'outputs/layer_{layer_name}/unit_{unit}_clusters.png', min_cluster_size=FLAGS.min_cluster_size,
                                    device=device)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
                prog='HDBScan_Clustering',
                description='clusters specified layer and gives unit features')
    parser.add_argument("--layer", default="features.10", type=str, help="specified layer to collect top k activations at")
    parser.add_argument("--k", default=1000, type=int, help="number of activations to collect and cluster with")
    parser.add_argument("--batch-size", default=64, type=int, help="batch size")
    parser.add_argument("--min-cluster-size", default=50, type=int, help="HDB clustering min cluster size")
    parser.add_argument("--device", default="cuda:0", type=str, help="device to run on")
    parser.add_argument("--plot", default=False, type=bool, help="whether to plot and save given units")
    parser.add_argument('--units', nargs='+', help='which units to plot the two classes of')
    parser.add_argument("--config-file", default="configs/alexnet_sparse_config.py", type=str,
        help="config file specifying model to run (default: alexnet-sparse)"
        )
    parser.add_argument("--data-path", default="image_data/imagenet", type=str,
        help="where the imagenet data is located")

    FLAGS = parser.parse_args()
    main(FLAGS)


# This is a pytorch implementation of biLSTM

import torch
from torch import nn
from torch.nn import functional as F
from torch.nn.utils.rnn import pack_sequence, pad_packed_sequence, pad_sequence

import utils
import os

import pandas as pd

class CNN_BiLSTM(nn.Module):
    """ Bidirectional long-short term memory network.

    It has a bi-LSTM as the main part, then follows a three-layer perception
    classifier. Dropout is applied to the output of the bi-LSTM and every layer
    of the perception, when the probability is set to be 0.5. Finally a log
    softmax is used to normalize the output.
    """
    def __init__(self, args):
        super(CNN_BiLSTM, self).__init__()
        self.args = args
        self.embed = nn.Embedding.from_pretrained(
            utils.embed_from_pretrained(args),
            freeze=self.args.freeze_embed
        )
        # self.conv1 = nn.Conv2d(in_channels=1,
        #                        out_channels=self.args.cnn_out_channels,
        #                        kernel_size=self.args.kernel_size,
        #                        stride=self.args.cnn_stride)
        self.conv1 = nn.Conv1d(in_channels=self.args.embed_num,
                               out_channels=self.args.cnn_out_channels,
                               kernel_size=self.args.kernel_size,
                               stride=self.args.cnn_stride,
                               bias=False)
        self.conv2 = nn.Conv1d(in_channels=self.args.embed_num,
                               out_channels=self.args.cnn_out_channels,
                               kernel_size=self.args.kernel_size,
                               stride=self.args.cnn_stride,
                               bias=False)
        self.maxpool = nn.MaxPool1d(kernel_size=3)
        self.batchnorm_forward = nn.BatchNorm1d(self.args.cnn_out_channels)
        self.batchnorm_backward = nn.BatchNorm1d(self.args.cnn_out_channels)
        self.lstm_forward = nn.LSTM(self.args.cnn_out_channels,
                                    self.args.hidden_dim,
                                    num_layers=1,
                                    dropout=self.args.dropout,
                                    bias=True)
        self.lstm_backward = nn.LSTM(self.args.cnn_out_channels,
                                     self.args.hidden_dim,
                                     num_layers=1,
                                     dropout=self.args.dropout,
                                     bias=True)
        self.fc1 = nn.Linear(self.args.hidden_dim * 2,
                             self.args.hidden_dim)
        self.fc2 = nn.Linear(self.args.hidden_dim,
                             1)
        self.softmax = nn.LogSoftmax()
        self.dropout = nn.Dropout(p=self.args.dropout)

    def forward(self, x):
        """ Overwrites the function for forward propagation.

        It accepts a list containing 1-d tensors that contains indexes to the
        embeddings.
        """
        code, code_rev = x

        code_embedded = pad_sequence(
            [self.embed(i.to(self.args.device)) for i in code])
        code_rev_embedded = pad_sequence(
            [self.embed(i.to(self.args.device)) for i in code_rev])

        code_embedded = torch.transpose(code_embedded, 0, 1)
        code_rev_embedded = torch.transpose(code_rev_embedded, 0, 1)

        code_embedded = code_embedded.view(code_embedded.shape[0],
                                           code_embedded.shape[1],
                                           code_embedded.shape[2])
        code_rev_embedded = code_rev_embedded.view(code_rev_embedded.shape[0],
                                                   code_rev_embedded.shape[1],
                                                   code_rev_embedded.shape[2])

        code_embedded = torch.transpose(code_embedded, 1, 2)
        code_rev_embedded = torch.transpose(code_rev_embedded, 1, 2)

        # 1d convolution layer
        code_embedded = self.conv2(code_embedded)
        code_rev_embedded = self.conv2(code_rev_embedded)

        # Dropout layer
        code_embedded = self.dropout(code_embedded)
        code_rev_embedded = self.dropout(code_rev_embedded)

        code_embedded = torch.transpose(code_embedded, 1, 2)
        code_rev_embedded = torch.transpose(code_rev_embedded, 1, 2)
        code_embedded = torch.transpose(code_embedded, 0, 1)
        code_rev_embedded = torch.transpose(code_rev_embedded, 0, 1)

        # bi-layer LSTM
        lstm_forward_out, _ = self.lstm_forward(code_embedded)
        lstm_backward_out, _ = self.lstm_backward(code_rev_embedded)

        # # Use the maxpool to get the output
        # lstm_forward_out = torch.transpose(lstm_forward_out, 0, 1)
        # lstm_forward_out = torch.transpose(lstm_forward_out, 1, 2)
        # lstm_forward_out = F.tanh(lstm_forward_out)
        # lstm_forward_out = F.max_pool1d(lstm_forward_out,
        #                                 lstm_forward_out.size(2)).squeeze(2)

        # lstm_backward_out = torch.transpose(lstm_backward_out, 0, 1)
        # lstm_backward_out = torch.transpose(lstm_backward_out, 1, 2)
        # lstm_backward_out = F.tanh(lstm_backward_out)
        # lstm_backward_out = F.max_pool1d(lstm_backward_out,
        #                                  lstm_backward_out.size(2)).squeeze(2)

        # bilstm_out = torch.cat((lstm_forward_out, lstm_backward_out), dim=1)

        # Use the last output of bilstm as output
        bilstm_out = torch.cat((lstm_forward_out, lstm_backward_out), dim=2)
        lengths = torch.tensor([i.shape[0] for i in code])
        lengths = ((lengths - self.args.kernel_size) /
                   self.args.cnn_stride + 1).to(torch.long)

        idx = (lengths - 1).view(-1, 1).expand(
            len(lengths), bilstm_out.size(2))

        # if batch is first, change 0 to 1
        idx = idx.unsqueeze(0).to(self.args.device)
        bilstm_out = bilstm_out.gather(0, idx).squeeze(0)

        bilstm_out = self.dropout(bilstm_out)
        y = F.relu(self.fc1(bilstm_out))
        y = self.fc2(y)

        return y

    def initialize(self):
        """ Initializes the parameters of the model itself.

        It initializes the weights of bilstm orthogonally, which is suggested
        by a lot of blogger on the Internet, and initializes the biases of the
        bilstm with zeros.

        Args:
            None

        Returns:
            None
        """
        # Initialize gates and hidden weights of layer 1 with orthogonal
        # normalization
        print(self.lstm_forward.bias_ih_l0)
        nn.init.orthogonal_(self.lstm_forward.weight_ih_l0)
        nn.init.orthogonal_(self.lstm_forward.weight_hh_l0)
        nn.init.orthogonal_(self.lstm_backward.weight_ih_l0)
        nn.init.orthogonal_(self.lstm_backward.weight_hh_l0)

        # nn.init.xavier_normal_(self.lstm_forward.weight_ih_l0)
        # nn.init.xavier_normal_(self.lstm_forward.weight_hh_l0)
        # nn.init.xavier_normal_(self.lstm_backward.weight_ih_l0)
        # nn.init.xavier_normal_(self.lstm_backward.weight_hh_l0)

        nn.init.zeros_(self.lstm_forward.bias_ih_l0)
        nn.init.ones_(self.lstm_forward.bias_ih_l0[1])
        nn.init.zeros_(self.lstm_forward.bias_hh_l0)
        nn.init.zeros_(self.lstm_backward.bias_ih_l0)
        nn.init.ones_(self.lstm_backward.bias_ih_l0[1])
        nn.init.zeros_(self.lstm_backward.bias_hh_l0)

        nn.init.xavier_normal_(self.conv2.weight)

        # # Initialize gates and hidden weights of layer 2 with orthogonal
        # # normalization
        # nn.init.orthogonal_(self.lstm_forward.weight_ih_l1)
        # nn.init.orthogonal_(self.lstm_forward.weight_hh_l1)
        # nn.init.orthogonal_(self.lstm_backward.weight_ih_l1)
        # nn.init.orthogonal_(self.lstm_backward.weight_hh_l1)

        # # nn.init.xavier_normal_(self.bilstm.weight_ih_l0)
        # # nn.init.xavier_normal_(self.bilstm.weight_hh_l0)
        # # nn.init.xavier_normal_(self.bilstm.weight_ih_l0)
        # # nn.init.xavier_normal_(self.bilstm.weight_hh_l0)

        # nn.init.zeros_(self.lstm_forward.bias_ih_l1)
        # nn.init.ones_(self.lstm_forward.bias_ih_l1[1])
        # nn.init.zeros_(self.lstm_forward.bias_hh_l1)
        # nn.init.zeros_(self.lstm_backward.bias_ih_l1)
        # nn.init.ones_(self.lstm_backward.bias_ih_l1[1])
        # nn.init.zeros_(self.lstm_backward.bias_hh_l1)

        pass


from contextlib import contextmanager

import zmq

from pairedmq.exc import TimeoutError

@contextmanager
def handshake_receiver(timeout=None):
    socket = zmq.Context().socket(zmq.REP)
    socket.setsockopt(zmq.LINGER, 0)
    port = socket.bind_to_random_port("tcp://127.0.0.1")
    try:
        def wait():
            poller = zmq.Poller()
            poller.register(socket, zmq.POLLIN)
            if poller.poll(timeout):
                received = int(socket.recv(zmq.DONTWAIT))
                socket.send_string("")  # Ack reception
            else:
                raise TimeoutError("Handshake timeout")
            return received

        yield (port, wait)
    finally:
        socket.close()

def handshake_sender(handshake_port, port, timeout=None):
    if timeout is None:
        timeout = -1
    socket = zmq.Context().socket(zmq.REQ)
    socket.setsockopt(zmq.LINGER, 0)
    socket.connect("tcp://127.0.0.1:%s" % handshake_port)
    socket.send_string(str(port))

    poller = zmq.Poller()
    poller.register(socket, zmq.POLLIN)
    if poller.poll(timeout):
        socket.recv()  # wait for ack
        socket.close()
    else:
        socket.close()
        raise TimeoutError("Handshake timeout")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import fnmatch
from tinytag import TinyTag
import os, re, requests, sys
import os.path
from os import path
import json
import pygame
import threading, random
NEXT = pygame.USEREVENT + 1
os.environ["SDL_VIDEODRIVER"] = "dummy"

class MuuzikPlayer:
    def __init__(self, config, language):
        self.language = language["musicPlayer"]
        self.information_path = config["information_path"]
        self.pattern = '*.mp3'
        self.playlist = []
        self.artist_list = []
        self.song_list = []
        self.current_track = 0
        self.pathToMusic = config['music_path'].split("\n")
        self.musicFile = []
        self.backgroundthread = None
        self.train_thread = None
        self.running = False
        self.stopTheMusic = False

    def setLanguage(self, language):
        self.language = language

    def scanMusic(self):
        print('test 1')
        numberOfSongsBeforeScan = self.countMySongs()

        rootPaths = ['/home/pi/Music']
        songlist = []
        artistlist = []
        album_list = []
        titlelist = []
        genre_list = []
        pattern = '*.mp3'
        print(rootPaths)
        for rootPath in rootPaths:
            for root, dirs, files in os.walk(rootPath):
                for filename in fnmatch.filter(files, pattern):
                    if path.exists(os.path.join(root, filename)):
                        try:
                            audiofile = TinyTag.get(os.path.join(root, filename))
                            print(audiofile)
                            album = str(audiofile.album)
                            title = str(audiofile.title)
                            genre = str(audiofile.genre)
                            albumArtist = str(audiofile.albumartist)
                            artist = str(audiofile.artist)
                            year = str(audiofile.year)

                            a = albumArtist
                            print(a)
                            if albumArtist == 'None':
                                print(artist)
                                a = artist
                            if title.strip() != '' and title.strip() not in titlelist:
                                titlelist.append(title.strip())

                            if album.strip() != '' and album.strip() not in album_list:
                                album_list.append(album.strip())

                            if genre.strip() not in genre_list:
                                genre_list.append(genre)

                            # print(genre + ' ' + a + ' ' + title + ' ' + os.path.join(root, filename))

                            if (a.strip()) not in artistlist:
                                artistlist.append(a)

                            songlist.append({
                                'song': '{} {} {} {}  {} {}  {} '.format(album.strip(), title.strip(), genre, albumArtist, artist, year,
                                                                         os.path.join(root, filename)),
                                'path': os.path.join(root, filename),
                                'artist': '{}'.format(a),
                                'song_name': '{}'.format(title)
                            })
                        except:
                            print('error')

        with open(self.information_path + '/musicFound.json', 'w') as f:
            json.dump(songlist, f, indent=4)

        print('writing in root')
        with open('/root/.config/rhasspy/profiles/de/slots/music_artists', "w") as txt_file:
            for line in artistlist:
                txt_file.write(re.sub('[^a-zA-Z0-9 ]', '', line) + "\n")
        print('finisched writing in root')

        with open(self.information_path + '/music_artists.ini', "w") as txt_file:
            for line in artistlist:
                txt_file.write(re.sub('[^a-zA-Z0-9 ]', '', line) + "\n")

        with open('/root/.config/rhasspy/profiles/de/slots/music_titles', "w") as txt_file:
            for line in titlelist:
                txt_file.write(re.sub('[^a-zA-Z0-9 ]', '', line) + "\n")

        with open(self.information_path + '/music_titles.ini', "w") as txt_file:
            for line in titlelist:
                txt_file.write(re.sub('[^a-zA-Z0-9 ]', '', line) + "\n")

        with open('/root/.config/rhasspy/profiles/de/slots/music_albums', "w") as txt_file:
            for line in album_list:
                txt_file.write(re.sub('[^a-zA-Z0-9 ]', '', line) + "\n")

        with open(self.information_path + '/music_albums.ini', "w") as txt_file:
            for line in album_list:
                txt_file.write(re.sub('[^a-zA-Z0-9 ]', '', line) + "\n")

        with open('/root/.config/rhasspy/profiles/de/slots/music_genres', "w") as txt_file:
            for line in genre_list:
                txt_file.write(re.sub('[^a-zA-Z0-9 ]', '', line) + "\n")

        with open(self.information_path + '/music_genres.ini', "w") as txt_file:
            for line in genre_list:
                txt_file.write(re.sub('[^a-zA-Z0-9 ]', '', line) + "\n")

        numberOfSongsAfterScan = self.countMySongs()
        if (numberOfSongsAfterScan < numberOfSongsBeforeScan):
            print(' neue Lieder')
            text = self.language["removedSongs"].format(numberOfSongsBeforeScan - numberOfSongsAfterScan)
            self.train()
            return text

        if (numberOfSongsAfterScan > numberOfSongsBeforeScan):
            print(' keine neue Lieder')
            text = self.language["foundNewSongs"].format(numberOfSongsAfterScan - numberOfSongsBeforeScan)
            #self.train(hermes, intentMessage)
            return text

            #return self.language["scanningIsDone"]
        print(' ende Scanning')
        #self.train(hermes, intentMessage)
        #TODO add found artists to slot artists
        #TODO check which songs are new and only add these artists to the artists slot
        # self.train('', hermes, intentMessage)
        return self.language["scanningIsDone"]

    def train(self):
        print('TRAINING')
        # hermes.publish("rhasspy/asr/" + intentMessage["site_id"] + "/train")
        r = requests.post('http://rhasspy:12101/api/train')
        #hermes.publish("hermes/tts/say", json.dumps({"text": 'Ich trainiere jetzt', "siteId": intentMessage["site_id"]}))
        return r

    def playPlaylist(self, playListName):
        if  path.isdir(self.information_path + '/playlists') == False:
            os.mkdir(self.information_path + '/playlists')

        musicFile = open(self.information_path + '/playlists')
        allSongs= json.load(musicFile)
        musicFile.close()
        return len(allSongs)

    def addToFavourites(self, playListName):
        print(self)
        if self.running:
            song = self.playlist(self.current_track)
            playListPath = self.information_path + '/Favourites.m3u'
            if path.exists(playListPath) == False:
                file = open(playListPath, "w+")
                file.append()
            else:
                with open(playListPath, 'a') as file:
                    file.write(song)
            return self.language["addMp3ToPlaylist"]["songAddedToPlaylist"]
        else:
            return self.language["addMp3ToPlaylist"]["noMusicIsRunning"]

    def addMp3ToPlaylist(self, playListName):
        if self.running:
            song = self.playlist(self.current_track)
            playListPath = self.information_path + '/playlists' + playListName +'.m3u'
            if path.exists(playListPath) == False:
                file = open(playListPath, "w+")
                file.append()
            else:
                with open(playListPath, 'a') as file:
                    file.write(song)
            return self.language["addMp3ToPlaylist"]["songAddedToPlaylist"]
        else:
            return self.language["addMp3ToPlaylist"]["noMusicIsRunning"]

    def removeMp3ToPlaylist(self, song):
        song = self.playlist(self.current_track)
        return song

    def getMp3FilesFromPlaylist(self, pathToPlayList):
        songs = []
        with open(pathToPlayList) as fp:
            line = fp.readline()
            cnt = 1
            while line:
                print("Line {}: {}".format(cnt, line.strip()))
                line = fp.readline().lower()
                if (".mp3" in line):
                    songs.append(line)
                cnt += 1
        return songs

    def countMySongs(self):
        try:
            musicFile = open(self.information_path  + '/musicFound.json')
            allSongs= json.load(musicFile)
            musicFile.close()
            return len(allSongs)
        except:
            return 0

    def play(self, music):
        self.playlist = []
        print(music)

        if not str(os.path.exists(self.information_path + '/musicFound.json')):
            print('play Musik')
            self.scanMusic()

        if len(self.musicFile) == 0:
            musicFile = open(self.information_path + '/musicFound.json')
            self.musicFile = json.load(musicFile)
            musicFile.close()

        title = music["title"].lower()
        album = music["album"].lower()
        artist = music["artist"].lower()
        genre = music["genre"].lower()

        i = 0
        while i < len(self.musicFile) -1:
            path = self.musicFile[i]['path']
            artist_temp = self.musicFile[i]['artist'].lower()
            song_temp = self.musicFile[i]['song_name'].lower()

            song = ' ' + self.musicFile[i]['song'].lower()
            i += 1

            if song.find(artist) > 0 \
                    or song.find(album) > 0 \
                    or song.find(title) > 0 \
                    or song.find(genre) > 0:
                self.playlist.append(path)
                self.artist_list.append(artist_temp)
                print(' artist_temp', artist_temp)
                print(' song_temp', song_temp)
                self.song_list.append(song_temp)

        self.current_track = 0
        if len(self.playlist) == 0:
            return self.language["notFound"]
        else:
            return self.playSong()

    def next(self):
        self.stop()
        print(' self.current_track', self.current_track)
        if self.current_track < len(self.playlist) - 1:
            self.current_track += 1
            self.playSong()
            print('AFTER 1 self.current_track', self.current_track)
            return self.language['nextSong']
        else:
            self.current_track = 0
            self.stop()
            print('AFTER 2 self.current_track', self.current_track)
            return self.language['playListEnded']

    def previous(self):
        if self.current_track == 0:
            self.current_track = len(self.playlist) - 1
        else:
            self.current_track -= 1
        self.playSong()
        return self.language['previuosSong']

    #Not implemented yet
    def repeat(self):
        self.playSong()
        return 'okay ich wieder hole'

    def stop(self):
        self.running = False
        self.stopTheMusic = True
        try:
            pygame.mixer.music.stop()
            pygame.quit()
        except Exception as ex:
            print('error while stopping but that is fine')
        # pygame.mixer.set_endevent(type)
        self.backgroundthread = None
        return self.language['stoppedTheMusic']

    def play_music_with_stars(self, stars):
        star_list_file_name = self.information_path + '/' + str(stars) + '_stars_list.json'
        star_list_file = open(star_list_file_name)
        star_list_file_list = json.load(star_list_file)
        star_list_file.close()
        self.playlist = star_list_file_list
        self.playSong()
        return  self.language['startedPlaying']

    def add_stars(self, stars):
        artist = self.artist_list[self.current_track]
        print(artist)
        song = self.song_list[self.current_track]
        print(song)
        path = self.playlist[self.current_track]
        print(path)
        star_list_file_name = self.information_path + '/' + str(stars) + '_stars_list.json'

        if not os.path.exists(star_list_file_name):
            print('Make new File')
            star_list_file_list = []
        else:
            star_list_file = open(star_list_file_name)
            star_list_file_list = json.load(star_list_file)
            star_list_file.close()
            print('loading file')

        write_new_file = True
        i = 0
        print(' total in file' +  str(len(star_list_file_list)))
        while i < len(star_list_file_list):
            #artist = star_list_file_list[i]['artist']
            #song = star_list_file_list[i]['song']
            #path = star_list_file_list[i]['path']

            if path == star_list_file_list[i]['path']:
                print(' found file')
                write_new_file = False
            i += 1

        if write_new_file:
            star_list_file_list.append({
                "song": song,
                "artist": artist,
                "path": path
            })
            with open(star_list_file_name, 'w') as f:
                json.dump(star_list_file_list, f, indent=4)

        print(star_list_file_list)

        return self.language['addedStars'].format(str(stars))

    def what_is_playing(self):
        print(self.current_track)
        print(self.playlist[self.current_track])
        artist = self.artist_list[self.current_track]
        song = self.song_list[self.current_track]

        print(artist)
        print(song)

        return self.language['whatIsPlaying'].format(song, artist)

    def playSong(self):
        # I am sure this is not the right way to use a thread but I have no clue how to play the music while
        # continueing the other scripts
        self.backgroundthread = None
        self.backgroundthread = threading.Thread(target=self.playing)
        self.backgroundthread.start()
        return self.language['startedPlaying']

    def playing(self):
        self.stopTheMusic = False
        # start first track
        self.tracks_number = len(self.playlist)
        NEXT = pygame.USEREVENT + 1
        print(' self.current_track ')
        print(self.current_track)
        print(' self.playlist ')
        print(self.playlist)

        # get the right speed of each song
        audiofile = TinyTag.get(self.playlist[self.current_track])
        pygame.mixer.init(frequency=audiofile.samplerate)
        print('-------------------------------------------------------------------')
        # sometimes needed I dont know why but on my rasp 4 I dont need this line anymore
        #try:
            #screen = pygame.display.set_mode((400, 300))
       # except Exception as ex:
        #    print('error but that is fine')

        print('-------------------------------------------------------------------')
        print(self.playlist)
        # start first track
        if path.exists(self.playlist[self.current_track]):
            pygame.mixer.music.load(self.playlist[self.current_track])
            pygame.mixer.music.play()
            pygame.mixer.music.set_endevent(NEXT)

            self.running = True

            while pygame.mixer.music.get_busy():
               nothing = True

            if self.stopTheMusic == False:
                self.next()
        else:
            self.stop()


# -*- coding: utf-8 -*-
import pytest
from django.core.management import call_command
from io import StringIO
from testing.validation import assert_test_database

@pytest.fixture
def django_db_setup(db, django_db_blocker):
    assert_test_database()
    new_io = StringIO()
    with django_db_blocker.unblock():
        call_command("migrate", stdout=new_io)
        call_command("create_permissions", stdout=new_io)
        call_command("populate", stdout=new_io)
        call_command(
            "createsuperuser",
            interactive=False,
            email="admin@yopmail.com",
            first_name="L",
            last_name="L",
            stdout=new_io,
        )


import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import seaborn as sns
import os
import argparse

output_dir = "../comparison/new_plots"

# compute deviation of alter/time passed inbetween visits
def compute_error(df):
    dage = df['alter'].shift(-1) - df['alter']
    dage = dage[dage.index % 16 != 15]  # drop diff between age 18 and next child's age 3
    dtime = df['time'].shift(-1) - df['time']
    dtime = dtime[dtime.index % 16 != 15]  # drop diff between age 18 and next child's age 3
    time_err = dtime - dage
    return time_err

def summing_error(df):
    macro_sum = df['EW_p'] + df['Fett_p'] + df['KH_p']
    macro_sum_err = macro_sum - 100
    return macro_sum_err

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Plot pearson correlations for both real and synthetic data')
    # positional arguments:
    parser.add_argument('output_dir', help='path to folder where the output files will be saved')
    parser.add_argument('syn_data_vambn', help='path to synthetic vambn data in csv format, one row per participant')
    parser.add_argument('syn_data_vambn_ft',
                        help='path to synthetic vambn_ft data in csv format, one row per participant')
    parser.add_argument('syn_data_vambn_mt',
                        help='path to synthetic vambn_mt data in csv format, one row per participant')

    args = parser.parse_args()
    # check whether output folder exists, if not create
    if not os.path.exists(args.output_dir):
        os.mkdir(args.output_dir)

    df_vambn = pd.read_csv(args.original_data_path)
    df_vambn_ft = pd.read_csv(args.syn_data_vambn_ft)
    df_vambn_mt = pd.read_csv(args.syn_data_vambn_mt)

    df_vambn_error = compute_error(df_vambn)
    df_vambn_ft_error = compute_error(df_vambn_ft)
    df_vambn_mt_error = compute_error(df_vambn_mt)

    for error, name in zip([df_vambn_error, df_vambn_ft_error, df_vambn_mt_error],
                           ["VAMBN", "VAMBN-FT", "VAMBN-MT"]):
        print(f'mean/median absolute time-error {name}:', error.abs().mean(), '/', error.abs().median())

    use_logscale = False
    logticks = [-50, -20, -10, -5, -2, -1, 0, 1, 2, 5, 10, 20, 50]

    # plot line
    fig, ax = plt.subplots()
    ax = sns.kdeplot(data=df_vambn_error, label='VAMBN', ax=ax)
    ax = sns.kdeplot(data=df_vambn_ft_error, label='VAMBN-FT', ax=ax)
    ax = sns.kdeplot(data=df_vambn_mt_error, label='VAMBN-MT', ax=ax)
    ax.set_xlabel('Δ$\it{time}$ - Δ$\it{alter}$, in years')
    ax.set_ylabel('Density')
    if use_logscale:
        ax.set_xscale('symlog')
        ax.set_xticks(logticks)
        ax.get_xaxis().set_major_formatter(ticker.ScalarFormatter())
    plt.legend()
    plt.xlim(-75, 75)
    plt.savefig(f'{args.output_dir}/time-error.pdf', bbox_inches='tight')

    # 100% SUMMING ERROR
    df_vambn_error_sum = compute_error(df_vambn)
    df_vambn_ft_error_sum = compute_error(df_vambn_ft)
    df_vambn_mt_error_sum = compute_error(df_vambn_mt)

    for error, name in zip([df_vambn_error_sum, df_vambn_ft_error_sum,
                            df_vambn_mt_error_sum],
                           ["VAMBN", "VAMBN-FT", "VAMBN-MT"]):
        print(f'mean/median absolute sum-error {name}:', error.abs().mean(), '/', error.abs().median())

    use_logscale = False
    # plot line
    fig, ax = plt.subplots()
    ax = sns.kdeplot(data=df_vambn_error_sum, label='VAMBN', ax=ax)
    ax = sns.kdeplot(data=df_vambn_ft_error_sum, label='VAMBN-FT', ax=ax)
    ax = sns.kdeplot(data=df_vambn_mt_error_sum, label='VAMBN-MT', ax=ax)

    ax.set_xlabel('(EW_p+Fett_p+KH_p) - 100')
    ax.set_ylabel('Density')
    # ax.set_title('Capability of summing to 100%')
    if use_logscale:
        ax.set_xscale('symlog')
        ax.set_xticks(logticks)
        ax.get_xaxis().set_major_formatter(ticker.ScalarFormatter())
    plt.legend()
    plt.savefig(f'{args.output_dir}/100-error.pdf', bbox_inches='tight')
    plt.show()

print('fin')


import cas.common.utilities as utilities
from cas.common.steamtools import SteamInstance
from cas.common.config import ConfigurationUtilities, LazyDynamicBase
from cas.common.cache import CacheManager

from pathlib import Path
from typing import Any, List, Union
import os
import logging
import subprocess

class BuildEnvironment:
    """
    Contains attributes about the current build environment
    """

    def __init__(self, path: str, config: dict):
        self.config = ConfigurationUtilities.parse_root_config(path, config)

        self.cache = CacheManager(path)
        self.cache.load()

        self.build_type = self.config.args.build_type
        self.build_categories = None

        categories = self.config.args.build_categories
        if categories:
            categories = categories.split(",")
            self.build_categories = frozenset(categories)

        self.verbose = self.config.args.verbose

        self.root = self.config.path.root
        self.content = self.config.path.content
        self.game = self.config.path.game
        self.src = self.config.path.src

        self.platform = utilities.resolve_platform_name()
        self.steam = None

        self._setup_bindir()

    def _get_appid_folder(self, appid: int) -> Path:
        if not self.steam:
            self.steam = SteamInstance()
        for app in self.steam.apps:
            if app.appid == appid:
                return app.path
        return None

    def _check_autodetect_appid(self) -> bool:
        if self.src.exists():
            logging.debug("appid autodetect skipped - src dir exists")
            return False
        elif self.get_tool("chaos").exists():
            logging.debug("appid autodetect skipped - chaos executable exists")
            return False

        return True

    def _setup_bindir(self):
        self.bindir = self.game.joinpath("bin", self.platform)
        override = self.config.options.get("bin_path")

        # we have a bin path override
        if override is not None:
            self.bindir = Path(override).joinpath(self.platform)
        # we're a mod (modwrapper present but no game executable) - autodetect bin path from appid
        elif self._check_autodetect_appid():
            appid = self.config.options.get("bin_appid")
            if appid is not None:
                logging.debug(f"mod detected - using base appid {str(appid)}")
                self.bindir = self._get_appid_folder(appid).joinpath(
                    "bin", self.platform
                )

        if not self.bindir.exists():
            raise Exception("Could not find the bin directory")

        logging.debug(f"using bin directory {self.bindir}")

    """
    Retrieves the absolute path to the tool at the specified source path.
    If the source path is None, it will default to self.bindir.
    """

    def get_tool(self, tool: str, src: Path = None) -> Path:
        if src is None:
            src = self.bindir
        assert not tool.endswith(".exe")
        if utilities.is_platform_windows():
            tool += ".exe"
        return src.joinpath(tool).resolve()

    def get_lib(self, lib: str) -> Path:
        if utilities.is_platform_windows():
            lib += ".dll"
        elif utilities.is_platform_osx():
            lib += ".dylib"
        elif utilities.is_platform_linux():
            lib += ".so"
        else:
            raise NotImplementedError()
        return self.bindir.joinpath(lib).resolve()

    def run_subprocess(self, *args, **kwargs):
        predef = {}
        if not self.verbose:
            predef["stdout"] = subprocess.DEVNULL
            predef["stderr"] = subprocess.DEVNULL
        return subprocess.run(*args, **dict(predef, **kwargs))

    def run_tool(
        self, args: List[str], source: bool = False, cwd: Union[str, Path] = None
    ) -> int:
        """
        High-level interface to run an executable with extra parameters
        """
        predef = {}
        predef["env"] = os.environ

        if source:
            predef["env"]["VPROJECT"] = str(self.config.path.vproject)
            predef["env"]["NOASSERT"] = "1"
        if cwd:
            predef["cwd"] = cwd

        try:
            result = self.run_subprocess(args, **predef)
        except Exception as e:
            raise Exception(f"failed to execute tool with parameters: {args}") from e
        return result.returncode

class BuildResult:
    """
    Represents a result returned from a subsystem
    """

    def __init__(self, success: bool, outputs: dict = {}):
        self.success = success
        self.outputs = outputs

class BuildSubsystem:
    """
    Represents a build system that implements custom behaviour.
    """

    def __init__(self, env: BuildEnvironment, config: dict):
        mod = self.__class__.__module__

        self.env = env
        self.config = config
        self._cache = env.cache["subsystems"][mod]
        self._logger = logging.getLogger(mod)

    def _get_config_raw(self) -> Any:
        # If we have a lazy config object then we need to extract its underlying raw data
        if isinstance(self.config, LazyDynamicBase):
            return self.config._data
        else:
            return self.config

    def needs_rebuild(self) -> bool:
        """
        Checks whether this subsystem needs to force a rebuild,
        based on whether its configuration hash has changed from the last successful build
        """
        old_hash = self._cache.get("config", None)
        new_hash = utilities.hash_object_sha256(self._get_config_raw())
        self._cache["config"] = new_hash

        if not old_hash:
            return False
        return old_hash != new_hash

    def rehash_config(self):
        """
        Rehashes the configuration and saves it in our cache
        """
        self._cache["config"] = utilities.hash_object_sha256(self._get_config_raw())

    def build(self, force: bool = False) -> BuildResult:
        """
        Invokes the build logic of the subsystem.
        Returns a BuildResult with the result.
        """
        raise NotImplementedError()

    def clean(self) -> bool:
        """
        Removes all the output files generated by this subsystem.
        """
        raise NotImplementedError()


#!/usr/bin/python3
"""This is automated testing for this api"""
import requests

url = 'https://hngmus2.onrender.com/api/'

#create person
person_information = {
    'name': 'tester1'
}
person = requests.post(url + 'peoples', json=person_information)
print(person.status_code)
print(f"person {person.json().get('id')} created sucessfully: {person.json()}")
person_information = {
    'name': 'tester2'
}
person = requests.post(url + 'peoples', json=person_information)
print(person.status_code)
print(f"person {person.json().get('id')} created sucessfully: {person.json()}")
person_information = {
    'name': 'tester3'
}
person = requests.post(url + 'peoples', json=person_information)
print(person.status_code)
user_id = person.json().get('id')
print(f"person {person.json().get('id')} created sucessfully: {person.json()}")

#get all person
person = requests.get(url + 'peoples')
print(f"All people in the database are {person.json()}")

#get a person
person = requests.get(url + 'people/2')
print(f"get information about person with {person.json().get('id')} is {person.json()}")

#update a person
person = requests.put(url + 'people/1', json={'name': '1-tester'})
print(f"The updated information about person with {person.json().get('id')} is {person.json()}")

#search a person
person = requests.get(url + 'people_search', params={'name': 'tester3'})
print(f"search a person with name  tester3 amd they are {person.json()}")

#delete a person
person = requests.delete(url + f'people/{user_id}')
print(f"Deleted the perdon with id {person.json().get('id')} successfully")

#gethjjjjjjhg
stats = requests.get(url + 'stats')
print(stats.status_code)
print(f"The database stats is {stats.json()}")


import keras
import pandas
import tensorflow as tf

from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.utils.np_utils import to_categorical

from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn.externals import joblib

# use CPU only
# configCPU = tf.ConfigProto(device_count={'CPU': 1, 'GPU': 0})
# sess = tf.Session(config=configCPU)

sess = tf.Session()  # use GPU

keras.backend.set_session(sess)

#--------------Preprocessing-----------------#

data = pandas.read_csv('You_Data_Set', header=None)

feature = data.drop([48], 1)
label = data[48]

scaler = preprocessing.StandardScaler().fit(feature)
joblib.dump(scaler, 'scalers/scaler.pkl')
# clf = joblib.load('filename.pkl')
# feature = scaler.transform(feature)
feature = feature.values

# --------------Train----------------- #

train_x, test_x, train_y, test_y = train_test_split(feature, label, test_size=0.2)

train_y = to_categorical(train_y, 2)
test_y = to_categorical(test_y, 2)

validation_x = train_x[:2000]
train_x = train_x[2000:]
validation_y = train_y[:2000]
train_y = train_y[2000:]

model = Sequential()
model.add(Dense(128, activation='relu', input_dim=48))
model.add(Dense(256, activation='relu'))
# model.add(Dropout(0.25))
model.add(Dense(128, activation='relu'))
# model.add(Dropout(0.25))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.25))
# model.add(Dense(40, activation='relu'))
model.add(Dense(32, activation='relu'))
model.add(Dropout(0.25))
# model.add(Lambda(lambda x: x / 10))
model.add(Dense(2, activation='softmax'))

opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6)
# model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) mean_squared_error
model.compile(optimizer='SGD', loss='categorical_crossentropy', metrics=['accuracy'])

model.fit(train_x, train_y,
                epochs=100,
                batch_size=1024,
                shuffle=True,
                validation_data=(validation_x, validation_y))

results = model.evaluate(test_x, test_y)
print("\n")
print(results)

model.save('models/FNN_Name.h5')


__version__ = '1.6.0'
import os

from setuptools import setup, find_packages

# README read-in
from os import path
this_directory = path.abspath(path.dirname(__file__))
with open(path.join(this_directory, 'README.rst'), encoding='utf-8') as f:
    long_description = f.read()
# END README read-in

setup(
    name='skytemple-eventserver',
    version=__version__,
    packages=find_packages(),
    description='Websocket server that emits SkyTemple UI events',
    long_description=long_description,
    long_description_content_type='text/x-rst',
    url='https://github.com/SkyTemple/skytemple-eventserver/',
    install_requires=[],
    classifiers=[
        'Development Status :: 4 - Beta',
        'Programming Language :: Python',
        'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',
        'Programming Language :: Python :: 3.8',
        'Programming Language :: Python :: 3.9',
        'Programming Language :: Python :: 3.10',
        'Programming Language :: Python :: 3.11',
    ],
    entry_points='''
        [skytemple.module]
        eventserver=skytemple_eventserver.module:EventserverModule
    '''
)


import numpy as np
from PyQt5.QtCore import pyqtSlot, QObject

class NPYHandler(QObject):

    def __init__(self):
        QObject.__init__(self)
        self.filePath = ''
        self.npyFile = None
        self.shape = []
        self.shapeString = ''
        self.length = 0

    @pyqtSlot(str,result=bool)
    def readNpyFile(self,filePath):
        self.filePath = filePath
        try:
            self.npyFile = np.load(self.filePath)
            self.shape = self.npyFile.shape
            self.updateShapeLength()
            self.updateShapeString()
            return True
        except BaseException as e:
            print('Error while reading .npy file: ' + str(e))
            return False

    def updateShapeString(self):
        #print(self.shape)
        if self.length == 0:
            return ''
        self.shapeString = str(self.shape[0])
        for i in range(self.length-1):
            self.shapeString += ' * ' + str(self.shape[i+1])
        #print(self.shapeString)

    def updateShapeLength(self):
        self.length = len(self.shape)

    @pyqtSlot(result=str)
    def getShapeSize(self):
        return self.shapeString

    @pyqtSlot(result=str)
    def getShapeLength(self):
        return str(self.length)

    @pyqtSlot(str,str,str,str,str,str,str,str,str)
    def draw(self,x,minX,maxX,y,minY,maxY,t,mint,maxt):
        print(x,minX,maxX,y,minY,maxY,t,mint,maxt)

npyHandler = NPYHandler()


# -*- coding: utf-8 -*-
import socket
import json
from MessageReceiver import MessageReceiver
from MessageParser import MessageParser

class Client:
    """
    This is the chat client class
    """
    def __init__(self, host, server_port):
        """
        This method is run when creating a new Client object
        """

        # Set up the socket connection to the server
        self.connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.server_port = server_port
        self.host = host
        self.run()

    def run(self):
        # Initiate the connection to the server
        self.connection.connect((self.host, self.server_port))

        MessageReceiver(self, self.connection)

        login_username = input("Login username: ")
        self.login(login_username)

        while True:
            message = input("> ")
            msg = message.split()
            self.create_request(msg[0], ' '.join(msg[1:]))

    def disconnect(self):
        self.connection.close()
        exit(0)

    def receive_message(self, message):
        message = message.decode()
        # print("<-- " + message)
        parser = MessageParser()
        result = parser.parse(message)
        if result is False:
            self.disconnect()

    def send_payload(self, data):
        data = json.dumps(data)

        self.connection.send(data.encode())
        # print("--> " + data)

        # More methods may be needed!

    def help(self):
        self.create_request('help')

    def login(self, username):
        self.create_request('login', username)
        self.username = username

    def logout(self):
        self.create_request('logout', self.username)

    def send_message(self, message):
        self.create_request('msg', message)

    def create_request(self, request, content=None):
        if not content:
            content = ''
        payload = {
            'request': request,
            'content': content,
        }
        self.send_payload(payload)

if __name__ == '__main__':
    """
    This is the main method and is executed when you type "python Client.py"
    in your terminal.

    No alterations are necessary
    """
    client = Client('localhost', 9998)


from aiogram import Bot, types, Dispatcher, executor

TOKEN = '1686424554:AAEUySebIA2fuA94jzhb8SQFfQbBSflvJp0'

bot = Bot(token=TOKEN)
dp = Dispatcher(bot)

@dp.message_handler(commands=['start'])
async def welcome(message: types.message):
    await bot.send_message(message.chat.id, 'Бот 1 Запущен')

if __name__ == "__main__":
    executor.start_polling(dp, skip_updates=True)