from cgitb import text
from re import I
from handlers.handler import Handler
from config.settings import KEYBOARD
from config.messages import MsgTemplates
from config.settings import ADMIN_ID
from telebot.types import CallbackQuery

class HandlerInlineQuery(Handler):

    def __init__(self, bot):
        super().__init__(bot)

    def send_new_student_for_admin(self, user):
        """
        send notification about new student for admin
        """
        student_lesson = self.BD.get_guest_lesson_by_user_id(user.id)
        lesson_type = self.BD.get_lesson_type_by_id(
            student_lesson.lessons_type_id)
        self.bot.send_message(ADMIN_ID,
                              f'{user.first_name} {user.last_name}, '
                              f'хочет прийти на пробное занятие по {lesson_type}. '
                              f'Вы можете связаться с ним по телефону {user.phone}',
                              reply_markup=self.keybords.guest_start_menu())

    def record_on_test_lesson(self, call: CallbackQuery) -> CallbackQuery:
        """
        write user on test lesson
        """
        guest = self.BD.get_user_by_user_id(call.from_user.id)
        last_msg = call.message
        self.BD._add_new_lesson(guest.id, call.data, True)
        self.send_new_student_for_admin(guest)
        self.bot.answer_callback_query(
            call.id, 'Вы записаны - ждите', show_alert=True)
        return last_msg

    def del_last_bot_message(self, message) -> None:
        """
        delete inline btn
        """
        self.bot.edit_message_text(
            chat_id=message.chat.id, text=message.text, message_id=message.id, reply_markup='')

    def more_about_lesson(self, call: CallbackQuery) -> None:
        """
        show extendent information about lessons
        """
        self.bot.answer_callback_query(call.id)

        user = self.BD.get_user_by_user_id(call.from_user.id)
        lesson = self.BD.get_guest_lesson_by_user_id(user.id)

        if call.data == 'Математика':
            if lesson:
                self.bot.send_message(
                    call.from_user.id, MsgTemplates.ABOUT_MATH_MSG)
            else:
                self.bot.send_message(call.from_user.id, MsgTemplates.ABOUT_MATH_MSG,
                                      reply_markup=self.keybords.record_on_lesson_menu(1))

        if call.data == 'Английский язык':
            if lesson:
                self.bot.send_message(
                    call.from_user.id, MsgTemplates.ABOUT_ENG_MSG)
            else:
                self.bot.send_message(call.from_user.id, MsgTemplates.ABOUT_ENG_MSG,
                                      reply_markup=self.keybords.record_on_lesson_menu(2))

        if call.data == 'Обществознание':
            if lesson:
                self.bot.send_message(
                    call.from_user.id, MsgTemplates.ABOUT_SOCIAL_MSG)
            else:
                self.bot.send_message(call.from_user.id, MsgTemplates.ABOUT_SOCIAL_MSG,
                                      reply_markup=self.keybords.record_on_lesson_menu(3))

    def handle(self):
        @self.bot.callback_query_handler(func=lambda call: True)
        def callback_inline(call: CallbackQuery):
            data = call.data
            print(type(call))
            if data.isdigit():
                last_msg = self.record_on_test_lesson(call)
                self.del_last_bot_message(last_msg)

            else:
                self.more_about_lesson(call)


from direct.directnotify import DirectNotifyGlobal
from direct.distributed import DistributedObject
from direct.interval.IntervalGlobal import *
from toontown.effects import DustCloud

def getDustCloudIval(toon):
    dustCloud = DustCloud.DustCloud(fBillboard=0)
    dustCloud.setBillboardAxis(2.0)
    dustCloud.setZ(3)
    dustCloud.setScale(0.4)
    dustCloud.createTrack()

    if hasattr(toon, 'laffMeter'):
        toon.laffMeter.color = toon.style.getBlackColor()

    sequence = Sequence(Wait(0.5), Func(dustCloud.reparentTo, toon), dustCloud.track, Func(dustCloud.destroy))

    if hasattr(toon, 'laffMeter'):
        sequence.append(Func(toon.laffMeter.adjustFace, toon.hp, toon.maxHp))

    return sequence

class DistributedBlackCatMgr(DistributedObject.DistributedObject):
    neverDisable = 1
    notify = DirectNotifyGlobal.directNotify.newCategory('DistributedBlackCatMgr')

    def announceGenerate(self):
        DistributedObject.DistributedObject.announceGenerate(self)
        base.cr.blackCatMgr = self

    def delete(self):
        base.cr.blackCatMgr = None
        DistributedObject.DistributedObject.delete(self)

    def requestBlackCatTransformation(self):
        self.sendUpdate('requestBlackCatTransformation')

    def doBlackCatTransformation(self):
        getDustCloudIval(base.localAvatar).start()


import numpy as np
import matplotlib.pyplot as plt
from matplotlib import cm

nx = 81
ny = 81
nt = 200
c = 1
dx = 2 / (nx - 1)
dy = 2 / (ny - 1)
sigma = .2
dt = sigma * dx

x = np.linspace(0, 2, nx)
y = np.linspace(0, 2, ny)

u = np.ones((nx, ny))
un = np.ones((nx, ny))

# initial conditions
u[int(.5 / dy):int(1 / dy + 1), int(.5 / dx):int(1 / dx + 1)] = 2

for n in range(nt + 1):
    un = u.copy()
    row, col = u.shape

    u[1:, 1:] = (un[1:, 1:] - (c * dt / dx * (un[1:, 1:] - un[1:, :-1])) - (c * dt / dy * (un[1:, 1:] - un[:-1, 1:])))

    u[0, :] = 1
    u[-1, :] = 1
    u[:, 0] = 1
    u[:, -1] = 1

fig, ax = plt.subplots(subplot_kw={"projection": "3d"})

#fig = plt.figure()
#ax = fig.gca(projection='3d')

X, Y = np.meshgrid(x, y)

surf = ax.plot_surface(X, Y, u[:], cmap=cm.viridis)

plt.show()


import tkinter as tk
from PIL import Image
from PIL import ImageTk

def callback():
    var.set('吹吧你')

root = tk.Tk()

frame1 = tk.Frame(root)
frame2 = tk.Frame(root)

var = tk.StringVar()
var.set('未满18岁禁止点击')

textLabel = tk.Label(frame1, textvariable=var, justify=tk.LEFT)
textLabel.pack(side=tk.LEFT)

img = Image.open('/Users/yangkaiqiang/Desktop/images/cat.jpg')
photo = ImageTk.PhotoImage(img)

imageLabel = tk.Label(frame1, image=photo)
imageLabel.pack(side=tk.RIGHT)

theButton = tk.Button(frame2, text='我已满18岁', command=callback)
theButton.pack()

frame1.pack(padx=10, pady=10)
frame2.pack(padx=10, pady=10)

tk.mainloop()


from typing import Callable
from functools import partial
import solara as sl
import pandas as pd
from solara.components.file_drop import FileInfo
import io

def load_file_df(file: FileInfo, set_df: Callable):
    if not file["data"]:
        return
    new_df = pd.read_csv(io.BytesIO(file["data"]))
    # Set it before embeddings so the user can see the df while embeddings load
    set_df(new_df)

@sl.component
def df_table(df: pd.DataFrame, num_samples: int, search: str) -> None:
    if not len(df):
        return
    filtered = df.copy()
    if "text" in df.columns:
        filtered = df[df["text"].str.contains(search)]
    sl.DataFrame(filtered[:num_samples])

@sl.component
def file_manager(set_df: Callable) -> None:
    sl.FileDrop(
        label="Drop CSV here (`text` col required)!",
        on_file=partial(load_file_df, set_df=set_df),
        lazy=False,
    )

@sl.component
def menu():
    df, set_df = sl.use_state(pd.DataFrame({}))
    num_samples, set_samples = sl.use_state(10)
    search, set_search = sl.use_state("")

    with sl.Sidebar():
        sl.InputText("What would you like to see?", search, on_value=set_search)
        sl.SliderInt("How many samples?", num_samples, max=100, on_value=set_samples)
        file_manager(set_df)
    df_table(df, num_samples, search)

@sl.component
def Page():
    menu()


import scrapy

from myprojct.items import ExampleItem

class ExampleSpider(scrapy.Spider):
    name = 'example'
    allowed_domains = ["example.com"]

    start_urls = [
        'http://www.example.com',
    ]

    # 先登录
    def start_requests(self):
        # Cookies数据
        cookies = {'uid': '"1083428ut78j8"', 'v': '30'}
        # 头信息
        headers = {
            'Connection': 'keep-alive',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36'
        }

        return [scrapy.FormRequest("http://www.example.com/articles",
                                   # 使用Cookies
                                   cookies=cookies,
                                   # 指定头信息
                                   headers=headers,
                                   # 指定回调函数
                                   callback=self.parse_page)]

    # 解析页面
    def parse_page(self, response):
        item = ExampleItem()
        item["name"] = response.css(".name").extract_first()
        yield item


import argparse
import os
from pickle import NONE
import numpy as np
from tqdm import tqdm
from torch.utils.tensorboard import SummaryWriter

from mydataloader import make_data_loader
from modeling.sync_batchnorm.replicate import patch_replication_callback
from modeling.deeplab import *
from utils.loss import SegmentationLosses
from utils.calculate_weights import calculate_weigths_labels
from utils.lr_scheduler import LR_Scheduler
from utils.saver import Saver
from utils.metrics import Evaluator

class Trainer(object):
    def __init__(self, args):

        self.args = args

        # 实例化训练信息保存的相关类
        self.saver = Saver(args)
        self.saver.save_experiment_config()
        # 创建Tensorboard Summary
        self.writer = SummaryWriter(log_dir=os.path.join(self.saver.experiment_dir))
        # 创建Dataloader
        self.train_loader, self.test_loader, self.nclass = make_data_loader(args)

        # 定义网络模型
        model = DeepLab(num_classes=self.nclass,
                        backbone=args.backbone,
                        output_stride=args.out_stride,
                        sync_bn=False,
                        freeze_bn=args.freeze_bn)
        # 设置是否固定BN层
        train_params = [{'params': model.get_1x_lr_params(), 'lr': args.lr},
                        {'params': model.get_10x_lr_params(), 'lr': args.lr * 10}]

        # Define Optimizer
        optimizer = torch.optim.SGD(train_params, momentum=args.momentum,
                                    weight_decay=args.weight_decay, nesterov=args.nesterov)

        # 是否使用class balanced weights，以平衡各类像素数量的不均衡
        if args.use_balanced_weights:
            classes_weights_path = os.path.join('./weizmann_horse_db','classes_weights.npy')
            # classes_weights_path = 'D:/code/数据集/MINIST_EXTENDED/3840_0_3/minist_classes_weights_change.npy'
            if os.path.isfile(classes_weights_path):
                weight = np.load(classes_weights_path)
                print('blanced weights:',weight)
            else:
                weight = calculate_weigths_labels(args.dataset, self.train_loader, self.nclass)
            weight = torch.from_numpy(weight.astype(np.float32))
        else:
            weight = None
        # 构建损失函数
        self.criterion = SegmentationLosses(weight=weight, cuda=args.cuda).build_loss(mode=args.loss_type)
        self.model, self.optimizer = model, optimizer

        # 创建计算指标所需的对象
        self.evaluator = Evaluator(self.nclass)
        # 定义 lr scheduler
        self.scheduler = LR_Scheduler(args.lr_scheduler, args.lr,
                                            args.epochs, len(self.train_loader))

        # 使用cuda训练
        if args.cuda:
            self.model = torch.nn.DataParallel(self.model, device_ids=self.args.gpu_ids)
            patch_replication_callback(self.model)
            self.model = self.model.cuda()

        # 加载之前训练好的模型，继续完成目标epoch数的训练
        self.best_pred = 0.0
        if args.resume is not None:
            if not os.path.isfile(args.resume):
                raise RuntimeError("=> no checkpoint found at '{}'" .format(args.resume))
            checkpoint = torch.load(args.resume)
            args.start_epoch = checkpoint['epoch']
            if args.cuda:
                self.model.module.load_state_dict(checkpoint['state_dict'])
            else:
                self.model.load_state_dict(checkpoint['state_dict'])
            if not args.ft:
                self.optimizer.load_state_dict(checkpoint['optimizer'])
            self.best_pred = checkpoint['best_pred']
            print("=> loaded checkpoint '{}' (epoch {})"
                  .format(args.resume, checkpoint['epoch']))

        # 若是fine-tuning则epoch次数从0开始
        if args.ft:
            args.start_epoch = 0

    def training(self, epoch):
        train_loss = 0.0
        self.model.train()
        tbar = tqdm(self.train_loader)
        num_img_tr = len(self.train_loader)    #数据集大小/BATCH_SIZE
        for i, (image,target) in enumerate(tbar):
            if self.args.cuda:
                image, target = image.cuda(), target.cuda()
            self.scheduler(self.optimizer, i, epoch, self.best_pred)
            self.optimizer.zero_grad()
            output = self.model(image)
            loss = self.criterion(output, target)
            loss.backward()
            self.optimizer.step()
            train_loss += loss.item()
            tbar.set_description('Train loss: %.3f' % (train_loss / (i + 1)))
            self.writer.add_scalar('train/total_loss_iter', loss.item(), i + num_img_tr * epoch)

            if i % (num_img_tr // 10) == 0:     # //是整数除法，向下取整
                global_step = i + num_img_tr * epoch
                print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, i * len(image), len(self.train_loader.dataset),
                100. * i / len(self.train_loader), loss.item()))

        self.writer.add_scalar('train/total_loss_epoch', train_loss, epoch)
        print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size + image.data.shape[0]))
        print('Loss: %.3f' % train_loss)

    def validation(self, epoch):
        self.model.eval()
        self.evaluator.reset()
        tbar = tqdm(self.test_loader, desc='\r')
        test_loss = 0.0

        for i, (image,target) in enumerate(tbar):
            if self.args.cuda:
                image, target = image.cuda(), target.cuda()
            with torch.no_grad():
                output = self.model(image)
            loss = self.criterion(output, target)
            test_loss += loss.item()
            tbar.set_description('Test loss: %.3f' % (test_loss / (i + 1)))
            pred = output.data.cpu().numpy()
            target = target.cpu().numpy()
            pred = np.argmax(pred, axis=1)
            # 利用混淆矩阵得到每个像素的分类情况
            self.evaluator.add_batch(target, pred)

        # 根据混淆矩阵得到各项指标
        Acc = self.evaluator.Pixel_Accuracy()
        mIoU = self.evaluator.Mean_Intersection_over_Union()
        FWIoU = self.evaluator.Frequency_Weighted_Intersection_over_Union()
        BIoU = self.evaluator.biou()
        self.writer.add_scalar('val/total_loss_epoch', test_loss, epoch)
        self.writer.add_scalar('val/mIoU', mIoU, epoch)
        self.writer.add_scalar('val/Acc', Acc, epoch)
        self.writer.add_scalar('val/fwIoU', FWIoU, epoch)

        print('Validation:')
        print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size + image.data.shape[0]))
        print("Acc:{}, mIoU:{}, BoundaryIoU: {}".format(Acc, mIoU, BIoU))
        print('Loss: %.3f' % test_loss)

        new_pred = mIoU
        if new_pred > self.best_pred:
            is_best = True
            self.best_pred = new_pred
            self.saver.save_checkpoint({
                'epoch': epoch + 1,
                'state_dict': self.model.module.state_dict(),
                'optimizer': self.optimizer.state_dict(),
                'best_pred': self.best_pred,
            }, is_best)
    def loadmodel(self,backbone='resnet'):
        PATH='checkpoint.pth.tar'
        checkpoint = torch.load(PATH)
        # self.args.start_epoch = checkpoint['epoch']
        if self.args.cuda:
            self.model.module.load_state_dict(checkpoint['state_dict'])
        else:
            self.model.load_state_dict(checkpoint['state_dict'])

def main():
    parser = argparse.ArgumentParser(description="PyTorch DeeplabV3Plus Training")
    parser.add_argument('--backbone', type=str, default='resnet',
                        choices=['resnet', 'xception'],
                        help='backbone name (default: resnet)')
    parser.add_argument('--out-stride', type=int, default=16,
                        help='network output stride (default: 16)')
    parser.add_argument('--dataset', type=str, default='kaggle',
                        choices=['kaggle', 'minist'],
                        help='dataset name (default: kaggle)')
    parser.add_argument('--use-sbd', action='store_true', default=True,
                        help='whether to use SBD dataset (default: True)')
    parser.add_argument('--base-size', type=int, default=513,
                        help='base image size')
    parser.add_argument('--crop-size', type=int, default=513,
                        help='crop image size')
    parser.add_argument('--freeze-bn', type=bool, default=False,
                        help='whether to freeze bn parameters (default: False)')
    parser.add_argument('--loss-type', type=str, default='ce',
                        choices=['ce', 'focal','ce_nolastch'],
                        help='loss func type (default: CrossEntropyLoss)')
    # 训练的超参数
    parser.add_argument('--epochs', type=int, default=None, metavar='N',
                        help='number of epochs to train (default: auto)')
    parser.add_argument('--start_epoch', type=int, default=0,
                        metavar='N', help='start epochs (default:0)')
    parser.add_argument('--batch-size', type=int, default=None,
                        metavar='N', help='input batch size for \
                                training (default: auto)')
    parser.add_argument('--test-batch-size', type=int, default=None,
                        metavar='N', help='input batch size for \
                                testing (default: auto)')
    parser.add_argument('--use-balanced-weights', action='store_true', default=True,  #False,
                        help='whether to use balanced weights (default: False)')
    # 优化器参数选择
    parser.add_argument('--lr', type=float, default=None, metavar='LR',
                        help='learning rate (default: auto)')
    parser.add_argument('--lr-scheduler', type=str, default='poly',
                        choices=['poly', 'step', 'cos'],
                        help='lr scheduler mode: (default: poly)')
    parser.add_argument('--momentum', type=float, default=0.9,
                        metavar='M', help='momentum (default: 0.9)')
    parser.add_argument('--weight-decay', type=float, default=5e-4,
                        metavar='M', help='w-decay (default: 5e-4)')
    parser.add_argument('--nesterov', action='store_true', default=False,
                        help='whether use nesterov (default: False)')
    # cuda and logging
    parser.add_argument('--no-cuda', action='store_true', default=
                        False, help='disables CUDA training')
    parser.add_argument('--gpu-ids', type=str, default='0',
                        help='use which gpu to train (default=0)')
    # 添加检查点(checkpoint)用于保存模型的参数
    parser.add_argument('--resume', type=str, default=None,
                        help='put the path to resuming file if needed')
    parser.add_argument('--checkname', type=str, default=None,
                        help='set the checkpoint name')
    # 微调预训练的模型，面向类似任务进行模型训练
    parser.add_argument('--ft', action='store_true', default=False,
                        help='finetuning on a different dataset')
    # eval-interval：评价间隔，默认一次1次epoch在测试集跑1次
    parser.add_argument('--eval-interval', type=int, default=1,
                        help='evaluation interval (default: 1)')
    parser.add_argument('--no-val', action='store_true', default=False,
                        help='skip validation during training')

    args = parser.parse_args()
    args.cuda = not args.no_cuda and torch.cuda.is_available()
    if args.cuda:
        try:
            #gpu_ids=[0],只有1块GPU
            args.gpu_ids = [int(s) for s in args.gpu_ids.split(',')]
        except ValueError:
            raise ValueError('Argument --gpu_ids must be a comma-separated list of integers only')

    args.backbone = 'resnet'  #'xception'

    ''' 加载之前训练的模型继续训练 '''
    # args.resume='../run/minist/deeplab-resnet/model_best.pth.tar'
    # default settings for epochs, batch_size and lr

    args.epochs = 90 # 50

    args.batch_size = 15

    args.test_batch_size = args.batch_size

    lrs = 0.05 #   0.07  #0.08 #初始学习率
    args.lr = lrs / (4 * len(args.gpu_ids)) * args.batch_size

    if args.checkname is None:
        args.checkname = 'deeplab-'+str(args.backbone)
    # print(args)

    trainer = Trainer(args)
    print('Starting Epoch:', trainer.args.start_epoch)
    print('Total Epoches:', trainer.args.epochs)

    # 在验证集上测试指标
    # trainer.loadmodel()
    # trainer.validation(0)

    # 训练模型
    for epoch in range(trainer.args.start_epoch, trainer.args.epochs):
        trainer.training(epoch)
        if not trainer.args.no_val and epoch % args.eval_interval == (args.eval_interval - 1):
            trainer.validation(epoch)

    trainer.writer.close()

if __name__ == "__main__":

    main()


import pandas as pd
import numpy as np
import pybedtools
import csv
import gzip

##deprecated by restrict_mutations_by_bed_efficient
def restrict_mutations_by_bed(df_mut, df_bed, unique=True, remove_X=True, replace_cols=False):
    """ Restrict mutations to only those that overlap elements in a bed file.
    """
    # df_mut = pd.read_table(f_mut, header=None, low_memory=False)
    # df_bed = pd.read_table(f_bed, header=None, low_memory=False)
    if remove_X:
        df_mut = df_mut[df_mut.iloc[:, 0] != "X"]
        df_bed = df_bed[df_bed.iloc[:, 0] != "X"]

    bed_mut = pybedtools.BedTool.from_dataframe(df_mut)
    bed_bed = pybedtools.BedTool.from_dataframe(df_bed)

    bed_inter = bed_mut.intersect(bed_bed)
    df_inter = bed_inter.to_dataframe()

    if unique:
        df_inter = df_inter.drop_duplicates()

    if replace_cols:
        # df_inter.columns = ['CHROM', 'START', 'END', 'REF', 'ALT', 'SAMPLE', 'GENE', 'ANNOT', 'MUT', 'CONTEXT']
        df_inter.columns = df_mut.columns

    return df_inter

def restrict_mutations_by_bed_efficient(f_mut, f_bed, bed12=False, drop_duplicates=False, drop_sex=False, replace_cols=False):
    """ Restrict mutations to only those that overlap elements in a bed file.
    """
    bed_mut = pybedtools.BedTool(f_mut)
    bed_bed = pybedtools.BedTool(f_bed)
    if bed12:
        bed_bed = bed_bed.bed12tobed6()

    bed_inter = bed_mut.intersect(bed_bed, wa=True)
    df_mut = read_mutation_file(bed_inter.fn, drop_duplicates=drop_duplicates, drop_sex=drop_sex)

    return df_mut

def read_mutation_file(path, drop_sex=True, drop_duplicates=False, unique_indels=True):
    # cols = ['CHROM', 'START', 'END', 'REF', 'ALT', 'SAMPLE', 'ANNOT', 'MUT_TYPE', 'CONTEXT']
    try:
        with open(path) as f:
            reader = csv.reader(f, delimiter='\t', skipinitialspace=True)
            first_row = next(reader)
    except UnicodeDecodeError:  ## file is probably gzipped
        with gzip.open(path, 'rt') as f:
            reader = csv.reader(f, delimiter='\t', skipinitialspace=True)
            first_row = next(reader)

    num_cols = len(first_row)

    if num_cols == 5:
        cols = ['CHROM', 'POS', 'REF', 'ALT', 'SAMPLE']
        dtype = dict(CHROM=str, POS=int, REF=str, ALT=str, SAMPLE=str)
    elif num_cols == 6:
        cols = ['CHROM', 'START', 'END', 'REF', 'ALT', 'SAMPLE']
        dtype = dict(CHROM=str, START=int, END=int, REF=str, ALT=str, SAMPLE=str)
    elif num_cols == 7:
        cols = ['CHROM', 'START', 'END', 'REF', 'ALT', 'SAMPLE', 'ANNOT']
        dtype = dict(CHROM=str, START=int, END=int, REF=str, ALT=str, SAMPLE=str, ANNOT=str)
    elif num_cols == 8:
        cols = ['CHROM', 'START', 'END', 'REF', 'ALT', 'SAMPLE', 'GENE', 'ANNOT']
        dtype = dict(CHROM=str, START=int, END=int, REF=str, ALT=str, SAMPLE=str, GENE=str, ANNOT=str)
    elif num_cols == 9:
        cols = ['CHROM', 'START', 'END', 'REF', 'ALT', 'SAMPLE', 'ANNOT', 'MUT_TYPE', 'CONTEXT']
        dtype = dict(CHROM=str, START=int, END=int, REF=str, ALT=str, SAMPLE=str, ANNOT=str, MUT_TYPE=str, CONTEXT=str)
    elif num_cols == 10:
        cols = ['CHROM', 'START', 'END', 'REF', 'ALT', 'SAMPLE', 'GENE', 'ANNOT', 'MUT_TYPE', 'CONTEXT']
        dtype = dict(CHROM=str, START=int, END=int, REF=str, ALT=str, SAMPLE=str, GENE=str, ANNOT=str, MUT_TYPE=str, CONTEXT=str)
    elif num_cols == 11:
        cols = ['CHROM', 'START', 'END', 'REF', 'ALT', 'SAMPLE', 'GENE', 'ANNOT', 'MUT_TYPE', 'CONTEXT', 'STRAND']
        dtype = dict(CHROM=str, START=int, END=int, REF=str, ALT=str, SAMPLE=str, GENE=str, ANNOT=str, MUT_TYPE=str, CONTEXT=str, STRAND=str)

    df = pd.read_csv(path, sep="\t", low_memory=False, names=cols, dtype=dtype)

    # df.columns = cols

    # df.CHROM = df.CHROM.astype(str)

    if drop_sex:
        if set(df.CHROM.unique()) - set([str(i) for i in range(1, 23)]):
            print('Restricting to autosomes')
            df = df[df.CHROM.isin([str(i) for i in range(1, 23)])]

        df['CHROM'] = df.CHROM.astype(int)

    # else:
    #     dtype['CHROM'] = str

    # df = df.astype(dtype)

    if drop_duplicates:
        df = drop_duplicate_mutations(df)

    if unique_indels:
        df = get_unique_indels(df)

    return df

def drop_duplicate_mutations(df_mut):
    df_dedup = df_mut.drop_duplicates(['CHROM', 'START', 'END', 'REF', 'ALT', 'SAMPLE'])
    return df_dedup

def get_unique_indels(df_mut):
    df_indel = df_mut[df_mut.ANNOT == 'INDEL']
    df_snv = df_mut[df_mut.ANNOT != 'INDEL']

    df_indel = df_indel.drop_duplicates(subset=['CHROM', 'START', 'END', 'REF', 'ALT', 'GENE'])

    return pd.concat([df_snv, df_indel])

def tabulate_nonc_mutations_split(f_nonc_bed, f_mut):
    try:
        # df_nonc = pd.read_csv(f_nonc_bed, sep= '\t', header=None, low_memory = False,
        # names=None)
        df_nonc = pd.read_table(f_nonc_bed, names=['CHROM', 'START', 'END', "ELT", "SCORE", "STRAND", 'thickStart', 'thickEnd', 'rgb', 'blockCount', 'blockSizes', 'blockStarts'], low_memory=False)
        df_nonc.CHROM = df_nonc.CHROM.astype(str)
        df_nonc = df_nonc[df_nonc.CHROM.isin([str(c) for c in range(1, 23)])]
        df_nonc.CHROM = df_nonc.CHROM.astype(int)
    except:
        raise Exception("ERROR: failed to load {}. make sure the bed file is in the correct bed12 format".format(df_nonc))
    df_mut = read_mutation_file(f_mut, drop_duplicates=True)
    assert ('GENE' in df_mut.columns and 'ANNOT' in df_mut.columns and'MUT_TYPE' in df_mut.columns)
    # try:
    #     df_mut = pd.read_csv(f_mut,sep='\t', low_memory=False, index_col=False,
    #     names=['CHROM', 'START', 'END', 'REF', 'ALT', 'ID', 'GENE', 'ANNOT', 'MUT','CONTEXT'])
    # except:
    #     raise Exception("ERROR: failed to load {}. make sure mut file is properly processed".format(df_mut))

    if 'chr' in str(df_nonc.CHROM[0]):
        df_nonc.CHROM = df_nonc.CHROM.map(lambda x: x.lstrip('chr'))

    bed_mut = pybedtools.BedTool.from_dataframe(df_mut)
    bed_bed = pybedtools.BedTool.from_dataframe(df_nonc)
    bed_split = bed_bed.bed6()

    bed_inter = bed_split.intersect(bed_mut, wao=True)
    df_inter = bed_inter.to_dataframe(header=None, names=np.arange(bed_inter.field_count()))
    df_inter = df_inter.drop(columns = [4,6,7,8,9,10,12,13,14,15])
    # df_split = df_inter.groupby([0,1,2,3,5]).agg({11:lambda x:len(set(x).difference(set('.'))), 16:np.sum}).reset_index()
    # df_split.columns = ['CHROM', 'START', 'END', 'ELT', 'STRAND','OBS_SAMPLES','OBS_MUT']

    df_whole = pd.pivot_table(df_inter, values = [1,2,11,16], index = [0,3,5], aggfunc={16:np.sum, 1:lambda x: sorted(set(x)), 2:lambda x: sorted(set(x)),11:lambda x:len(set(x).difference(set('.')))}).reset_index()
    df_whole.columns = ['CHROM', 'ELT', 'STRAND','BLOCK_STARTS', 'BLOCK_ENDS', 'OBS_SAMPLES', 'OBS_MUT']
    return None, df_whole
    # return df_split, df_whole

def tabulate_mutations_in_element(f_mut, f_elt_bed, bed12=False, drop_duplicates=False, all_elements=False,
    max_muts_per_sample=1e9, max_muts_per_elt_per_sample=3e9, return_blacklist=False):
    df_cnt = tabulate_muts_per_sample_per_element(f_mut, f_elt_bed, bed12=bed12, drop_duplicates=drop_duplicates)
    df_cnt.rename({'SAMPLE': 'OBS_SAMPLES'}, axis=1, inplace=True)

    # Remove hypermutated samples and cap total mutations from a sample in an element
    # *if there are any samples
    if len(df_cnt) > 0:
        df_cnt_sample = df_cnt.pivot_table(index='OBS_SAMPLES', values='OBS_MUT', aggfunc=np.sum)
        blacklist = df_cnt_sample[df_cnt_sample.OBS_MUT > max_muts_per_sample].index
        df_cnt = df_cnt[~df_cnt.OBS_SAMPLES.isin(blacklist)]
    else:
        blacklist = []
    df_cnt.loc[df_cnt.OBS_SNV > max_muts_per_elt_per_sample, 'OBS_SNV'] = max_muts_per_elt_per_sample
    df_cnt.loc[df_cnt.OBS_INDEL > max_muts_per_elt_per_sample, 'OBS_INDEL'] = max_muts_per_elt_per_sample

    df_summary = df_cnt.pivot_table(index='ELT',
        values=['OBS_SAMPLES', 'OBS_SNV', 'OBS_INDEL'],
        aggfunc={'OBS_SAMPLES': len, 'OBS_SNV': np.sum, 'OBS_INDEL': np.sum})
    if len(df_summary) == 0:
        df_summary = pd.DataFrame({'OBS_SAMPLES':[], 'OBS_SNV':[], 'OBS_INDEL':[], 'ELT':[]})
        df_summary = df_summary.set_index('ELT')
    ## Add elements with zero counts if required
    if all_elements:
        df_bed = pd.read_csv(f_elt_bed, sep="\t", header=None).set_index(3)
        df_bed.index.rename('ELT', inplace=True)
        df_summary = df_bed.merge(df_summary, left_index=True, right_index=True, how='left')
        df_summary.loc[df_summary.OBS_SNV.isna(), 'OBS_SNV'] = 0
        df_summary.loc[df_summary.OBS_INDEL.isna(), 'OBS_INDEL'] = 0
        df_summary.loc[df_summary.OBS_SAMPLES.isna(), 'OBS_SAMPLES'] = 0

    if return_blacklist:
        return df_summary[['OBS_SAMPLES', 'OBS_SNV', 'OBS_INDEL']], blacklist
    else:
        return df_summary[['OBS_SAMPLES', 'OBS_SNV', 'OBS_INDEL']]

def tabulate_muts_per_sample_per_element(f_mut, f_elt_bed, bed12=False, drop_duplicates=False, unique_indels=True):
    ## Convert dataframes to bedtools objects
    bed_mut = pybedtools.BedTool(f_mut)
    bed_elt = pybedtools.BedTool(f_elt_bed)

    if bed12:
        bed_elt = bed_elt.bed12tobed6()

    ## Intersect bedtools objects
    bed_inter = bed_mut.intersect(bed_elt, wa=True, wb=True)
    if len(bed_inter) == 0:
        return  pd.DataFrame({'ELT':[], 'SAMPLE':[], 'OBS_SNV':[], 'OBS_INDEL':[], 'OBS_MUT':[]})

    n_field = bed_inter.field_count()
    df_inter = bed_inter.to_dataframe(header=None, names=range(n_field), low_memory=False)

    if drop_duplicates:
        df_inter = df_inter.drop_duplicates([0, 1, 2, 3, 4, 5, 13]) ## Drop mutations duplicated by genic annotations

    # Separate SNVs and INDELs
    df_inter.loc[df_inter[7] != 'INDEL', 7] = 'SNV'
    df_inter_snv = df_inter[df_inter[7] == 'SNV']
    df_inter_ind = df_inter[df_inter[7] == 'INDEL']

    # if unique_indels:
    #     df_inter_ind = df_inter_ind.drop_duplicates(subset=[0, 1, 2, 3, 4, 6])

    ## Aggregate over subelements of a single annotation
    df_cnt_snv = df_inter_snv.groupby([13, 5]).size().reset_index(name='OBS_SNV')
    df_cnt_ind = df_inter_ind.groupby([13, 5]).size().reset_index(name='OBS_INDEL')
    df_cnt = df_cnt_snv.merge(df_cnt_ind, how='outer')
    df_cnt.loc[df_cnt.OBS_SNV.isna(), 'OBS_SNV'] = 0
    df_cnt.loc[df_cnt.OBS_INDEL.isna(), 'OBS_INDEL'] = 0
    df_cnt['OBS_MUT'] = df_cnt.OBS_SNV + df_cnt.OBS_INDEL
    df_cnt = df_cnt[[13, 5, 'OBS_SNV', 'OBS_INDEL', 'OBS_MUT']]
    df_cnt.columns = ['ELT', 'SAMPLE', 'OBS_SNV', 'OBS_INDEL', 'OBS_MUT']
    # df_cnt = df_inter.groupby([13, 5]).size().reset_index(name='COUNT')
    # df_cnt.columns = ['ELT', 'SAMPLE', 'OBS_SNV']

    return df_cnt

def tabulate_nonc_mutations_at_sites(f_sites, f_mut, return_sites = False):
    """ tabulate mutations occuring at given sites.
        * sites file is assumed to be annotated with corresponding gene
        ** This function no longer filters by cds regions using -use_subelts flag, instead this
            is to be carried out by the user with the f_sites file prior to pretraining
    """
    df_sites = read_mutation_file(f_sites)
    df_sites.rename({"SAMPLE": "ELT"}, axis=1, inplace=True)
    assert ('GENE' in df_sites.columns and 'ANNOT' in df_sites.columns and 'MUT_TYPE' in df_sites.columns)
    if not 'STRAND' in df_sites.columns:
        print("WARNING: strand column not detected in sites file. Defaulting all sites to + strand.")
        df_sites['STRAND'] = "+"

    df_mut = read_mutation_file(f_mut, drop_duplicates = False)
    assert ('GENE' in df_mut.columns and 'ANNOT' in df_mut.columns and'MUT_TYPE' in df_mut.columns)

    if len(df_mut[df_mut.ANNOT == 'INDEL']):
        print('WARNING: INDELS found in mutation file. Dig sites model is only applicable to SNVs. INDELS will be dropped.')

    df_mut = df_mut[df_mut.ANNOT != 'INDEL']

    muts_atsites = df_mut.merge(df_sites, on =['CHROM', 'START', 'END', 'REF', 'ALT', 'GENE', 'ANNOT','MUT_TYPE', 'CONTEXT'], how='inner')
    # print(muts_atsites[0:5])
    ## use sites annotations as gene
    # muts_atsites = muts_atsites.drop(columns=['GENE']).rename(columns={'SAMPLE_x':'SAMPLE', 'SAMPLE_y':'GENE'})

    counts = muts_atsites.groupby(['ELT']).agg(
        {'SAMPLE': lambda x: len(set(x)), 'CHROM': lambda x: len(x)}).reset_index()
    counts.columns = ['ELT', 'OBS_SAMPLES', "OBS_SNV"]

    # ## Add ELTs with zero observed mutatios
    # ALL_SITES = set(df_sites.ELT.unique())
    # ZERO_SITES = ALL_SITES - set(counts.ELT.unique())
    # counts_zero = pd.DataFrame({'ELT': list(ZERO_SITES), 'OBS_SAMPLES': [0]*len(ZERO_SITES), 'OBS_MUT': [0]*len(ZERO_SITES)})
    # counts_aug = pd.concat([counts, counts_zero]).sort_values(by='ELT')
    counts_aug = counts

    if return_sites:
        #trim unecessary columns and set index for fast lookup
        df_sites = df_sites.drop(columns = ['GENE','ANNOT','REF','ALT']).rename(columns={'SAMPLE':'GENE'}).set_index('GENE')
        return counts_aug, df_sites
    else:
        return counts_aug

#wrapper for transfer learning model on sites
def tabulate_sites_in_element(f_sites, f_mut):
    df_res = tabulate_nonc_mutations_at_sites(f_sites, f_mut)
    df_res = df_res.set_index('ELT')
    return df_res[['OBS_SAMPLES', 'OBS_SNV']]

def _genic_fill_empty_cols(df_gene_counts):
    """
    ensures that every mutation type is tabulated
    """
    needed_cols = set(['Essential_Splice', 'Missense','Nonsense','Stop_loss','Synonymous'])
    cur_cols  = set(df_gene_counts.columns)
    dif_cols = needed_cols.difference(cur_cols)
    for c in dif_cols:
        df_gene_counts[c] = 0

def filter_hypermut_samples(df_mut, max_muts_per_sample, return_blacklist=False):
    """ Remove samples that have more mutations than the specified threshold
    """
    sample_cnt = df_mut.SAMPLE.value_counts()
    samples_blacklist = sample_cnt[sample_cnt > max_muts_per_sample].index.to_list()

    df_whitelist = df_mut[~df_mut.SAMPLE.isin(samples_blacklist)]

    if return_blacklist:
        return df_whitelist, samples_blacklist

    return df_whitelist

def filter_samples_by_stdev(df_mut, stdev_cutoff):
    """ Remove samples where # mutations > stdev_cutoff*stdev of cohort mutation counts
    """
    sample_cnt = df_mut.SAMPLE.value_counts()
    stdev = sample_cnt.std()
    print(stdev)
    samples_blacklist = sample_cnt[sample_cnt > stdev * stdev_cutoff].index.to_list()

    df_whitelist = df_mut[~df_mut.SAMPLE.isin(samples_blacklist)]

    return df_whitelist

def cap_muts_per_element_per_sample(df_mut_elt_samp, max_muts_per_elt_per_sample):
    """ Upper-bound number of mutations any one sample can contribute to an element

    Args:
        df_mut_elt_samp     dataframe produced by tabulate_muts_per_sample_per_element
    """
    mask = df_mut_elt_samp.OBS_MUT > max_muts_per_elt_per_sample
    df_mut_elt_samp.loc[mask, 'OBS_MUT'] = max_muts_per_elt_per_sample

    return df_mut_elt_samp

def mutations_per_gene(df_mut_cds, max_muts_per_gene_per_sample=3e9):
    """ Count the number of mutations per gene in a mutation dataframe
    """
    # df_counts = pd.crosstab(df_mut_cds.GENE, df_mut_cds.ANNOT)
    df_group = df_mut_cds.groupby(['GENE', 'SAMPLE', 'ANNOT']).size().reset_index(name='COUNT')
    df_group.loc[df_group.COUNT > max_muts_per_gene_per_sample, 'COUNT'] = max_muts_per_gene_per_sample
    df_pivot = df_group.pivot_table(index=['GENE', 'ANNOT'], values='COUNT', aggfunc=np.sum).reset_index()
    df_counts = df_pivot.pivot(index='GENE', columns='ANNOT', values='COUNT')
    df_counts[df_counts.isna()] = 0
    df_counts = df_counts.astype(int)
    df_counts.columns = df_counts.columns.to_list()

    if not "Missense" in df_counts.columns:
        df_counts['Missense'] = 0
    if not "Nonsense" in df_counts.columns:
        df_counts['Nonsense'] = 0
    if not "Synonymous" in df_counts.columns:
        df_counts['Synonymous'] = 0
    if not "Essential_Splice" in df_counts.columns:
        df_counts['Essential_Splice'] = 0
    if not "INDEL" in df_counts.columns:
        df_counts['INDEL'] = 0

    df_counts.rename({'Missense': 'OBS_MIS',
                      'Nonsense': 'OBS_NONS',
                      'Synonymous': 'OBS_SYN',
                      'Essential_Splice': 'OBS_SPL',
                      'INDEL': 'OBS_INDEL'
                     },
        axis=1, inplace=True
    )

    return df_counts

def mutations_by_element(f_mut, f_elt_bed, bed12=False, drop_duplicates=False):
    """ Get all mutations falling within a set of elements
    """
    bed_mut = pybedtools.BedTool(f_mut)
    bed_elt = pybedtools.BedTool(f_elt_bed)
    if bed12:
        bed_elt = bed_elt.bed12tobed6()

    ## Intersect bedtools objects
    bed_inter = bed_mut.intersect(bed_elt, wa=True, wb=True)
    df_hits = bed_inter.to_dataframe(header=None, names=range(18), low_memory=False)
    df_hits = df_hits.iloc[:, [0,1,2,3,4,5,6,7,8,9,13]]

    if drop_duplicates:
        df_hits = df_hits.drop_duplicates([0, 1, 2, 3, 4, 5, 13]) ## Drop mutations duplicated by genic annotations

    df_hits.columns = ['CHROM', 'START', 'END', 'REF', 'ALT', 'SAMPLE', 'GENE', 'ANNOT', 'TYPE', 'CONTEXT', 'ELT']

    return df_hits

def bed12_boundaries(f_bed):
    df_nonc = pd.read_table(f_bed, names=['CHROM', 'START', 'END', "ELT", "SCORE", "STRAND", 'thickStart', 'thickEnd', 'rgb', 'blockCount', 'blockSizes', 'blockStarts'], low_memory=False)
    df_nonc.CHROM = df_nonc.CHROM.astype(str)
    df_nonc.blockSizes = df_nonc.blockSizes.astype(str)
    df_nonc.blockStarts = df_nonc.blockStarts.astype(str)
    if 'chr' in str(df_nonc['CHROM'][0]):
        df_nonc['CHROM'] = df_nonc['CHROM'].map(lambda x: x.lstrip('chr'))
    df_nonc = df_nonc[df_nonc.CHROM.isin([str(c) for c in range(1, 23)])]
    df_nonc.CHROM = df_nonc.CHROM.astype(int)

    def _get_starts(row):
        str_starts = row.blockStarts
        if str_starts.endswith(','):
            str_starts = str_starts[:-1]

        return [int(x)+row.START for x in str_starts.split(",")]

    def _get_ends(row):
        str_sizes = row.blockSizes
        if str_sizes.endswith(','):
            str_sizes = str_sizes[:-1]

        sizes = [int(x) for x in str_sizes.split(',')]

        return [START + SIZE for START, SIZE in zip(row.BLOCK_STARTS, sizes)]

    df_nonc['BLOCK_STARTS'] = df_nonc.apply(_get_starts, axis=1)
    df_nonc['BLOCK_ENDS'] = df_nonc.apply(_get_ends, axis=1)

    df_nonc = df_nonc[['CHROM', 'ELT', 'STRAND', 'BLOCK_STARTS', 'BLOCK_ENDS']]

    return df_nonc


import re
from urllib import parse
import scrapy
from scrapy.http import Request
from scrapy.selector import Selector
from QADoorSpider.items import ArticleItem

class TuikuSpider(scrapy.Spider):
    name = "tuiku"
    # allowed_domains = ["tuicool.com"]
    start_urls = (
        'http://www.tuicool.com/topics/11130000?st=0&lang=1&pn=0',
    )

    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate, sdch',
        'Accept-Language': 'zh-CN,zh;q=0.8',
        'Connection': 'keep-alive',
        'Cookie': 'UM_distinctid=15b13ff61121b1-0679fa0202aad5-1421150f-140000-15b13ff611330d; CNZZDATA5541078=cnzz_eid%3D604530066-1490687638-null%26ntime%3D1500176950; _tuicool_session=BAh7C0kiD3Nlc3Npb25faWQGOgZFRkkiJWZjNTc3MjQxMWU0ZjQ5NmU1N2M4NGViNDc3OTRiODZjBjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMVpEL29teHRTTnhRR1FIeWwycVlOTjIxNzBFUVJpMHhiY1Vnb2swMUh2b009BjsARkkiDHVzZXJfaWQGOwBGaQMWkgJJIg5yZXR1cm5fdG8GOwBGSSIsaHR0cDovL3d3dy50dWljb29sLmNvbS9hcnRpY2xlcy9CYlVKRmI2BjsARkkiDnVzZXJfY2l0eQY7AEZJIgvlhajlm70GOwBUSSITdXNlcl9jaXR5X2NvZGUGOwBGSSIIYWxsBjsAVA%3D%3D--7424f2078d34e840c9197dd1f58c2db057aab636',
        'Host': 'www.tuicool.com',
        'If-None-Match': 'W/"6da29458c549b4788210314b12c8e718"',
        'Referer': 'http://www.tuicool.com/topics',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'
    }

    def start_requests(self):
        for page in range(0, 10):
            if page>1:
                self.headers['Referer'] = 'http://www.tuicool.com/topics/11130000?st=0&lang=1&pn={}'.format(page-1)
            yield Request('http://www.tuicool.com/topics/11130000?st=0&lang=1&pn={}'.format(page), headers=self.headers, callback=self.parse_list)

    def parse_list(self, response):
        selector = Selector(response)
        titles = selector.xpath('//div[@class="title"]/a/@title').extract()
        urls = selector.xpath('//div[@class="title"]/a/@href').extract()
        for title, url in zip(titles, urls):
            q_item = ArticleItem()
            q_item['title'] = title.strip()
            q_item['url'] = parse.urljoin('http://www.tuicool.com/', url)
            # q_item['_id'] = q_item['url'].split('/')[-1]
            yield Request(url=q_item['url'], headers=self.headers, meta={'item': q_item}, callback=self.parse_detail)

    def parse_detail(self, response):
        q_item = response.meta['item']
        selector = Selector(response)
        q_item['pub_date'] = selector.xpath('//span[@class="timestamp"]/text()').extract_first().strip('时间').strip()
        q_item['source'] = selector.xpath('//div[@class="source"]/a[@class="cut cut70"]/text()').extract_first().strip()
        q_item['tags'] = selector.xpath('//span[@class="new-label"]/text()').extract()
        q_item['content'] = selector.xpath('//div[@class="article_body"]').extract_first()
        yield q_item


# using opencv built in face recognizer

import os
import cv2 as cv
import numpy as np

people = []
train_dir = r'A:\GitHub\opencv_practice\Faces\train'
haar_cascade = cv.CascadeClassifier('Face_detection_using_haar_cascade\haar_cascade_face.xml')
for i in os.listdir(r'A:\GitHub\opencv_practice\Faces\train'):
    people.append(i)

features = []
labels = []
def train():
    for person in people:
        path = os.path.join(train_dir, person)
        label = people.index(person)

        for img in os.listdir(path):
            img_path = os.path.join(path, img)

            img_array = cv.imread(img_path)
            gray = cv.cvtColor(img_array, cv.COLOR_BGR2GRAY)
            faces_rect = haar_cascade.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=1)

            for (x,y,w,h) in faces_rect:
                faces_roi = gray[y:y+h, x:x+w]
                features.append(faces_roi)
                labels.append(label)

train()
np.save('Features.npy', features)
np.save('Labels.npy', labels)
face_recognizer = cv.face.LBPHFaceRecognizer_create()
face_recognizer.train(np.array(features, dtype='object'), np.array(labels))
np.save('Features.npy', features)
np.save('Labels.npy', labels)
face_recognizer.save('face_trained.yml')


import argparse
import logging
from multiprocessing.connection import wait
import os
import glob
import numpy as np
import torch
import torch.nn.functional as F
from PIL import Image
from torchvision import transforms
import monai
from monai.data import list_data_collate
from utils.data_loading import BasicDataset
from torch.utils.data import DataLoader
from unet import UNet
from utils.dice_score import dice_loss, dice_coeff, multiclass_dice_coeff
from utils.utils import plot_img_and_mask
import SimpleITK as sitk
from scipy.signal import resample_poly
from dataload import Data
# from myshow3d import myshow, myshow3d
import cv2
def predict_img(net,
                img,
                device,
                scale_factor=1,
                out_threshold=0.5):
    net.eval()
    # img = torch.from_numpy(full_img)
    # img = img.unsqueeze(0)
    img = img.to(device=device, dtype=torch.float32)

    with torch.no_grad():
        mask_pred = net(img)

        if net.n_classes == 1:
            mask_pred = (F.sigmoid(mask_pred) > 0.5).float()
            # compute the Dice score
            # dice_score += dice_coeff(mask_pred, mask_true, reduce_batch_first=False)
        else:
            mask_pred = F.one_hot(mask_pred.argmax(dim=1), net.n_classes).permute(0, 3, 1, 2).float()
            # compute the Dice score, ignoring background
            # dice_score += multiclass_dice_coeff(mask_pred[:, 1:, ...], mask_true[:, 1:, ...], reduce_batch_first=False)

        return torch.softmax(mask_pred, dim=1).argmax(dim=1)[0].float().cpu()

def get_args():
    parser = argparse.ArgumentParser(description='Predict masks from input images')
    parser.add_argument('--model', '-m', default='MODEL.pth', metavar='FILE',
                        help='Specify the file in which the model is stored')
    parser.add_argument('--input', '-i', metavar='INPUT', nargs='+', help='Filenames of input images', required=True)
    parser.add_argument('--output', '-o', metavar='INPUT', nargs='+', help='Filenames of output images')
    parser.add_argument('--viz', '-v', action='store_true',
                        help='Visualize the images as they are processed')
    parser.add_argument('--no-save', '-n', action='store_true', help='Do not save the output masks')
    parser.add_argument('--mask-threshold', '-t', type=float, default=0.5,
                        help='Minimum probability value to consider a mask pixel white')
    parser.add_argument('--scale', '-s', type=float, default=0.5,
                        help='Scale factor for the input images')

    return parser.parse_args()

def get_output_filenames(args):
    def _generate_name(fn):
        split = os.path.splitext(fn)
        return f'{split[0]}_OUT{split[1]}'

    return args.output or list(map(_generate_name, args.input))

def mask_to_image(mask: np.ndarray):
    if mask.ndim == 2:
        return Image.fromarray((mask * 255).astype(np.uint8))
    elif mask.ndim == 3:
        return Image.fromarray((np.argmax(mask, axis=0)/ mask.shape[0]).astype(np.uint8))

def resolution_match(result_array, data_util, coord1, coord2):
    result_array_np = np.transpose(result_array, (2, 1, 0))
    result_array_temp = sitk.GetImageFromArray(result_array_np)
    result_array_temp.SetSpacing((data_util.resolution) + (1, ))

    # save temporary label
    writer = sitk.ImageFileWriter()
    writer.SetFileName('temp_seg.nii')
    writer.Execute(result_array_temp)

    files = [{"image": 'temp_seg.nii'}]

    files_ds = monai.data.Dataset(data=files, transform=data_util.reverse_transforms)
    files_loader = DataLoader(files_ds, batch_size=1, num_workers=0)

    for files_data in files_loader:
        files_images = files_data["image"]

        res = files_images.squeeze().data.numpy()

    # result_array = np.rint(res)
    result_array = res
    os.remove('./temp_seg.nii')

    # empty_array = np.zeros(original_shape)
    # empty_array[coord1[0]:coord2[0],coord1[1]:coord2[1],coord1[2]:coord2[2]] = result_array

    itk_image = sitk.GetImageFromArray(result_array)
    return itk_image

if __name__ == '__main__':
    input_path = '/Users/mona/workSpace/github_repo/DeepShim/code/python/Pytorch-UNet/data/target_data_originalresolution'
    path = '/Users/mona/workSpace/github_repo/DeepShim/code/python/Pytorch-UNet/dataoutput/test_resolution_2'
    model = f"checkpoints/test_resolution/best_model_fold.pth"

    if not os.path.exists(path):
        os.makedirs(path)

    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    data_util = Data()

    print(f'Using device {device}')

    net = UNet(n_channels=1, n_classes=2)
    net.to(device=device)
    net.load_state_dict(torch.load(model, map_location=device))

    print('Model loaded!')

    input_files = glob.glob(f"{input_path}/*.nii")
    input_files.sort(key=lambda x:int((x.split("/")[-1]).split('_')[-1][:-4]))
    masks = []
    images = []
    for image in input_files:
        original_shape, crop_shape, coord1, coord2, resampled_size, original_resolution = data_util.statistics_crop(image, data_util.resolution)
        files = [{"image": image}]
        test_ds = monai.data.Dataset(data=files, transform=data_util.test_transforms)
        test_loader = DataLoader(test_ds, batch_size=1, num_workers=0, collate_fn=list_data_collate, pin_memory=False)
        for test_data in test_loader:
            test_image = test_data["image"].to(device)
            mask_slice = predict_img(net=net,
                            img=test_image,
                            scale_factor=1,
                            out_threshold=0.5,
                            device=device)
        result_array = mask_slice.squeeze().data.cpu().numpy()
        result_array = result_array[0:resampled_size[0],0:resampled_size[1]]

        result_seg = data_util.from_numpy_to_itk(result_array, image)
        result = os.path.join(path, image.split("/")[-1])
        writer = sitk.ImageFileWriter()
        writer.SetFileName(result)
        writer.Execute(result_seg)

        masks.append(result_array)
        images.append((test_image[0].squeeze().data.cpu().numpy())[0:resampled_size[0],0:resampled_size[1]])

    masks = np.dstack(masks)
    images = np.dstack(images)

    # masks_img = data_util.from_numpy_to_itk(masks, image)
    # images_img = data_util.from_numpy_to_itk(images, image)

    if not (original_resolution == data_util.resolution):
        data_util.init_reverse(crop_shape=(crop_shape+(len(input_files), )))
        masks_img = resolution_match(masks, data_util, coord1, coord2)
        images_img = resolution_match(images, data_util, coord1, coord2)

    else:
        masks_img = sitk.GetImageFromArray(masks)
        images_img = sitk.GetImageFromArray(images)

    sitk.WriteImage(masks_img, os.path.join(path, "3Doutput.nii"))
    sitk.WriteImage(images_img, os.path.join(path, "3Dinput.nii"))


from functools import wraps
from time import time

def timing(f):
    """
    From https://stackoverflow.com/a/27737385/1775741
    """
    @wraps(f)
    def wrap(*args, **kw):
        ts = time()
        result = f(*args, **kw)
        te = time()
        dur = te - ts
        print('func:{!r} args:[{!r}, {!r}] took: {:2.4f} sec'.format(
            f.__name__, args, kw, dur)
        )
        return dur, result
    return wrap


import kivy
from kivy.app import App
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.button import  Button

class Tela1(BoxLayout):
    def on_press_bt(self):
        janela.root_window.remove_widget(janela.root)
        janela.root_window.add_widget(Tela2())
    def __init__(self, **kwargs):
        super(Tela1, self).__init__(**kwargs)
        self.orientation = "vertical"
        bt1 = Button(text="CLICK")
        bt1.on_press = self.on_press_bt
        bt2 = Button(text="CLICK2")
        bt3 = Button(text="CLICK3")
        self.add_widget(bt1)
        self.add_widget(bt2)
        self.add_widget(bt3)

class Tela2(BoxLayout):
    def on_press_bt(self):
        janela.root_window.remove_widget(janela.root)
        janela.root_window.add_widget(Tela1())

    def __init__(self, **kwargs):
        super(Tela2, self).__init__(**kwargs)
        self.orientation = "vertical"
        bt = Button (text= "clique")
        bt.on_press = self.on_press_bt
        self.add_widget(bt)

class PY(App):
    def build(self):
        return Tela1()

janela = PY()
janela.run()


import matplotlib.pyplot as plt
def corelation(px,py):
    n=len(px)
    sumx=0
    sumy=0
    sumXY=0
    sumX2=0
    sumY2=0
    for i in range (n):
        x = px[i]
        y = py[i]
        sumx += x
        sumy += y
    meanx=sumx/n
    meany=sumy/n

    for i in range(n):
         X=px[i]-meanx
         Y=py[i]-meany

         sumX2+= X*X
         sumY2+= Y*Y
         sumXY+= X*Y
    print(sumX2,sumY2,sumXY)
    r=(sumXY)/(sumX2*sumY2)**(1/2)
    return r
x=[105,104,102,101,100,99,98,96,93,92]
y=[101,103,100,98,95,96,104,92,97,94]
result=corelation(x,y)
print(result)
#plt.scatter(x,y,color='red',marker='o')
plt.bar(x,y,color='red')
plt.show()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
__copyright__ = """ This code is licensed under the 3-clause BSD license.
Copyright ETH Zurich, Laboratory of Physical Chemistry, Reiher Group.
See LICENSE.txt for details.
"""
"""
Functions that abstract sending and receiving data
between a server and a client.
The data is now serialized using json.
"""
import json
import struct
import typing
from socket import socket as Socket

def send_data(data: typing.Any, socket: Socket) -> None:
    """
    Serialise and send data through a socket.
    The size of the dataset is prepended to the real data,
    so that the receiver knows how large is the dataset.
    Retry until it is done.
    """
    serialized = json.dumps(data)
    raw_data = bytes(serialized, encoding="utf-8")
    sizeinfo = struct.pack("!i", len(raw_data))  # Sending an integer

    socket.sendall(sizeinfo)
    socket.sendall(raw_data)

def recv_data(connection: Socket) -> typing.Any:
    """
    Read data from a socket.
    It is expected that the first datum that comes out
    will be the size of the data sent through.
    """
    buf = bytearray()
    while len(buf) < 4:
        buf += connection.recv(4 - len(buf))
    (size_to_receive,) = struct.unpack("!i", buf)
    size_received = 0
    data_received = bytearray()
    while len(data_received) < size_to_receive:
        chunksize = min(4096, size_to_receive - size_received)
        data_received += connection.recv(chunksize)
    data_decoded = data_received.decode("utf-8")

    return json.loads(data_decoded)


from django.urls import re_path

from . import views

lang_urlpatterns = [
    re_path(r"^recurring/?$", views.contribute, name="recurring_payment_initial"),
    re_path(
        r"^recurring/subscription/?$",
        views.contribute,
        name="recurring_payment_subscription",
    ),
    re_path(r"^terms/?$", views.payment_terms, name="payment_terms"),
    re_path(
        r"^recurring/management/?$",
        views.recurring_payment_management,
        name="recurring_payment_management",
    ),
]


from django.shortcuts import render

# End points:
def index(request):
    return render(request,'index.html')

def submission(request):
    # Taking values:
    text = request.POST.get('text', 'default')
    remover = request.POST.get('remover', 'off')
    capital = request.POST.get('capital', 'off')
    lower = request.POST.get('lower', 'off')
    counter = request.POST.get('counter', 'off')
    # Logic implementation:
    puncs = '''!()-[]{};:'"\,<>./?@#$%^&*_~'''
    if remover=="on":
        result=""
        for char in text:
            if char not in puncs:
                result = result + char
                text = result
    if capital=="on":
        result = ""
        for char in text:
            result = result + char.upper()
            text =result
    if lower=="on":
        result = ""
        for char in text:
            result = result + char.lower()
            text = result
    if counter=="on":
        resultone = 0
        for char in text:
            resultone = resultone+1
        final = {'clear':resultone}
        return render(request,'submission.html',final)
    if text=="":
        return render(request,'error.html')
    params = {'clear': result}
    return render(request, 'submission.html', params)

def about(request):
    return render(request, 'about.html')


import json
from urllib import response
from unittest import mock
from django.urls import reverse
import pytest
from apps.fyle.models import Expense, ExpenseGroup, Reimbursement, get_default_expense_group_fields, get_default_expense_state, \
    ExpenseGroupSettings, _group_expenses, get_default_ccc_expense_state
from apps.workspaces.models import Configuration, Workspace
from apps.tasks.models import TaskLog
from apps.fyle.tasks import create_expense_groups
from .fixtures import data

@pytest.mark.django_db()
def test_create_expense(create_temp_workspace):
    mock_expenes = data['expenses']
    expense_count = len(Expense.objects.filter(org_id='or79Cob97KSh'))
    Expense.create_expense_objects(
        mock_expenes
    )

    expense = Expense.objects.filter(org_id='or79Cob97KSh').order_by('created_at')
    assert len(expense) == expense_count+2

    expense = expense.last()
    assert expense.employee_email == 'jhonsnow@fyle.in'
    assert expense.currency == 'USD'
    assert expense.fund_source == 'PERSONAL'

def test_default_fields():
    expense_group_field = get_default_expense_group_fields()
    expense_state = get_default_expense_state()
    ccc_expense_state = get_default_ccc_expense_state()

    assert expense_group_field == ['employee_email', 'report_id', 'claim_number', 'fund_source']
    assert expense_state == 'PAYMENT_PROCESSING'
    assert ccc_expense_state == 'PAID'

@pytest.mark.django_db
def test_expense_group_settings(create_temp_workspace):
    payload = data['expense_group_setting_payload']

    ExpenseGroupSettings.update_expense_group_settings(
        payload, 3
    )

    settings = ExpenseGroupSettings.objects.last()

    assert settings.expense_state == 'PAYMENT_PROCESSING'
    assert settings.ccc_expense_state == 'PAID'
    assert settings.ccc_export_date_type == 'spent_at'

def test_create_expense_groups_by_report_id_fund_source_spent_at(db):
    expenses = data['expenses_spent_at']

    expense_objects = Expense.create_expense_objects(expenses)

    configuration = Configuration.objects.get(workspace_id=1)
    workspace = Workspace.objects.get(id=1)

    expense_group_setting = ExpenseGroupSettings.objects.get(workspace_id=1)
    expense_group_setting.reimbursable_export_date_type = 'spent_at'
    reimbursable_expense_group_fields = expense_group_setting.reimbursable_expense_group_fields
    reimbursable_expense_group_fields.append('spent_at')
    expense_group_setting.reimbursable_expense_group_fields = reimbursable_expense_group_fields
    expense_group_setting.save()

    assert len(expense_objects) == 3

    ExpenseGroup.create_expense_groups_by_report_id_fund_source(expense_objects, configuration, 1)

    expense_group = ExpenseGroup.objects.filter(workspace=workspace).order_by('-created_at').first()

    assert expense_group.expenses.count() == 2

def test_create_expense_groups_by_report_id_fund_source(db):
    expenses = data['expenses']

    expense_objects = Expense.create_expense_objects(expenses)

    configuration = Configuration.objects.get(workspace_id=49)
    workspace = Workspace.objects.get(id=1)

    ExpenseGroup.create_expense_groups_by_report_id_fund_source(expense_objects, configuration, 49)

    expense_groups = ExpenseGroup.objects.filter(workspace=workspace)

    assert len(expense_groups) == 2

    expense_group_setting = ExpenseGroupSettings.objects.get(workspace_id=49)
    corporate_credit_card_expense_group_fields = expense_group_setting.corporate_credit_card_expense_group_fields
    corporate_credit_card_expense_group_fields.append('dummy')
    expense_group_setting.corporate_credit_card_expense_group_fields = corporate_credit_card_expense_group_fields
    expense_group_setting.save()

    ExpenseGroup.create_expense_groups_by_report_id_fund_source(expense_objects, configuration, 49)

    expense_groups = ExpenseGroup.objects.filter(workspace=workspace)

    assert len(expense_groups) == 2

def test_create_reimbursement(db):

    reimbursements = data['reimbursements']

    Reimbursement.create_or_update_reimbursement_objects(reimbursements=reimbursements, workspace_id=1)

    pending_reimbursement = Reimbursement.objects.get(reimbursement_id='reimgCW1Og0BcM')

    pending_reimbursement.state = 'PENDING'
    pending_reimbursement.settlement_id= 'setgCxsr2vTmZ'

    reimbursements[0]['is_paid'] = True

    Reimbursement.create_or_update_reimbursement_objects(reimbursements=reimbursements, workspace_id=1)

    paid_reimbursement = Reimbursement.objects.get(reimbursement_id='reimgCW1Og0BcM')
    paid_reimbursement.state == 'PAID'

def test_get_last_synced_at(db):

    reimbursement = Reimbursement.get_last_synced_at(1)

    assert reimbursement.workspace_id == 1
    assert reimbursement.settlement_id == 'setqi0eM6HUgZ'
    assert reimbursement.state == 'COMPLETE'


import os, shlex
import tempfile, subprocess

def editContent(content=None):
    """ Edit the content with vi editor, the input
    'content' is byte, the returned one is byte also
    """
    tmpfile = tempfile.NamedTemporaryFile(delete=False)
    if content:
        tmpfile.write(content)
        tmpfile.flush()
    cmd = 'vi ' + tmpfile.name
    p = subprocess.Popen(shlex.split(cmd))
    p.communicate()
    p.wait()

    # vim may delete the original file,
    # so we reopen the file to read.
    content = open(tmpfile.name, 'rb').read()
    os.unlink(tmpfile.name)
    return content


"""Definition of the documentRRHH contenttype
"""
from zope.interface import implements
from Products.Archetypes import atapi
from Products.ATContentTypes.content import folder
from Products.ATContentTypes.content import schemata
from notes.rrhh.interfaces import IDocumentRRHH
from notes.rrhh.config import PROJECTNAME
from Products.ATVocabularyManager import NamedVocabulary

DocumentRRHHSchema = folder.ATFolderSchema.copy() + atapi.Schema((

            atapi.TextField(
                name='body',
                allowable_content_types=('text/plain',
                                               'text/structured',
                                               'text/html',),
                 default_output_type='text/x-html-safe',
                 widget=atapi.RichWidget(
                     label='Body',
                     label_msgid='label_body',
                     i18n_domain='notes.rrhh',
                     rows=40,
                 ),
             required=False,
             searchable=True,
             ),

            #CATEGORIES
            #atapi.LinesField(
            #    name='categoryRRHH',
            #    widget=atapi.InAndOutWidget(
            #        format="select",
            #        label_msgid='categoryRRHH_label',
            #        description_msgid='categoryRRHH_help',
            #        i18n_domain='notes.rrhh',
            #    ),
            #    languageIndependent=True,
            #    required=False,
            #    schemata="categorization",
            #    vocabulary=NamedVocabulary('categoryRRHH_keywords'),
            #    enforceVocabulary=True,
            #),

))

DocumentRRHHSchema['title'].storage = atapi.AnnotationStorage()
DocumentRRHHSchema['description'].storage = atapi.AnnotationStorage()

# Hide default category option
DocumentRRHHSchema['description'].widget.visible = {'view': 'invisible', 'edit': 'invisible'}
DocumentRRHHSchema['language'].widget.visible = {'view': 'invisible', 'edit': 'invisible'}
DocumentRRHHSchema['relatedItems'].widget.visible = {'view': 'invisible', 'edit': 'invisible'}
DocumentRRHHSchema['location'].widget.visible = {'view': 'invisible', 'edit': 'invisible'}

schemata.finalizeATCTSchema(DocumentRRHHSchema, folderish=True, moveDiscussion=False)

class DocumentRRHH(folder.ATFolder):
    """Description of the documentRRHH"""
    implements(IDocumentRRHH)

    meta_type = "DocumentRRHH"
    schema = DocumentRRHHSchema

    title = atapi.ATFieldProperty('title')
    description = atapi.ATFieldProperty('description')

atapi.registerType(DocumentRRHH, PROJECTNAME)


#!/usr/bin/env python2
# -*- coding: utf-8 -*-

import config
import math
import numpy as np
from twisted.internet import reactor, threads
from score import score

class ProgressBox:
    def __init__(self, nPossibilities):
        self.nPossibilities = nPossibilities
        self.nCompleted = 0

    def getProgress(self):
        return float(self.nCompleted) / self.nPossibilities

    def getPercentProgress(self):
        return (self.nCompleted * 100) / self.nPossibilities

    def addCompleted(self):
        if self.nCompleted < self.nPossibilities:
            self.nCompleted = self.nCompleted + 1

def produceMatrix(chosenSections):
    schedule = np.empty(7, dtype=object)
    for i in range(0,7):
        schedule[i] = np.empty(config.SLOTS_PER_DAY, dtype=object)

    for section in chosenSections:
        sessions = section.sessions
        for sess in sessions:
            tss = sess.timeslots
            for ts in tss:
                sday = int(math.floor(ts / config.SLOTS_PER_DAY))
                sslot = ts % config.SLOTS_PER_DAY

                if schedule[sday][sslot] is not None:
                    return None

                schedule[sday][sslot] = sess

    return schedule

def solve2(socket, nCourse, sectionsForCourse, chosenSections, timeEquivalencies, progressBox, start, end, freeDays):
    # TODO: A better implementation here?
    if nCourse <= 3:
        progress = progressBox.getPercentProgress()
        reactor.callFromThread(socket.sendProgress, progress)

        cancelled = threads.blockingCallFromThread(reactor, socket.isCancelled)
        if cancelled:
            raise ValueError("The request has been cancelled")

    if nCourse == len(sectionsForCourse):
        schedule = produceMatrix(chosenSections)
        progressBox.addCompleted()
        progress = progressBox.getPercentProgress()

        if schedule is not None:
            sc = score(schedule, start, end, freeDays)

            # Copy sections
            chosenSectionsForOutput = []
            for chosenSection in chosenSections:
                chosenSectionsForOutput.append(chosenSection)

            reactor.callFromThread(socket.sendProgress, progress, chosenSectionsForOutput, timeEquivalencies, sc)

        return

    course = sectionsForCourse[nCourse][0]
    sections = sectionsForCourse[nCourse][1]

    for section in sections:
        chosenSections[nCourse] = section
        solve2(socket, nCourse + 1, sectionsForCourse, chosenSections, timeEquivalencies, progressBox, start, end, freeDays)

def startSolution(socket, courses, start, end, freeDays):
    solutions = []
    sectionsForCourse = []
    timeEquivalencies = {}

    nPossibilities = 1
    for course in courses:
        if course is not None:
            sectionsForThisCourse = {}

            for section in course.sections:
                hash = section.time_hash

                if not hash in sectionsForThisCourse:
                    sectionsForThisCourse[hash] = section
                else:
                    timeEquivalentSection = sectionsForThisCourse[hash]

                    if not section in timeEquivalencies:
                        timeEquivalencies[timeEquivalentSection] = []

                    timeEquivalencies[timeEquivalentSection].append(section)

            sectionList = [value for key,value in sectionsForThisCourse.iteritems()]
            sectionsForCourse.append((course, sectionList))

            nPossibilities = nPossibilities * len(sectionList)

    chosenSections = np.empty(len(sectionsForCourse), dtype=object)
    progressBox = ProgressBox(nPossibilities)

    solve2(socket, 0, sectionsForCourse, chosenSections, timeEquivalencies, progressBox, start, end, freeDays)

    reactor.callFromThread(socket.sendCompletion)


import logging, time, json

from flask import Flask, request, Response, make_response
from flask_cors import CORS
from flask_expects_json import expects_json

from randomizer.iogr_rom import Randomizer, generate_filename, VERSION
from randomizer.models.randomizer_data import RandomizerData as Settings

from config import Config
from database import Database

from encoder import JSONEncoder
from models.http.seed_request import SeedRequest
from models.http.seed_response import SeedResponse
from models.http.version_response import VersionResponse
from models.patch import Patch
from models.spoiler import Spoiler
from models.result import Result

logging.basicConfig(level=logging.DEBUG, filename="output.log")

app = Flask(__name__)
app.config['CORS_HEADERS'] = 'Content-Type'
app.json_encoder = JSONEncoder

cors = CORS(app, resources={
    r"/v1/*": {"origins": "*"}
})

config = Config()
database = Database(logging, config)

@app.route("/v1/seed/generate", methods=["POST"])
@expects_json(SeedRequest.schema)
def generateSeed() -> Response:
    request_data = SeedRequest(request.get_json())
    settings = Settings(request_data.seed, request_data.difficulty, request_data.goal,
                        request_data.logic, request_data.statues, request_data.statue_req,
                        request_data.enemizer, request_data.start_location, request_data.firebird,
                        request_data.ohko, request_data.red_jewel_madness, request_data.allow_glitches,
                        request_data.boss_shuffle, request_data.open_mode, request_data.z3_mode,
                        request_data.overworld_shuffle, request_data.entrance_shuffle, request_data.generate_race_rom,
                        request_data.fluteless, request_data.sprite, request_data.dungeon_shuffle)

    randomizer = Randomizer("./data/gaia.bin")
    result = __generate(randomizer, settings, request_data.generate_race_rom, 0, request_data.hide_settings)
    if result is None:
        return make_response("Failed to generate a seed", 500)

    if result.patch is None:
        return make_response("Failed to generate a seed", 500)

    patch = result.patch
    spoiler = result.spoiler
    permalink = result.permalink

    response = SeedResponse(patch.patch, patch.patchName, patch.version, permalink)
    if spoiler is not None:
        response.spoiler = spoiler.spoiler
        response.spoilerName = spoiler.spoilerName

    if permalink is not None:
        response.permalink_id = permalink

    return make_response(response.to_json(), 200)

@app.route("/v1/seed/permalink/<link_id>", methods=["GET"])
def getPermalinkedSeed(link_id: str = "") -> Response:
    try:
        if link_id == "":
            return make_response("Did not send permalink ID", 401)

        logging.info("Attempting to get permalink: " + link_id)
        document = database.get(link_id)
        if document is None:
            return make_response("Permalink Not Found", 404)

        return make_response(json.dumps(document, cls=JSONEncoder), 200)
    except Exception as e:
        logging.exception(e)
        return make_response("An unknown error has occurred", 500)

@app.route("/v1/api/version", methods=["GET"])
def getRandomizerVersion() -> Response:
    version = VersionResponse(VERSION)
    return make_response(version.to_json())

def __generate(randomizer: Randomizer, settings: Settings, race: bool = False, retries: int = 0, hide_settings: bool = False) -> Result:
    if retries >= 3:
        return None

    patch = None
    permalink = None
    spoiler = None

    try:
        patch_start_time = time.perf_counter()
        logging.info(f"({settings.seed}) Generating patch ({retries + 1} / 3)...")
        logging.info(f"({settings.seed}) Settings: {str(request.get_json())}")

        patch = __generatePatch(randomizer, settings, hide_settings)
        if patch is None:
            logging.info(f"({settings.seed}) Failed to generate patch in {time.perf_counter() - patch_start_time} seconds!")
            return __generate(randomizer, settings, race, retries + 1, hide_settings)
        else:
            logging.info(f"({settings.seed}) Generated patch in {time.perf_counter() - patch_start_time} seconds!")

        if not race:
            logging.info(f"({settings.seed}) Race Mode off, generating spoiler...")
            spoiler_start_time = time.perf_counter()
            spoiler = __generateSpoiler(randomizer, settings, hide_settings)
            logging.info(f"({settings.seed}) Generated spoiler in {time.perf_counter() - spoiler_start_time} seconds!")
        else:
            logging.info(f"({settings.seed}) Race Mode on, not generating spoiler...")

        if database.enabled:
            logging.info(f"({settings.seed}) Generating permalink...")
            permalink_start_time = time.perf_counter()
            permalink = database.create(patch, spoiler, settings, hide_settings)
            logging.info(
                f"({settings.seed}) Permalink generated in {time.perf_counter() - permalink_start_time} seconds!")

        return Result(patch, spoiler, permalink)

    except Exception as e:
        logging.exception(e)
        return __generate(randomizer, settings, race, retries + 1, hide_settings)

def __generateFileName(settings: Settings, extension: str, obfuscate_options: bool = False) -> str:
    if obfuscate_options:
        file_name = "IoGR_V" + VERSION + "_" + str(settings.seed) + "." + extension
        return file_name
    return generate_filename(settings, extension)

def __generatePatch(randomizer: Randomizer, settings: Settings, hide_settings: bool = False) -> Patch:
    patch_filename = __generateFileName(settings, "sfc", hide_settings)
    patch = randomizer.generate_rom(patch_filename, settings)
    return Patch(patch, patch_filename, VERSION)

def __generateSpoiler(randomizer: Randomizer, settings: Settings, hide_settings: bool = False) -> Spoiler:
    spoiler_filename = __generateFileName(settings, "txt", hide_settings)
    spoiler = randomizer.generate_spoiler()
    return Spoiler(spoiler, spoiler_filename)

if __name__ == "__main__":
    app.debug = config.DEBUG
    app.run(host="0.0.0.0", port=5000)


import requests
import re

url = "https://discordrollout.nekos.sh/api/data/status"

def get_info():
    nitro_info = ""
    nonnitro_info = ""

    response = requests.request("GET", url)

    match = re.search(r'"nitro":"([^"]+)"', response.text)
    if match:
        nitro_info = match.group(1)

    match = re.search(r'"nonnitro":"([^"]+)"', response.text)
    if match:
        nonnitro_info = match.group(1)

    complete = nitro_info + "|" + nonnitro_info

    return complete


import numpy as np
from scipy.stats import norm, truncnorm

def _em_step_body_(args):
    """
    Does a step of the EM algorithm, needed to dereference args to support parallelism
    """
    return _em_step_body(*args)

def _em_step_body(Z, r_lower, r_upper, sigma, num_ord_updates=1):
    """
    Iterate the rows over provided matrix
    """
    num, p = Z.shape
    Z_imp = np.copy(Z)
    C = np.zeros((p,p))
    for i in range(num):
        try:
            c, z_imp, z = _em_step_body_row(Z[i,:], r_lower[i,:], r_upper[i,:], sigma)
        except:
            np.savetxt("Z.txt", Z)
            print(Z)
        Z_imp[i,:] = z_imp
        Z[i,:] = z
        C += c
    return C, Z_imp, Z

def _em_step_body_row(Z_row, r_lower_row, r_upper_row, sigma, num_ord_updates=1):
    """
    The body of the em algorithm for each row
    Returns a new latent row, latent imputed row and C matrix, which, when added
    to the empirical covariance gives the expected covariance
    Args:
        Z_row (array): (potentially missing) latent entries for one data point
        r_lower_row (array): (potentially missing) lower range of ordinal entries for one data point
        r_upper_row (array): (potentially missing) upper range of ordinal entries for one data point
        sigma (matrix): estimate of covariance
        num_ord (int): the number of ordinal columns

    Returns:
        C (matrix): results in the updated covariance when added to the empircal covariance
        Z_imp_row (array): Z_row with latent ordinals updated and missing entries imputed
        Z_row (array): inpute Z_row with latent ordinals updated
    """
    Z_imp_row = np.copy(Z_row)
    p = Z_imp_row.shape[0]
    num_ord = r_upper_row.shape[0]
    C = np.zeros((p,p))

    obs_indices = np.where(~np.isnan(Z_row))[0]

    missing_indices = np.setdiff1d(np.arange(p), obs_indices)
    ord_in_obs = np.where(obs_indices < num_ord)[0]
    ord_obs_indices = obs_indices[ord_in_obs]
    # obtain correlation sub-matrices
    # obtain submatrices by indexing a "cartesian-product" of index arrays
    sigma_obs_obs = sigma[np.ix_(obs_indices,obs_indices)]
    sigma_obs_missing = sigma[np.ix_(obs_indices, missing_indices)]
    sigma_missing_missing = sigma[np.ix_(missing_indices, missing_indices)]

    if len(missing_indices) > 0:
        tot_matrix = np.concatenate((np.identity(len(sigma_obs_obs)), sigma_obs_missing), axis=1)
        intermed_matrix = np.linalg.solve(sigma_obs_obs, tot_matrix)
        sigma_obs_obs_inv = intermed_matrix[:, :len(sigma_obs_obs)]
        J_obs_missing = intermed_matrix[:, len(sigma_obs_obs):]
    else:
        sigma_obs_obs_inv = np.linalg.solve(sigma_obs_obs, np.identity(len(sigma_obs_obs)))

    # initialize vector of variances for observed ordinal dimensions
    var_ordinal = np.zeros(p)

    # OBSERVED ORDINAL ELEMENTS
    # when there is an observed ordinal to be imputed and another observed dimension, impute this ordinal
    if len(obs_indices) >= 2 and len(ord_obs_indices) >= 1:
        for update_iter in range(num_ord_updates):
            # used to efficiently compute conditional mean
            sigma_obs_obs_inv_Z_row = np.dot(sigma_obs_obs_inv, Z_row[obs_indices])
            for ind in range(len(ord_obs_indices)):
                j = obs_indices[ind]
                not_j_in_obs = np.setdiff1d(np.arange(len(obs_indices)),ind)
                v = sigma_obs_obs_inv[:,ind]
                new_var_ij = np.asscalar(1.0/v[ind])

                new_mean_ij = Z_row[j] - new_var_ij*sigma_obs_obs_inv_Z_row[ind]
                mean, var = truncnorm.stats(
                    a=(r_lower_row[j] - new_mean_ij) / np.sqrt(new_var_ij),
                    b=(r_upper_row[j] - new_mean_ij) / np.sqrt(new_var_ij),
                    loc=new_mean_ij,
                    scale=np.sqrt(new_var_ij),
                    moments='mv')
                if np.isfinite(var):
                    var_ordinal[j] = var
                    if update_iter == num_ord_updates - 1:
                        C[j,j] = C[j,j] + var
                if np.isfinite(mean):
                    Z_row[j] = mean

    # MISSING ELEMENTS
    Z_obs = Z_row[obs_indices]
    Z_imp_row[obs_indices] = Z_obs
    if len(missing_indices) > 0:
        Z_imp_row[missing_indices] = np.matmul(J_obs_missing.T,Z_obs)
        # variance expectation and imputation
        if len(ord_obs_indices) >= 1 and len(obs_indices) >= 2 and np.sum(var_ordinal) > 0:
            cov_missing_obs_ord = J_obs_missing[ord_in_obs].T * var_ordinal[ord_obs_indices]
            C[np.ix_(missing_indices, ord_obs_indices)] += cov_missing_obs_ord
            C[np.ix_(ord_obs_indices, missing_indices)] += cov_missing_obs_ord.T
            C[np.ix_(missing_indices, missing_indices)] += sigma_missing_missing - np.matmul(J_obs_missing.T, sigma_obs_missing) \
                                                           + np.matmul(cov_missing_obs_ord, J_obs_missing[ord_in_obs])
        else:
            C[np.ix_(missing_indices, missing_indices)] += sigma_missing_missing - np.matmul(J_obs_missing.T, sigma_obs_missing)
    return C, Z_imp_row, Z_row


#!/usr/bin/python3

import intcode
from collections import defaultdict
from screen import display
canvas = defaultdict(lambda: 0)

code = [int(c) for c in open('day13.txt').read().split(',')]
machine = intcode.IntProc(code)
try:
    while True:
        x, y, t = machine.read(3)
        canvas[(x,y)] = t
except intcode.Terminated:
    print(sum(1 for _, t in canvas.items()
              if t == 2))

cmp = lambda a, b: (a > b) - (a < b)

BLANK = 0
WALL = 1
BLOCK = 2
PADDLE = 3
BALL = 4

# part 2
SYMBOLS = {
    BLANK:  ' ',
    WALL:   '#',
    BLOCK:  '*',
    PADDLE: '_',
    BALL:   'O',
}
code[0] = 2
score = 0
ball = 0
paddle = 0
outbuf = []

import asyncio

@intcode.grouped(3)
async def handle_output(x, y, t):
    global score, ball, paddle
    if x == -1 and y == 0:
        score = t
    else:
        canvas[(x,y)] = t
        if t == BALL:
            ball = x
        elif t == PADDLE:
            paddle = x

async def provide_input():
    return cmp(ball, paddle)

cpu = intcode.AsyncIntCode(code, input=provide_input, output=handle_output)
loop = asyncio.get_event_loop()
try:
    loop.run_until_complete(cpu.run())
    print(score)
finally:
    loop.close()


import matplotlib.pyplot as plt
import numpy as np
import argparse

parser=argparse.ArgumentParser(description="Plot 1d differential scattering cross section output from GGADT")
parser.add_argument('file_name',metavar='ggadt-output-file',type=str, nargs=1, help="a file containing the output of GGADT")

args = parser.parse_args()

file_name = args.file_name[0]

dscs = np.loadtxt(file_name, dtype=np.dtype([('theta', np.float_), ('dCdOm', np.float_)]))
dscs['theta'] *= 206264.806 # convert to arcseconds

# now plot!
f = plt.figure()
ax = f.add_subplot(111)

ax.plot(dscs['theta'], dscs['dCdOm'])
ax.set_xlabel("$\\theta$ [arcseconds]")
ax.set_ylabel("$dQ_{\\rm sca}/d\\Omega$ [sr$^{-1}$]")
ax.set_yscale('log')
plt.show(block=True)


from abc import abstractmethod, ABCMeta

from octobot_commons.logging.logging_util import get_logger

class AbstractWebsocket:
    __metaclass__ = ABCMeta

    def __init__(self, config, exchange_manager):
        self.config = config
        self.exchange_manager = exchange_manager
        self.client = None
        self.name = self.get_name()
        self.logger = get_logger(f"WebSocket - {self.name}")

    # Abstract methods
    @classmethod
    def get_name(cls):
        raise NotImplementedError("get_name not implemented")

    @classmethod
    def has_name(cls, name: str) -> bool:
        raise NotImplementedError("has_name not implemented")

    @staticmethod
    def get_websocket_client(config, exchange_manager):
        raise NotImplementedError("get_websocket_client not implemented")

    @abstractmethod
    def is_handling(self, feed_name):
        raise NotImplementedError("is_handling not implemented")

    @abstractmethod
    async def init_websocket(self, time_frames, trader_pairs):
        raise NotImplementedError("init_websocket not implemented")

    @abstractmethod
    async def start_sockets(self):
        raise NotImplementedError("start_sockets not implemented")

    @abstractmethod
    async def wait_sockets(self):
        raise NotImplementedError("wait_sockets not implemented")

    @abstractmethod
    async def close_and_restart_sockets(self):
        raise NotImplementedError("close_and_restart_sockets not implemented")

    @abstractmethod
    async def stop_sockets(self):
        raise NotImplementedError("stop_sockets not implemented")


import torch
import torch.nn as nn
import torch.nn.functional as F
from model.general.attention.additive import AdditiveAttention

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

class TextEncoder(torch.nn.Module):
    def __init__(self, word_embedding, word_embedding_dim, num_filters,
                 window_size, query_vector_dim, dropout_probability):
        super(TextEncoder, self).__init__()
        self.word_embedding = word_embedding
        self.dropout_probability = dropout_probability
        self.CNN = nn.Conv2d(1,
                             num_filters, (window_size, word_embedding_dim),
                             padding=(int((window_size - 1) / 2), 0))
        self.additive_attention = AdditiveAttention(query_vector_dim,
                                                    num_filters)

    def forward(self, text):
        # batch_size, num_words_text, word_embedding_dim
        text_vector = F.dropout(self.word_embedding(text),
                                p=self.dropout_probability,
                                training=self.training)
        # batch_size, num_filters, num_words_title
        convoluted_text_vector = self.CNN(
            text_vector.unsqueeze(dim=1)).squeeze(dim=3)
        # batch_size, num_filters, num_words_title
        activated_text_vector = F.dropout(F.relu(convoluted_text_vector),
                                          p=self.dropout_probability,
                                          training=self.training)

        # batch_size, num_filters
        text_vector = self.additive_attention(
            activated_text_vector.transpose(1, 2))
        return text_vector

class ElementEncoder(torch.nn.Module):
    def __init__(self, embedding, linear_input_dim, linear_output_dim):
        super(ElementEncoder, self).__init__()
        self.embedding = embedding
        self.linear = nn.Linear(linear_input_dim, linear_output_dim)

    def forward(self, element):
        return F.relu(self.linear(self.embedding(element)))

class NewsEncoder(torch.nn.Module):
    def __init__(self, config, pretrained_word_embedding):
        super(NewsEncoder, self).__init__()
        self.config = config
        if pretrained_word_embedding is None:
            word_embedding = nn.Embedding(config.num_words,
                                          config.word_embedding_dim,
                                          padding_idx=0)
        else:
            word_embedding = nn.Embedding.from_pretrained(
                pretrained_word_embedding, freeze=False, padding_idx=0)
        assert len(config.dataset_attributes['news']) > 0
        text_encoders_candidates = ['title', 'abstract']
        self.text_encoders = nn.ModuleDict({
            name:
            TextEncoder(word_embedding, config.word_embedding_dim,
                        config.num_filters, config.window_size,
                        config.query_vector_dim, config.dropout_probability)
            for name in (set(config.dataset_attributes['news'])
                         & set(text_encoders_candidates))
        })
        category_embedding = nn.Embedding(config.num_categories,
                                          config.category_embedding_dim,
                                          padding_idx=0)
        element_encoders_candidates = ['category', 'subcategory']
        self.element_encoders = nn.ModuleDict({
            name:
            ElementEncoder(category_embedding, config.category_embedding_dim,
                           config.num_filters)
            for name in (set(config.dataset_attributes['news'])
                         & set(element_encoders_candidates))
        })
        if len(config.dataset_attributes['news']) > 1:
            self.final_attention = AdditiveAttention(config.query_vector_dim,
                                                     config.num_filters)

    def forward(self, news):
        """
        Args:
            news:
                {
                    "category": batch_size,
                    "subcategory": batch_size,
                    "title": batch_size * num_words_title,
                    "abstract": batch_size * num_words_abstract,
                }
        Returns:
            (shape) batch_size, num_filters
        """
        text_vectors = [
            encoder(news[name].to(device))
            for name, encoder in self.text_encoders.items()
        ]
        element_vectors = [
            encoder(news[name].to(device))
            for name, encoder in self.element_encoders.items()
        ]

        all_vectors = text_vectors + element_vectors

        if len(all_vectors) == 1:
            final_news_vector = all_vectors[0]
        else:
            final_news_vector = self.final_attention(
                torch.stack(all_vectors, dim=1))
        return final_news_vector


from django.conf.urls import url, include
from . import views
from django.views.generic.edit import CreateView
from django.contrib.auth.forms import UserCreationForm

app_name = 'movieratings'
urlpatterns = [
    url(r'^$', views.index, name='index'),
    url(r'^toprated', views.TopRated.as_view(), name='toprated'),
    # url(r'^$', views.IndexView.as_view(), name='index'),
    url(r'^search/$', views.SearchView.as_view(), name='search'),
    url(r'^genre/$', views.GenreView.as_view(), name='genre'),
    url(r'^allmovies/', views.AllMovies.as_view(), name='all_movies'),
    url(r'^movie/(?P<pk>[0-9]+)/$', views.MovieDetail.as_view(), name='movie_detail'),
    url(r'^rater/(?P<pk>[0-9]+)/$', views.RaterDetail.as_view(), name='rater_detail'),
    url(r'^newrating/', views.NewRating.as_view(), name='new_rating'),
    url(r'^register/$', views.register, name='register'), # these 3: conflicting with prior auths urls?
    url(r'^login/$', views.user_login, name='login'),
    url(r'^logout/$', views.user_logout, name='logout'),
    # url(r'^index/$', views.IndexView.as_view() name='index'),
    ]


from django.shortcuts import render_to_response, redirect
from Inventar.models import Geraet, GeraeteTyp
from django.db.models import Q
from django.template import RequestContext
from django.contrib.auth.decorators import login_required
from ClientManagement.models import ComputerType, WindowsKey, Computer
from ClientManagement.forms import ComputerTypeForm, WindowsKeyForm, ComputerForm
from django.contrib import messages
import re

@login_required
def index(request):
	computers = Computer.objects.all().order_by('name')
	types = ComputerType.objects.all().order_by('vendor', 'name')
	keys = WindowsKey.objects.all()

	return render_to_response('clientmanagement/index.html', {'computers': computers, 'types': types, 'keys': keys, 'active': 'clientManagement'}, context_instance=RequestContext(request))

@login_required
def computer_edit(request, id):
	try:
		c = Computer.objects.get(pk=id)
	except Computer.DoesNotExist:
			return render_to_response('clientmanagement/404.html')

	if request.method == 'POST':
		form = ComputerForm(request.POST, instance=c)
		if form.is_valid():
			c = form.save()
			messages.add_message(request, messages.SUCCESS, 'Der Computer "' + c.name + '" wurde erfolgreich bearbeitet.')
			return redirect('/pool/')
	else:
		form = ComputerForm(instance=c)

	return render_to_response('clientmanagement/computer_form.html', {'form': form, 'action': 'Speichern', 'active': 'clientManagement'}, context_instance=RequestContext(request))

@login_required
def computer_create(request):
	if request.method == 'POST':
		form = ComputerForm(request.POST)
		if form.is_valid():
			c = form.save()
			messages.add_message(request, messages.SUCCESS, 'Der Computer "' + c.name + '" wurde erfolgreich erstellt.')
			return redirect('/pool/')
	else:
		form = ComputerForm()

	return render_to_response('clientmanagement/computer_form.html', {'form': form, 'action': 'Erstellen', 'active': 'clientManagement'}, context_instance=RequestContext(request))

@login_required
def type_edit(request, id):
	try:
		t = ComputerType.objects.get(pk=id)
	except ComputerType.DoesNotExist:
			return render_to_response('clientmanagement/404.html')

	if request.method == 'POST':
		form = ComputerTypeForm(request.POST, instance=t)
		if form.is_valid():
			t = form.save()
			messages.add_message(request, messages.SUCCESS, 'Der Computertyp "' + t.name + '" wurde erfolgreich bearbeitet.')
			return redirect('/pool/')
	else:
		form = ComputerTypeForm(instance=t)

	return render_to_response('clientmanagement/type_form.html', {'form': form, 'action': 'Speichern', 'active': 'clientManagement'}, context_instance=RequestContext(request))

@login_required
def type_create(request):
	if request.method == 'POST':
		form = ComputerTypeForm(request.POST)
		if form.is_valid():
			t = form.save()
			messages.add_message(request, messages.SUCCESS, 'Der Computertyp "' + t.name + '" wurde erfolgreich erstellt.')
			return redirect('/pool/')
	else:
		form = ComputerTypeForm()

	return render_to_response('clientmanagement/type_form.html', {'form': form, 'action': 'Erstellen', 'active': 'clientManagement'}, context_instance=RequestContext(request))

@login_required
def key_create(request):
	if request.method == 'POST':
		form = WindowsKeyForm(request.POST)
		if form.is_valid():
			k = form.save()
			messages.add_message(request, messages.SUCCESS, 'Der Windows Key wurde erfolgreich erstellt.')
			return redirect('/pool/')
	else:
		form = WindowsKeyForm()

	return render_to_response('clientmanagement/key_form.html', {'form': form, 'action': 'Erstellen', 'active': 'clientManagement'}, context_instance=RequestContext(request))

@login_required
def key_edit(request, id):
	try:
		k = WindowsKey.objects.get(pk=id)
	except WindowsKey.DoesNotExist:
			return render_to_response('clientmanagement/404.html')

	if request.method == 'POST':
		form = WindowsKeyForm(request.POST, instance=k)
		if form.is_valid():
			k = form.save()
			messages.add_message(request, messages.SUCCESS, 'Der Windows Key wurde erfolgreich bearbeitet.')
			return redirect('/pool/')
	else:
		form = WindowsKeyForm(instance=k)

	return render_to_response('clientmanagement/key_form.html', {'form': form, 'action': 'Speichern', 'active': 'clientManagement'}, context_instance=RequestContext(request))

@login_required
def dhcp_config(request):
	computers = Computer.objects.filter(~Q(name='') and ~Q(ip_address=''))

	computer_list = []
	for c in computers:
		if re.search('([a-fA-F0-9]{2}[:|\-]?){6}', c.mac_address):
			computer_list.append(c)

	return render_to_response('clientmanagement/dhcp_config.txt', {'computers': computer_list}, content_type='text/plain', context_instance=RequestContext(request))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Aug  4 02:50:47 2018

@author: Kazuki
"""

import numpy as np
import pandas as pd
import gc
import os
from multiprocessing import Pool, cpu_count
NTHREAD = cpu_count()
import utils_agg
import utils
#utils.start(__file__)
#==============================================================================
PREF = 'f412_'

KEY = 'SK_ID_CURR'

SHIFT = 1

month   =  12*1

#os.system(f'rm ../feature/t*_{PREF}*')
# =============================================================================
# curr
# =============================================================================
col_app_money = ['app_AMT_INCOME_TOTAL', 'app_AMT_CREDIT', 'app_AMT_ANNUITY', 'app_AMT_GOODS_PRICE']
col_app_day = ['app_DAYS_BIRTH', 'app_DAYS_EMPLOYED', 'app_DAYS_REGISTRATION', 'app_DAYS_ID_PUBLISH', 'app_DAYS_LAST_PHONE_CHANGE']

def get_trte():
    usecols = ['SK_ID_CURR', 'AMT_INCOME_TOTAL', 'AMT_CREDIT', 'AMT_ANNUITY', 'AMT_GOODS_PRICE']
    usecols += ['DAYS_BIRTH', 'DAYS_EMPLOYED', 'DAYS_REGISTRATION', 'DAYS_ID_PUBLISH', 'DAYS_LAST_PHONE_CHANGE']
    rename_di = {
                 'AMT_INCOME_TOTAL':       'app_AMT_INCOME_TOTAL',
                 'AMT_CREDIT':             'app_AMT_CREDIT',
                 'AMT_ANNUITY':            'app_AMT_ANNUITY',
                 'AMT_GOODS_PRICE':        'app_AMT_GOODS_PRICE',
                 'DAYS_BIRTH':             'app_DAYS_BIRTH',
                 'DAYS_EMPLOYED':          'app_DAYS_EMPLOYED',
                 'DAYS_REGISTRATION':      'app_DAYS_REGISTRATION',
                 'DAYS_ID_PUBLISH':        'app_DAYS_ID_PUBLISH',
                 'DAYS_LAST_PHONE_CHANGE': 'app_DAYS_LAST_PHONE_CHANGE',
                 }
    trte = pd.concat([pd.read_csv('../input/application_train.csv.zip', usecols=usecols).rename(columns=rename_di),
                      pd.read_csv('../input/application_test.csv.zip',  usecols=usecols).rename(columns=rename_di)],
                      ignore_index=True)
    return trte

# =============================================================================
#
# =============================================================================
df = pd.read_csv('../input/credit_card_balance.csv.zip')
df.sort_values(['SK_ID_PREV', 'MONTHS_BALANCE'], inplace=True)
df.reset_index(drop=True, inplace=True)

df = pd.merge(df, get_trte(), on='SK_ID_CURR', how='left')

df['emp_month'] = df['app_DAYS_EMPLOYED']/30

col = ['SK_ID_CURR', 'AMT_BALANCE',
       'AMT_CREDIT_LIMIT_ACTUAL', 'AMT_DRAWINGS_ATM_CURRENT',
       'AMT_DRAWINGS_CURRENT', 'AMT_DRAWINGS_OTHER_CURRENT',
       'AMT_DRAWINGS_POS_CURRENT', 'AMT_INST_MIN_REGULARITY',
       'AMT_PAYMENT_CURRENT', 'AMT_PAYMENT_TOTAL_CURRENT',
       'AMT_RECEIVABLE_PRINCIPAL', 'AMT_RECIVABLE', 'AMT_TOTAL_RECEIVABLE',
       'CNT_DRAWINGS_ATM_CURRENT', 'CNT_DRAWINGS_CURRENT',
       'CNT_DRAWINGS_OTHER_CURRENT', 'CNT_DRAWINGS_POS_CURRENT',
       'CNT_INSTALMENT_MATURE_CUM', 'NAME_CONTRACT_STATUS', 'SK_DPD',
       'SK_DPD_DEF']

df_before = df.loc[df.MONTHS_BALANCE.between(df['emp_month']-month, df['emp_month']), col]
df_after  = df.loc[df.MONTHS_BALANCE.between(df['emp_month'], df['emp_month']+month), col]

df_before = df_before.groupby(KEY).mean()#.add_prefix('bf_')
df_after  = df_after.groupby(KEY).mean()#.add_prefix('af_')

df_ratio = pd.DataFrame()
for c in df_before.columns:
    df_ratio[c+'_ratio'] = df_before[c] / df_after[c]

df_ratio.reset_index(inplace=True)
df_ratio.replace(np.inf, np.nan, inplace=True)
df_ratio.replace(-np.inf, np.nan, inplace=True)

# =============================================================================
# output
# =============================================================================
train = utils.load_train([KEY])
test = utils.load_test([KEY])

tmp = pd.merge(train, df_ratio, on=KEY, how='left').drop(KEY, axis=1)
utils.to_feature(tmp.add_prefix(PREF), '../feature/train')

tmp = pd.merge(test, df_ratio, on=KEY, how='left').drop(KEY, axis=1)
utils.to_feature(tmp.add_prefix(PREF),  '../feature/test')

#==============================================================================
utils.end(__file__)