from MemoryCache import MemoryCache
from MainMemory import MainMemory
import os

from bitarray import bitarray

mainMemory = MainMemory()

memoryCache = MemoryCache()

def readContentFromMemory():
    memoryAdress = input("Digite o endereço de memória em hexa. Ex: 0x12: ")
    memoryAdress = verificaEndereco(memoryAdress, 0)
    adressInteger = int(memoryAdress, 16)
    adressBits = bin(adressInteger)[2:]
    arrayBits = bitarray(adressBits)
    arrayAsString = arrayBits.to01()
    acertos = 0
    faltas = 0

    # Verificar se esta na cache
    bloco = memoryCache.isHere(arrayBits)

    if bloco == None:
        bloco = memoryCache.leBlocoMemoria(arrayBits, mainMemory)
        faltas = 1
        print("Valor não encontrado na memória cache!")
    else:
        acertos = 1
        print("Valor encontrado na memória cache!")

    #print("Conteudo lido para a memória com sucesso!")

    #ler apenas a celula certa a partir do deslocamento do bloco
    info = memoryCache.getInfoFromBits(arrayBits)
    conjunto = info['conjunto']
    label = info['label']
    quadro = memoryCache.thisQuadro(conjunto, label)
    deslocamento = bloco.leDeslocamento(arrayAsString[-2:])

    print("Endereço:", hex(adressInteger))
    print("O Bloco é:", label)
    print("O Quadro é:", quadro)
    print("O Conjunto é:", conjunto)
    print("O Deslocamento é:", deslocamento)
    print("Dados: ", end = "")
    bloco.line[deslocamento].printCell()

    return {
        "acertos": acertos,
        "faltas": faltas
    }

def writeContentInMemory():
    memoryAdress = input("Digite o endereço de memória em hexa. Ex: 0x12: ")
    memoryAdress = verificaEndereco(memoryAdress, 0)
    dados = input("Digite os dados que serão armazenados: ")
    bitsDados = verificaEntrada(dados, 0)
    acertos = 0
    faltas = 0

    adressInteger = int(memoryAdress, 16)
    adressBits = bin(adressInteger)[2:]
    arrayBits = bitarray(adressBits)

    # Verificar se esta na memoria cache
    bloco = memoryCache.isHere(arrayBits)

    # Se não, ler o endereço de memoria na cache
    if bloco == None:
        bloco = memoryCache.leBlocoMemoria(arrayBits, mainMemory)
        faltas += 1
    else:
        acertos += 1

    # Escrever os dados na cache
    memoryCache.writeData(arrayBits, bitsDados)

    return {
        "acertos": acertos,
        "faltas": faltas
        }

def findTwoscomplement(valor):
    n = len(valor)

    # Traverse the valoring to get first
    # '1' from the last of valoring
    i = n - 1
    while(i >= 0):
        if (valor[i] == '1'):
            break

        i -= 1

    # If there exists no '1' concatenate 1
    # at the starting of valoring
    if (i == -1):
        return '1'+valor

    # Continue traversal after the
    # position of first '1'
    k = i - 1
    while(k >= 0):

        # Just flip the values
        if (valor[k] == '1'):
            valor = list(valor)
            valor[k] = '0'
            valor = ''.join(valor)
        else:
            valor = list(valor)
            valor[k] = '1'
            valor = ''.join(valor)

        k -= 1
    valor = complete8Bits(valor)
    # return the modified valoring
    return valor

def complete8Bits(entrada):
    aux = ''
    i = 8-len(entrada)
    for _ in range(0, i):
        aux+='0'
    aux += entrada
    return aux

def verificaEndereco(entrada, flag):
    if(flag == 1):
        clearConsole()
        entrada = input("Endereço inválido!\nDigite um endereço entre 0x00(dec=0) a 0x7f(dec=127): ")
    try:
        hexa = int(entrada, 16)
        if hexa >= 0 and hexa <= 127 : # 128 células
            return entrada
        else:
            verificaEndereco(entrada, 1)
    except:
        verificaEndereco(entrada, 1)

def clearConsole():
    os.system('cls' if os.name == 'nt' else 'clear')

def verificaEntrada(entrada, flag):
    if(flag == 1):
        clearConsole()
        entrada = input("Valor inválido!\nDigite um inteiro entre -128 a 127 ou dois caracteres(limite == 8 bits): ")
    try:
        inteiro = int(entrada)
        if inteiro >= -128 and inteiro <= 127 : # 8 bits
            entrada = bin(inteiro)
            entrada = findTwoscomplement(entrada[2:])
            return entrada
        else:
            verificaEntrada(entrada, 1)
    except:
        if len(entrada) > 2:                              # 8 bits
            verificaEntrada(entrada, 1)
        else:
            saida = bin(ord(entrada[0]))
            saida = saida[2:]
            saida += bin(ord(entrada[1]))
            saida = saida.replace('0b', '')
            return saida
            # converter string

def maiorLength(valor0, valor1):
    try:
        a = len(str(valor0))
        b = len(str(valor1))
        return a if a > b else b
    except:
        return 0

def padronaizaPorcentagem(porcentagem):
    dec = ""
    #try:
    aux = str(round(porcentagem, 2)).split(".")
    try:
        for _ in range(0, len(aux[1])):
            dec += "0"
        porcentagem = str(aux[0]+"."+aux[1]+dec+"%")
        return porcentagem
    except:
        porcentagem = (aux[0]+".00%")
        return porcentagem

def statistics(numeroEscrita, numeroLeitura, acertosEscrita, acertosLeitura, faltasEscrita, faltasLeitura):
    acertosGeralPorcentagem = 0
    faltasGeralPorcentagem = 0
    porcentagemEscritaAcertos = 0
    porcentagemEscritaFaltas = 0
    porcentagemLeituraAcertos = 0
    porcentagemLeituraFaltas = 0
    numeroGeral = 0

    if(numeroEscrita != 0):
        porcentagemEscritaAcertos = (acertosEscrita/numeroEscrita) * 100
        porcentagemEscritaFaltas = (faltasEscrita/numeroEscrita) * 100
        numeroGeral += numeroEscrita

    if(numeroLeitura != 0):
        porcentagemLeituraAcertos = (acertosLeitura/numeroLeitura) * 100
        porcentagemLeituraFaltas = (faltasLeitura/numeroLeitura) * 100
        numeroGeral += numeroLeitura

    acertosGeral = acertosEscrita + acertosLeitura

    if((numeroEscrita + numeroLeitura) != 0):
        #print("OLA")
        #print("Acertos gerais ", acertosGeral)
        acertosGeralPorcentagem = (acertosEscrita + acertosLeitura)/(numeroEscrita + numeroLeitura) * 100
        faltasGeralPorcentagem = (faltasEscrita + faltasLeitura)/(numeroEscrita + numeroLeitura) * 100

    faltasGeral = faltasEscrita + faltasLeitura

    clearConsole()

    print("\n{:-^63s}".format("ESCRITA"))
    exibe0 = "Quantidade de Escrita: "+str(numeroEscrita)
    print("|{: ^61s}|".format(exibe0))
    exibe0 = "Acertos absolutos: "+str(acertosEscrita)
    exibe1 = "Porcentagem: "+padronaizaPorcentagem(porcentagemEscritaAcertos)
    print("|{: ^30s}|{: ^30s}|".format(exibe0, exibe1))
    exibe0 = "Faltas absolutas: "+str(faltasEscrita)
    exibe1 = "Porcentagem: "+padronaizaPorcentagem(porcentagemEscritaFaltas)
    print("|{: ^30s}|{: ^30s}|".format(exibe0, exibe1))

    print("\n{:-^63s}".format("LEITURA"))
    exibe0 = "Quantidade de Leitura: "+str(numeroLeitura)
    print("|{: ^61s}|".format(exibe0))
    exibe0 = "Acertos absolutos: "+str(acertosLeitura)
    exibe1 = "Porcentagem: "+padronaizaPorcentagem(porcentagemLeituraAcertos)
    print("|{: ^30s}|{: ^30s}|".format(exibe0, exibe1))
    exibe0 = "Faltas absolutas: "+str(faltasLeitura)
    exibe1 = "Porcentagem: "+padronaizaPorcentagem(porcentagemLeituraFaltas)
    print("|{: ^30s}|{: ^30s}|".format(exibe0, exibe1))

    print("\n{:-^63s}".format("GERAL"))
    exibe0 = "Quantidade Geral: "+str(numeroGeral) # REVER ISSO
    print("|{: ^61s}|".format(exibe0))
    #maiorLen = maiorLength(acertosGeral, acertosGeralPorcentagem)
    exibe0 = "Acertos absolutos: "+str(acertosGeral)
    exibe1 = "Porcentagem: "+padronaizaPorcentagem(acertosGeralPorcentagem)
    print("|{: ^30s}|{: ^30s}|".format(exibe0, exibe1))
    exibe0 = "Faltas absolutas: "+str(faltasGeral)
    exibe1 = "Porcentagem: "+padronaizaPorcentagem(faltasGeralPorcentagem)
    print("|{: ^30s}|{: ^30s}|".format(exibe0, exibe1))

    print("\n")

    pass

# Main

numeroEscrita = 0
numeroLeitura = 0
acertosEscrita = 0
acertosLeitura = 0
faltasEscrita = 0
faltasLeitura = 0

clearConsole()

messageInput = "Digite:\n1 - Ler o conteúdo de um endereço de memória\n"
messageInput += "2 - Escrever em um determinado endereço de memória\n"
messageInput += "3 - Estatisticas\n"
messageInput += "4 - Mostrar toda memória principal\n"
messageInput += "5 - Mostrar toda memória cache\n"
messageInput += "6 - Encerrar o programa\n"

#info = ['00000000', '00000001', '00000010', '00000011']

while True:
    option = input(messageInput)

    try:
        option = int(option)
    except:
        clearConsole()
        print("Entrada inválida, tente novamente")
        continue

    if option == 1:
        aux = readContentFromMemory()
        acertosLeitura += aux['acertos']
        faltasLeitura += aux['faltas']
        numeroLeitura += 1

    elif option == 2:
        aux = writeContentInMemory()
        acertosEscrita += aux['acertos']
        faltasEscrita += aux['faltas']

        numeroEscrita += 1

    elif option == 3:
        statistics(numeroEscrita, numeroLeitura, acertosEscrita, acertosLeitura, faltasEscrita, faltasLeitura)
    elif option == 4:
        mainMemory.printAllCellsMain()

    elif option == 5:
        #print all from cache
        memoryCache.printAllCellsCache()
    elif option == 6:
        break


import fileinput
import json
import logging

import torch
from torch.utils.data import Dataset
from transformers import AutoConfig

config = AutoConfig.from_pretrained(
    "THUDM/chatglm-6b", trust_remote_code=True, revision="main"
)

MAX_LENGTH = 2048
pad_to = 1
device = "cuda"
eos_id = config.eos_token_id
pad_token_id = config.pad_token_id

class DocMTDataset(Dataset):
    def __init__(self, pairs) -> None:
        super().__init__()
        self.pairs = pairs

    def __getitem__(self, index):
        import ipdb
        ipdb.set_trace()
        return self.pairs[index]

    def __len__(self):
        return len(self.pairs)

def collate_fn(batch):
    global device
    input_ids = []
    labels = []
    position_ids = []

    _max_length = max([len(obj["prompt"]) + len(obj["completion"]) for obj in batch])
    _max_length = (_max_length // pad_to + (_max_length % pad_to > 0)) * pad_to

    attention_mask = torch.ones((len(batch), _max_length, _max_length), device=device)
    attention_mask.tril_()

    for i, obj in enumerate(batch):
        context_length = obj["prompt"].index(130004)
        attention_mask[i, :, :context_length] = 1

        to_pad = _max_length - len(obj["prompt"]) - len(obj["completion"])

        input_ids.append(obj["prompt"] + obj["completion"] + [pad_token_id] * to_pad)

        position_ids.append(
            torch.stack(
                [
                    torch.arange(0, _max_length, device=device),
                    torch.concat(
                        [
                            torch.zeros(context_length - 1, device=device),
                            torch.arange(
                                0, _max_length - context_length + 1, device=device
                            ),
                        ]
                    ),
                ]
            ).long()
        )

        labels.append(
            torch.tensor(
                [-100] * len(obj["prompt"]) + obj["completion"] + [-100] * to_pad,
                device=device,
            ).long()
        )

    attention_mask.unsqueeze_(1)
    attention_mask = (attention_mask < 0.5).bool()
    return {
        "input_ids": torch.tensor(input_ids).long(),
        "attention_mask": attention_mask,
        "labels": torch.stack(labels),
        "position_ids": torch.stack(position_ids),
    }

def encode_pairs(pairs, tokenizer, with_eos=True):
    prompt_ids = tokenizer.batch_encode_plus([pair["prompt"] for pair in pairs])[
        "input_ids"
    ]
    completion_ids = tokenizer.batch_encode_plus(
        [pair["completion"] for pair in pairs], add_special_tokens=False
    )["input_ids"]
    if with_eos:
        pairs_encoded = [
            {"prompt": prompt_ids[i], "completion": completion_ids[i] + [eos_id]}
            for i in range(len(pairs))
        ]
    else:
        pairs_encoded = [
            {"prompt": prompt_ids[i], "completion": completion_ids[i]}
            for i in range(len(pairs))
        ]
    return pairs_encoded

PROMPT_DICT = {
    "prompt_input": ("{instruction}\n\n### 输入文本:\n{input}\n\n### 输出文本:\n"),
}

def load(path, tokenizer, with_eos=True):

    pairs = []

    logging.info("Loading data from %s", path)
    for line in fileinput.input(path):
        line = json.loads(line)
        prompt = PROMPT_DICT["prompt_input"].format_map(line)
        completion = line["output"]
        pairs.append({"prompt": prompt, "completion": completion})

    logging.info(f"Encoding {len(pairs)} pairs")
    pairs = encode_pairs(pairs, tokenizer, with_eos=with_eos)
    logging.info(f"Loaded {len(pairs)} pairs from {path}")
    return pairs


import os
import time
import torch
import datetime
import visdom

import torch.nn as nn
from torch.autograd import Variable
from torchvision.utils import save_image

from dcgan_models import Generator, Discriminator
from utils import *

import matplotlib.pyplot as plt

class Trainer(object):

    def setup_seed(seed):
        torch.manual_seed(seed)
        torch.cuda.manual_seed_all(seed)
        torch.backends.cudnn.deterministic = True

    # 设置随机数种子
    setup_seed(1000)

    def __init__(self, data_loader, config):

        # history
        # self.history_d_loss_fake = []
        # self.history_d_out_real = []
        self.history_d_loss = []
        self.history_g_loss_fake = []

        # Data loader
        self.data_loader = data_loader

        # Model hyper-parameters
        self.imsize = config.imsize                      # [64]
        # self.g_num = config.g_num                        # 5
        self.z_dim = config.z_dim                        # 128
        self.parallel = config.parallel

        self.channels = config.channels
        self.total_step = config.total_step              # [100000]
        # self.d_iters = config.d_iters                    # 5
        self.batch_size = config.batch_size              # 64
        self.num_workers = config.num_workers            # 2
        self.g_lr = config.g_lr                          # 0.001
        self.d_lr = config.d_lr                          # 0.004
        self.beta1 = config.beta1                        # 0.0  这两个β就是 《lhy 2020.08-09 Optimization》 第16页的那 2 个参数
        self.beta2 = config.beta2                        # 0.9
        self.pretrained_model = config.pretrained_model  # [25536]

        self.dataset = config.dataset                    # [celeb]
        self.use_tensorboard = config.use_tensorboard    # type=str2bool, default=False
        self.image_path = config.image_path              # ./data
        self.log_path = config.log_path                  # ./log
        self.model_save_path = config.model_save_path    # ./models
        self.sample_path = config.sample_path            # ./samples
        self.log_step = config.log_step                  # [10]
        self.sample_step = config.sample_step            # [400]
        self.model_save_step = config.model_save_step    # [1.0]
        self.version = config.version                    # [dcgan_10]

        # Path
        self.log_path = os.path.join(config.log_path, self.version)
        self.sample_path = os.path.join(config.sample_path, self.version)
        self.model_save_path = os.path.join(config.model_save_path, self.version)

        self.build_model()                               # 下面的 build_model(self)方法

        if self.use_tensorboard:
            self.build_tensorboard()

        # Start with trained model
        if self.pretrained_model:
            self.load_pretrained_model()

    def train(self):
        # Data iterator 数据迭代器
        data_iter = iter(self.data_loader)
        step_per_epoch = len(self.data_loader)  # 21551/64 = 336
        model_save_step = int(self.model_save_step * step_per_epoch) # 1*336 = 336

        # Fixed input for debugging [64个这个 → [128个这个 → 标准正态分布出来的数] ]
        # fixed_z = tensor2var(torch.randn(self.batch_size, self.z_dim))
        fixed_z = tensor2var(torch.randn(self.batch_size, self.z_dim)) # 转成 cuda() 数据

        # Start with trained model
        if self.pretrained_model:
            start = self.pretrained_model + 1
        else:
            start = 0

        # Start time
        start_time = time.time()
        # Visulization
        viz = visdom.Visdom() # python -m visdom.server
        viz.line([[0, 0]], [0], win='loss', opts=dict(title='loss', legend=['D_loss', 'G_loss']))
        for step in range(start, self.total_step):

            # ================== Train D ================== #

            # model.train() 启用 Batch Normalization 和 Dropout。 如果模型中有BN层(Batch Normalization）和Dropout，需要在训练时添加model.train()。
            # model.train()是保证BN层能够用到每一批数据的均值和方差。对于Dropout，model.train()是随机取一部分网络连接来训练更新参数。
            # model.eval()则是不启用
            self.D.train()
            self.G.train()

            try:
                # 64,3,64,64
                real_images, _ = next(data_iter) # 如果可以，用下一波数据训练
            except:
                data_iter = iter(self.data_loader)
                real_images, _ = next(data_iter)

            # Compute loss with real images
            real_images = tensor2var(real_images) # 转成cuda数据
            d_out_real = self.D(real_images) # 让D认识真图
            # 使用铰链损失函数 得到D对真实图片的loss
            d_loss_real = torch.nn.ReLU()(1.0 - d_out_real).mean()

            # apply Gumbel Softmax
            # z = tensor2var(torch.randn(real_images.size(0), self.z_dim))
            # real_images.size() torch.Size([64,3,64,64])
            z = tensor2var(torch.randn(real_images.size(0), self.z_dim)) # 生成cuda随机噪声
            fake_images = self.G(z) # 用于产生假图
            d_out_fake = self.D(fake_images) # 让D去分辨这些假图
            # 使用铰链损失函数 得到D对虚假图片的loss
            d_loss_fake = torch.nn.ReLU()(1.0 + d_out_fake).mean()

            # Backward + Optimize
            d_loss = d_loss_real + d_loss_fake  # D的总loss
            self.reset_grad()                   # 梯度归零
            d_loss.backward()                   # 自动计算梯度
            self.d_optimizer.step()             # 然后优化

            # ================== Train G and gumbel ================== #
            # Create random noise
            z = tensor2var(torch.randn(real_images.size(0), self.z_dim))
            fake_images = self.G(z)

            # Compute loss with fake images
            g_out_fake = self.D(fake_images)  # batch x n
            g_loss_fake = - g_out_fake.mean()

            self.reset_grad()
            g_loss_fake.backward()
            self.g_optimizer.step()

            # # 记录一些history data
            # # self.history_d_loss_fake.append(d_loss_fake.item())
            # # self.history_d_out_real.append(d_loss_real.item())
            # self.history_d_loss.append(d_loss.item())
            # self.history_g_loss_fake.append(g_loss_fake.item())

            # Print out log info
            if (step + 1) % self.log_step == 0:
                viz.line([[d_loss.item(), g_loss_fake.item()]], [step], win='loss', update='append')
                elapsed = time.time() - start_time
                elapsed = str(datetime.timedelta(seconds=elapsed))
                print("Elapsed [{}], step [{}/{}], d_loss_real: {:.4f}".format(elapsed, step + 1,self.total_step,d_loss_real.item()))

            # Sample images
            if (step + 1) % self.sample_step == 0:
                fake_images = self.G(fixed_z)
                save_image(denorm(fake_images.data[:64]),
                           os.path.join(self.sample_path, '{}_fake.jpg'.format(step + 1)))
                # # 独立小图
                # for i in range(0, fake_images.size(0)):
                #     save_image(denorm(fake_images.data[i]),os.path.join(self.sample_path, '{}_fake{}.png'.format(step + 1, i + 1)))

            if (step+1) % model_save_step==0:
                torch.save(self.G.state_dict(),
                           os.path.join(self.model_save_path, '{}_G.pth'.format(step + 1)))
                torch.save(self.D.state_dict(),
                           os.path.join(self.model_save_path, '{}_D.pth'.format(step + 1)))

    def build_model(self):

        self.G = Generator(self.imsize, self.z_dim).cuda()
        self.D = Discriminator(self.imsize, self.channels).cuda()
        if self.parallel:                                # 因为 Flase ，所以不使用多个GPU
            self.G = nn.DataParallel(self.G)
            self.D = nn.DataParallel(self.D)

        # Loss and optimizer
        # self.g_optimizer = torch.optim.Adam(self.G.parameters(), self.g_lr, [self.beta1, self.beta2])
        self.g_optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, self.G.parameters()), self.g_lr, [self.beta1, self.beta2])
        self.d_optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, self.D.parameters()), self.d_lr, [self.beta1, self.beta2])

        self.c_loss = torch.nn.CrossEntropyLoss()
        # print networks
        print(self.G)
        print(self.D)

    def build_tensorboard(self):
        from logger import Logger
        self.logger = Logger(self.log_path)

    def load_pretrained_model(self):
        self.G.load_state_dict(torch.load(os.path.join(
            self.model_save_path, '{}_G.pth'.format(self.pretrained_model))))
        self.D.load_state_dict(torch.load(os.path.join(
            self.model_save_path, '{}_D.pth'.format(self.pretrained_model))))
        print('loaded trained models (step: {})..!'.format(self.pretrained_model))

    def reset_grad(self):
        """根据pytorch中的backward()函数的计算，当网络参量进行反馈时，梯度是被积累的而不是被替换掉；但是在每一个batch时毫无疑问并不需要将两个batch的梯度混合起来累积，因此这里就需要每个batch设置一遍zero_grad 了"""
        self.d_optimizer.zero_grad()
        self.g_optimizer.zero_grad()

    # 这个data_iter作用机理还没摸明白，以后研究
    def save_sample(self, data_iter):
        real_images, _ = next(data_iter)
        save_image(denorm(real_images), os.path.join(self.sample_path, 'real.png'))


from nemo.collections import nlp as nemo_nlp
from nemo.utils.exp_manager import exp_manager
from nemo.collections.nlp.parts.nlp_overrides import NLPDDPPlugin
import traceback

import os
import wget
import torch
import pytorch_lightning as pl
from omegaconf import OmegaConf

from pathlib import Path
import argparse

import sys
sys.path.append(Path(__file__).parent.absolute())

from new_bio_eval import BioEval

def load_gs_labels(label_file):
  labels = []
  with open(label_file, "r") as f:
    lines = f.readlines()
  for line in lines:
    line = line.strip()
    if len(line) < 1:
      continue
    labels.append(line.split(" "))

  return labels

def predict_in_sent(text_file, model_ner, bz=32, combine=True):
    idx2label = {v: k for k, v in model_ner.cfg.label_ids.items()}

    res = []
    with open(text_file, "r") as f:
        lines = f.readlines()

    ll = [len(line.split(" ")) for line in lines]
    all_preds = model_ner._infer(lines, bz)

    for idx in ll:
        tmp = [idx2label[each] for each in all_preds[:idx]]
        if combine:
            res.append(" ".join(tmp))
        else:
            res.append(tmp)
        all_preds = all_preds[idx:]

    return res

def main(args):
    config_path = args.ner_config
    config = OmegaConf.load(config_path)

    config.trainer.gpus = args.gpus
    print(OmegaConf.to_yaml(config))

    trainer = pl.Trainer(plugins=[NLPDDPPlugin()], **config.trainer)
    exp_dir = exp_manager(trainer, config.get("exp_manager", None))
    model_ner = None

    if args.do_train:
        if not config.pretrained_model:
            model_ner = nemo_nlp.models.TokenClassificationModel(cfg=config.model, trainer=trainer)
        else:
            model_ner = nemo_nlp.models.TokenClassificationModel.restore_from(config.pretrained_model)

        trainer.fit(model_ner)

        try:
            if config.model.nemo_path:
                model_ner.save_to(config.model.nemo_path)
        except:
            traceback.print_exc()
            config.model.nemo_path = Path(str(exp_dir)) / "checkpoints/default.nemo"

        dev_text = Path(config.model.dataset.data_dir) / config.model.test_ds.text_file
        dev_label = Path(config.model.dataset.data_dir) / config.model.test_ds.labels_file
        dev_preds = predict_in_sent(dev_text, model_ner, bz=config.model.test_ds.batch_size, combine=False)

        try:
            dev_gs = load_gs_labels(dev_label)
            bio_eval = BioEval()
            bio_eval.eval_mem(dev_gs, dev_preds, do_flat=False)
            bio_eval.show_evaluation()
        except:
            pass

        dev_preds = predict_in_sent(dev_text, model_ner, bz=config.model.test_ds.batch_size, combine=True)
        pout = Path(args.pred_output)
        pout.mkdir(exist_ok=True)
        ofn = pout / "labels_test.txt"
        with open(ofn, "w") as f:
            f.write("\n".join(dev_preds))

    if args.do_pred and args.pred_output:
        trainer = pl.Trainer(plugins=[NLPDDPPlugin()], **config.trainer)
        exp_dir = exp_manager(trainer, config.get("exp_manager", None))

        try:
            model_ner = nemo_nlp.models.TokenClassificationModel.restore_from(config.model.nemo_path, trainer=trainer, strict=False)
        except:
            traceback.print_exc()
            config.model.nemo_path = str(exp_dir) + "/checkpoints/default.nemo"
            model_ner = nemo_nlp.models.TokenClassificationModel.restore_from(config.model.nemo_path, trainer=trainer, strict=False)

        text_file = Path(config.model.dataset.data_dir) / config.model.test_ds.text_file

        res = predict_in_sent(text_file, model_ner, combine=True)

        test_label = Path(config.model.dataset.data_dir) / config.model.test_ds.labels_file
        test_gs = load_gs_labels(test_label)
        bio_eval = BioEval()
        bio_eval.eval_mem(test_gs, res, do_flat=False)
        bio_eval.show_evaluation()

        pout = Path(args.pred_output)
        pout.mkdir(exist_ok=True)
        ofn = pout / "labels_test1.txt"
        with open(ofn, "w") as f:
            f.write("\n".join(res))

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--pred_output", type=str, default=None,
                        help="where we store all the models and prediction")
    parser.add_argument("--ner_config", type=str, required=True,
                        help="config file .yaml")
    parser.add_argument("--gpus", type=int, default=1,
                        help="how many gpus to use")
    parser.add_argument("--do_train", action='store_true',
                        help="training model")
    parser.add_argument("--do_dev", action='store_true',
                        help="run evaluation using f1 on dev set")
    parser.add_argument("--do_pred", action='store_true',
                        help="run prediction on test data")
    parser.add_argument("--local_rank", type=int, default=0,
                        help="local rank")
    parser.add_argument("--nproc_per_node", type=int, default=1,
                        help="local rank")
    parser.add_argument("--nnodes", type=int, default=1,
                        help="local rank")

    global_args = parser.parse_args()
    main(global_args)


import numpy as np
from numpy import linalg
from numpy.linalg import norm
from scipy.spatial.distance import squareform, pdist

import sklearn
from sklearn.manifold import TSNE
from sklearn.datasets import load_digits
from sklearn.preprocessing import scale

from sklearn.metrics.pairwise import pairwise_distances
from sklearn.manifold.t_sne import (_joint_probabilities,
                                    _kl_divergence)
# from sklearn.utils.extmath import _ravel

RS = 20150101

import matplotlib.pyplot as plt
import matplotlib.patheffects as PathEffects
import matplotlib

import seaborn as sns
sns.set_style('darkgrid')
sns.set_context('notebook', font_scale=1.5, rc={"lines.linewidth":2.5})

from moviepy.video.io.bindings import mplfig_to_npimage
import moviepy.editor as mpy

digits = load_digits()
print(digits.data.shape)
# print(digits['DESCR'])

def dataset():
    nrows, ncols = 2, 5
    plt.figure(figsize=(6,3))
    plt.gray()
    for i in range(ncols * nrows):
        ax = plt.subplot(nrows, ncols, i+1)
        ax.matshow(digits.images[i,...])
        plt.xticks([]); plt.yticks([])
        plt.title(digits.target[i])
    # plt.show()

X = np.vstack([digits.data[digits.target == i]
                   for i in range(10)])
y = np.hstack([digits.target[digits.target == i]
                   for i in range(10)])

def scatter(x, colors):
    # choose a color palette with seaborn
    palette = np.array(sns.color_palette('hls', 10))

    # create a scatter plot
    f = plt.figure(figsize=(8, 8))
    ax = plt.subplot(aspect='equal')
    sc = ax.scatter(x[:, 0], x[:, 1], lw=0, s=40, c=palette[colors.astype(np.int)])
    plt.xlim(-25, 25)
    plt.ylim(-25, 25)
    ax.axis('off')
    ax.axis('tight')

    # add a labels for each digit
    txts = []
    for i in range(10):
        xtext, ytext = np.median(x[colors == i, :], axis=0)
        txt = ax.text(xtext, ytext, str(i), fontsize=24)
        txt.set_path_effects([PathEffects.Stroke(linewidth=5, foreground='w'),
                              PathEffects.Normal()])
        txts.append(txt)
    return f, ax, sc, txts

# reorder the data points according to the handwritten numbers
def tsne():
    digits_proj = TSNE(random_state=RS).fit_transform(X)
    # print(digits_proj.shape)
    scatter(digits_proj, y)
    plt.show()
    # f, ax, sc, txts = scatter(digits_proj, y)
    #
    # def make_frame_mpl(t):
    #     i = int(t*40)
    #     x = digits_proj[..., i]
    #     sc.set_offsets(x)
    #     for j, txt in zip(range(10), txts):
    #         xtext, ytext = np.median(x[y == j, :], axis=0)
    #         txt.set_x(xtext)
    #         txt.set_y(ytext)
    #     return mplfig_to_npimage(f)
    #
    # animation = mpy.VideoClip(make_frame_mpl, duration=digits_proj.shape[1]/40.)
    # animation.write_gif("D://animation.gif", fps=20)

def similarity_matrix():
    def _joint_probabilities_constant_sigma(D, sigma):
        P = np.exp(-D**2/2 * sigma**2)
        P /= np.sum(P, axis=1)
        return P

    # Pairwise_distances between all data points
    D = pairwise_distances(X, squared=True)
    # Similarity with constant sigma
    P_constant = _joint_probabilities_constant_sigma(D, .002)
    # Similarity with variable sigma
    P_binary = _joint_probabilities(D, 30., False)
    # The output of this function needs to be reshaped to a square matrix
    P_binary_s = squareform(P_binary)

    plt.figure(figsize=(12, 4))
    pal = sns.light_palette("blue", as_cmap=True)

    plt.subplot(131)
    plt.imshow(D[::10, ::10], interpolation='none', cmap=pal)
    plt.axis('off')
    plt.title('Distance matrix', fontdict={'fontsize':16})

    plt.subplot(132)
    plt.imshow(P_constant[::10, ::10], interpolation='none', cmap=pal)
    plt.axis('off')
    plt.title("$p_{j|i}$ (constant $\sigma$)", fontdict={'fontsize':16})

    plt.subplot(133)
    plt.imshow(P_binary_s[::10, ::10], interpolation='none', cmap=pal)
    plt.axis('off')
    plt.title("$p_{j|i}$ (variable $\sigma$)", fontdict={'fontsize':16})
    # plt.show()

def physical_analogy():
    positions = []

    def _gradient_descent(objective, p0, it, n_iter, n_iter_check=1, n_iter_without_progress=30,
                          momentum=0.5, learning_rate=1000.0,
                          min_gain=0.01, min_grad_norm=1e-7,
                          min_error_diff=1e-7, verbose=0,
                          args=None, kwargs=None):
        if args is None:
            args = []
        if kwargs is None:
            kwargs = {}
        p = p0.copy().ravel()
        update = np.zeros_like(p)
        gains = np.ones_like(p)
        error = np.finfo(np.float).max  # finfo机器精度
        best_error = np.finfo(np.float).max
        best_iter = 0

        for i in range(it, n_iter):
            positions.append(p.copy())

            new_error, grad = objective(p, *args)
            error_diff = np.abs(new_error - error)
            error = new_error
            grad_norm = linalg.norm(grad)

            if error < best_error:
                best_error = error
                best_iter = i
            elif i - best_iter > n_iter_without_progress:
                break
            if min_grad_norm >= grad_norm:
                break
            if min_error_diff >= error_diff:
                break

            inc = update * grad >= 0.0
            dec = np.invert(inc)
            gains[inc] += 0.05
            gains[dec] *= 0.95
            np.clip(gains, min_gain, np.inf)
            grad *= gains
            update = momentum * update - learning_rate * grad
            p += update

        return p, error, i
    sklearn.manifold.t_sne._gradient_descent = _gradient_descent

    X_proj = TSNE(random_state=RS).fit_transform(X)
    X_iter = np.dstack(position.reshape(-1, 2)
                       for position in positions)

    f, ax, sc, txts = scatter(X_iter[..., -1], y)

    def make_frame_mpl(t):
        i = int(t*40)
        x = X_iter[..., i]
        sc.set_offsets(x)
        for j, txt in zip(range(10), txts):
            xtext, ytext = np.median(x[y == j, :], axis=0)
            txt.set_x(xtext)
            txt.set_y(ytext)
        return mplfig_to_npimage(f)

    animation = mpy.VideoClip(make_frame_mpl, duration=X_iter.shape[2]/40.)
    animation.write_gif("D://animation.gif", fps=20)

if __name__ == '__main__':
    physical_analogy()
    # tsne()


"""Support for ThinQ device switches."""

from __future__ import annotations

from dataclasses import dataclass
from datetime import timedelta
import logging
from typing import Any, Awaitable, Callable, Tuple

from .wideq import (
    FEAT_ECOFRIENDLY,
    FEAT_EXPRESSFRIDGE,
    FEAT_EXPRESSMODE,
    FEAT_ICEPLUS,
    WM_DEVICE_TYPES,
    DeviceType,
)

from homeassistant.components.switch import (
    SwitchDeviceClass,
    SwitchEntity,
    SwitchEntityDescription,
)
from homeassistant.config_entries import ConfigEntry
from homeassistant.const import STATE_OFF, STATE_ON
from homeassistant.core import HomeAssistant
from homeassistant.helpers.entity_platform import AddEntitiesCallback
from homeassistant.helpers.update_coordinator import CoordinatorEntity

from . import LGEDevice
from .const import DOMAIN, LGE_DEVICES
from .device_helpers import (
    STATE_LOOKUP,
    LGEBaseDevice,
    get_entity_name,
    get_multiple_devices_types,
)

# general sensor attributes
ATTR_POWER_OFF = "power_off"

SCAN_INTERVAL = timedelta(seconds=120)

_LOGGER = logging.getLogger(__name__)

@dataclass
class ThinQSwitchEntityDescription(SwitchEntityDescription):
    """A class that describes ThinQ switch entities."""

    available_fn: Callable[[Any], bool] | None = None
    turn_off_fn: Callable[[Any], Awaitable[None]] | None = None
    turn_on_fn: Callable[[Any], Awaitable[None]] | None = None
    value_fn: Callable[[Any], bool] | None = None

WASH_DEV_SWITCH: Tuple[ThinQSwitchEntityDescription, ...] = (
    ThinQSwitchEntityDescription(
        key=ATTR_POWER_OFF,
        name="Power off",
        value_fn=lambda x: x.is_power_on,
        turn_off_fn=lambda x: x.device.power_off(),
        available_fn=lambda x: x.is_power_on,
    ),
)
REFRIGERATOR_SWITCH: Tuple[ThinQSwitchEntityDescription, ...] = (
    ThinQSwitchEntityDescription(
        key=FEAT_ECOFRIENDLY,
        name="Eco friendly",
        icon="mdi:gauge-empty",
        turn_off_fn=lambda x: x.device.set_eco_friendly(False),
        turn_on_fn=lambda x: x.device.set_eco_friendly(True),
        available_fn=lambda x: x.is_power_on,
    ),
    ThinQSwitchEntityDescription(
        key=FEAT_EXPRESSFRIDGE,
        name="Express fridge",
        icon="mdi:coolant-temperature",
        turn_off_fn=lambda x: x.device.set_express_fridge(False),
        turn_on_fn=lambda x: x.device.set_express_fridge(True),
        available_fn=lambda x: x.device.set_values_allowed,
    ),
    ThinQSwitchEntityDescription(
        key=FEAT_EXPRESSMODE,
        name="Express mode",
        icon="mdi:snowflake",
        turn_off_fn=lambda x: x.device.set_express_mode(False),
        turn_on_fn=lambda x: x.device.set_express_mode(True),
        available_fn=lambda x: x.device.set_values_allowed,
    ),
    ThinQSwitchEntityDescription(
        key=FEAT_ICEPLUS,
        name="Ice plus",
        icon="mdi:snowflake",
        turn_off_fn=lambda x: x.device.set_ice_plus(False),
        turn_on_fn=lambda x: x.device.set_ice_plus(True),
        available_fn=lambda x: x.device.set_values_allowed,
    ),
)

AC_DUCT_SWITCH = ThinQSwitchEntityDescription(
    key="duct-zone",
    name="Zone",
)

def _switch_exist(lge_device: LGEDevice, switch_desc: ThinQSwitchEntityDescription) -> bool:
    """Check if a switch exist for device."""
    if switch_desc.value_fn is not None:
        return True

    feature = switch_desc.key
    if feature in lge_device.available_features:
        return True

    return False

async def async_setup_entry(
    hass: HomeAssistant, entry: ConfigEntry, async_add_entities: AddEntitiesCallback
) -> None:
    """Set up the LGE switch."""
    entry_config = hass.data[DOMAIN]
    lge_devices = entry_config.get(LGE_DEVICES)
    if not lge_devices:
        return

    _LOGGER.debug("Starting LGE ThinQ switch setup...")
    lge_switch = []

    # add WM devices
    lge_switch.extend(
        [
            LGESwitch(lge_device, switch_desc)
            for switch_desc in WASH_DEV_SWITCH
            for lge_device in get_multiple_devices_types(lge_devices, WM_DEVICE_TYPES)
            if _switch_exist(lge_device, switch_desc)
        ]
    )

    # add refrigerators
    lge_switch.extend(
        [
            LGESwitch(lge_device, switch_desc)
            for switch_desc in REFRIGERATOR_SWITCH
            for lge_device in lge_devices.get(DeviceType.REFRIGERATOR, [])
            if _switch_exist(lge_device, switch_desc)
        ]
    )

    # add AC duct zone switch
    lge_switch.extend(
        [
            LGEDuctSwitch(lge_device, duct_zone)
            for lge_device in lge_devices.get(DeviceType.AC, [])
            for duct_zone in lge_device.device.duct_zones
        ]
    )

    async_add_entities(lge_switch)

class LGESwitch(CoordinatorEntity, SwitchEntity):
    """Class to control switches for LGE device"""

    entity_description = ThinQSwitchEntityDescription

    def __init__(
            self,
            api: LGEDevice,
            description: ThinQSwitchEntityDescription,
    ):
        """Initialize the switch."""
        super().__init__(api.coordinator)
        self._api = api
        self._wrap_device = LGEBaseDevice(api)
        self.entity_description = description
        self._attr_name = get_entity_name(api, description.key, description.name)
        self._attr_unique_id = f"{api.unique_id}-{description.key}-switch"
        self._attr_device_class = SwitchDeviceClass.SWITCH
        self._attr_device_info = api.device_info

    @property
    def should_poll(self) -> bool:
        """Return True if entity has to be polled for state.

        We overwrite coordinator property default setting because we need
        to poll to avoid the effect that after changing switch state
        it is immediately set to prev state. The async_update method here
        do nothing because the real update is performed by coordinator.
        """
        return True

    async def async_update(self) -> None:
        """Update the entity.

        This is a fake update, real update is done by coordinator.
        """
        return

    @property
    def is_on(self):
        """Return the state of the switch."""
        ret_val = self._get_switch_state()
        if ret_val is None:
            return False
        if isinstance(ret_val, bool):
            return ret_val
        if ret_val == STATE_ON:
            return True
        state = STATE_LOOKUP.get(ret_val, STATE_OFF)
        return state == STATE_ON

    @property
    def available(self) -> bool:
        """Return True if entity is available."""
        is_avail = True
        if self.entity_description.available_fn is not None:
            is_avail = self.entity_description.available_fn(self._wrap_device)
        return self._api.available and is_avail

    async def async_turn_off(self, **kwargs):
        """Turn the entity off."""
        if self.entity_description.turn_off_fn is None:
            raise NotImplementedError()
        if self.is_on:
            await self.entity_description.turn_off_fn(self._wrap_device)

    async def async_turn_on(self, **kwargs):
        """Turn the entity on."""
        if self.entity_description.turn_on_fn is None:
            raise NotImplementedError()
        if not self.is_on:
            await self.entity_description.turn_on_fn(self._wrap_device)

    def _get_switch_state(self):
        """Get current switch state"""
        if self.entity_description.value_fn is not None:
            return self.entity_description.value_fn(self._wrap_device)

        if self._api.state:
            feature = self.entity_description.key
            return self._api.state.device_features.get(feature)

        return None

class LGEDuctSwitch(LGESwitch):
    """Class to control switches for LGE AC duct device"""

    def __init__(
            self,
            api: LGEDevice,
            duct_zone: str
    ):
        """Initialize the switch."""
        super().__init__(api, AC_DUCT_SWITCH)
        self._attr_name += f" {duct_zone}"
        self._attr_unique_id += f"-{duct_zone}"
        self._zone = duct_zone

    @property
    def is_on(self):
        """Return the state of the switch."""
        return self._wrap_device.device.get_duct_zone(self._zone)

    @property
    def available(self) -> bool:
        """Return True if entity is available."""
        return (
            self._wrap_device.device.is_duct_zone_enabled(self._zone)
            and self._wrap_device.is_power_on
        )

    async def async_turn_off(self, **kwargs):
        """Turn the entity off."""
        self._wrap_device.device.set_duct_zone(self._zone, False)

    async def async_turn_on(self, **kwargs):
        """Turn the entity on."""
        self._wrap_device.device.set_duct_zone(self._zone, True)


from typing import Optional
from os.path import join

from rastervision2.core.data.label_store import (LabelStoreConfig,
                                                 ObjectDetectionGeoJSONStore)
from rastervision2.pipeline.config import register_config

@register_config('object_detection_geojson_store')
class ObjectDetectionGeoJSONStoreConfig(LabelStoreConfig):
    uri: Optional[str] = None

    def build(self, class_config, crs_transformer, extent=None, tmp_dir=None):
        return ObjectDetectionGeoJSONStore(self.uri, class_config,
                                           crs_transformer)

    def update(self, pipeline=None, scene=None):
        if pipeline is not None and scene is not None:
            if self.uri is None:
                self.uri = join(pipeline.predict_uri,
                                '{}.json'.format(scene.id))


import shutil
import zipfile

import win32api
import os
import json
from zipfile import ZipFile

def sys_info():
    total, used, free = shutil.disk_usage("/")
    drives = win32api.GetLogicalDriveStrings()
    drives = drives.split('\000')[:-1]
    print("[+] Имена логических дисков - " + str(drives))
    print("[+] Всего : %d Gb" % (total // (2**30)))
    print("[+] Используется : %d Gb" % (used // (2**30)))
    print("[+] Свободно : %d Gb" % (free // (2**30)))
    input("[*] Нажмите кнопку для продолжения")

def work_with_files():
    print("[*] Выберите нужный пункт ")
    print("[1] Создать файл")
    print("[2] Записать строку в файл")
    print("[3] Прочитать файл")
    print("[4] Удалить файл")
    print("[5] Выйти")
    usr_input = int(input("-> "))
    if usr_input == 1:
        file_name = input("[*] Введите имя файла (без разрешения) ")
        file_name = file_name + ".txt"
        if not os.path.exists(file_name):
            with open(file_name, 'w'): pass
            input("[*] Файл создан. Нажмите кнопку для продолжения.")
            work_with_files()
    elif usr_input == 2:
        file_name = input("[*] Введите имя файла (без разрешения) ")
        usr_string = input("[*] Введите строку")
        file_name = file_name + ".txt"
        if os.path.exists(file_name):
            f = open(file_name, 'w')
            f.write(usr_string)
            f.close()
            input("[*] Строка записана. Нажмите кнопку для продолжения.")
            work_with_files()
        else:
            input("[!] Такого файла нет!")
            work_with_files()
    elif usr_input == 3:
        file_name = input("[*] Введите имя файла (без разрешения) ")
        file_name = file_name + ".txt"
        if os.path.exists(file_name):
            with open(file_name) as f:
                lines = f.readlines()
                print(lines)
                input("[*] Файл прочитан. Нажмите кнопку для продолжения.")
                work_with_files()
        else:
            input("[!] Такого файла нет!")
            work_with_files()
    elif usr_input == 4:
        file_name = input("[*] Введите имя файла (без разрешения) ")
        file_name = file_name + ".txt"
        if os.path.exists(file_name):
            os.remove(file_name)
            input("[*] Файл удален. Нажмите кнопку для продолжения.")
            work_with_files()
        else:
            input("[!] Такого файла нет!")
            work_with_files()
    else:
        pass

def work_with_json():
    print("[*] Выберите нужный пункт ")
    print("[1] Создать JSON на основе")
    print("[2] Создать объект")
    print("[3] Прочитать JSON")
    print("[4] Удалить JSON")
    print("[5] Выйти")
    usr_input = int(input("-> "))
    if usr_input == 1:
        file_name = input("[*] Введите имя файла (без разрешения) ")
        file_name = file_name + ".json"
        base_file = input("[*] Введите имя файла базы (без разрешения) ")
        base_file = base_file + ".json"
        with open(base_file) as f:
            data = json.load(f)
        with open(file_name, 'w') as f:
            json.dump(data, f, indent=2)
            input("[*] Создан новый файл на основе " + str(base_file))
            work_with_json()
    elif usr_input == 2:
        color = input("[*] Введите цвет ")
        value = input("[*] Введите значение")
        data = {"color": color, "value": value}
        file_name = input("[*] Введите имя файла (без разрешения) ")
        file_name = file_name + ".json"
        with open(file_name, 'a', encoding='utf-8') as f:
            json.dump(data, f, ensure_ascii=False, indent=2)
            work_with_json()
    elif usr_input == 3:
        file_name = input("[*] Введите имя файла (без разрешения) ")
        file_name = file_name + ".json"
        if os.path.exists(file_name):
            with open(file_name, 'r') as f:
                data = json.load(f)
                print(data)
                input("[*] Файл прочитан ")
                work_with_json()
        else:
            input("[*] Такого файла не существует")
            work_with_json()
    elif usr_input == 4:
        file_name = input("[*] Введите имя файла (без разрешения) ")
        file_name = file_name + ".json"
        if os.path.exists(file_name):
            os.remove(file_name)
            input("[*] Файл удален. Нажмите кнопку для продолжения.")
            work_with_json()
        else:
            input("[!] Такого файла нет!")
            work_with_json()
    else:
        pass

def work_with_rar():
    print("[*] Выберите нужный пункт ")
    print("[1] Создать zip архив")
    print("[2] Разархивировать файл")
    print("[3] Удалить файл и архив")
    print("[4] Выйти")
    usr_input = int(input("-> "))
    if usr_input == 1:
        zip_file_name = input("[*] Введите имя архива (без разрешения) ")
        zip_file_name = zip_file_name + ".zip"
        file_name = input("[*] Введите имя файла (с разрешением) ")
        try:
            zipfile.ZipFile(zip_file_name, mode='w').write(file_name)
            input("[*] Файл разархивирован!")
            work_with_rar()
        except:
            work_with_rar()
            pass
    elif usr_input == 2:
        zip_file_name = input("[*] Введите имя архива (без разрешения) ")
        zip_file_name = zip_file_name + ".zip"
        if os.path.exists(zip_file_name):
            with zipfile.ZipFile(zip_file_name, 'r') as zip_ref:
                zip_ref.extractall()
            input("[*] Архив разархивирован!")
            work_with_rar()
        else:
            input("[*] Такого архива нет!")
            work_with_rar()
    elif usr_input == 3:
        zip_file_name = input("[*] Введите имя архива (без разрешения) ")
        zip_file_name = zip_file_name + ".zip"
        if os.path.exists(zip_file_name):
            os.remove(zip_file_name)
            input("[*] Архив удален!")
            work_with_rar()
        else:
            input("[*] Такого архива нет!")
            work_with_rar()
    else:
        pass

def main():
    while True:
        print("[*] Работа с файлами. Выберите необходимый пункт")
        print("[1] Вывести информацию о системе")
        print("[2] Работа с файлами")
        print("[3] Работа с JSON")
        print("[5] Работа с архивами")
        print("[0] Выйти")
        usr_input = int(input("-> "))
        if (usr_input == 1):
            sys_info()
        elif (usr_input == 2):
            work_with_files()
        elif (usr_input == 3):
            work_with_json()
        elif (usr_input == 5):
            work_with_rar()
        elif (usr_input == 0):
            break

if __name__ == "__main__":
    main()


#!/usr/bin/python3

import sys
from bson import ObjectId
import json
from function import *
import datetime

sensors = []
db = db.dbmongo()

collection = "device"

def add(fillData):
    insertQuery = {
        'group_code_name':fillData.get('group_code_name', None),
        'key_access':fillData.get('key_access', None),
        'device_code':fillData.get('device_code', None),
        'name':fillData.get('name', None),
        'field':fillData.get('field', None), #arraylist [field on sensor]
        'date_add': datetime.datetime.utcnow(),
        'add_by':fillData.get('add_by', None),
        'active':fillData.get('active', False),
        'information':fillData.get('information', None), #array[location, detail, purpose]
    }
    result = db.insertData(collection,insertQuery)
    if result == []:
        response = {'status':False, 'message':"Add Failed"}
    else:
        response = {'status':True,'message':'Success','data':result}
    return cloud9Lib.jsonObject(response)

def find(query):
    result = db.find(collection,query)
    if result == []:
        response = {"status":False, "data":query}
    else:
        response = {'status':True, 'data':result}
    return cloud9Lib.jsonObject(response)

def findOne(query):
    result = db.findOne(collection,query, None)
    if result is None or result is False:
        response = {"status":False, "data":query}
    else:
        response = {'status':True,'message':'Success','data':result}
    return cloud9Lib.jsonObject(response)

def update(query,data):
    updateData = {}
    queryUpdate = {}
    if 'code_name' in query: queryUpdate['code_name'] = query['code_name']
    if '_id' in query: queryUpdate['_id'] = query['_id']

    if 'name' in data: updateData['name'] = data['name']
    if 'active' in data: updateData['active'] = data['active']
    if 'key_access' in data: updateData['key_access'] = data['key_access']
    if 'device_code' in data: updateData['device_code'] = data['device_code']
    if 'field' in data: updateData['field'] = data['field']
    if 'information' in data: updateData['information'] = data['information']
    if 'updated_by' in data: updateData['updated_by'] = data['updated_by']
    if 'token_access' in data: updateData['token_access'] = data['token_access']

    if updateData == []:
        return {"status":False, "message":"UPDATE NONE"}
    result = db.updateData(collection,queryUpdate,updateData)
    if not result :
        response = {"status":False, "message":"UPDATE FAILED"}
    else:
        response = {'status':True,'message':'Success','data':result}
    return cloud9Lib.jsonObject(response)

def delete(query):
    result = db.deleteData(collection,query)
    if not result:
        response = {"status":False, "message":"DELETE FAILED"}
    else:
        response = {'status':True,'message':'Success','data':result}
    return cloud9Lib.jsonObject(response)


import os
import matplotlib.pyplot as plt
import numpy as np
# kshot = "0shot/robert"
kshot = "5shot/updatelr0.01.metalr0.005.numstep5"
log_dir = "D:/연구/프로젝트/maml_result/"

paths = [log_dir+"1/0shot/", log_dir + "1/10shot/updatelr0.005.metalr0.005.numstep5/test/400/"]
# paths = [log_dir+"1/0shot/", log_dir + "0/10shot/updatelr0.01.metalr0.005.numstep5/test/500/"]
iterations = ['0shot','10shot']

data = []
for _ in range(len(paths)):
    data.append({"ICC" : [], "MSE" : [], "ACC" : [], "F1-b": []})

for i in range(len(paths)):
    files = []
    for file_name in os.listdir(paths[i]):
        if (file_name.startswith("outa")):
            files.append((int(file_name.split(".")[0].split("_")[1]), file_name))
    print(len(files))

    files.sort(key=lambda f:f[0])
    for f in files:
        print(paths[i] + f[1])
        file = open(paths[i] + f[1])
        for line in file:
            if(line.startswith("ICC")):
                fig = line.split(" ")
                data[i]['ICC'].append(float(fig[len(fig)-1]))
            elif(line.startswith("RMSE")):
                fig = line.split(" ")
                data[i]['MSE'].append(float(fig[len(fig)-1]))
            elif(line.startswith("ACC")):
                fig = line.split(" ")
                data[i]['ACC'].append(float(fig[len(fig)-1]))
            elif(line.startswith("F1-b")):
                fig = line.split(" ")
                data[i]['F1-b'].append(float(fig[len(fig)-1]))
    print("===================")

subjects = 13
opacity = 0.6
bar_width = 0.3
plt.figure(figsize = (10,7))
x_range = np.arange(0, subjects)
plt.suptitle("compare test results for each subject", fontsize=16)

plt.subplot(221)
plt.bar(x_range - bar_width/2, data[0]['ICC'], bar_width, alpha=opacity, color = "blue", label = iterations[0])
plt.bar(x_range + bar_width/2, data[1]['ICC'], bar_width, alpha=opacity, color= "red", label = iterations[1])
# plt.plot(x_range, data[0]['ICC'],label = iterations[0])
# plt.plot(x_range, data[1]['ICC'], label = iterations[1])
plt.xticks(range(0, subjects, 1))
plt.legend(loc=0)
plt.xlabel('Subject')
plt.ylabel('ICC')
plt.title('ICC')
# plt.tight_layout()

plt.subplot(222)
y = data[0]['MSE']
plt.bar(x_range - bar_width/2, data[0]['MSE'], bar_width, alpha=opacity, color = "blue", label = iterations[0])
plt.bar(x_range + bar_width/2, data[1]['MSE'], bar_width, alpha=opacity, color= "red", label = iterations[1])
# plt.plot(x_range, data[0]['MSE'], "o")
# plt.plot(x_range, data[0]['MSE'],label = iterations[0])
# plt.plot(x_range, data[1]['MSE'], "r*")
# plt.plot(x_range, data[1]['MSE'], label = iterations[1])
plt.xticks(range(0, subjects, 1))
plt.legend(loc=1)
plt.xlabel('Subject')
plt.ylabel('RMSE')
plt.title('RMSE')

plt.subplot(223)
plt.bar(x_range - bar_width/2, data[0]['ACC'], bar_width, alpha=opacity, color = "blue", label = iterations[0])
plt.bar(x_range + bar_width/2, data[1]['ACC'], bar_width, alpha=opacity, color= "red", label = iterations[1])
# plt.plot(x_range, data[0]['ACC'], "o")
# plt.plot(x_range, data[0]['ACC'],label = iterations[0])
# plt.plot(x_range, data[1]['ACC'], "r*")
# plt.plot(x_range, data[1]['ACC'], label = iterations[1])
plt.xticks(range(0, subjects, 1))
plt.legend(loc=0)
plt.xlabel('Subject')
plt.ylabel('ACC')
plt.title('ACC')

plt.subplot(224)
plt.bar(x_range - bar_width/2, data[0]['F1-b'], bar_width, alpha=opacity, color = "blue", label = iterations[0])
plt.bar(x_range + bar_width/2, data[1]['F1-b'], bar_width, alpha=opacity, color= "red", label = iterations[1])
# plt.plot(x_range, data[0]['F1-b'], "o")
# plt.plot(x_range, data[0]['F1-b'],label = iterations[0])
# plt.plot(x_range, data[1]['F1-b'], "r*")
# plt.plot(x_range, data[1]['F1-b'], label = iterations[1])
plt.xticks(range(0, subjects, 1))
#plt.yticks(range(0, 1, 10))
plt.legend(loc=0)
plt.xlabel('Subject')
plt.ylabel('F1-b')
plt.title('F1-b')

plt.show()


from os import path

import json
import mock
import six

from oslo_config import cfg
from oslo_log import log as logging

from congress.cfg_validator.agent import agent
from congress.cfg_validator.agent import opts
from congress import opts as opts_congress
from congress.tests import base

# pylint: disable=protected-access

LOG = logging.getLogger(__name__)

NAMESPACE_FILE_CONTENT = """[DEFAULT]
output_file = etc/svc.conf.sample
wrap_width = 79
namespace = congress
namespace = congress-agent
"""

CONF_FILE1_CONTENT = """
[agent]
version:v0
"""

CONF_FILE2_CONTENT = """
[agent]
host:h0
"""

CONF_AGENT_CONTENT = """
[agent]
host:hhh
version:vvv
services: svc: {svc1.conf: svc.tpl}
"""

TEMPLATES = {
    "tpl1": agent.Template("TPL1", "out"),
    "tpl2": agent.Template("TPL2", "out")}
CONFIGS = {
    "cfg1": agent.Config("CFG1", TEMPLATES["tpl1"], "svc"),
    "cfg2": agent.Config("CFG2", TEMPLATES["tpl2"], "svc")}
NAMESPACES = {
    "ns1": agent.Namespace("NS1"),
    "ns2": agent.Namespace("NS2")}

def _gen_ns_data_fake(namespace):
    return u"{\"DEFAULT\": {\"namespaces\": [[\"" + namespace + u"\", []]]}}"

class TestTemplate(base.TestCase):

    "Test template loading"

    @mock.patch('oslo_config.cfg.open')
    def test_parse_template(self, mock_open):
        "Test loading a template"
        mock.mock_open(mock=mock_open, read_data=NAMESPACE_FILE_CONTENT)
        # Patch the mock_open file to support iteration.
        mock_open.return_value.__iter__ = lambda x: iter(x.readline, '')

        tpl, out_file = agent.Template._parse_template_conf('template')
        self.assertEqual(len(tpl), 2)
        self.assertEqual(out_file, 'etc/svc.conf.sample')

class TestCfgConfig(base.TestCase):

    "Test config handling"

    def test_sanitize(self):
        "test config sanitization"
        conf = cfg.ConfigOpts()
        conf._namespace = cfg._Namespace(conf)
        conf._namespace._normalized = []

        opt_1 = cfg.StrOpt(name='lorem', secret=True)
        opt_2 = cfg.StrOpt(name='ipsum', secret=False)
        conf.register_opts([opt_1, opt_2])

        parsed = {'DEFAULT': {'lorem': ['mysecret'], 'ipsum': ['notsecret']}}
        conf._namespace._normalized.append(parsed)

        agent.Config.sanitize_config(conf)

        self.assertNotIn('mysecret', json.dumps(conf._namespace._normalized))
        self.assertIn(
            'notsecret', json.dumps(conf._namespace._normalized))

        self.assertEqual(conf.lorem, '****')
        self.assertEqual(conf.ipsum, 'notsecret')

    def test_get_info(self):
        "test basic get_info"
        cfg_mock = mock.Mock(spec=agent.Config)
        tpl = mock.Mock()
        tpl.hash = 'lorem'
        cfg_mock.template = tpl
        cfg_mock.path = 'ipsum'
        cfg_mock.data = 'dolor'
        cfg_mock.service = 'svc'

        info = agent.Config.get_info(cfg_mock)

        self.assertIn('template', info)
        self.assertEqual(info['template'], 'lorem')
        self.assertIn('path', info)
        self.assertEqual(info['path'], 'ipsum')
        self.assertIn('data', info)
        self.assertEqual(info['data'], 'dolor')

class TestCfgNamespace(base.TestCase):
    "Test namespace handling"

    @mock.patch('congress.cfg_validator.agent.agent.'
                'validator_generator.generate_ns_data')
    def test_load(self, gen_ns_data_mock):
        "Test load namespace"
        gen_ns_data_mock.return_value = _gen_ns_data_fake('lorem')

        ns_mock = agent.Namespace.load('ipsum')

        self.assertEqual(ns_mock.name, 'ipsum')
        self.assertEqual(json.dumps(ns_mock.data), _gen_ns_data_fake('lorem'))
        self.assertIsNotNone(ns_mock.hash)

        same_data_ns = agent.Namespace.load('other_ipsum')
        self.assertEqual(same_data_ns.hash, ns_mock.hash)

        gen_ns_data_mock.return_value = _gen_ns_data_fake('other_lorem')
        other_data_ns = agent.Namespace.load('ipsum')
        self.assertNotEqual(ns_mock.hash, other_data_ns.hash)

    def test_get_info(self):
        "Test basic info on namespace"
        ns_mock = mock.Mock(spec=agent.Namespace)
        ns_mock.name = 'foo'
        ns_mock.data = 'bar'

        info = agent.Namespace.get_info(ns_mock)

        self.assertIn('name', info)
        self.assertEqual(info['name'], 'foo')
        self.assertIn('data', info)
        self.assertEqual(info['data'], 'bar')

class TestCfgTemplate(base.TestCase):
    """Test the handling of templates"""

    @mock.patch('congress.cfg_validator.agent.agent.'
                'cfg.ConfigOpts._parse_config_files')
    def test_template_conf(self, parse_mock):
        """Test the parsing of template"""
        ns_mock = cfg._Namespace(None)
        ns_mock._normalized.append({
            'DEFAULT':
                {
                    'output_file': ['etc/congress.conf.sample'],
                    'wrap_width': ['79'],
                    'namespace': ['congress', 'oslo.log'],
                }
        })

        parse_mock.return_value = ns_mock

        namespace, out_file = agent.Template._parse_template_conf('somewhere')

        self.assertEqual(out_file, 'etc/congress.conf.sample')
        self.assertIn('congress', namespace)
        self.assertIn('oslo.log', namespace)

    @mock.patch('congress.cfg_validator.agent.agent.Namespace.load')
    @mock.patch('congress.cfg_validator.agent.agent.'
                'Template._parse_template_conf')
    def test_load(self, parse_tpl_conf_mock, load_ns_mock):
        """Test loading a template"""
        parse_tpl_conf_mock.return_value = (['congress_h', 'oslo.log'],
                                            'some/where.sample')

        load_ns_mock.side_effect = [mock.MagicMock(hash='lorem'),
                                    mock.MagicMock(hash='ipsum')]

        tpl = agent.Template.load('path/to/template')

        self.assertEqual(tpl.path, 'path/to/template')
        self.assertEqual(tpl.output_file, 'some/where.sample')
        self.assertEqual(tpl.name, 'where')
        self.assertIsNotNone(tpl.hash)
        self.assertTrue(tpl.namespaces)
        self.assertIn('lorem', [ns.hash for ns in tpl.namespaces])
        self.assertIn('ipsum', [ns.hash for ns in tpl.namespaces])

    def test_get_info(self):
        """Test basic info on template"""
        tpl = mock.Mock(spec=agent.Template)
        tpl.path = 'lorem'
        tpl.output_file = 'ipsum'
        tpl.name = 'dolor'
        ns_mock = mock.Mock()
        ns_mock.hash = 'sit'
        tpl.namespaces = [ns_mock]

        info = agent.Template.get_info(tpl)

        self.assertIn('name', info)
        self.assertEqual(info['name'], 'dolor')
        self.assertIn('path', info)
        self.assertEqual(info['path'], 'lorem')
        self.assertIn('output_file', info)
        self.assertEqual(info['output_file'], 'ipsum')
        self.assertIn('namespaces', info)
        self.assertEqual(info['namespaces'], ['sit'])

def _file_mock(file_spec):
    def _mk_content(spec):
        fval = mock.mock_open(read_data=spec)
        fval.return_value.__iter__ = lambda x: iter(x.readline, '')
        return fval
    file_map = {name: _mk_content(spec)
                for name, spec in six.iteritems(file_spec)}

    def _give_file(name):
        basename = path.basename(name)
        return file_map.get(basename, None)(name)

    return mock.MagicMock(name='open', spec=open, side_effect=_give_file)

def _fake_opt_loader(namespaces):
    def fake_entry_point(namespace):
        if namespace == 'congress':
            return opts_congress.list_opts
        if namespace == 'congress-agent':
            return opts.list_opts
        else:
            return None
    return [(ns, fake_entry_point(ns)) for ns in namespaces]

class TestCfgManager(base.TestCase):
    """Config manager tests"""

    @mock.patch(
        'oslo_config.generator._get_raw_opts_loaders', _fake_opt_loader)
    @mock.patch(
        'oslo_config.cfg.open',
        _file_mock({
            "svc.tpl": NAMESPACE_FILE_CONTENT,
            "svc1.conf": CONF_FILE1_CONTENT,
            "svc2.conf": CONF_FILE2_CONTENT}))
    def test_init(self):
        """Test the creation of the config manager"""
        cfg_manager = agent.ConfigManager(
            'host', {"svc": {"svc1.conf": "svc.tpl", "svc2.conf": "svc.tpl"}})
        self.assertEqual('host', cfg_manager.host)
        self.assertEqual(2, len(cfg_manager.configs))
        self.assertEqual(2, len(cfg_manager.namespaces))
        self.assertEqual(1, len(cfg_manager.templates))
        for conf in six.itervalues(cfg_manager.configs):
            self.assertIsInstance(conf, agent.Config)
        for nspc in six.itervalues(cfg_manager.namespaces):
            self.assertIsInstance(nspc, agent.Namespace)
        for tpl in six.itervalues(cfg_manager.templates):
            self.assertIsInstance(tpl, agent.Template)

def _setup_endpoint():
    with mock.patch(
        'oslo_config.cfg.open',
        _file_mock({"agent.conf": CONF_AGENT_CONTENT})),\
        mock.patch(
            'congress.cfg_validator.agent.agent.ConfigManager',
            autospec=True) as mock_manager,\
        mock.patch(
            'congress.cfg_validator.agent.rpc.ValidatorDriverClient',
            autospec=True) as mock_client:
        conf = cfg.ConfigOpts()
        opts.register_validator_agent_opts(conf)
        conf(args=['--config-file', 'agent.conf'])
        mock_manager.return_value.configs = CONFIGS
        mock_manager.return_value.templates = TEMPLATES
        mock_manager.return_value.namespaces = NAMESPACES
        endpoint = agent.ValidatorAgentEndpoint(conf=conf)
        return endpoint, mock_client, mock_manager

class TestValidatorAgentEndpoint(base.TestCase):
    """Test the endpoint for the agent communications"""

    # pylint: disable=no-self-use

    @mock.patch(
        'oslo_config.cfg.open',
        _file_mock({
            "agent.conf": CONF_AGENT_CONTENT}))
    @mock.patch(
        'congress.cfg_validator.agent.agent.ConfigManager',
        autospec=True)
    @mock.patch(
        'congress.cfg_validator.agent.rpc.ValidatorDriverClient',
        autospec=True)
    def test_publish_template_hashes(self, mock_client, mock_manager):
        "Test a request to publish hashes"
        conf = cfg.ConfigOpts()
        opts.register_validator_agent_opts(conf)
        conf(args=['--config-file', 'agent.conf'])
        templates = {"tpl1": {}, "tpl2": {}}
        mock_manager.return_value.templates = templates
        endpoint = agent.ValidatorAgentEndpoint(conf=conf)
        endpoint.publish_templates_hashes({})
        mock_client.return_value.process_templates_hashes.assert_called_with(
            {}, set(templates), "hhh")

    def test_publish_configs_hashes(self):
        "Test a request to publish hashes"
        endpoint, mock_client, _ = _setup_endpoint()
        endpoint.publish_configs_hashes({})
        mock_client.return_value.process_configs_hashes.assert_called_with(
            {}, set(CONFIGS), "hhh")

    def test_get_config(self):
        "Test reply to an explicit config request"
        endpoint, _, _ = _setup_endpoint()
        ret = endpoint.get_config({}, cfg_hash="cfg1")
        expected = {
            'data': None,
            'path': 'CFG1',
            'service': 'svc',
            'template': None,
            'version': 'vvv'}
        self.assertEqual(expected, ret)
        ret = endpoint.get_config({}, cfg_hash="XXX")
        self.assertEqual(None, ret)

    def test_get_namespace(self):
        "Test reply to an explicit config request"
        endpoint, _, _ = _setup_endpoint()
        ret = endpoint.get_namespace({}, ns_hash="ns1")
        expected = {
            'version': 'vvv',
            'data': None,
            'name': 'NS1'}
        self.assertEqual(expected, ret)
        ret = endpoint.get_namespace({}, ns_hash="XXX")
        self.assertEqual(None, ret)

    def test_get_template(self):
        "Test reply to an explicit config request"
        endpoint, _, _ = _setup_endpoint()
        ret = endpoint.get_template({}, tpl_hash="tpl1")
        expected = {
            'name': 'out',
            'namespaces': [],
            'output_file': 'out',
            'path': 'TPL1',
            'version': 'vvv'}
        self.assertEqual(expected, ret)
        ret = endpoint.get_template({}, tpl_hash="XXX")
        self.assertEqual(None, ret)


import logging
import os
import site
import sys

# Add the path to the site-packages of your installation. Typically, if you use
# virtual environments, you may enter the path to its site-packages directory
# in this list (ex: /path/to/virtualenv/lib/python3.x/site-packages/).
SITE_PACKAGES = []

# Set here the path to the mushi package. Typically, it is the path to the
# the directory this file is in.
APP_PATH = '/path/to/mushi/'

# Remember original sys.path.
old_syspath = list(sys.path)

# Add each additional site-packages directory.
for sp in SITE_PACKAGES:
    site.addsitedir(sp)

# Reorder sys.path to place new directories at the front.
new_syspath = []
for item in list(sys.path):
    if item not in old_syspath:
        new_syspath.append(item)
        sys.path.remove(item)
sys.path[:0] = new_syspath

# Insert application path.
sys.path.insert(0, APP_PATH)

# Redirect logging to sys.stderr.
logging.basicConfig(stream=sys.stderr)

from werkzeug.serving import run_simple
from werkzeug.wsgi import DispatcherMiddleware

from mushi.apps import api, webui

def application(req_environ, start_response):
    os.environ['MUSHI_SETTINGS'] = req_environ['MUSHI_SETTINGS']

    fn = DispatcherMiddleware(
        webui.create_app(),
        {
            '/api': api.create_app()
        }
    )

    return fn(req_environ, start_response)

if __name__ == '__main__':
    run_simple('0.0.0.0', 5000, application, use_reloader=True, use_debugger=True)


#!/usr/bin/env python
#
# midiin_poll.py
#
"""Show how to receive MIDI input by polling an input port."""

from __future__ import print_function

import logging
import sys
import time
import os

from rtmidi.midiutil import open_midiinput
import pandas
import datetime

log = logging.getLogger('midiin_poll')
logging.basicConfig(level=logging.DEBUG)

# Prompt user to indicate what hand(s) they will be practicing with
hand = input("Pleast type 'left', 'right', or 'both' to indicate the hand(s) you are practicing: ")

# Prompt user for practice module name
module = input("Please type the name of the practice module: ")

# Prompts user for MIDI input port, unless a valid port number or name
# is given as the first argument on the command line.
# API backend defaults to ALSA on Linux.
port = sys.argv[1] if len(sys.argv) > 1 else None

# Opens a midi input port and assigns a midiin object and captures name of port
midiin, port_name = open_midiinput(port)

# Prompt for performing a hard exit
print("Entering main loop. Press Control-C to exit.")

# Starts time before any keys are pressed e.g.: beginning of practice session
timer = time.time()

# Create a dataframe to store pressed key data in locally
df = pandas.DataFrame(columns = ['Date', 'Module', 'Hand', 'Time', 'Stroke', 'Key', 'Speed'])

# Main loop for capturing pressed key data
while True:
    # Retrieves most recent midi input message
    msg = midiin.get_message()

    if msg: # Following creates variables and adds them to the dataframe
        message, deltatime = msg
        timer += deltatime
        date = datetime.datetime.now().date()
        df = df.append({
            'Date' : date,
            'Module' : module,
            'Hand' : hand,
            'Time' : timer,
            'Stroke' : message[0],
            'Key' : message[1],
            'Speed' : message[2]
            },
                ignore_index = True)

        # Filters dataframe for beginning and end of key stroke AND foot pedal presses
        df = df[(df.Stroke == 144) | (df.Stroke == 128) | (df.Stroke == 177)]

        # Removes the duplicate foot pedal speed data; foot pedal data can now be IDed as Stroke=177 and Soeed=0
        df = df[-((df.Stroke == 177) & (df.Speed == 127))]

        # Create a unique csv name based on module and date
        filename = date.strftime('%Y-%m-%d') + module + '.csv'

        # Write the data to the datastorace csv
        df.to_csv(filename, mode='w', header=True, index=True)

        time.sleep(0.01)


import os
import yagmail

gmail = os.getenv("GMAIL")
gmailpw = os.getenv("GMAILPW")

def send_email(subject):
    try:
        content = ['']
        with yagmail.SMTP(gmail, gmailpw) as yag:
            yag.send(gmail, subject, content)
        print("Email sent successfully")
    except:
        print("Error in sending email")


import operator
from datetime import datetime
from typing import Optional

from aiogram import Bot

import app.chain as abstract_chain

class KufarHandlerChain(abstract_chain.AbstractHandlerChain):
    @staticmethod
    def action_name():
        return "kuf"

    def __init__(self, bot: Optional[Bot]):
        self.bot = bot

    @staticmethod
    def site_name():
        return "kufar.by"

    async def search_request(self, query, date: datetime = None) -> list:
        saved_time = datetime.now()
        if not date:
            saved_time = datetime.now()
        if isinstance(date, str):
            saved_time = datetime.fromisoformat(date)

        response = await super().get_request(
            "https://cre-api-v2.kufar.by/items-search/v1/engine/v1/search/rendered-paginated",
            {'rgn': "7", 'ot': "1", 'query': query, 'size': "42", 'lang': "ru"}
        )
        response_elements = response['ads']
        if len(response_elements) > 0:
            result = []
            for elem in response_elements:
                response_time = datetime.strptime(elem.get('list_time'), '%Y-%m-%dT%H:%M:%SZ')
                if saved_time < response_time:
                    result.append(self._search_request_object(response_time, elem.get('ad_link')))
            if len(result) > 0:
                sorted(result, key=operator.itemgetter('time'), reverse=True)
                return result
            elif not date:
                last_response_elem = sorted(response_elements, key=operator.itemgetter('list_time'), reverse=True)[0]
                response_time = datetime.strptime(last_response_elem.get('list_time'), '%Y-%m-%dT%H:%M:%SZ')
                return [self._search_request_object(response_time, last_response_elem.get('ad_link'))]
        else:
            if not date:
                return [self._search_request_object(saved_time)]
            return []

    @classmethod
    def _search_request_object(cls, time, url=None):
        return {'time': time.isoformat(), 'link': url}


from data_loading import SpotifyRecommenderDataset
import visualization
from architecture import Autoencoder
from train import train_autoencoder
from webapp import WebApp
import torch
import os
import argparse
import shutil
from model_testing import test

model_store_path = "./model.pth"
dataframe_store_path = "data/spotify_recommender_dataset"

parser = argparse.ArgumentParser()
parser.add_argument("-t", "--train", help="force retraining of the model", action="store_true")
parser.add_argument("-v", "--verbose", help="verbose mode", action="store_true")
parser.add_argument("-c", "--clean", help="clean all cached dataframe files in \
    data/spotify_recommender_dataset/ and produce new ones (might take some time)", action="store_true")
parser.add_argument("-n", "--num-epochs", help="the number of epochs you want the model to \
    train. Only useful with -t option.", type=int, default=10)
args = parser.parse_args()

if args.clean and os.path.exists(dataframe_store_path):
    shutil.rmtree(dataframe_store_path)

trainModel = args.train
if not os.path.exists(model_store_path) or not os.path.exists(dataframe_store_path):
    trainModel = True

dataset = SpotifyRecommenderDataset()
if trainModel:
    print("starting to train model...")
    train_dataloader = torch.utils.data.DataLoader(dataset, batch_size=256, shuffle=True)
    model = Autoencoder()
    model = train_autoencoder(model, train_dataloader, verbose=args.verbose, num_epochs=args.num_epochs)
    with torch.no_grad():
        print("Evaluation loss:", test(model, dataset.model_input_tensor))
        encodings = model.encode(dataset.model_input_tensor)
    dataset.add_encoding_tensor(encodings)
    torch.save(model, model_store_path)

knn = visualization.KNN(dataset)
webapp = WebApp(dataset, knn)

webapp.run()


from django.shortcuts import render, redirect, get_object_or_404
from django.utils import timezone
from .models import Post
from .forms import PostForm
import requests
from bs4 import BeautifulSoup
import re
from unidecode import unidecode as un

def post_list(request):
    """
    A function that retrieves and returns plant siting posts information ordered by decreasing published date.
    For the submission form in the page if a POST request is detected, save the inputted information and return it to the database.
    Otherwise, render an empty form"""
    posts = Post.objects.filter(published_date__lte=timezone.now()).order_by('published_date')

    if request.method == "POST":
        form = PostForm(request.POST, request.FILES)
        if form.is_valid():
            post = form.save(commit=False)
            post.author = request.user
            post.published_date = timezone.now()
            post.save()
            return redirect("/")
    elif request.method == "GET":
        form = PostForm()
    return render(request, 'sitings/post_list.html', {'posts': posts, 'form':form})

def post_detail(request, pk):
    """ After clicking a plant siting on the main page send the plant name to a natural products database,
    scrape the contents and return chemicals found in the plant/mushroom.  Count the chemicals according
    to chemical structure by checking suffixes and send the count of compounds to the page to be displayed

    """
    post = get_object_or_404(Post,pk=pk)
    url = 'http://pkuxxj.pku.edu.cn/UNPD/print_fz_zwln.php'
    r = requests.post(url, data={'fangji':str(post)}) #Insert the plant name into the input box
    plant_data = r.content
    BS_data = BeautifulSoup(plant_data, "lxml")
    decoded_data = BS_data.decode('utf-8') #Convert from bytes to string
    cleaned_data = decoded_data.split('<td>\n')
    number_cleaner = "^\+?(\d+-?)*\d+$" #Regex to remove extraneous numbers (e.g. 23-5124-1251)
    chem_raw = []
    chems = []
    flavonoids = ('one','in',)
    flavonoid_count = 0
    terpenes = ('sterol', 'ene', 'ol','en','enin',)
    terpene_count = 0
    amines = ('ine',)
    amine_count = 0
    acids = ('acid','saeure',)
    acid_count = 0
    glycoside_count = 0

    for chem in cleaned_data:
        if '|' in chem:
            chem_raw.append((chem.replace('</td>', '').replace('</n>', '').replace('<n>','').strip().split('|')[-1].
            capitalize())) #identify tr elements containing chemicals by the presence of |
            # The | is used to separate chemical synonyms and only one name is wanted so the last one is retrieved
            # Example: compounds = 'Bicyclo[3.1.1.]hept-2-ene,2,6,6-trimethyl | 2-Pinene' ---> result = '2-Pinene'
    for value in chem_raw:
        if not re.match(number_cleaner, value): #Remove strings composed of only numbers and dashe
            if not value.startswith('['):
                chems.append(value)
    for compound in chems: # Check the suffixes and count occurrences
        if compound.endswith(flavonoids):
            flavonoid_count += 1
        elif compound.endswith(terpenes):
            terpene_count += 1
        elif compound.endswith(amines):
            amine_count += 1
        elif compound[:-2].endswith('side') or compound.endswith('side'):
            glycoside_count += 1
        if compound.endswith(acids):
            acid_count += 1

    return render(request, 'sitings/post_detail.html', {'post': post, 'chems':sorted(chems),
    'flavonoid_count':flavonoid_count, 'terpene_count':terpene_count, 'amine_count':amine_count,
    'acid_count':acid_count, 'glycoside_count':glycoside_count})


#
# Arquivo com exemplos para manipulação de dados na Internet
#

import urllib.request
from bs4 import BeautifulSoup

def ConectaInternet():
    objUrl = urllib.request.urlopen("https://www.investopedia.com/terms/f/forex.asp")

    if objUrl.getcode() == 200:
        dados = objUrl.read()
        soup = BeautifulSoup(dados, 'html.parser')
        print(soup.get_text())

ConectaInternet()

Python 3 - WebScraping/conexaoInternet_finished.py


from django.db import models
from django.utils.html import format_html
import django_tables2 as tables

from fornecedores.models import Fornecedores

# Create your models here.

class NotasFiscais(models.Model):
    codnotafiscal = models.IntegerField(
        db_column="codnotafiscal", blank=True, null=False, primary_key=True
    )
    numeronotafiscal = models.CharField(
        db_column="numeronotafiscal", max_length=255, blank=True, null=True
    )

    fornecedor = models.ForeignKey(
        Fornecedores,
        on_delete=models.PROTECT,
        db_column="codfornecedor",
        to_field="codfornecedor",
        blank=True,
        null=True,
    )

    datanotafiscal = models.DateField(db_column="datanotafiscal", blank=True, null=True)
    valornotafiscal = models.DecimalField(
        db_column="valornotafiscal",
        blank=True,
        null=True,
        max_digits=10,
        decimal_places=2,
    )
    observacao = models.CharField(
        db_column="observacao", max_length=255, blank=True, null=True
    )
    formapagamento = models.CharField(
        db_column="formapagamento", max_length=255, blank=True, null=True
    )

    class Meta:
        managed = False
        db_table = "notasfiscais"

    def __str__(self):
        return self.numeronotafiscal[:20]

    def save(self, *args, **kwargs):
        if not self.codnotafiscal:
            max = NotasFiscais.objects.aggregate(models.Max("codnotafiscal"))[
                "codnotafiscal__max"
            ]
            self.codnotafiscal = max + 1
        super().save(*args, **kwargs)

class NotasFiscaisTable(tables.Table):
    codnotafiscal = tables.Column(
        verbose_name="Cód", attrs={"th": {"class": "text-center"}}
    )
    numeronotafiscal = tables.Column(verbose_name="Nota Fiscal")
    fornecedor = tables.Column(verbose_name="Fornecedor")
    datanotafiscal = tables.Column(verbose_name="Data", localize=True)
    valornotafiscal = tables.Column(
        verbose_name="Valor", attrs={"th": {"class": "text-right"}}
    )
    observacao = tables.Column(
        verbose_name="Obs.", attrs={"th": {"class": "text-center"}}, default=""
    )
    formapagamento = tables.Column(verbose_name="Forma de Pagamento")

    def render_valornotafiscal(self, value):
        return "R$ {:0.2f}".format(value).replace(".", ",")

    def render_datanotafiscal(self, value):
        return value.strftime("%d/%m/%Y")

    def render_observacao(self, value):
        html = "   "

        return format_html(html, value)

    class Meta:
        model = NotasFiscais
        fields = [
            "codnotafiscal",
            "numeronotafiscal",
            "fornecedor",
            "datanotafiscal",
            "valornotafiscal",
            "observacao",
            "formapagamento",
        ]

# codcontapagar, codnotafiscal, parcela, datavencimento, valorparcela, datapagamento, formapagamento, observacao

class ContasPagar(models.Model):
    codcontapagar = models.IntegerField(
        db_column="codcontapagar", blank=True, null=False, primary_key=True
    )

    notafiscal = models.ForeignKey(
        NotasFiscais,
        on_delete=models.PROTECT,
        db_column="codnotafiscal",
        to_field="codnotafiscal",
        blank=True,
        null=True,
    )

    parcela = models.CharField(
        db_column="parcela",
        max_length=255,
        blank=True,
        null=True,
    )

    datavencimento = models.DateField(db_column="datavencimento", blank=True, null=True)
    valorparcela = models.DecimalField(
        db_column="valorparcela",
        blank=True,
        null=True,
        max_digits=10,
        decimal_places=2,
    )

    datapagamento = models.DateField(db_column="datapagamento", blank=True, null=True)

    formapagamento = models.CharField(
        db_column="formapagamento", max_length=255, blank=True, null=True
    )
    observacao = models.CharField(
        db_column="observacao", max_length=255, blank=True, null=True
    )

    def save(self, *args, **kwargs):
        if not self.codcontapagar:
            max = ContasPagar.objects.aggregate(models.Max("codcontapagar"))[
                "codcontapagar__max"
            ]
            self.codcontapagar = max + 1
        super().save(*args, **kwargs)

    class Meta:
        managed = False
        db_table = "contaspagar"

class ContasPagarTable(tables.Table):
    class Meta:
        model = ContasPagar


import torch

def expected_gaussian_kernel(X, Y, gamma=1.0):
    """Compute the average of k(X, Y) where k is the gaussian rbf kernel with
    γ=`gamma`.

    In particular, compute

        k(X, Y) = ||exp(-γ ||Χ-Υ||_2^2)||_1 / N / M
                = ||exp(-γ (||Χ||^2 + ||Υ||^2 - 2Χ Υ^Τ))||_1 / N / M

    Arguments
    ---------
        X: tensor (N, D) Features sampled from the first distribution
        Y: tensor (M, D) Features sampled from the second distribution
        gamma: The γ parameter for the rbf kernel.
    """
    N, D = X.shape
    M, _ = Y.shape

    X2 = torch.einsum("nd,nd->n", X, X)
    # minor optimization for the case that Y = X
    if Y is X:
        Y2 = X2
    else:
        Y2 = torch.einsum("md,md->m", Y, Y)
    XY = torch.einsum("nd,md->nm", X, Y)

    # prepare them for addition by making sure
    # X2.shape == (N, 1) and Y2.shape == (1, M)
    X2 = X2.view(N, 1)
    Y2 = Y2.view(1, M)

    return torch.exp(-gamma * (X2 + Y2 - 2*XY)).mean()

def mmd(X, Y, gamma=1.0):
    """Compute the mmd loss with gaussian rbf kernel with γ=`gamma`.

    This function computes

        MMD(X, Y) ~= E_p,p[k(x, x')] + E_q,q[k(y, y')] - 2 E_p,q[k(x, y)]

    Arguments
    ---------
        X: tensor (N, D) Features sampled from the first distribution
        Y: tensor (M, D) Features sampled from the second distribution
        gamma: The γ parameter for the rbf kernel.
    """
    Exx = expected_gaussian_kernel(X, X, gamma)
    Eyy = expected_gaussian_kernel(Y, Y, gamma)
    Exy = expected_gaussian_kernel(X, Y, gamma)

    return Exx + Eyy - 2 * Exy


r'''
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import collections
import glob
import os
import stat
import socket
import ssl
import sys
import uuid
import logging
import urllib

from OpenSSL import crypto
from nogotofail.clients.linux.pyblame import util

Application = collections.namedtuple("Application", ["application", "version"])
class BlameConnection(object):
    host = ""
    port = -1
    install_id = ""
    blame_sock = None
    resp_headers = {}
    probability = 0
    override_attacks = True
    attacks = []
    data_attacks = []
    platform_info = ""
    ssl = True
    fingerprint_callback = None
    vuln_callback = None
    info_callback = None
    handshake_completed = False

    def __init__(self, host, port, ssl=True, fingerprint_callback=None,
            install_id=None, platform_info="Unix", probability=None,
            attacks=None, data_attacks=None,
            vuln_callback=None, info_callback=None):
        self.host = host
        self.port = port
        self.ssl = ssl
        self.fingerprint_callback = fingerprint_callback
        self.install_id = install_id or uuid.uuid4()
        self.platform_info = platform_info
        self.probability = probability
        self.attacks = attacks
        self.data_attacks = data_attacks
        self.vuln_callback = vuln_callback
        self.info_callback = info_callback
        self.logger = logging.getLogger("pyblame")

    def build_headers(self):
        """Create the dictionary headers to send the server in the handshake.
        """
        headers = {}
        headers["Installation-ID"] = self.install_id
        headers["Platform-Info"] = self.platform_info
        if self.attacks is not None:
            headers["Attacks"] = ",".join(self.attacks)
        if self.data_attacks is not None:
            headers["Data-Attacks"] = ",".join(self.data_attacks)
        if self.probability is not None:
            headers["Attack-Probability"] = self.probability
        return headers

    def handshake(self):
        """Do the blame handshake with the server.

        connect MUST have been called before this
        """
        if not self.blame_sock:
            raise ValueError("Connection not created before call to handshake")
        headers = self.build_headers()
        f = self.blame_sock.makefile()
        f.write("nogotofail_ctl/1.0\n" +
                "\n".join(("%s: %s" % (key, value) for key, value in headers.iteritems())) +
                "\n\n")
        f.flush()

        line = f.readline().strip()
        if line.split(" ")[0] != "0":
            raise ValueError("Server rejected handshake")
        while True:
            line = f.readline().strip()
            if not line:
                break
            key, value = line.split(":", 1)
            self.resp_headers[key.strip()] = value.strip()
        self.handshake_completed = True

    def connect(self):
        """Create a connection with the blame server.
        If self.ssl this will also do the SSL handshake
        """
        sock = socket.create_connection((self.host, self.port))
        self.logger.info("Connected...")
        if self.ssl:
            sock = ssl.wrap_socket(sock)
            cert = crypto.load_certificate(crypto.FILETYPE_ASN1, sock.getpeercert(True))
            digest = cert.digest("sha256")
            if self.fingerprint_callback and not self.fingerprint_callback(digest):
                raise Exception("Untrusted endpoint")
            self.logger.info("SSL connection established")
        self.blame_sock = sock

    def on_vuln_notify(self, params):
        """Called when a vuln_notify command is sent from the server
        Command arguments format is:
        connection_id vuln_type server_addr server_port app1 version1, app2 version 2..

        connection_id is an opaque blob identifying the connection (currently a uuid4)
        vuln_type is one of nogotofail.mitm.util.vuln.*
        server_addr and server_port are the remote of the vulnerable connnection.

        In addition there are 0 or more applications sent, each a (name, version) tuple as
        sent by a response to a tcp_client_id request
        """
        id, type, server_addr, server_port = params[:4]
        remaining = " ".join(params[4:])
        applications = remaining.split(", ")
        applications = [Application(*map(urllib.unquote, app.split())) for app in applications]
        if self.vuln_callback:
            self.vuln_callback(id, type, server_addr, server_port, applications)
        return "OK"

    def on_tcp_client_id(self, params):
        """Called when a tcp_client_id command is sent from the server
        Command arguments format is:
        source_port encoded_dest_ip dest_port

        source_port is the source port of the connection
        encoded_ip is the hex encoded output of socket.inet_pton of the destination IP
        dest_port is the destination port in the connection
        """
        source_port = int(params[0])
        dest_ip = None
        dest_port = -1
        if len(params) > 1:
            dest_ip_bytes = params[1].decode("hex")
            if len(dest_ip_bytes) == 4:
                family = socket.AF_INET
            elif len(dest_ip_bytes) == 16:
                family = socket.AF_INET6
            else:
                raise ValueError("Wrong size dest_ip")
            dest_ip = socket.inet_ntop(family, dest_ip_bytes)
        if len(params) > 2:
            dest_port = int(params[2])
        if self.info_callback:
            applications = self.info_callback(source_port, dest_ip, dest_port)
            if applications:
                return ", ".join(("%s %s" % (urllib.quote(app.application), app.version)
                    for app in applications))
        return ""

    command_handlers = {
      "tcp_client_id" : on_tcp_client_id,
      "vuln_notify": on_vuln_notify,
            }

    def run(self):
        """Run the command loop for the blame connection.

        handshake() MUST have been called before this.
        """
        if not self.handshake_completed:
            raise ValueError("Handshake not completed")
        f = self.blame_sock.makefile()
        while True:
            line = f.readline().strip()
            if not line:
                break
            params = line.split(" ")
            tx_id, command = params[:2]
            handler = self.command_handlers.get(command)
            reply_payload = None
            if handler:
                reply_payload = handler(self, params[2:])
            else:
                reply_payload = "ERROR: Unknown command"
            if not reply_payload:
                reply_payload = ""
            f.write("%s %s\r\n" % (tx_id, reply_payload))
            f.flush()


from .utils import get_from_github
import distutils.ccompiler

import subprocess
from distutils.cmd import Command
from distutils.spawn import find_executable
import sys

import os

class build_ninja(Command):
    description = "build Ninja build system"

    user_options = [
        ('build-base='         , None, 'base directory for LLVM/Clang'),
        ('ninja-build-dir='    , None, 'build directory for Ninja'),
        ('ninja-github-user='  , None, 'Github Ninja username'),
        ('ninja-github-repo='  , None, 'Github Ninja repository'),
        ('ninja-github-branch=', None, 'Github Ninja branch'),
        ('compiler='           , None, 'Compiler'),
    ]

    def initialize_options(self):
        self.build_base = None
        self.compiler = None
        self.ninja_build_dir = 'ninja'
        self.ninja_github_user = 'martine'
        self.ninja_github_repo = 'ninja'
        self.ninja_github_branch = 'release'

    def finalize_options(self):
        self.set_undefined_options('build',
            ('build_base', 'build_base'),
            ('compiler', 'compiler'),)

    def run(self):
        if self.compiler is None:
            self.compiler = distutils.ccompiler.get_default_compiler()
        bootstrap_args = []
        win64 = sys.maxsize > 2**32
        if self.compiler == 'msvc':
            bootstrap_args = ['--platform=msvc']
            distutils.msvc9compiler.VERSION = 11.0
            compiler = distutils.ccompiler.new_compiler(compiler='msvc')
            compiler.initialize('win-amd64' if win64 else 'win32')
        elif self.compiler == 'mingw32':
            bootstrap_args = ['--platform=mingw']

        ninja_info = dict(user=self.ninja_github_user,
            repo=self.ninja_github_repo, branch=self.ninja_github_branch)
        self.ninja_build_dir += '_' + self.compiler
        final_dir = os.path.join(self.build_base, self.ninja_build_dir)
        if os.path.isdir(final_dir):
            print("Found '{}', assuming it contains {}:{}".format(
                final_dir, ninja_info['repo'], ninja_info['branch']))
        else:
            get_from_github(ninja_info, final_dir, self.build_base)
        self.ninja_exe = find_executable('ninja', final_dir)
        if self.ninja_exe is None:
            subprocess.check_call([sys.executable, 'bootstrap.py'] +
                bootstrap_args, cwd=final_dir)
            self.ninja_exe = find_executable('ninja', final_dir)
            assert self.ninja_exe