import rsa, hashlib

class Chain:
	def __init__(self):
		self.validatedBlocks = [Block(self, 'Genisis Block')]
		self.validatedBlocks[0].hash = hashlib.sha256('Genisis Block')
		self.pendingBlocks = []
		self.transactionBlock = Block(self, hashlib.sha256('Genisis Block'))
		self.accounts = {}
		self.validators = []

class Block:
	def __init__(self, chain, prevBlockHash):
		self.transactions = []
		self.chain = chain
		self.capacity = 3
		self.previousHash = prevBlockHash
		self.hash = None
		self.data = ''
		self.votes = []

	def addTransaction(self, transaction):
		self.transactions.append(transaction)
		if len(self.transactions) >= self.capacity:
			self.data = ' | '.join([' - '.join((transaction.sender, transaction.amount, transaction.reciver)) \
				for transaction in self.transactions] + ' || ' + self.previousHash).encode('ascii')
			self.hash = hashlib.sha256(self.data)
			self.chain.pendingBlocks.append(self)

	def vote(self, vote, blockSignature, publicKey):
		if publicKey in self.chain.validators and rsa.decrypt(blockSignature, publicKey) == self.data:
			self.votes.append((vote, blockSignature, publicKey))

class Transaction:
	def __init__(self, amount, senderPubKey, senderPrivKey, reciver):
		self.amount = amount
		self.sender = senderPubKey
		self.reciver = reciver
		self.signature = self.sign(senderPrivKey)

	def sign(self, key):
		data = f'{self.sender} sending {self.amount} to {self.reciver}'
		return rsa.encrypt(data.encode('ascii'), key)

class Wallet:
	def __init__(self, chain):
		self.balance = 0
		self.publicKey, self.privateKey = rsa.newkeys(4096)
		chain.accounts[self.publicKey] = 0

	def send(self, amount, reciver):
		transaction = Transaction(amount, self.publicKey, self.privateKey, reciver)

class Validator:
	def __init__(self, chain, wallet):
		self.chain = chain
		self.wallet = wallet
		self.chain.validators.append(self.wallet.publickKey)

	def validateBlock(self):
		block = self.chain.pendingBlocks[0]
		for transaction in block.transactions:
			if rsa.decrypt(transaction.signature, transaction.sender) == \
			f'{transaction.sender} sending {transaction.amount} to {transaction.reciver}' and \
			self.chain.accounts[transaction.sender] >= transaction.amount:
				block.vote(True, rsa.encrypt(block.data.encode('ascii'), self.wallet.privateKey), self.wallet.publicKey)


from unittest import skip
from django.http import HttpRequest
from django.test import TestCase, Client, RequestFactory
from django.urls import reverse

from store.models import Product, Category
from store.views import ProductDetailView

@skip("demonstrating skipping")
class TestSkip(TestCase):
    def test_skip_example(self):
        pass

class TestViewResponses(TestCase):
    def setUp(self):
        self.client = Client()
        self.factory = RequestFactory()

        self.category1 = Category.objects.create(
            name="Books",
            slug="books",
            description="Book Category",
            # image = models.ImageField(blank=True),
        )

        self.category2 = Category.objects.create(
            name="CDs",
            slug="cds",
            description="CD Category",
            # image = models.ImageField(blank=True),
        )

        self.product1 = Product.objects.create(
            name="Book Product",
            slug="book_product",
            description="This is Testing Product.",
            price=1000,
            image="test_image",
            stock=100,
        )
        self.product1.categories.add(self.category1)

        self.product2 = Product.objects.create(
            name="CD Product",
            slug="cd_product",
            description="This is Testing Product.",
            price=1000,
            image="test_image",
            stock=100,
        )
        self.product2.categories.add(self.category2)

    def test_allowed_host(self):
        """
        Test allowed host.
        """
        response = self.client.get('/')
        self.assertEqual(response.status_code, 200)

    def test_store_page(self):
        """
        Test store response status
        """

        # Send get request to index page and store response
        response = self.client.get('/store/')

        # Make sure status code is 200
        self.assertEqual(response.status_code, 200)

        # Make sure all products are returned in the context
        self.assertEqual(response.context["products"].count(), 2)

    def test_products_by_category(self):
        """
        Test products_by_category filter
        """

        # Send get request to index page and store response
        response = self.client.get(reverse('store:products_by_category', args=['books']))
        # response = self.client.get('/store/categories/books/')

        # Make sure status code is 200
        self.assertEqual(response.status_code, 200)

        # Make sure just selected category products are returned in the context
        self.assertEqual(response.context["products"].count(), 1)

    def test_product_detail_page(self):
        """
        Test product_detail page response status
        """

        # Send get request to index page and store response
        response = self.client.get(reverse('store:product_detail', args=['book_product']))
        # response = self.client.get('/store/products/book_product/')

        # Make sure status code is 200
        self.assertEqual(response.status_code, 200)

        # Make sure Product is correct.
        html = response.content.decode('utf8')
        self.assertIn('Book Product', html)

    def test_no_product_found(self):
        """
        Test no matching product
        """
        response = self.client.get('/store/products/dummy/')
        self.assertEqual(response.status_code, 404)


from models.DataLoader import DataLoader
from models.DataSet import DataSet
from models.models import DeepFMatNet, DeepFMatAlex, DeepFMatVGG16, DeepFMatResNet18
from models.Regularizer import L2Regularizer, L1Regularizer
import torch.optim as optim
import torch.nn as nn
from torch.optim.lr_scheduler import StepLR
import numpy as np
import torch
import os
import cv2
import pickle
import time
from torch.utils.tensorboard import SummaryWriter
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
import argparse
import datetime

# torch.autograd.set_detect_anomaly(True)

def epipoline(x, formula):
    '''

    :param x:
    :param formula:
    :return:
    '''
    array = formula.flatten()
    a = array[0]
    b = array[1]
    c = array[2]
    return int((-c - a * x) / b)

def verify_xfx(line, point):
    '''

    :param line:
    :param point:
    :return:
    '''
    l = np.array(line).flatten()
    a = l[0]
    b = l[1]
    return abs(line.dot(point))/np.sqrt(a*a+b*b)

def visualize2Images(args, left_paths, right_paths, f_mats, loss, epoch, img_idx, visualizeDir = "visualization"):
    '''

    :param args:
    :param left_paths:
    :param right_paths:
    :param f_mats:
    :param loss:
    :param epoch:
    :param img_idx:
    :param visualizeDir:
    :return:
    '''
    colors = [
        (255, 102, 102),
        (102, 255, 255),
        (125, 125, 125),
        (204, 229, 255),
        (0, 0, 204)
    ]
    f_mats = f_mats.cpu().numpy()
    THRESHOLD = 0.12
    sift = cv2.xfeatures2d.SIFT_create()
    bf = cv2.BFMatcher()
    images = []
    errors = {}
    for idx, (left_path, right_path, f_mat) in enumerate(zip(left_paths, right_paths, f_mats)):
        f_mat = np.array(f_mat.reshape((3,3)))

        left_img = cv2.imread(left_path)
        #-------
        hl, wl = left_img.shape[0], left_img.shape[1]
        left_img = left_img[int(hl / 2) - 128: int(hl / 2) + 128, int(wl / 2) - 128: int(wl / 2) + 128]
#--------------------------

        left_imgG = cv2.cvtColor(left_img.copy(), cv2.COLOR_BGR2GRAY)
        left_img_line = left_img.copy()

        right_img = cv2.imread(right_path)
#-------------------------------
        hr, wr = right_img.shape[0], right_img.shape[1]
        right_img = right_img[int(hr / 2) - 128: int(hr / 2) + 128, int(wr / 2) - 128: int(wr / 2) + 128]

        right_imgG = cv2.cvtColor(right_img.copy(), cv2.COLOR_BGR2GRAY)
        right_img_line = right_img.copy()

        (kps_left, descs_left) = sift.detectAndCompute(left_imgG, None)
        (kps_right, descs_right) = sift.detectAndCompute(right_imgG, None)

        matches = bf.knnMatch(descs_left, descs_right, k=2)
        good = []
        for m, n in matches:
            if m.distance < THRESHOLD * n.distance:
                good.append([m])

        err_l = []
        err_r = []
        img_W = left_img.shape[1] - 1
#---------------------------------------------------------------------
        for color_idx, g in enumerate(good):
            # get the ids of matching feature points
            id_l, id_r = g[0].queryIdx, g[0].trainIdx
            # x: column
            # y: row
            # get the feature points in both left and right images
            x_l, y_l = kps_left[id_l].pt
            x_r, y_r = kps_right[id_r].pt

            '''Color for line'''
            color = colors[color_idx % len(colors)]

            '''Epi line on the left image'''
            # epi line of right points on the left image
            point_r = np.array([x_r, y_r, 1])
            line_l = np.dot(f_mat.T, point_r)

            # calculating 2 points on the line
            y_0 = epipoline(0, line_l)
            y_1 = epipoline(img_W, line_l)
            # drawing the line and feature points on the left image
            left_img_line = cv2.circle(left_img_line, (int(x_l), int(y_l)), radius=4, color=color)
            left_img_line = cv2.line(left_img_line, (0, y_0), (img_W, y_1), color=color, lineType=cv2.LINE_AA)
            # displaying just feature points
            left_img = cv2.circle(left_img, (int(x_l), int(y_l)), radius=4, color=color)

            '''Epi line on the right image'''
            # epi line of left points on the right image
            point_l = np.array([x_l, y_l, 1])
            line_r = np.dot(f_mat, point_l)

            # verifying points
            err_R = verify_xfx(line_r, point_r)
            err_r.append(err_R)
            # verifying points
            err_L = verify_xfx(line_l, point_l)
            err_l.append(err_L)
            # calculating 2 points on the line
            y_0 = epipoline(0, line_r)
            y_1 = epipoline(img_W, line_r)

            # drawing the line on the right image
            right_img_line = cv2.circle(right_img_line, (int(x_r), int(y_r)), radius=4, color=color)
            right_img_line = cv2.line(right_img_line, (0, y_0), (img_W, y_1), color=color, lineType=cv2.LINE_AA)
            # displaying just feature points
            right_img = cv2.circle(right_img, (int(x_r), int(y_r)), radius=4, color=color)

        l_avgErr = np.average(err_l) if err_l else 0
        r_avgErr = np.average(err_r) if err_r else 0

        vis = np.concatenate((left_img_line, right_img_line), axis=0)
        font = cv2.FONT_HERSHEY_SIMPLEX

        img_H = vis.shape[0]
        cv2.putText(vis, '{:.4f}'.format(float(l_avgErr)), (10, 20), font, 0.3, color=(0, 255, 0), lineType=cv2.LINE_AA)
        cv2.putText(vis, '{:.4f}'.format(float(r_avgErr)), (10, img_H - 10), font, 0.3, color=(0, 255, 0), lineType=cv2.LINE_AA)
        cv2.putText(vis, '{:.4f}'.format(float(loss.data.cpu())), (int(img_W-img_W/2), img_H - 10), font, 0.3, color=(0, 255, 0), lineType=cv2.LINE_AA)

        sqResultDir = os.path.join(ROOT_DIR, visualizeDir, '{}'.format(epoch))
        if not os.path.exists(sqResultDir):
            os.makedirs(sqResultDir)

        cv2.imwrite(os.path.join(sqResultDir, 'epipoLine_sift_batch{}_img{}.png'.format(img_idx, idx)), vis)
        print("Writing image ... " + 'epipoLine_sift_batch{}_img{}.png'.format(img_idx, idx))
        images.append(vis)
        errors['batch{}_img{}_left'.format(img_idx, idx)] = l_avgErr
        errors['batch{}_img{}_right'.format(img_idx, idx)] = r_avgErr
    return np.array(images), errors

def training(args, model, device, trainLoader, optimizer, criterion, epoch, writer, allParamsRegularized= False):
    '''
    Training the model.
    :param args: input arguments
    :param model: training model
    :param device: device
    :param trainLoader: training loader
    :param optimizer: optimizer
    :param criterion: criterion
    :param epoch: current epoch
    :param writer: tensorboard writer
    :param allParamsRegularized: regularize all params?
    :return:
    '''
    # enter train mode
    model.train()
    # saving losses
    totalLoss = []
    print(50 * "*")
    print("Training Epoch ... ", epoch)
    print(50 * "*")

    # number of batches in the training dataset
    l = len(trainLoader)
    # length of the whole training dataset.
    L = len(trainLoader.dataset)
    # Regularizer
    reg_loss = L2Regularizer(model=model, lambda_reg=0.01)
    for batch_idx, (data, target, (_,_)) in enumerate(trainLoader):
        data, target = data.to(device, dtype=torch.float), target.to(device, dtype=torch.float)
        optimizer.zero_grad()
        output = model(data)
        loss = criterion(output, target)
        if allParamsRegularized:
            loss = reg_loss.regularized_all_param(reg_loss_function=loss)
        totalLoss.append(loss.item())

        loss.backward()

        optimizer.step()
        # writing the loss of the current batch to tensorboard
        writer.add_scalar('Batch Loss',
                          loss.data.cpu(),
                          epoch * l + batch_idx)
        if batch_idx % args.log_interval == args.log_interval-1:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, (batch_idx+1) * len(data), L ,
                       100. * (batch_idx+1) / l, loss.data.cpu()))

    # writing the loss of the current epoch to tensorboard
    writer.add_scalars('Train Epoch Loss', {'Training': np.mean(np.array(totalLoss))}, epoch)
    writer.flush()

def validating(args, model, device, valLoader, epoch, writer, log, trainloader=None):
    '''

    :param args:
    :param model:
    :param device:
    :param valLoader:
    :param epoch:
    :param writer:
    :param log:
    :param trainloader:
    :return:
    '''
    visualDir = "visualization/model_{}".format(args.exp)
    model.eval()
    totalLoss = []
    trainTotalLoss = []
    criterion = nn.MSELoss()

    print(50 * "*")
    print("Validating Epoch ... ", epoch)
    print(50 * "*")
    L = len(valLoader.dataset)
    with torch.no_grad():
        for id, (data, target, (left_img, right_img)) in enumerate(valLoader):
            data, target = data.to(device, dtype=torch.float), target.to(device, dtype=torch.float)
            output = model(data)
            loss = criterion(output, target)
            totalLoss.append(loss.item())

            if id == 0:
                print("Visualize ... Batch {}".format(id))
                imageBatch, errors = visualize2Images(args, left_img, right_img, output, loss, epoch, id, visualizeDir=visualDir)
                log["epoch_{}_batch_{}".format(epoch, id)] = errors

                nameErr = 'batch{}_img{}_right'.format(id, 0)
                print(nameErr)
                writer.add_scalars("Testing Image Errors", {"exp_{}_{}".format(args.exp, nameErr): errors[nameErr]}, epoch)
                writer.add_images("VisualResult_Exp_{}".format(args.exp),  imageBatch, global_step=epoch, dataformats='NHWC')

        if trainloader:
            for id, (data, target, (_, _)) in enumerate(trainloader):
                data, target = data.to(device, dtype=torch.float), target.to(device, dtype=torch.float)
                output = model(data)
                loss = criterion(output, target)
                trainTotalLoss.append(loss.item())

    valMean = np.mean(np.array(totalLoss))
    writer.add_scalars('Epoch Loss',{'Validate': valMean} ,epoch)
    if trainloader:
        trainMean = np.mean(np.array(trainTotalLoss))
        writer.add_scalars('Epoch Loss', {'Train': trainMean}, epoch)
        print('Trainning Set: Average Error: {:.6f}'.format(trainMean))
    writer.flush()
    print('Validation set: Average Error: {:.6f}. Length of set : {}.'.format(valMean, L))

def main():
    '''
    Running the models here
    :return:
    '''
    parser = argparse.ArgumentParser(description='DeepF_noCorrs')

    parser.add_argument('--deviceID', type=int, default=0, metavar='N',
                        help='The GPU ID (default: 0)')
    parser.add_argument('--batch-size', type=int, default=8, metavar='N',
                        help='batch size for training set (default: 8)')
    parser.add_argument('--test-batch-size', type=int, default=8, metavar='N',
                        help='batch size for testing set (default: 8)')
    parser.add_argument('--epochs', type=int, default=200, metavar='N',
                        help='number of epochs (default: 200)')
    parser.add_argument('--lr', type=float, default=0.0000001, metavar='LR',
                        help='learning rate (default: 0.0000001)')
    parser.add_argument('--exp', type=int, default=0, metavar='experiment ID',
                        help='naming the experiment ID (default: 0)')

    parser.add_argument('--log-interval', type=int, default=100, metavar='N',
                        help='how many iterations to wait before printing the loss status')

    parser.add_argument('--k-fold', type=int, default=3, metavar='1-9',
                        help='how many folds in the test set (default: 3)')

    parser.add_argument('--kth', type=int, default=0, metavar='0-10',
                        help='The kth fold (default: 0)')

    parser.add_argument("--model", type=str, default='deepfmat', metavar="deepfmat, resnet, vgg16, alex",
                        help='Selecting the training models (default: deepfmat)')

    parser.add_argument("--norm", type=str, default='ETR', metavar='ETR, ABS, FBN',
                        help="Selecting the normalization method (default: ETR)")

    args = parser.parse_args()

    # -------------Dataset Path-----------------------------------
    POSES_PATH = "/media/slark/Data/Projects/dataset/data_kitti/dataset/poses"
    SEQUENCE_PATH = "/media/slark/Data/Projects/dataset/data_kitti/dataset/sequences"
    log_dir = os.path.join("logs/batchLosses/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
    resultDir = os.path.join(ROOT_DIR, "analysis/experiment_{}_lr{}_batch{}".format(args.exp, args.lr, args.batch_size))
    # ------------------------------------------------------------

    # -------------Loading dataset--------------------------------
    db = DataSet(SEQUENCE_PATH, POSES_PATH)
    train, val, test = db.dataSets(k_fold=args.k_fold, kth=args.kth)
    poses = db.poses
    train_loader = DataLoader(dataSet=train
                              , poses=poses
                              , dType="train"
                              , camera=0)
    val_loader = DataLoader(dataSet=val
                            , poses=poses
                            , dType="validate"
                            , camera=0)
    # ------------------------------------------------------------
    use_cuda = torch.cuda.is_available()
    kwargs = {'num_workers': 4, 'pin_memory': True} if use_cuda else {}
    device = torch.device(args.deviceID if use_cuda else "cpu")
    trainLoader = torch.utils.data.DataLoader(train_loader
                                              , batch_size=args.batch_size
                                              , shuffle=True
                                              , **kwargs)
    valLoader = torch.utils.data.DataLoader(val_loader
                                            , batch_size=args.test_batch_size
                                            , shuffle=False
                                            , **kwargs)

    resultModelFile = os.path.join(resultDir, log_dir, "result_{}".format(args.exp))

    print(50*"*")
    print("Running experiment {}".format(args.exp))
    print(50 * "*")

    outputSize = 9
    writer = SummaryWriter(os.path.join(resultDir, log_dir))
    if args.model == "deepfmat":
        model = DeepFMatNet(outputSize=outputSize, norm=args.norm).to(device)
    elif args.model == "alex":
        model = DeepFMatAlex(outputSize=outputSize, norm=args.norm).to(device)
    elif args.model == "vgg16":
        model = DeepFMatVGG16(outputSize=outputSize, norm=args.norm).to(device)
    elif args.model == "resnet":
        model = DeepFMatResNet18(outputSize=outputSize, norm=args.norm).to(device)

    if os.path.isfile(resultModelFile):
        try:
            model.load_state_dict(torch.load(resultModelFile))
        except:
            print("Cannot load the saved model")

    optimizer = optim.Adam(model.parameters(), lr=args.lr)
    criterion = nn.MSELoss()
    scheduler = StepLR(optimizer, step_size=5, gamma=0.1)
    testErrorLog = os.path.join(resultDir, "log.txt")
    log = {}
    if os.path.isfile(testErrorLog):
        with open(testErrorLog, "rb") as f:
            log = pickle.load(f)
    for epoch in range(1, args.epochs+1):
        startTime = time.time()
        training(args, model, device, trainLoader, optimizer, criterion, epoch, writer)

        validating(args, model, device, valLoader, epoch, writer, log, trainloader=trainLoader)
        scheduler.step()
        torch.save(model.state_dict(), resultModelFile)
        torch.save(model.state_dict(), resultModelFile + "_epoch_{}".format(epoch))
        with open(testErrorLog, 'wb') as f:
            pickle.dump(log, f)
        endTime = time.time()
        writer.add_scalar('Time Epoch', endTime, epoch)
        print('--------{}--------\n'.format(endTime - startTime))
    writer.close()

if __name__ == '__main__':
    main()


import logging
import random

from chess.models.conf import DATA_TOURNAMENTS
from tinydb import Query

from chess.models.round import Round

class Tournament:
    """ """

    MAX_PLAYERS = 4
    table = DATA_TOURNAMENTS

    def __init__(
            self,

            name="",
            place="",
            start_date="",
            end_date="",
            rounds_number=4,
            current_round=-1,
            rounds_list=[],
            players_list=[],
            description="",
            status="created",  # créé, en cours, terminé,

    ) -> None:
        """ """
        self.name = name
        self.place = place
        self.start_date = start_date
        self.end_date = end_date
        self.rounds_number = rounds_number
        self.current_round = current_round
        self.rounds_list = rounds_list
        self.players_list = players_list
        self.description = description
        self.status = status

    def create(self):
        """ """
        self.table.insert(self.__dict__)

    def end_tournament(self):
        """ """
        Obj = Query()
        self.table.update({"status": "closed"}, Obj.name == self.name)

    def remove_one(self):
        """ """
        Obj = Query()
        self.table.remove(Obj.name == self.name)

    @classmethod
    def remove_all(cls):
        """ """
        cls.table.truncate()

    def update(self, tournament_value, value_to_change):
        """ """
        Obj = Query()
        self.table.update({tournament_value: value_to_change},
                          Obj.name == self.name)

    @classmethod
    def find_one(cls, data, value):
        """ Chercher un tournois """
        Obj = Query()
        t_list = cls.table.search(Obj[data] == value)

        if len(t_list) != 1:
            raise AttributeError("More than 1 tournament")
        return t_list[0]

    @classmethod
    def find_all(cls):
        """ """
        list_doc = cls.table.all()
        list_dict = [dict(doc) for doc in list_doc]

        return list_dict

    def add_player(self, ine_player: str):
        """ """
        if (self.status == "created") and (self.n_players < self.MAX_PLAYERS):

            if ine_player not in self.players_list:
                self.players_list.append(ine_player)
                self.table.update({'players_list': self.players_list},
                                  Query().name == self.name)
                logging.warning("Joueur inscrit")

            else:
                logging.warning(" Attention : joueur déjà inscrit")

        elif self.n_players >= self.MAX_PLAYERS:
            logging.warning("Maximum de joueurs atteint")

        else:
            logging.warning("Attention : tournois non créé")

    def add_players(self, players_list: list):
        """ """
        for player in players_list:
            self.add_player(player)

    def start_tournament(self):
        """ """
        if (self.status == "created") and (self.n_players == self.MAX_PLAYERS):
            self.status = "running"
            self.table.update({"status": self.status},
                              Query().name == self.name)

        elif self.status != "created":
            logging.error(f"Erreur de status {self.status}")
        else:
            logging.error("problème nombre de players")

    def create_first_round(self):
        """ """
        if self.current_round == -1 and self.status == "running":
            self.current_round += 1
            self.table.update({"current_round": self.current_round},
                              Query().name == self.name)

            shuffled_list = self.players_list
            random.shuffle(shuffled_list)
            match_1 = (f"{shuffled_list[0]}", 0), (f"{shuffled_list[1]}", 0)
            match_2 = (f"{shuffled_list[2]}", 0), (f"{shuffled_list[3]}", 0)
            matchs_list = [match_1, match_2]
            r = Round(matchs_list)
            r.create()
            self.rounds_list.append(r.round_id)
            self.table.update({"rounds_list": self.rounds_list},
                              Query().name == self.name)
            return r.round_id

    @classmethod
    def get_instance(cls, document):
        name = document["name"]
        place = document["place"]
        start_date = document["start_date"]
        end_date = document["end_date"]
        rounds_number = document["rounds_number"]
        current_round = document["current_round"]
        rounds_list = document["rounds_list"]
        players_list = document["players_list"]
        description = document["description"]
        status = document["status"]
        return Tournament(name, place, start_date, end_date, rounds_number,
                          current_round, rounds_list, players_list,
                          description, status)

    @property
    def scores(self):
        """ """
        if not self.n_players:
            return {}

            # no score before 1st round

        scores = {id_player: 0 for id_player in self.players_list}
        if self.current_round < 1:
            return scores

        # else
        for round_id in self.rounds_list:
            ronde = Round.find_one("round_id", round_id)
            matchs_list = ronde.matchs_list
            for match in matchs_list:
                scores[match[0][0]] += match[0][1]
                scores[match[1][0]] += match[1][1]

        return scores

    @property
    def classement(self):
        """ """
        classement = [(i, j) for i, j in self.scores.items()]

        classement = sorted(classement, key=lambda i: i[1], reverse=True)

        classement = [i[0] for i in classement]
        self.players_list = classement
        self.table.update({"players_list": self.players_list},
                          Query().name == self.name)

        return classement

    def have_already_played(self, player_x, player_0):
        """"""
        flatten_match_list = []
        for round_id in self.rounds_list:
            rounde = Round.find_one("round_id", round_id)

            matchs_list = rounde.matchs_list

            for match in matchs_list:
                m = (match[0][0], match[1][0])
                flatten_match_list.append(m)

        cand_match = (player_0, player_x)

        for match in flatten_match_list:

            if match == cand_match:
                return True

        cand_match = (player_x, player_0)
        for match in flatten_match_list:

            if match == cand_match:
                return True

        return False

    def compute_round(self):
        """define the roundes"""

        logging.warning("_compute_ronde  called")

        # seulement si c'est la 1er la current_round 0
        if self.current_round == 0:
            logging.warning("not self._current_round:")

            # we need 3 object storage
            match_list = []
            players_choisis = []
            players_non_choisis = [i for i in self.players_list]

            while len(players_non_choisis) != 0:
                # just to be more readable
                p1 = players_non_choisis[0]
                p2 = players_non_choisis[1]

                # match and match list
                match = [(p1, 0), (p2, 0)]
                match_list.append(match)

                # update players_choisis & non choisis
                players_choisis.extend([p1, p2])

                players_non_choisis.remove(p1)
                players_non_choisis.remove(p2)

            return match_list

        # else
        match_list = []
        players_choisis = []
        self.players_list = self.classement
        players_non_choisis = [i for i in self.players_list]

        i = 0
        while len(players_non_choisis) != 0:
            p1 = players_non_choisis[i]

            if p1 in players_choisis:
                continue

            for p_id in self.players_list:
                if p1 == p_id:
                    continue

                if p_id in players_choisis:
                    continue

                already_played = self.have_already_played(p1, p_id)
                if not already_played:
                    break

                # match and match list
            match = [(p1, 0), (p_id, 0)]
            match_list.append(match)

            # update players_choisis & non choisis
            players_choisis.extend([p1, p_id])

            if p_id in players_non_choisis:
                players_non_choisis.remove(p_id)
            players_non_choisis.remove(p1)

        return match_list

    def next_round(self):
        """ """
        i = self.rounds_list[self.current_round]
        previous_round = Round.find_one("round_id",
                                        f"{i}")
        previous_round.end()

        self.current_round += 1
        self.table.update({"current_round": self.current_round},
                          Query().name == self.name)

        if int(self.current_round) >= int(self.rounds_number):
            self.status = "closed"
            self.table.update({"status": self.status},
                              Query().name == self.name)

            return logging.warning("Tournois terminé !")
        new_match_list = self.compute_round()

        new_round = Round(new_match_list)

        self.rounds_list.append(new_round.round_id)
        self.table.update({"rounds_list": self.rounds_list},
                          Query().name == self.name)
        new_round.create()

    @property
    def n_players(self):
        """ """
        return len(self.players_list)

    def __repr__(self):
        """ """
        rep = 'Tournois(' + self.name + ',' + self.place + str(
            self.players_list) + ')'
        return rep


from array import array
from ctypes import Array
from typing import Tuple

def binary_search(numbers: Array, value: int) -> Tuple[bool, int]:
    p=0
    k=len(numbers)-1
    while(p<=k):
        middle = int((p+k)/2)
        if value==numbers[middle]:
            return True,middle
        elif value>numbers[middle]:
            p+=1
        else:
            k-=1
    return False,-1

x = array('i',[0,4,7,11,55,99])
print(binary_search(x,2))
print(binary_search(x,55))


import torch
torch.manual_seed(1741)
import random
random.seed(1741)
import numpy as np
np.random.seed(1741)
from collections import defaultdict
import pickle
import os
import tqdm
from itertools import combinations
from data_loader.reader import mulerx_tsvx_reader, tbd_tml_reader, tdd_tml_reader, tml_reader, tsvx_reader
from utils.tools import create_target, padding, pos_to_id
from sklearn.model_selection import train_test_split
from utils.SentenceEncoder import SentenceEncoder
import gc

class Reader(object):
    def __init__(self, type) -> None:
        super().__init__()
        self.type = type

    def read(self, dir_name, file_name):
        if self.type == 'tsvx':
            return tsvx_reader(dir_name, file_name)
        elif self.type == 'tml':
            return tml_reader(dir_name, file_name)
        elif self.type == 'mulerx':
            return mulerx_tsvx_reader(dir_name, file_name)
        elif self.type == 'tbd_tml':
            return tbd_tml_reader(dir_name, file_name)
        elif self.type == 'tdd_man':
            return tdd_tml_reader(dir_name, file_name, type_doc='man')
        elif self.type == 'tdd_auto':
            return tdd_tml_reader(dir_name, file_name, type_doc='auto')
        else:
            raise ValueError("We have not supported {} type yet!".format(self.type))

class C2V(object):
    def __init__(self, emb_file:str) -> None:
        super().__init__()
        with open(emb_file, 'r', encoding='UTF-8') as f:
            lines = f.readlines()
        self.c2v = {}
        for line in lines:
            tokens = line.split(" ")
            concept = tokens[0]
            emb = [float(tok) for tok in tokens[1:]]
            self.c2v[concept] = emb

    def get_emb(self, concept):
        _concept = "_".join(concept.split(" ")).lower()
        # print(cp)
        try:
            return self.c2v[_concept]
        except:
            return [0.0]*300

def load_dataset(dir_name, type):
    reader = Reader(type)
    onlyfiles = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
    corpus = []
    # i = 0
    for file_name in tqdm.tqdm(onlyfiles):
        # if i == 1:
        #     break
        # i = i + 1
        if type == 'i2b2_xml':
            if file_name.endswith('.xml'):
                my_dict = reader.read(dir_name, file_name)
                if my_dict != None:
                    corpus.append(my_dict)
        else:
            my_dict = reader.read(dir_name, file_name)
            if my_dict != None:
                corpus.append(my_dict)
    return corpus

global_sent_encoder = SentenceEncoder('roberta-base')
global_c2v = C2V('./datasets/numberbatch-en-19.08.txt')

def loader(dataset, min_ns, file_type=None, file_path=None, label_type=None):
    sent_encoder = global_sent_encoder
    c2v = global_c2v
    def get_data_point(my_dict, flag):
        data = []
        eids = my_dict['event_dict'].keys()
        pair_events = list(combinations(eids, 2))

        ctx_id_augm = []
        for pair in pair_events:
            x, y = pair

            x_sent_id = my_dict['event_dict'][x]['sent_id']
            y_sent_id = my_dict['event_dict'][y]['sent_id']
            ctx_id = list(range(len( my_dict["sentences"])))
            if  x_sent_id != y_sent_id:
                ctx_id.remove(x_sent_id)
                ctx_id.remove(y_sent_id)
            else:
                ctx_id.remove(x_sent_id)
            id_augm = [tuple(sorted([x_sent_id, y_sent_id, id])) for id in ctx_id]
            ctx_id_augm.extend(id_augm)
        ctx_id_augm = list(set(ctx_id_augm))
        # print(len(ctx_id_augm))

        ctx_augm = []
        ctx_augm_mask = []
        _augm_emb = []
        for ids in ctx_id_augm:
            ids = set(ids)
            sent = []
            for id in ids:
                sent = sent + my_dict["sentences"][id]["roberta_subword_to_ID"][1:]
            sent = [0] + sent
            pad, mask = padding(sent, max_sent_len=512)
            _augm_emb.append(sent_encoder.encode(pad, mask, is_ctx=True))
            ctx_augm.append(pad)
            ctx_augm_mask.append(mask)
        _augm_emb = torch.cat(_augm_emb, dim=0)
        if len(ctx_augm) == 0:
            _augm_emb = []
        _ctx_augm_emb = {}
        for i in range(len(ctx_id_augm)):
            _ctx_augm_emb[ctx_id_augm[i]] = _augm_emb[i]

        doc = []
        doc_mask = []
        doc_emb = []
        for sent_id in list(range(len( my_dict["sentences"]))):
            sent = my_dict["sentences"][sent_id]["roberta_subword_to_ID"]
            pad, mask = padding(sent, max_sent_len=512)
            doc_emb.append(sent_encoder.encode(pad, mask))
            doc.append(pad)
            doc_mask.append(mask)
        doc_emb = torch.cat(doc_emb, dim=0)

        sent_ev = defaultdict(list)
        sent_ev_ids = defaultdict(list)
        ev_kg_emb = {}
        # print(eids)
        for eid in eids:
            sent_id = my_dict['event_dict'][eid]['sent_id']
            e_possition = my_dict["event_dict"][eid]["roberta_subword_id"]
            sent_ev[sent_id].append(e_possition)
            sent_ev_ids[sent_id].append(eid)
            mention = my_dict["event_dict"][eid]["mention"] ######
            # print(mention)
            kg_emb = c2v.get_emb(mention)
            # print(kg_emb)
            ev_kg_emb[eid] = kg_emb
        sent_ev = dict(sent_ev)
        sent_ev_ids = dict(sent_ev_ids)
        # print(ev_kg_emb)

        for pair in pair_events:
            x, y = pair

            x_sent_id = my_dict['event_dict'][x]['sent_id']
            y_sent_id = my_dict['event_dict'][y]['sent_id']

            x_sent = my_dict["sentences"][x_sent_id]["roberta_subword_to_ID"]
            y_sent = my_dict["sentences"][y_sent_id]["roberta_subword_to_ID"]

            x_position = my_dict["event_dict"][x]["roberta_subword_id"]
            y_position = my_dict["event_dict"][y]["roberta_subword_id"]

            x_sent_pos = pos_to_id(my_dict["sentences"][x_sent_id]["roberta_subword_pos"])
            y_sent_pos = pos_to_id(my_dict["sentences"][y_sent_id]["roberta_subword_pos"])

            target, x_position_new, y_position_new = create_target(x_sent, y_sent, x_sent_id, y_sent_id, x_position, y_position)
            target_encode = sent_encoder.encode(target)
            target_emb = target_encode[:, 0].squeeze()
            target_len = len(target)

            x_ev_embs = target_encode[:, x_position_new].squeeze()
            y_ev_embs = target_encode[:, y_position_new].squeeze()

            x_kg_ev_emb = torch.tensor(ev_kg_emb[x])
            y_kg_ev_emb = torch.tensor(ev_kg_emb[y])
            # print(x_kg_ev_emb)

            ctx = []
            _ctx_emb = []
            ctx_pos = []
            ctx_len = []
            _ctx_ev_embs = []
            _ctx_ev_kg_embs = []
            ctx_id = list(range(len( my_dict["sentences"])))
            num_ev_sents = []
            if  x_sent_id != y_sent_id:
                ctx_id.remove(x_sent_id)
                ctx_id.remove(y_sent_id)
            else:
                ctx_id.remove(x_sent_id)
            id_mapping = {}
            if len(ctx_id) != 0:
                i = 0
                for sent_id in ctx_id:
                    id_mapping[i] = sent_id
                    sent = my_dict["sentences"][sent_id]['roberta_subword_to_ID']
                    sent_pos = pos_to_id(my_dict["sentences"][sent_id]['roberta_subword_pos'])
                    ctx.append(sent)
                    ctx_pos.append(sent_pos)
                    ctx_len.append(len(sent))
                    sent_emb = _ctx_augm_emb[tuple(sorted([x_sent_id, y_sent_id, sent_id]))]
                    assert sent_emb != None
                    _ctx_emb.append(sent_emb)
                    e_possitions = sent_ev.get(sent_id)
                    if e_possitions == None:
                        e_possitions = []
                    num_ev_sents.append(len(e_possitions))
                    if len(e_possitions) != 0:
                        ev_embs = torch.max(doc_emb[sent_id, e_possitions, :], dim=0)[0] # 768
                        _ctx_ev_embs.append(ev_embs)

                    eids = sent_ev_ids.get(sent_id)
                    if eids == None:
                        eids = []
                    # print(eids)
                    if len(eids) != 0:
                        sent_ev_kg_emb = [ev_kg_emb[eid] for eid in eids]
                        # print("before: ", torch.tensor(sent_ev_kg_emb))
                        sent_ev_kg_emb = torch.max(torch.tensor(sent_ev_kg_emb), dim=0)[0] # 300
                        # print("max: ", sent_ev_kg_emb)
                        _ctx_ev_kg_embs.append(sent_ev_kg_emb)
                    else:
                        # print("Sent no ev")
                        _ctx_ev_embs.append(torch.ones(768)*-1000.0)
                        _ctx_ev_kg_embs.append(torch.ones(300)*-1000.0)
                    i = i + 1
                # print(ctx_ev_kg_embs)
                ctx_ev_kg_embs =torch.stack(_ctx_ev_kg_embs, dim=0)
                ctx_ev_embs = torch.stack(_ctx_ev_embs, dim=0)
                ctx_emb = torch.stack(_ctx_emb, dim=0) # ns x 768
            # print(ctx_emb.size())
            xy = my_dict["relation_dict"].get((x, y))
            yx = my_dict["relation_dict"].get((y, x))

            candidates = [
                [str(x), str(y), x_sent, y_sent, x_sent_id, y_sent_id, x_sent_pos, y_sent_pos, x_position, y_position, x_ev_embs, y_ev_embs, x_kg_ev_emb,
                y_kg_ev_emb, id_mapping, target, target_emb, target_len, ctx, ctx_emb, ctx_ev_embs, num_ev_sents, ctx_ev_kg_embs, ctx_len, ctx_pos, flag, xy],
                [str(y), str(x), y_sent, x_sent, y_sent_id, x_sent_id, y_sent_pos, x_sent_pos, y_position, x_position, y_ev_embs, x_ev_embs, y_kg_ev_emb,
                x_kg_ev_emb, id_mapping, target, target_emb, target_len, ctx, ctx_emb, ctx_ev_embs, num_ev_sents, ctx_ev_kg_embs, ctx_len, ctx_pos, flag, yx],
            ]
            for item in candidates:
                if item[-1] != None:
                    data.append(item)
        return data

    train_set = []
    train_short = []
    test_set = []
    test_short = []
    validate_set = []
    validate_short = []
    if dataset == "MATRES":
        print("MATRES Loading .......")
        aquaint_dir_name = "./datasets/MATRES/TBAQ-cleaned/AQUAINT/"
        timebank_dir_name = "./datasets/MATRES/TBAQ-cleaned/TimeBank/"
        platinum_dir_name = "./datasets/MATRES/te3-platinum/"
        validate = load_dataset(aquaint_dir_name, 'tml')
        train = load_dataset(timebank_dir_name, 'tml')
        test = load_dataset(platinum_dir_name, 'tml')
        _tt = train + validate
        _tt = list(sorted(_tt, key=lambda x: x["doc_id"]))
        train, validate = train_test_split(_tt, test_size=0.1, train_size=0.9)

        processed_dir = "./datasets/MATRES/docEvR_processed_kg/"
        if not os.path.exists(processed_dir):
            os.mkdir(processed_dir)

        for my_dict in tqdm.tqdm(train):
            file_name = my_dict["doc_id"] + ".pkl"
            if not os.path.exists(processed_dir+file_name):
                data = get_data_point(my_dict, 2)
                with open(processed_dir+file_name, 'wb') as f:
                    pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
            else:
                with open(processed_dir+file_name, 'rb') as f:
                    data = pickle.load(f)
            for item in data:
                if len(item[-4]) >= min_ns:
                    train_set.append(item)
                if len(item[-4]) < min_ns:
                    train_short.append(item)

        for my_dict in tqdm.tqdm(validate):
            file_name = my_dict["doc_id"] + ".pkl"
            if not os.path.exists(processed_dir+file_name):
                data = get_data_point(my_dict, 2)
                with open(processed_dir+file_name, 'wb') as f:
                    pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
            else:
                with open(processed_dir+file_name, 'rb') as f:
                    data = pickle.load(f)
            for item in data:
                if len(item[-4]) >= min_ns:
                    validate_set.append(item)
                if len(item[-4]) < min_ns:
                    validate_short.append(item)

        for my_dict in tqdm.tqdm(test):
            file_name = my_dict["doc_id"] + ".pkl"
            if not os.path.exists(processed_dir+file_name):
                data = get_data_point(my_dict, 2)
                with open(processed_dir+file_name, 'wb') as f:
                    pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
            else:
                with open(processed_dir+file_name, 'rb') as f:
                    data = pickle.load(f)
            for item in data:
                if len(item[-4]) >= min_ns:
                    test_set.append(item)
                if len(item[-4]) < min_ns:
                    test_short.append(item)

        print("Train_size: {}".format(len(train_set) + len(train_short)))
        print("Test_size: {}".format(len(test_set) + len(test_short)))
        print("Validate_size: {}".format(len(validate_set) + len(validate_short)))

    if dataset == "HiEve":
        print("HiEve Loading .....")
        dir_name = "./datasets/hievents_v2/processed/"
        corpus = load_dataset(dir_name, 'tsvx')
        corpus = list(sorted(corpus, key=lambda x: x["doc_id"]))
        train, test = train_test_split(corpus, train_size=0.8, test_size=0.2)
        train, validate = train_test_split(train, train_size=0.75, test_size=0.25)
        sample = 0.4

        processed_dir = "./datasets/hievents_v2/processed/docEvR_processed_kg/"
        if not os.path.exists(processed_dir):
            os.mkdir(processed_dir)

        for my_dict in tqdm.tqdm(train):
            file_name = my_dict["doc_id"] + ".pkl"
            if os.path.exists(processed_dir+file_name):
                with open(processed_dir+file_name, 'rb') as f:
                    data = pickle.load(f)
            else:
                data = get_data_point(my_dict, 1)
                with open(processed_dir+file_name, 'wb') as f:
                    pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
            for item in data:
                if item[-1] == 3:
                    if random.uniform(0, 1) < sample:
                        if len(item[-4]) >= min_ns:
                            train_set.append(item)
                        else:
                            train_short.append(item)
                else:
                    if len(item[-4]) >= min_ns:
                            train_set.append(item)
                    else:
                        train_short.append(item)

        for my_dict in tqdm.tqdm(test):
            file_name = my_dict["doc_id"] + ".pkl"
            if os.path.exists(processed_dir+file_name):
                with open(processed_dir+file_name, 'rb') as f:
                    data = pickle.load(f)
            else:
                data = get_data_point(my_dict, 1)
                with open(processed_dir+file_name, 'wb') as f:
                    pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
            for item in data:
                if item[-1] == 3:
                    if random.uniform(0, 1) < sample:
                        if len(item[-4]) >= min_ns:
                            test_set.append(item)
                        else:
                            test_short.append(item)
                else:
                    if len(item[-4]) >= min_ns:
                            test_set.append(item)
                    else:
                        test_short.append(item)

        for my_dict in tqdm.tqdm(validate):
            file_name = my_dict["doc_id"] + ".pkl"
            if os.path.exists(processed_dir+file_name):
                with open(processed_dir+file_name, 'rb') as f:
                    data = pickle.load(f)
            else:
                data = get_data_point(my_dict, 1)
                with open(processed_dir+file_name, 'wb') as f:
                    pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
            for item in data:
                if item[-1] == 3:
                    if random.uniform(0, 1) < sample:
                        if len(item[-4]) >= min_ns:
                            validate_set.append(item)
                        else:
                            validate_short.append(item)
                else:
                    if len(item[-4]) >= min_ns:
                            validate_set.append(item)
                    else:
                        validate_short.append(item)

        print("Train_size: {}".format(len(train_set) + len(train_short)))
        print("Test_size: {}".format(len(test_set) + len(test_short)))
        print("Validate_size: {}".format(len(validate_set) + len(validate_short)))

    if dataset == 'TBD':
        print("Timebank Dense Loading .....")
        train_dir = "./datasets/TimeBank-dense/train/"
        test_dir = "./datasets/TimeBank-dense/test/"
        validate_dir = "./datasets/TimeBank-dense/dev/"
        train = load_dataset(train_dir, 'tbd_tml')
        test = load_dataset(test_dir, 'tbd_tml')
        validate = load_dataset(validate_dir, 'tbd_tml')
        _tt = train + validate
        _tt = list(sorted(_tt, key=lambda x: x["doc_id"]))
        train, validate = train_test_split(_tt, test_size=0.1, train_size=0.9)

        processed_dir = "./datasets/TimeBank-dense/docEvR_processed_kg/"
        if not os.path.exists(processed_dir):
            os.mkdir(processed_dir)

        for my_dict in tqdm.tqdm(train):
            file_name = my_dict["doc_id"] + ".pkl"
            if not os.path.exists(processed_dir+file_name):
                data = get_data_point(my_dict, 4)
                with open(processed_dir+file_name, 'wb') as f:
                    pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
            else:
                with open(processed_dir+file_name, 'rb') as f:
                    data = pickle.load(f)
            for item in data:
                if len(item[-4]) >= min_ns:
                    train_set.append(item)
                if len(item[-4]) < min_ns:
                    train_short.append(item)

        for my_dict in tqdm.tqdm(test):
            file_name = my_dict["doc_id"] + ".pkl"
            if not os.path.exists(processed_dir+file_name):
                data = get_data_point(my_dict, 4)
                with open(processed_dir+file_name, 'wb') as f:
                    pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
            else:
                with open(processed_dir+file_name, 'rb') as f:
                    data = pickle.load(f)
            for item in data:
                if len(item[-4]) >= min_ns:
                    test_set.append(item)
                if len(item[-4]) < min_ns:
                    test_short.append(item)

        for my_dict in tqdm.tqdm(validate):
            file_name = my_dict["doc_id"] + ".pkl"
            if not os.path.exists(processed_dir+file_name):
                data = get_data_point(my_dict, 4)
                with open(processed_dir+file_name, 'wb') as f:
                    pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
            else:
                with open(processed_dir+file_name, 'rb') as f:
                    data = pickle.load(f)
            for item in data:
                if len(item[-4]) >= min_ns:
                    validate_set.append(item)
                if len(item[-4]) < min_ns:
                    validate_short.append(item)

        print("Train_size: {}".format(len(train_set) + len(train_short)))
        print("Test_size: {}".format(len(test_set) + len(test_short)))
        print("Validate_size: {}".format(len(validate_set) + len(validate_short)))

    if dataset == 'TDD_man':
        print("TDD_man Loading .....")
        train_dir = "./datasets/TimeBank-dense/train/"
        test_dir = "./datasets/TimeBank-dense/test/"
        validate_dir = "./datasets/TimeBank-dense/dev/"
        train = load_dataset(train_dir, 'tdd_man')
        test = load_dataset(test_dir, 'tdd_man')
        validate = load_dataset(validate_dir, 'tdd_man')
        # _tt = train + validate
        # _tt = list(sorted(_tt, key=lambda x: x["doc_id"]))
        # train, validate = train_test_split(_tt, test_size=0.1, train_size=0.9)

        processed_dir = "./datasets/TDDiscourse/TDDMan/docEvR_processed_kg/"
        if not os.path.exists(processed_dir):
            os.mkdir(processed_dir)

        for my_dict in tqdm.tqdm(train):
            file_name = my_dict["doc_id"] + ".pkl"
            if not os.path.exists(processed_dir+file_name):
                data = get_data_point(my_dict, 5)
                with open(processed_dir+file_name, 'wb') as f:
                    pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
            else:
                with open(processed_dir+file_name, 'rb') as f:
                    data = pickle.load(f)
            for item in data:
                if len(item[-4]) >= min_ns:
                    train_set.append(item)
                if len(item[-4]) < min_ns:
                    train_short.append(item)

        for my_dict in tqdm.tqdm(test):
            file_name = my_dict["doc_id"] + ".pkl"
            if not os.path.exists(processed_dir+file_name):
                data = get_data_point(my_dict, 5)
                with open(processed_dir+file_name, 'wb') as f:
                    pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
            else:
                with open(processed_dir+file_name, 'rb') as f:
                    data = pickle.load(f)
            for item in data:
                if len(item[-4]) >= min_ns:
                    test_set.append(item)
                if len(item[-4]) < min_ns:
                    test_short.append(item)

        for my_dict in tqdm.tqdm(validate):
            file_name = my_dict["doc_id"] + ".pkl"
            if not os.path.exists(processed_dir+file_name):
                data = get_data_point(my_dict, 5)
                with open(processed_dir+file_name, 'wb') as f:
                    pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
            else:
                with open(processed_dir+file_name, 'rb') as f:
                    data = pickle.load(f)
            for item in data:
                if len(item[-4]) >= min_ns:
                    validate_set.append(item)
                if len(item[-4]) < min_ns:
                    validate_short.append(item)

        print("Train_size: {}".format(len(train_set) + len(train_short)))
        print("Test_size: {}".format(len(test_set) + len(test_short)))
        print("Validate_size: {}".format(len(validate_set) + len(validate_short)))

    if dataset == 'TDD_auto':
        print("TDD_auto Loading .....")
        train_dir = "./datasets/TimeBank-dense/train/"
        test_dir = "./datasets/TimeBank-dense/test/"
        validate_dir = "./datasets/TimeBank-dense/dev/"
        train = load_dataset(train_dir, 'tdd_auto')
        test = load_dataset(test_dir, 'tdd_auto')
        validate = load_dataset(validate_dir, 'tdd_auto')
        # _tt = train + validate
        # _tt = list(sorted(_tt, key=lambda x: x["doc_id"]))
        # train, validate = train_test_split(_tt, test_size=0.1, train_size=0.9)

        processed_dir = "./datasets/TDDiscourse/TDDAuto/docEvR_processed_kg/"
        if not os.path.exists(processed_dir):
            os.mkdir(processed_dir)

        for my_dict in tqdm.tqdm(train):
            file_name = my_dict["doc_id"] + ".pkl"
            if not os.path.exists(processed_dir+file_name):
                data = get_data_point(my_dict, 5)
                with open(processed_dir+file_name, 'wb') as f:
                    pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
            else:
                with open(processed_dir+file_name, 'rb') as f:
                    data = pickle.load(f)
            for item in data:
                if len(item[-4]) >= min_ns:
                    train_set.append(item)
                if len(item[-4]) < min_ns:
                    train_short.append(item)

        for my_dict in tqdm.tqdm(test):
            file_name = my_dict["doc_id"] + ".pkl"
            if not os.path.exists(processed_dir+file_name):
                data = get_data_point(my_dict, 5)
                with open(processed_dir+file_name, 'wb') as f:
                    pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
            else:
                with open(processed_dir+file_name, 'rb') as f:
                    data = pickle.load(f)
            for item in data:
                if len(item[-4]) >= min_ns:
                    test_set.append(item)
                if len(item[-4]) < min_ns:
                    test_short.append(item)

        for my_dict in tqdm.tqdm(validate):
            file_name = my_dict["doc_id"] + ".pkl"
            if not os.path.exists(processed_dir+file_name):
                data = get_data_point(my_dict, 5)
                with open(processed_dir+file_name, 'wb') as f:
                    pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
            else:
                with open(processed_dir+file_name, 'rb') as f:
                    data = pickle.load(f)
            for item in data:
                if len(item[-4]) >= min_ns:
                    validate_set.append(item)
                if len(item[-4]) < min_ns:
                    validate_short.append(item)

        print("Train_size: {}".format(len(train_set) + len(train_short)))
        print("Test_size: {}".format(len(test_set) + len(test_short)))
        print("Validate_size: {}".format(len(validate_set) + len(validate_short)))

    if dataset=='infer':
        reader = Reader(file_type)
        print(f'Reading file {file_path} ....')
        my_dict = reader.read('', file_path)
        data = get_data_point(my_dict, label_type)
        for item in data:
            if len(item[-4]) >= min_ns:
                test_set.append(item)
            else:
                test_short.append(item)
        print("Train_size: {}".format(len(train_set)))
        print("Test_size: {}".format(len(test_set)))
        print("Validate_size: {}".format(len(validate_set)))
        print("Train_size: {}".format(len(train_short)))
        print("Test_size: {}".format(len(test_short)))
        print("Validate_size: {}".format(len(validate_short)))

    del sent_encoder
    del c2v
    gc.collect()

    return train_set, test_set, validate_set, train_short, test_short, validate_short


# -*- coding: utf-8 -*-

#Importing required libraries
from string import punctuation
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.tokenize import sent_tokenize
import urllib.request
import bs4 as BeautifulSoup
import nltk
from nltk.cluster.util import cosine_distance
import numpy as np
import networkx as nx
import re

def read_article(text):

    sentences =[]

    sentences = sent_tokenize(text) #module in nltk
    for sentence in sentences:
        sentence.replace("[^a-zA-Z0-9]"," ")

    return sentences

def generate_summary2(lnk):
    text = urllib.request.urlopen(lnk)
    article = text.read()
    article_parsed = BeautifulSoup.BeautifulSoup(article,'html.parser')
    paragraphs = article_parsed.find_all('p')
    article_content = ''
    for p in paragraphs:
        article_content += p.text
    sentencel = read_article(article_content)
    tokens = word_tokenize(article_content)
    nltk.download("stopwords")
    stop_words = stopwords.words('english')
    from string import punctuation
    punctuation = punctuation + '\n'
    word_frequencies = {}
    for word in tokens:
        if word.lower() not in stop_words:
            if word.lower() not in punctuation:
                if word not in word_frequencies.keys():
                    word_frequencies[word] = 1
                else:
                    word_frequencies[word] += 1
    max_frequency = max(word_frequencies.values())
    for word in word_frequencies.keys():
        word_frequencies[word] = word_frequencies[word]/max_frequency
    sent_token = sent_tokenize(article_content)
    sentence_scores = {}
    for sent in sent_token:
        sentence = sent.split(" ")
        for word in sentence:
            if word.lower() in word_frequencies.keys():
                if sent not in sentence_scores.keys():
                    sentence_scores[sent] = word_frequencies[word.lower()]
                else:
                    sentence_scores[sent] += word_frequencies[word.lower()]
    from heapq import nlargest
    select_length = int(len(sent_token)*0.3)
    summary = nlargest(select_length, sentence_scores, key = sentence_scores.get)
    final_summary = [word for word in summary]
    summary = ' '.join(final_summary)
    summaryl = read_article(summary)
    return summary,len(sentencel),len(summaryl)


from django.shortcuts import render
from .models import Post, Contact, Social, WebElement, Soupiska
from .forms import NewUserForm
from django.contrib.auth import login
from django.contrib import messages

social = Social.objects
webelement = WebElement.objects
post = Post.objects.order_by("-pub_date")

# co všechno se má odeslat z databáze na stránku a podle jakého templatu má stránka vypadat

def IndexView(response):
    return render(
        response,
        "website/index.php",
        {"post": post, "social": social, "webelement": webelement},
    )

def ContactView(response):
    contacts = Contact.objects
    return render(
        response,
        "website/contacts.php",
        {"contacts": contacts, "social": social, "webelement": webelement},
    )

def GroupView(response, group):
    return render(
        response,
        "website/group.php",
        {"group": group, "post": post, "social": social, "webelement": webelement},
    )

def AboutUsView(response):
    return render(
        response,
        "website/about.php",
        {"social": social, "webelement": webelement},
    )

def SoupiskaView(response, group):
    soupiska = Soupiska.objects
    return render(
        response,
        "website/soupiska.php",
        {
            "group": group,
            "soupiska": soupiska,
            "social": social,
            "webelement": webelement,
        },
    )

def SignUpView(response):
    if response.method == "POST":
        form = NewUserForm(response.POST)
        if form.is_valid():
            user = form.save()
            login(response, user)
            messages.success(response, "Registration successful.")
            return render(
                response,
                "website/index.php",
                {"post": post, "social": social, "webelement": webelement},
            )
        messages.error(response, "Unsuccessful registration. Invalid information.")
    form = NewUserForm()
    return render(
        response,
        "website/signup.php",
        {"register_form": form, "social": social, "webelement": webelement},
    )


from django.urls import path

from . import views

app_name = "shifter_files"
urlpatterns = [
    path('', views.FileUploadView.as_view(), name='index'),
    path('files', views.FileListView.as_view(), name='myfiles'),
    path('files/<str:file_hex>', views.FileDetailView.as_view(),
         name='file-details'),
    path('download/<str:file_hex>', views.FileDownloadLandingView.as_view(),
         name='file-download-landing'),
    path('f/<str:file_hex>', views.FileDownloadView.as_view(),
         name='file-download'),
    path('files/<str:file_hex>/delete', views.FileDeleteView.as_view(),
         name='file-delete'),
]


from typing import Literal, Optional

from ._base import BaseWidget, TkWidget
from ._variables import Float

class ProgressBar(BaseWidget):
    _tcl_class = "ttk::progressbar"
    _keys = {
        "focusable": (bool, "takefocus"),
        "max": float,
        "mode": str,
        "orientation": (str, "orient"),
        "value": float,
        "variable": Float,
    }

    def __init__(
        self,
        parent: Optional[TkWidget] = None,
        focusable: Optional[bool] = None,
        max: Optional[int] = 100,
        mode: Optional[Literal["determinate", "indeterminate"]] = None,
        orientation: Optional[Literal["horizontal", "vertical"]] = None,
        value: Optional[int] = None,
        variable: Optional[Float] = None,
    ) -> None:
        BaseWidget.__init__(
            self,
            parent,
            maximum=max,
            mode=mode,
            orient=orientation,
            takefocus=focusable,
            value=value,
            variable=variable,
        )

    def _repr_details(self):
        return f"mode={self.mode!r}, max={self.max!r}, value={self.value!r}"

    def get(self) -> float:
        return self.value

    def set(self, value: float = 0) -> None:
        self.value = value

    def start(self, steps_per_second: int = 20) -> None:
        if steps_per_second > 1000:
            raise ValueError("error")
        interval = int(1000 / steps_per_second)
        self._tcl_call(None, self, "start", interval)

    def stop(self) -> None:
        self._tcl_call(None, self, "stop")

    def step(self, amount: int = 1) -> None:
        self._tcl_call(None, self, "step", amount)

    def __add__(self, other: int):
        self.set(self.get() + other)
        return self

    def __sub__(self, other: int):
        self.set(self.get() - other)
        return self


'''
Input: __init__(feature_size), eeg_features
Output: the noised data from gaussian layer
'''

import torch
from torch import nn

class MoGLayer(nn.Module):

    def __init__(self, noise_dim: tuple):
        super(MoGLayer, self).__init__()

        pre_std = torch.zeros(noise_dim)
        pre_std = torch.nn.init.uniform_(pre_std, -0.2, 0.2)
        self.std = nn.Parameter(pre_std, requires_grad=True)

        pre_mean = torch.zeros(noise_dim)
        pre_mean = torch.nn.init.uniform_(pre_mean, -1.0, 1.0)
        self.mean = nn.Parameter(pre_mean, requires_grad=True)

    def set_dev(self, DEV):
        self.dev = DEV

    def forward(self, noise):
        return self.mean.to(self.dev) + (self.std.to(self.dev) * noise.to(self.dev))


import collections, math
def solution(fees, records):
    cars_park_record, cars_park_time = collections.defaultdict(list), collections.defaultdict(int)
    for record in records:
        time, car_num, _ = record.split()
        cars_park_record[car_num].append(time)

    for car_num in cars_park_record:
        times = cars_park_record[car_num]
        for i in range((len(times) + 1) // 2):
            in_hour, in_minute = map(int, times[i * 2].split(':'))
            print(in_hour, in_minute)
            if i * 2 + 1 >= len(times):
                out_hour, out_minute = 23, 59
            else:
                out_hour, out_minute = map(int, times[i * 2 + 1].split(':'))
            time = (out_hour - in_hour) * 60 + (out_minute - in_minute)
            cars_park_time[car_num] += time

    answer = []
    for car_num in sorted(cars_park_time):
        time = cars_park_time[car_num]
        if time <= fees[0]:
            answer.append(fees[1])
        else:
            fee = fees[1] + math.ceil((time - fees[0]) / fees[2]) * fees[3]
            answer.append(fee)

    return answer

주차 요금 계산_HW.py


import time
import torch

def train_hebb(model, loader, device, measures=None, criterion=None):
    """
    Train only the hebbian blocks
    """
    t = time.time()
    loss_acc = (not model.is_hebbian()) and (criterion is not None)
    with torch.no_grad():
        for inputs, target in loader:
            # print(inputs.min(), inputs.max(), inputs.mean(), inputs.std())
            ## 1. forward propagation
            inputs = inputs.float().to(device)  # , non_blocking=True)
            output = model(inputs)

            # print(r"%s"%(time.time()-t))

            if loss_acc:
                target = target.to(device, non_blocking=True)

                ## 2. loss calculation
                loss = criterion(output, target)

                ## 3. Accuracy assessment
                predict = output.data.max(1)[1]
                acc = predict.eq(target.data).sum()
                # Save if measurement is wanted
                conv, r1 = model.convergence()
                measures.step(target.shape[0], loss.clone().detach().cpu(), acc.cpu(), conv, r1, model.get_lr())
            model.update()

    info = model.radius()
    convergence, R1 = model.convergence()
    return measures, model.get_lr(), info, convergence, R1

def train_unsup(model, loader, device,
                blocks=[]):  # fixed bug as optimizer is not used or pass in the only use it has in this repo currently
    """
    Unsupervised learning only works with hebbian learning
    """
    model.train(blocks=blocks)  # set unsup blocks to train mode
    _, lr, info, convergence, R1 = train_hebb(model, loader, device)
    return lr, info, convergence, R1


'''

PROBLEM:

Given a string, rearrange it so that any two adjacent characters are not the same.
If such a reorganization of the characters is possible, output any possible valid
arrangement. Otherwise, return an empty string.

------------------------
PATTERN: TOP K ELEMENTS
------------------------

'''

# importing libraries
from collections import Counter
import heapq

def reorganize_string(input_string):

    output = [""] * len(input_string)
    counter = Counter(input_string)

    max_heap = [[-val, key] for key, val in counter.items()]
    heapq.heapify(max_heap)

    if len(input_string) % 2 == 0 and (-max_heap[0][0]) > len(input_string) // 2 :
        return ""
    elif (-max_heap[0][0]) > (len(input_string) // 2) + 1 :
        return ""

    i = 0
    previous = None
    while i < len(input_string) :

        if max_heap :

            freq, char = heapq.heappop(max_heap)

            output[i] = char

            i += 1
            freq += 1

            if previous :
                heapq.heappush(max_heap, previous)
                previous = None

            if i < len(input_string) and freq < 0 :
                previous = [freq,char]

        elif previous : return ""
        else : break

    return ''.join(output)


#!/usr/bin/env python
"""
Tests covering the parsing functions that read in light characteristics.
"""

import pytest

from light_character import light_character

@pytest.mark.parametrize(
    "test_input",
    [
        'F', 'Fl', 'LFl', 'Oc', 'Iso', 'Q'
    ]
)
def test_parse_pattern_simple(test_input):
    """
    Parse pattern returns the pattern name in lower case.
    If there are no explicit groups, there is an implicit single
    group of 1.
    """
    actual = light_character.parse_pattern(test_input)
    assert actual == (test_input.lower(), [1])

@pytest.mark.parametrize(
    "test_input, expected",
    [
        ('F.', ('f', [1])),
        ('Fl.(3+1)', ('fl', [3, 1])),
        ('L.Fl.(2)', ('lfl', [2])),
        ('Oc.(1+2+3)', ('oc', [1, 2, 3]))
    ]
)
def test_parse_pattern(test_input, expected):
    """
    Parse pattern returns the pattern name in lower case, filtered for dots.
    If there are explicit groups, they are also returned.
    """
    assert light_character.parse_pattern(test_input) == expected

def test_period_error():
    try:
        light_character.parse_period([])
        assert False
    except IndexError:
        assert True

@pytest.mark.parametrize(
    "fragments,expected",
    [
        (['xyz', '1'], 1000),
        (['xyz', '10s'], 10000),
        (['xyz', '15', 's'], 15000),
        (['xyz', '1.5s'], 1500)
    ]
)
def test_period(fragments, expected):
    """
    The period is the last part of the characteristic.
    It may or may not be marked with 's' for seconds.
    The output value is in milliseconds.
    """
    assert light_character.parse_period(fragments) == expected

@pytest.mark.parametrize(
    "fragments,expected",
    [
        (['R'], 'R'),
        (['W'], 'W'),
        (['G', '1s'], 'G'),
        (['Y'], 'Y'),
    ]
)
def test_colour_code(fragments, expected):
    assert light_character.get_colour_code(fragments) == (
        expected,
        fragments[1:]
    )

def test_colour_code_default():
    """
    A characteristic without an explicit colour is white
    :return:
    """
    assert light_character.get_colour_code(['1s']) == (
        'W',
        ['1s']
    )


import sys

from pyspark import SparkConf, SparkContext
from pyspark.sql import SQLContext
import pyspark.sql.functions as F

from users_datamart import distance

def main():

    date = sys.argv[1]
    events_base_path = sys.argv[2]
    geo_base_path = sys.argv[3]
    second_table_save = sys.argv[4]

    conf = SparkConf().setAppName(f"ZonesDatamart-{date}")
    sc = SparkContext(conf=conf)
    sql = SQLContext(sc)

    date_pivot=F.to_date(F.lit(date), "yyyy-MM-dd")
    def count_event_zone(df,event,slice):
        return df\
            .select(F.col('id').alias('zone_id'),F.col('event_type'))\
            .filter(F.col('event_type')==event)\
            .groupBy('zone_id')\
            .agg(F.count('*').alias(f'{slice}_{event}'))

    def count_registration(df, message, slice, date):
        if slice == 'month':
            messages_without_pivot = message.select('event.message_from').where(
                (F.month("date") < F.month(date_pivot)) & (F.year("date") <= F.year(date_pivot)))
        elif slice == 'week':
            messages_without_pivot = message.select('event.message_from').where(
                (F.weekofyear("date") < F.weekofyear(date_pivot)) & (F.year("date") <= F.year(date_pivot)))

        return df.select(F.col('event.message_from'),F.col('id').alias('zone_id'))\
                .join(messages_without_pivot, 'message_from', 'left_anti') \
                .distinct() \
                .groupBy('zone_id')\
                .agg(F.count('*').alias(f'{slice}_user'))

    date_pivot=F.to_date(F.lit(date), "yyyy-MM-dd")

    message = sql.read.parquet(events_base_path) \
                .where(F.col('date') <= date_pivot)\
                .withColumn('lat1', F.radians(F.col('lat'))) \
                .withColumn('lon1', F.radians(F.col('lon'))) \
                .drop('lat','lon')

    event_week = sql.read.parquet(events_base_path) \
        .where((F.col('date') <= F.lit(date_pivot)) &
            (F.weekofyear(F.col('date')) == F.weekofyear(date_pivot)) &
            (F.year(F.col('date')) == F.year(date_pivot))) \
        .withColumn('lat1', F.radians(F.col('lat'))) \
        .withColumn('lon1', F.radians(F.col('lon'))) \
        .drop('lat','lon')

    event_month = sql.read.parquet(events_base_path) \
                .where((F.col('date') <= F.lit(date_pivot)) &
                    (F.month(F.col('date')) == F.month(date_pivot)) &
                    (F.year(F.col('date')) == F.year(date_pivot)))\
                .withColumn('lat1', F.radians(F.col('lat'))) \
                .withColumn('lon1', F.radians(F.col('lon'))) \
                .drop('lat','lon')

    geo = sql.read.option("delimiter", ";").option("header", True).csv(geo_base_path) \
            .withColumn('lat2', F.radians(F.col('lat'))) \
            .withColumn('lon2', F.radians(F.col('lng'))) \
            .drop('lat', 'lng')

    event_week_city = distance(event_week, geo)
    event_month_city = distance(event_month, geo)

    count_event_week = count_event_zone(event_week_city, 'message', 'week') \
        .join(count_event_zone(event_week_city, 'reaction', 'week'), 'zone_id','full') \
        .join(count_event_zone(event_week_city, 'subscription', 'week'), 'zone_id','full') \
        .join(count_registration(event_week_city, message, 'week', date_pivot),'zone_id','full')

    count_event_month = count_event_zone(event_month_city, 'message', 'month').drop('user_id','event_type') \
        .join(count_event_zone(event_month_city, 'reaction', 'month'), 'zone_id','full') \
        .join(count_event_zone(event_month_city, 'subscription', 'month'), 'zone_id','full') \
        .join(count_registration(event_month_city, message, 'month', date_pivot),'zone_id','full')

    zones_datamart = geo.select(
        F.weekofyear(date_pivot).alias('week'),
        F.month(date_pivot).alias('month'),
        F.col('id').alias('zone_id'))\
        .join(count_event_week, 'zone_id', 'full')\
        .join(count_event_month, 'zone_id', 'full')

    zones_datamart.write.parquet(f"{second_table_save}/date={date}")

if __name__ == '__main__':
    main()


#!/usr/bin/env python

import os
import sys

import setuptools

if sys.argv[-1] == "publish":
    os.system("python setup.py sdist upload")
    sys.exit()

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="pyflowater",
    version="0.5.2",
    packages=["pyflowater"],
    description="Python interface for Flo by Moen API",
    #      long_description=long_description,
    url="https://github.com/rsnodgrass/pyflowater",
    author="Ryan Snodgrass",
    author_email="rsnodgrass@gmail.com",
    license="Apache Software License",
    install_requires=["requests>=2.0", "google-cloud-firestore"],
    keywords=["flo", "home automation", "water monitoring"],
    zip_safe=True,
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: Apache Software License",
        "Operating System :: OS Independent",
    ],
)


import errno
import logging
import os
import subprocess
import re
import shutil
import socket
import tempfile

from . import qmp

LOG = logging.getLogger(__name__)

# Mapping host architecture to any additional architectures it can
# support which often includes its 32 bit cousin.
ADDITIONAL_ARCHES = {
    "x86_64" : "i386",
    "aarch64" : "armhf"
}

def kvm_available(target_arch=None):
    host_arch = os.uname()[4]
    if target_arch and target_arch != host_arch:
        if target_arch != ADDITIONAL_ARCHES.get(host_arch):
            return False
    return os.access("/dev/kvm", os.R_OK | os.W_OK)

#: Maps machine types to the preferred console device types
CONSOLE_DEV_TYPES = {
    r'^clipper$': 'isa-serial',
    r'^malta': 'isa-serial',
    r'^(pc.*|q35.*|isapc)$': 'isa-serial',
    r'^(40p|powernv|prep)$': 'isa-serial',
    r'^pseries.*': 'spapr-vty',
    r'^s390-ccw-virtio.*': 'sclpconsole',
    }

class QEMUMachineError(Exception):
    """
    Exception called when an error in QEMUMachine happens.
    """

class QEMUMachineAddDeviceError(QEMUMachineError):
    """
    Exception raised when a request to add a device can not be fulfilled

    The failures are caused by limitations, lack of information or conflicting
    requests on the QEMUMachine methods.  This exception does not represent
    failures reported by the QEMU binary itself.
    """

class MonitorResponseError(qmp.QMPError):
    """
    Represents erroneous QMP monitor reply
    """
    def __init__(self, reply):
        try:
            desc = reply["error"]["desc"]
        except KeyError:
            desc = reply
        super(MonitorResponseError, self).__init__(desc)
        self.reply = reply

class QEMUMachine(object):
    """
    A QEMU VM

    Use this object as a context manager to ensure the QEMU process terminates::

        with VM(binary) as vm:
            ...
        # vm is guaranteed to be shut down here
    """

    def __init__(self, binary, args=None, wrapper=None, name=None,
                 test_dir="/var/tmp", monitor_address=None,
                 socket_scm_helper=None):
        '''
        Initialize a QEMUMachine

        @param binary: path to the qemu binary
        @param args: list of extra arguments
        @param wrapper: list of arguments used as prefix to qemu binary
        @param name: prefix for socket and log file names (default: qemu-PID)
        @param test_dir: where to create socket and log file
        @param monitor_address: address for QMP monitor
        @param socket_scm_helper: helper program, required for send_fd_scm()
        @note: Qemu process is not started until launch() is used.
        '''
        if args is None:
            args = []
        if wrapper is None:
            wrapper = []
        if name is None:
            name = "qemu-%d" % os.getpid()
        self._name = name
        self._monitor_address = monitor_address
        self._vm_monitor = None
        self._qemu_log_path = None
        self._qemu_log_file = None
        self._popen = None
        self._binary = binary
        self._args = list(args)     # Force copy args in case we modify them
        self._wrapper = wrapper
        self._events = []
        self._iolog = None
        self._socket_scm_helper = socket_scm_helper
        self._qmp = None
        self._qemu_full_args = None
        self._test_dir = test_dir
        self._temp_dir = None
        self._launched = False
        self._machine = None
        self._console_device_type = None
        self._console_address = None
        self._console_socket = None

        # just in case logging wasn't configured by the main script:
        logging.basicConfig()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.shutdown()
        return False

    # This can be used to add an unused monitor instance.
    def add_monitor_null(self):
        self._args.append('-monitor')
        self._args.append('null')

    def add_fd(self, fd, fdset, opaque, opts=''):
        """
        Pass a file descriptor to the VM
        """
        options = ['fd=%d' % fd,
                   'set=%d' % fdset,
                   'opaque=%s' % opaque]
        if opts:
            options.append(opts)

        # This did not exist before 3.4, but since then it is
        # mandatory for our purpose
        if hasattr(os, 'set_inheritable'):
            os.set_inheritable(fd, True)

        self._args.append('-add-fd')
        self._args.append(','.join(options))
        return self

    # Exactly one of fd and file_path must be given.
    # (If it is file_path, the helper will open that file and pass its
    # own fd)
    def send_fd_scm(self, fd=None, file_path=None):
        # In iotest.py, the qmp should always use unix socket.
        assert self._qmp.is_scm_available()
        if self._socket_scm_helper is None:
            raise QEMUMachineError("No path to socket_scm_helper set")
        if not os.path.exists(self._socket_scm_helper):
            raise QEMUMachineError("%s does not exist" %
                                   self._socket_scm_helper)

        # This did not exist before 3.4, but since then it is
        # mandatory for our purpose
        if hasattr(os, 'set_inheritable'):
            os.set_inheritable(self._qmp.get_sock_fd(), True)
            if fd is not None:
                os.set_inheritable(fd, True)

        fd_param = ["%s" % self._socket_scm_helper,
                    "%d" % self._qmp.get_sock_fd()]

        if file_path is not None:
            assert fd is None
            fd_param.append(file_path)
        else:
            assert fd is not None
            fd_param.append(str(fd))

        devnull = open(os.path.devnull, 'rb')
        proc = subprocess.Popen(fd_param, stdin=devnull, stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT, close_fds=False)
        output = proc.communicate()[0]
        if output:
            LOG.debug(output)

        return proc.returncode

    @staticmethod
    def _remove_if_exists(path):
        """
        Remove file object at path if it exists
        """
        try:
            os.remove(path)
        except OSError as exception:
            if exception.errno == errno.ENOENT:
                return
            raise

    def is_running(self):
        return self._popen is not None and self._popen.poll() is None

    def exitcode(self):
        if self._popen is None:
            return None
        return self._popen.poll()

    def get_pid(self):
        if not self.is_running():
            return None
        return self._popen.pid

    def _load_io_log(self):
        if self._qemu_log_path is not None:
            with open(self._qemu_log_path, "r") as iolog:
                self._iolog = iolog.read()

    def _base_args(self):
        if isinstance(self._monitor_address, tuple):
            moncdev = "socket,id=mon,host=%s,port=%s" % (
                self._monitor_address[0],
                self._monitor_address[1])
        else:
            moncdev = 'socket,id=mon,path=%s' % self._vm_monitor
        args = ['-chardev', moncdev,
                '-mon', 'chardev=mon,mode=control',
                '-display', 'none', '-vga', 'none']
        if self._machine is not None:
            args.extend(['-machine', self._machine])
        if self._console_device_type is not None:
            self._console_address = os.path.join(self._temp_dir,
                                                 self._name + "-console.sock")
            chardev = ('socket,id=console,path=%s,server,nowait' %
                       self._console_address)
            device = '%s,chardev=console' % self._console_device_type
            args.extend(['-chardev', chardev, '-device', device])
        return args

    def _pre_launch(self):
        self._temp_dir = tempfile.mkdtemp(dir=self._test_dir)
        if self._monitor_address is not None:
            self._vm_monitor = self._monitor_address
        else:
            self._vm_monitor = os.path.join(self._temp_dir,
                                            self._name + "-monitor.sock")
        self._qemu_log_path = os.path.join(self._temp_dir, self._name + ".log")
        self._qemu_log_file = open(self._qemu_log_path, 'wb')

        self._qmp = qmp.QEMUMonitorProtocol(self._vm_monitor,
                                            server=True)

    def _post_launch(self):
        self._qmp.accept()

    def _post_shutdown(self):
        if self._qemu_log_file is not None:
            self._qemu_log_file.close()
            self._qemu_log_file = None

        self._qemu_log_path = None

        if self._console_socket is not None:
            self._console_socket.close()
            self._console_socket = None

        if self._temp_dir is not None:
            shutil.rmtree(self._temp_dir)
            self._temp_dir = None

    def launch(self):
        """
        Launch the VM and make sure we cleanup and expose the
        command line/output in case of exception
        """

        if self._launched:
            raise QEMUMachineError('VM already launched')

        self._iolog = None
        self._qemu_full_args = None
        try:
            self._launch()
            self._launched = True
        except:
            self.shutdown()

            LOG.debug('Error launching VM')
            if self._qemu_full_args:
                LOG.debug('Command: %r', ' '.join(self._qemu_full_args))
            if self._iolog:
                LOG.debug('Output: %r', self._iolog)
            raise

    def _launch(self):
        """
        Launch the VM and establish a QMP connection
        """
        devnull = open(os.path.devnull, 'rb')
        self._pre_launch()
        self._qemu_full_args = (self._wrapper + [self._binary] +
                                self._base_args() + self._args)
        LOG.debug('VM launch command: %r', ' '.join(self._qemu_full_args))
        self._popen = subprocess.Popen(self._qemu_full_args,
                                       stdin=devnull,
                                       stdout=self._qemu_log_file,
                                       stderr=subprocess.STDOUT,
                                       shell=False,
                                       close_fds=False)
        self._post_launch()

    def wait(self):
        """
        Wait for the VM to power off
        """
        self._popen.wait()
        self._qmp.close()
        self._load_io_log()
        self._post_shutdown()

    def shutdown(self):
        """
        Terminate the VM and clean up
        """
        if self.is_running():
            try:
                self._qmp.cmd('quit')
                self._qmp.close()
            except:
                self._popen.kill()
            self._popen.wait()

        self._load_io_log()
        self._post_shutdown()

        exitcode = self.exitcode()
        if exitcode is not None and exitcode < 0:
            msg = 'qemu received signal %i: %s'
            if self._qemu_full_args:
                command = ' '.join(self._qemu_full_args)
            else:
                command = ''
            LOG.warn(msg, -exitcode, command)

        self._launched = False

    def qmp(self, cmd, conv_keys=True, **args):
        """
        Invoke a QMP command and return the response dict
        """
        qmp_args = dict()
        for key, value in args.items():
            if conv_keys:
                qmp_args[key.replace('_', '-')] = value
            else:
                qmp_args[key] = value

        return self._qmp.cmd(cmd, args=qmp_args)

    def command(self, cmd, conv_keys=True, **args):
        """
        Invoke a QMP command.
        On success return the response dict.
        On failure raise an exception.
        """
        reply = self.qmp(cmd, conv_keys, **args)
        if reply is None:
            raise qmp.QMPError("Monitor is closed")
        if "error" in reply:
            raise MonitorResponseError(reply)
        return reply["return"]

    def get_qmp_event(self, wait=False):
        """
        Poll for one queued QMP events and return it
        """
        if len(self._events) > 0:
            return self._events.pop(0)
        return self._qmp.pull_event(wait=wait)

    def get_qmp_events(self, wait=False):
        """
        Poll for queued QMP events and return a list of dicts
        """
        events = self._qmp.get_events(wait=wait)
        events.extend(self._events)
        del self._events[:]
        self._qmp.clear_events()
        return events

    def event_wait(self, name, timeout=60.0, match=None):
        """
        Wait for specified timeout on named event in QMP; optionally filter
        results by match.

        The 'match' is checked to be a recursive subset of the 'event'; skips
        branch processing on match's value None
           {"foo": {"bar": 1}} matches {"foo": None}
           {"foo": {"bar": 1}} does not matches {"foo": {"baz": None}}
        """
        def event_match(event, match=None):
            if match is None:
                return True

            for key in match:
                if key in event:
                    if isinstance(event[key], dict):
                        if not event_match(event[key], match[key]):
                            return False
                    elif event[key] != match[key]:
                        return False
                else:
                    return False

            return True

        # Search cached events
        for event in self._events:
            if (event['event'] == name) and event_match(event, match):
                self._events.remove(event)
                return event

        # Poll for new events
        while True:
            event = self._qmp.pull_event(wait=timeout)
            if (event['event'] == name) and event_match(event, match):
                return event
            self._events.append(event)

        return None

    def get_log(self):
        """
        After self.shutdown or failed qemu execution, this returns the output
        of the qemu process.
        """
        return self._iolog

    def add_args(self, *args):
        """
        Adds to the list of extra arguments to be given to the QEMU binary
        """
        self._args.extend(args)

    def set_machine(self, machine_type):
        """
        Sets the machine type

        If set, the machine type will be added to the base arguments
        of the resulting QEMU command line.
        """
        self._machine = machine_type

    def set_console(self, device_type=None):
        """
        Sets the device type for a console device

        If set, the console device and a backing character device will
        be added to the base arguments of the resulting QEMU command
        line.

        This is a convenience method that will either use the provided
        device type, of if not given, it will used the device type set
        on CONSOLE_DEV_TYPES.

        The actual setting of command line arguments will be be done at
        machine launch time, as it depends on the temporary directory
        to be created.

        @param device_type: the device type, such as "isa-serial"
        @raises: QEMUMachineAddDeviceError if the device type is not given
                 and can not be determined.
        """
        if device_type is None:
            if self._machine is None:
                raise QEMUMachineAddDeviceError("Can not add a console device:"
                                                " QEMU instance without a "
                                                "defined machine type")
            for regex, device in CONSOLE_DEV_TYPES.items():
                if re.match(regex, self._machine):
                    device_type = device
                    break
            if device_type is None:
                raise QEMUMachineAddDeviceError("Can not add a console device:"
                                                " no matching console device "
                                                "type definition")
        self._console_device_type = device_type

    @property
    def console_socket(self):
        """
        Returns a socket connected to the console
        """
        if self._console_socket is None:
            self._console_socket = socket.socket(socket.AF_UNIX,
                                                 socket.SOCK_STREAM)
            self._console_socket.connect(self._console_address)
        return self._console_socket


import numpy as np
from matplotlib import pyplot as plt
import os
from . import settings

plt.rcParams['font.sans-serif'] = ['Microsoft YaHei']  # 用来正常显示中文标签
plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示负号
plt.rcParams['figure.dpi'] = 300  # 分辨率
plt.rcParams['figure.figsize'] = (5.771, 5.771 * 0.618)  # 图片大小，单位是inches
np.set_printoptions(edgeitems=30, linewidth=200000)  # 输出不换行
np.set_printoptions(precision=2, suppress=True)  # 精度，不使用科学计数
np.set_printoptions(threshold=np.inf)  # 过多的行，不会变成省略号

def check_and_set_ticks(axes, xticks, yticks):
    xticks is not None and axes.xaxis.set_ticks(xticks)
    xticks is not None and axes.xaxis.limit_range_for_scale(min(xticks), max(xticks))
    yticks is not None and axes.yaxis.set_ticks(yticks)
    yticks is not None and axes.yaxis.limit_range_for_scale(min(yticks), max(yticks))

def plot_line(
        x, y, *args, figure: plt.Figure = None, axes: plt.Axes = None,
        xticks=None, yticks=None, xlabel=None, ylabel=None, **kwargs):
    figure: plt.Figure = plt.gcf() if figure is None else figure
    axes: plt.Axes = plt.gca() if axes is None else axes

    # 设置刻度
    check_and_set_ticks(axes, xticks, yticks)

    xlabel is None or axes.set_xlabel(xlabel)
    ylabel is None or axes.set_ylabel(ylabel)

    axes.plot(x, y, *args, **kwargs)
    return figure

def plot_surface(
        data, show_colorbar=True, extent=None, vmin=None, vmax=None, xticks=None, yticks=None,
        axes: plt.Axes = None, figure: plt.Figure = None, reverse=True):
    assert len(set(map(len, data))) == 1
    # 对数据进行倒序，以保证初值在最下边
    if reverse:
        data = data[::-1]
    # 获取图像和坐标轴对象
    figure = plt.figure() if figure is None else figure
    axes = plt.axes() if axes is None else axes
    # 计算最大值和最小值
    vmin = vmin is None and np.min(np.min(data)) or vmin
    vmax = vmax is None and np.max(np.max(data)) or vmax
    # 设置刻度
    check_and_set_ticks(axes, xticks, yticks)
    # 计算图像范围
    if not extent:
        if xticks is not None and yticks is not None:
            extent = (min(xticks), max(xticks), min(yticks), max(yticks))
        else:
            extent = (-1, 1, -1, 1)
    # 绘制二维图
    image = axes.imshow(
        data, vmin=vmin, vmax=vmax, interpolation='nearest',
        cmap=plt.cm.jet, extent=extent, aspect='auto')
    # 显示彩虹条图例
    if show_colorbar:
        plt.colorbar(image, ax=axes)

    return figure

def get_figure_path(file_name):
    path = os.path.join(settings.FIGURE_PATH, file_name)
    dir_name = os.path.dirname(path)
    os.path.exists(dir_name) or os.makedirs(dir_name)
    return path