from turtle import Screen
from paddle import Paddle
from ball import Ball
from scoreboard import Scoreboard
import time

SCREEN_HEIGHT = 600
SCREEN_WIDTH = 800
LEFT_PADDLE_X_COORD = -350
RIGHT_PADDLE_X_COORD = 350
PADDLE_Y_COORD = 0
FRONT_OF_LEFT_PADDLE_X_COORD = -330
FRONT_OF_RIGHT_PADDLE_X_COORD = 330
SCREEN_UPPER_BOUND = 280
SCREEN_LOWER_BOUND = -280
LEFT_SIDE_BOUNDARY = -352
RIGHT_SIDE_BOUNDARY = 352
LEFT_SCOREBOARD_COORD = (-100, 260)
RIGHT_SCOREBOARD_COORD = (100, 260)
GAME_ENDING_SCORE = 5
SECONDS_DELAY_AFTER_POINT = 0.5
DISTANCE_FROM_PADDLE_FOR_PADDLE_BOUNCE = 50

screen = Screen()
screen.tracer(0)
screen.bgcolor("black")
screen.setup(width=SCREEN_WIDTH, height=SCREEN_HEIGHT)
screen.title("PONG")

left_paddle = Paddle(LEFT_PADDLE_X_COORD, PADDLE_Y_COORD)
right_paddle = Paddle(RIGHT_PADDLE_X_COORD, PADDLE_Y_COORD)

ball = Ball()

left_score = Scoreboard(LEFT_SCOREBOARD_COORD)
right_score = Scoreboard(RIGHT_SCOREBOARD_COORD)

screen.listen()
screen.onkey(left_paddle.move_up, "w")
screen.onkey(left_paddle.move_down, "s")
screen.onkey(right_paddle.move_up, "o")
screen.onkey(right_paddle.move_down, "l")

game_is_running = True
while game_is_running:
    screen.update()
    time.sleep(ball.move_speed)
    ball.move()
    if ball.ycor() >= SCREEN_UPPER_BOUND or ball.ycor() <= SCREEN_LOWER_BOUND:
        ball.y_direction_reversal()
    if ball.xcor() >= FRONT_OF_RIGHT_PADDLE_X_COORD:
        if right_paddle.distance(ball) <= DISTANCE_FROM_PADDLE_FOR_PADDLE_BOUNCE and ball.direction == "right":
            ball.direction = "left"
            ball.x_direction_reversal()
            ball.increase_speed()
        elif ball.xcor() >= RIGHT_SIDE_BOUNDARY:
            ball.reset_position()
            left_score.add_point()
            time.sleep(SECONDS_DELAY_AFTER_POINT)
    if ball.xcor() <= FRONT_OF_LEFT_PADDLE_X_COORD:
        if left_paddle.distance(ball) <= DISTANCE_FROM_PADDLE_FOR_PADDLE_BOUNCE and ball.direction == "left":
            ball.direction = "right"
            ball.x_direction_reversal()
            ball.increase_speed()
        elif ball.xcor() <= LEFT_SIDE_BOUNDARY:
            ball.reset_position()
            right_score.add_point()
            time.sleep(SECONDS_DELAY_AFTER_POINT)
    if left_score.score == GAME_ENDING_SCORE:
        game_is_running = False
        left_score.game_over("left")
    elif right_score.score == GAME_ENDING_SCORE:
        game_is_running = False
        right_score.game_over("right")

screen.exitonclick()


'''Train CIFAR10 with PyTorch.'''
import socket

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torch.backends.cudnn as cudnn

import torchvision
import torchvision.transforms as transforms

import os
import shutil
import argparse

# from Trainer import Trainer
from Trainer_loss import Trainer

from models import *
import time

DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'

def write_train_time(s,rank):
    with open('./output/train_output_' + str(rank) + '.txt', 'a') as f:
        f.write(s+' ')

def write_test_time(s,rank):
    with open('./output/test_output_' + str(rank) + '.txt', 'a') as f:
        f.write(s+' ')

def write_delay_k(s,rank):
    with open('./output/delay_' + str(rank) + '.txt', 'a') as f:
        f.write(s+' ')

def main():
    parser = argparse.ArgumentParser(description='PyTorch CIFAR10 Training')
    parser.add_argument('--lr', default=0.1, type=float, help='learning rate')
    parser.add_argument('--resume', '-r', action='store_true',
                        help='resume from checkpoint')

    # Distributed
    parser.add_argument('--rank', type=int, help='the rank of part')
    parser.add_argument('--size', type=int, help='size of client in the world')
    parser.add_argument('--port', type=int, help='self port')
    parser.add_argument('--n_ip', type=str, help='next ring allreduce server ip')
    parser.add_argument('--n_port', type=int, help='next ring allreduce server port')
    parser.add_argument('--epoch', type=int, default=120 ,help='epoch')
    parser.add_argument('--dl', type=int, default=0 ,help='dynamic loss')
    args = parser.parse_args()

    # Distributed
    rank = args.rank
    world_size = args.size
    self_port = args.port
    next_addr = (args.n_ip, args.n_port)
    # Distributed End

    # clean env
    if os.path.exists(r'output'):
        shutil.rmtree(r'output')
    if not os.path.isdir('output'):
        os.mkdir('output')

    best_acc = 0  # best test accuracy
    start_epoch = 0  # start from epoch 0 or last checkpoint epoch

    # Data
    print('==> Preparing data..')
    transform_train = transforms.Compose([
        transforms.RandomCrop(32, padding=4),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
    ])

    transform_test = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
    ])

    trainset = torchvision.datasets.CIFAR10(
        root='./data', train=True, download=True, transform=transform_train)

    # Distributed
    sampler = torch.utils.data.DistributedSampler(trainset, num_replicas=world_size, rank=rank)
    # Distributed End

    trainloader = torch.utils.data.DataLoader(
        trainset, batch_size=32, shuffle=(sampler is None), sampler=sampler, num_workers=2)

    testset = torchvision.datasets.CIFAR10(
        root='./data', train=False, download=True, transform=transform_test)
    testloader = torch.utils.data.DataLoader(
        testset, batch_size=100, shuffle=False, num_workers=2)

    classes = ('plane', 'car', 'bird', 'cat', 'deer',
               'dog', 'frog', 'horse', 'ship', 'truck')

    # Init distributed
    _socket_recv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    _socket_recv.bind(('0.0.0.0', self_port))
    _socket_recv.listen(5)

    socket_send = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    if rank == 0:
        socket_send.connect(next_addr)
        print('Rank %d client connected' % rank)

        socket_recv, recv_addr = _socket_recv.accept()
        print('Rank %d server connected' % rank, recv_addr)
    else:
        socket_recv, recv_addr = _socket_recv.accept()
        print('Rank %d server connected' % rank, recv_addr)

        socket_send.connect(next_addr)
        print('Rank %d client connected' % rank)

    # Model
    print('==> Building model..')
    # net = VGG('VGG19')
    net = ResNet18()
    # net = PreActResNet18()
    # net = GoogLeNet()
    # net = DenseNet121()
    # net = ResNeXt29_2x64d()
    # net = MobileNet()
    # net = MobileNetV2()
    # net = DPN92()
    # net = ShuffleNetG2()
    # net = SENet18()
    # net = ShuffleNetV2(1)
    # net = EfficientNetB0()
    # net = RegNetX_200MF()
    net = net.to(DEVICE)
    if DEVICE == 'cuda':
        net = torch.nn.DataParallel(net)
        cudnn.benchmark = True

    if args.resume:
        # Load checkpoint.
        print('==> Resuming from checkpoint..')
        assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!'
        checkpoint = torch.load('./checkpoint/ckpt.pth')
        net.load_state_dict(checkpoint['net'])
        best_acc = checkpoint['acc']
        start_epoch = checkpoint['epoch']

    criterion = nn.CrossEntropyLoss()
    optimizer = optim.SGD(net.parameters(), lr=args.lr,
                          momentum=0.9, weight_decay=5e-4)

    # socket_send.setsockopt(
    #     socket.SOL_SOCKET,
    #     socket.SO_SNDBUF,
    #     1024 * 15000
    # )

    trainer = Trainer(net, rank, world_size, trainloader, testloader, optimizer, criterion, socket_send, socket_recv, args.dl)
    trainer.sync_model()
    for epoch in range(start_epoch, start_epoch + args.epoch):
        write_train_time(str(time.time()),rank)
        trainer.train(epoch)
        print("----- = test = -----")
        write_test_time(str(time.time()),rank)
        trainer.test(epoch)
    write_delay_k(str(trainer.delay)+'\n\n'+str(trainer.k_list),rank)
    # print(trainer.delay)
    # print(trainer.k_list)

if __name__ == '__main__':
    torch.multiprocessing.freeze_support()
    main()


import os

import math
import numpy as np
import torch.optim as optim
import torch
from torch_geometric.loader import DataLoader
from lifelines.utils import concordance_index as get_cindex
from torch.utils.data import Subset
from sklearn.model_selection import KFold

# from .MgraphDTA.dataset import *
from .MgraphDTA.preprocessing import GNNDataset
from .MgraphDTA.model import *
from .MgraphDTA.utils import *
from tqdm import tqdm
from lifelines.utils import concordance_index as ci
from sklearn.metrics import mean_squared_error
# from MgraphDTA.preprocessing import *

from .structure_dataset import DTADataset
import pandas as pd
import torch_geometric.data as data
from .model_NHGNN import NHGNN_DTA

class MergedDataset(data.Dataset):
    def __init__(self, dataset1, dataset2, transform=None, pre_transform=None):
        super(MergedDataset, self).__init__(None, transform, pre_transform)
        assert len(dataset1) == len(dataset2), "Datasets must be of equal length!"
        self.dataset1 = dataset1
        self.dataset2 = dataset2

    def len(self):
        return len(self.dataset1)

    def get(self, idx):
        data1 = self.dataset1[idx]
        data2 = self.dataset2[idx]

        # Simply return a tuple of the two data points
        return data1, data2

class MGraphDTA(nn.Module):
    def __init__(self, data_input_path, data_path, drug_vocab, target_vocab, auto_dataset: bool = True, filter_num=32, out_dim=1,
                 model_name="Mgraph_1", model_dir="saved_models", dataset_name="kiba", seed=42, test_ratio=0.15, tar_len=2600):
        super().__init__()
        # data
        self.data_input_path = data_input_path
        self.dataset_name = dataset_name
        if auto_dataset:
            self.dataset = self.get_dataset(data_input_path, dataset_name, data_path, target_vocab, tar_len)
            self.train_set, self.test_set = self.split_dataset(self.dataset, seed, test_ratio)

        # model
        self.model_name = model_name
        self.model_dir = model_dir
        # self.protein_encoder = TargetRepresentation(block_num, vocab_protein_size, embedding_size)
        self.protein_struc_model = NHGNN_DTA(pretrain_path=f"saved_models/TFusion_{dataset_name}.pt",
                                             data_path=f"data_input/{dataset_name}/raw/data.csv",
                                             saved_path=f"saved_models/NHGNN_{dataset_name}_test1.pkl",
                                             drug_vocab=drug_vocab, target_vocab=target_vocab, seed=seed, test_ratio=test_ratio)
        self.ligand_encoder = GraphDenseNet(num_input_features=22, out_dim=filter_num * 3, block_config=[8, 8, 8],
                                            bn_sizes=[2, 2, 2])

        self.classifier = nn.Sequential(
            nn.Linear(352, 1024),
            nn.ReLU(),
            nn.Dropout(0.1),
            nn.Linear(1024, 1024),
            nn.ReLU(),
            nn.Dropout(0.1),
            nn.Linear(1024, 256),
        )
        self.classifier2 = nn.Sequential(
            nn.ReLU(),
            nn.Dropout(0.1),
            nn.Linear(256, 256)
        )

        # self.classifier3 = nn.Sequential(
        #     nn.Linear(256, 256),
        #     nn.ReLU(),
        #     nn.Dropout(0.1),
        #     nn.Linear(256, 1)
        # )
        # self.dd1 = nn.Linear(256, 256)
        # self.dd2 = nn.Linear(256, 1)

    def get_dataset(self, data_input_path, dataset_name, data_path, target_vocab, tar_len):
        def target_to_graph(target_key, target_sequence, contact_dir):
            target_edge_index = []
            target_size = len(target_sequence)
            contact_file = os.path.join(contact_dir, target_key + '.npy')
            contact_map = np.load(contact_file)
            contact_map += np.matrix(np.eye(contact_map.shape[0]))
            index_row, index_col = np.where(contact_map > 0.8)
            for i, j in zip(index_row, index_col):
                target_edge_index.append([i, j])
            target_edge_index = np.array(target_edge_index)
            return target_size, target_edge_index

        print(f"processing proterin structure dataset of {dataset_name}...")

        df = pd.read_csv(data_path)
        smiles = set(df['compound_iso_smiles'])
        target = set(df['target_key'])

        target_seq = {}
        for i in range(len(df)):
            target_seq[df.loc[i, 'target_key']] = df.loc[i, 'target_sequence']
        target_graph = {}
        pconsc4_path = os.path.join(os.path.dirname(os.path.dirname(data_path)), "pconsc4")
        for k in target_seq:
            seq = target_seq[k]
            _, graph = target_to_graph(k, seq, pconsc4_path)
            target_graph[seq] = graph

        target_emb = {}
        target_len = {}
        for k in target_seq:
            seq = target_seq[k]
            content = []
            flag = 0
            for i in range(len(seq)):
                if flag >= len(seq):
                    break
                if (flag + 1 < len(seq)):
                    if target_vocab.stoi.__contains__(seq[flag:flag + 2]):
                        content.append(target_vocab.stoi.get(seq[flag:flag + 2]))
                        flag = flag + 2
                        continue
                content.append(target_vocab.stoi.get(seq[flag], target_vocab.unk_index))
                flag = flag + 1

            if len(content) > tar_len - 2:
                content = content[:tar_len - 2]

            X = [target_vocab.sos_index] + content + [target_vocab.eos_index]
            target_len[k] = len(content)
            if tar_len > len(X):
                padding = [target_vocab.pad_index] * (tar_len - len(X))
                X.extend(padding)
            target_emb[seq] = torch.tensor(X)

        fpath = os.path.join(data_input_path, dataset_name)
        root_path = os.path.dirname(os.path.dirname(data_path))
        data_set1 = GNNDataset(fpath)
        data_set2 = DTADataset(root=root_path, path=data_path, target_graph=target_graph, target_len=target_len)
        data_set = MergedDataset(data_set1, data_set2)
        return data_set

    def split_dataset(self, dataset, seed, test_ratio):
        test_size = round(len(dataset) * test_ratio)
        train_dataset, test_dataset = torch.utils.data.random_split(
            dataset=dataset,
            lengths=[len(dataset) - test_size, test_size],
            generator=torch.Generator().manual_seed(seed)
        )
        return train_dataset, test_dataset

    def forward(self, data, data2, output_vector=False, freeze_front=False):
        protein_x = self.protein_struc_model.forward_pro(data2)
        ligand_x = self.ligand_encoder(data)

        xx = torch.cat([protein_x, ligand_x], dim=-1)
        x = self.classifier(xx)
        out = self.classifier2(x)

        if output_vector:
            return out

        # if freeze_front:
        #     out = self.dd1(out) * out
        #     out = self.dd2(out)
        #     return out

        out = torch.norm(out, dim=-1)

        # if output_vector:
        #     return torch.cat((xx, out), dim=-1), torch.cat((x, out), dim=-1)
        return out

    def val(self, test_set, batch_size=128, use_cuda=True):
        dataloader_test = DataLoader(test_set, batch_size=batch_size, shuffle=False)
        # dataloader_test = torch.utils.data.DataLoader(test_set, batch_size=batch_size, shuffle=False, collate_fn=self.collate)
        model = self
        model.eval()
        total_pred = torch.Tensor()
        total_label = torch.Tensor()
        print("validating...")
        with torch.no_grad():
            for data, data2 in tqdm(dataloader_test, ncols=80):
                if use_cuda:
                    data = data.cuda()
                out = model(data, data2)
                total_pred = torch.cat((total_pred, out.cpu()), 0)
                total_label = torch.cat((total_label, data.y.cpu()), 0)
        all_ci = ci(total_label.detach().numpy().flatten(), total_pred.detach().numpy().flatten())
        all_mse = mean_squared_error(total_label.detach().numpy().flatten(), total_pred.detach().numpy().flatten())
        model.train()
        return all_mse, all_ci

    def fit(self, train_set, test_set, save_model=True, device=torch.device('cuda'), early_stop_epoch=400, lr=5e-4,
            freeze_front=False):
        print(f"len(train_set)={len(train_set)}")
        print(f"len(test_set)={len(test_set)}")

        train_loader = DataLoader(train_set, batch_size=256, shuffle=True)

        epochs = 3000
        steps_per_epoch = 50
        num_iter = math.ceil((epochs * steps_per_epoch) / len(train_loader))
        break_flag = False

        model = self.to(device)

        optimizer = optim.Adam(model.parameters(), lr=lr)
        criterion = nn.MSELoss()

        global_step = 0
        global_epoch = 0

        running_loss = AverageMeter()
        running_cindex = AverageMeter()
        running_best_mse = BestMeter("min")

        model.train()

        if freeze_front:
            for param in model.parameters():
                param.requires_grad = False
            for param in model.dd1.parameters():
                param.requires_grad = True
            for param in model.dd2.parameters():
                param.requires_grad = True
            optimizer = optim.Adam(filter(lambda p : p.requires_grad, model.parameters()), lr=lr)

        for i in range(num_iter):
            if break_flag:
                break

            for data, data2 in train_loader:
                model.train()

                global_step += 1
                data = data.to(device)
                pred = model(data, data2, freeze_front=freeze_front)

                loss = criterion(pred.view(-1), data.y.view(-1))
                cindex = get_cindex(data.y.detach().cpu().numpy().reshape(-1),
                                    pred.detach().cpu().numpy().reshape(-1))

                optimizer.zero_grad()
                loss.backward()
                optimizer.step()

                running_loss.update(loss.item(), data.y.size(0))
                running_cindex.update(cindex, data.y.size(0))

                if global_step % steps_per_epoch == 0:

                    global_epoch += 1

                    epoch_loss = running_loss.get_average()
                    epoch_cindex = running_cindex.get_average()
                    running_loss.reset()
                    running_cindex.reset()

                    mse, ci = self.val(test_set)

                    msg = "epoch-%d, loss-%.4f, cindex-%.4f, mse-%.4f ci-%.4f" % (
                    global_epoch, epoch_loss, epoch_cindex, mse, ci)
                    print(msg)

                    if mse < running_best_mse.get_best():
                        running_best_mse.update(mse)
                        if save_model:
                            save_model_dict(model, self.model_dir, self.model_name)
                    else:
                        count = running_best_mse.counter()
                        if count > early_stop_epoch:
                            break_flag = True
                            break


import logging
import re
import time
from urllib.parse import parse_qs
from urllib.parse import urlparse

import pandas as pd
import requests
from bs4 import BeautifulSoup

from covidata import config
from covidata.persistencia.dao import persistir_dados_hierarquicos

def pt_PortoAlegre():
    url = config.url_pt_PortoAlegre
    parsed_uri = urlparse(url)
    url_base = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)

    page = requests.get(url)
    parser = BeautifulSoup(page.content, 'lxml')

    proximas_paginas = __get_paginacao(parser, url_base)

    nomes_colunas_documentos = []
    linhas_df = []
    linhas_df_documentos = []
    linhas_df_itens = []

    linhas_df_documentos, linhas_df_itens, nomes_colunas_documentos = __processar_pagina(linhas_df,
                                                                                linhas_df_documentos,
                                                                                linhas_df_itens,
                                                                                nomes_colunas_documentos,
                                                                                parser, url_base)

    for pagina in proximas_paginas:
        parser = __parse(pagina)
        linhas_df_documentos, linhas_df_itens, nomes_colunas_documentos = __processar_pagina(linhas_df,
                                                                                linhas_df_documentos,
                                                                                linhas_df_itens,
                                                                                nomes_colunas_documentos,
                                                                                parser, url_base)

    nomes_colunas = ['Número da licitação', 'Objeto', 'Tipo', 'Status', 'Aplicar o Decreto 10.024/2019',
                     'Início de propostas', 'Final de propostas', 'Limite para impugnações', 'Data de Abertura',
                     'Pregoeiro', 'Autoridade Competente', 'Apoio', 'Origem dos Recursos', 'Operação']
    df = pd.DataFrame(linhas_df, columns=nomes_colunas)

    df_documentos = pd.DataFrame(linhas_df_documentos, columns=nomes_colunas_documentos)

    nomes_colunas_itens = ['Número da Licitação', 'Item', 'Descrição', 'Unidade', 'Quantidade', 'Melhor Lance (R$',
                           'Situação', 'Modalidade']
    df_itens = pd.DataFrame(linhas_df_itens, columns=nomes_colunas_itens)

    persistir_dados_hierarquicos(df, {'Documentos': df_documentos, 'Itens': df_itens}, 'portal_transparencia',
                                 'Licitacoes', 'RS', 'Porto Alegre')

def __get_paginacao(parser, url_base):
    tag_paginacao = parser.findAll("ul", {"class": "pagination"})

    if len(tag_paginacao) > 0:
        paginador = tag_paginacao[0]
        paginas = paginador.findAll('li')
        proximas_paginas = []

        for pagina in paginas[1:]:
            link = url_base + pagina.findAll('a')[0].attrs['href']
            proximas_paginas.append(link)

        return proximas_paginas
    else:
        return []

def __get_paginacao_itens(parser, url_base, codigo_interno_licitacao):
    tag_paginacao = parser.findAll("ul", {"class": "pagination"})
    proximas_paginas = []

    if len(tag_paginacao) > 0:
        paginador = tag_paginacao[0]
        paginas = paginador.findAll('li')
        numero_pagina = 2

        for _ in paginas[1:]:
            link = url_base + f'18/SessaoPublica/lances/htmlAba.asp?param=0&ttTipo=0&ttCD_LICITACAO=' \
                f'{codigo_interno_licitacao}&ttPagina={numero_pagina}'
            numero_pagina += 1
            proximas_paginas.append(link)
    else:
        link = url_base + f'18/SessaoPublica/lances/htmlAba.asp?param=0&ttTipo=0&ttCD_LICITACAO=' \
            f'{codigo_interno_licitacao}&ttPagina=1'
        proximas_paginas.append(link)

    return proximas_paginas

def __processar_pagina(linhas_df, linhas_df_documentos, linhas_df_itens, nomes_colunas_documentos, parser, url_base):
    registros = parser.findAll('div', {'class': 'item-registro xs-col-12'})

    for registro in registros:
        textos = re.split('\r\n|\n\t|\t\n', registro.findAll('h2')[0].get_text().strip())

        numero_licitacao = textos[0]

        conteudo = registro.findAll('div')[0]
        divs = conteudo.findAll('div')
        div_descricao_registro = divs[0]
        objeto = div_descricao_registro.findAll('p')[0].get_text().strip()
        div_detalhes = divs[1]
        spans = div_detalhes.findAll('span')

        tipo = spans[0].get_text().strip()
        tipo = tipo[tipo.find('\n') + 1:len(tipo)].strip()

        status = spans[1].get_text().strip()
        status = status[status.find('\n') + 1:len(status)].strip()

        # página de detalhes
        parser_detalhes, codigo_interno_licitacao = __processar_detalhes(numero_licitacao, linhas_df, objeto, spans,
                                                                         status, tipo, url_base)

        # documentos
        linhas_df_documentos, nomes_colunas_documentos = __processar_documentos(numero_licitacao, linhas_df_documentos,
                                                                                parser_detalhes, url_base)

        # itens
        linhas_df_itens = __processar_itens(numero_licitacao, linhas_df_itens, parser_detalhes, url_base,
                                            codigo_interno_licitacao)

    return linhas_df_documentos, linhas_df_itens, nomes_colunas_documentos

def __processar_detalhes(numero_licitacao, linhas_df, objeto, spans, status, tipo, url_base):
    dados_adicionais = spans[2]
    url_detalhes = dados_adicionais.findAll('a', {'title': 'Dados do Processo'})[0].attrs['href']
    codigo_interno_licitacao = url_detalhes[url_detalhes.rfind('-') + 1:url_detalhes.rfind('/')]
    url_detalhes = url_base + url_detalhes

    parser_detalhes = __parse(url_detalhes)

    dados_licitacao = parser_detalhes.findAll('p', {'class': 'ff1 f400 fs16 fcW'})[0]
    bs = dados_licitacao.findAll('b')
    aplicar_decreto = bs[1].next_sibling.string.strip()
    inicio_propostas = bs[3].next_sibling.string.strip()
    final_propostas = bs[4].next_sibling.string.strip()
    limite_impugnacoes = bs[5].next_sibling.string.strip()
    data_abertura = bs[6].next_sibling.string.strip()
    pregoeiro = bs[7].next_sibling.string.strip()
    autoridade_competente = bs[8].next_sibling.string.strip()
    apoio = bs[9].next_sibling.string.strip()
    origem_recursos = ''
    operacao = ''

    i = 11
    if 'Origem dos Recursos:' in bs[i].get_text():
        origem_recursos = bs[i].next_sibling.string.strip()
        i += 1

    if 'Operação:' in bs[i].get_text():
        operacao = bs[i].next_sibling.string.strip()

    linhas_df.append([numero_licitacao, objeto, tipo, status, aplicar_decreto, inicio_propostas, final_propostas,
                      limite_impugnacoes, data_abertura, pregoeiro, autoridade_competente, apoio, origem_recursos,
                      operacao])

    return parser_detalhes, codigo_interno_licitacao

def __parse(url):
    page = requests.get(url)
    return BeautifulSoup(page.content, 'lxml')

def __processar_documentos(numero_licitacao, linhas_df_documentos, parser_detalhes, url_base):
    div_documentos = parser_detalhes.findAll('div', {'class': 'col-xs-12 col-sm-6 bgMidGreen'})[0]
    url_documentos = div_documentos.findAll('a', {'title': 'Atas e demais documentos do processo'})[0].attrs['href']
    url_documentos = url_base + url_documentos
    parsed_url_documentos = urlparse(url_documentos)
    chave = parse_qs(parsed_url_documentos.query)['ttCD_CHAVE'][0]

    parser_documentos = __parse(url_documentos)

    span = parser_documentos.findAll('span', {'class': 'green'})
    numero_processo_interno = span[0].findAll('b')[0].get_text()
    tabela = parser_documentos.findAll('table')[0]
    cabecalho = tabela.findAll('th')
    nomes_colunas_documentos = ['Número da Licitação', 'Número do Processo Interno'] + \
                               [coluna.get_text() for coluna in cabecalho] + \
                               ['Comentário']
    linhas_documentos = tabela.findAll('tbody')[0].findAll('tr')

    for linha_documento in linhas_documentos:
        tds = linha_documento.findAll('td')
        a = tds[0].findAll('a')

        if len(a) > 0 and a[0].attrs['title'] == 'Clique para expandir':
            comentario = a[0].get_text()
            # Execita a chamada que a página executaria caso o clique fosse realizado
            onclick = linha_documento.attrs['onclick']
            codigo_fornecedor = onclick[onclick.find('(') + 1:onclick.find(',')]
            url_documentos_fornecedores = url_base + f'18/Atas/DocumentoFornecedor/?ttCD_LICITACAO={chave}' \
                f'&ttCD_FORNECEDOR={codigo_fornecedor}'

            # parser_documentos_fornecedor = __parse(driver, url_documentos_fornecedores)
            parser_documentos_fornecedor = __parse(url_documentos_fornecedores)

            links = parser_documentos_fornecedor.findAll('a')

            for link in links:
                nome_arquivo = link.get_text().strip()
                if nome_arquivo != '':
                    linhas_df_documentos.append(
                        [numero_licitacao, numero_processo_interno, nome_arquivo, '', url_base + link.attrs['href'],
                         comentario])
        else:
            linhas_df_documentos.append(
                [numero_licitacao, numero_processo_interno, tds[0].get_text(), tds[1].get_text(),
                 tds[2].findAll('a')[0].attrs['href'], ''])

    return linhas_df_documentos, nomes_colunas_documentos

def __processar_itens(numero_licitacao, linhas_df_itens, parser, url_base, codigo_interno_licitacao):
    proximas_paginas = __get_paginacao_itens(parser, url_base, codigo_interno_licitacao)

    for pagina in proximas_paginas:
        parser = __parse(pagina)
        __processar_pagina_itens(numero_licitacao, linhas_df_itens, parser)

    return linhas_df_itens

def __processar_pagina_itens(numero_licitacao, linhas_df_itens, parser):
    try:
        div_itens = parser.findAll('div', {'class': 'list-itens-processo'})[0]
    except IndexError:
        print('aqui')

    divs_itens = div_itens.findAll('div', {'class': 'item-processo'})

    for div_item in divs_itens:
        spans = div_item.findAll('span')
        codigo_item = spans[0].contents[1].string
        descricao = spans[1].get_text()
        unidade = spans[2].contents[2].string
        quantidade = spans[3].contents[2].string
        melhor_lance = spans[4].contents[2].string
        svgs = spans[5].findAll('svg')

        situacao = ''
        modalidade = ''

        if len(svgs) > 0:
            situacao = svgs[0].get_text()
            modalidade = svgs[1].find('title').get_text()

        linhas_df_itens.append(
            [numero_licitacao, codigo_item, descricao, unidade, quantidade, melhor_lance, situacao, modalidade])

def main():
    logger = logging.getLogger('covidata')
    logger.info('Portal de transparência da capital...')
    start_time = time.time()

    pt_PortoAlegre()

    logger.info("--- %s segundos ---" % (time.time() - start_time))

main()


import MetaTrader5 as mt5

mt5.initialize()

forex = ['EUR', 'GBP', 'AUD', 'NZD', 'USD', 'CHF', 'CAD', 'SGD', 'NOK', 'SEK']

commodity = ['XAU', 'XPD', 'XPT', 'XNG', 'XTI', 'XBR', 'DXY']

positions = mt5.positions_get(symbol='AUDJPY')
# print(positions)

def volAdjust(vol, asset):

    if asset[0:3] in forex:
        return vol * 100000
    elif asset[0:3] in commodity:
        return vol * 100
    elif asset[0:3] == 'XAG':
        return vol * 1000
    else:
        return vol

def volume(positions):

    if len(positions) == 1:         #For one trade/asset
        if positions[0].type == 0:
            lot = positions[0].volume
        else:
            lot = -(positions[0].volume)
        return volAdjust(lot, positions[0].symbol)
    else:                           #For more than one trades
        volume_ = 0
        for i in positions:
            if i.type == 0:         #for a long position
                volume_ += i.volume
            else:                   #For a short position
                volume_ -= i.volume
        return volAdjust(volume_, positions[0].symbol)

def exposure(positions):

    exposure_ = 0
    if len(positions) == 1:
        exposure_ = volume(positions) * positions[0].price_open
        return exposure_
    else:
        exposure_ = 0
        for i in positions:
            if i.type == 0:
                exposure_ += volAdjust(i.volume, i.symbol) * i.price_open
            else:
                exposure_ -= volAdjust(i.volume, i.symbol) * i.price_open
            return exposure_

print(f"Volumes: {volume(positions)}")
print(f"Exposure: {exposure(positions)}")

mt5.shutdown()