import torch
import argparse
import os
import pandas as pd
from tqdm import tqdm

import glob
from data_helper import TaskManager, BenchmarkTaskManager, all_normal_form
from typing import List

from fol import (
    BetaEstimator4V,
    order_bounds,
)
from utils.util import (
    Writer,
    load_data_with_indexing,
    load_task_manager,
    read_from_yaml,
    set_global_seed,
)

from collections import defaultdict

def eval_step(model, eval_iterator, device, mode, allowed_easy_ans=False):
    logs = defaultdict(lambda: defaultdict(float))
    with torch.no_grad():
        for data in tqdm(eval_iterator):
            for key in data:
                pred = data[key]["emb"]
                all_logit = model.compute_all_entity_logit(
                    pred, union=("u" in key or "U" in key)
                )  # batch*nentity
                argsort = torch.argsort(all_logit, dim=1, descending=True)
                ranking = argsort.clone().to(torch.float)
                #  create a new torch Tensor for batch_entity_range
                if device != torch.device("cpu"):
                    ranking = ranking.scatter_(
                        1,
                        argsort,
                        torch.arange(model.n_entity)
                        .to(torch.float)
                        .repeat(argsort.shape[0], 1)
                        .to(device),
                    )
                else:
                    ranking = ranking.scatter_(
                        1,
                        argsort,
                        torch.arange(model.n_entity)
                        .to(torch.float)
                        .repeat(argsort.shape[0], 1),
                    )
                # achieve the ranking of all entities
                for i in range(all_logit.shape[0]):
                    if mode == "train":
                        easy_ans = []
                        hard_ans = data[key]["answer_set"][i]
                    else:
                        if allowed_easy_ans:
                            easy_ans = []
                            hard_ans = list(
                                set(data[key]["hard_answer_set"][i]).union(
                                    set(data[key]["easy_answer_set"][i])
                                )
                            )
                        else:
                            easy_ans = data[key]["easy_answer_set"][i]
                            hard_ans = data[key]["hard_answer_set"][i]

                    num_hard = len(hard_ans)
                    num_easy = len(easy_ans)
                    assert len(set(hard_ans).intersection(set(easy_ans))) == 0
                    # only take those answers' rank
                    cur_ranking = ranking[i, list(easy_ans) + list(hard_ans)]
                    cur_ranking, indices = torch.sort(cur_ranking)
                    masks = indices >= num_easy
                    if device != torch.device("cpu"):
                        answer_list = (
                            torch.arange(num_hard + num_easy).to(torch.float).to(device)
                        )
                    else:
                        answer_list = torch.arange(num_hard + num_easy).to(torch.float)

                    cur_ranking = cur_ranking - answer_list + 1
                    # filtered setting: +1 for start at 0, -answer_list for ignore other answers

                    cur_ranking = cur_ranking[masks]
                    # only take indices that belong to the hard answers
                    mrr = torch.mean(1.0 / cur_ranking).item()
                    h1 = torch.mean((cur_ranking <= 1).to(torch.float)).item()
                    h3 = torch.mean((cur_ranking <= 3).to(torch.float)).item()
                    h10 = torch.mean((cur_ranking <= 10).to(torch.float)).item()
                    add_hard_list = torch.arange(num_hard).to(torch.float).to(device)
                    hard_ranking = (
                        cur_ranking + add_hard_list
                    )  # for all hard answer, consider other hard answer
                    logs[key]["retrieval_accuracy"] += torch.mean(
                        (hard_ranking <= num_hard).to(torch.float)
                    ).item()
                    logs[key]["MRR"] += mrr
                    logs[key]["HITS1"] += h1
                    logs[key]["HITS3"] += h3
                    logs[key]["HITS10"] += h10
                num_query = all_logit.shape[0]
                logs[key]["num_queries"] += num_query
        for key in logs.keys():
            for metric in logs[key].keys():
                if metric != "num_queries":
                    logs[key][metric] /= logs[key]["num_queries"]
            # print(key, logs[key]['MRR'])
    # torch.cuda.empty_cache()

    return logs

def save_benchmark(log, writer, taskmanger: BenchmarkTaskManager):
    form_log = defaultdict(lambda: defaultdict(float))
    for normal_form in all_normal_form:
        formula = taskmanger.form2formula[normal_form]
        if formula in log:
            form_log[normal_form] = log[formula]
            print(normal_form, log[formula]['MRR'])
    # writer.save_dataframe(form_log, f"eval_{taskmanger.type_str}.csv")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--config", default="config/benchmark_BetaE.yaml", type=str)
    parser.add_argument("--prefix", default="dev", type=str)
    parser.add_argument("--checkpoint_path", default=None, type=str)
    parser.add_argument("--load_step", default=0, type=int)
    parser.add_argument("--test", default=None, type=str)
    args = parser.parse_args()

    assert args.test, 'enter test file'
    """
        Load config
    """
    configure = read_from_yaml(args.config)
    print("loaded configuration")
    """
        Load model
    """

    if configure.get("cuda", -1) >= 0 and torch.cuda.is_available():
        device = torch.device("cuda:{}".format(configure["cuda"]))
        # logging.info('Device use cuda: %s' % configure['cuda'])
    else:
        device = torch.device("cpu")

    print(device)
    data_folder = configure["data"]["data_folder"]
    (
        entity_dict,
        relation_dict,
        projection_train,
        reverse_projection_train,
        projection_valid,
        reverse_projection_valid,
        projection_test,
        reverse_projection_test,
    ) = load_data_with_indexing(data_folder)

    print("loaded data")
    n_entity, n_relation = len(entity_dict), len(relation_dict)

    case_name = f'dev/{args.checkpoint_path.split("/")[-1]}'
    writer = Writer(case_name=case_name, config=configure, log_path="benchmark_log")

    model_name = configure["estimator"]["embedding"]
    model_params = configure["estimator"][model_name]
    model_params["n_entity"], model_params["n_relation"] = n_entity, n_relation
    model_params["negative_sample_size"] = configure["train"]["negative_sample_size"]
    model_params["device"] = device
    if model_name == "beta":
        model = BetaEstimator4V(**model_params)
        allowed_norm = ["DeMorgan", "DNF+MultiIU"]
    else:
        assert False, "Not valid model name!"
    model.to(device)

    print("loaded model")
    if args.checkpoint_path:
        torch.load(args.checkpoint_path, map_location="cpu")
    """
        Load data
    """

    test_tm_list = []
    # query_files = glob.glob(os.path.join(configure["data"]["data_folder"], "*.csv"))
    query_files = [args.test] # Training code have to be used
    print(f"Queries files: {query_files}")
    assert query_files, "Probably wrong dataset folder"
    for query_file in query_files:
        formula_id_data = pd.read_csv(configure["evaluate"]["formula_id_file"])
        type_str = os.path.splitext(os.path.split(query_file)[1])[0].split("-")[1]
        test_tm = BenchmarkTaskManager(
            formula_id_data, data_folder, type_str, device, model
        )
        test_iterator = test_tm.build_iterators(
            model, batch_size=configure["evaluate"]["batch_size"]
        )
        test_tm_list.append(test_tm)
    # print(configure)
    print("data processed. Start inferencing")
    # print(test_tm_list)
    for test_tm in test_tm_list:
        test_iterator = test_tm.build_iterators(
            model, batch_size=configure["evaluate"]["batch_size"]
        )
        _log = eval_step(model, test_iterator, device, mode="test")
        """
        test_iterator = test_tm.build_iterators(model, batch_size=configure['evaluate']['batch_size'])
        _log_easy = eval_step(model, test_iterator, device, mode='test', allowed_easy_ans=True)
        for formula in _log_easy:
            for metrics in _log_easy[formula]:
                _log[formula][f'easy_{metrics}'] = _log_easy[formula][metrics]
        """
        save_benchmark(_log, writer, test_tm)

    """
        MRR
    """


from metaflow import FlowSpec,card,step,batch,current,Parameter

CHART_OPTIONS = [
    {
        "caption":"This is a first dummy chart", # Caption of the chart
        "x_key" : "x_1",  # The key to match in Task object
        "y_key" : "y_1",  # The key to match in Task object
        "xlabel": "some x label",
        "ylabel": "some y label",
        "chart_type":"line",
        "id" : "cid1",
    },
    {
        "caption":"This is a second dummy chart", # Caption of the chart
        "x_key" : "x_2",  # The key to match in Task object
        "y_key" : "y_2",  # The key to match in Task object
        "xlabel": "some x label",
        "ylabel": "some y label",
        "chart_type":"line",
        "id" : "cid2",
    },
]
TABLES = [
    {
        "heading":"Model Metadata",
        "cells" : [
            {
                "name":"Path of model : (S3 path to the model weights)", # Name is the name of the property to show in the table.
                "key" : "s3_url"  # The key to match in Task object
            },
            {
                "name":"Wandb dashboard URL",
                "key" :"wandb_url"
            },
            {
                "name":"Job Execution Meduim", #
                "key": "exec_medium"
            }
        ]
    },
]

IMAGES = [
   {
        "caption":"Image From Remote URL", # Caption for the image.
        # The "key" is the key to match in Task object to retrieve the image.
        "key" : "",
        # The artifact Actual path to the image.
        # This can be a remote path
        "path_key": "remote_image"
    },
    {
        "caption":"Image from a Metaflow artifact",
        "key" : "random_image",
        "path_key": ""
    },
]
class CardPipelineFlow(FlowSpec):
    num_rows = Parameter('num-rows',default = 1000000,type=int,help='The number of rows from the dataset to use for Training.')

    batch_size = Parameter('batch-size',default = 64,type=int,help='Batch size to use for training the model.')

    browsing_path_parquet = Parameter(
        'browsing-path-parquet',\
        envvar="BROWSING_PATH_PARQUET",\
        default=None,type=str,help='Path to the browsing.parquet file in S3. This file contains browsing related data in the Coveo Challenge'
    )

    sku_path_parquet = Parameter(
        'sku-path-parquet',\
        envvar="SKU_PATH_PARQUET",\
        default=None,type=str,help='Path to the sku_to_content.parquet files in S3; This file contains unique productids.'
    )

    max_epochs = Parameter(
        'max-epochs',\
        envvar="MAX_EPOCHS",\
        default=1,type=int,help='Maximum number of epochs to train model.'
    )

    num_gpus = Parameter(
        'num-gpus',
        envvar="NUM_GPUS",\
        default=0,type=int,help='Number of GPUs to use when training the model.'
    )

    @step
    def start(self):
        self.next(self.train)

    # @batch(cpu=4,memory=30000,gpu=1,image='valayob/coveo-challenge-flow-image:0.4')
    @card(type='coveo_data_card',\
        options={\
            "charts": CHART_OPTIONS,\
            "tables": TABLES,\
            "images":IMAGES,\
            "show_parameters":True
        },\
        id='testcard')
    @step
    def train(self):
        import random
        from metaflow import current
        import numpy as np
        self.params = current.parameter_names
        print(self.params)
        self.wandb_url= "<WANDBURL COMES HERE>"
        self.model_wieghts_path= "<MODEL WEIGHTS COMES HERE>"
        self.exec_medium = "local"
        self.random_image = np.random.randn(1024,768).tolist()
        self.remote_image = "https://picsum.photos/1024/768"
        self.y_1 = np.random.randn(10).tolist()
        self.x_1 = [i for i in range(1,10)]
        self.y_2 = [random.randint(0,10) for _ in range(10)]
        self.x_2 = [i for i in range(1,10)]

        self.next(self.end)

    @step
    def end(self):
        print("Done Computation")

if __name__ == "__main__":
    CardPipelineFlow()

{'random': 'random', 'current': 'metaflow.current', 'np': 'numpy'}


from typing import TextIO
from sssom.parsers import parse_sssom_table
from oaklib import OntologyResource
from oaklib.implementations import SqlImplementation
import click
from sssom.writers import write_table

SSSOM_FILE = "../ontology/tmp/mondo.sssom.tsv"
RESOURCE = "../ontology/tmp/mondo-ingest.db"
OBJECT_LABEL_COLUMN = "object_label"
OBJECT_ID_COLUMN = "object_id"

input_argument = click.argument("input", required=True, type=click.Path(exists=True))
output_option = click.option(
    "-o",
    "--output",
    help="Path of SSSOM output file.",
    type=click.File(mode="w"),
    default=SSSOM_FILE,
)

@click.group()
def main():
    """Run the CLI."""
    pass

@main.command()
@input_argument
@output_option
def run(input: str = SSSOM_FILE, output: TextIO = SSSOM_FILE):
    """Get object_label from resource via oaklib."""
    msdf = parse_sssom_table(input)
    ontology_resource = OntologyResource(slug=RESOURCE, local=True)
    oi = SqlImplementation(ontology_resource)

    # Condition to check if OBJECT_LABEL_COLUMN is empty
    is_empty = msdf.df[OBJECT_LABEL_COLUMN].isnull() | (
        msdf.df[OBJECT_LABEL_COLUMN] == ""
    )

    # Apply the function only where OBJECT_LABEL_COLUMN is empty
    msdf.df.loc[is_empty, OBJECT_LABEL_COLUMN] = msdf.df.loc[
        is_empty, OBJECT_ID_COLUMN
    ].apply(lambda x: oi.label(x))

    write_table(msdf=msdf, file=output)

    # if OBJECT_LABEL_COLUMN not in msdf.df.columns:
    #     msdf.df[OBJECT_LABEL_COLUMN] = msdf.df[OBJECT_ID_COLUMN].apply(lambda x: oi.label(x))
    #     write_table(msdf=msdf,file=output)

if __name__ == "__main__":
    main()


import numpy as np
from joblib import dump
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import recall_score, precision_score, roc_auc_score
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB

from service.constants import RANDOM_STATE, TEST_SET_SIZE, MIN_TEST_ITEMS_COUNT, VECTORIZER_PATH_TEMPLATE, \
    VECTORIZER_PARAMS_PATH_TEMPLATE, MODEL_PATH_TEMPLATE

TEXT_COLUMN_KEY = 'text'
SCORE_COLUMN_KEY = 'score'

def train_model(df, working_directory):
    if TEXT_COLUMN_KEY not in df.columns or SCORE_COLUMN_KEY not in df.columns:
        raise Exception(f'Data should contain "{TEXT_COLUMN_KEY}" and "{SCORE_COLUMN_KEY}" columns')

    if len(df) == 0:
        raise Exception('Data should contain at least 1 entry')

    path_vectorizer = VECTORIZER_PATH_TEMPLATE.substitute(working_directory=working_directory)
    path_vectorizer_params = VECTORIZER_PARAMS_PATH_TEMPLATE.substitute(working_directory=working_directory)
    path_model = MODEL_PATH_TEMPLATE.substitute(working_directory=working_directory)

    labels = (df['score'] >= 3).astype(np.int32).values

    min_df = min(round(len(df) * 0.0001), 1000)  # not greater when 1000
    vectorizer = TfidfVectorizer(
        strip_accents='unicode',
        lowercase=True,
        stop_words='english',
        token_pattern=r'(?u)(\b[a-z]{2,}\b|[\u263a-\U0001f645])',
        ngram_range=(1, 2),
        min_df=min_df,
        binary=True
    )
    dump(vectorizer, path_vectorizer)

    text_vectorized = vectorizer.fit_transform(df['text'])

    vectorizer_params = {
        'vocabulary_': vectorizer.vocabulary_,
        'idf_': vectorizer.idf_,
    }
    dump(vectorizer_params, path_vectorizer_params)

    if len(df) * TEST_SET_SIZE >= MIN_TEST_ITEMS_COUNT:
        x_train, x_test, y_train, y_test = train_test_split(text_vectorized, labels, test_size=TEST_SET_SIZE,
                                                            random_state=RANDOM_STATE)
        test_dataset_type = 'test'
    else:
        x_train, x_test, y_train, y_test = text_vectorized, text_vectorized, labels, labels
        test_dataset_type = 'train'

    model = MultinomialNB()
    model.fit(x_train, y_train)
    dump(model, path_model)

    prediction_probas = model.predict_proba(x_test)
    predictions = np.argmax(prediction_probas, axis=1)
    recall = recall_score(y_test, predictions)
    precision = precision_score(y_test, predictions)
    roc_auc = roc_auc_score(y_test, prediction_probas[:, 1])
    result = f'Model has been trained successfully!\nTested on {test_dataset_type} dataset of size: {x_test.shape[0]}\nrecall: {recall}\nprecision: {precision}\nroc-auc: {roc_auc}'

    return result


from django.urls import path

from .views import (CreateRateGenerics, DetailCarGenerics, ListCarGenerics,
                    PopularCarGenerics)

urlpatterns = [
    path("cars/", ListCarGenerics.as_view()),
    path("cars/<int:pk>/", DetailCarGenerics.as_view()),
    path("rate/", CreateRateGenerics.as_view()),
    path("popular/", PopularCarGenerics.as_view()),
]


from __future__ import unicode_literals, print_function, division

import numpy as np
import torch
import random
import torch.nn as nn
import torch.nn.functional as F
from torch import optim
import time, json
from utils import timeSince, showPlot, Lang, read_lines, tensorsFromPair, tensorFromSentence, prepareData, load_glove_model

DEVICE = "cpu"

class EncoderRNN(nn.Module):
    def __init__(self, hidden_size, input_size = None, wembeddings = None):
        super(EncoderRNN, self).__init__()
        self.hidden_size = hidden_size
        if wembeddings is None:
            self.embedding = nn.Embedding(input_size, hidden_size)
        else:
            print("Embeddings loaded!")
            self.embedding = torch.nn.Embedding.from_pretrained(torch.from_numpy(wembeddings).float())
        self.gru = nn.GRU(hidden_size, hidden_size)

    def forward(self, input, hidden):
        embedded = self.embedding(input).view(1, 1, -1)
        output = embedded
        output, hidden = self.gru(output, hidden)
        return output, hidden

    def initHidden(self):
        return torch.zeros(1, 1, self.hidden_size, device=DEVICE)

class AttnDecoderRNN(nn.Module):
    def __init__(self, hidden_size, output_size, dropout_p=0.1, max_length= 15):
        super(AttnDecoderRNN, self).__init__()
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.dropout_p = dropout_p
        self.max_length = max_length

        self.embedding = nn.Embedding(self.output_size, self.hidden_size)
        self.attn = nn.Linear(self.hidden_size * 2, self.max_length)
        self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)
        self.dropout = nn.Dropout(self.dropout_p)
        self.gru = nn.GRU(self.hidden_size, self.hidden_size)
        self.out = nn.Linear(self.hidden_size, self.output_size)

    def forward(self, input, hidden, encoder_outputs):
        embedded = self.embedding(input).view(1, 1, -1)
        embedded = self.dropout(embedded)

        attn_weights = F.softmax(
            self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)
        attn_applied = torch.bmm(attn_weights.unsqueeze(0),
                                 encoder_outputs.unsqueeze(0))

        output = torch.cat((embedded[0], attn_applied[0]), 1)
        output = self.attn_combine(output).unsqueeze(0)

        output = F.relu(output)
        output, hidden = self.gru(output, hidden)

        output = F.log_softmax(self.out(output[0]), dim=1)
        return output, hidden, attn_weights

    def initHidden(self):
        return torch.zeros(1, 1, self.hidden_size, device=DEVICE)

def train_lstm(lines, name, path, iters, pretrained=True):
    hidden_size = 300
    input_lang, pairs, _ = prepareData('questions', lines)
    print(random.choice(pairs))

    words = input_lang.word2count.keys()
    glove_model = load_glove_model("/home/brenda/Documents/master/semestre3/independent_study/generate_graph_clean/output/glove.6B.300d.txt")
    words_embedding = []

    words_embedding.append(np.random.rand(300))
    words_embedding.append(np.random.rand(300))
    for w in words:
        words_embedding.append(glove_model[w])

    wembeddings = np.array(words_embedding)

    print("Embeddings shape: ", wembeddings.shape, " words: ", input_lang.n_words)
    encoder = EncoderRNN(hidden_size, wembeddings=wembeddings).to(DEVICE)
    attn_decoder = AttnDecoderRNN(hidden_size, input_lang.n_words, dropout_p=0.1).to(DEVICE)

    if pretrained:
        encoder.load_state_dict(torch.load(path + "encoder_{}".format(name)))
        attn_decoder.load_state_dict(torch.load(path + "decoder_{}".format(name)))

    trainIters(encoder, attn_decoder, pairs, input_lang, n_iters=iters, print_every=100, name=name, path_model=path)
    evaluate_randomly(encoder, attn_decoder, pairs, input_lang)

def trainIters(encoder, decoder, pairs, input_lang, n_iters, print_every=100, plot_every=100, learning_rate=0.01, name="default", path_model = None):
    start = time.time()
    plot_losses = []
    print_loss_total = 0  # Reset every print_every
    plot_loss_total = 0  # Reset every plot_every

    encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)
    decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)
    training_pairs = [tensorsFromPair(random.choice(pairs), input_lang) for i in range(n_iters)]
    criterion = nn.NLLLoss()

    for iter in range(1, n_iters + 1):
        training_pair = training_pairs[iter - 1]
        input_tensor = training_pair[0]
        target_tensor = training_pair[0]  # no 1

        loss = train(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion)

        print_loss_total += loss
        plot_loss_total += loss

        if iter % print_every == 0:
            print_loss_avg = print_loss_total / print_every
            print_loss_total = 0
            print('%s (%d %d%%) %.4f' % (timeSince(start, iter / n_iters),
                                         iter, iter / n_iters * 100, print_loss_avg))
            #print(path_model + "encoder_{}".format(name))
            torch.save(encoder.state_dict(), path_model + "encoder_{}".format(name))
            torch.save(decoder.state_dict(), path_model + "decoder_{}".format(name))

        if iter % plot_every == 0:
            plot_loss_avg = plot_loss_total / plot_every
            plot_losses.append(plot_loss_avg)
            plot_loss_total = 0

    #showPlot(plot_losses)

def train(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, max_length=15, SOS_token = 0, EOS_token = 1):
    teacher_forcing_ratio =  0.5
    encoder_hidden = encoder.initHidden()

    encoder_optimizer.zero_grad()
    decoder_optimizer.zero_grad()

    input_length = input_tensor.size(0)
    target_length = target_tensor.size(0)

    encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=DEVICE)

    loss = 0

    #print("input_length ", input_length)
    for ei in range(input_length):
        encoder_output, encoder_hidden = encoder(input_tensor[ei], encoder_hidden)
        #print("encoder_output ", encoder_output.shape)
        #print(ei)
        encoder_outputs[ei] = encoder_output[0, 0]

    decoder_input = torch.tensor([[SOS_token]], device=DEVICE)
    decoder_hidden = encoder_hidden

    use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False
    if use_teacher_forcing:
        # Teacher forcing: Feed the target as the next input
        for di in range(target_length):
            decoder_output, decoder_hidden, decoder_attention = decoder(
                decoder_input, decoder_hidden, encoder_outputs)

            loss += criterion(decoder_output, target_tensor[di])
            decoder_input = target_tensor[di]  # Teacher forcing

    else:
        # Without teacher forcing: use its own predictions as the next input
        for di in range(target_length):
            decoder_output, decoder_hidden, decoder_attention = decoder(
                decoder_input, decoder_hidden, encoder_outputs)
            topv, topi = decoder_output.topk(1)
            decoder_input = topi.squeeze().detach()  # detach from history as input

            loss += criterion(decoder_output, target_tensor[di])
            if decoder_input.item() == EOS_token:
                break

    loss.backward()

    encoder_optimizer.step()
    decoder_optimizer.step()

    return loss.item() / target_length

def test_lstm(lines, name, encoder, attn_decoder, path_model):
    input_lang, pairs, _ = prepareData('questions', lines)
    print(random.choice(pairs))

    encoder.load_state_dict(torch.load(path_model + "encoder_{}".format(name)))
    attn_decoder.load_state_dict(torch.load(path_model + "decoder_{}".format(name)))
    evaluate_randomly(encoder, attn_decoder, pairs, input_lang)

def evaluate_randomly(encoder, decoder, pairs, input_lang, n=10):
    for i in range(n):
        pair = random.choice(pairs)
        print('>', pair[0])
        print('=', pair[0])
        output_words, attentions = evaluate(encoder, decoder, pair[0], input_lang)
        output_sentence = ' '.join(output_words)
        print('<', output_sentence)
        print('')

def evaluate(encoder, decoder, sentence, input_lang, max_length=15, SOS_token = 0, EOS_token = 1):
    with torch.no_grad():
        input_tensor = tensorFromSentence(input_lang, sentence)
        input_length = input_tensor.size()[0]
        encoder_hidden = encoder.initHidden()

        encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=DEVICE)

        for ei in range(input_length):
            encoder_output, encoder_hidden = encoder(input_tensor[ei], encoder_hidden)
            encoder_outputs[ei] += encoder_output[0, 0]

        decoder_input = torch.tensor([[SOS_token]], device=DEVICE)  # SOS
        decoder_hidden = encoder_hidden

        decoded_words = []
        decoder_attentions = torch.zeros(max_length, max_length)

        for di in range(max_length):
            decoder_output, decoder_hidden, decoder_attention = decoder(decoder_input, decoder_hidden, encoder_outputs)
            decoder_attentions[di] = decoder_attention.data
            topv, topi = decoder_output.data.topk(1)
            if topi.item() == EOS_token:
                decoded_words.append('<EOS>')
                break
            else:
                decoded_words.append(input_lang.index2word[topi.item()])

            decoder_input = topi.squeeze().detach()

        return decoded_words, decoder_attentions[:di + 1]

def predict_encode(encoder, input_lang, sentence, max_length=15):
    #print("input size: ", input_lang.n_words)

    with torch.no_grad():
        input_tensor = tensorFromSentence(input_lang, sentence)
        input_length = input_tensor.size()[0]
        encoder_hidden = encoder.initHidden()

        encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=DEVICE)

        for ei in range(input_length):
            encoder_output, encoder_hidden = encoder(input_tensor[ei], encoder_hidden)
            encoder_outputs[ei] += encoder_output[0, 0]
    return encoder_hidden, encoder_outputs

def decode_question(decoder, decoder_input, input_lang, decoder_hidden, encoder_outputs):
    with torch.no_grad():
        EOS_token = 1
        decoder_output, decoder_hidden, decoder_attention = decoder(decoder_input, decoder_hidden, encoder_outputs)
        topv, topi = decoder_output.data.topk(1)
        if topi.item() == EOS_token:
            decoded_words = '<EOS>'
        else:
            decoded_words = input_lang.index2word[topi.item()]

        decoder_input = topi.squeeze().detach()

        return decoded_words, decoder_input, decoder_hidden


# Part 1 and Part 2
# Zheng Lin Lei

from pathlib import Path
import time

SCRIPT_DIR = Path(__file__).parent
INPUT_FILE = Path(SCRIPT_DIR, "../input.txt")

def main():
    with open(INPUT_FILE, mode="rt") as f:
        elf_meals = f.read().split("\n\n") # split on empty lines

    elf_calories = [] # store total calories for each elf
    for elf in elf_meals:
        calories = sum(map(int, elf.splitlines()))
        elf_calories.append(calories)

    print(f"Part 1: {max(elf_calories)}")

    elf_calories = sorted(elf_calories)
    print(f"Part 2: {elf_calories[-3:]}")
    print(f"Part 2: {sum(elf_calories[-3:])}")

if __name__ == "__main__":
    t1 = time.perf_counter()
    main()
    t2 = time.perf_counter()
    print(f"Execution time: {t2 - t1:0.4f} seconds")


import requests
import json
from stream import Stream

game_to_id = {}

def prepare_game_id_cache(games):
    for game in games:
        if game in game_to_id:
            continue

        payload = {
            'q': game,
            'liveonly': True
        }

        r = requests.get('https://www.hitbox.tv/api/games', params=payload)
        if r.status_code == 200:
            data = r.json()
            for category in data.get('categories', {}):
                if category.get('category_name', '').lower() == game.lower():
                    game_to_id[game] = category['category_id']
                    break

def get_streams(games):
    prepare_game_id_cache(games)
    streams = []
    for game in games:
        game_id = game_to_id.get(game)
        if game_id is None:
            continue

        params = {
            'game': game_id,
            'liveonly': True,
            'showHidden': False,
            'publicOnly': True
        }
        r = requests.get('https://www.hitbox.tv/api/media/live/list', params=params)
        print('The hitbox API returned {} for {}'.format(r.status_code, game))
        if r.status_code == 200:
            data = r.json()
            for stream in data.get('livestream', []):
                kwargs = {
                    'url': stream.get('channel', {}).get('channel_link', ''),
                    'viewers': int(stream.get('media_views', 0)),
                    'display_name': stream.get('media_user_name', ''),
                    'game': game,
                    'status': stream.get('media_status', '')
                }

                streams.append(Stream(**kwargs))

    return streams


from typing import Dict, List, Optional, Union

from azure.core.exceptions import HttpResponseError
import msrest.serialization

from ._automanage_client_enums import *

class Resource(msrest.serialization.Model):
    """Resource.

    Variables are only populated by the server, and will be ignored when sending a request.

    :ivar id: Fully qualified resource Id for the resource. Ex -
     /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}.
    :vartype id: str
    :ivar name: The name of the resource.
    :vartype name: str
    :ivar type: The type of the resource. Ex- Microsoft.Compute/virtualMachines or
     Microsoft.Storage/storageAccounts.
    :vartype type: str
    """

    _validation = {
        'id': {'readonly': True},
        'name': {'readonly': True},
        'type': {'readonly': True},
    }

    _attribute_map = {
        'id': {'key': 'id', 'type': 'str'},
        'name': {'key': 'name', 'type': 'str'},
        'type': {'key': 'type', 'type': 'str'},
    }

    def __init__(
        self,
        **kwargs
    ):
        super(Resource, self).__init__(**kwargs)
        self.id = None
        self.name = None
        self.type = None

class TrackedResource(Resource):
    """The resource model definition for a ARM tracked top level resource.

    Variables are only populated by the server, and will be ignored when sending a request.

    All required parameters must be populated in order to send to Azure.

    :ivar id: Fully qualified resource Id for the resource. Ex -
     /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}.
    :vartype id: str
    :ivar name: The name of the resource.
    :vartype name: str
    :ivar type: The type of the resource. Ex- Microsoft.Compute/virtualMachines or
     Microsoft.Storage/storageAccounts.
    :vartype type: str
    :param tags: A set of tags. Resource tags.
    :type tags: dict[str, str]
    :param location: Required. The geo-location where the resource lives.
    :type location: str
    """

    _validation = {
        'id': {'readonly': True},
        'name': {'readonly': True},
        'type': {'readonly': True},
        'location': {'required': True},
    }

    _attribute_map = {
        'id': {'key': 'id', 'type': 'str'},
        'name': {'key': 'name', 'type': 'str'},
        'type': {'key': 'type', 'type': 'str'},
        'tags': {'key': 'tags', 'type': '{str}'},
        'location': {'key': 'location', 'type': 'str'},
    }

    def __init__(
        self,
        *,
        location: str,
        tags: Optional[Dict[str, str]] = None,
        **kwargs
    ):
        super(TrackedResource, self).__init__(**kwargs)
        self.tags = tags
        self.location = location

class Account(TrackedResource):
    """Definition of the Automanage account.

    Variables are only populated by the server, and will be ignored when sending a request.

    All required parameters must be populated in order to send to Azure.

    :ivar id: Fully qualified resource Id for the resource. Ex -
     /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}.
    :vartype id: str
    :ivar name: The name of the resource.
    :vartype name: str
    :ivar type: The type of the resource. Ex- Microsoft.Compute/virtualMachines or
     Microsoft.Storage/storageAccounts.
    :vartype type: str
    :param tags: A set of tags. Resource tags.
    :type tags: dict[str, str]
    :param location: Required. The geo-location where the resource lives.
    :type location: str
    :param identity: The identity of the Automanage account.
    :type identity: ~automanage_client.models.AccountIdentity
    """

    _validation = {
        'id': {'readonly': True},
        'name': {'readonly': True},
        'type': {'readonly': True},
        'location': {'required': True},
    }

    _attribute_map = {
        'id': {'key': 'id', 'type': 'str'},
        'name': {'key': 'name', 'type': 'str'},
        'type': {'key': 'type', 'type': 'str'},
        'tags': {'key': 'tags', 'type': '{str}'},
        'location': {'key': 'location', 'type': 'str'},
        'identity': {'key': 'identity', 'type': 'AccountIdentity'},
    }

    def __init__(
        self,
        *,
        location: str,
        tags: Optional[Dict[str, str]] = None,
        identity: Optional["AccountIdentity"] = None,
        **kwargs
    ):
        super(Account, self).__init__(tags=tags, location=location, **kwargs)
        self.identity = identity

class AccountIdentity(msrest.serialization.Model):
    """Identity for the Automanage account.

    Variables are only populated by the server, and will be ignored when sending a request.

    :ivar principal_id: The principal id of Automanage account identity.
    :vartype principal_id: str
    :ivar tenant_id: The tenant id associated with the Automanage account.
    :vartype tenant_id: str
    :param type: The type of identity used for the Automanage account. Currently, the only
     supported type is 'SystemAssigned', which implicitly creates an identity. Possible values
     include: "SystemAssigned", "None".
    :type type: str or ~automanage_client.models.ResourceIdentityType
    """

    _validation = {
        'principal_id': {'readonly': True},
        'tenant_id': {'readonly': True},
    }

    _attribute_map = {
        'principal_id': {'key': 'principalId', 'type': 'str'},
        'tenant_id': {'key': 'tenantId', 'type': 'str'},
        'type': {'key': 'type', 'type': 'str'},
    }

    def __init__(
        self,
        *,
        type: Optional[Union[str, "ResourceIdentityType"]] = None,
        **kwargs
    ):
        super(AccountIdentity, self).__init__(**kwargs)
        self.principal_id = None
        self.tenant_id = None
        self.type = type

class AccountList(msrest.serialization.Model):
    """The response of the list Account operation.

    :param value: Result of the list Account operation.
    :type value: list[~automanage_client.models.Account]
    """

    _attribute_map = {
        'value': {'key': 'value', 'type': '[Account]'},
    }

    def __init__(
        self,
        *,
        value: Optional[List["Account"]] = None,
        **kwargs
    ):
        super(AccountList, self).__init__(**kwargs)
        self.value = value

class UpdateResource(msrest.serialization.Model):
    """Represents an update resource.

    :param tags: A set of tags. The tags of the resource.
    :type tags: dict[str, str]
    """

    _attribute_map = {
        'tags': {'key': 'tags', 'type': '{str}'},
    }

    def __init__(
        self,
        *,
        tags: Optional[Dict[str, str]] = None,
        **kwargs
    ):
        super(UpdateResource, self).__init__(**kwargs)
        self.tags = tags

class AccountUpdate(UpdateResource):
    """Definition of the Automanage account.

    :param tags: A set of tags. The tags of the resource.
    :type tags: dict[str, str]
    :param identity: The identity of the Automanage account.
    :type identity: ~automanage_client.models.AccountIdentity
    """

    _attribute_map = {
        'tags': {'key': 'tags', 'type': '{str}'},
        'identity': {'key': 'identity', 'type': 'AccountIdentity'},
    }

    def __init__(
        self,
        *,
        tags: Optional[Dict[str, str]] = None,
        identity: Optional["AccountIdentity"] = None,
        **kwargs
    ):
        super(AccountUpdate, self).__init__(tags=tags, **kwargs)
        self.identity = identity

class ConfigurationProfileAssignment(Resource):
    """Configuration profile assignment is an association between a VM and automanage profile configuration.

    Variables are only populated by the server, and will be ignored when sending a request.

    :ivar id: Fully qualified resource Id for the resource. Ex -
     /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}.
    :vartype id: str
    :ivar name: The name of the resource.
    :vartype name: str
    :ivar type: The type of the resource. Ex- Microsoft.Compute/virtualMachines or
     Microsoft.Storage/storageAccounts.
    :vartype type: str
    :param properties: Properties of the configuration profile assignment.
    :type properties: ~automanage_client.models.ConfigurationProfileAssignmentProperties
    """

    _validation = {
        'id': {'readonly': True},
        'name': {'readonly': True},
        'type': {'readonly': True},
    }

    _attribute_map = {
        'id': {'key': 'id', 'type': 'str'},
        'name': {'key': 'name', 'type': 'str'},
        'type': {'key': 'type', 'type': 'str'},
        'properties': {'key': 'properties', 'type': 'ConfigurationProfileAssignmentProperties'},
    }

    def __init__(
        self,
        *,
        properties: Optional["ConfigurationProfileAssignmentProperties"] = None,
        **kwargs
    ):
        super(ConfigurationProfileAssignment, self).__init__(**kwargs)
        self.properties = properties

class ConfigurationProfileAssignmentCompliance(msrest.serialization.Model):
    """The compliance status for the configuration profile assignment.

    Variables are only populated by the server, and will be ignored when sending a request.

    :ivar update_status: The state of compliance, which only appears in the response. Possible
     values include: "Succeeded", "Failed", "Created".
    :vartype update_status: str or ~automanage_client.models.UpdateStatus
    """

    _validation = {
        'update_status': {'readonly': True},
    }

    _attribute_map = {
        'update_status': {'key': 'updateStatus', 'type': 'str'},
    }

    def __init__(
        self,
        **kwargs
    ):
        super(ConfigurationProfileAssignmentCompliance, self).__init__(**kwargs)
        self.update_status = None

class ConfigurationProfileAssignmentList(msrest.serialization.Model):
    """The response of the list configuration profile assignment operation.

    :param value: Result of the list configuration profile assignment operation.
    :type value: list[~automanage_client.models.ConfigurationProfileAssignment]
    """

    _attribute_map = {
        'value': {'key': 'value', 'type': '[ConfigurationProfileAssignment]'},
    }

    def __init__(
        self,
        *,
        value: Optional[List["ConfigurationProfileAssignment"]] = None,
        **kwargs
    ):
        super(ConfigurationProfileAssignmentList, self).__init__(**kwargs)
        self.value = value

class ConfigurationProfileAssignmentProperties(msrest.serialization.Model):
    """Automanage configuration profile assignment properties.

    Variables are only populated by the server, and will be ignored when sending a request.

    :param configuration_profile: A value indicating configuration profile. Possible values
     include: "Azure virtual machine best practices – Dev/Test", "Azure virtual machine best
     practices – Production".
    :type configuration_profile: str or ~automanage_client.models.ConfigurationProfile
    :param target_id: The target VM resource URI.
    :type target_id: str
    :param account_id: The Automanage account ARM Resource URI.
    :type account_id: str
    :param configuration_profile_preference_id: The configuration profile custom preferences ARM
     resource URI.
    :type configuration_profile_preference_id: str
    :ivar provisioning_status: The state of onboarding, which only appears in the response.
     Possible values include: "Succeeded", "Failed", "Created".
    :vartype provisioning_status: str or ~automanage_client.models.ProvisioningStatus
    :param compliance: The configuration setting for the configuration profile.
    :type compliance: ~automanage_client.models.ConfigurationProfileAssignmentCompliance
    """

    _validation = {
        'provisioning_status': {'readonly': True},
    }

    _attribute_map = {
        'configuration_profile': {'key': 'configurationProfile', 'type': 'str'},
        'target_id': {'key': 'targetId', 'type': 'str'},
        'account_id': {'key': 'accountId', 'type': 'str'},
        'configuration_profile_preference_id': {'key': 'configurationProfilePreferenceId', 'type': 'str'},
        'provisioning_status': {'key': 'provisioningStatus', 'type': 'str'},
        'compliance': {'key': 'compliance', 'type': 'ConfigurationProfileAssignmentCompliance'},
    }

    def __init__(
        self,
        *,
        configuration_profile: Optional[Union[str, "ConfigurationProfile"]] = None,
        target_id: Optional[str] = None,
        account_id: Optional[str] = None,
        configuration_profile_preference_id: Optional[str] = None,
        compliance: Optional["ConfigurationProfileAssignmentCompliance"] = None,
        **kwargs
    ):
        super(ConfigurationProfileAssignmentProperties, self).__init__(**kwargs)
        self.configuration_profile = configuration_profile
        self.target_id = target_id
        self.account_id = account_id
        self.configuration_profile_preference_id = configuration_profile_preference_id
        self.provisioning_status = None
        self.compliance = compliance

class ConfigurationProfilePreference(TrackedResource):
    """Definition of the configuration profile preference.

    Variables are only populated by the server, and will be ignored when sending a request.

    All required parameters must be populated in order to send to Azure.

    :ivar id: Fully qualified resource Id for the resource. Ex -
     /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}.
    :vartype id: str
    :ivar name: The name of the resource.
    :vartype name: str
    :ivar type: The type of the resource. Ex- Microsoft.Compute/virtualMachines or
     Microsoft.Storage/storageAccounts.
    :vartype type: str
    :param tags: A set of tags. Resource tags.
    :type tags: dict[str, str]
    :param location: Required. The geo-location where the resource lives.
    :type location: str
    :param properties: Properties of the configuration profile preference.
    :type properties: ~automanage_client.models.ConfigurationProfilePreferenceProperties
    """

    _validation = {
        'id': {'readonly': True},
        'name': {'readonly': True},
        'type': {'readonly': True},
        'location': {'required': True},
    }

    _attribute_map = {
        'id': {'key': 'id', 'type': 'str'},
        'name': {'key': 'name', 'type': 'str'},
        'type': {'key': 'type', 'type': 'str'},
        'tags': {'key': 'tags', 'type': '{str}'},
        'location': {'key': 'location', 'type': 'str'},
        'properties': {'key': 'properties', 'type': 'ConfigurationProfilePreferenceProperties'},
    }

    def __init__(
        self,
        *,
        location: str,
        tags: Optional[Dict[str, str]] = None,
        properties: Optional["ConfigurationProfilePreferenceProperties"] = None,
        **kwargs
    ):
        super(ConfigurationProfilePreference, self).__init__(tags=tags, location=location, **kwargs)
        self.properties = properties

class ConfigurationProfilePreferenceAntiMalware(msrest.serialization.Model):
    """Automanage configuration profile Antimalware preferences.

    :param enable_real_time_protection: Enables or disables Real Time Protection. Possible values
     include: "True", "False".
    :type enable_real_time_protection: str or ~automanage_client.models.EnableRealTimeProtection
    :param exclusions: Extensions, Paths and Processes that must be excluded from scan.
    :type exclusions: object
    :param run_scheduled_scan: Enables or disables a periodic scan for antimalware. Possible values
     include: "True", "False".
    :type run_scheduled_scan: str or ~automanage_client.models.RunScheduledScan
    :param scan_type: Type of scheduled scan. Possible values include: "Quick", "Full".
    :type scan_type: str or ~automanage_client.models.ScanType
    :param scan_day: Schedule scan settings day.
    :type scan_day: str
    :param scan_time_in_minutes: Schedule scan settings time.
    :type scan_time_in_minutes: str
    """

    _attribute_map = {
        'enable_real_time_protection': {'key': 'enableRealTimeProtection', 'type': 'str'},
        'exclusions': {'key': 'exclusions', 'type': 'object'},
        'run_scheduled_scan': {'key': 'runScheduledScan', 'type': 'str'},
        'scan_type': {'key': 'scanType', 'type': 'str'},
        'scan_day': {'key': 'scanDay', 'type': 'str'},
        'scan_time_in_minutes': {'key': 'scanTimeInMinutes', 'type': 'str'},
    }

    def __init__(
        self,
        *,
        enable_real_time_protection: Optional[Union[str, "EnableRealTimeProtection"]] = None,
        exclusions: Optional[object] = None,
        run_scheduled_scan: Optional[Union[str, "RunScheduledScan"]] = None,
        scan_type: Optional[Union[str, "ScanType"]] = None,
        scan_day: Optional[str] = None,
        scan_time_in_minutes: Optional[str] = None,
        **kwargs
    ):
        super(ConfigurationProfilePreferenceAntiMalware, self).__init__(**kwargs)
        self.enable_real_time_protection = enable_real_time_protection
        self.exclusions = exclusions
        self.run_scheduled_scan = run_scheduled_scan
        self.scan_type = scan_type
        self.scan_day = scan_day
        self.scan_time_in_minutes = scan_time_in_minutes

class ConfigurationProfilePreferenceList(msrest.serialization.Model):
    """The response of the list ConfigurationProfilePreference operation.

    :param value: Result of the list ConfigurationProfilePreference operation.
    :type value: list[~automanage_client.models.ConfigurationProfilePreference]
    """

    _attribute_map = {
        'value': {'key': 'value', 'type': '[ConfigurationProfilePreference]'},
    }

    def __init__(
        self,
        *,
        value: Optional[List["ConfigurationProfilePreference"]] = None,
        **kwargs
    ):
        super(ConfigurationProfilePreferenceList, self).__init__(**kwargs)
        self.value = value

class ConfigurationProfilePreferenceProperties(msrest.serialization.Model):
    """Automanage configuration profile preference properties.

    :param vm_backup: The custom preferences for Azure VM Backup.
    :type vm_backup: ~automanage_client.models.ConfigurationProfilePreferenceVmBackup
    :param anti_malware: The custom preferences for Azure Antimalware.
    :type anti_malware: ~automanage_client.models.ConfigurationProfilePreferenceAntiMalware
    """

    _attribute_map = {
        'vm_backup': {'key': 'vmBackup', 'type': 'ConfigurationProfilePreferenceVmBackup'},
        'anti_malware': {'key': 'antiMalware', 'type': 'ConfigurationProfilePreferenceAntiMalware'},
    }

    def __init__(
        self,
        *,
        vm_backup: Optional["ConfigurationProfilePreferenceVmBackup"] = None,
        anti_malware: Optional["ConfigurationProfilePreferenceAntiMalware"] = None,
        **kwargs
    ):
        super(ConfigurationProfilePreferenceProperties, self).__init__(**kwargs)
        self.vm_backup = vm_backup
        self.anti_malware = anti_malware

class ConfigurationProfilePreferenceUpdate(UpdateResource):
    """Definition of the configuration profile preference.

    :param tags: A set of tags. The tags of the resource.
    :type tags: dict[str, str]
    :param properties: Properties of the configuration profile preference.
    :type properties: ~automanage_client.models.ConfigurationProfilePreferenceProperties
    """

    _attribute_map = {
        'tags': {'key': 'tags', 'type': '{str}'},
        'properties': {'key': 'properties', 'type': 'ConfigurationProfilePreferenceProperties'},
    }

    def __init__(
        self,
        *,
        tags: Optional[Dict[str, str]] = None,
        properties: Optional["ConfigurationProfilePreferenceProperties"] = None,
        **kwargs
    ):
        super(ConfigurationProfilePreferenceUpdate, self).__init__(tags=tags, **kwargs)
        self.properties = properties

class ConfigurationProfilePreferenceVmBackup(msrest.serialization.Model):
    """Automanage configuration profile VM Backup preferences.

    :param time_zone: TimeZone optional input as string. For example: Pacific Standard Time.
    :type time_zone: str
    :param instant_rp_retention_range_in_days: Instant RP retention policy range in days.
    :type instant_rp_retention_range_in_days: int
    :param retention_policy: Retention policy with the details on backup copy retention ranges.
    :type retention_policy: str
    :param schedule_policy: Backup schedule specified as part of backup policy.
    :type schedule_policy: str
    """

    _attribute_map = {
        'time_zone': {'key': 'timeZone', 'type': 'str'},
        'instant_rp_retention_range_in_days': {'key': 'instantRpRetentionRangeInDays', 'type': 'int'},
        'retention_policy': {'key': 'retentionPolicy', 'type': 'str'},
        'schedule_policy': {'key': 'schedulePolicy', 'type': 'str'},
    }

    def __init__(
        self,
        *,
        time_zone: Optional[str] = None,
        instant_rp_retention_range_in_days: Optional[int] = None,
        retention_policy: Optional[str] = None,
        schedule_policy: Optional[str] = None,
        **kwargs
    ):
        super(ConfigurationProfilePreferenceVmBackup, self).__init__(**kwargs)
        self.time_zone = time_zone
        self.instant_rp_retention_range_in_days = instant_rp_retention_range_in_days
        self.retention_policy = retention_policy
        self.schedule_policy = schedule_policy

class ErrorAdditionalInfo(msrest.serialization.Model):
    """The resource management error additional info.

    Variables are only populated by the server, and will be ignored when sending a request.

    :ivar type: The additional info type.
    :vartype type: str
    :ivar info: The additional info.
    :vartype info: object
    """

    _validation = {
        'type': {'readonly': True},
        'info': {'readonly': True},
    }

    _attribute_map = {
        'type': {'key': 'type', 'type': 'str'},
        'info': {'key': 'info', 'type': 'object'},
    }

    def __init__(
        self,
        **kwargs
    ):
        super(ErrorAdditionalInfo, self).__init__(**kwargs)
        self.type = None
        self.info = None

class ErrorResponse(msrest.serialization.Model):
    """The resource management error response.

    :param error: The error object.
    :type error: ~automanage_client.models.ErrorResponseError
    """

    _attribute_map = {
        'error': {'key': 'error', 'type': 'ErrorResponseError'},
    }

    def __init__(
        self,
        *,
        error: Optional["ErrorResponseError"] = None,
        **kwargs
    ):
        super(ErrorResponse, self).__init__(**kwargs)
        self.error = error

class ErrorResponseError(msrest.serialization.Model):
    """The error object.

    Variables are only populated by the server, and will be ignored when sending a request.

    :ivar code: The error code.
    :vartype code: str
    :ivar message: The error message.
    :vartype message: str
    :ivar target: The error target.
    :vartype target: str
    :ivar details: The error details.
    :vartype details: list[~automanage_client.models.ErrorResponse]
    :ivar additional_info: The error additional info.
    :vartype additional_info: list[~automanage_client.models.ErrorAdditionalInfo]
    """

    _validation = {
        'code': {'readonly': True},
        'message': {'readonly': True},
        'target': {'readonly': True},
        'details': {'readonly': True},
        'additional_info': {'readonly': True},
    }

    _attribute_map = {
        'code': {'key': 'code', 'type': 'str'},
        'message': {'key': 'message', 'type': 'str'},
        'target': {'key': 'target', 'type': 'str'},
        'details': {'key': 'details', 'type': '[ErrorResponse]'},
        'additional_info': {'key': 'additionalInfo', 'type': '[ErrorAdditionalInfo]'},
    }

    def __init__(
        self,
        **kwargs
    ):
        super(ErrorResponseError, self).__init__(**kwargs)
        self.code = None
        self.message = None
        self.target = None
        self.details = None
        self.additional_info = None

class Operation(msrest.serialization.Model):
    """Automanage REST API operation.

    :param name: Operation name: For ex.
     providers/Microsoft.Automanage/configurationProfileAssignments/write or read.
    :type name: str
    :param is_data_action: Indicates whether the operation is a data action.
    :type is_data_action: str
    :param display: Provider, Resource, Operation and description values.
    :type display: ~automanage_client.models.OperationDisplay
    :param status_code: Service provider: Microsoft.Automanage.
    :type status_code: str
    """

    _attribute_map = {
        'name': {'key': 'name', 'type': 'str'},
        'is_data_action': {'key': 'isDataAction', 'type': 'str'},
        'display': {'key': 'display', 'type': 'OperationDisplay'},
        'status_code': {'key': 'properties.statusCode', 'type': 'str'},
    }

    def __init__(
        self,
        *,
        name: Optional[str] = None,
        is_data_action: Optional[str] = None,
        display: Optional["OperationDisplay"] = None,
        status_code: Optional[str] = None,
        **kwargs
    ):
        super(Operation, self).__init__(**kwargs)
        self.name = name
        self.is_data_action = is_data_action
        self.display = display
        self.status_code = status_code

class OperationDisplay(msrest.serialization.Model):
    """Provider, Resource, Operation and description values.

    :param provider: Service provider: Microsoft.Automanage.
    :type provider: str
    :param resource: Resource on which the operation is performed:  For ex.
    :type resource: str
    :param operation: Operation type: Read, write, delete, etc.
    :type operation: str
    :param description: Description about operation.
    :type description: str
    """

    _attribute_map = {
        'provider': {'key': 'provider', 'type': 'str'},
        'resource': {'key': 'resource', 'type': 'str'},
        'operation': {'key': 'operation', 'type': 'str'},
        'description': {'key': 'description', 'type': 'str'},
    }

    def __init__(
        self,
        *,
        provider: Optional[str] = None,
        resource: Optional[str] = None,
        operation: Optional[str] = None,
        description: Optional[str] = None,
        **kwargs
    ):
        super(OperationDisplay, self).__init__(**kwargs)
        self.provider = provider
        self.resource = resource
        self.operation = operation
        self.description = description

class OperationList(msrest.serialization.Model):
    """The response model for the list of Automanage operations.

    :param value: List of Automanage operations supported by the Automanage resource provider.
    :type value: list[~automanage_client.models.Operation]
    """

    _attribute_map = {
        'value': {'key': 'value', 'type': '[Operation]'},
    }

    def __init__(
        self,
        *,
        value: Optional[List["Operation"]] = None,
        **kwargs
    ):
        super(OperationList, self).__init__(**kwargs)
        self.value = value

class ProxyResource(Resource):
    """The resource model definition for a ARM proxy resource. It will have everything other than required location and tags.

    Variables are only populated by the server, and will be ignored when sending a request.

    :ivar id: Fully qualified resource Id for the resource. Ex -
     /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}.
    :vartype id: str
    :ivar name: The name of the resource.
    :vartype name: str
    :ivar type: The type of the resource. Ex- Microsoft.Compute/virtualMachines or
     Microsoft.Storage/storageAccounts.
    :vartype type: str
    """

    _validation = {
        'id': {'readonly': True},
        'name': {'readonly': True},
        'type': {'readonly': True},
    }

    _attribute_map = {
        'id': {'key': 'id', 'type': 'str'},
        'name': {'key': 'name', 'type': 'str'},
        'type': {'key': 'type', 'type': 'str'},
    }

    def __init__(
        self,
        **kwargs
    ):
        super(ProxyResource, self).__init__(**kwargs)


import os

import fitz
import openpyxl

import config_module
import exit_module
import global_data_module

SOURCE_EXCEL = 'sourceExcel'
SOURCE_EXCEL_FILE_NAME_COLUMN = 'sourceExcelFileNameColumn'
SOURCE_EXCEL_START_ROW = 'sourceExcelStartRow'
SOURCE_PDF = 'sourcePdf'
IMAGE_SCALE = 'scale'
OUTPUT_DIR = 'outputDir'

def read_asin_data(excel_path: str) -> []:
    workbook = openpyxl.load_workbook(excel_path)
    row_index = 0
    name_list = []
    for row in workbook.worksheets[0].rows:
        if row[0].value is not None and row_index >= config_module.get_config_obj()[SOURCE_EXCEL_START_ROW]:
            if row[0].value.strip() != '':
                name_list.append(row[config_module.get_config_obj()[SOURCE_EXCEL_FILE_NAME_COLUMN]].value)
        row_index = row_index + 1
    return name_list

print('正在初始化程序数据...')
global_data_module.init()
print('正在解析配置文件并读取Excel...')
# 读取Excel
file_names = read_asin_data(config_module.get_config_obj()[SOURCE_EXCEL])
print('正在解码PDF...')
# 读取PDF
pdf_doc = fitz.open(config_module.get_config_obj()[SOURCE_PDF])
print('PDF页数：' + str(pdf_doc.pageCount) + ', EXCEL 记录数：' + str(len(file_names)))
if pdf_doc.pageCount != len(file_names):
    exit_module.tip_and_wait_then_exit('Excel的记录数量与PDF页数不匹配，无法继续操作')
mat = fitz.Matrix(config_module.get_config_obj()[IMAGE_SCALE], config_module.get_config_obj()[IMAGE_SCALE])
if not os.path.exists(config_module.get_config_obj()[OUTPUT_DIR]):
    os.makedirs(config_module.get_config_obj()[OUTPUT_DIR])
for page_index in range(pdf_doc.pageCount):
    pdf_page = pdf_doc[page_index]
    print('开始转换写出第 ' + str(page_index + 1) + ' / ' + str(len(file_names)) + ' 页图片数据')
    pdf_page.getPixmap(matrix=mat).writePNG(
        config_module.get_config_obj()[OUTPUT_DIR] + os.sep + file_names[page_index] + '.png')

print('任务执行完毕')


# from the command line, run with
# mpiexec -n 4 python -m mpi4py mpitest.py

from mpi4py import MPI
import sys
import numpy as np

size = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
name = MPI.Get_processor_name()

#sys.stdout.write(
#    "Hello, World! I am process %d of %d on %s.\n" % (rank, size, name))

# geneate a list of n random numbers between 0 and 1
n = 2000
localArray = np.random.random(n)

# sort the local array
localArray.sort()

# find the start and end of the array based on the range of 0 to 1
# and the number of processor
arrayBoundaries = []

# loop over the processors
bufferSize = n/2

arecv = np.zeros([size, int(bufferSize)], dtype='f')
for i in range(0, size):
    #  find the elements with in range
    xmin = float(i)/size
    xmax = float(i+1)/size
    asend = localArray[localArray >= xmin]
    asend = asend[asend < xmax]

    # copy this data into the send buffer
    sendbuf = np.zeros(int(bufferSize), dtype='f')
    sendbuf[0:len(asend)] = asend
    MPI.COMM_WORLD.Gather(sendbuf, arecv, root=i )

# compress the data into a local array from the gather
# ignore the zeros
newLocal = []
for i in range(0, size):
    for j in range(0, int(bufferSize)):
        if arecv[i,j] != 0:
            newLocal.append(arecv[i,j])

newLocal.sort()
#print("rank = ", rank, "local size = ", len(newLocal), "localMin = ", newLocal[0], "localMax = ", newLocal[-1])
localData = np.array( [len(newLocal), newLocal[0], newLocal[-1]] , dtype='f')
globalData = np.zeros([size, 3], dtype='f')

MPI.COMM_WORLD.Gather(localData, globalData, root=0)
if rank == 0:
    #print(globalData)
    for i in range(0, size):
        print("node = ", i, " node data size = ", globalData[i,0], "node Min = ", globalData[i,1], "node Max = ", globalData[i,2])

MPI.COMM_WORLD.Barrier()


from sched import scheduler
import dgl
import tqdm
import dataset
import numpy as np
import pandas as pd
from config import *
import torchtuples as tt
from datetime import datetime
from pycox.evaluation import EvalSurv
from torchtuples.callbacks import Callback
from sksurv.ensemble import RandomSurvivalForest
from sksurv.metrics import concordance_index_censored
from sksurv.linear_model import CoxPHSurvivalAnalysis
from pycox.models import LogisticHazard, PMF, CoxPH, DeepHitSingle
from pycox.preprocessing.label_transforms import LabTransDiscreteTime
from models import MLP, CoxModel, NonCoxModel, EarlyStop, GNN, MVP, DeepSurv, Nnetsurvival

def eval_surv(surv_df, y_df):
    time_grid = np.linspace(y_df.time.min(), y_df.time.max(), IBS_GRID_N)
    ev = EvalSurv(surv_df, y_df.time.values, y_df.event.values, censor_surv='km')
    Ctd = ev.concordance_td()
    IBS = ev.integrated_brier_score(time_grid)
    return Ctd, IBS

def sksurv(train, test, method_name, graph=None, print=tqdm.tqdm.write):
    start_time = datetime.now()
    label_type = [('cens', '?'), ('time', '<f8')]
    y_train = np.array([(bool(i), d) for i, d in train.iloc[:,:2].to_numpy()], dtype=label_type)
    y_test = np.array([(bool(i), d) for i, d in test.iloc[:,:2].to_numpy()], dtype=label_type)

    X_train = train.iloc[:,2:].to_numpy()
    X_test = test.iloc[:,2:].to_numpy()

    if method_name == 'CPH':
        model = CoxPHSurvivalAnalysis(alpha=CPH_PENALTY)
    elif method_name == 'RSF':
        model = RandomSurvivalForest(
                n_estimators=RSF_N_ESTIMATORS,
                min_samples_split=10,
                min_samples_leaf=15,
                max_features="sqrt",
                n_jobs=-1,
                random_state=GLOBALSEED)

    model.fit(X_train, y_train)
    # == sksurv.metrics.concordance_index_censored == lifelines.utils.concordance_index
    # >= sksurv.metrics.concordance_index_ipcw
    train_C = model.score(X_train, y_train)
    test_C = model.score(X_test, y_test)
    test_pred = model.predict(X_test)

    test_surv_fn = model.predict_survival_function(X_test)
    test_surv_df = pd.DataFrame(np.stack([fn.y for fn in test_surv_fn],axis=1), index=test_surv_fn[0].x)
    test_Ctd, test_IBS = eval_surv(test_surv_df, test)

    print(f'[{method_name}] [train] C: {train_C:.4f} | [test] C: {test_C:.4f}, Ctd: {test_Ctd:.4f}, IBS: {test_IBS:.4f} | time: {datetime.now() - start_time}')
    return train_C, test_C, test_Ctd, test_IBS, test_pred

class LRScheduler(Callback):
    def __init__(self, lr, lr_decay=0.01):
        self.epoch = 0
        self.lr = lr
        self.lr_decay = lr_decay
        self.scheduler = scheduler

    def on_epoch_end(self):
        for p in self.model.optimizer.param_groups:
            p['lr'] = self.lr / (1 + self.epoch * self.lr_decay)

        self.epoch += 1

def pycox(train, test, method_name, graph=None, lr=LR, net_param=None,
          training=True,
          save_model=None,
          return_hg=False,
          return_fusion=False,
          print=tqdm.tqdm.write):
    if method_name == 'Nnet-survival':
        method = LogisticHazard
    elif method_name == 'PMF':
        method = PMF
    elif method_name == 'DeepSurv':
        method = CoxPH
    elif method_name == 'DeepHit':
        method = DeepHitSingle
    elif method_name.endswith('Multi'):
        method = DeepHitSingle
    else:
        method = CoxPH

    start_time = datetime.now()
    valid = train.sample(frac=0.1)
    train = train.drop(valid.index)

    x_train = train.iloc[:,2:].values.astype(np.float32)
    y_train = (train.time.values.astype(np.float32), train.event.values.astype(np.float32))
    x_val = valid.iloc[:,2:].values.astype(np.float32)
    y_val = (valid.time.values.astype(np.float32), valid.event.values.astype(np.float32))
    x_test = test.iloc[:,2:].values.astype(np.float32)

    params = {}
    if method is not CoxPH:
        labtrans = LogisticHazard.label_transform(INTERVALS)
        y_train = labtrans.fit_transform(*y_train)
        y_val = labtrans.transform(*y_val)
        params = {
            'duration_index': labtrans.cuts
        }

    inp_dim = x_train.shape[1]
    L2_reg = 0

    callbacks = [tt.cb.EarlyStopping()]

    if method_name[:3] in ['GCN', 'GAT', 'GIN', 'SAG']:
        if net_param:
            g_dims = [G_INP_DIM] + [net_param['g_hid_dim']] * net_param['g_hid_layers']
            c_embed_dim = net_param['c_embed_dim']
            haz_dim = net_param['haz_dim']
            dropout = net_param['dropout']
            readout = net_param['readout']
            fusion = net_param['fusion']
        else:
            g_dims = [G_INP_DIM] + [G_HID_DIM] * G_HID_LAYER
            c_embed_dim = EMB_DIM
            haz_dim = HAZ_DIM
            dropout = DROPOUT
            readout = G_READOUT
            fusion = G_FUSION
        out_dim = 1 if method is CoxPH else INTERVALS
        net = GNN(method_name[:3], inp_dim, graph.to(DEVICE), g_dims, c_embed_dim, haz_dim, out_dim, dropout, readout=readout, fusion=fusion)
    elif method_name.startswith('MicrobeSurv'):
        g_dims = [G_INP_DIM] + [G_HID_DIM] * G_HID_LAYER
        c_embed_dim = EMB_DIM
        haz_dim = HAZ_DIM
        readout = G_READOUT
        fusion = G_FUSION
        out_dim = 1 if method is CoxPH else INTERVALS
        g = graph.to(DEVICE) if graph else None
        net = MVP(method_name[3:6], inp_dim, g, g_dims, c_embed_dim, haz_dim, out_dim, DROPOUT, MVPOOL_RATIO, readout=readout, fusion=fusion)
    elif method_name == 'DeepSurv':
        # METABRIC
        # lr = 0.01
        # L2_reg = 10.891
        # lr_decay = 4.169e-3
        # hyperparams = {
        #     'batch_norm': False,
        #     'dropout': 0.16,
        #     'hidden_layers_sizes': [41],
        #     'n_in': inp_dim,
        #     'activation': 'selu'
        # }

        # SUPPORT
        lr = 0.047
        L2_reg = 8.12
        lr_decay = 2.573e-3
        hyperparams = {
            'batch_norm': False,
            'dropout': 0.255,
            'hidden_layers_sizes': [44],
            'n_in': inp_dim,
            'activation': 'selu'
        }

        callbacks += [LRScheduler(lr, lr_decay)]
        net = DeepSurv(**hyperparams)
    elif method_name == 'Nnet-survival':
        # SUPPORT
        hid_dims = [7]
        out_dim = 1 if method is CoxPH else INTERVALS
        L2_reg = 0.1

        net = Nnetsurvival(inp_dim, out_dim, hid_dims)
    else:
        inp_g = graph.num_nodes() if graph else 0
        inp_c = inp_dim - inp_g
        out_dim = 1 if method is CoxPH else INTERVALS
        net = MLP(inp_c, inp_g, EMB_DIM, G_HID_DIM, G_HID_LAYER, HAZ_DIM, out_dim, DROPOUT)

    opt = tt.optim.Adam(lr, weight_decay=L2_reg)
    model = method(net, opt, device=DEVICE, **params)

    if training:
        log = model.fit(x_train, y_train, BATCH, EPOCHS, callbacks, val_data=(x_val, y_val), verbose=False)
        log_df = log.to_pandas()
        if method is CoxPH: model.compute_baseline_hazards()
        # if save_model: model.save_net(save_model)
        if save_model:
            torch.save(model.net.state_dict(), save_model)
            if hasattr(model, 'baseline_hazards_'):
                model.baseline_hazards_.to_pickle(save_model.split('.')[0]+'_blh.pickle')
    else:
        log_df = pd.DataFrame([0], columns=['train_loss'])
        # model.load_net(save_model, map_location=DEVICE) # this will go wrong
        model.net.load_state_dict(torch.load(save_model, map_location=DEVICE))
        blh_path = save_model.split('.')[0]+'_blh.pickle'
        if os.path.isfile(blh_path):
            model.baseline_hazards_ = pd.read_pickle(blh_path)
            model.baseline_cumulative_hazards_ = model.baseline_hazards_.cumsum()

    train_C = test_C = pd.NA
    if method is CoxPH:
        train_pred = model.predict(x_train)
        train_C = concordance_index_censored(train.event.astype(bool), train.time, train_pred.squeeze())[0]
        test_pred = model.predict(x_test)
        test_C = concordance_index_censored(test.event.astype(bool), test.time, test_pred.squeeze())[0]
    else:
        test_pred = model.predict(x_test).mean(1)

    test_surv_df = model.predict_surv_df(x_test)
    test_Ctd, test_IBS = eval_surv(test_surv_df, test)

    print(f'[{method_name}] [train] epoch: {len(log_df)}, loss: {log_df.train_loss.iloc[-1]:.4f}, C: {train_C:.4f} | [test] C: {test_C:.4f}, Ctd: {test_Ctd:.4f}, IBS: {test_IBS:.4f} | time: {datetime.now() - start_time}')

    if return_hg:
        return model, model.net(torch.tensor(x_test).to(DEVICE), return_hg=return_hg).cpu()
    if return_fusion:
        return model, model.net(torch.tensor(x_test).to(DEVICE), return_fusion=return_fusion).cpu()

    return train_C, test_C, test_Ctd, test_IBS, test_pred

def gen_data_for_graph_model(x, dim, g, is_mvp=False):
    inp_c = x[:, :dim]
    inp_n = x[:, dim:]
    inp_n = inp_n.reshape(-1).unsqueeze(1)
    inp_g = dgl.batch([g] * x.shape[0])
    if is_mvp:
        edge_index = torch.stack(inp_g.edges()).to(DEVICE)
        which_graph = torch.tensor([[i] * (x.shape[1] - dim) for i in range(x.shape[0])]).flatten().to(DEVICE)
        return (inp_n, inp_c, edge_index, which_graph)
    return (inp_c, inp_n, inp_g.to(DEVICE))

def coxbased(train, test,
            method_name='MLP',
            graph=None,
            training=True,
            save_model=None,
            return_hg=False,
            verbose=False,
            print=tqdm.tqdm.write):
    start_time = datetime.now()

    inp_dim = train.shape[1] - 2
    if graph:
        inp_dim -= graph.num_nodes()
        is_mvp = method_name.startswith('MVP')

    if method_name.startswith('MLP'):
        net = MLP(inp_dim, EMB_DIM, HAZ_DIM, OUT_DIM, DROPOUT).to(DEVICE)
    elif method_name.startswith('GCN'):
        net = GNN('GCN', G_INP_DIM, G_HID_DIM, inp_dim, EMB_DIM, HAZ_DIM, OUT_DIM, DROPOUT).to(DEVICE)
    elif method_name.startswith('GAT'):
        net = GNN('GAT', G_INP_DIM, G_HID_DIM, inp_dim, EMB_DIM, HAZ_DIM, OUT_DIM, DROPOUT, num_heads=NUM_HEADS).to(DEVICE)
    elif method_name.startswith('GIN'):
        net = GNN('GIN', G_INP_DIM, G_HID_DIM, inp_dim, EMB_DIM, HAZ_DIM, OUT_DIM, DROPOUT).to(DEVICE)
    elif method_name.startswith('SAG'):
        net = GNN('SAG', G_INP_DIM, G_HID_DIM, inp_dim, EMB_DIM, HAZ_DIM, OUT_DIM, DROPOUT, pool_ratio=SAGPOOL_RATIO).to(DEVICE)
    elif method_name.startswith('MVPGCN'):
        net = MVP('GCN', G_INP_DIM, G_HID_DIM, inp_dim, EMB_DIM, HAZ_DIM, OUT_DIM, DROPOUT, MVPOOL_RATIO).to(DEVICE)
    elif method_name.startswith('MVPGINGAT'):
        net = MVP('GIN', G_INP_DIM, G_HID_DIM, inp_dim, EMB_DIM, HAZ_DIM, OUT_DIM, DROPOUT, MVPOOL_RATIO).to(DEVICE)
    elif method_name.startswith('TEST'):
        net = TEST('GCN', G_INP_DIM, G_HID_DIM, inp_dim, EMB_DIM, HAZ_DIM, OUT_DIM, DROPOUT, MVPOOL_RATIO,
                    readout='avg', fusion='dot').to(DEVICE)

    model = CoxModel(net, LR)

    df2t = lambda df: map(lambda d: torch.FloatTensor(d.to_numpy()).to(DEVICE), [df.iloc[:,2:], df.time, df.event])
    train_log = ''
    if training:
        valid = train.sample(frac=0.1)
        train = train.drop(valid.index)
        train_loader = dataset.loader(train)

        min_valid_loss = np.inf
        best_net_param = None
        scheduler = EarlyStop()

        for e in range(EPOCHS):
            losses = []
            model.net.train()
            for batch in train_loader:
                x, yt, ye = map(lambda d: d.to(DEVICE), batch)
                if graph: x = gen_data_for_graph_model(x, inp_dim, graph, is_mvp)
                loss = model.update(x, yt, ye)
                if loss is not None: losses.append(loss)

            model.net.eval()
            with torch.no_grad():
                x, yt, ye = df2t(valid)
                if graph: x = gen_data_for_graph_model(x, inp_dim, graph, is_mvp)
                loss = model.update(x, yt, ye, False)

                if loss < min_valid_loss:
                    min_valid_loss = loss
                    best_net_param = model.net.state_dict()

                if scheduler.add(loss): break

            if verbose:
                print(f'[{method_name}] epoch {e+1:2d}, loss: [train] {np.mean(losses):.4f}, [valid] {loss:.4f}, [best] {min_valid_loss:.4f}, cnt: {scheduler.cnt}')

        train_log = f'epoch: {e+1:2d}, loss: {np.mean(losses):.4f}, '
        model.net.load_state_dict(best_net_param)
        if save_model: torch.save(model.net.state_dict(), save_model)
    else:
        model.net.load_state_dict(torch.load(save_model, map_location=DEVICE))

    model.net.eval()
    with torch.no_grad():
        x_train = torch.FloatTensor(train.iloc[:,2:].to_numpy()).to(DEVICE)
        x_test = torch.FloatTensor(test.iloc[:,2:].to_numpy()).to(DEVICE)
        if graph:
            inps_train = gen_data_for_graph_model(x_train, inp_dim, graph, is_mvp)
            inps_test = gen_data_for_graph_model(x_test, inp_dim, graph, is_mvp)
        else:
            inps_train = (x_train,)
            inps_test = (x_test,)
        model.compute_haz(inps_train, train[['time','event']])

        train_pred = model.net(*inps_train).cpu()
        train_C = concordance_index_censored(train.event.astype(bool), train.time, train_pred.squeeze())[0]
        test_pred = model.net(*inps_test).cpu()
        test_C = concordance_index_censored(test.event.astype(bool), test.time, test_pred.squeeze())[0]
        test_surv_df = model.pred_surv(inps_test)
        test_Ctd, test_IBS = eval_surv(test_surv_df, test)

        print(f'[{method_name}] [train] {train_log}C: {train_C:.4f} | [test] C: {test_C:.4f}, Ctd: {test_Ctd:.4f}, IBS: {test_IBS:.4f} | time: {datetime.now() - start_time}')

        if return_hg:
            return model.net(*inps_test, return_hg=return_hg).cpu()

    return train_C, test_C, test_Ctd, test_IBS

def noncox(train, test, method_name='MLP', graph=None, save_model=None, verbose=False, print=tqdm.tqdm.write):
    start_time = datetime.now()

    valid = train.sample(frac=0.1)
    train = train.drop(valid.index)

    labtrans = LabTransDiscreteTime(INTERVALS)
    yt_train, _ = labtrans.fit_transform(train.time, train.event)
    yt_val, _ = labtrans.transform(valid.time, valid.event)
    valid.time = (yt_val - valid.event + 1).apply(lambda x: [1]*x+[0]*(INTERVALS-x))
    train_loader = dataset.loader(train, yt_train)

    # visulaze discrete time intervals
    # k=20
    # print('time:     '+' '.join(map(lambda x:f'{int(x):3d}', train.time.values[:k])))
    # print('interval: '+' '.join(map(lambda x:f'{int(x):3d}', yt_train[:k])))
    # print('event:    '+' '.join(map(lambda x:f'{int(x):3d}', train.event.values[:k])))
    # print('cuts: '+' '.join(map(lambda x:f'{x:.1f}', labtrans.cuts)))

    inp_dim = train.shape[1] - 2
    if graph:
        inp_dim -= graph.num_nodes()
        is_mvp = method_name.startswith('MVP')

    if method_name.startswith('MLP'):
        net = MLP(inp_dim, EMB_DIM, HAZ_DIM, INTERVALS, DROPOUT).to(DEVICE)
    elif method_name.startswith('GIN'):
        net = GNN('GIN', G_INP_DIM, G_HID_DIM, inp_dim, EMB_DIM, HAZ_DIM, INTERVALS, DROPOUT).to(DEVICE)
    elif method_name.startswith('MVP'):
        net = MVP('GIN', G_INP_DIM, G_HID_DIM, inp_dim, EMB_DIM, HAZ_DIM, INTERVALS, DROPOUT, MVPOOL_RATIO).to(DEVICE)
    model = NonCoxModel(net, LR, labtrans.cuts)

    df2t = lambda df: map(lambda d: torch.FloatTensor(d).to(DEVICE), [df.iloc[:,2:].to_numpy(), df.time.to_list(), df.event.to_numpy()])
    min_valid_loss = np.inf
    best_net_param = None
    scheduler = EarlyStop()

    for e in range(EPOCHS):
        losses = []
        model.net.train()
        for batch in train_loader:
            x, yt, ye = map(lambda d: d.to(DEVICE), batch)
            if graph: x = gen_data_for_graph_model(x, inp_dim, graph, is_mvp)
            loss = model.update(x, yt, ye)
            if loss is not None: losses.append(loss)

        model.net.eval()
        with torch.no_grad():
            x, yt, ye = df2t(valid)
            if graph: x = gen_data_for_graph_model(x, inp_dim, graph, is_mvp)
            loss = model.update(x, yt, ye, False)

            if loss < min_valid_loss:
                min_valid_loss = loss
                best_net_param = model.net.state_dict()

            if scheduler.add(loss): break

        if verbose:
            print(f'[{method_name}] epoch {e+1:2d}, loss: [train] {np.mean(losses):.4f}, [valid] {loss:.4f}, [best] {min_valid_loss:.4f}, cnt: {scheduler.cnt}')

    train_C = test_C = pd.NA
    model.net.load_state_dict(best_net_param)
    model.net.eval()
    with torch.no_grad():
        x_test = torch.FloatTensor(test.iloc[:,2:].to_numpy()).to(DEVICE)
        if graph:
            inps_test = gen_data_for_graph_model(x_test, inp_dim, graph, is_mvp)
        else:
            inps_test = (x_test,)

        test_surv_df = model.pred_surv(inps_test)
        test_Ctd, test_IBS = eval_surv(test_surv_df, test)

    print(f'[{method_name}] [train] epoch: {e+1:2d}, loss: {np.mean(losses):.4f}, C: {train_C:.4f} | [test] C: {test_C:.4f}, Ctd: {test_Ctd:.4f}, IBS: {test_IBS:.4f} | time: {datetime.now() - start_time}')
    if save_model: torch.save(model.net.state_dict(), save_model)

    return train_C, test_C, test_Ctd, test_IBS


import re

from django import template

register = template.Library()

@register.simple_tag(takes_context=True)
def line_break(context, value):
    if type(value) == str:
        value = re.sub(r'^\s+|\n|\s+$', '<br>', value)
    return value


from scipy import stats
import numpy as np

distributions = [stats.chi2(6), stats.chi2(7)]
n_dim = len(distributions)
num_mixtures = 15
rho = .1
n_rows = 100
p_mcar = 0.1
shift = 0
regression_parameter = 2
cov = np.array([[1, rho], [rho, 1]])
method = "shrinkage"

eps = 0.003
M = np.concatenate((np.array(20 * [50]), np.array(5 * [1000])))
n_sample = 10000


from torch.utils import data
import h5py
# X is the 2D input and Y is the classification result
# This func just append them together into a dataset.
def make_dataset(file_path, train=True):

  datasets = []
  dataset = []

  if train:
    fhd5 = h5py.File(file_path,'r')
    for gname, group in fhd5.items():
      datasets.append(group)
    idx = 0
    for things in datasets[0]:
      # dataset.append(datasets[0][idx],datasets[1][idx])
      dataset.append([idx,idx])
      idx = idx +1

  else:
    fhd5 = h5py.File(file_path,'r')
    for gname, group in fhd5.items():
      datasets.append(group)
    idx = 0
    for things in datasets[0]:
      dataset.append([idx,idx])
      # dataset.append(datasets[0][idx],datasets[1][idx])
      idx = idx +1
# #
  return dataset

class MyDataSet(data.Dataset):
  # The "train" parameter would tell which dataset to use.
  def __init__(self, file_path, transform = None, train = True):
    self.train = train
    if self.train:
      traindatasets = []
      fhd5 = h5py.File(file_path,'r')
      for gname, group in fhd5.items():
          # print(group)
        traindatasets.append(group)
      self.train_set_path = make_dataset(file_path, train)
    else:
      testdatasets = []
      fhd5 = h5py.File(file_path,'r')
      for gname, group in fhd5.items():
          # print(group)
        testdatasets.append(group)
      self.test_set_path = make_dataset(file_path, train)

  def __getitem__(self, index):
    if self.train:
      img_path, label_path = self.train_set_path[index]
      img = self.traindatasets[0][img_path]
      label = self.traindatasets[1][label_path]
      if self.transform is not None:
        img = self.transform(img)
      return img,label
    else:
      img_path, label_path = self.test_set_path[index]
      img = self.testdatasets[0][img_path]
      label = self.testdatasets[1][label_path]
      if self.transform is not None:
        img = self.transform(img)
      return img,label

  def __len__(self):
    if self.train:
      return len(self.train_set_path)

    else:
      return len(self.test_set_path)


import logging, re

from django.contrib.auth.decorators import login_required
from django.views.generic.simple import direct_to_template

from intensity.models import AssetInfo, Activity, ServerInstance
from intensity.views import getpost_form
from intensity.components.map_wizard__forms import MapWizardForm
from intensity.utility import DummyRequest
from intensity.tracker.views import asset_clone, activity_new, activity_requisition
from intensity.components.instance_selector__views import select

## Do the actual work
def wizard(account, location, original, requisition):
    dummy_request = DummyRequest(account)

    # Clone the original asset

    response = asset_clone(dummy_request, original.uuid)
    url = response['Location']
    m = re.match('/tracker/asset/view/(\w+)/$', url)
    asset_id = m.group(1)
    assert(asset_id != original.uuid)
    asset = AssetInfo.objects.get(uuid=asset_id)
    asset.location = location
    asset.save()

    # Create an activity

    dummy_request.POST['asset_id'] = asset_id
    response = activity_new(dummy_request)
    url = response['Location']
    m = re.match('/tracker/activity/view/(\w+)/$', url)
    activity_id = m.group(1)
    activity = Activity.objects.get(uuid=activity_id)
    activity.name = location.replace('base/', '').replace('.tar.gz', '') + ' v1.0'
    activity.save()

    # If requested, requisition and select a server
    if requisition:
        # Requisition
        activity_requisition(dummy_request, activity_id)
        message = dummy_request.session['message'].lower()
        success = 'success' in message
        if not success:
            return 'Map creation wizard made your map asset and activity, but could not find a server. Try to requisition one later for your new activity.'

        # Select
        instances = ServerInstance.objects.filter(activity = activity).filter(requisitioner = account)
        assert(len(instances) == 1)
        select(dummy_request, instances[0].uuid)

    return True

@login_required
def view(request):
    def on_valid(form):
        logging.info('Map wizard - view')

        result = wizard(request.account, form.cleaned_data['location'], form.cleaned_data['original'], form.cleaned_data['requisition'])

        if type(result) is str:
            request.session['message'] = result
        else:
            if form.cleaned_data['requisition']:
                request.session['message'] = 'Map creation wizard finished successfully. To start editing your map, open the client program, log in and then select "connect to selected."'
            else:
                request.session['message'] = 'Map creation wizard finished successfully. To start editing your map, run it on a server and connect to that server.'

    return getpost_form(request, 'map_wizard.html', MapWizardForm, on_valid, '/tracker/account/')


import pandas as pd
from sklearn.linear_model import LinearRegression
import matplotlib.pyplot as plt

salary = pd.read_csv("salary.csv")
# empty fields in salary change to NA
salary['salaryBrutto'] = pd.to_numeric(salary['salaryBrutto'], errors='coerce')
salary_original = salary.copy()
# erase all rows with NA values
salary = salary.dropna()

workedYears = salary.drop('salaryBrutto', axis = 1)

# teaching
lm = LinearRegression()
lm.fit(workedYears, salary.salaryBrutto)

# here I could test only values from range salary_original[47:]
# like Krzysztof has proposed
# prediction
X = salary_original.drop('salaryBrutto', axis = 1)
Y = lm.predict(X)

salary_prediction = salary_original.copy()
salary_prediction.salaryBrutto = Y

salary_prediction.index.names = ['id']
salary_prediction.columns = ['worked_years', 'salary']

salary_prediction[47:].to_csv('salary_predicted.csv')


from datetime import datetime
from pandas import read_csv, DataFrame
from seaborn import regplot, jointplot
from matplotlib.backends.backend_pdf import PdfPages
import matplotlib.pyplot as mpt
import patsy as pt
import numpy as np
import os
from fpdf import FPDF

def handle_uploaded_file(data):
    save_path = '/tmp/' + datetime.now().strftime('%d%m%y %H:%M:%S') + '.csv'

    with open(save_path, 'wb+') as dest:
        for chunk in data.chunks():
            dest.write(chunk)
    return save_path
'''
def ploting(save_path):
    table = read_csv(save_path, delimiter = ',')
    way = save_path.split('.')[0]
    os.mkdir(way)
    index = table.columns
    for i in range(0,table.shape[1] - 1):
        g = regplot(x = index[i], y = index[-1],  data = table[[index[i], index[-1]]] )
        mpt.savefig(way + '/' + way.split('/')[-1] + index[i] + '.jpg')

    pdf = FPDF()
    for image in os.listdir(way):
        pdf.add_page()
        pdf.image(way + '/' + image, w = 200, h = 200)
    pdf.output( way + '/result.pdf', 'F')

    return way + '/result.pdf'

'''
def ploting(save_path):
    table = read_csv(save_path, delimiter = ',')
    way = save_path.split('.')[0] + '.pdf'
    pp = PdfPages(way)
    index = table.columns
    for i in range(0,table.shape[1] - 1):
        regplot(x = index[i], y = index[-1], data = table[[index[i], index[-1]]])
        pp.savefig()
    pp.close()

    return way

def eqs(t):
    st = 'y ~ ' + ' + '.join([ e for e in table.columns if e[0] == 'x'])
    print(st)
    pt_y, pt_x = pt.dmatrices(st, t)
    res = np.linalg.lstsq(pt_x, pt_y)
    print(res[0].ravel())


import urllib.request
import http.cookiejar
from bs4 import BeautifulSoup
import codecs
import time

url = "https://hackerone.com/programs"
#local_copy = "Programs - HackerOne.html"

local_copy = "../input/Directory_ Report Vulnerabilities to Companies's Security Teams.html"

# Currently, one should download the programs page manually and then
# use this script to parse the local copy. This is probability because
# the page contains javascript code which later fetches
# the list of the programs. Our crawler thus cannot get the information directly.
# Also, if you click view source in Chrome, you won't see the
# programs either.
getDataOnline = False

resultFilePath = "../output/programList.txt"
resultFile = codecs.open(resultFilePath, "w", "utf-8", buffering=0)

programStatFile = codecs.open("../output/programStat.txt", "w", "utf-8", buffering=0)

def is_program_name(tag):
    #print(tag)
    return  tag.name == 'span' and 'itemprop' in tag.attrs and "data-reactid" in tag.attrs

def is_program_link(tag):

    return tag.name == "a" and 'class' in tag.attrs and 'spec-program-name' in tag.attrs['class']

# HackerOne has changed their website. So we cannot use the following code anymore.
# Or we need to use some interaction simulation to get the pop up window.

#def is_item_value(tag):
#    return  tag.name == 'div' and 'class' in tag.attrs and "program-profile-meta-item-value" in tag.attrs['class']

#def is_bug_closed(tag):
#    if (tag.name != "div") or ('class' not in tag.attrs) or ("program-profile-meta-item-value" not in tag.attrs['class']):
#        return False
#    print(tag)
#    return True

if getDataOnline:

    cj = http.cookiejar.CookieJar()
    opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
    opener.addheaders = [('User-agent','Mozilla/5.0 (Windows NT 6.3; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0')]
    urllib.request.install_opener(opener)
    req = opener.open(url)

    while req.code != 200:
        print("Request fail, code = " + str(req.code))
        time.sleep(5)
        req = opener.open(url)

    soup = BeautifulSoup(req.read())

else:

    soup = BeautifulSoup(codecs.open(local_copy, encoding="utf-8").read())

entries = soup.find_all(is_program_link)

for entry in entries:

    #bugClosedTag = entry.parent.parent.findNext(is_item_value)

    #bugClosed = bugClosedTag.string

    #progName = entry.findNext("span").string

    progLink = entry['href']
    progName = entry.findNext(is_program_name).string

    minBounty = None

    #if "Minimum bounty" in str(bugClosedTag.parent.parent):
    #    minBounty = bugClosedTag.findNext(is_item_value).string

    #programStatFile.write(progName + "," + bugClosed + "," + str(minBounty) + "\n")

    print(progLink)

    resultFile.write(progName + "," + progLink + "\n")

resultFile.close()
programStatFile.close()


"""
import speech_recognition as sr
from PyQt5.uic import loadUi
from PyQt5.QtWidgets import QDialog

r = sr.Recognizer()

class input_voice():
    def __init__(self):
        loadUi("./outputwindow.ui", self)
        self.ClockOutButton.setChecked(False)
        self.ClockInButton.setChecked(False)
        self.function()

    def function(self):

        while True:
            try:
                with sr.Microphone() as source:
                    print("Speak 'check in' to checkin and 'check out' to checkout ")
                    r.adjust_for_ambient_noise(source, duration=0.2)
                    audio = r.listen(source)
                    MyText = r.recognize_google(audio)
                    MyText = MyText.lower()
                    if MyText == 'check in':
                        self.ClockInButton.setChecked(True)
                        print("~~~~~You Checked In~~~~~")
                    if MyText == 'check out':
                        self.ClockOutButton.setChecked(True)
                        print("~~~~~You Checked Out~~~~~")
                    if MyText == 'thank you':
                        print("~~~~~Your Welcome~~~~~")
                        break
            except sr.RequestError as e:
                print("Could not request results; {0}".format(e))

            except sr.UnknownValueError:
                print("unknown error occured")
"""
"""
haha = input_voice()
haha.function
"""

import speech_recognition as sr
import multiprocessing
import time

def speech(string_message):
    r = sr.Recognizer()
    try:
        with sr.Microphone() as source:
            print("Speak 'IN' to checkin and 'OUT' to checkout ")
            r.adjust_for_ambient_noise(source, duration=0.2)
            audio = r.listen(source)
            MyText = r.recognize_google(audio)
            MyText = MyText.lower()
            if MyText == 'in':
                print("~~~~~You Checked In~~~~~")
                string_message = 'clockin'
            if MyText == 'out':
                print("~~~~~You Checked Out~~~~~")
                string_message = 'clockout'

    except sr.RequestError as e:
        print("Could not request results; {0}".format(e))

    except sr.UnknownValueError:
        print("unknown error occured")

print("haha")
string_message = ' '
p = multiprocessing.Process(target= speech, name="Function", args=(string_message, ))
p.start()
time.sleep(10)
p.terminate()
p.join()
print(string_message)


"""Methods and utilities for sampling experiences"""
from collections import defaultdict
import numpy as np

import torch

class MultiBatch:
    """Represents a multi-agent experience batch"""

    OBS = "obs"
    ACTION = "actions"
    REWARD = "rewards"
    DONE = "dones"

    def __init__(self):
        self._policy_batches = defaultdict(list)
        self._agent_episodes = None
        self._policy_map = None

    def _store_episode(self):
        for agent_id, episode in self._agent_episodes.items():
            for key in episode.keys():
                episode[key] = np.array(episode[key])

            if self._policy_map is not None:
                batch = self._policy_batches[self._policy_map[agent_id]]
            else:
                batch = self._policy_batches[agent_id]

            batch.append(episode)

    def start_episode(self, policy_map=None):
        if self._agent_episodes is not None:
            self._store_episode()

        self._agent_episodes = defaultdict(lambda: defaultdict(list))
        self._policy_map = policy_map

    def end_episode(self):
        if self._agent_episodes is not None:
            self._store_episode()

        self._agent_episodes = None
        self._policy_map = None

    def append(self, field, values):
        for agent_id, value in values.items():
            self._agent_episodes[agent_id][field].append(value)

    def step(self, obs, actions, rewards, dones, fetches):
        for agent_id in obs.keys():
            episode = self._agent_episodes[agent_id]

            episode[MultiBatch.OBS].append(obs[agent_id])
            episode[MultiBatch.ACTION].append(actions[agent_id])
            episode[MultiBatch.REWARD].append(rewards[agent_id])
            episode[MultiBatch.DONE].append(dones[agent_id])

            for key, value in fetches[agent_id].items():
                episode[key].append(value)

    def policy_batch(self, policy_id):
        return self._policy_batches[policy_id]

    def items(self):
        return self._policy_batches.items()

class FrozenPolicy:
    """Wrapper for frozen Torch policies"""

    def __init__(self, model):
        self._model = model

    def make_agent(self):

        class Agent:

            def __init__(self, model):
                self._state = model.initial_state(batch_size=1)
                self._model = model

            def act(self, obs):
                obs = torch.as_tensor(obs, dtype=torch.float32)
                obs = obs.unsqueeze(0)  # NOTE: Add batch dimension
                obs = obs.unsqueeze(0)  # NOTE: Add time dimension (for RNNs)

                action, self._state = self._model(obs, self._state)
                action.squeeze(0)  # NOTE: Remove time dimension
                action.squeeze(0)  # NOTE: Remove batch dimension

                return action.numpy(), {}

        return Agent(self._model)

    def update(self, update):
        """Dummy to make sure we are not trying to update frozen polices"""
        raise NotImplementedError("Frozen policies cannot be updated")

    @property
    def model(self):
        return self._model

def sample(env, policies, num_episodes, max_steps, policy_fn=None):
    """Generates a batch of episodes"""
    batch = MultiBatch()
    total_steps = 0

    for episode in range(num_episodes):
        obs = env.reset()
        step = 0

        agents = {}
        map = {}
        dones = {}
        for agent_id in obs.keys():
            if policy_fn is not None:
                policy_id = policy_fn(agent_id)
            else:
                policy_id = agent_id

            agents[agent_id] = policies[policy_id].make_agent()
            map[agent_id] = policy_id
            dones[agent_id] = False

        batch.start_episode(map)  # NOTE: Why do we need the policy map
        batch.append(MultiBatch.OBS, obs)

        while step < max_steps and not all(dones.values()):
            actions = {}
            fetches = {}
            for agent_id, ob in obs.items():
                actions[agent_id], fetches[agent_id] = agents[agent_id].act(ob)

            obs, rewards, dones, _ = env.step(actions)

            batch.step(obs, actions, rewards, dones, fetches)
            step += 1

        batch.end_episode()
        total_steps += step

    stats = {"mean_reward": 0}
    for policy_id, agent_batch in batch.items():
        mean_reward = 0
        for episode in agent_batch:
            mean_reward += np.sum(episode[MultiBatch.REWARD])

        mean_reward /= len(agent_batch)
        stats[str(policy_id) + "/mean_reward"] = mean_reward
        stats["mean_reward"] += mean_reward

    stats["episodes"] = num_episodes
    stats["samples"] = total_steps

    return batch, stats


import sys, collections
sys.stdin = open("숨바꼭질_input.txt")

def BFS():
    while q:
        x = q.popleft()
        if x == K:
            return visit[x]
        nextPos(x - 1, x)
        nextPos(x + 1, x)
        nextPos(x * 2, x)

def nextPos(nex, x):
    # nex 지점이 범위 내에 있고
    # 한번도 방문하지 않았거나, 최소 time으로 해당 방을 방문한 경우
    if 0 <= nex < 100001 and (visit[nex] == 0 or visit[nex] > visit[x] + 1):
        visit[nex] = visit[x] + 1
        q.append(nex)

N, K = map(int, input().split())

visit = [0 for _ in range(100001)]
q = collections.deque([N])
print(BFS())


from typing import Optional

from torch import Tensor

from recsys_metrics.utils import _check_beyond_accuracy_inputs

# Efficient implementation: see https://ojs.aaai.org/index.php/AAAI/article/view/17886/17691
def mean_interlist_diversity(preds: Tensor, indexes: Tensor, k: Optional[int] = None) -> Tensor:
    preds, indexes, k = _check_beyond_accuracy_inputs(preds, indexes, k=k)

    _, topk_idx = preds.topk(k, dim=-1)
    sorted = False  # turn off sorting speeds up ~40% and we do NOT need sorted result for `numel`
    _, topk_cnt = indexes.take_along_dim(topk_idx, dim=-1).unique(sorted=sorted, return_counts=True)

    squared_cnt = (topk_cnt**2).sum(dim=-1)  # `tensor` dtype, no need to convert other raw dtypes below
    n_users = len(preds)
    n_pairs = n_users**2 - n_users
    return (n_users**2 - squared_cnt/k) / n_pairs