import argparse
import glob
import json
import logging
import os
import random

import numpy as np
import torch
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
from torch.utils.data.distributed import DistributedSampler
from tqdm import tqdm, trange

from transformers import (
    WEIGHTS_NAME,
    AdamW,
    get_linear_schedule_with_warmup,
    BertConfig,
    BertModel,
    BertPreTrainedModel,
    BertTokenizer,
)
from transformers import glue_compute_metrics as compute_metrics
from transformers import glue_convert_examples_to_features as convert_examples_to_features
from transformers import glue_output_modes as output_modes
from transformers import glue_processors as processors

from transformers.data.processors.utils import InputExample, DataProcessor

try:
    from torch.utils.tensorboard import SummaryWriter
except ImportError:
    from tensorboardX import SummaryWriter

import code
import os
import pickle
from sklearn.metrics.pairwise import cosine_similarity

import torch
import torch.nn as nn
import torch.nn.functional as F

import numpy as np
import pandas as pd

logger = logging.getLogger(__name__)
model_path = "/share/data/lang/users/zeweichu/projs/qa-chinese/chinese_wwm_ext_pytorch"

MODEL_CLASSES = {
    "bert": (BertConfig, BertTokenizer),
}

def set_seed(args):
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    if args.n_gpu > 0:
        torch.cuda.manual_seed_all(args.seed)

class FAQProcessor(DataProcessor):
    """Processor for the CoLA data set (GLUE version)."""

    def get_example_from_tensor_dict(self, tensor_dict):
        """See base class."""
        return InputExample(
            tensor_dict["idx"].numpy(),
            tensor_dict["sentence"].numpy().decode("utf-8"),
            None,
            str(tensor_dict["label"].numpy()),
        )

    def prepare_data(self, file_dir):
        train_df = pd.read_csv(file_dir, sep="\t")
        self.candidate_title = train_df["title"].tolist()
        self.candidate_reply = train_df["reply"].tolist()
        self.candidate_translated = train_df["translated"].tolist()

    def get_train(self):
        return self._create_pos_examples(self.candidate_title, self.candidate_translated, "pos"), self._create_neg_examples(self.candidate_title, "neg")

    def _create_pos_examples(self, lines_a, lines_b, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, (line_a, line_b)) in enumerate(zip(lines_a, lines_b)):
            guid = "%s-%s" % (set_type, i)
            examples.append(InputExample(guid=guid, text_a=line_a, text_b=line_b, label=1))
        return examples

    def _create_neg_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line_a) in enumerate(zip(lines)):
            guid = "%s-%s" % (set_type, i)
            line_b = random.sample(lines, 1)[0]
            while line_b == line_a:
                line_b = random.sample(1)[0]
            examples.append(InputExample(guid=guid, text_a=line_a, text_b=line_b, label=1))
        return examples

class BertForFAQHinge(BertPreTrainedModel):
    r"""
        **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
            Labels for computing the sequence classification/regression loss.
            Indices should be in ``[0, ..., config.num_labels - 1]``.
            If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss),
            If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy).
    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
        **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
            Classification (or regression if config.num_labels==1) loss.
        **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)``
            Classification (or regression if config.num_labels==1) scores (before SoftMax).
        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
            of shape ``(batch_size, sequence_length, hidden_size)``:
            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
    Examples::
        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
        model = BertForSequenceClassification.from_pretrained('bert-base-uncased')
        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
        labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
        outputs = model(input_ids, labels=labels)
        loss, logits = outputs[:2]
    """

    def __init__(self, config):
        super(BertPreTrainedModel, self).__init__(config)
        self.num_labels = config.num_labels

        self.bert = BertModel(config)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)
        self.scoring = nn.Linear(config.hidden_size, 1)

        self.init_weights()

    def forward(
        self,
        input_ids=None,
        attention_mask=None,
        token_type_ids=None,
        position_ids=None,
        head_mask=None,
        inputs_embeds=None,
        labels=None,
    ):
        outputs = self.bert(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
        )

        pooled_output = outputs[1]
        pooled_output = self.dropout(pooled_output)
        score = self.scoring(pooled_output)

        outputs = (score,) + outputs[2:]  # add hidden states and attention if they are here

        return outputs  # (loss), logits, (hidden_states), (attentions)

def train(args, train_dataset, model, processor, tokenizer):
    """ Train the model """
    if args.local_rank in [-1, 0]:
        tb_writer = SummaryWriter()

    # Prepare optimizer and schedule (linear warmup and decay)
    no_decay = ["bias", "LayerNorm.weight"]
    optimizer_grouped_parameters = [
        {
            "params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
            "weight_decay": args.weight_decay,
        },
        {"params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], "weight_decay": 0.0},
    ]

    t_total = len(train_dataset) // args.gradient_accumulation_steps * args.num_train_epochs
    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
    scheduler = get_linear_schedule_with_warmup(
        optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total
    )

    # Train!
    logger.info("***** Running training *****")
    logger.info("  Num examples = %d", len(train_dataset))
    logger.info("  Num Epochs = %d", args.num_train_epochs)
    logger.info("  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size)
    logger.info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
    logger.info("  Total optimization steps = %d", t_total)

    epochs_trained = 0
    global_step = 0
    steps_trained_in_current_epoch = 0

    tr_loss, logging_loss = 0.0, 0.0
    model.zero_grad()
    train_iterator = trange(
        epochs_trained, int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]
    )
    set_seed(args)  # Added here for reproductibility
    for k in train_iterator:
        if k != 0:
            dataset, _, _ = load_examples(args, args.task_name, tokenizer, processor)
        args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)
        train_sampler = SequentialSampler(train_dataset)
        train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)
        epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0])
        for step, batch in enumerate(epoch_iterator):

            # Skip past any already trained steps if resuming training
            if steps_trained_in_current_epoch > 0:
                steps_trained_in_current_epoch -= 1
                continue

            model.train()
            batch = tuple(t.to(args.device) for t in batch)
            pos_inputs = {"input_ids": batch[0], "attention_mask": batch[1], "token_type_ids": batch[2]}
            neg_inputs = {"input_ids": batch[3], "attention_mask": batch[4], "token_type_ids": batch[5]}
            pos_outputs = model(**pos_inputs)
            neg_outputs = model(**neg_inputs)

            pos_score = pos_outputs[0]
            neg_score = neg_outputs[0]

            loss = -(pos_score - neg_score - args.margin) # pos必须要比neg高一定的margin
            loss[loss < 0] = 0
            loss = torch.mean(loss)
            loss.backward()

            tr_loss += loss.item()
            if (step + 1) % args.gradient_accumulation_steps == 0:
                torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)

                optimizer.step()
                scheduler.step()  # Update learning rate schedule
                model.zero_grad()
                global_step += 1
        logger.info("average loss: " + str(tr_loss/global_step))

    return global_step, tr_loss / global_step

def evaluate(args, model, eval_dataset):
    # 把所有的问题全部用BERT编码
    # Loop to handle MNLI double evaluation (matched, mis-matched)
    scores = []

    args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)
    # Note that DistributedSampler samples randomly
    eval_sampler = SequentialSampler(eval_dataset)
    eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)

    # multi-gpu eval
    if args.n_gpu > 1:
        model = torch.nn.DataParallel(model)

    # Eval!
    logger.info("  Num examples = %d", len(eval_dataset))
    logger.info("  Batch size = %d", args.eval_batch_size)
    eval_loss = 0.0
    nb_eval_steps = 0
    preds = None
    out_label_ids = None
    for batch in tqdm(eval_dataloader, desc="Evaluating"):
        model.eval()
        batch = tuple(t.to(args.device) for t in batch)

        with torch.no_grad():
            inputs = {"input_ids": batch[0], "attention_mask": batch[1], "token_type_ids": batch[2]}
            outputs = model(**inputs)

            score = outputs[0]
            scores.append(score)

    scores = [score.data.cpu().view(-1) for score in scores]
    scores = torch.cat(scores)

    return scores.numpy()

def load_examples(args, task, tokenizer, processor):
    if args.local_rank not in [-1, 0] and not evaluate:
        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache

    pos_data, neg_data = processor.get_train()

    # Load data features from cache or dataset file
    logger.info("Creating features from dataset file at %s", args.data_dir)
    pos_features = convert_examples_to_features(
        pos_data,
        tokenizer,
        label_list=[1],
        output_mode="classification",
        max_length=args.max_seq_length,
        pad_on_left=bool(args.model_type in ["xlnet"]),  # pad on the left for xlnet
        pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],
        pad_token_segment_id=4 if args.model_type in ["xlnet"] else 0,
    )
    neg_features = convert_examples_to_features(
        neg_data,
        tokenizer,
        label_list=[1],
        output_mode="classification",
        max_length=args.max_seq_length,
        pad_on_left=bool(args.model_type in ["xlnet"]),  # pad on the left for xlnet
        pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],
        pad_token_segment_id=4 if args.model_type in ["xlnet"] else 0,
    )

    if args.local_rank == 0 and not evaluate:
        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache

    # Convert to Tensors and build dataset
    pos_input_ids = torch.tensor([f.input_ids for f in pos_features], dtype=torch.long)
    pos_attention_mask = torch.tensor([f.attention_mask for f in pos_features], dtype=torch.long)
    pos_token_type_ids = torch.tensor([f.token_type_ids for f in pos_features], dtype=torch.long)
    neg_input_ids = torch.tensor([f.input_ids for f in neg_features], dtype=torch.long)
    neg_attention_mask = torch.tensor([f.attention_mask for f in neg_features], dtype=torch.long)
    neg_token_type_ids = torch.tensor([f.token_type_ids for f in neg_features], dtype=torch.long)
    dataset = TensorDataset(pos_input_ids, pos_attention_mask, pos_token_type_ids, neg_input_ids, neg_attention_mask, neg_token_type_ids)

    return dataset, processor.candidate_title, processor.candidate_reply

def main():
    parser = argparse.ArgumentParser()

    # Required parameters
    parser.add_argument(
        "--data_dir",
        default=None,
        type=str,
        required=True,
        help="The input data dir. Should contain the .tsv files (or other data files) for the task.",
    )
    parser.add_argument(
        "--model_type",
        default=None,
        type=str,
        required=True,
        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
    )
    parser.add_argument(
        "--model_name_or_path",
        default=None,
        type=str,
        required=True,
        help="Path to pre-trained model or shortcut name selected in the list",
    )
    parser.add_argument(
        "--task_name",
        default=None,
        type=str,
        required=True,
        help="The name of the task to train selected in the list: " + ", ".join(processors.keys()),
    )
    parser.add_argument(
        "--output_dir",
        default=None,
        type=str,
        required=True,
        help="The output directory where the model predictions and checkpoints will be written.",
    )

    # Other parameters
    parser.add_argument(
        "--config_name", default="", type=str, help="Pretrained config name or path if not the same as model_name"
    )
    parser.add_argument(
        "--tokenizer_name",
        default="",
        type=str,
        help="Pretrained tokenizer name or path if not the same as model_name",
    )
    parser.add_argument(
        "--cache_dir",
        default="",
        type=str,
        help="Where do you want to store the pre-trained models downloaded from s3",
    )
    parser.add_argument(
        "--max_seq_length",
        default=128,
        type=int,
        help="The maximum total input sequence length after tokenization. Sequences longer "
        "than this will be truncated, sequences shorter will be padded.",
    )
    parser.add_argument("--do_train", action="store_true", help="Whether to run training.")
    parser.add_argument("--do_eval", action="store_true", help="Whether to run eval on the dev set.")
    parser.add_argument(
        "--evaluate_during_training", action="store_true", help="Rul evaluation during training at each logging step."
    )
    parser.add_argument(
        "--do_lower_case", action="store_true", help="Set this flag if you are using an uncased model."
    )

    parser.add_argument("--per_gpu_train_batch_size", default=8, type=int, help="Batch size per GPU/CPU for training.")
    parser.add_argument(
        "--per_gpu_eval_batch_size", default=8, type=int, help="Batch size per GPU/CPU for evaluation."
    )
    parser.add_argument(
        "--gradient_accumulation_steps",
        type=int,
        default=1,
        help="Number of updates steps to accumulate before performing a backward/update pass.",
    )
    parser.add_argument("--margin", default=5, type=float, help="The margin of hinge loss.")
    parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.")
    parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.")
    parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.")
    parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.")
    parser.add_argument(
        "--num_train_epochs", default=3.0, type=float, help="Total number of training epochs to perform."
    )
    parser.add_argument(
        "--max_steps",
        default=-1,
        type=int,
        help="If > 0: set total number of training steps to perform. Override num_train_epochs.",
    )
    parser.add_argument("--warmup_steps", default=0, type=int, help="Linear warmup over warmup_steps.")

    parser.add_argument("--logging_steps", type=int, default=50, help="Log every X updates steps.")
    parser.add_argument("--save_steps", type=int, default=50, help="Save checkpoint every X updates steps.")
    parser.add_argument(
        "--eval_all_checkpoints",
        action="store_true",
        help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number",
    )
    parser.add_argument("--no_cuda", action="store_true", help="Avoid using CUDA when available")
    parser.add_argument(
        "--overwrite_cache", action="store_true", help="Overwrite the cached training and evaluation sets"
    )
    parser.add_argument("--seed", type=int, default=42, help="random seed for initialization")

    parser.add_argument(
        "--fp16",
        action="store_true",
        help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",
    )
    parser.add_argument(
        "--fp16_opt_level",
        type=str,
        default="O1",
        help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
        "See details at https://nvidia.github.io/apex/amp.html",
    )
    parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank")
    parser.add_argument("--server_ip", type=str, default="", help="For distant debugging.")
    parser.add_argument("--server_port", type=str, default="", help="For distant debugging.")
    args = parser.parse_args()

    # Setup distant debugging if needed
    if args.server_ip and args.server_port:
        # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
        import ptvsd

        print("Waiting for debugger attach")
        ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
        ptvsd.wait_for_attach()

    # Setup CUDA, GPU & distributed training
    if args.local_rank == -1 or args.no_cuda:
        device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
        args.n_gpu = torch.cuda.device_count()
    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
        torch.cuda.set_device(args.local_rank)
        device = torch.device("cuda", args.local_rank)
        torch.distributed.init_process_group(backend="nccl")
        args.n_gpu = 1
    args.device = device

    # Setup logging
    logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
        datefmt="%m/%d/%Y %H:%M:%S",
        level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,
    )
    logger.warning(
        "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
        args.local_rank,
        device,
        args.n_gpu,
        bool(args.local_rank != -1),
        args.fp16,
    )

    # Set seed
    set_seed(args)

    # Prepare GLUE task
    args.task_name = args.task_name.lower()

    # Load pretrained model and tokenizer
    if args.local_rank not in [-1, 0]:
        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab

    args.model_type = args.model_type.lower()
    config_class, tokenizer_class = MODEL_CLASSES[args.model_type]
    model_class = BertForFAQHinge
    config = config_class.from_pretrained(
        args.config_name if args.config_name else args.model_name_or_path,
        finetuning_task=args.task_name,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )
    tokenizer = tokenizer_class.from_pretrained(
        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
        do_lower_case=args.do_lower_case,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )
    model = model_class.from_pretrained(
        args.model_name_or_path,
        from_tf=bool(".ckpt" in args.model_name_or_path),
        config=config,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )

    if args.local_rank == 0:
        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab

    model.to(args.device)

    logger.info("Training/evaluation parameters %s", args)

    processor = FAQProcessor()
    processor.prepare_data(args.data_dir)
    dataset, candidate_title, candidate_reply = load_examples(args, args.task_name, tokenizer, processor)
    if args.do_train:
        train(args, dataset, model, processor, tokenizer)
        # Create output directory if needed
        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
            os.makedirs(args.output_dir)

        logger.info("Saving model checkpoint to %s", args.output_dir)
        # Save a trained model, configuration and tokenizer using `save_pretrained()`.
        # They can then be reloaded using `from_pretrained()`
        model_to_save = (
            model.module if hasattr(model, "module") else model
        )  # Take care of distributed/parallel training
        model_to_save.save_pretrained(args.output_dir)
        tokenizer.save_pretrained(args.output_dir)

        # Good practice: save your training arguments together with the trained model
        torch.save(args, os.path.join(args.output_dir, "training_args.bin"))

        # Load a trained model and vocabulary that you have fine-tuned
        model = model_class.from_pretrained(args.output_dir)
        tokenizer = tokenizer_class.from_pretrained(args.output_dir)
        model.to(args.device)

    if args.do_eval:
        while True:
            title = input("你的问题是？\n")
            if len(title.strip()) == 0:
                continue

            examples = [InputExample(guid=0, text_a=title, text_b=c, label=1) for c in candidate_title]
            features = convert_examples_to_features(
                examples,
                tokenizer,
                label_list=[1],
                output_mode="classification",
                max_length=args.max_seq_length,
                pad_on_left=bool(args.model_type in ["xlnet"]),  # pad on the left for xlnet
                pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],
                pad_token_segment_id=4 if args.model_type in ["xlnet"] else 0,
            )

            # Convert to Tensors and build dataset
            all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
            all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long)
            all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long)

            dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids)
            scores = evaluate(args, model, dataset)
            top5_indices = scores.argsort()[-5:][::-1]
            for index in top5_indices:
                print("可能的答案，参考问题：" + candidate_title[index] + "\t答案：" + candidate_reply[index] + "\t得分：" + str(scores[index]))
                print()

if __name__ == "__main__":
    main()


from sympy import (Eq, solve)
from symplyphysics import (Symbol, units, Quantity, print_expression, validate_input,
    validate_output)
from symplyphysics.core.operations.sum_array import SumArray
from symplyphysics.core.symbols.symbols import tuple_of_symbols

# Description
## sum(I) = 0
# Where:
## I is a current flowing through electrical node.

## In other words, as electrical charge is being neither created nor accumulated in electrical node,
## sum of all input currents are equal to sum of output current. If we assert input current is positive and output is negative, we gain summary current as 0.
## This property of electrical node is called Kirchhoff law #1.
## Assert there are minimum 2 currents flowing through any node

currents = Symbol("currents", units.current)
law = Eq(SumArray(currents), 0, evaluate=False)

def print_law() -> str:
    return print_expression(law)

@validate_input(currents_=currents)
@validate_output(units.current)
def calculate_current_from_array(currents_: list[Quantity]) -> Quantity:
    current_symbols = tuple_of_symbols("current", units.current, len(currents_) + 1)
    unknown_current = current_symbols[len(currents_)]
    currents_law = law.subs(currents, current_symbols).doit()
    solved = solve(currents_law, unknown_current, dict=True)[0][unknown_current]
    for (from_, to_) in zip(current_symbols, currents_):
        solved = solved.subs(from_, to_)
    return Quantity(solved)


from django.contrib.auth.decorators import login_required
from django.urls import path
from .views import (TestCreateView, QuestionsCreateView, AnswersCreateView, TestsListView, TestDetailView,
                    testing_finishing_view, TestingBeginningView, ContinueTestCreateRedirectView)

urlpatterns = [
    path('', TestsListView.as_view(), name='main'),
    path('test/create/step1/', login_required(TestCreateView.as_view()), name='test_create'),
    path('test/create/step2/', login_required(QuestionsCreateView.as_view()), name='quest_create'),
    path('test/create/step3/', login_required(AnswersCreateView.as_view()), name='answer_create'),
    path('test/create/continue/<str:test_slug>/', login_required(ContinueTestCreateRedirectView.as_view()), name='continue_create'),

    path('tests/<str:slug>/', login_required(TestDetailView.as_view()), name='test_detail'),

    path('tests/start/<int:test_pk>/', login_required(TestingBeginningView.as_view()), name='test_start'),
    path('tests/finish/<int:test_pk>/', testing_finishing_view, name='test_finish'),
]


''' Calibration phase of the LU_urb.py '''

import pickle
import os
import metrics
import numpy as np
import parameters
##from pcraster.framework import *
##from numba import njit
from scipy.spatial import distance

#### Script to find calibrate LU model

# Work directory:
work_dir = parameters.getWorkDir()

# Get metrics
metricList = parameters.getSumStats()
locationalMetric = parameters.getLocationalAccuracyMetric()
all_metrices = metricList+locationalMetric
# Get case studies
case_studies = parameters.getCaseStudies()
# Get calibration scenarios
scenarios = parameters.getCalibrationScenarios()

# Get the number of parameter iterations and number of time step defined in the parameter.py script
nrOfTimesteps=parameters.getNrTimesteps()
numberOfIterations = parameters.getNumberofIterations()
iterations = range(1, numberOfIterations+1, 1)
timeSteps=range(1,nrOfTimesteps+1,1)

# Get the observed time steps. Time steps relate to the year of the CLC data, where 1990 was time step 0.
obsSampleNumbers = [1] #range(1,20+1,1) <- for stochastic model
obsTimeSteps = parameters.getObsTimesteps()

# Path to the folder with the metrics npy arrays stored
country = parameters.getCountryName()
arrayFolder = os.path.join(work_dir,'results',country,'metrics')

#################
### FUNCTIONS ###
#################

def getModelledArray(metric,scenario=None,aim=None,case=None):
  if case is None:
    folder = arrayFolder
  else:
    folder = os.path.join(work_dir,'results',case,'metrics')
  return np.load(os.path.join(folder, metric + '.npy'),allow_pickle=True)

def getObservedArray(metric,scenario=None,aim=None,case=None):
  if case is None:
    folder = arrayFolder
  else:
    folder = os.path.join(work_dir,'results',case,'metrics')
  return np.load(os.path.join(folder, metric + '_obs.npy'),allow_pickle=True)

def getSubsetArray(metric,index,scenario=None,aim=None,case=None):
  if case is None:
    folder = arrayFolder
  else:
    folder = os.path.join(work_dir,'results',case,'metrics')
  return np.load(os.path.join(folder, metric+'_subset_' + str(index) + '.npy'),allow_pickle=True)

def getParameterConfigurations(case=None):
  if case is None:
    folder = arrayFolder
  else:
    folder = os.path.join(work_dir,'results',case,'metrics')
  return np.load(os.path.join(folder, 'parameter_sets.npy'), allow_pickle=True)

def getKappaArray(scenario=None,aim=None,case=None):
  if case is None:
    folder = arrayFolder
  else:
    folder = os.path.join(work_dir,'results',case,'metrics')
  return np.load(os.path.join(folder, 'kappa.npy'),allow_pickle=True)

def getKappaSimulationArray(aim=None,scenario=None, case=None):
  if case is None:
    folder = arrayFolder
  else:
    folder = os.path.join(work_dir,'results',case,'metrics')
  return np.load(os.path.join(folder, 'kappa_simulation.npy'),allow_pickle=True)

def getAllocationArray(aim=None,scenario=None,case=None):
  if case is None:
    folder = arrayFolder
  else:
    folder = os.path.join(work_dir,'results',case,'metrics')
  return np.load(os.path.join(folder, 'allocation_disagreement.npy'),allow_pickle=True)

def getAveragedArray(array, scenario, aim):
  # period is a list containing indexes of selected years, it is defined by the scenario
  period = parameters.getCalibrationPeriod()[scenario][aim]
  # get average values
  a_mean = np.mean(array[period],axis=0)
  return a_mean

def getNormalizedArray(array, kappa=None):
  a_max = np.amax(array)
  a_min = np.amin(array)
  if kappa==True:
    a_norm = [(x - a_min) / (a_max - a_min) for x in array]
  else:
    a_norm = [(a_max - x) / (a_max - a_min) for x in array]
  return a_norm

def createDiffArray(metric, scenario, aim, case=None):
  # Get the modelled and observed data
  modelled = getModelledArray(metric, scenario, aim, case)
  observed = getObservedArray(metric, scenario, aim, case)
  # Get the number of parameter configurations
  noParameterConfigurations = modelled.shape[1]
  # Get the number of zones
  numberZones = len(modelled[0][0][1])
  # Create a 3D array: no of rows = no of observed timesteps, no of columns = no of parameter sets
  # Each cell store: the parameter set and the values of zones
  theArray = np.zeros((len(obsTimeSteps),noParameterConfigurations,numberZones,1))

  for row in range(0,len(obsTimeSteps)):
    for col in range(0,noParameterConfigurations):
      for zone in range(0,numberZones):
        theArray[row,col][zone][0] = modelled[obsTimeSteps[row]-1,col][1][zone][0] - observed[row,0][1][zone][0]
  return theArray

def saveArray(array,name):
  fileName = os.path.join(arrayFolder, str(name))
  # Clear the directory if needed
  if os.path.exists(fileName + '.npy'):
      os.remove(fileName + '.npy')
  # Save the data
  np.save(fileName, array)

#@njit(parallel=True)
def getValidCells(u_array):
  # Get the number of valid cells in the map
  cells = len(u_array)
  nanCells = float(np.sum(np.isnan(u_array)))
  validCells = cells - nanCells
  return validCells

#@njit(parallel=True)
def K_calculations(row,col,obs0,obs1,mod0,mod1,cArray,allocationArray,kappaArray,validCells):
  # Find states. Each cell has only one of four states.
  # Observed -> modelled
  # 1. urban -> urban
  # 2. urban -> non-urban
  # 3. non-urban -> urban
  # 4. non-urban -> non-urban
  state1 = np.where(obs1 & mod1,1,0)
  state2 = np.where(obs1 & mod0,2,0)
  state3 = np.where(obs0 & mod1,3,0)
  state4 = np.where(obs0 & mod0,4,0)

  # Save state of each cell into the array
  allStates = state1+state2+state3+state4

  # Fill the contingency table (Pontius and Millones, 2011, Table 2):
  cArray[0,0] = np.sum(allStates == 1)/validCells
  cArray[1,0] = np.sum(allStates == 2)/validCells
  cArray[0,1] = np.sum(allStates == 3)/validCells
  cArray[1,1] = np.sum(allStates == 4)/validCells
  cArray[2,0] = (np.sum(allStates == 1)+np.sum(allStates == 2))/validCells
  cArray[2,1] = (np.sum(allStates == 3)+np.sum(allStates == 4))/validCells
  cArray[0,2] = (np.sum(allStates == 1)+np.sum(allStates == 3))/validCells
  cArray[1,2] = (np.sum(allStates == 2)+np.sum(allStates == 4))/validCells
  cArray[2,2] = 1

  # Calculate quantity disagreemnt (Pontius and Millones, 2011) (2)
  q_urban = np.absolute(cArray[0,0]+cArray[1,0]-(cArray[0,0]+cArray[0,1]))
  q_non_urban = np.absolute(cArray[0,1]+cArray[1,1]-(cArray[1,0]+cArray[1,1]))
  Q = (q_urban + q_non_urban) / 2 # (3)

  # Calculate allocation disagreement (Pontius and Millones, 2011): (4)
  a_urban = 2 * np.minimum(cArray[1,0], cArray[0,1])
  a_non_urban = 2 * np.minimum(cArray[0,1], cArray[1,0])
  A = (a_urban + a_non_urban) / 2 # (5)

  # Calculate the proportion correct
  C = cArray[0,0] + cArray[1,1] # (6)

  # Calculate the total disagreement
  D = 1 - C # = Q + A (7)

  # Calculate fractions of agreement:
  E = cArray[0,2]*cArray[2,0] + cArray[1,2]*cArray[2,1] # (8,9)

  # Calculate Kappa
  K_standard = (C-E)/(1-E) # (11)
  #print('year: ', str(row[1]),', parameter set: ', str(col), ', Kappa standard:',K_standard)

  # Save Kapa in array
  kappaArray[row[0],col] = K_standard
  # Save errors
  allocationArray[row[0],col] = A

def calculateKappa(scenario=None, aim=None):
  # Create arrays to store the metrices by Pontius (allocation disagreement)
  allocationArray = np.zeros((len(obsTimeSteps),numberOfIterations))
  # Create array to store Kappa
  kappaArray = np.zeros((len(obsTimeSteps),numberOfIterations))
  # Load data:
  urbObs = np.array(getObservedArray('urb'))
  # Loop observed years. For each year, calculate Kappa between observed map and modelled map:
  for row in enumerate(obsTimeSteps):
    # Get the number of valid cells in the map
    validCells = getValidCells(urbObs[row[0],0,1])
    # Load modelled urban for the given obs year
    urbMod = getSubsetArray('urb',row[1])
    # Define conditions to compare simulated and actual (observed) maps
    obs1 = (urbObs[row[0],0,1] == 1)
    obs0 = (urbObs[row[0],0,1] == 0)

    # Loop parameter sets:
    for col in range(0,numberOfIterations):
      # Create contingency table, full of zeros (to calculate Kappa)
      cArray = np.zeros((3,3))

      # For year 1990 (first observation time step) there is a perfect agreement:
      if row[0] == 0:
        kappaArray[row[0],col] = 1.0

      else:
        # Define conditions to compare simulated and actual (observed) maps
        mod1 = (urbMod[0,col,1] == 1)
        mod0 = (urbMod[0,col,1] == 0)

        K_calculations(row,col,obs0,obs1,mod0,mod1,cArray,allocationArray,kappaArray,validCells)

  # Save all the metrices
  saveArray(kappaArray,'kappa')
  saveArray(allocationArray,'allocation_disagreement')
  print('Kappa and Pontius metrices calculated and saved as npy files')

######################################################
# KAPPA SIMULATION                                   #
######################################################
def calculateKappaSimulation(scenario=None, aim=None):
  # Create array to store Kappa Simulation (by Van Vliet):
  kappaSimulation = np.zeros((len(obsTimeSteps),numberOfIterations))
  # Load data:
  urbObs = getObservedArray('urb')
  # Get the number of not NaN cells
  validCells = getValidCells(urbObs[0,0,1])
  # Select the original map
  originals = getOriginal(urbObs[0,0,1], validCells)
  #original_0,original_1,p_org0,p_org1 = getOriginal(urbObs[0,0,1], validCells)
  # Loop observed years:
  for row in enumerate(obsTimeSteps):
    if row == 3:
      #original_0,original_1,p_org0,p_org1 = getOriginal(urbObs[2,0,1], validCells)
      originals = getOriginal(urbObs[2,0,1], validCells)
    # Load modelled urban for the given obs year
    urbMod = getSubsetArray('urb',row[1])
    # Define conditions to compare simulated and actual (observed) maps
    obs1 = (urbObs[row[0],0,1] == 1)
    obs0 = (urbObs[row[0],0,1] == 0)
    # Get transitions from the original map
    transitions = getTransitions(obs0,obs1,originals[0],originals[1],validCells)
    # Loop parameter sets:
    for col in range(0,len(urbMod[0])):
      # For year 1990 (first observation time step) there is a perfect agreement:
      if row[0] in [0]:
        kappaSimulation[row[0],col] = 1.0
      else:
        # Define conditions to compare simulated and actual (observed) maps
        mod1 = (urbMod[0,col,1] == 1)
        mod0 = (urbMod[0,col,1] == 0)
        # Calculate P0 between simulated and actual map (by Van Vliet):
        PO = getPO(obs0,obs1,mod0,mod1,validCells)
        # Calculate PE Transition (Van Vliet, 2013, Eq. 2.7):
        PE_transition = getPE_transition(mod0,mod1,originals,validCells,transitions)
        # Calculate Kappa Simulation (Van Vliet, 2013)
        Ks_values(row,col,kappaSimulation,PO,PE_transition)
  # Save the data
  saveArray(kappaSimulation,'kappa_simulation')

#@njit(parallel=True)
def getOriginal(urb_array, validCells):
  # Define conditions (urban and non-urban cells) in the original map (1990 or 2006):
  original_0 = (urb_array == 0) #non-urban
  original_1 = (urb_array == 1) #urban
  p_org0 = float(np.sum(original_0)/validCells)
  p_org1 = float(np.sum(original_1)/validCells)
  return original_0,original_1,p_org0,p_org1

#@njit(parallel=True)
def getTransitions(obs0,obs1,original_0,original_1,validCells):
  # Define conditions and calculate the transitions:
  act_0_0 = np.where(obs0 & original_0,1,0)
  act_0_1 = np.where(obs1 & original_0,1,0)
  act_1_0 = np.where(obs0 & original_1,1,0)
  act_1_1 = np.where(obs1 & original_1,1,0)
  p_act_0_0 = float(np.sum(act_0_0)/validCells)
  p_act_0_1 = float(np.sum(act_0_1)/validCells)
  p_act_1_0 = float(np.sum(act_1_0)/validCells)
  p_act_1_1 = float(np.sum(act_1_1)/validCells)
  return p_act_0_0,p_act_0_1,p_act_1_0,p_act_1_1

#@njit(parallel=True)
def getPO(obs0,obs1,mod0,mod1,validCells):
  PO_urban = float(np.sum(obs1 & mod1)/validCells)
  PO_non_urban = float(np.sum(obs0 & mod0)/validCells)
  PO = PO_urban + PO_non_urban
  return PO

#@njit(parallel=True)
def getPE_transition(mod0,mod1,originals,validCells,transitions):
  # Get proportion of LU classes in original map
  original_0,original_1,p_org0,p_org1 = originals
  # Get transitions from the original map
  p_act_0_0,p_act_0_1,p_act_1_0,p_act_1_1 = transitions
  # Define conditions to compare transitions in original and simulated maps
  sim_0_0 = np.where(mod0 & original_0,1,0)
  sim_0_1 = np.where(mod1 & original_0,1,0)
  sim_1_0 = np.where(mod0 & original_1,1,0)
  sim_1_1 = np.where(mod1 & original_1,1,0)
  # Get transitions from the simulated (modelled) map
  p_sim_0_0 = float(np.sum(sim_0_0)/validCells)
  p_sim_0_1 = float(np.sum(sim_0_1)/validCells)
  p_sim_1_0 = float(np.sum(sim_1_0)/validCells)
  p_sim_1_1 = float(np.sum(sim_1_1)/validCells)
  # Calculate PE Transition (Van Vliet, 2013, Eq. 2.7):
  PE_transition = p_org0 * (p_act_0_0 * p_sim_0_0 + p_act_0_1 * p_sim_0_1) +\
                  p_org1 * (p_act_1_0 * p_sim_1_0 + p_act_1_1 * p_sim_1_1)
  return PE_transition

#@njit()
def Ks_values(row,col,kappaSimulation,PO,PE_transition):
  # Calculate Kappa Simulation (Van Vliet, 2013, Eq. 2.9):
  K_simulation = (PO - PE_transition) / (1 - PE_transition)
  # Save Kappa Simulation in array
  kappaSimulation[row[0],col] = K_simulation

######################################################

def calcRMSE(metric, scenario, aim, case=None):
  diffArray = createDiffArray(metric,scenario,aim,case)
  # Get the number of parameter sets and number of zones
  pSets = diffArray.shape[1]
  zones = diffArray.shape[2]
  # Create empty array. Rows = nr of observed timesteps, columns = nr of parameter sets
  rmseArray = np.zeros((diffArray.shape[0],diffArray.shape[1]))

  # Calculate RMSE for each timestep and parameter set. Number of observations = number of zones
  for row in range(0,len(obsTimeSteps)):
    for col in range(0,pSets):
      # Create a list containing difference between the modelled and observed metric for the zones
      x = diffArray[row,col].flatten()
      # Remove nan values
      x = x[~np.isnan(x)]
      # Calculate RMSE for each zones
      rmseArray[row,col] = np.sqrt(np.mean(x**2))

  return rmseArray

def getRankedRMSE(metric,scenario,aim,case=None):
  # get the array with RMSE
  aRMSE = calcRMSE(metric, scenario, aim,case)
  # get the RMSE for the selected year
  sRMSE = getAveragedArray(aRMSE, scenario,aim)
  # order the parameter sets
  RMSEorder = sRMSE.argsort()
  # rank the parameter sets. rank == 0 indifies the best parameter set
  RMSEranks = RMSEorder.argsort()

  return RMSEranks

def getKappaIndex(scenario,aim,case=None):
  # get the array with kappa values for the whole study area or the selected zones only
  kappaArr = getKappaArray(scenario, aim, case)
  # get the kappa for the selected year
  aKAPPA = getAveragedArray(kappaArr,scenario,aim)
  # order the parameter sets
  KAPPAorder = aKAPPA.argsort()
  # rank the parameter sets
  KAPPAranks = KAPPAorder.argsort()
  # reverse the ranking of the parameter sets, as the higher value indicates a better result
  KAPPAranks = np.subtract(np.amax(KAPPAranks),KAPPAranks)

  return KAPPAranks

def getAllocationIndex(scenario,aim,case=None):
  # get the array with A values for the whole study area or the selected zones only
  allocationArr = getAllocationArray(scenario, aim, case)
  # get the A for the selected year
  aAllocation = getAveragedArray(allocationArr,scenario,aim)
  # order the parameter sets
  allocationOrder = aAllocation.argsort()
  # rank the parameter sets
  allocationRanks = allocationOrder.argsort()

  return allocationRanks

def getCalibratedIndeks(metric,scenario,case=None):
  if metric in ['kappa','Ks']:
    # Get the ranked parameter sets based on Kappa
    ranked = getKappaIndex(scenario,'calibration',case)
  elif metric =='A':
    # Get the ranked parameter sets based on A
    ranked = getAllocationIndex(scenario,'calibration',case)
  else:
    # Get the ranked parameter sets based on RMSE
    ranked = getRankedRMSE(metric,scenario,'calibration',case)
  calibratedIndex, = np.where(ranked == 0)

  return int(calibratedIndex)

def getRankedMultiobjective(metric,weights,scenario,aim='calibration',case=None):
  # get the array with RMSE
  aRMSE = calcRMSE(metric, scenario, aim,case)
  # get the RMSE for the selected year
  sRMSE = getAveragedArray(aRMSE, scenario,aim)
  # get the normalized RMSE
  n_RMSE = getNormalizedArray(sRMSE)
  # Get the Kappa array for the given aim and scenario
  aKAPPA = getKappaArray(scenario,aim, case)
  # Get the calibration value
  sKappa = getAveragedArray(aKAPPA, scenario,aim)
  # Get the normalized Kappa
  n_Kappa = getNormalizedArray(sKappa, kappa=True)
  # Join two goal function
  n_RMSE_n_Kappa = weights[0] * np.array(n_RMSE) + weights[1] * np.array(n_Kappa) # the bigger the better
  # order the parameter sets
  n_RMSE_n_Kappa_order = n_RMSE_n_Kappa.argsort()
  # rank the parameter sets
  n_RMSE_n_Kappa_ranks = n_RMSE_n_Kappa_order.argsort()
  # reverse the ranking of the parameter sets
  n_RMSE_n_Kappa_ranks = np.subtract(np.amax(n_RMSE_n_Kappa_ranks),n_RMSE_n_Kappa_ranks)

  return n_RMSE_n_Kappa_ranks

def getMultiObjectiveIndex(metric, w_RMSE,w_Kappa,scenario, case=None):
  # Get the ranked parameters based on multi-objective calibration
  ranked = getRankedMultiobjective(metric,[w_RMSE,w_Kappa],scenario,'calibration',case)
  # Best index is ranked 0
  calibratedIndex, = np.where(ranked == 0)

  return int(calibratedIndex)

def getCalibratedParameters(calibrationScenario):
  # create an array of metric names, indexes of the parameters and the parameters' values
  pArray = np.zeros((len(metricList+['kappa']),6), dtype='object')
  parameterSets = getParameterConfigurations()
  for row,m in enumerate(metricList):
    # Get the calibrated parameter index
    calibratedIndex = getCalibratedIndeks(m,calibrationScenario)
    p = parameterSets[calibratedIndex]
    pArray[row,:] = m, calibratedIndex, p[0],p[1],p[2],p[3]

  # Get the parameter index with the highest Kappa
  kappaIndex = getCalibratedIndeks('kappa',calibrationScenario)
  # Get the best parameter set
  k = parameterSets[kappaIndex]
  # Fill the array
  pArray[row+1,:] = 'kappa',kappaIndex, k[0],k[1],k[2],k[3]

  return pArray

def getTopCalibratedParameters(metric, scenario, numberOfTopPerformers,case=None):
  """
  Returns an array of size numberOfTopPerformers x 7
  Each rown contains [metric name, index, validation result, p1, p2, p3, p4]
  """
  topArray = np.zeros((numberOfTopPerformers,7), dtype='object')
  parameterSets = getParameterConfigurations()
  if metric == 'kappa':
    # Get the ranked parameter sets based on Kappa
    ranked = getKappaIndex(scenario,'calibration',case)
    # Get the validation error
    error = getKappaArray(scenario,'validation',case)
  else:
    # Get the ranked parameter sets based on RMSE
    ranked = getRankedRMSE(metric,scenario,'calibration',case)
    # Get the validation error
    error = calcRMSE(metric,scenario,'validation',case)

  for i in range(0,numberOfTopPerformers):
    # Get the parameter index depending on the rank i
    calibratedIndex, = np.where(ranked == i)
    sError = getAveragedArray(error, scenario, 'validation')
    p = parameterSets[calibratedIndex]
    topArray[i,[0,1,2]] = metric, calibratedIndex[0], sError[int(calibratedIndex)]
    for j in [3,4,5,6]:
      topArray[i, j] = p[0][j-3]
  return topArray

def getTopCalibratedParameters_multiobjective(metric, weights, scenario, numberOfTopPerformers,case=None):
  """
  Returns an array of size numberOfTopPerformers x 8
  Each rown contains [metric name, index, validation result RMSE, validation result Kappa, p1, p2, p3, p4]
  weights = [w_RMSE, w_Kappa] # sum=1
  """
  topArray = np.zeros((numberOfTopPerformers,8), dtype='object')
  parameterSets = getParameterConfigurations()
  # Get the ranked parameter sets based on RMSE
  multi_ranked = getRankedMultiobjective(metric,weights,scenario,aim='calibration',case=case)
  # Get the validation RMSE error
  RMSE = calcRMSE(metric,scenario,'validation',case)
  # Get the validation kappa metric
  Kappa = getKappaArray(scenario,'validation',case)

  for i in range(0,numberOfTopPerformers):
    # Get the parameter index depending on the rank i
    calibratedIndex, = np.where(multi_ranked == i)
    avRMSE = getAveragedArray(RMSE, scenario, 'validation')
    avKappa = getAveragedArray(RMSE, scenario, 'validation')
    p = parameterSets[calibratedIndex]
    topArray[i,[0,1,2,3]] = metric,calibratedIndex[0],avRMSE[int(calibratedIndex)],avKappa[int(calibratedIndex)]

    for j in [4,5,6,7]:
      topArray[i, j] = p[0][j-4]
  return topArray

def getResultsEverySet(aim):
  ''' Create a dictionairy: results[country][scenario]
      Dict stores an array with shape: number of parameter sets x number of metrics
      Array stores calibration/validation values for each set for both scenarios '''

  # Create array to store calibration/validation results for all metrics
  results = {}
  # Get the number of parameter sets
  parameters=range(0,numberOfIterations)

  for c in case_studies:
    results[c]={}
    for s in scenarios:
      results[c][s] = np.empty([numberOfIterations,len(all_metrices)])

  # Loop all the countries:
  for country in case_studies:
    # Get data
    resultFolder = os.path.join(os.getcwd(),'results',country, 'metrics')

    i=0
    # Loop all the metrics:
    for m in all_metrices:
      # Loop calibration scenarios:
      for scenario in scenarios:
        # Load data
        if m == 'kappa':
          an_array = getKappaArray(scenario,aim,case=country)
        elif m == 'Ks':
          an_array = getKappaSimulationArray(scenario,aim,case=country)
        elif m == 'A':
          an_array = getAllocationArray(scenario,aim,case=country)
        else:
          an_array = calcRMSE(m,scenario,aim,case=country)

        # get the calibration value of the metric for every set
        av_array = getAveragedArray(an_array,scenario,aim)
        # Save results inside the array
        results[country][scenario][:,i] = av_array
      i = i+1

  return results

def getValidationResults():
  """
  Create a 2D array matrix:
  ## shape validation metrics x 5 goal functions * cases * 2 scenarios
  ## each cell containes RMSE or Kappa Standard, Kappa Simulation or Allocation Disagreement value
  ## this value represent the value of the validation metric (row), obtained using the given goal function,
  ## in a given case study, for a given scenario (column).
  """
  goal_functions = all_metrices
  validation_metrices = all_metrices
  # validation on 3 case studies and 2 scenarios
  validationResults = np.empty((len(validation_metrices),
                                len(goal_functions)*len(case_studies)*len(scenarios)))
  # Lop validation metrices, to get their values:
  for i,v_m in enumerate(validation_metrices):
    j=0
    for country in case_studies:
      for scenario in scenarios:
        for m in goal_functions:
          # Get the index of the calibrated parameter set
          index = getCalibratedIndeks(m, scenario,country)

          # an_array is an array of arrays (contains rmse/kappa arrays for every p set)
          if v_m == 'kappa':
            an_array = getKappaArray(scenario,'validation',case=country)
          elif v_m == 'Ks':
            an_array = getKappaSimulationArray('validation',case=country)
          elif v_m == 'A':
            an_array = getAllocationArray('validation',case=country)
          else:
            # Get the error for the validation metric
            an_array = calcRMSE(v_m,scenario,'validation',case=country)

          # Get the values for the validation period
          av_results = getAveragedArray(an_array,scenario,'validation')

          validationResults[i,j] = av_results[index]

          j+=1

  return validationResults

def getValidationResults_multiobjective(weights):
  """
  Create an array matrix:
  ## shape 7x24 (7 validation metrics x 4 goal functions (metrics determining parameter set) * 3 cases * 2 scenarios)
  ## each cell containes RMSEs, Kappa Standard, Kappa Simulation or Allocation Disagreement value
  ## this value represent the value of the validation metric (row),
  ## obtained using the given multiobjective goal function, combinaning patch and cell-based goal function,
  ## in a given case study, for a given scenario (column).
  """
  goal_functions = metricList
  validation_metrices = all_metrices + ['Ks','A']
  # validation on 3 case studies and 2 scenarios
  validationResults = np.empty((len(validation_metrices),len(goal_functions)*3*2))
  # Lop validation metrices, to get their values:
  for i,v_m in enumerate(validation_metrices):
    j=0
    for m in goal_functions:
      for country in case_studies:
        for scenario in scenarios:
          # Get the index of the calibrated parameter set
          index = getMultiObjectiveIndex(m, weights[0],weights[1],scenario,case=country)

          # an_array is an array of arrays (contains rmse/kappa arrays for every p set)
          if v_m == 'kappa':
            an_array = getKappaArray(scenario,'validation',case=country)
          elif v_m == 'Ks':
            an_array = getKappaSimulationArray('validation',case=country)
          elif v_m == 'A':
            an_array = getAllocationArray('validation',case=country)
          else:
            # Get the error for the validation metric
            an_array = calcRMSE(v_m,scenario,'validation',case=country)
          # Get the values for the validation period
          av_results = getAveragedArray(an_array,scenario,'validation')
          validationResults[i,j] = av_results[index]
          j+=1
  return validationResults

######################
#### NON DOMINATED ###
######################

def get_ND_input(scenario,aim,case):
  ''' Get the calibration\validation values for each set '''
  results = {
    'calibration':getResultsEverySet('calibration'),
    'validation':getResultsEverySet('validation')}
  # ...And get them for a selected case
  v = results[aim][case][scenario]

  return v

def get_ND_mask(case, scenario):
  """
  Create a boolean mask of the pareto-efficient points
  :param costs: An (n_points, n_costs) array
  :return: A (n_points, ) boolean array, indicating whether each point is Pareto efficient
  """
  # Get the metric values to find the non dominated solutions
  costs = get_ND_input(scenario,'calibration',case)
  # Create a boolean array to select nondominated solutions
  is_efficient = np.ones(costs.shape[0], dtype = bool)
  for i, c in enumerate(costs):
    if is_efficient[i]:
      # Keep any point with a lower cost
      is_efficient[is_efficient] = np.any(costs[is_efficient]<c, axis=1)
      is_efficient[i] = True  # And keep self
  return is_efficient

def get_ND(case, scenario, aim):
  """Filter an array and leave the non-dominated rows only"""
  # Get the calibration or validation values array
  in_results = get_ND_input(scenario,aim,case)
  # Get the non-dominated mask
  non_dominated = get_ND_mask(case, scenario)

  return in_results[non_dominated]

def get_ND_indices(case, scenario):
  '''Returns an array with the indices of the all non-dominated solutions'''
  v_c = get_ND_input(scenario,'calibration',case)
  # Get the mask of the non dominated solutions
  c_mask = get_ND_mask(case, scenario)
  # Get the list of the indices of the non dominated solutions
  nd_indices = np.array(range(len(v_c)))[c_mask]

  return nd_indices

def getWeights(solutions=None):
  ''' Returns an array with weights for all solutions or selected solutions only
      Selected solutions are identified by the indices
      Array size: number of drivers x number of solutions'''
  # Get the paramter sets weights
  parameterSets = getParameterConfigurations()
  # Selec indices if no specific points are requested
  if solutions is None:
    solutions = range(len(parameterSets))
  # Get the drivers
  drivers = parameters.getSuitFactorDict()[1]
  # Create an array to store the weights
  weights = np.zeros((len(drivers), len(solutions)))
  # Fill the data for each point
  for i, col in enumerate(weights.T):
    weights[:,i] = parameterSets[solutions[i]]

  return weights

def getAverageWeights(indices=None):
  ''' Returns the average importance (weight) of a given urban growth driver
      for all solutions or selected solutions only '''
  # Get the drivers
  drivers = parameters.getSuitFactorDict()[1]
  # Get weights:
  weights = getWeights(indices)
  # Create a list with the average weights
  av_weights = np.average(weights, axis=1)

  return av_weights

def get_ND_n_masks(case, scenario, trade_off = False, eps=0.9):
  """
  Create a boolean mask of the selected n solutions selected from the non dominated points
  :param nd_results: An (n_points, costs) array of non-dominated solutions
  :return:
    a mask for n points for of each objective, points in order of objectives
  """
  # Get the results values
  in_results = get_ND(case, scenario, aim='calibration')
  # Create an array to store condition boolean arrays
  c_list =[]
  # Save a boolean list of lists masking the minimum errors from ND domain
  for i, col in enumerate(in_results.T):
    # Get the masking list
    b_mask = in_results[:,i] == in_results.min(axis=0)[i]
    # If more than one solution provides minimum error, select randomly
    if sum(b_mask) > 1:
      # Get indexes of true values
      true_i = [i for i, x in enumerate(b_mask) if x]
      # Select one random index
      random_i = np.random.choice(true_i)
      # Leave only the random true value
      random_b_mask = [False for i in b_mask]
      # Change one value to True
      random_b_mask[random_i] = True
      # Append the list of masks
      c_list.append(random_b_mask)
    else:
      #append the list of masks
      c_list.append(b_mask)
  # Change into an array
  c_array = np.array(c_list)

  return c_array

def get_ND_n_indices(case, scenario):
  # Get non dominated indices
  indices_nd = get_ND_indices(case, scenario)
  # Get the n solutions mask array
  n_mask_array = get_ND_n_masks(case, scenario)
  # Subset indices of the n solutions, keeping the order of n objectives
  n_indices = np.array([int(indices_nd[condition]) for condition in n_mask_array])

  return n_indices

def get_av_mask(case, scenario, solution_space, objectives):
  ''' Return a mask of the 1 solution, selected as the closest to the average structure of the
      n objectives or all non-dominated solutions.
      This solution is selected from the all posible solutions or the non-dominated solutions only

      objectives in ['n_objectives','nd_solutions']
      solution_space in ['all', 'nondominated'] '''
  # Get the indices of the selected solutions to find the averaged structure
  the_indices = {
    'n_objectives': get_ND_n_indices(case, scenario),
    'nd_solutions': get_ND_indices(case, scenario)
    }
  # Get the average value of the weights
  n_weights_av = getAverageWeights(the_indices[objectives])
  # Get the weights for every possible paramter set
  weights = {
    'all': getWeights(),
    'nondominated': getWeights(get_ND_indices(case, scenario)) }
  # Calculate the distance to the averaged weight for every possible set
  distance_to_averaged = np.array(
    [distance.euclidean(n_weights_av, col) for col in weights[solution_space].T])

  # Find the closest point
  # condition: if in the averaged weights, none has a zero-weight, this should
  # be the case in the closest point too; i.e. we disregard zero-weights
  if np.min(n_weights_av) > 0.0:
    ##print('weights', weights[solution_space])
    condition = (weights[solution_space] < 0.05)
    idx = condition.any(axis=0)
    if np.any(idx):
      ##print(idx)
      ##print(weights[solution_space][:, ~idx])
      ##print(distance_to_averaged.min())
      ##print(distance_to_averaged[~idx].min())
      ##print(distance_to_averaged == np.amin(distance_to_averaged))
      ##print(np.logical_and(distance_to_averaged == np.amin(distance_to_averaged[~idx]), ~idx))
      theMask = np.logical_and(distance_to_averaged == np.amin(distance_to_averaged[~idx]), ~idx)
    else:
      theMask = distance_to_averaged==distance_to_averaged.min()
  else:
    theMask = distance_to_averaged==distance_to_averaged.min()
  return theMask

def get_av_index(case, scenario, solution_space, objectives):
  ''' Returns index of the 1 solution, selected as the closest to the average structure of the
      n objectives or all non-dominated solutions.
      This solution is selected from the all posible solutions or the non-dominated solutions only

      objectives in ['n_objectives','nd_solutions']
      solution_space in ['all', 'nondominated']'''
  # Get the indices from the solutions space
  indices_space = {
    'all': range(len(getParameterConfigurations())),
    'nondominated': get_ND_indices(case, scenario) }
  # Get the mask with the solution
  aMask = get_av_mask(case, scenario, solution_space, objectives)
  # Get the inbdex
  av_index = np.array(indices_space[solution_space])[aMask]

  return av_index

def get_ND_n_1_indices(case, scenario, solution_space, objectives):
  ''' Returns an array with the n+1 indices for n objectives from the non-dominated solutions
      solution_space in ['all', 'nondominated'] <- point to be selected from
      objectives in ['n_objectives','nd_solutions']'''
  # Get the indices of the n objectives
  nd_n_indices = get_ND_n_indices(case, scenario)
  # Get the index of the multiobjective solution
  av_index = get_av_index(case, scenario, solution_space, objectives)
  # If there is more than one closest point, randomly select one only
  if len(av_index) > 0:
    ##print('points', av_index)
    av_index = [np.random.choice(av_index)]
  # Get the array of the selected indices
  nd_n_1_indices = np.append(nd_n_indices, av_index)

  return nd_n_1_indices

def get_ND_n_1_masks(case, scenario, solution_space, objectives): # to mask the non-dominated solutions!
  ''' Create a boolean masks of the selected n+1 solutions selected from the solution space
      mask size = len(non dominated solutions)
      solution_space in ['all', 'nondominated']
      objectives in ['n_objectives','nd_solutions'] <-- parameter space for multi-objective optimisation'''
  # Get all indices possible
  indices_all = np.arange(len(getParameterConfigurations()))
  # Get the indices of n+1 objectives
  indices_n_1 = get_ND_n_1_indices(case, scenario, solution_space, objectives)
  # Create an array of masks
  masks = np.array([aidx == indices_all for aidx in indices_n_1])
  # Create one mask, without the order of points
  mask = np.array([ np.any(col) for col in masks.T ])
  # Subset non dominated solutions only. Results array has shape = non dominated solutions
  c_array = mask[get_ND_mask(case, scenario)]

  return c_array

def get_ND_n_1(case, scenario, aim, solution_space, objectives):
  '''
  Returns n+1 points from the results, keeping their order in line with the objectives
  (n single-objective and 1 multi-objective)

  Multi-objective approach parameters:
  solution_space: selection of points closest to the 'ideal' point in ['all', 'nondominated']
  objectives: recquirement for selecting the 'ideal' point in ['n_objectives','nd_solutions']
  '''

  # Get results depending on aim
  results_nd = get_ND_input(scenario,aim,case)
  # Get the indices of the n+1 solutions
  idx_n_1 = get_ND_n_1_indices(case, scenario, solution_space, objectives)
  # Subset points
  results_nd_n_1 = np.array([results_nd[i] for i in idx_n_1])

  return results_nd_n_1


import datetime
import threading

from simvestr import create_app
from simvestr import update_portfolio
from simvestr.helpers.utils import get_delay, load_yaml_config

if __name__ == '__main__':
    config_yml = load_yaml_config()
    delay = get_delay()

    app = create_app()

    update_config = dict(
        duration=datetime.timedelta(**config_yml["UPDATE_DURATION"]),
        interval=datetime.timedelta(**config_yml["UPDATE_INTERVAL"]),
        app=app
    )

    daily_update_thread = threading.Timer(
        interval=delay,
        function=update_portfolio,
        kwargs=update_config,
    )

    daily_update_thread.start()


from django.dispatch import receiver
from django.db.models.signals import pre_save, pre_delete
from .models import OrderItem
from basket.models import Basket

@receiver(pre_save, sender=OrderItem)
@receiver(pre_save, sender=Basket)
def product_quantity_update_save(sender, update_fields, instance, **kwargs):
   if update_fields is 'quantity' or 'product':
       if instance.pk:
           instance.product.quantity -= instance.quantity - \
                                        sender.get_item(instance.pk).quantity
       else:
           instance.product.quantity -= instance.quantity
       instance.product.save()

@receiver(pre_delete, sender=OrderItem)
@receiver(pre_delete, sender=Basket)
def product_quantity_update_delete(sender, instance, **kwargs):
   instance.product.quantity += instance.quantity
   instance.product.save()


from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.neural_network import MLPClassifier, MLPRegressor
from sklearn.ensemble import HistGradientBoostingRegressor
from sklearn import linear_model
from sklearn.svm import SVR
from sklearn.metrics import r2_score
import statsmodels.formula.api as sm
import statsmodels.formula.api as smf
import numpy as np
from sklearn.utils import resample

class SCM:
    def __init__(self, X, S, A, Y, \
                X_cat_columns, X_cont_columns, X_count_columns, \
                S_type, A_type, Y_type, rmse_noise_add_to_reg=True, \
                    duplicate_exogenous=False, bootstrap_exogenous=False):
        self.X_cat_columns = X_cat_columns
        self.X_cont_columns = X_cont_columns
        self.X_count_columns = X_count_columns
        self.A_type = A_type
        self.S_type = S_type
        self.Y_type = Y_type
        self.rmse_noise_add_to_reg = rmse_noise_add_to_reg
        self.duplicate_exogenous = duplicate_exogenous
        self.bootstrap_exogenous = bootstrap_exogenous
        self.X = X
        self.S = S
        self.A = A
        self.Y = Y

        self.fitted_SCM = {}
        self.fit_SCM()

    def fit_target(self, data, target, var_type, var_name):
        # wrapper function, fit model based on target variable type
        if var_type == 'cat':
            fit_fn = self.fit_catgoritcal_target(data, target, var_name)
        elif var_type == 'cont':
            fit_fn = self.fit_continuos_target(data, target, var_name)
        elif var_type == 'count':
            fit_fn = self.fit_count_target(data, target, var_name)
        return fit_fn

    def fit_catgoritcal_target(self, data, target, var_name):
        # generic helper function, fit categorical target with a Decision Tree Classifier
        # input: data and target to fit
        # output: fitted model
        fit_fn = DecisionTreeClassifier(random_state=42, max_depth=12)
        fit_fn.fit(data, target)
        self.fitted_SCM[var_name+'_fn'] = fit_fn

    def fit_continuos_target(self, data, target, var_name):
        # generic helper function, fit continuous target with a Decision Tree Regressor
        # input: data and target to fit
        # output: fitted model
        fit_fn = DecisionTreeRegressor(random_state=42, max_depth=12)
        fit_fn.fit(data, target)
        predict = fit_fn.predict(data)
        resid = target-predict
        rmse = np.sqrt(np.mean(resid ** 2))
        r2 = r2_score(target, predict)
        print("r2 score {}: {}".format(var_name, r2))
        if r2 == 1:
            print("rmse: ", rmse)
        self.fitted_SCM[var_name+'_fn'] = fit_fn
        self.fitted_SCM[var_name+'_rmse'] = rmse

    def fit_count_target(self, data, target, var_name):
        # generic helper function, fit continuous target with a Support Vector Regression
        # input: data and target to fit
        # output: fitted model
        fit_fn = HistGradientBoostingRegressor(loss="poisson", random_state=42, max_leaf_nodes=128)
        fit_fn.fit(data, target)
        predict = fit_fn.predict(data)
        resid = target-predict
        rmse = np.sqrt(np.mean(resid ** 2))
        r2 = r2_score(target, predict)
        print("r2 score {}: {}".format(var_name, r2))
        self.fitted_SCM[var_name+'_fn'] = fit_fn
        self.fitted_SCM[var_name+'_rmse'] = rmse
        self.fitted_SCM[var_name+'_mean'] = np.mean(target)

    def sample_categorical_target(self, data, variable_name):
        predict_proba_variable = self.fitted_SCM[variable_name+'_fn'].predict_proba(data)
        variable_SCM = np.vstack([np.random.choice(predict_proba_variable.shape[1],
                            size=(1, 1), p=predict_proba_variable[row, :])
                            for row in range(predict_proba_variable.shape[0])])

        return variable_SCM

    def sample_continuous_target(self, data, variable_name):
        print("variable_name from cont sample: ", variable_name)
        prediction = self.fitted_SCM[variable_name+'_fn'].predict(data)
        if self.rmse_noise_add_to_reg:
            if 'count' in variable_name:
                data_adaptive_noise = \
                    np.random.poisson(lam=self.fitted_SCM[variable_name+'_rmse'],
                                                        size=prediction.shape)
            else:
                data_adaptive_noise = \
                    np.random.normal(scale=self.fitted_SCM[variable_name+'_rmse'],
                                                    size=prediction.shape)
            variable_SCM = (prediction + data_adaptive_noise).reshape(-1, 1)
        else:
            variable_SCM = prediction.reshape(-1, 1)
        return variable_SCM

    def sample_target(self, data, var_type, variable_name):
        if (var_type == 'cat'):
            sampled_target = self.sample_categorical_target(data, variable_name)
        elif (var_type == 'cont') or (var_type == 'count'):
            sampled_target = self.sample_continuous_target(data, variable_name)
        else:
            raise NotImplementedError
        return sampled_target

    def fit_SCM(self):
        np.random.seed(42)
        S_reshaped = np.array(self.S).reshape(-1, 1)
        A_reshaped = np.array(self.A).reshape(-1, 1)
        assert np.all(np.vstack((self.S))==S_reshaped)
        if (not self.duplicate_exogenous) and (not self.bootstrap_exogenous):
            # fit X_categorical (col by col)
            [self.fit_target(S_reshaped,
                self.X[self.X_cat_columns].iloc[:, col], 'cat', 'X_cat_'+str(col))
                for col in range(len(self.X_cat_columns))]

            # fit X_continuous (col by col)
            [self.fit_target(S_reshaped,
                self.X[self.X_cont_columns].iloc[:, col], 'cont', 'X_cont_'+str(col))
                for col in range(len(self.X_cont_columns))]

            # fit X_count (col by col)
            [self.fit_target(S_reshaped,
                self.X[self.X_count_columns].iloc[:, col], 'count', 'X_count_'+str(col))
                for col in range(len(self.X_count_columns))]

        # fit A
        self.fit_target(np.hstack((S_reshaped, self.X)), self.A, self.A_type, 'A')
        # fit Y
        self.fit_target(np.hstack((S_reshaped, self.X, A_reshaped)), self.Y, self.Y_type, 'Y')

    def exogenous_generate(self, n):
        if self.duplicate_exogenous:
            times_to_dup = int(n/len(self.S))
            dup_S = np.array(self.S)
            dup_S = np.tile(dup_S, times_to_dup).reshape(-1, 1)

            dup_X = np.array(self.X)
            dup_X = np.tile(dup_X, (times_to_dup, 1))

            return dup_S, dup_X
        elif self.bootstrap_exogenous:
            np_S = np.array(self.S)
            boot_S = resample(np_S, replace=True, n_samples=n, random_state=42).reshape(-1, 1)

            np_X = np.array(self.X)
            boot_X = resample(np_X, replace=True, n_samples=n, random_state=42)

            return boot_S, boot_X
        else:
            if self.S_type == 'cat':
                # check if binary sensitive attribute
                if np.all(np.unique(np.array(self.S)) == [0,1]):
                    num_1s = np.sum(np.array(self.S))
                    prob_1 = num_1s/len(self.S)
                    sampled_S = np.random.binomial(1, prob_1, size=n)
            else:
                raise NotImplementedError

            return sampled_S

    def sample_SCM(self, n):
        if (self.duplicate_exogenous) or (self.bootstrap_exogenous):
            sampled_S, sampled_X = self.exogenous_generate(n)
        else:
            # sample S
            sampled_S = self.exogenous_generate(n)
            sampled_S = sampled_S.reshape(-1, 1)
            # sample X cat
            sampled_X_cat = \
                np.hstack([self.sample_target(sampled_S, 'cat', 'X_cat_'+str(col))
                                            for col in range(len(self.X_cat_columns))])
            assert sampled_X_cat.shape==(n, len(self.X_cat_columns))
            # sample X cont
            samples_cont = [self.sample_target(sampled_S, 'cont', 'X_cont_'+str(col))
                                            for col in range(len(self.X_cont_columns))]
            sampled_X_cont = \
                np.hstack(samples_cont)
            assert sampled_X_cont.shape==(n, len(self.X_cont_columns))
            # sample X count
            samples_count = [self.sample_target(sampled_S, 'count', 'X_count_'+str(col))
                                            for col in range(len(self.X_count_columns))]
            sampled_X_count = \
                np.hstack(samples_count)
            assert sampled_X_count.shape==(n, len(self.X_count_columns))

            sampled_X = np.hstack((sampled_X_cat, sampled_X_cont, sampled_X_count))

        # sample A
        sampled_A = \
            self.sample_target(
                np.hstack((sampled_S, sampled_X)), self.A_type, 'A')
        # find A_col size
        if len(self.A.shape)==1:
            A_size = 1
        else:
            A_size = self.A.shape[1]
        assert sampled_A.shape==(n, A_size)
        # sample Y
        sampled_Y = \
            self.sample_target(
                np.hstack((sampled_S, sampled_X, sampled_A)), self.Y_type, 'Y')
        # find Y_col size
        if len(self.Y.shape)==1:
            Y_size = 1
        else:
            Y_size = self.Y.shape[1]
        assert sampled_Y.shape==(n, Y_size)

        if (self.duplicate_exogenous) or (self.bootstrap_exogenous):
            return sampled_S, sampled_X, sampled_A, sampled_Y
        else:
            return sampled_S, sampled_X_cat, sampled_X_cont, sampled_X_count, sampled_A, sampled_Y


from argparse import REMAINDER, ArgumentParser
from typing import Any, List, Callable, Optional

from .command import (
    handle_hook,
    handle_list,
    handle_commit,
    handle_search,
    handle_no_subcommand,
)

parser = ArgumentParser("nonemoji", description="Simple gitmoji cli written in python")
parser.set_defaults(func=handle_no_subcommand)
_subparsers = parser.add_subparsers(title="subcommands")

# hook
hook_parser = _subparsers.add_parser("hook", help="Run as a git hook.")
hook_parser.add_argument("commit_msg_file", help="Path to the commit message file")
hook_parser.set_defaults(func=handle_hook)

# commit
commit_parser = _subparsers.add_parser("commit", help="Start to commit changes.")
commit_parser.add_argument("-e", "--emoji", help="Emoji to use")
commit_parser.add_argument("-m", "--message", help="Commit message")
commit_parser.add_argument(
    "--", nargs=REMAINDER, help="Argument to pass to the git", dest="args"
)
commit_parser.set_defaults(func=handle_commit)

# list
list_parser = _subparsers.add_parser("list", help="List all available emoji.")
list_parser.set_defaults(func=handle_list)

# search
search_parser = _subparsers.add_parser("search", help="Search for an emoji.")
search_parser.set_defaults(func=handle_search)

def main(argv: Optional[List[str]] = None):
    result = parser.parse_args(argv)
    result = vars(result)
    func: Callable[..., Any] = result.pop("func")
    func(**result)


import logging as log
import sys, os
from telegram.ext import Application, CommandHandler, MessageHandler, CallbackQueryHandler, filters

def main():
    import handlers

    #Init bot
    TOKEN = os.getenv("TELEGRAM_BOT_TOKEN_PROD")
    bot = Application.builder().token(TOKEN).build()

    # Init input handlers
    bot.add_handler(CommandHandler("start", handlers.start))
    bot.add_handler(CommandHandler("empezar", handlers.start))
    bot.add_handler(CallbackQueryHandler(handlers.InlineKeyboardHandler))
    bot.add_handler(MessageHandler(filters.TEXT, handlers.text_input_handler))
    bot.add_handler(MessageHandler(filters.LOCATION, handlers.location_input_handler))

    bot.run_polling()

if __name__ == '__main__':

    # Init and config logger
    log.basicConfig(
        stream=sys.stdout,
        format='%(asctime)s %(levelname)-8s %(message)s',
        level=log.INFO,
        datefmt='%Y-%m-%d %H:%M:%S'
    )
    main()


import json

from .database import Database

QUERIES = {
    'select_row': 'SELECT * FROM user_status WHERE user_id = ?',
    'update_status': 'UPDATE user_status SET status = ?, data = ? WHERE user_id = ?',
    'insert_status': 'INSERT INTO user_status(user_id) SELECT ? WHERE (SELECT Changes() = 0)'
}

class UserStatusTable:
    """
    Class for manipulation of the 'user_status' table in the database.
    """

    def __enter__(self):
        self._connect = Database.open()
        self._cursor = self._connect.cursor()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self._connect.close()

    def _select_row(self, user_id):
        row = self._cursor.execute(QUERIES['select_row'], [user_id]).fetchone()
        return row

    def _update_status(self, user_id, status, data=None):
        self._cursor.execute(QUERIES['update_status'], [status, data, user_id])
        self._cursor.execute(QUERIES['insert_status'], [user_id])

    def status(self, user_id):
        """
        ???
        :param user_id:
        :return:
        """
        row = self._select_row(user_id)
        if row is None:
            return 0, None
        try:
            data = json.loads(row['data'])
        except ValueError:
            data = None
        return int(row['status']), data

    def update(self, user_id, status, data=None):
        """
        ???
        :param user_id:
        :param status:
        :param data:
        :return:
        """
        dump = json.dumps(data)
        with self._connect:
            self._update_status(user_id, status, dump)


"""
Author: Jasjit Rudra
Date: 03 May 2023
Description: (Mean Time to Recover) Time it takes for a service to bounce back from failure.
Note: JQL Query can be edited to find any ticket (not just bugfixes) details.
"""

import json
import requests
from datetime import datetime
import matplotlib.pyplot as plt

jira_server = "https://jira.xxx.com"
jira_url = "https://jira.xxx.com/rest/api/3/issue/"
jira_token = "YOUR JIRA TOKEN HERE"

headers = {
    "Accept": "application/json",
    "Authorization": f"Bearer {jira_token}"
}

def cycle_time(jira_project, start_date, end_date):
    jql_query = f"(issuetype = Bug OR issuetype = Defect) AND status in (Accepted, Completed, Closed, Verified) AND resolved >= '{start_date}' AND resolved <= '{end_date}' and project = '{jira_project}'"
    response = requests.get(
        url=f"{jira_server}/rest/api/latest/search",
        headers=headers,
        params={
            'jql': jql_query,
            'expand': 'changelog'
        }
    )

    response_json = json.loads(response.text)
    issues = response_json["issues"]
    if not issues:
        return "There are no bugs resolved within the given time frame"

    total_time = 0
    x_axis = []
    y_axis = []
    for issue in reversed(issues):
        created_date_str = issue["fields"]["created"]
        resolved_date_str = issue["fields"]["resolutiondate"]

        created_date = datetime.strptime(created_date_str[:19], "%Y-%m-%dT%H:%M:%S")
        x_axis.append(f"{created_date.month} / {created_date.day}")
        resolved_date = datetime.strptime(resolved_date_str[:19], "%Y-%m-%dT%H:%M:%S")
        diff_days = (resolved_date - created_date).days
        y_axis.append(diff_days)
        print(f"Issue {issue['key']} resolved in {diff_days} days.")
        total_time += diff_days

    # Calculate the average time taken to resolve an issue
    num_issues = len(issues)
    if num_issues > 0:
        avg_time = total_time / num_issues
        plt.switch_backend('Agg')
        plt.plot(x_axis, y_axis)
        plt.title(f"Mean time to Recover = {round(avg_time)} days")
        plt.xlabel("Timeline (of issues discovered)", fontsize=10)
        plt.ylabel("Days Taken to Resolve", fontsize=10)
        plt.rc('axes', labelsize=5)

        plt.savefig(fname='static/images/mean_time_to_recover.png')
        print(f"Average time to resolve an issue: {avg_time} days.")
        return f"Average time taken to resolve defect: {avg_time} days"


"""Classes representing when medicine needs to be taken and refilled."""

import pickle
import random
from dataclasses import dataclass

import arrow

import db
from models.time import TimeOfDay

@dataclass
class Schedule:
    """A schedule for taking medicine for morning, afternoon, and evening."""

    morning_medication: list
    afternoon_medication: list
    evening_medication: list
    patient_id: int = None
    id: int = random.Random(10000)

    def get_morning_medication(self):
        """Get the morning medication."""
        return self.morning_medication

    def get_afternoon_medication(self):
        """Get the afternoon medication."""
        return self.afternoon_medication

    def get_evening_medication(self):
        """Get the evening medication."""
        return self.evening_medication

    def get_all_medication(self):
        """Get all medication."""
        return (
            self.morning_medication
            + self.afternoon_medication
            + self.evening_medication
        )

    def get_medication_by_time_of_day(self, time_of_day):

        if time_of_day == TimeOfDay.MORNING:
            return self.get_morning_medication()
        elif time_of_day == TimeOfDay.AFTERNOON:
            return self.get_afternoon_medication()
        elif time_of_day == TimeOfDay.EVENING:
            return self.get_evening_medication()
        else:
            return None

    def check_for_refills(self):
        """Check for refills."""
        for medication in self.get_all_medication():
            if medication.calculate_days_left() < 7:
                medication.send_refill_text()

    def save(self):
        """Save the schedule."""
        # pickle the medication lists
        pickled_morning_medication = pickle.dumps(self.morning_medication)
        pickled_afternoon_medication = pickle.dumps(self.afternoon_medication)
        picked_evening_medication = pickle.dumps(self.evening_medication)
        try:
            self.id = db.insert(
                "schedules",
                morning_medication=pickled_morning_medication,
                afternoon_medication=pickled_afternoon_medication,
                evening_medication=picked_evening_medication,
                patient_id=self.patient_id,
            )
            return True
        except Exception as e:
            print(e)
            print("There was an error saving the schedule.")
            return False

    def load(self, id):
        """Load the schedule."""
        schedule = db.get("schedules", id)
        # unpickle the medication lists
        self.morning_medication = pickle.loads(schedule["morning_medication"])
        self.afternoon_medication = pickle.loads(schedule["afternoon_medication"])
        self.evening_medication = pickle.loads(schedule["evening_medication"])
        self.patient_id = schedule["patient_id"]
        self.id = schedule["id"]
        return self.__dict__


from bs4 import BeautifulSoup as bs
from splinter import Browser
import pandas as pd
import datetime as dt

def scrape_all():

    browser = Browser("chrome")

    # Get these variable values from function
    news_title, news_p = mars_news(browser)

    # Run all scraping functions and store in dictionary.
    data = {
        "news_title": news_title,
        "news_paragraph": news_p,
        "featured_image": featured_image(browser),
        "hemispheres": hemispheres(browser),
        "weather": mars_weather(browser),
        "facts": mars_facts(),
        "last_modified": dt.datetime.now()
    }

    # Stop webdriver and return data
    browser.quit()
    return data

def mars_news(browser):
    browser.visit('https://mars.nasa.gov/news/')

    # Parse it for news title and news paragraph using Beautiful Soup
    html = browser.html
    news_soup = bs(html, 'html.parser')
    news_title = news_soup.find('div', class_ = 'content_title').get_text()
    news_p = news_soup.find('div', class_ = 'article_teaser_body').get_text()

    return news_title, news_p

def featured_image(browser):

    # Visit site
    browser.visit('https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars')

    # Navigate to the correct page with info using splinter browser
    full_image = browser.find_by_id('full_image')
    full_image.click()

    browser.is_element_present_by_text("more info", wait_time=1)
    more_info = browser.find_link_by_partial_text('more info')
    more_info.click()

    # Parse it for image using Beautiful Soup
    html = browser.html
    fi_soup = bs(html, 'html.parser')
    featured_image = fi_soup.find('img', class_ = 'main_image')['src']
    featured_image_url = f'https://jpl.nasa.gov{featured_image}'

    return featured_image_url

def mars_weather(browser):

    # Visit site
    browser.visit('https://twitter.com/marswxreport?lang=en')

    # Parse it for first tweet's content
    html = browser.html
    weather_soup = bs(html, 'html.parser')
    mars_weather = weather_soup.find(class_='TweetTextSize TweetTextSize--normal js-tweet-text tweet-text').get_text()
    mars_weather

    return mars_weather

def mars_facts():
    # Read the webpage into pandas and grab the second table
    df = pd.read_html('https://space-facts.com/mars/')[1]

    # Rename columns
    df.columns = ['Description', 'Value']

    df.set_index('Description')

    # Add some bootstrap styling to <table>
    return df.to_html(classes="table table-striped")

def hemispheres(browser):

    browser.visit('https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars')

    # Set up empty list for storing dictionaries
    hemisphere_image_urls = []

    # Set up loop to go through each hemisphere page
    for i in range(4):

        # empty dictionary to hold url and title to be scraped
        hemisphere = {}

        # Click on the next link in the loop
        browser.find_by_css('a.itemLink.product-item h3')[i].click()

        # Grab the href link and store it in the dictionary
        hemisphere['img_url'] = browser.find_link_by_text('Sample').first['href']

        # Grab the title and store it in the dictionary
        hemisphere['title'] = browser.find_by_css('h2.title').text

        # Append the dictionary to the list
        hemisphere_image_urls.append(hemisphere)

        # Go back in order to repeat the loop
        browser.back()

    return hemisphere_image_urls

if __name__ == "__main__":

    # If running as script, print scraped data
    print(scrape_all())