from collections import defaultdict
from statistics import mean
from typing import Dict, List, Literal, Optional, Set, Tuple, Union

import torch
from cohesion_tools.evaluation import CohesionScore, CohesionScorer
from cohesion_tools.extractors import PasExtractor
from rhoknp import BasePhrase, Document, Morpheme, Phrase, Sentence
from rhoknp.props import DepType, FeatureDict, MemoTag
from seqeval.metrics import f1_score as seqeval_f1_score
from seqeval.scheme import IOB2
from sklearn.metrics import accuracy_score, f1_score, precision_recall_fscore_support

from kwja.callbacks.utils import (  # add_discourse,
    add_base_phrase_features,
    add_cohesion,
    add_dependency,
    add_named_entities,
    build_morphemes,
    chunk_morphemes,
    get_morpheme_attribute_predictions,
)
from kwja.datamodule.datasets import WordDataset
from kwja.metrics.base import BaseModuleMetric
from kwja.metrics.conll18_ud_eval import main as conll18_ud_eval
from kwja.metrics.utils import unique
from kwja.utils.constants import (
    BASE_PHRASE_FEATURES,
    CONJFORM_TAGS,
    CONJTYPE_TAGS,
    DISCOURSE_RELATIONS,
    IGNORE_INDEX,
    IGNORE_VALUE_FEATURE_PAT,
    MASKED,
    POS_TAGS,
    SUBPOS_TAGS,
    WORD_FEATURES,
    CohesionTask,
    WordTask,
)
from kwja.utils.reading_prediction import get_word_level_readings
from kwja.utils.sub_document import extract_target_sentences, to_orig_doc_id

class WordModuleMetric(BaseModuleMetric):
    STATE_NAMES = (
        "example_ids",
        "reading_predictions",
        "reading_subword_map",
        "pos_logits",
        "subpos_logits",
        "conjtype_logits",
        "conjform_logits",
        "word_feature_probabilities",
        "ne_predictions",
        "base_phrase_feature_probabilities",
        "dependency_predictions",
        "dependency_type_predictions",
        "cohesion_logits",
        "discourse_predictions",
        "discourse_labels",
    )

    def __init__(self, max_seq_length: int) -> None:
        super().__init__(max_seq_length)
        self.dataset: Optional[WordDataset] = None
        self.reading_id2reading: Optional[Dict[int, str]] = None
        self.training_tasks: Optional[List[WordTask]] = None

        self.example_ids: torch.Tensor
        self.reading_predictions: torch.Tensor
        self.reading_subword_map: torch.Tensor
        self.pos_logits: torch.Tensor
        self.subpos_logits: torch.Tensor
        self.conjtype_logits: torch.Tensor
        self.conjform_logits: torch.Tensor
        self.word_feature_probabilities: torch.Tensor
        self.ne_predictions: torch.Tensor
        self.base_phrase_feature_probabilities: torch.Tensor
        self.dependency_predictions: torch.Tensor
        self.dependency_type_predictions: torch.Tensor
        self.cohesion_logits: torch.Tensor
        self.discourse_predictions: torch.Tensor
        self.discourse_labels: torch.Tensor

    def _pad(self, kwargs: Dict[str, torch.Tensor]) -> None:
        for key, value in kwargs.items():
            if key in {"example_ids"} or value.numel() == 0:
                continue
            elif key in {"reading_subword_map", "cohesion_logits", "discourse_predictions", "discourse_labels"}:
                dims = [value.ndim - 2, value.ndim - 1]
            else:
                dims = [1]
            for dim in dims:
                size = [self.max_seq_length - s if i == dim else s for i, s in enumerate(value.size())]
                if key in {"discourse_labels"}:
                    fill_value: Union[float, int] = IGNORE_INDEX
                else:
                    fill_value = MASKED if torch.is_floating_point(value) else 0
                padding = torch.full(size, fill_value, dtype=value.dtype, device=value.device)
                value = torch.cat([value, padding], dim=dim)
            kwargs[key] = value

    def compute(self) -> Dict[str, float]:
        assert self.training_tasks is not None, "training_tasks isn't set"

        if isinstance(self.example_ids, torch.Tensor) is False:
            self.example_ids = torch.cat(self.example_ids, dim=0)  # type: ignore

        sorted_indices = unique(self.example_ids)
        for state_name in self.STATE_NAMES:
            state = getattr(self, state_name)
            if isinstance(state, torch.Tensor) is False:
                state = torch.cat(state, dim=0)
            if state_name == "reading_subword_map":
                state = state.bool()
            setattr(self, state_name, state[sorted_indices])

        predicted_documents, partly_gold_document1, partly_gold_document2, gold_documents = self._build_documents()

        metrics: Dict[str, float] = {}
        if WordTask.READING_PREDICTION in self.training_tasks:
            metrics.update(self.compute_reading_prediction_metrics(predicted_documents, gold_documents))
        if WordTask.MORPHOLOGICAL_ANALYSIS in self.training_tasks:
            metrics.update(self.compute_morphological_analysis_metrics(predicted_documents, gold_documents))
        if WordTask.WORD_FEATURE_TAGGING in self.training_tasks:
            metrics.update(self.compute_word_feature_tagging_metrics(predicted_documents, gold_documents))
        if WordTask.NER in self.training_tasks:
            metrics.update(self.compute_ner_metrics(partly_gold_document1, gold_documents))
        if WordTask.BASE_PHRASE_FEATURE_TAGGING in self.training_tasks:
            metrics.update(self.compute_base_phrase_feature_tagging_metrics(partly_gold_document1, gold_documents))
        if WordTask.DEPENDENCY_PARSING in self.training_tasks:
            metrics.update(self.compute_dependency_parsing_metrics(partly_gold_document1, gold_documents))
        if WordTask.COHESION_ANALYSIS in self.training_tasks:
            metrics.update(self.compute_cohesion_analysis_metrics(partly_gold_document2, gold_documents))
        if WordTask.DISCOURSE_RELATION_ANALYSIS in self.training_tasks:
            predictions = self.discourse_predictions.view(-1)
            labels = self.discourse_labels.view(-1)
            metrics.update(self.compute_discourse_relation_analysis_metrics(predictions, labels))
        return metrics

    def _build_documents(self) -> Tuple[List[Document], ...]:
        assert self.dataset is not None, "dataset isn't set"
        assert self.reading_id2reading is not None, "reading_id2reading isn't set"

        doc_id2predicted_sentences: Dict[str, List[Sentence]] = defaultdict(list)
        doc_id2partly_gold_sentences1: Dict[str, List[Sentence]] = defaultdict(list)
        doc_id2partly_gold_sentences2: Dict[str, List[Sentence]] = defaultdict(list)
        doc_id2gold_sentences: Dict[str, List[Sentence]] = defaultdict(list)
        for (
            example_id,
            reading_predictions,
            reading_subword_map,
            *morpheme_attribute_logits,  # pos_logits, subpos_logits, conjtype_logits, conjform_logits
            word_feature_probabilities,
            ne_predictions,
            base_phrase_feature_probabilities,
            dependency_predictions,
            dependency_type_predictions,
            cohesion_logits,
            _,  # discourse_predictions
            _,  # discourse_labels
        ) in zip(*[getattr(self, state_name).tolist() for state_name in self.STATE_NAMES]):
            example = self.dataset.examples[example_id]
            gold_document = self.dataset.doc_id2document[example.doc_id]
            orig_doc_id = to_orig_doc_id(gold_document.doc_id)

            word_reading_predictions = get_word_level_readings(
                [self.reading_id2reading[reading_id] for reading_id in reading_predictions],
                [self.dataset.tokenizer.decode(input_id) for input_id in example.encoding.ids],
                reading_subword_map,
            )
            morpheme_attribute_predictions = get_morpheme_attribute_predictions(
                *(logits[: len(gold_document.morphemes)] for logits in morpheme_attribute_logits)
            )
            morphemes = build_morphemes(
                [m.surf for m in gold_document.morphemes],
                [m.lemma for m in gold_document.morphemes],
                word_reading_predictions,
                morpheme_attribute_predictions,
            )
            predicted_document = chunk_morphemes(gold_document, morphemes, word_feature_probabilities)
            predicted_document.doc_id = gold_document.doc_id
            for sentence in extract_target_sentences(predicted_document):
                doc_id2predicted_sentences[orig_doc_id].append(sentence)

            # goldの基本句区切り・基本句主辞を使用
            partly_gold_document1 = gold_document.reparse()
            partly_gold_document1.doc_id = gold_document.doc_id
            self.refresh(partly_gold_document1, level=1)
            for sentence in extract_target_sentences(partly_gold_document1):
                add_named_entities(sentence, ne_predictions)
                add_base_phrase_features(sentence, base_phrase_feature_probabilities)
                add_dependency(
                    sentence, dependency_predictions, dependency_type_predictions, example.special_token_indexer
                )
                doc_id2partly_gold_sentences1[orig_doc_id].append(sentence)

            # goldの基本句区切り・基本句主辞・基本句素性を使用
            partly_gold_document2 = gold_document.reparse()
            partly_gold_document2.doc_id = gold_document.doc_id
            self.refresh(partly_gold_document2, level=2)
            add_cohesion(
                partly_gold_document2,
                cohesion_logits,
                self.dataset.cohesion_task2extractor,
                self.dataset.cohesion_task2rels,
                self.dataset.restrict_cohesion_target,
                example.special_token_indexer,
            )
            for sentence in extract_target_sentences(partly_gold_document2):
                doc_id2partly_gold_sentences2[orig_doc_id].append(sentence)

            for sentence in extract_target_sentences(gold_document):
                doc_id2gold_sentences[orig_doc_id].append(sentence)
        predicted_documents = self._convert_doc_id2sentences_into_documents(doc_id2predicted_sentences)
        partly_gold_documents1 = self._convert_doc_id2sentences_into_documents(doc_id2partly_gold_sentences1)
        partly_gold_documents2 = self._convert_doc_id2sentences_into_documents(doc_id2partly_gold_sentences2)
        gold_documents = self._convert_doc_id2sentences_into_documents(doc_id2gold_sentences)
        return predicted_documents, partly_gold_documents1, partly_gold_documents2, gold_documents

    @staticmethod
    def refresh(document: Document, level: int = 1) -> None:
        """Refresh document

        NOTE:
            level1: clear discourse relations, rel tags, dependencies, and base phrase features.
            level2: clear discourse relations and rel tags.
        """
        assert level in (1, 2), f"invalid level: {level}"
        try:
            for clause in document.clauses:
                clause.discourse_relations.clear()
        except AttributeError:
            pass

        for phrase in document.phrases:
            if level == 1:
                phrase.features.clear()
                phrase.parent_index = None
                phrase.dep_type = None

        for base_phrase in document.base_phrases:
            base_phrase.rel_tags.clear()
            base_phrase.memo_tag = MemoTag()
            if level == 1:
                base_phrase.features.clear()
                base_phrase.parent_index = None
                base_phrase.dep_type = None

    @staticmethod
    def _convert_doc_id2sentences_into_documents(doc_id2sentences: Dict[str, List[Sentence]]) -> List[Document]:
        return [Document.from_sentences(ss) for ss in doc_id2sentences.values()]

    @staticmethod
    def compute_reading_prediction_metrics(
        predicted_documents: List[Document], gold_documents: List[Document]
    ) -> Dict[str, float]:
        predictions = [m.reading for d in predicted_documents for m in d.morphemes]
        labels = [m.reading for d in gold_documents for m in d.morphemes]
        return {"reading_prediction_accuracy": accuracy_score(y_true=labels, y_pred=predictions)}

    @staticmethod
    def compute_morphological_analysis_metrics(
        predicted_documents: List[Document], gold_documents: List[Document]
    ) -> Dict[str, float]:
        metrics: Dict[str, float] = {}
        for attribute_name, supported_attribute_values in zip(
            ("pos", "subpos", "conjtype", "conjform"), (POS_TAGS, SUBPOS_TAGS, CONJTYPE_TAGS, CONJFORM_TAGS)
        ):
            metrics[f"{attribute_name}_f1"] = WordModuleMetric._calc_morpheme_attribute_f1_score(
                [m for d in predicted_documents for m in d.morphemes],
                [m for d in gold_documents for m in d.morphemes],
                attribute_name,
                supported_attribute_values,
            )
        metrics["morphological_analysis_f1"] = mean(metrics.values())
        return metrics

    @staticmethod
    def _calc_morpheme_attribute_f1_score(
        predicted_morphemes: List[Morpheme],
        gold_morphemes: List[Morpheme],
        attribute_name: str,
        supported_attribute_values: Tuple[str, ...],
    ) -> float:
        labels: List[str] = []
        for morpheme in gold_morphemes:
            attribute_value = getattr(morpheme, attribute_name)
            labels.append(attribute_value if attribute_value in supported_attribute_values else "UNKNOWN")
        predictions: List[str] = [getattr(morpheme, attribute_name) for morpheme in predicted_morphemes]
        return f1_score(y_true=labels, y_pred=predictions, average="micro").item()

    @staticmethod
    def compute_word_feature_tagging_metrics(
        predicted_documents: List[Document], gold_documents: List[Document]
    ) -> Dict[str, float]:
        metrics: Dict[str, float] = {}
        all_labels = []
        all_predictions = []
        for feature_label in WORD_FEATURES:
            labels: List[List[str]] = []
            predictions: List[List[str]] = []
            gold_document: Document
            predicted_document: Document
            for gold_document, predicted_document in zip(gold_documents, predicted_documents):
                if feature_label == "基本句-区切":
                    label = WordModuleMetric._convert_units_into_segmentation_tags(gold_document.base_phrases)
                    prediction = WordModuleMetric._convert_units_into_segmentation_tags(predicted_document.base_phrases)
                elif feature_label == "文節-区切":
                    label = WordModuleMetric._convert_units_into_segmentation_tags(gold_document.phrases)
                    prediction = WordModuleMetric._convert_units_into_segmentation_tags(predicted_document.phrases)
                else:
                    label = [
                        ("B" if feature_label in WordModuleMetric._convert_features_to_labels(m.features) else "O")
                        for m in gold_document.morphemes
                    ]
                    prediction = [
                        ("B" if feature_label in WordModuleMetric._convert_features_to_labels(m.features) else "O")
                        for m in predicted_document.morphemes
                    ]
                labels.append(label)
                predictions.append(prediction)
            metrics[f"{feature_label}_f1"] = seqeval_f1_score(
                y_true=labels, y_pred=predictions, mode="strict", scheme=IOB2, average="micro"
            ).item()
            all_labels += labels
            all_predictions += predictions
        metrics["macro_word_feature_tagging_f1"] = mean(
            metrics[f"{feature_label}_f1"] for feature_label in WORD_FEATURES
        )
        metrics["micro_word_feature_tagging_f1"] = seqeval_f1_score(
            y_true=all_labels, y_pred=all_predictions, mode="strict", scheme=IOB2, average="micro"
        ).item()
        return metrics

    @staticmethod
    def _convert_units_into_segmentation_tags(units: Union[List[Phrase], List[BasePhrase]]) -> List[str]:
        return ["B" if idx == 0 else "I" for u in units for idx in range(len(u.morphemes))]

    @staticmethod
    def compute_ner_metrics(predicted_documents: List[Document], gold_documents: List[Document]) -> Dict[str, float]:
        for document in predicted_documents + gold_documents:
            for named_entity in document.named_entities:
                for i, morpheme in enumerate(named_entity.morphemes):
                    bi = "B" if i == 0 else "I"
                    morpheme.features["NE"] = f"{bi}-{named_entity.category.value}"
        labels = [[m.features.get("NE") or "O" for m in d.morphemes] for d in gold_documents]
        predictions = [[m.features.get("NE") or "O" for m in d.morphemes] for d in predicted_documents]
        return {
            "ner_f1": seqeval_f1_score(
                y_true=labels, y_pred=predictions, mode="strict", scheme=IOB2, average="micro"
            ).item()
        }

    @staticmethod
    def compute_base_phrase_feature_tagging_metrics(
        partly_gold_documents1: List[Document], gold_documents: List[Document]
    ) -> Dict[str, float]:
        metrics: Dict[str, float] = {}
        all_labels: List[bool] = []
        all_predictions: List[bool] = []
        for feature_label in BASE_PHRASE_FEATURES:
            labels: List[bool] = []
            predictions: List[bool] = []
            gold_document: Document
            predicted_document: Document
            for gold_document, predicted_document in zip(gold_documents, partly_gold_documents1):
                labels += [
                    feature_label in WordModuleMetric._convert_features_to_labels(base_phrase.features)
                    for base_phrase in gold_document.base_phrases
                ]
                predictions += [
                    feature_label in WordModuleMetric._convert_features_to_labels(base_phrase.features)
                    for base_phrase in predicted_document.base_phrases
                ]
            # 正解ラベルがない基本句素性は評価対象外
            if any(labels):
                metrics[f"{feature_label}_f1"] = f1_score(
                    y_true=labels, y_pred=predictions, pos_label=True, average="binary"
                ).item()
            all_labels += labels
            all_predictions += predictions
        metrics["macro_base_phrase_feature_tagging_f1"] = mean(
            metrics[f"{feature_label}_f1"] for feature_label in BASE_PHRASE_FEATURES if f"{feature_label}_f1" in metrics
        )
        metrics["micro_base_phrase_feature_tagging_f1"] = f1_score(
            y_true=all_labels, y_pred=all_predictions, pos_label=True, average="binary"
        ).item()
        return metrics

    @staticmethod
    def _convert_features_to_labels(features: FeatureDict) -> Set[str]:
        # {"用言": "動", "体言": True, "節-機能-原因・理由": "ので"} -> {"用言:動", "体言", "節-機能-原因・理由"}
        return {
            k + (f":{v}" if isinstance(v, str) and IGNORE_VALUE_FEATURE_PAT.match(k) is None else "")
            for k, v in features.items()
            if v not in (None, False)
        }

    @staticmethod
    def compute_dependency_parsing_metrics(
        partly_gold_documents1: List[Document], gold_documents: List[Document]
    ) -> Dict[str, float]:
        metrics: Dict[str, float] = {}
        unit_names: List[Literal["base_phrase", "morpheme"]] = ["base_phrase", "morpheme"]
        for unit_name in unit_names:
            metrics.update(
                {
                    f"{unit_name}_{k}": v
                    for k, v in conll18_ud_eval(
                        *WordModuleMetric._to_conll_lines(partly_gold_documents1, gold_documents, unit_name)
                    ).items()
                }
            )
        return metrics

    @staticmethod
    def _to_conll_lines(
        partly_gold_documents1: List[Document],
        gold_documents: List[Document],
        unit_name: Literal["base_phrase", "morpheme"],
    ) -> Tuple[List[str], List[str]]:
        gold_lines = []
        system_lines = []
        for gold_document, partly_gold_document1 in zip(gold_documents, partly_gold_documents1):
            for sentence in gold_document.sentences:
                gold_lines += [WordModuleMetric._to_conll_line(u) for u in getattr(sentence, f"{unit_name}s")]
                gold_lines.append("\n")
            for sentence in partly_gold_document1.sentences:
                system_lines += [WordModuleMetric._to_conll_line(u) for u in getattr(sentence, f"{unit_name}s")]
                system_lines.append("\n")
        return gold_lines, system_lines

    @staticmethod
    def _to_conll_line(unit: Union[BasePhrase, Morpheme]) -> str:
        id_ = unit.index + 1  # 0-origin -> 1-origin
        if isinstance(unit, BasePhrase):
            form = "".join(m.surf for m in unit.morphemes)
            lemma = "".join(m.lemma for m in unit.morphemes)
            head = unit.parent_index + 1 if unit.parent_index is not None else 0
            deprel = unit.dep_type or DepType.DEPENDENCY
        else:
            assert isinstance(unit, Morpheme)
            form = unit.surf
            lemma = unit.lemma
            head = unit.parent.index + 1 if unit.parent is not None else 0
            if unit == unit.base_phrase.head and unit.base_phrase.dep_type is not None:
                deprel = unit.base_phrase.dep_type
            else:
                deprel = DepType.DEPENDENCY
        upos = xpos = feats = deps = misc = "_"
        return "\t".join(map(str, (id_, form, lemma, upos, xpos, feats, head, deprel, deps, misc)))

    def compute_cohesion_analysis_metrics(
        self, partly_gold_documents2: List[Document], gold_documents: List[Document]
    ) -> Dict[str, float]:
        assert self.dataset is not None, "dataset isn't set"
        pas_extractor = self.dataset.cohesion_task2extractor[CohesionTask.PAS_ANALYSIS]
        assert isinstance(pas_extractor, PasExtractor), "pas utils isn't set correctly"

        scorer = CohesionScorer(
            exophora_referent_types=self.dataset.exophora_referent_types,
            pas_cases=pas_extractor.cases if CohesionTask.PAS_ANALYSIS in self.dataset.cohesion_tasks else [],
            pas_verbal=pas_extractor.verbal_predicate,
            pas_nominal=pas_extractor.nominal_predicate,
            bridging=(CohesionTask.BRIDGING_REFERENCE_RESOLUTION in self.dataset.cohesion_tasks),
            coreference=(CohesionTask.COREFERENCE_RESOLUTION in self.dataset.cohesion_tasks),
        )
        score: CohesionScore = scorer.run(partly_gold_documents2, gold_documents)

        metrics: Dict[str, float] = {}
        for task, analysis2metric in score.to_dict().items():
            for analysis, metric in analysis2metric.items():
                if task not in ("bridging", "coreference"):
                    key = "pas"
                    if task != "all_case":
                        key += f"_{task}"
                else:
                    key = task
                if analysis != "all":
                    key += f"_{analysis}"
                key += "_f1"
                metrics[key] = metric.f1
        cohesion_analysis_f1s = []
        if CohesionTask.PAS_ANALYSIS in self.dataset.cohesion_tasks:
            cohesion_analysis_f1s.append(metrics["pas_f1"])
        if CohesionTask.BRIDGING_REFERENCE_RESOLUTION in self.dataset.cohesion_tasks:
            cohesion_analysis_f1s.append(metrics["bridging_f1"])
        if CohesionTask.COREFERENCE_RESOLUTION in self.dataset.cohesion_tasks:
            cohesion_analysis_f1s.append(metrics["coreference_f1"])
        metrics["cohesion_analysis_f1"] = mean(cohesion_analysis_f1s) if cohesion_analysis_f1s else 0.0
        return metrics

    @staticmethod
    def compute_discourse_relation_analysis_metrics(
        predictions: torch.Tensor,  # (N)
        labels: torch.Tensor,  # (N)
    ) -> Dict[str, float]:
        mask = labels.ne(IGNORE_INDEX)
        predictions = predictions[mask].cpu()
        labels = labels[mask].cpu()

        if labels.numel() == 0:
            accuracy = 0.0
        else:
            accuracy = accuracy_score(y_true=labels, y_pred=predictions)

        precision, recall, f1, _ = precision_recall_fscore_support(
            y_true=labels,
            y_pred=predictions,
            labels=[idx for idx, relation in enumerate(DISCOURSE_RELATIONS) if relation != "談話関係なし"],
            average="micro",
            zero_division=0.0,
        )

        return {
            "discourse_relation_analysis_accuracy": accuracy,
            "discourse_relation_analysis_precision": precision.item(),
            "discourse_relation_analysis_recall": recall.item(),
            "discourse_relation_analysis_f1": f1.item(),
        }


import os
import torch
import torch.nn as nn
import torch.optim as optim

from typing import List
from functools import partial
from ..constants import TORCH_MODELS_DIR

def get_activation(name):
    if type(name) != str:
        return name
    if name == 'relu':
        return nn.ReLU()
    elif name == 'tanh':
        return nn.Tanh()
    elif name == 'sigmoid':
        return nn.Sigmoid()
    else:
        raise Exception('Unknown activation function: ' + name)

def get_optimizer(name, params={}):
    if isinstance(name, optim.Optimizer):
        return name
    if name == 'rmsprop':
        return partial(optim.RMSprop, **dict(params, lr=0.001))
    elif name == 'adam':
        return partial(optim.Adam, **params)
    elif name == 'adadelta':
        return partial(optim.Adadelta, **params)
    elif name == 'adagrad':
        return partial(optim.Adagrad, **dict(params, lr=0.001))
    else:
        raise Exception('Invalid optimizer: ' + str(name))

def atleast_dim(x: torch.Tensor, dim: int, axis=0) -> torch.Tensor:
    while x.dim() < dim:
        x = x.unsqueeze(axis)
    return x

def one_hot(indices, dim) -> torch.Tensor:
    """
    Creates a one-hot tensor from a tensor of integers.
    indices.size() should be [seq,batch] or [batch,]
    result size() would be [seq,batch,dim] or [batch,dim]
    """
    out = torch.zeros(indices.size()+torch.Size([dim]))
    d = len(indices.size())
    return out.scatter_(d, indices.unsqueeze(d).to(dtype=torch.int64), 1)

def xavier_init(m: nn.Module):
    if type(m) == nn.Linear:
        nn.init.xavier_uniform_(m.weight)

class FFNetwork(nn.Sequential):
    """
    Simple feed-forward network with fully connected layers
    """

    def __init__(self, input_dim: int, output_dim: int, layers: List[int], activation='relu'):
        super().__init__()
        act_module = get_activation(activation)

        prev_dim = input_dim
        for (i, layer) in enumerate(layers):
            if type(layer) == int:
                lsize = layer
                self.add_module('fc{}'.format(i+1), nn.Linear(prev_dim, lsize))
                self.add_module('activation{}'.format(i+1), act_module)
                prev_dim = lsize
            elif layer == 'dropout':
                self.add_module('dropout_{}'.format(i+1), nn.Dropout())

        self.add_module('output', nn.Linear(prev_dim, output_dim))

class SaveableModel(nn.Module):
    """
    Mixin which provides `save` and `restore`
    methods for (de)serializing the model.
    """
    def _savedir(self):
        dir = TORCH_MODELS_DIR
        if self._scope is not None:
            dir += '/' + self._scope
        return dir

    def _savepath(self):
        return self._savedir() + '/' + self._label

    def save(self):
        os.makedirs(self._savedir(), exist_ok=True)
        return torch.save(self.state_dict(), self._savepath())

    def restore(self):
        return self.load_state_dict(torch.load(self._savepath()))


#!/Library/Frameworks/Python.framework/Versions/3.5/bin/python3.5
import torch.nn as nn
import torch
from .GDN import GDN
import math

class Analysis_net_17(nn.Module):
    '''
    Analysis net
    '''
    def __init__(self, out_channel_N=192):
        super(Analysis_net_17, self).__init__()
        self.conv1 = nn.Conv2d(3, out_channel_N, 9, stride=4, padding=4)
        torch.nn.init.xavier_normal_(self.conv1.weight.data, (math.sqrt(2 * (3 + out_channel_N) / (6))))
        torch.nn.init.constant_(self.conv1.bias.data, 0.01)
        self.gdn1 = GDN(out_channel_N)
        self.conv2 = nn.Conv2d(out_channel_N, out_channel_N, 5, stride=2, padding=2)
        torch.nn.init.xavier_normal_(self.conv2.weight.data, math.sqrt(2))
        torch.nn.init.constant_(self.conv2.bias.data, 0.01)
        self.gdn2 = GDN(out_channel_N)
        self.conv3 = nn.Conv2d(out_channel_N, out_channel_N, 5, stride=2, padding=2, bias=False)
        torch.nn.init.xavier_normal_(self.conv3.weight.data, math.sqrt(2))
        # torch.nn.init.constant_(self.conv3.bias.data, 0.01)
        # self.gdn3 = GDN(out_channel_N)
        # self.conv4 = nn.Conv2d(out_channel_N, out_channel_M, 5, stride=2, padding=2)
        # torch.nn.init.xavier_normal_(self.conv4.weight.data, (math.sqrt(2 * (out_channel_M + out_channel_N) / (out_channel_N + out_channel_N))))
        # torch.nn.init.constant_(self.conv4.bias.data, 0.01)

    def forward(self, x):
        x = self.gdn1(self.conv1(x))
        x = self.gdn2(self.conv2(x))
        x = self.conv3(x)
        return x

def build_model():
        input_image = torch.zeros([4, 3, 256, 256])

        analysis_net = Analysis_net_17()
        feature = analysis_net(input_image)

        print(feature.size())

if __name__ == '__main__':
    build_model()


import os
from random import randint

from flask import Flask, render_template, jsonify, request, redirect, url_for, session
from flask_session import Session
from flask_socketio import SocketIO, emit
from flask_cors import CORS, cross_origin

app = Flask(__name__)

app.config["SESSION_PERMANENT"] = False
app.config["SESSION_TYPE"] = "filesystem"
app.config["SECRET_KEY"] = os.getenv("SECRET_KEY")
app.config['CORS_HEADERS'] = 'Content-Type'
Session(app)

cors = CORS(app)
socketio = SocketIO(app)

hexa = ["0", "1", "2", "3", "4", "5", "6", "7",
        "8", "9", "A", "B", "C", "D", "E", "F"]

channels_multi = []
channels = []
users = []
inicio = True
msg = {"asdf": ["hola como estas?", "bien y tu", "cuando y donde?", "bien y tu",
                "cuando y donde?", "bien y tu", "cuando y donde?", "bien y tu", "cuando y donde?"]}

def random_color():
    c = "#"
    for i in range(6):
        c += hexa[(randint(0, 1000)*randint(0, 1000) +
                   randint(0, 500)*randint(0, 500)) % len(hexa)]
    return c

@app.route("/", methods=["POST", "GET"])
def index():
    codigo = 1

    if request.method == "POST":
        channel = request.form.get("channel_name")
        if (channel not in channels):
            channels_multi.append((channel, session['user'], session['color']))
            channels.append(channel)
            msg[channel] = []
        else:
            codigo = 420
    return render_template("index.html", channels=channels_multi, codigo=codigo)

@app.route("/logout")
def logout():
    users.remove(session["user"])
    session.clear()
    return redirect(url_for("index"))

@app.route("/register", methods=["POST", "GET"])
def register():
    codigo = 1
    print(session, "/////////////SESSION")
    if 'user' in session:
        print("--------------------------------qqqqqqqqqqqqqqqqqqqqq\n-----------")
        redirect(url_for("index"))
    elif request.method == "POST":
        user = request.form.get("nickname")
        if (user not in users):
            users.append(user)
            session["user"] = user
            # session["color"] = colores[randint(0, len(colores)});-1)]
            #session["color"] = colores[i[0] % len(colores)]
            session["color"] = random_color()
            print(session, "------\n----")

            return render_template("logged.html", user=user, color=session["color"])
        else:
            codigo = 69
    return render_template("register.html", users=users, codigo=codigo)

    # if request.method == "GET":
    #    return render_template("register.html")
    # else:
    #    return redirect(url_for('index'))

@app.route("/channel/<string:channel>", methods=["GET"])
def channel_creation(channel):
    try:
        return render_template(
            "channels.html",
            channel=channel,
            msg=msg[channel],
            user=session["user"]
        )
    except:
        print("-------------------------")

        return redirect(url_for('index'))

@socketio.on('send message')
def newMessage(msg2):
    if (len(msg[msg2['channel']]) == 100):
        msg[msg2['channel']].pop(0)
    msg[msg2['channel']].append(msg2)

    print(msg2, "*************************************")
    emit(
        "new message",
        {
            "mensaje": msg2['mensaje'],
            "username": msg2['username'],
            "date": msg2['date'],
            "channel": msg2['channel'],
            "color": msg2['color']

        },
        broadcast=True
    )


#!/usr/bin/env python

###############################
import json
import logging
import shlex
import subprocess
import sys
import time
import boto3
from Naked.toolshed.shell import execute_rb
# AWS Region
from botocore.exceptions import ClientError
#import main.py
import os

############################################################################
# Config you might want to change
############################################################################

region = "ap-south-1"

#VPC stack Variables
VPCStackOuput ={ }

############################################################################
# Config you might not want to change
############################################################################

# Names for VPC ,Netapp and worker stack
vpc_stack =  'myEfsVPC'
efs_stack='myEfs-stack'

# CloudFormation templates for VPC and worker nodes
vpc_template = "https://myeksfiles.s3.ap-south-1.amazonaws.com/MyEfs/MyVpc.yaml"
efs_template='https://myeksfiles.s3.ap-south-1.amazonaws.com/MyEfs/MyEfs.yaml'

############################################################################
# Connect AWS
############################################################################

# Connect to AWS, EC2, cloudformation, IAM and EKS
s = boto3.Session(region_name=region)
ec2 = s.client("ec2")
cf = s.client("cloudformation")
iam = s.client("iam")
eks = s.client("eks")

############################################################################
# IAM role for K8s
############################################################################

############################################################################
# VPC stack
############################################################################

# The VPC stack is a VPC and subnetworks to allow K8s communication.

# Check if a stack exists.
def stack_exists(cf, name):
    try:
        stack = cf.describe_stacks(StackName=name)
        return True
    except:
        return False

print("*** VPC stack")
if stack_exists(cf, vpc_stack):
    # If stack exists, do nothing.
    print("VPC stack already exists")

else:

    print("Creating VPC stack...")

    # Create VPC stack.
    response = cf.create_stack(
        StackName=vpc_stack,
        TemplateURL=vpc_template,
        Parameters=[],
        TimeoutInMinutes=15,
        OnFailure='DELETE'
    )

    if response == None:
        print("Could not create VPC stack")
        sys.exit(1)

    if not "StackId" in response:
        print("Could not create VPC stack")
        sys.exit(1)

    # Get stack ID for later.
    stack_id = response["StackId"]

    print("Created stack " + vpc_stack)
    print("Waiting for VPC stack creation to complete...")

    try:

        # A waiter is something which polls AWS to find out if an operation
        # has completed.
        waiter = cf.get_waiter('stack_create_complete')

        # Wait for stack creation to complet
        res = waiter.wait(
            StackName=vpc_stack,
        )

    except:

        # If waiter fails, that'll be the thing taking too long to deploy.
        print("Gave up waiting for stack to create")
        sys.exit(1)

    print("Stack created")

# Get output information from the stack: VPC ID, security group and subnet IDs.
stack = cf.describe_stacks(StackName=vpc_stack)
vpc_sg = None
vpc_subnet_ids = None
vpc_id = None

# Loop over outputs grabbing information.
for v in stack["Stacks"][0]["Outputs"]:
    if v["OutputKey"] == "SecurityGroups":
        vpc_sg = v["OutputValue"]
    if v["OutputKey"] == "VpcId":
        vpc_id = v["OutputValue"]
    VPCStackOuput['vpc_id'] = vpc_id
    if v["OutputKey"] == "SubnetIds":
        vpc_subnet_ids = v["OutputValue"]
    if v["OutputKey"] == "PublicSubnetIds":
        vpc_public_subnet_ids = v["OutputValue"]
        VPCStackOuput ['vpc_public_subnet_ids'] = vpc_public_subnet_ids
    if v["OutputKey"] == "PrivateSubnetIds":
        vpc_private_subnet_ids = v["OutputValue"]

print("VPC ID: %s" % vpc_id)
print("VPC security group: %s" % vpc_sg)
print("VPC All subnet IDs: %s" % vpc_subnet_ids)
print("VPC Public subnet IDs: %s" % vpc_public_subnet_ids)
print("VPC Private subnet IDs: %s" % vpc_private_subnet_ids)

# Split subnet IDs - it's comma separated.
vpc_subnet_ids = vpc_subnet_ids.split(",")
vpc_public_subnet_ids = vpc_public_subnet_ids.split(",")

VPCStackOuput['pubsubnet1_a'] =  vpc_subnet_ids[0]
VPCStackOuput['pubsubnet1_b'] =  vpc_subnet_ids[1]
VPCStackOuput['prisubnet1_a'] =  vpc_subnet_ids[2]
VPCStackOuput['prisubnet1_b'] =  vpc_subnet_ids[3]
############################################################################
# Netapp Stack
############################################################################

def stack_exists(cf, efs_stack):
    try:
        stack = cf.describe_stacks(StackName=efs_stack)
        return True
    except:
        return False

print("*** Efs stack")

if stack_exists(cf, efs_stack):
    # If stack exists, do nothing.
    print("Efs stack already exists")

else:
    print("Creating Efs stack...")
    # Create Netapp stack.
    response = cf.create_stack(
        StackName=efs_stack,
        Capabilities=["CAPABILITY_IAM","CAPABILITY_AUTO_EXPAND","CAPABILITY_NAMED_IAM"],
        TemplateURL=efs_template,
        Parameters=[
        {
        'ParameterKey': 'VPC',
            'ParameterValue': vpc_id
        },
        {
        'ParameterKey': 'PublicSubnet1A',

            'ParameterValue': vpc_public_subnet_ids[0]
        },
        {
        'ParameterKey': 'PublicSubnet1B',

            'ParameterValue': vpc_subnet_ids[1]
        },
        {
        'ParameterKey': 'SourceSecurityGroupId',

            'ParameterValue': vpc_sg
        },

        ],
        TimeoutInMinutes=45,
        OnFailure='DELETE'
    )

    if response == None:
        print("Could not create Efs stack")
        sys.exit(1)

    if not "StackId" in response:
        print("Could not create Efs stack")
        sys.exit(1)

    # Get stack ID for later.
    stack_id = response["StackId"]

    print("Created stack " + stack_id)
    print("Waiting for Efs stack creation to complete...")

    try:

        # A waiter is something which polls AWS to find out if an operation
        # has completed.
        waiter = cf.get_waiter('stack_create_complete')

        # Wait for stack creation to complete
        res = waiter.wait(
            StackName=efs_stack,
        )
    except:
       # If waiter fails, that'll be the thing taking too long to deploy.
        print("Gave up waiting for Efsstack to create")
        sys.exit(1)

    print("efs Stack created")

efsstack = cf.describe_stacks(StackName=efs_stack)
    # Loop over outputs grabbing information.
for efs in efsstack["Stacks"][0]["Outputs"]:
    if efs["OutputKey"] == "EFS":
            efs_Id = efs["OutputValue"]
    if efs["OutputKey"] == "EFSMountTarget1":
        EFSMountTarget1_Id = efs["OutputValue"]
    if efs["OutputKey"] == "EFSMountTarget2":
        EFSMountTarget2_Id = efs["OutputValue"]

print("EFS Id is : %s" % efs_Id)
print("EFSMountTarget1 Id is: %s" % EFSMountTarget1_Id )
print("EFSMountTarget2 Id is: %s" % EFSMountTarget2_Id )

################DONE#################################################################

# file = open("VPCNetAppOutput.txt", "w")
# str_dictionary = VPCStackOuput
# file.write(str_dictionary["vpc_id"])
# file.write("\n")
# file.write(str_dictionary["pubsubnet1_a"])
# file.write("\n")
# file.write(str_dictionary["pubsubnet1_b"])
# file.write("\n")
# file.write(str_dictionary["prisubnet1_a"])
# file.write("\n")
# file.write(str_dictionary["prisubnet1_b"])
# file.write("\n")
# file.close()
# ############ Create EKS cluster ##################################################
# clustercount=1
# try:
#  if clustercount == 0:
#     success = execute_rb('read_config.rb')
#     time.sleep(5)
#     if success:
#         print("managedcluster.yaml is generated")
#         subprocess.call(shlex.split('managedcluster.sh'))
#         time.sleep(1800)
#         print(" Cluster is created")
#     else:
#         sys.exit()
#     clustercount += 1
# except ClientError as e:
#         logging.error(e)
# print( "EKS Cluster created")
# ############ Setup the EKSSETUP ##################################################
# setupcount=1
# try:
#     if setupcount==0:
#         print("execute eks setup ")
#         subprocess.call(shlex.split('ekssetup.sh'))
#         time.sleep(10)
#     else:
#         print ( 'Already setup is done')
# except ClientError as e:
#         logging.error(e)
# print(" Execute Cluster setup done and active now")
# ############Setup the NFS Storage Class  ##################################################
# storagecount=1
# try:
#     if storagecount==0:
#         print("execute storage class setup")
#         subprocess.call(shlex.split('netapp-nfs-sc.sh'))
#         time.sleep(10)
#     else:
#         print ( 'Already SC is created')
#
# except ClientError as e:
#         logging.error(e)
# print(" storage class is created and active now")
#
# ##################### Generating netapp-nfs-pv.yaml file ##################################
#
# with open('NetAppVolume.txt', 'w') as f:
#     f.write(netapp_url.split('/')[-1])
#     f.write("\n")
#     f.write(str(CVOMountPt).split(':')[-1][:15])
# time.sleep(5)
# success = execute_rb('Net_App_Volume.rb')
#
# ############Setup the NSF PV  ##################################################
# pvcount=1
# try:
#     if pvcount==0:
#         print("execute PV setup")
#         subprocess.call(shlex.split('netapp-nfs-pv.sh'))
#         time.sleep(12)
#     else:
#         print('Already PV is created')
#
# except ClientError as e:
#         logging.error(e)
# print(" PV is created and active now")
#
# ############Finally WEB Deployment  ##################################################
#
# wedepcount=0
# try:
#     if wedepcount==0:
#         print("execute WEB Deployment setup")
#         subprocess.call(shlex.split('myweb-deployment-nfs-netapp-pv.sh'))
#         time.sleep(20)
#     else:
#         print('Already WEB Deplmnt is created')
#
# except ClientError as e:
#         logging.error(e)
# print(" WEB Deplmnt is created and active now")


# Nikolaus Howe, May 2016
# Some functions to streamline the NN notebooks

import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, auc

def draw_histogram(test_data, test_labels, model):
    # Get the test signal and background to make the histogram
    test_signal = test_data[np.where(test_labels==1)]
    test_bkg    = test_data[np.where(test_labels==0)]

    # Calculate the probabilities for the test sets
    p_signal    = model.predict(test_signal)
    p_bkg       = model.predict(test_bkg)

    # Draw classification histogram
    plt.hist(p_signal, 50, normed=1, facecolor='blue', alpha=0.4, label='gamma')
    plt.hist(p_bkg , 50, normed=1, facecolor='red' , alpha=0.4, label='pi0')
    plt.xlabel('Prediction')
    plt.ylabel('Probability')
    plt.title('Binary Classification Histogram')
    plt.yscale('log')
    plt.grid(True)
    plt.legend(loc='upper right')
    plt.show()

def draw_roc_curve(test_data, test_labels, model):
    # Get classification predictions
    predictions = model.predict(test_data)

    # Draw the ROC curve
    fpr, tpr, _ = roc_curve(test_labels, predictions)
    plt.xlim([.0, 1.01])
    plt.ylim([.0, 1.01])
    plt.title("ROC Curve")
    plt.plot( tpr, 1-fpr )

def draw_loss_history(my_fit):
    plt.ylim(bottom=0)
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.title('Training Error by Epoch')
    plt.plot(my_fit.history['loss'])

def draw_val_loss_history(my_fit):
    plt.ylim(bottom=0)
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.title('Training Error by Epoch')
    plt.plot(my_fit.history['val_loss'])

def draw_list(label_list, predictions, histories):
    # Set up the plot
    plt.figure(figsize=(10,10))
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.title('Training Error by Epoch')
    plt.ylim(bottom=0)

    # Get the colors
    cmap = plt.get_cmap('jet_r')
    N = len(label_list)

    # Draw the histories
    for i, label in enumerate(label_list):
        color = cmap(float(i)/N) # get color for this plot
        plt.plot(histories[label]['loss'], label = label + ' loss', linewidth=2, c=color)
        plt.plot(histories[label]['val_loss'], label = label + ' val_loss', linewidth=2, linestyle='dashed', c=color)

    # Put in the legend (with thick lines!)
    leg = plt.legend(loc='upper right')
    for legobj in leg.legendHandles:
        legobj.set_linewidth(8.0)
    plt.show()

    # Set up plot
    plt.figure(figsize=(10,10))
    plt.xlim([.7, 1.01])
    plt.ylim([.7, 1.01])
    plt.title("ROC Curve")
    # Draw the roc curves
    for label in label_list:
        pred = predictions[label][0].reshape(predictions[label][0].shape[0])
        truth = predictions[label][1].astype(int)
        fpr, tpr, _ = roc_curve(truth, pred)
        plt.plot( tpr, 1-fpr , label = label, linewidth=1.5)
        plt.xlabel('True Positive Rate')
        plt.ylabel('True Negative Rate')

    # Draw the legend (with thick lines!)
    leg = plt.legend(loc='lower left')
    for legobj in leg.legendHandles:
        legobj.set_linewidth(8.0)
    plt.show()


""" Eop visualisation.

This module contains all methods used for visualising Eop series and their residuals, using numpy and matplotlib as backend.

The public methods availabe in this module are:

    *plot_serie, plots a data serie and its uncertainty
    *plot_residuals, plots residuals between two data series and their combined uncertainties.
    *plot_serie_to_ax, plots a data serie and its uncertainty to given axes objects
    *plot_residuals_to_ax, plots residuals between two data series and their combined uncertainties togiven axes objects.

"""

import matplotlib.pyplot as plt
import numpy as np
import EopTool.units as units

from EopTool.calc import predict_residuals

def plot_serie(df, selected_serie, ax, value_unit = units.one, sigma_unit = units.one):
    ax = plt.subplot()
    plot_serie_to_ax(df, selected_serie, ax, value_unit , sigma_unit)
    plt.show()

def plot_residuals():
    pass

def plot_serie_to_ax(df, selected_serie, ax, value_unit = units.one, sigma_unit = units.one):

    serie = df[selected_serie]
    ax[0].scatter(serie.index, serie*value_unit.convertion_factor, s=4)
    ax[0].set_xlabel(f"year")
    ax[0].set_ylabel(value_unit.symbol)
    ax[0].grid()

    sigma = df[selected_serie + "_SIGMA"]
    ax[1].scatter(sigma.index, sigma*sigma_unit.convertion_factor, s=4)
    ax[1].set_xlabel(f"year")
    ax[1].set_ylabel(sigma_unit.symbol)
    ax[1].grid()

    plt.tight_layout()

def plot_residuals_to_ax(df_from, df_to, selected_serie, t0, parameters, axes, value_unit = units.one, sigma_unit = units.one):

    if not df_from is None and not df_to is None:

        min_x = min(df_from.index)
        max_x = max(df_to.index)

        regression_x = np.linspace(min_x, max_x, 400)
        regression_y = predict_residuals(regression_x, t0, parameters)

        axes[0].plot(regression_x, regression_y*value_unit.convertion_factor, c="k")
        axes[0].scatter(df_from.index, (df_from[selected_serie] - df_to[selected_serie])*value_unit.convertion_factor, 2)
        axes[0].set_xlabel(f"year")
        axes[0].set_ylabel(value_unit.symbol)
        axes[0].grid()

        axes[1].scatter(df_from.index, np.sqrt(df_from[selected_serie + "_SIGMA"]**2 + df_to[selected_serie + "_SIGMA"]**2)*sigma_unit.convertion_factor, 2)
        axes[1].set_xlabel(f"year")
        axes[1].set_ylabel(sigma_unit.symbol)
        axes[1].grid()

        plt.tight_layout()

def plot_transformed_residuals_to_ax(df_from, df_to, selected_serie, t0, parameters, axes, value_unit = units.one, sigma_unit = units.one):
    if not df_from is None and not df_to is None:

        predicted_residuals = predict_residuals(df_from.index, t0, parameters)
        axes[0].scatter(df_from.index, (df_from[selected_serie] - df_to[selected_serie] - predicted_residuals)*value_unit.convertion_factor, 2)
        axes[0].set_xlabel(f"year")
        axes[0].set_ylabel(value_unit.symbol)
        axes[0].grid()

        axes[1].scatter(df_from.index, np.sqrt(df_from[selected_serie + "_SIGMA"]**2 + df_to[selected_serie + "_SIGMA"]**2)*sigma_unit.convertion_factor, 2)
        axes[1].set_xlabel(f"year")
        axes[1].set_ylabel(sigma_unit.symbol)
        axes[1].grid()


from itertools import tee

with open('./input.txt') as input:
    handle1, handle2 = tee(input)
    bitcount = len(next(handle1).strip())
    count = 0
    print(bitcount)
    occurences = [0] * bitcount
    for row in handle2:
        for index, bit in enumerate(row.strip()):
            occurences[index] += int(bit)
        count += 1

    gamma = sum([2 ** (bitcount - 1 - index) * (occurence > count / 2) for index, occurence in enumerate(occurences)])
    epsilon = 2 ** bitcount - 1 - gamma
    print(gamma * epsilon) # 3374136

Day 03 - Binary Diagnostic/part1-solution.py


import pytest
import numpy as np
from collections import namedtuple

from ppo_worm.env_wrapper import WormGymWrapper

@pytest.fixture(scope="session")
def worm_env():
    return WormGymWrapper("worm/worm.x86_64", no_graphics=True)

def test_reset(worm_env):
    obs = worm_env.reset()
    assert obs.shape == (10, 64)

def test_step(worm_env):
    action = np.random.normal(size=(10, 9))
    observation, reward, done, info = worm_env.step(action)
    assert observation.shape == (10, 64)
    assert reward.shape == (10,)
    assert done == False

def test_decision_to_observation(worm_env):
    DecisionStep = namedtuple("DecisionStep", ["obs", "reward"])

    # Each observation is 64 array
    # Each reward is a float
    decision_steps = [DecisionStep(np.ones((1, 64)), 1.) for _ in range(10)]
    observations, rewards  = worm_env._decision_to_observation(decision_steps)

    # Check shapes match when converting
    assert observations.shape == (10, 64)
    assert rewards.shape == (10,)


from __future__ import unicode_literals
import ezdxf
from ezdxf.tools import standards

def main():
    dwg = ezdxf.new('R12')
    for name, desc, pattern in standards.linetypes():
        if name in dwg.linetypes:
            continue
        dwg.linetypes.new(name, dxfattribs={
            'description': desc,
            'pattern': pattern,
        })
    for name, font in standards.styles():
        if name in dwg.styles:
            continue
        dwg.styles.new(name, dxfattribs={
            'font': font,
        })

    msp = dwg.modelspace()
    for n, ltype in enumerate(dwg.linetypes):
        name = ltype.dxf.name
        msp.add_line((0, n), (10, n), dxfattribs={'linetype': name})
        msp.add_text(name, dxfattribs={
            'insert': (0, n + 0.1),
            'height': 0.25,
            'style': 'ARIAL'
        })
    dwg.saveas('standard_linetypes.dxf')

if __name__ == '__main__':
    main()


"""
Example DAG using GCSToBigQueryOperator.
"""

import os

from airflow import DAG
from airflow.providers.google.cloud.operators.bigquery import (
    BigQueryCreateEmptyDatasetOperator,
    BigQueryDeleteDatasetOperator,
)
from airflow.providers.google.cloud.transfers.gcs_to_bigquery import GCSToBigQueryOperator
from airflow.utils.dates import days_ago
from datetime import datetime, timedelta

DATASET_NAME = os.environ.get("GCP_DATASET_NAME", 'airflow_ds')
TABLE_NAME = os.environ.get("GCP_TABLE_NAME", 'gcs_to_bq_table')

default_args = {
    'owner': 'airflow',
    'depends_on_past': False,
    'start_date': days_ago(2), #datetime(2019, 6, 30),
    'retries': 1,
    'retry_delay': timedelta(minutes=1)
}

dag = DAG(
    dag_id='gcs_to_bigquery_operator',
	catchup=False,
	default_args=default_args,
    schedule_interval=None
)

create_dataset = BigQueryCreateEmptyDatasetOperator(
    task_id='create_airflow_ds_dataset', dataset_id=DATASET_NAME, dag=dag
)

# [START howto_operator_gcs_to_bigquery]
load_csv = GCSToBigQueryOperator(
    task_id='gcs_to_bigquery',
    bucket='third-campus-303308-bigmart',
    source_objects=['bigmart_data.csv'],
    destination_project_dataset_table=f"{DATASET_NAME}.{TABLE_NAME}",
    schema_fields=None,
	skip_leading_rows=1,
	field_delimiter=',',
	autodetect=True,
    write_disposition='WRITE_TRUNCATE',
    dag=dag,
)
# [END howto_operator_gcs_to_bigquery]

#delete_test_dataset = BigQueryDeleteDatasetOperator(
#    task_id='delete_airflow_ds_dataset', dataset_id=DATASET_NAME, delete_contents=True, dag=dag
#)

create_dataset >> load_csv #>> delete_test_dataset


from Block import Blockchain, Block
from flask import Flask, jsonify, request
import json
app = Flask(__name__)

blockchain = Blockchain()

@app.route("/")
def home():
    return "Hello, Flask!"

@app.route('/SendWeapon/GametoServer', methods = ['POST'])
def GametoServer():
    values = request.get_json()
    print(values)

    required = ['name', 'discription', 'damage', 'attackSpeed']

    if not all(k in values for k in required):
        return 'Missing data', 400

    # Create a new block
    blockchain.mine(Block(values))

    return 'Success'

@app.route("/PrintData", methods = ['GET'])
def PrintChain():
    # Print out each block in the blockchain
    #iter = blockchain.head
    #response = ''
    #while iter != None:
        #print(inum)
        # response = response + iter
        # iter = iter.next
    response = {
        'Message' : "You did it!",
        'Block' : "This Block"
    }
    return jsonify(response), 200

@app.route("/GetChain", methods = ['GET'])
def GetChain():
    itr = blockchain.head
    toGameJsn = {}
    toGameJsn[itr.blockNo] = {
        'BlockNumber' : itr.blockNo,
        'Hash' : str(itr.hash()),
        'Previous Hash' : itr.previousHash,
        'Data' : itr.data
    }
    itr = itr.next
    while itr != None:
        toGameJsn[itr.blockNo] = {
            'BlockNumber' : itr.blockNo,
            'Hash' : str(itr.hash()),
            'Previous Hash' : itr.previousHash,
            'Data' : itr.data
        }

        itr = itr.next

    return jsonify(json.dumps(toGameJsn)), 200


import copy
import os.path
from pathlib import Path

from nni.experiment.config import ExperimentConfig

def expand_path(path):
    return os.path.realpath(os.path.join(os.path.dirname(__file__), path))

## minimal config ##

minimal_json = {
    'searchSpace': {'a': 1},
    'trialCommand': 'python main.py',
    'trialConcurrency': 2,
    'tuner': {
        'name': 'random',
    },
    'trainingService': {
        'platform': 'local',
    },
}

minimal_class = ExperimentConfig('local')
minimal_class.search_space = {'a': 1}
minimal_class.trial_command = 'python main.py'
minimal_class.trial_concurrency = 2
minimal_class.tuner.name = 'random'

minimal_canon = {
    'experimentType': 'hpo',
    'searchSpace': {'a': 1},
    'trialCommand': 'python main.py',
    'trialCodeDirectory': os.path.realpath('.'),
    'trialConcurrency': 2,
    'useAnnotation': False,
    'debug': False,
    'logLevel': 'info',
    'experimentWorkingDirectory': str(Path.home() / 'nni-experiments'),
    'tuner': {'name': 'random'},
    'trainingService': {
        'platform': 'local',
        'trialCommand': 'python main.py',
        'trialCodeDirectory': os.path.realpath('.'),
        'debug': False,
        'maxTrialNumberPerGpu': 1,
        'reuseMode': False,
    },
}

minimal_canon_2 = copy.deepcopy(minimal_canon)
minimal_canon_2['tuner']['classArgs'] = {}

## detailed config ##

detailed_canon = {
    'experimentName': 'test case',
    'experimentType': 'hpo',
    'searchSpaceFile': expand_path('assets/search_space.json'),
    'searchSpace': {'a': 1},
    'trialCommand': 'python main.py',
    'trialCodeDirectory': expand_path('assets'),
    'trialConcurrency': 2,
    'trialGpuNumber': 1,
    'maxExperimentDuration': '1.5h',
    'maxTrialNumber': 10,
    'maxTrialDuration': 60,
    'nniManagerIp': '1.2.3.4',
    'useAnnotation': False,
    'debug': True,
    'logLevel': 'warning',
    'experimentWorkingDirectory': str(Path.home() / 'nni-experiments'),
    'tunerGpuIndices': [0],
    'assessor': {
        'name': 'assess',
    },
    'tuner': {
        'className': 'Tuner',
        'codeDirectory': expand_path('assets'),
        'classArgs': {'random_seed': 0},
    },
    'trainingService': {
        'platform': 'local',
        'trialCommand': 'python main.py',
        'trialCodeDirectory': expand_path('assets'),
        'trialGpuNumber': 1,
        'debug': True,
        'useActiveGpu': False,
        'maxTrialNumberPerGpu': 2,
        'gpuIndices': [1, 2],
        'reuseMode': True,
    },
    'sharedStorage': {
        'storageType': 'NFS',
        'localMountPoint': expand_path('assets'),
        'remoteMountPoint': '/tmp',
        'localMounted': 'usermount',
        'nfsServer': 'nfs.test.case',
        'exportedDirectory': 'root',
    },
}

## test function ##

def test_all():
    minimal = ExperimentConfig(**minimal_json)
    assert minimal.json() == minimal_canon

    assert minimal_class.json() == minimal_canon_2

    detailed = ExperimentConfig.load(expand_path('assets/config.yaml'))
    assert detailed.json() == detailed_canon

if __name__ == '__main__':
    test_all()


#!/usr/bin/env python

from utils import Utils
from dataset import Dataset
import argparse
import sys

def main(args):
    d = Dataset()
    if args.gather:
        d.load_csv(args.infile, args.num_accounts, args.skip, args.verbose)
    elif args.load:
        d.load(args.infile, args.verbose)
    else:
        print('--gather and --load are mutually exclusive options')
        sys.exit(1)

    print('Done!')
    sys.exit(0)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-g', '--gather', action='store_true', help='Gather relevant account data')
    parser.add_argument('-l', '--load', action='store_true', help='Load previously gathered data')
    parser.add_argument('-if', '--infile', help='CSV file to load accounts from')
    parser.add_argument('-of', '--outfile', help='File to store JSON output to')
    parser.add_argument('-s', '--skip', type=int, help='Skip this many accounts', default=0)
    parser.add_argument('-n', '--num-accounts', type=int, help='How many accounts to process')
    parser.add_argument('-v', '--verbose', action='store_true', help='Display more information during process')

    args = parser.parse_args()
    main(args)


import io
import json
from PIL import Image

TILE_SIZE = 4096

# disable PIL maximum pixels limit
Image.MAX_IMAGE_PIXELS = None

# open image
image = Image.open('moon.jpg')

# get image size
width, height = image.size

# get scale factors
SCALE_FACTORS = [pow(2, factor) for factor in range(
    0, round(max(width, height) / TILE_SIZE))]

# get sizes levels
sizes = [{
    'width': int(width / factor),
    'height': int(height / factor)} for factor in SCALE_FACTORS]

# get tiles
tiles = {'width': TILE_SIZE, 'scaleFactors': SCALE_FACTORS}

id = 'http://localhost/'

# set base default content
content = {
    '@context': 'http://iiif.io/api/image/2/context.json',
    # server url
    '@id': id,
    'protocol': 'http://iiif.io/api/image',
    # size of original image
    'width': width,
    'height': height,
    'profile': ['http://iiif.io/api/image/2/level2.json'],
    # list of image sizes
    'sizes': sizes,
    'tiles': tiles
}

# create json file
with io.open('info.json', 'w', encoding='utf-8') as file:
    json.dump(content, file, ensure_ascii=False, indent=4)


from fastapi import FastAPI,APIRouter

# /api/v1/student
# student
import student
from config import settings

router = APIRouter(prefix="{}".format(settings.API_V1_STR),
                   tags=["core"]
                   )

@router.get("/")
async def hello():
    return {"message": "Hello World"}

app = FastAPI()

app.include_router(router)
app.include_router(student.router)


from .cube import TrainableCube, PredictorCube, CubeLabel
from ..utils.functions import sorted_labels

import re
import os
import json
from collections import defaultdict

class PatternMatcher(TrainableCube, PredictorCube):
    """Matcher based on regexps"""

    def __init__(self):
        self.labels = []
        self.patterns = []

    def train(self, labels, patterns):
        """Arguments:

            labels:  [[..], [..]]  nested lists of patterns
            patterns: [[..], [..]]  nested list of corresponded labels
        """

        self.labels = labels
        self.patterns = [[p.lower() for p in ptrns] for ptrns in patterns]

    def forward(self, query):
        unique_labels = set()
        labels_probas = defaultdict(int)
        prepared_query = query.strip().lower()

        for labels, patterns in zip(self.labels, self.patterns):
            unique_labels.update(labels)
            for pattern in patterns:
                if re.match(pattern, prepared_query):
                    for label in labels:
                        labels_probas[label] = 1

                    break

        return sorted_labels([CubeLabel(label, labels_probas[label])
                              for label in unique_labels])

    def save(self, path, name="pattern_matcher.cube"):
        super().save(path, name)

        cube_params = {
            'cube': self.__class__.__name__,
            'labels': self.labels,
            'patterns': self.patterns,
        }

        cube_path = os.path.join(path, name)

        with open(cube_path, 'w') as out:
            out.write(json.dumps(cube_params))

        return cube_path

    @classmethod
    def load(cls, path):
        with open(path, 'r') as f:
            cube_params = json.loads(f.read())

        pattern_matcher = cls()
        pattern_matcher.train(cube_params["labels"], cube_params["patterns"])

        return pattern_matcher


from typing import Tuple

import pytest

from y2021.day_9.main import puzzle_1, Point, calculate_top_bottom_left_right

@pytest.mark.parametrize(
    "input_, expected",
    [
        pytest.param(
            [
                "2199943210",
                "3987894921",
                "9856789892",
                "8767896789",
                "9899965678",
            ],
            15,
            id="example",
        ),
        pytest.param(
            [
                "00000000000000",
                "00000000000000",
                "00000000000000",
                "00000000000000",
                "00000000000000",
            ],
            0,
            id="plane",
        ),
        pytest.param(
            [
                "00000000000000",
                "00000010000000",
                "00000201000000",
                "00000020000000",
                "00000000000000",
            ],
            1,
            id="center",
        ),
        pytest.param(
            [
                "00000000000010",
                "00000000000001",
                "00000000000000",
                "00000000000001",
                "00000000000020",
            ],
            2,
            id="top-left corner and bot left-corner",
        ),
        pytest.param(
            [
                "01000000000000",
                "40000000000000",
                "00000000000000",
                "20000000000000",
                "06000000000000",
            ],
            2,
            id="top-left corner and bot left-corner",
        ),
    ],
)
def test_puzzle_1(input_, expected):
    assert puzzle_1(input_) == expected

_adjacent_0 = (1, 2, 3, 4)
_adjacent_1 = (4, 5, 6, 7)
_adjacent_2 = (4, None, None, 7)

@pytest.mark.parametrize(
    "point, adjacent, expected_is_lower_point, expected_adjacent_point",
    [
        pytest.param(
            Point(0, (0, 0), *_adjacent_0),
            _adjacent_0,
            True,
            min(_adjacent_0),
            id="Lower point and 1 as min adjacent point is 1",
        ),
        pytest.param(
            Point(6, (1, 1), *_adjacent_0),
            _adjacent_0,
            False,
            min(_adjacent_0),
            id="Not lower point and 1 as min adjacent point",
        ),
        pytest.param(
            Point(5, (1, 1), *_adjacent_1),
            _adjacent_1,
            False,
            min(_adjacent_1),
            id="Not lower point and 4 as min adjacent point",
        ),
        pytest.param(
            Point(3, (1, 1), *_adjacent_1),
            _adjacent_1,
            True,
            min(_adjacent_1),
            id="Lower point and 4 as min adjacent point",
        ),
    ],
)
def test_point_is_lower_and_adjacent(
    point: Point,
    adjacent: Tuple[int, int, int, int],
    expected_is_lower_point: bool,
    expected_adjacent_point: int,
):
    assert point.is_low_point() == expected_is_lower_point
    assert point.adjacent == expected_adjacent_point

_matrix = [
    [2, 1, 9, 9, 9, 4, 3, 2, 1, 0],
    [3, 9, 8, 7, 8, 9, 4, 9, 2, 1],
    [9, 8, 5, 6, 7, 8, 9, 8, 9, 2],
    [8, 7, 6, 7, 8, 9, 6, 7, 8, 9],
    [9, 8, 9, 9, 9, 6, 5, 6, 7, 8],
]

@pytest.mark.parametrize(
    "x, y, input_, expected",
    [
        pytest.param(
            0,
            0,
            _matrix,
            (None, 1, None, 3),
            id="x: 0, y: 0 expect Left: None, Right: 1, Top: None, Bottom: 3",
        ),
        pytest.param(
            0,
            4,
            _matrix,
            (None, 8, 8, None),
            id="x: 0, y: 0 expect Left: None, Right: 8, Top: 8, Bottom: None",
        ),
        pytest.param(
            9,
            0,
            _matrix,
            (1, None, None, 1),
            id="x: 0, y: 0 expect Left: 1, Right: None, Top: None, Bottom: 1",
        ),
        pytest.param(
            9,
            4,
            _matrix,
            (7, None, 9, None),
            id="x: 0, y: 0 expect Left: 7, Right: 9, Top: None, Bottom: None",
        ),
        pytest.param(
            1,
            1,
            _matrix,
            (3, 8, 1, 8),
            id="x: 0, y: 0 expect Left: 3, Right: 8, Top: 1, Bottom: 8",
        ),
    ],
)
def test_calculate_top_bottom_left_right(x, y, input_, expected):
    assert calculate_top_bottom_left_right(x, y, input_) == expected


import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision
from torch.utils.data import DataLoader, Dataset
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from torchvision.models import mobilenet_v2, MobileNet_V2_Weights
from tqdm import tqdm
from models import Conv_Model, MobileNet_Model

# constants
num_workers = 4
batch_size = 128
num_epochs = 15
learning_rate = 0.001
progress_bar = True

# get device
device = f'cuda:0' if torch.cuda.is_available() else 'cpu'
print(f"using device: {device}")
# EDA
# create dataset

def get_loaders(transforms):
	train_dataset = torchvision.datasets.CIFAR10('../datasets/', train=True, download=True, transform=transforms)
	_dataset = torchvision.datasets.CIFAR10('../datasets/', train=False, download=True, transform=transforms)

	# split nontraining dataset into validation and testing datasets
	nontrain_length = len(_dataset)//2
	valid_dataset, test_dataset = torch.utils.data.random_split(_dataset, [nontrain_length, nontrain_length], generator=torch.Generator().manual_seed(0))

	# create dataloaders
	train_loader = DataLoader(dataset = train_dataset, batch_size = batch_size, num_workers = 2, shuffle = False, persistent_workers = True)
	valid_loader = DataLoader(dataset = valid_dataset, batch_size = batch_size, num_workers = 2, shuffle = False, persistent_workers = True)
	test_loader = DataLoader(dataset = test_dataset, batch_size = batch_size, num_workers = 1, shuffle = False, persistent_workers = True)

	return train_loader, valid_loader, test_loader

def train(model, train_loader, valid_loader):
	# initialize optimizer
	optimizer = optim.Adam(model.parameters(), lr = learning_rate)

	# initialize loss function
	loss_func = nn.CrossEntropyLoss()

	# prev_loss is used to store validation losses -> training is stopped
	# once validation loss is above a 5-epoch rolling mean
	prev_loss = []

	# iterate for specified number of epochs
	for epoch in range(num_epochs):
		model.train()
		sum_loss = 0
		for batch_idx, (images, labels) in enumerate(tqdm(train_loader, disable = not progress_bar, desc = f'Epoch {epoch:02d}', ncols=60)):
			# send tensors to device
			images, labels = images.to(device), labels.to(device)

			# zero out gradients
			optimizer.zero_grad()

			# forward pass
			preds = model(images)

			# calculate loss
			loss = loss_func(preds, labels)
			sum_loss += loss.item()

			# backward pass
			loss.backward()

			# step optimizer
			optimizer.step()

		print(f'\tTrain loss =      {sum_loss/(batch_idx+1)/batch_size:.6f}')

		# validation loop
		model.eval()
		valid_loss = 0
		with torch.no_grad():
			for batch_idx, (images, labels) in enumerate(valid_loader):
				# send tensors to device
				images, labels = images.to(device), labels.to(device)

				# forward pass
				preds = model(images)

				# calculate loss
				loss = loss_func(preds, labels)
				valid_loss += loss.item()

		# append current loss to prev_loss list
		prev_loss.append(valid_loss/(batch_idx+1)/batch_size)

		print(f'\tValidation loss = {valid_loss/(batch_idx+1)/batch_size:.6f}')

		# # if valid_loss exceedes the 5-epoch rolling sum, break from training
		if valid_loss/(batch_idx+1)/batch_size > np.mean(prev_loss[-5:]):
			continue
			break

	return model, prev_loss

def test(model, test_loader):
	loss_func = nn.CrossEntropyLoss()

	sum_loss = 0
	num_correct = 0
	total = 0
	model.eval()
	with torch.no_grad():
		for batch_idx, (images, labels) in enumerate(tqdm(test_loader, disable = not progress_bar, desc = 'Testing', ncols=60)):
			# send tensors to device
			images, labels = images.to(device), labels.to(device)

			# forward pass
			preds = model(images)

			# calculate loss
			# loss = loss_func(preds.float(), F.one_hot(labels, num_classes = 10).float())
			loss = loss_func(preds, labels)
			sum_loss += loss.item()

			# calc number correct
			preds = torch.argmax(preds, dim = 1)
			num_correct += torch.sum(preds == labels)
			total += len(preds)

	print(f'Test loss: {sum_loss/(batch_idx+1)/batch_size:.6f}')
	print(f'Test acc:  {num_correct/total:.6f}')
	return sum_loss

# train and test each model
print('--------Conv---------')
normalize = torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.225, 0.225, 0.225])
convert = torchvision.transforms.ConvertImageDtype(torch.float)
transforms = torchvision.transforms.Compose([torchvision.transforms.ToTensor(),normalize, convert])
train_loader, valid_loader, test_loader = get_loaders(transforms)
Conv_model = Conv_Model(6*6*16, 128, 10)
Conv_model.to(device)

Conv_model, Conv_losses = train(Conv_model, train_loader, valid_loader)
Conv_loss = test(Conv_model, test_loader)

print('------MobileNet------')
train_loader, valid_loader, test_loader = get_loaders(MobileNet_V2_Weights.IMAGENET1K_V1.transforms())
MobileNet_model = MobileNet_Model()

MobileNet_model.to(device)

MobileNet_model, MobileNet_losses = train(MobileNet_model, train_loader, valid_loader)
MobileNet_loss = test(MobileNet_model, test_loader)

# affine transformations -> -45 to +45 degree rotationsm -20% to +20% translations in x and y directions, and 90% to 110% scale factor
affine = torchvision.transforms.RandomAffine(45, translate = (0.2, 0.2), scale = (0.9, 1.1))

# same transformations from earlier
normalize = torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.225, 0.225, 0.225])
convert = torchvision.transforms.ConvertImageDtype(torch.float)
transforms = torchvision.transforms.Compose([torchvision.transforms.ToTensor(), normalize, convert, affine])
train_loader, valid_loader, test_loader = get_loaders(transforms)

# create model and send to gpu
Conv_model = Conv_Model(6*6*16, 128, 10)
Conv_model.to(device)

Conv_model, Conv_losses = train(Conv_model, train_loader, valid_loader)
Conv_loss = test(Conv_model, test_loader)

'''
print('--------LSTM--------')
LSTM_model, LSTM_losses = train(LSTM_model, train_loader, valid_loader)
LSTM_loss = test(LSTM_model, test_loader)

print('--------GRU---------')
GRU_model, GRU_losses = train(GRU_model, train_loader, valid_loader)
GRU_loss = test(GRU_model, test_loader)

# plot losses over time
xx = np.arange(num_epochs)
plt.plot(xx, RNN_losses, label = 'RNN')
plt.plot(xx, LSTM_losses, label = 'LSTM')
plt.plot(xx, GRU_losses, label = 'GRU')
plt.legend()
plt.title('Validation Loss')
plt.show()

# store embeddings as dict
embeddings = {}
with open('glove.6B.50d.txt', 'r', encoding='cp437') as f:
	for row in f:
		vals = row.strip().split(' ')
		# the first value is the word, all the rest are embeddings
		embeddings[vals[0]] = np.array(vals[1:]).astype(np.float32)

def cosine_similarity(embeddings, w1, w2):
	# get embeddings for each word
	v1 = embeddings[w1]
	v2 = embeddings[w2]

	# calculate cosine similarity
	return (v1.dot(v2))/(np.linalg.norm(v1) * np.linalg.norm(v2))

def dissimilarity_metric(embeddings, w1, w2):
	# get embeddings for each word
	v1 = embeddings[w1]
	v2 = embeddings[w2]

	# calculate euclidean distance
	return np.linalg.norm(v1 - v2)
'''


from PIL import Image, ImageDraw, ImageFont
import random
import io
import base64
import os
from pathlib import Path

def encode(key, msg):
    enc = list()
    for i in range(len(msg)):
        key_c = key[i % len(key)]
        enc.append(chr((ord(msg[i]) + ord(key_c)) % 256))
    return base64.urlsafe_b64encode("".join(enc).encode()).decode()

def decode(key, msg):
    dec = list()
    message = base64.urlsafe_b64decode(msg).decode()
    for i in range(len(message)):
        key_c = key[i % len(key)]
        dec.append(chr((256 + ord(message[i]) - ord(key_c)) % 256))
    return "".join(dec)

# Set the image size
width = 500
height = 500
key = "kEY"
BASE_DIR = Path(__file__).resolve().parent.parent

def createCaptcha() -> tuple:

    # Generate a random background color
    bg_color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))

    # Create a new image with the same size as the background
    img = Image.new('RGB', (width, height), color=bg_color)

    # Generate a random text string
    alphanumber = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
    text = "".join(random.choice(alphanumber) for _ in range(5))

    # Choose a random font
    fontname = random.choice(os.listdir(os.path.join(BASE_DIR, "fonts")))
    fontsize = random.randint(80, 100)
    font = ImageFont.truetype(os.path.join(BASE_DIR, "fonts", fontname), size=fontsize)

    # Choose a random position for the text
    text_width, text_height = font.getsize(text)
    x = random.randint(0, max(0, width - text_width))
    y = random.randint(text_height, max(0, height - text_height))

    # Choose a random text color
    text_color = (random.randint(100, 255), random.randint(100, 255), random.randint(100, 255))

    # Draw the text on the image
    draw = ImageDraw.Draw(img)

    # Generate random shapes
    for _ in range(10):
        shape = random.choice(['square', 'circle'])
        shape_color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
        x1 = random.randint(0, max(0, width - 50))
        y1 = random.randint(0, max(0, height - 50))
        x2 = x1 + random.randint(20, 50)
        y2 = y1 + random.randint(20, 50)
        if shape == 'square':
            draw.rectangle((x1, y1, x2, y2), fill=shape_color)
        else:
            draw.ellipse((x1, y1, x2, y2), fill=shape_color)

    # Generate random lines
    for _ in range(10):
        line_color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
        x1 = random.randint(0, max(0, width - 50))
        y1 = random.randint(0, max(0, height - 50))
        x2 = x1 + random.randint(20, 50)
        y2 = y1 + random.randint(20, 50)
        draw.line((x1, y1, x2, y2), fill=line_color, width=random.randint(2, 6))

    draw.text((x, y), text, font=font, fill=text_color)

    # Save the image to a buffer
    buffer = io.BytesIO()
    img.save(buffer, format='JPEG')

    # Convert the buffer to a base64-encoded string
    base64_str = base64.b64encode(buffer.getvalue()).decode('utf-8')

    #returning data
    return (encode(key, text), base64_str)

def checkCaptcha(token, value) -> bool:
    if decode(key, token) == value:
        return True
    else: return False


import vcf
import os
from joblib import Parallel, delayed
import math
import itertools
import multiprocessing
import csv

vcf_file_ending = ".vcf"

variant_cap = 5
use_variant_cap = False

include_filenames = False

def open_files(country, main_dir, res_dir):
    input_dir = "{}/{}/Vcf".format(main_dir, country)
    #input_dir = "{}/{}".format(main_dir, country)
    vars_dir = "{}/vars".format(res_dir)
    vars_outfile = "{}/{}_variants.txt".format(vars_dir, country)

    all_vars = {}
    if not os.path.exists(vars_dir):
        os.makedirs(vars_dir)

    print('Reading VCF files...\n')
    # Stores all the files with specified ending in the input directory

    vcf_filenames = [os.path.join(input_dir, file) for file in os.listdir(input_dir) if file.endswith(vcf_file_ending)]

    print(
        "Found {} Vcf files in {}, checking validity and processing records...\n".format(len(vcf_filenames), input_dir))
    total_files_num = len(vcf_filenames)
    n_jobs = 40

    with Parallel(n_jobs=n_jobs) as parallelizer:
        group_size = math.ceil(len(vcf_filenames) / n_jobs)

        # Splits the filenames list into groups of size group_size
        buffer_names = [iter(vcf_filenames)] * group_size
        split_names = itertools.zip_longest(fillvalue=None, *buffer_names)

        # Assings split_names groups to different
        task_iterator = (delayed(read_files)(files) for files in split_names)
        results = parallelizer(task_iterator)
        for r in results:
            for p, d in r.items():
                if p in all_vars:
                    all_vars[p].update(d)
                else:
                    all_vars[p] = d

        return process_variants(all_vars, total_files_num, vars_outfile)

def read_files(filenames):
    files_remaining = len(filenames)
    temp_vars = {}
    for filename in filenames:
        files_remaining -= 1

        print("{}: {} files remaining".format(multiprocessing.current_process().name, files_remaining))
        if filename is None:
            continue
        vcf_values = [line for line in vcf.Reader(open(filename), 'r')]
        new_vars = check_variants(filename.split("/")[-1].split(".vcf")[0], vcf_values)
        for p, d in new_vars.items():
            if p in temp_vars:
                temp_vars[p].update(d)
            else:
                temp_vars[p] = d
    return temp_vars

def check_variants(filename, vcf_vals):
    variants = {}

    for record in vcf_vals:
        ref = record.REF
        alts = record.ALT
        if None in alts:
            continue
        alt = str(alts[0])
        unformatted_geno = [sample['GT'] for sample in record.samples][0]
        genotype = unformatted_geno.replace('0', ref).replace('1', alt)
        if record.POS not in variants.keys():
            variants[record.POS] = {filename: genotype}
        else:
            variants[record.POS].update({filename: genotype})

    return variants

def process_variants(avar, total_files_num, outfile):
    count_ciel = variant_cap
    single_variant_files = {}
    multi_variant_files = {}

    for pos, variants in avar.items():
        num_variants = len(variants.values())
        if num_variants > count_ciel:
            continue
        elif num_variants == 1:
            single_variant_files[pos] = variants
        else:
            multi_variant_files[pos] = variants

    return print_variants(single_variant_files, multi_variant_files, total_files_num, outfile)

def print_variants(single, multiple, total_num, outfile):
    single_num = len(single.keys())
    multi_num = len(multiple.keys())
    print("Found {} single variant files".format(single_num))
    print("Found {} multi variant files".format(multi_num))
    print("Writing results to {}".format(outfile))

    with open(outfile, 'w') as w:
        w.write("Single Variant Files ({}/{})\n".format(single_num, total_num))

        for pos, vars in single.items():
            w.write("Position: {}".format(pos))
            for f, g in vars.items():
                w.write("\tFile: {}\tVariant: {}".format(f, g))
            w.write("\n\n")

        w.write("\n\nMulti Variant Files ({}/{})\n".format(multi_num, total_num))

        for pos, vars in multiple.items():
            w.write("Position: {}".format(pos))
            for f, g in vars.items():
                w.write("\tFile: {}\tVariant: {}".format(f, g))
            w.write("\n\n")

    return {"Single": single_num, "Multiple": multi_num, "Total": total_num}

def run_all_countries():
    exclude = ["Vcf"]
    main_dir = "/home/jurgen/Results/"

    #main_dir = "input"
    countries = [file for file in os.listdir(main_dir) if os.path.isdir(os.path.join(main_dir, file))]
    results = "results"
    if not os.path.exists(results):
        os.makedirs(results)
    with open("{}/variant_counts.txt".format(results), 'w') as w:
        writer = csv.writer(w, delimiter="\t")
        writer.writerow(["Country", "Single", "Multiple", "Total"])
        for c in countries:
            if c in exclude:
                continue
            print("\nPreparing variant counter for {}\n\n".format(c))
            counts_dict = open_files(c, main_dir, results)
            single_percent = counts_dict["Single"] / counts_dict["Total"]
            multi_percent = counts_dict["Multiple"] / counts_dict["Total"]
            line = [c, "{} ({})".format(counts_dict["Single"], single_percent),
                    "{} ({})".format(counts_dict["Multiple"], multi_percent), counts_dict["Total"]]
            writer.writerow(line)

run_all_countries()


#!/usr/bin/env python
"""Script to import tags into hoover.

Receives a csv with tags as stdin. Use it like: tags.csv | -c collection -u username
"""
import subprocess
from argparse import ArgumentParser
import sys

parser = ArgumentParser()
parser.add_argument('-c', '--collection', dest='collection', required=True,
                    help='Collection for which the new tags should be added.')
parser.add_argument('-u', '--user', dest='username', required=True,
                    help='User for which tags should be added.')
parser.add_argument('-p', '--public', dest='public',
                    action='store_true', default=False,
                    help='Flag to set if the new tags should be public.')

args = parser.parse_args()

# check if any data has been received as stdin
if sys.stdin.isatty():
    print('No stdin received. Pass the csv with tags as stdin')
    print('Exiting')
    sys.exit(1)

# use devnull as stdin to prevent consuming the data that has been passed to the script
uuid_output = subprocess.run(['./liquid', 'dockerexec', 'hoover:search', './manage.py',
                              'getuuid', args.username],
                             capture_output=True,
                             stdin=subprocess.DEVNULL)

uuid = uuid_output.stdout.strip().decode('utf-8')

if not uuid:
    print(f'Error: Could not find uuid for user: {args.username}')
    sys.exit(1)

# stdin is passed to this subprocess
import_command = ['./liquid', 'dockerexec', 'hoover:snoop', './manage.py', 'importtags',
                  '-c', args.collection,
                  '--uuid', uuid,
                  '--user', args.username]
if args.public:
    import_command.append('--public')

subprocess.run(import_command)


from PIL import Image

def hide_text_in_image(image_path, message):
    # Ouvre l'image en mode lecture
    img = Image.open(image_path)
    # Convertit le message en une chaîne binaire
    message_bin = ''.join(format(ord(c), '08b') for c in message)
    message=f'[{len(message)}]'+message
    message_bin = ''.join(format(ord(c), '08b') for c in message)
    print(message)

    # Vérifie si le message est trop long pour être caché dans l'image
    if len(message_bin) > img.width * img.height:
        raise ValueError("Le message est trop long pour être caché dans l'image.")
    # Cachage du message dans les pixels de l'image
    pixels = img.load()
    index = 0
    for y in range(img.height):
        for x in range(img.width):
            if index < len(message_bin):
                pixel = pixels[x, y]
                r = pixel[0]
                # Remplace le bit de poids faible de chaque canal de couleur par un bit du message
                pixels[x, y] = (r & ~1 | int(message_bin[index]))
                index +=1
            else:
                break
    # Enregistre l'image modifiée avec le message caché
    img.save("../Decrypt_cpp/image_avec_message.png")

# Exemple d'utilisation
image_path = "img.png"
message = "Chiens qui baisent"
hide_text_in_image(image_path, message)


import argparse
import torch
import torch.onnx
from lib.config import config, update_config, hrnet
from collections import OrderedDict

def parse_args():
    parser = argparse.ArgumentParser(description='Train classification network')
    parser.add_argument('--cfg',
                        help='experiment configure file name',
                        required=False,
                        type=str,
                        default='./experiments/cls_hrnet_w18_sgd_lr5e-2_wd1e-4_bs32_x100.yaml')

    parser.add_argument('--modelDir',
                        help='model directory',
                        type=str,
                        default='')
    parser.add_argument('--logDir',
                        help='log directory',
                        type=str,
                        default='')
    parser.add_argument('--dataDir',
                        help='data directory',
                        type=str,
                        default='')
    parser.add_argument('--testModel',
                        help='testModel',
                        type=str,
                        default='')

    args = parser.parse_args()
    update_config(config, args)

    return args

def pth2onnx(input_file="", output_file=""):
    args = parse_args()

    checkpoint = torch.load("./output/imagenet/cls_hrnet_w18_sgd_lr5e-2_wd1e-4_bs32_x100/model_best.pth.tar", map_location='cpu')
    model = hrnet.get_cls_net(config)
    model.load_state_dict(checkpoint)
    model.eval()
    print(model)

    input_names = ["image_input"]
    output_names = ["output_1"]
    dynamic_axes = {'image': {0: '-1'}, 'class': {0: '-1'}}
    dummy_input = torch.randn(16, 3, 224, 224)
    torch.onnx.export(model, dummy_input, "hrnet_npu_16.onnx", input_names=input_names, dynamic_axes=dynamic_axes, output_names=output_names, verbose=True, opset_version=11)

if __name__ == "__main__":
    pth2onnx()


from zipfile import ZipFile

FILES = [ 'file-a', 'file-b', 'file-c' ]

for i in range(1, 100):
    filename = f'sub{str(i).rjust(3, "0")}.zip'
    with ZipFile(f'data/{filename}') as file:
        contents = file.namelist()
        if len(contents) != 3 or not all(f in contents for f in FILES):
            print(filename)