import math

import torch
import torch.nn as nn
import torch.nn.functional as F

from fairseq import options
from fairseq import utils

from fairseq.modules import (
    AdaptiveInput, AdaptiveSoftmax, CharacterTokenEmbedder, LearnedPositionalEmbedding, MultiheadAttention,
    SinusoidalPositionalEmbedding,
    ElmoTokenEmbedder)

from fairseq.tasks.fb_bert import BertTask
from . import (
    FairseqIncrementalDecoder, FairseqEncoder, FairseqModel, register_model,
    register_model_architecture,
)

def gelu(x):
    """Implementation of the gelu activation function.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

class BertLayerNorm(nn.Module):
    def __init__(self, hidden_size, eps=1e-12):
        """Construct a layernorm module in the TF style (epsilon inside the square root).
        """
        super(BertLayerNorm, self).__init__()
        self.gamma = nn.Parameter(torch.ones(hidden_size))
        self.beta = nn.Parameter(torch.zeros(hidden_size))
        self.variance_epsilon = eps

    def forward(self, x):
        u = x.mean(-1, keepdim=True)
        s = (x - u).pow(2).mean(-1, keepdim=True)
        x = (x - u) / torch.sqrt(s + self.variance_epsilon)
        return self.gamma * x + self.beta

@register_model('summerization_transformer')
class SummerizationTransformerModel(FairseqModel):
    """
    Transformer model from `"Attention Is All You Need" (Vaswani, et al, 2017)
    <https://arxiv.org/abs/1706.03762>`_.

    Args:
        encoder (TransformerEncoder): the encoder
        decoder (TransformerDecoder): the decoder

    The Transformer model provides the following named architectures and
    command-line arguments:

    .. argparse::
        :ref: fairseq.models.transformer_parser
        :prog:
    """

    def __init__(self, encoder, decoder):
        super().__init__(encoder, decoder)

    @staticmethod
    def add_args(parser):
        """Add model-specific arguments to the parser."""
        parser.add_argument('--dropout', type=float, metavar='D',
                            help='dropout probability')
        parser.add_argument('--attention-dropout', type=float, metavar='D',
                            help='dropout probability for attention weights')
        parser.add_argument('--relu-dropout', type=float, metavar='D',
                            help='dropout probability after ReLU in FFN')
        parser.add_argument('--encoder-embed-path', type=str, metavar='STR',
                            help='path to pre-trained encoder embedding')
        parser.add_argument('--encoder-embed-dim', type=int, metavar='N',
                            help='encoder embedding dimension')
        parser.add_argument('--encoder-ffn-embed-dim', type=int, metavar='N',
                            help='encoder embedding dimension for FFN')
        parser.add_argument('--encoder-attention-heads', type=int, metavar='N',
                            help='num encoder attention heads')
        parser.add_argument('--encoder-normalize-before', action='store_true',
                            help='apply layernorm before each encoder block')
        parser.add_argument('--encoder-learned-pos', action='store_true',
                            help='use learned positional embeddings in the encoder')
        parser.add_argument('--decoder-embed-dim', type=int, metavar='N',
                            help='decoder embedding dimension')
        parser.add_argument('--decoder-ffn-embed-dim', type=int, metavar='N',
                            help='decoder embedding dimension for FFN')
        parser.add_argument('--decoder-layers', type=int, metavar='N',
                            help='num decoder layers')
        parser.add_argument('--decoder-attention-heads', type=int, metavar='N',
                            help='num decoder attention heads')
        parser.add_argument('--decoder-learned-pos', action='store_true',
                            help='use learned positional embeddings in the decoder')
        parser.add_argument('--decoder-normalize-before', action='store_true',
                            help='apply layernorm before each decoder block')
        parser.add_argument('--share-decoder-input-output-embed', action='store_true',
                            help='share decoder input output embed')

    @classmethod
    def build_model(cls, args, task):
        """Build a new model instance."""

        # make sure all arguments are present in older models
        base_architecture(args)

        if not hasattr(args, 'max_source_positions'):
            args.max_source_positions = 512
        if not hasattr(args, 'max_target_positions'):
            args.max_target_positions = 512

        dict = task.source_dictionary

        def build_embedding(dictionary, embed_dim, path):

            assert path is not None
            args.short_seq_prob = 0.0
            task = BertTask(args, dictionary)
            models, _ = utils.load_ensemble_for_inference([path], task, {'remove_head': True, 'remove_pooled' : True, 'save_masks' : False})
            assert len(models) == 1, 'ensembles are currently not supported for elmo embeddings'
            pretrain_model = models[0]
            return pretrain_model

        encoder_embed_tokens = build_embedding(dict, args.encoder_embed_dim, args.encoder_embed_path)
        decoder_embed_tokens = encoder_embed_tokens.decoder.bert.embeddings.word_embeddings
        decoder_embed_positions = encoder_embed_tokens.decoder.bert.embeddings.position_embeddings

        encoder = EmbeddingEncoder(dict, encoder_embed_tokens)
        decoder = TransformerDecoder(args, dict, decoder_embed_tokens, decoder_embed_positions)

        return SummerizationTransformerModel(encoder, decoder)

class EmbeddingEncoder(FairseqEncoder):

    def __init__(self, dictionary, embed_tokens):
        super().__init__(dictionary)

        self.embedder = embed_tokens
        self.padding_idx = embed_tokens.task.dictionary.pad()

    def forward(self, src_tokens, src_lengths, segment):
        # embed tokens and positions
        x, _ = self.embedder(src_tokens, segment, apply_mask=False)

        # B x T x C -> T x B x C
        x = x.transpose(0, 1)

        # compute padding mask
        encoder_padding_mask = src_tokens.eq(self.padding_idx)
        if not encoder_padding_mask.any():
            encoder_padding_mask = None

        return {
            'encoder_out': x,  # T x B x C
            'encoder_padding_mask': encoder_padding_mask,  # B x T
        }

    def reorder_encoder_out(self, encoder_out, new_order):
        if encoder_out['encoder_out'] is not None:
            encoder_out['encoder_out'] = \
                encoder_out['encoder_out'].index_select(1, new_order)
        if encoder_out['encoder_padding_mask'] is not None:
            encoder_out['encoder_padding_mask'] = \
                encoder_out['encoder_padding_mask'].index_select(0, new_order)
        return encoder_out

    def max_positions(self):
        """Maximum input length supported by the encoder."""
        return float('inf')

class TransformerEncoder(FairseqEncoder):
    """
    Transformer encoder consisting of *args.encoder_layers* layers. Each layer
    is a :class:`TransformerEncoderLayer`.

    Args:
        args (argparse.Namespace): parsed command-line arguments
        dictionary (~fairseq.data.Dictionary): encoding dictionary
        embed_tokens (torch.nn.Embedding): input embedding
        left_pad (bool, optional): whether the input is left-padded. Default:
            ``True``
    """

    def __init__(self, args, dictionary, embed_tokens, embed_scale=None, left_pad=True):
        super().__init__(dictionary)
        self.dropout = args.dropout

        embed_dim = embed_tokens.embedding_dim
        self.padding_idx = embed_tokens.padding_idx
        self.max_source_positions = args.max_source_positions
        self.eos_idx = dictionary.eos()

        self.embed_tokens = embed_tokens
        self.embed_scale = math.sqrt(args.encoder_embed_dim) if embed_scale is None else embed_scale
        self.embed_positions = PositionalEmbedding(
            args.max_source_positions, embed_dim, self.padding_idx,
            left_pad=left_pad,
            learned=args.encoder_learned_pos,
        ) if not args.no_enc_token_positional_embeddings else None

        self.layers = nn.ModuleList([])
        self.layers.extend([
            TransformerEncoderLayer(args)
            for i in range(args.encoder_layers)
        ])
        self.register_buffer('version', torch.Tensor([2]))
        self.normalize = args.encoder_normalize_before
        if self.normalize:
            self.layer_norm = LayerNorm(embed_dim)

    def forward(self, src_tokens, src_lengths):
        """
        Args:
            src_tokens (LongTensor): tokens in the source language of shape
                `(batch, src_len)`
            src_lengths (torch.LongTensor): lengths of each source sentence of
                shape `(batch)`

        Returns:
            dict:
                - **encoder_out** (Tensor): the last encoder layer's output of
                  shape `(src_len, batch, embed_dim)`
                - **encoder_padding_mask** (ByteTensor): the positions of
                  padding elements of shape `(batch, src_len)`
        """
        # embed tokens and positions
        x = self.embed_scale * self.embed_tokens(src_tokens)
        if self.embed_positions is not None:
            x += self.embed_positions(src_tokens)
        x = F.dropout(x, p=self.dropout, training=self.training)

        # B x T x C -> T x B x C
        x = x.transpose(0, 1)

        # compute padding mask
        encoder_padding_mask = src_tokens.eq(self.padding_idx)
        if not encoder_padding_mask.any():
            encoder_padding_mask = None

        # encoder layers
        for layer in self.layers:
            x = layer(x, encoder_padding_mask)

        if self.normalize:
            x = self.layer_norm(x)

        return {
            'encoder_out': x,  # T x B x C
            'encoder_padding_mask': encoder_padding_mask,  # B x T
        }

    def reorder_encoder_out(self, encoder_out, new_order):
        """
        Reorder encoder output according to *new_order*.

        Args:
            encoder_out: output from the ``forward()`` method
            new_order (LongTensor): desired order

        Returns:
            *encoder_out* rearranged according to *new_order*
        """
        if encoder_out['encoder_out'] is not None:
            encoder_out['encoder_out'] = \
                encoder_out['encoder_out'].index_select(1, new_order)
        if encoder_out['encoder_padding_mask'] is not None:
            encoder_out['encoder_padding_mask'] = \
                encoder_out['encoder_padding_mask'].index_select(0, new_order)
        return encoder_out

    def max_positions(self):
        """Maximum input length supported by the encoder."""
        if self.embed_positions is None:
            return self.max_source_positions
        return min(self.max_source_positions, self.embed_positions.max_positions())

    def upgrade_state_dict(self, state_dict):
        """Upgrade a (possibly old) state dict for new versions of fairseq."""
        if isinstance(self.embed_positions, SinusoidalPositionalEmbedding):
            if 'encoder.embed_positions.weights' in state_dict:
                del state_dict['encoder.embed_positions.weights']
            state_dict['encoder.embed_positions._float_tensor'] = torch.FloatTensor(1)
        if utils.item(state_dict.get('encoder.version', torch.Tensor([1]))[0]) < 2:
            # earlier checkpoints did not normalize after the stack of layers
            self.layer_norm = None
            self.normalize = False
            state_dict['encoder.version'] = torch.Tensor([1])
        return state_dict

class TransformerDecoder(FairseqIncrementalDecoder):
    """
    Transformer decoder consisting of *args.decoder_layers* layers. Each layer
    is a :class:`TransformerDecoderLayer`.

    Args:
        args (argparse.Namespace): parsed command-line arguments
        dictionary (~fairseq.data.Dictionary): decoding dictionary
        embed_tokens (torch.nn.Embedding): output embedding
        no_encoder_attn (bool, optional): whether to attend to encoder outputs.
            Default: ``False``
        left_pad (bool, optional): whether the input is left-padded. Default:
            ``False``
    """

    def __init__(self, args, dictionary, embed_tokens, embed_positions, embed_scale=None, no_encoder_attn=False, left_pad=False,
                 final_norm=True):
        super().__init__(dictionary)
        self.dropout = args.dropout
        self.share_input_output_embed = args.share_decoder_input_output_embed

        input_embed_dim = embed_tokens
        self.embed_dim = args.decoder_embed_dim
        output_embed_dim = args.decoder_output_dim

        padding_idx = dictionary.pad()
        self.max_target_positions = args.max_target_positions

        self.embed_tokens = embed_tokens
        self.embed_scale = math.sqrt(self.embed_dim) if embed_scale is None else embed_scale

        self.project_in_dim = Linear(input_embed_dim, self.embed_dim,
                                     bias=False) if self.embed_dim != input_embed_dim.weight.size(1) else None

        self.embed_positions = embed_positions

        self.layers = nn.ModuleList([])
        self.layers.extend([
            TransformerDecoderLayer(args, no_encoder_attn)
            for _ in range(args.decoder_layers)
        ])

        self.adaptive_softmax = None

        self.project_out_dim = Linear(self.embed_dim, output_embed_dim, bias=False) \
            if self.embed_dim != output_embed_dim and not args.tie_adaptive_weights else None

        self.load_softmax = not getattr(args, 'remove_head', False)

        if self.load_softmax:
            if not self.share_input_output_embed:
                self.embed_out = nn.Parameter(torch.Tensor(len(dictionary), output_embed_dim))
                nn.init.normal_(self.embed_out, mean=0, std=output_embed_dim ** -0.5)
        self.normalize = args.decoder_normalize_before and final_norm
        if self.normalize:
            self.layer_norm = LayerNorm(self.embed_dim)

        self.apply(self.init_bert_weights)

    def init_bert_weights(self, module):
        """ Initialize the weights.
        """
        if isinstance(module, (nn.Linear, nn.Embedding)):
            # Slightly different from the TF version which uses truncated_normal for initialization
            # cf https://github.com/pytorch/pytorch/pull/5617
            module.weight.data.normal_(mean=0.0, std=0.02)
        elif isinstance(module, BertLayerNorm):
            module.beta.data.zero_()
            module.gamma.data.fill_(1.0)
        if isinstance(module, nn.Linear) and module.bias is not None:
            module.bias.data.zero_()

    def forward(self, prev_output_tokens, encoder_out=None, incremental_state=None):
        """
        Args:
            prev_output_tokens (LongTensor): previous decoder outputs of shape
                `(batch, tgt_len)`, for input feeding/teacher forcing
            encoder_out (Tensor, optional): output from the encoder, used for
                encoder-side attention
            incremental_state (dict): dictionary used for storing state during
                :ref:`Incremental decoding`

        Returns:
            tuple:
                - the last decoder layer's output of shape `(batch, tgt_len,
                  vocab)`
                - the last decoder layer's attention weights of shape `(batch,
                  tgt_len, src_len)`
        """
        # embed positions

        lm_decoder = not isinstance(self.embed_tokens, nn.Embedding)

        position_ids = torch.arange(prev_output_tokens.size(1)).type_as(prev_output_tokens)
        positions = self.embed_positions(
            position_ids,
        ) if self.embed_positions is not None else None

        if incremental_state is not None and not lm_decoder:
            prev_output_tokens = prev_output_tokens[:, -1:]
            if positions is not None:
                positions = positions[:, -1:]

        # embed tokens and positions
        x = self.embed_scale * self.embed_tokens(prev_output_tokens)

        if incremental_state is not None and lm_decoder:
            x = x[:, -1:]
            if positions is not None:
                positions = positions[:, -1:]

        if self.project_in_dim is not None:
            x = self.project_in_dim(x)

        if positions is not None:
            x += positions
        x = F.dropout(x, p=self.dropout, training=self.training)

        # B x T x C -> T x B x C
        x = x.transpose(0, 1)
        attn = None

        inner_states = [x]

        # decoder layers
        for layer in self.layers:
            x, attn = layer(
                x,
                encoder_out['encoder_out'] if encoder_out is not None else None,
                encoder_out['encoder_padding_mask'] if encoder_out is not None else None,
                incremental_state,
                self_attn_mask=self.buffered_future_mask(x) if incremental_state is None else None,
            )
            inner_states.append(x)

        if self.normalize:
            x = self.layer_norm(x)

        # T x B x C -> B x T x C
        x = x.transpose(0, 1)

        if self.project_out_dim is not None:
            x = self.project_out_dim(x)

        if self.adaptive_softmax is None and self.load_softmax:
            # project back to size of vocabulary
            if self.share_input_output_embed:
                x = F.linear(x, self.embed_tokens.weight)
            else:
                x = F.linear(x, self.embed_out)

        return x, {'attn': attn, 'inner_states': inner_states}

    def max_positions(self):
        """Maximum output length supported by the decoder."""
        return self.max_target_positions

    def buffered_future_mask(self, tensor):
        dim = tensor.size(0)
        if not hasattr(self, '_future_mask') or self._future_mask is None or self._future_mask.device != tensor.device:
            self._future_mask = torch.triu(utils.fill_with_neg_inf(tensor.new(dim, dim)), 1)
        if self._future_mask.size(0) < dim:
            self._future_mask = torch.triu(utils.fill_with_neg_inf(self._future_mask.resize_(dim, dim)), 1)
        return self._future_mask[:dim, :dim]

    def upgrade_state_dict_named(self, state_dict, name):
        pass

class TransformerEncoderLayer(nn.Module):
    """Encoder layer block.

    In the original paper each operation (multi-head attention or FFN) is
    postprocessed with: `dropout -> add residual -> layernorm`. In the
    tensor2tensor code they suggest that learning is more robust when
    preprocessing each layer with layernorm and postprocessing with:
    `dropout -> add residual`. We default to the approach in the paper, but the
    tensor2tensor approach can be enabled by setting
    *args.encoder_normalize_before* to ``True``.

    Args:
        args (argparse.Namespace): parsed command-line arguments
    """

    def __init__(self, args):
        super().__init__()
        self.embed_dim = args.encoder_embed_dim
        self.self_attn = MultiheadAttention(
            self.embed_dim, args.encoder_attention_heads,
            dropout=args.attention_dropout,
            pos_emb_type=args.attn_position_embedding_type,
            max_k_pos_emb=args.max_key_position_embedding,
            max_v_pos_emb=args.max_val_position_embedding,
            left_pad=LanguagePairDataset.LEFT_PAD_SOURCE,
        )
        self.dropout = args.dropout
        self.relu_dropout = args.relu_dropout
        self.normalize_before = args.encoder_normalize_before
        self.fc1 = Linear(self.embed_dim, args.encoder_ffn_embed_dim)
        self.fc2 = Linear(args.encoder_ffn_embed_dim, self.embed_dim)
        self.layer_norms = nn.ModuleList([LayerNorm(self.embed_dim) for i in range(2)])

    def forward(self, x, encoder_padding_mask):
        """
        Args:
            x (Tensor): input to the layer of shape `(seq_len, batch, embed_dim)`
            encoder_padding_mask (ByteTensor): binary ByteTensor of shape
                `(batch, src_len)` where padding elements are indicated by ``1``.

        Returns:
            encoded output of shape `(batch, src_len, embed_dim)`
        """
        residual = x
        x = self.maybe_layer_norm(0, x, before=True)
        x, _ = self.self_attn(query=x, key=x, value=x, key_padding_mask=encoder_padding_mask)
        x = F.dropout(x, p=self.dropout, training=self.training)
        x = residual + x
        x = self.maybe_layer_norm(0, x, after=True)

        residual = x
        x = self.maybe_layer_norm(1, x, before=True)
        x = F.relu(self.fc1(x))
        x = F.dropout(x, p=self.relu_dropout, training=self.training)
        x = self.fc2(x)
        x = F.dropout(x, p=self.dropout, training=self.training)
        x = residual + x
        x = self.maybe_layer_norm(1, x, after=True)
        return x

    def maybe_layer_norm(self, i, x, before=False, after=False):
        assert before ^ after
        if after ^ self.normalize_before:
            return self.layer_norms[i](x)
        else:
            return x

class TransformerDecoderLayer(nn.Module):
    """Decoder layer block.

    In the original paper each operation (multi-head attention, encoder
    attention or FFN) is postprocessed with: `dropout -> add residual ->
    layernorm`. In the tensor2tensor code they suggest that learning is more
    robust when preprocessing each layer with layernorm and postprocessing with:
    `dropout -> add residual`. We default to the approach in the paper, but the
    tensor2tensor approach can be enabled by setting
    *args.decoder_normalize_before* to ``True``.

    Args:
        args (argparse.Namespace): parsed command-line arguments
        no_encoder_attn (bool, optional): whether to attend to encoder outputs.
            Default: ``False``
    """

    def __init__(self, args, no_encoder_attn=False):
        super().__init__()
        self.embed_dim = args.decoder_embed_dim
        self.self_attn = MultiheadAttention(
            self.embed_dim, args.decoder_attention_heads,
            dropout=args.attention_dropout,
            #pos_emb_type=args.attn_position_embedding_type,
            #max_k_pos_emb=args.max_key_position_embedding,
            #max_v_pos_emb=args.max_val_position_embedding,
            #left_pad=LanguagePairDataset.LEFT_PAD_SOURCE,
        )
        self.dropout = args.dropout
        self.relu_dropout = args.relu_dropout
        self.normalize_before = args.decoder_normalize_before

        self.self_attn_layer_norm = BertLayerNorm(self.embed_dim)

        if no_encoder_attn:
            self.encoder_attn = None
            self.encoder_attn_layer_norm = None
        else:
            self.encoder_attn = MultiheadAttention(
                self.embed_dim, args.decoder_attention_heads,
                dropout=args.attention_dropout,
            )
            self.encoder_attn_layer_norm = BertLayerNorm(self.embed_dim)

        self.fc1 = nn.Linear(self.embed_dim, args.decoder_ffn_embed_dim)
        self.fc2 = nn.Linear(args.decoder_ffn_embed_dim, self.embed_dim)

        self.final_layer_norm = BertLayerNorm(self.embed_dim)
        self.need_attn = True

        self.onnx_trace = False

    def prepare_for_onnx_export_(self):
        self.onnx_trace = True

    def forward(self, x, encoder_out, encoder_padding_mask, incremental_state,
                prev_self_attn_state=None, prev_attn_state=None, self_attn_mask=None,
                self_attn_padding_mask=None):
        """
        Args:
            x (Tensor): input to the layer of shape `(seq_len, batch, embed_dim)`
            encoder_padding_mask (ByteTensor): binary ByteTensor of shape
                `(batch, src_len)` where padding elements are indicated by ``1``.

        Returns:
            encoded output of shape `(batch, src_len, embed_dim)`
        """
        residual = x
        x = self.maybe_layer_norm(self.self_attn_layer_norm, x, before=True)
        if prev_self_attn_state is not None:
            if incremental_state is None:
                incremental_state = {}
            prev_key, prev_value = prev_self_attn_state
            saved_state = {"prev_key": prev_key, "prev_value": prev_value}
            self.self_attn._set_input_buffer(incremental_state, saved_state)
        x, _ = self.self_attn(
            query=x,
            key=x,
            value=x,
            key_padding_mask=self_attn_padding_mask,
            incremental_state=incremental_state,
            need_weights=False,
            attn_mask=self_attn_mask,
        )
        x = F.dropout(x, p=self.dropout, training=self.training)
        x = residual + x
        x = self.maybe_layer_norm(self.self_attn_layer_norm, x, after=True)

        attn = None
        if self.encoder_attn is not None:
            residual = x
            x = self.maybe_layer_norm(self.encoder_attn_layer_norm, x, before=True)
            if prev_attn_state is not None:
                if incremental_state is None:
                    incremental_state = {}
                prev_key, prev_value = prev_attn_state
                saved_state = {"prev_key": prev_key, "prev_value": prev_value}
                self.encoder_attn._set_input_buffer(incremental_state, saved_state)
            x, attn = self.encoder_attn(
                query=x,
                key=encoder_out,
                value=encoder_out,
                key_padding_mask=encoder_padding_mask,
                incremental_state=incremental_state,
                static_kv=True,
                need_weights=(not self.training and self.need_attn),
            )
            x = F.dropout(x, p=self.dropout, training=self.training)
            x = residual + x
            x = self.maybe_layer_norm(self.encoder_attn_layer_norm, x, after=True)

        residual = x
        x = self.maybe_layer_norm(self.final_layer_norm, x, before=True)
        x = gelu(self.fc1(x))
        x = F.dropout(x, p=self.relu_dropout, training=self.training)
        x = self.fc2(x)
        x = F.dropout(x, p=self.dropout, training=self.training)
        x = residual + x
        x = self.maybe_layer_norm(self.final_layer_norm, x, after=True)
        if self.onnx_trace:
            saved_state = self.self_attn._get_input_buffer(incremental_state)
            self_attn_state = saved_state["prev_key"], saved_state["prev_value"]
            return x, attn, self_attn_state
        return x, attn

    def maybe_layer_norm(self, layer_norm, x, before=False, after=False):
        assert before ^ after
        if after ^ self.normalize_before:
            return layer_norm(x)
        else:
            return x

    def make_generation_fast_(self, need_attn=False, **kwargs):
        self.need_attn = need_attn

@register_model_architecture('summerization_transformer', 'vanilla_small')
def base_architecture(args):
    args.encoder_embed_path = getattr(args, 'encoder_embed_path', None)
    args.encoder_embed_dim = getattr(args, 'encoder_embed_dim', 512)
    args.encoder_ffn_embed_dim = getattr(args, 'encoder_ffn_embed_dim', 2048)
    args.encoder_attention_heads = getattr(args, 'encoder_attention_heads', 8)
    args.encoder_normalize_before = getattr(args, 'encoder_normalize_before', False)
    args.decoder_embed_dim = getattr(args, 'decoder_embed_dim', args.encoder_embed_dim)
    args.decoder_ffn_embed_dim = getattr(args, 'decoder_ffn_embed_dim', args.encoder_ffn_embed_dim)
    args.decoder_layers = getattr(args, 'decoder_layers', 3)
    args.decoder_attention_heads = getattr(args, 'decoder_attention_heads', 8)
    args.decoder_normalize_before = getattr(args, 'decoder_normalize_before', False)
    args.decoder_learned_pos = getattr(args, 'decoder_learned_pos', False)
    args.attention_dropout = getattr(args, 'attention_dropout', 0.)
    args.relu_dropout = getattr(args, 'relu_dropout', 0.)
    args.dropout = getattr(args, 'dropout', 0.1)
    args.decoder_output_dim = getattr(args, 'decoder_output_dim', args.decoder_embed_dim)
    args.decoder_input_dim = getattr(args, 'decoder_input_dim', args.decoder_embed_dim)

# parameters used in the "Attention Is All You Need" paper (Vaswani, et al, 2017)
@register_model_architecture('summerization_transformer', 'summerization_bert')
def transformer_vaswani_wmt_en_de_big(args):
    args.encoder_embed_dim = getattr(args, 'encoder_embed_dim', 768)
    args.encoder_ffn_embed_dim = getattr(args, 'encoder_ffn_embed_dim', 3072)
    args.encoder_attention_heads = getattr(args, 'encoder_attention_heads', 12)
    args.encoder_normalize_before = getattr(args, 'encoder_normalize_before', False)
    args.decoder_embed_dim = getattr(args, 'decoder_embed_dim', 768)
    args.decoder_ffn_embed_dim = getattr(args, 'decoder_ffn_embed_dim', 3072)
    args.decoder_attention_heads = getattr(args, 'decoder_attention_heads', 12)
    args.dropout = getattr(args, 'dropout', 0.1)
    base_architecture(args)


# mod5
import os
import zipfile
import pandas as pd
import numpy as np
import re
from re import search
from nltk.tokenize import word_tokenize
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import classification_report, plot_confusion_matrix
import matplotlib.pyplot as plt
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.pipeline import Pipeline
from sklearn.ensemble import RandomForestClassifier

DATA_PATH = '../data/'
# if the path or names of the data hosted on Kaggle changes the following
# four constants will need updated
FAKE_TRUE_DATA_ARCHIVE = 'fake-and-real-news-dataset.zip'
FAKE_DATA_FILE = 'Fake.csv'
TRUE_DATA_FILE = 'True.csv'
FAKE_TRUE_KAGGLE_LOCATION = 'clmentbisaillon/fake-and-real-news-dataset'

GUARDIAN_DATA_ARCHIVE = 'guardian-news-dataset.zip'
GUARDIAN_DATA_FILE = 'combined_filtered.csv'
GUARDIAN_KAGGLE_DATA_LOCATION = 'sameedhayat/guardian-news-dataset'

RANDOM_STATE = 42

def remove_single_characters(word_list, exception_list):
    """Remove all the single characters, except those on the exception list"""
    return [w for w in word_list if (len(w) > 1 or w in exception_list)]

def remove_words(word_list, words_to_remove):
    """Remove all the words in the words_to_remove list from the words_list"""
    return [w for w in word_list if w not in words_to_remove]

def lower_unless_all_caps(string_):
    """
    Make all words in the input string lowercase unless that
    word is in all caps
    """
    words = string_.split()
    processed_words = [w.lower() if not (w.isupper() and len(w) > 1) else w for w in words]
    return ' '.join(processed_words)

def tokenize_and_normalize_title_and_text(title, text):
    """Combine, tokenize, and normalize the title and text of a news story"""

    URL_REGEX = r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»“”‘’]))"
    TWITTER_HANDLE_REGEX = r'(?<=^|(?<=[^\w]))(@\w{1,15})\b'
    DATE_WORDS = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday',
              'saturday', 'sunday', 'january', 'february', 'march', 'april',
             'may', 'june', 'july', 'august', 'september', 'october',
             'november', 'december']

    title_text = ' '.join([title, text])
    title_text = re.sub(URL_REGEX, '{link}', title_text)
    title_text = re.sub(TWITTER_HANDLE_REGEX, '@twitter-handle', title_text)
    title_text = lower_unless_all_caps(title_text)
    title_text = re.sub(r'\d+', ' ', title_text)
    title_text = re.sub(r'\(reuters\)', ' ', title_text)
    tokens = word_tokenize(title_text)
    tokens = remove_single_characters(tokens, ['i', '!'])
    tokens = remove_words(tokens, ["'s"])
    tokens = remove_words(tokens, DATE_WORDS)

    return tokens

def not_a_date(string):
    """
    Check if the input string can be converted to a date.

    Return:
    True if it is not a date
    False if it is a date
    """
    try:
        date = pd.to_datetime(string)
        return False
    except ValueError:
        return True

def lowercase_and_only_expanded_stopwords(doc):
    """Remove stopwords and lowercase tokens"""

    # Get list of common words (stop words)

    # using expanded stopwords list from https://gist.github.com/sebleier/554280

    gist_file = open(DATA_PATH + "gist_stopwords.txt", "r")
    try:
        content = gist_file.read()
        expanded_stopwords = content.split(",")
    finally:
        gist_file.close()

    # Remove some words that are particular to the dataset

    expanded_stopwords.remove('via')
    expanded_stopwords.remove('eu')
    expanded_stopwords.remove('uk')

    return [token.lower() for token in doc if token.lower() in expanded_stopwords]

def passthrough(doc):
    """passthrough function for use in the pipeline because the text is already tokenized"""
    return doc

# def download_data():
#     """
#     Download data from Kaggle and save in DATA_PATH directory
#     """
#
#     try:
#         os.mkdir(DATA_PATH)
#     except FileExistsError:
#         print('Data directory already exists')
#         pass
#
#     # Download Fake and True News Dataset
#
#     # assumes that kaggle has been installed and an api key is correctly installed
#     if not os.path.exists(DATA_PATH + FAKE_TRUE_DATA_ARCHIVE):
#         !kaggle datasets download -d $FAKE_TRUE_KAGGLE_LOCATION -p $DATA_PATH
#
#     if not os.path.exists(DATA_PATH + FAKE_DATA_FILE):
#         with zipfile.ZipFile(DATA_PATH + DATA_ARCHIVE, 'r') as zip_ref:
#             zip_ref.extractall(path=DATA_PATH)
#
#     # Download Guardian Dataset
#
#     # assumes that kaggle has been installed and an api key is correctly installed
#     if not os.path.exists(DATA_PATH + GUARDIAN_DATA_ARCHIVE):
#         !kaggle datasets download -d $GUARDIAN_KAGGLE_DATA_LOCATION -p $DATA_PATH
#
#     if not os.path.exists(DATA_PATH + GUARDIAN_DATA_FILE):
#         with zipfile.ZipFile(DATA_PATH + GUARDIAN_DATA_ARCHIVE, 'r') as zip_ref:
#             zip_ref.extractall(path=DATA_PATH)

def load_data():
    """
    Load data stored in .csv files into dataframes
    """

    fake_df = pd.read_csv(DATA_PATH + FAKE_DATA_FILE)
    true_df = pd.read_csv(DATA_PATH + TRUE_DATA_FILE)
    guard_df = pd.read_csv(DATA_PATH + GUARDIAN_DATA_FILE)

    # add labels to data frames. All the Guardian news stories are assumed to be
    # true because The Guardian is not designated as a "Fake" news source.

    fake_df['label'] = 'fake'
    true_df['label'] = 'true'
    guard_df['label'] = 'true'

    # Filter the Guardian dataset to only contain "Politics" stories

    guard_politics_df = guard_df[guard_df.sectionName == 'Politics']

    # Remove unneeded columns from Guardian Dataset

    guard_clean_df = guard_politics_df[['fields.bodyText', 'webTitle', 'label']]

    # Update columns in guard_clean_df to match fake and true dfs

    guard_clean_df.columns = ['text', 'title', 'label']

    # Combine fake and true clean dfs for futher cleaning in case there are any
    # stories that are labeled Fake and True

    fake_true_clean_df = pd.concat([fake_df, true_df], ignore_index=True)

    return fake_true_clean_df, guard_clean_df

def clean_data(fake_true_clean_df, guard_clean_df):
    """
    Clean the input dataframes
    """
    # Remove Duplicates

    fake_true_clean_df = fake_true_clean_df.drop_duplicates(ignore_index=True)
    guard_clean_df = guard_clean_df.drop_duplicates(ignore_index=True)

    # Remove rows with Nan, first assigning spaces to Nan

    fake_true_clean_df = fake_true_clean_df.replace(r'^\s*$', np.nan, regex=True)
    guard_clean_df = guard_clean_df.replace(r'^\s*$', np.nan, regex=True)

    fake_true_clean_df.dropna(inplace=True)
    guard_clean_df.dropna(inplace=True)

    # Drop Duplicate News Stories.  Duplicate Titles are OK

    fake_true_clean_df = fake_true_clean_df.drop_duplicates(['text'], ignore_index=True)
    guard_clean_df = guard_clean_df.drop_duplicates(['text'], ignore_index=True)

    # Check the date column in fake_true_clean_df to find malformed data and remove it

    bad_date_indexes = fake_true_clean_df[fake_true_clean_df['date'].apply(not_a_date)].index

    fake_true_clean_df = fake_true_clean_df.drop(bad_date_indexes, axis=0)

    return fake_true_clean_df, guard_clean_df

def normalize_and_tokenize(fake_true_clean_df, guard_clean_df):
    """
    Normalize the title and text and the tokenize the combined columns
    """

    fake_true_clean_df['tt_tokens'] = fake_true_clean_df.apply(lambda row: tokenize_and_normalize_title_and_text(row['title'], row['text']), axis=1)

    guard_clean_df['tt_tokens'] = guard_clean_df.apply(lambda row: tokenize_and_normalize_title_and_text(row['title'], row['text']), axis=1)

    return fake_true_clean_df, guard_clean_df

def integrate_datasets(fake_true_clean_df, guard_clean_df):
    """
    Take a random sample of the true news stories from fake_true_clean_df that is 1/2
    the size of the fake stories. Take a random sample that is 1/2
    the size of the fake stories of the stories in guard_clean_df
    which are also labeled true.  Join these two samples with all the fake News
    stories.
    """

    df_fake = fake_true_clean_df[fake_true_clean_df.label == 'fake']
    df_true = fake_true_clean_df[fake_true_clean_df.label == 'true']

    true_sample_size = int(len(df_fake) / 2)

    df_true_sample = df_true.sample(true_sample_size, random_state=RANDOM_STATE)
    df_guard_sample = guard_clean_df.sample(true_sample_size, random_state=RANDOM_STATE)

    df_all = pd.concat([df_fake[['tt_tokens', 'label']],
                        df_true_sample[['tt_tokens', 'label']],
                        df_guard_sample[['tt_tokens', 'label']]],
                       axis=0)

    return df_all

def train_model(df):
    """
    Train a Random Forest Classifier on the input news dataset using tf/idf to
    represent the text.  Only words from a set list of common words will be used
    in the model.
    """

    # Split the data
    y = df.label
    X = df.drop('label', axis=1)

    X_train, X_test, y_train, y_test = train_test_split(X,
                                                        y,
                                                        test_size=0.2,
                                                        random_state=RANDOM_STATE,
                                                        stratify=y)

    # Encode labels

    le = LabelEncoder()
    y_train_enc = le.fit_transform(y_train)
    y_test_enc = le.transform(y_test)

    # Define pipeline for the model

    pipeline = Pipeline([('bow',CountVectorizer(#min_df = 5,
                                                preprocessor = lowercase_and_only_expanded_stopwords,
                                                tokenizer = passthrough,
                                                #max_df = 1.0,
                                                ngram_range = (1, 1))),
                         ('tfidf', TfidfTransformer()),
                         ('classifier', RandomForestClassifier(n_jobs = -1,
                                                               random_state = RANDOM_STATE,
                                                               min_samples_split = 0.005,
                                                               max_depth = None)),
                        ])

    # Fit the model

    pipeline.fit(X_train, y_train_enc)

    # Display classification reports for the test sets

    y_hat_test = pipeline.predict(X_test)

    print(f'Classification Report for the Test Set')
    print(classification_report(y_test_enc, y_hat_test, target_names=le.classes_))

    matrix = plot_confusion_matrix(pipeline,
                                   X_test,
                                   y_test_enc,
                                   display_labels = le.classes_,
                                   cmap = plt.cm.Blues,
                                   xticks_rotation = 70,
                                   values_format = 'd')

    matrix.ax_.set_title(f'Test Set Confustion Matrix, without Normalization')

    plt.show()

def main():
    # download_data()
    fake_true_clean_df, guard_clean_df = load_data()
    fake_true_clean_df, guard_clean_df = clean_data(fake_true_clean_df, guard_clean_df)
    fake_true_clean_df, guard_clean_df = normalize_and_tokenize(fake_true_clean_df, guard_clean_df)
    df_all = integrate_datasets(fake_true_clean_df, guard_clean_df)
    print(len(df_all))
    train_model(df_all)

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
# -*- encoding: utf-8 -*-
'''
@File    :  sp_mvg.py
@Time    :  2020/12/02 21:12:19
@Author  :  Kearney
@Version :  0.0.0
@Contact :  191615342@qq.com
@License :  GPL 3.0
@Desc    :  一个精灵的原地动画-超级玛丽-多精灵组图。一张图上有多个精灵的多个形态
    ref：   https://blog.csdn.net/qq_39687901/article/details/88422493#commentBox
'''
import pygame
import sys

BLACK = (0, 0, 0)

class Block(pygame.sprite.Sprite):
    def __init__(self, screen, filename, position):
        pygame.sprite.Sprite.__init__(self)
        self.screen = screen
        try:
            self.image = pygame.image.load(filename)
        except Exception as e:  # 图片文件记载错误
            print("温馨提示： ", e, "， 请正确配置图片文件")
            raise
        self.rect = self.image.get_rect()  # 获取矩形区域
        # (self.rect.x, self.rect.y) = position  # 设置初始位置
        self.position = position

    def update(self):
        # blit参数： 显示的图像，显示图像的位置，要显示图片的那个矩形区域
        # self.screen.blit(self.image, self.position)
        self.screen.blit(self.image, self.position, (0, 0, 95, 60))
        self.screen.blit(self.image, (50, 400), (190, 32, 320, 64))

pygame.init()
screen = pygame.display.set_mode([800, 600])
clock = pygame.time.Clock()

player = Block(screen, './img/mario.png', (50, 200))  # 创建

while True:
    clock.tick(30)  # 设置帧速率为30
    screen.fill(BLACK)  # 全屏填充背景色
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            pygame.quit()  # 退出循环
            sys.exit(0)  # 结束程序

    player.update()
    pygame.display.update()  # 更新画面


from __future__ import unicode_literals

from unittest import TestCase

import six
from hamcrest import assert_that, equal_to, none, only_contains

from storops.lib.parser import PropMapper, PropDescriptor, OutputParser

__author__ = 'Cedric Zhuang'

class PropMapperTest(TestCase):
    def test_camel_case_to_under_score(self):
        test_data = {
            'AbcDef': 'ABC_DEF',
            'abc def': 'ABC_DEF',
            'abc:': 'ABC',
            'SPAWrites': 'SPA_WRITES',
            'Is Thin LUN': 'IS_THIN_LUN',
            'TestCIMElement': 'TEST_CIM_ELEMENT'
        }

        for (k, v) in six.iteritems(test_data):
            assert_that(PropMapper.camel_case_to_under_score(k).upper(),
                        equal_to(v))

    def test_camel_case_to_under_score_with_delimiter(self):
        test_data = {
            'AbcDef': 'ABC.DEF',
            'abc def': 'ABC.DEF',
            'Is Thin LUN': 'IS.THIN.LUN',
            'abc:': 'ABC',
            'SPAWrites': 'SPA.WRITES',
            'TestCIMElement': 'TEST.CIM.ELEMENT',
            'VALUE_ARRAY': 'VALUE.ARRAY',
            'LUNs': 'LUNS',
            'Source LUN(s)': 'SOURCE.LUNS',
            'Capacity (GBs)': 'CAPACITY.GBS'
        }

        for (k, v) in six.iteritems(test_data):
            assert_that(PropMapper.camel_case_to_under_score(k, '.').upper(),
                        equal_to(v))

A = PropDescriptor('-a', 'Prop A (name):', 'prop_a')
B = PropDescriptor('-b', 'Prop B:')
C = PropDescriptor('-c', 'Prop C:')
ID = PropDescriptor(None, 'ID:', is_index=True)

class DemoParser(OutputParser):
    def __init__(self):
        super(DemoParser, self).__init__()
        self.add_property(A, B, C, ID)

class OutputParserTest(TestCase):
    def test_sequence(self):
        parser = DemoParser()
        index = parser.index_property
        assert_that(index.sequence, equal_to(3))

    def test_property_key(self):
        parser = DemoParser()
        assert_that(parser.has_property_key('prop_a'), equal_to(True))
        assert_that(parser.has_property_key('prop_b'), equal_to(True))
        assert_that(parser.has_property_key('D'), equal_to(False))

    def test_get_property_label_found(self):
        parser = DemoParser()
        assert_that(parser.get_property_label('prop_a'),
                    equal_to('Prop A (name):'))

    def test_get_property_label_not_found(self):
        parser = DemoParser()
        assert_that(parser.get_property_label('na'), none())

    def test_get_property_key_found(self):
        parser = DemoParser()
        assert_that(parser.get_property_key('Prop A (name):'),
                    equal_to('prop_a'))

    def test_get_property_key_not_found(self):
        parser = DemoParser()
        assert_that(parser.get_property_key('Prop B (name):'), none())

    def test_property_names(self):
        parser = DemoParser()
        assert_that(parser.property_names,
                    only_contains('id', 'prop_a', 'prop_b', 'prop_c'))


import sys

import numpy as np
import pyqtgraph as pg
from PIL import Image
from PyQt5.QtWidgets import QApplication, QVBoxLayout, QPushButton, QWidget, QFileDialog
from pyqtgraph import ImageView

# 设置 PyQtGraph 显示配置
########################################################################################################################
# 设置显示背景色为白色，默认为黑色
pg.setConfigOption('background', 'w')
# 设置显示前景色为黑色，默认为灰色
pg.setConfigOption('foreground', 'k')
# 设置图像显示以行为主，默认以列为主
pg.setConfigOption('imageAxisOrder', 'row-major')

class PyQtGraphicDemo(QWidget):
    def __init__(self, parent=None):
        super(PyQtGraphicDemo, self).__init__(parent)

        self.resize(600, 400)

        # 图像显示控件
        self.graphicsView = ImageView(self)
        # 隐藏直方图，菜单按钮，ROI
        self.graphicsView.ui.histogram.hide()
        self.graphicsView.ui.menuBtn.hide()
        self.graphicsView.ui.roiBtn.hide()

        image = Image.open(r'./stockimg/a.jpg')
        if image is not None:
            # 如果之前未设置显示选项以行为主，这里需要对显示图像进行转置
            self.graphicsView.setImage(np.array(image))

        self.verticalLayout = QVBoxLayout(self)
        self.verticalLayout.addWidget(self.graphicsView)

        # 设置窗口布局
        self.setLayout(self.verticalLayout)

if __name__ == '__main__':
    app = QApplication(sys.argv)
    demo = PyQtGraphicDemo()
    demo.show()
    sys.exit(app.exec_())


import sys
import torch
import torch_xla
import torch_xla.core.xla_model as xm
import torch_xla.distributed.xla_multiprocessing as xmp

def _mp_fn(index):
  device = xm.xla_device()
  real_device = xm.xla_real_devices([str(device)])[0]
  if real_device.startswith('TPU:'):
    ones = torch.ones((2, 3))
    twos = ones + 1.0
    xones = ones.to(device)
    xtwos = twos.to(device)
    xm.all_reduce('sum', [xones, xtwos])

    if (not xones.cpu().allclose(ones * float(xm.xrt_world_size())) or
        not xtwos.cpu().allclose(twos * float(xm.xrt_world_size()))):
      print('CrossReplicaSum produced wrong reductions')
      print(xones, file=sys.stderr)
      sys.exit(1)
  else:
    print(
        'Default device {} is not a TPU device'.format(real_device),
        file=sys.stderr)

if __name__ == '__main__':
  xmp.spawn(_mp_fn, args=())


from django.http import Http404
from django.shortcuts import render
from django.views.generic import ListView, DetailView

from main.models import ProductAttribute, Product, FeatureProduct, Category
from main.views import prepare_context

class ProductView(ListView):
    model = ProductAttribute
    paginate_by = 15
    context_object_name = 'productattributes'
    template_name = 'shop/product.html'

    def get_queryset(self):
        if 'pk' in self.kwargs:
            return ProductAttribute.objects.filter(product_id=self.kwargs['pk'])
        else:
            return ProductAttribute.objects.all()

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context = prepare_context(context)
        if 'pk' in self.kwargs:
            model = Product.objects.get(product_id=self.kwargs['pk'])
            context['model'] = model

        context['categories'] = Category.objects.all()[:4]
        context['products'] = Product.objects.all()[:4]

        categories = Category.objects.all()[:4]
        products = Product.objects.all()[:4]
        context['group'] = zip(categories, products)

        return context

class ProductAtributeView(DetailView):
    model = ProductAttribute
    template_name = 'shop/product_detail.html'
    """
    A base view for displaying a single object
    """

    # def get(self, request, *args, **kwargs):
    #     self.object = self.get_object()
    #     context = self.get_context_data(object=self.object)
    #     return self.render_to_response(context)

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context = prepare_context(context)
        context['categories'] = Category.objects.all()[:4]
        context['products'] = Product.objects.all()[:4]
        context['model'] = ProductAttribute.objects.get(pk=self.object.pk)
        if context['model'] is not None:
            obj = context['model']
            context['obj_features'] = FeatureProduct.objects.filter(product_id=obj.product_id).order_by('-feature_id',
                                                                                'feature_value_id')
            context['obj_attributes'] = obj.attributes.all()

            context['models'] = ProductAttribute.objects.filter(product_id=obj.product_id)
        else:
            context['models'] = ProductAttribute.objects.all()[:4]
        return context

class ShopView(ListView):
    model = ProductAttribute
    paginate_by = 8
    queryset = ProductAttribute.objects.all()
    context_object_name = 'productattributes'
    template_name = 'shop/index.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context = prepare_context(context)
        context['categories'] = Category.objects.all()[:4]
        context['products'] = Product.objects.all()[:4]
        return context

def get_detail(request, pk):
    try:
        pks = pk.split("_")
    except ProductAttribute.DoesNotExist:
        raise Http404('pk does not exist')

    return render(request, 'shop/details.html',
                  context={'pks': pks})


# coding=utf-8

# from django.forms import ModelForm
# from universities.models import University
from django import forms

# class FormUniversity(ModelForm):
#     class Meta:
#         model = University
#         exclude = ['id']

MY_CHOICES = (
    ('1', 'Carrera'),
    ('2', 'Universidad'),
    ('3', 'Facultad'),
    ('4', 'Sede'),
    ('5', 'Departamento'),
    ('6', 'Año'),
    ('7', 'Tipo Universidad'),
    )

class FormSearch(forms.Form):
    text = forms.CharField(label='Busqueda', error_messages={'required': 'Este campo es obligatorio'})
    options = forms.ChoiceField(choices=MY_CHOICES, widget=forms.RadioSelect, error_messages={'required': 'Este campo es obligatorio'})

# class FormUniversity(forms.Form):
#     university = forms.ModelMultipleChoiceField(queryset=University.objects.all(), error_messages={'required': 'Debe ingresar una universidad'})


import os
import numpy as np

import tensorflow as tf
import tensorflow_hub as hub
import tensorflow_datasets as tfds
import tensorflow_text as text  # A dependency of the preprocessing model
import tensorflow_addons as tfa

from absl import app
from absl import flags
from absl import logging

from official.nlp import optimization

TFHUB_HANDLE_ENCODER = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3'
TFHUB_HANDLE_PREPROCESS = 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3'
TFDS_NAME = 'glue/cola'
NUM_CLASSES = 2
SENTENCE_FEATURE = 'sentence'

LOCAL_MODEL_DIR = '/tmp/saved_model'
LOCAL_TB_DIR = '/tmp/logs'
LOCAL_CHECKPOINT_DIR = '/tmp/checkpoints'

FLAGS = flags.FLAGS
flags.DEFINE_integer('epochs', 2, 'Nubmer of epochs')
flags.DEFINE_integer('per_replica_batch_size', 16, 'Per replica batch size')
flags.DEFINE_float('init_lr', 2e-5, 'Initial learning rate')
flags.DEFINE_float('dropout_ratio', 0.1, 'Dropout ratio')

def make_bert_preprocess_model(sentence_features, seq_length=128):
    """Returns a model mapping string features to BERT inputs."""

    input_segments = [
        tf.keras.layers.Input(shape=(), dtype=tf.string, name=ft)
        for ft in sentence_features]

    # Tokenize the text to word pieces.
    bert_preprocess = hub.load(TFHUB_HANDLE_PREPROCESS)
    tokenizer = hub.KerasLayer(bert_preprocess.tokenize, name='tokenizer')
    segments = [tokenizer(s) for s in input_segments]

    # Pack inputs. The details (start/end token ids, dict of output tensors)
    # are model-dependent, so this gets loaded from the SavedModel.
    packer = hub.KerasLayer(bert_preprocess.bert_pack_inputs,
                            arguments=dict(seq_length=seq_length),
                            name='packer')
    model_inputs = packer(segments)
    return tf.keras.Model(input_segments, model_inputs)

def build_classifier_model(num_classes, dropout_ratio):
    """Creates a text classification model based on BERT encoder."""

    inputs = dict(
        input_word_ids=tf.keras.layers.Input(shape=(None,), dtype=tf.int32),
        input_mask=tf.keras.layers.Input(shape=(None,), dtype=tf.int32),
        input_type_ids=tf.keras.layers.Input(shape=(None,), dtype=tf.int32),
    )

    encoder = hub.KerasLayer(TFHUB_HANDLE_ENCODER, trainable=True, name='encoder')
    net = encoder(inputs)['pooled_output']
    net = tf.keras.layers.Dropout(rate=dropout_ratio)(net)
    net = tf.keras.layers.Dense(num_classes, activation=None, name='classifier')(net)
    return tf.keras.Model(inputs, net, name='prediction')

def get_data_pipeline(in_memory_ds, info, split,
                      batch_size,  bert_preprocess_model):
    """Creates a sentence preprocessing pipeline."""

    is_training = split.startswith('train')
    dataset = tf.data.Dataset.from_tensor_slices(in_memory_ds[split])
    num_examples = info.splits[split].num_examples

    if is_training:
        dataset = dataset.shuffle(num_examples)
        dataset = dataset.repeat()
    dataset = dataset.batch(batch_size)
    dataset = dataset.map(lambda ex: (bert_preprocess_model(ex), ex['label']))
    dataset = dataset.cache().prefetch(buffer_size=tf.data.AUTOTUNE)
    return dataset, num_examples

def set_job_dirs():
    """Sets job directories based on env variables set by Vertex AI."""

    model_dir = os.getenv('AIP_MODEL_DIR', LOCAL_MODEL_DIR)
    tb_dir = os.getenv('AIP_TENSORBOARD_LOG_DIR', LOCAL_TB_DIR)
    checkpoint_dir = os.getenv('AIP_CHECKPOINT_DIR', LOCAL_CHECKPOINT_DIR)

    return model_dir, tb_dir, checkpoint_dir

def main(argv):
    """Starts a training run."""

    del argv
    logging.info('Setting up training.')
    logging.info('   epochs: {}'.format(FLAGS.epochs))
    logging.info('   per_replica_batch_size: {}'.format(FLAGS.per_replica_batch_size))
    logging.info('   init_lr: {}'.format(FLAGS.init_lr))
    logging.info('   dropout_ratio: {}'.format(FLAGS.dropout_ratio))

    # Set distribution strategy
    strategy = tf.distribute.MirroredStrategy()

    global_batch_size = (strategy.num_replicas_in_sync *
                         FLAGS.per_replica_batch_size)

    # Configure input data pipelines
    tfds_info = tfds.builder(TFDS_NAME).info
    num_classes = tfds_info.features['label'].num_classes
    num_examples = tfds_info.splits.total_num_examples
    available_splits = list(tfds_info.splits.keys())
    labels_names = tfds_info.features['label'].names

    with tf.device('/job:localhost'):
        in_memory_ds = tfds.load(TFDS_NAME, batch_size=-1, shuffle_files=True)

    bert_preprocess_model = make_bert_preprocess_model([SENTENCE_FEATURE])

    train_dataset, train_data_size = get_data_pipeline(
        in_memory_ds, tfds_info, 'train', global_batch_size, bert_preprocess_model)

    validation_dataset, validation_data_size = get_data_pipeline(
        in_memory_ds, tfds_info, 'validation', global_batch_size, bert_preprocess_model)

    # Configure the model
    steps_per_epoch = train_data_size // global_batch_size
    num_train_steps = steps_per_epoch * FLAGS.epochs
    num_warmup_steps = num_train_steps // 10
    validation_steps = validation_data_size // global_batch_size

    with strategy.scope():
        classifier_model = build_classifier_model(NUM_CLASSES, FLAGS.dropout_ratio)
        optimizer = optimization.create_optimizer(
            init_lr=FLAGS.init_lr,
            num_train_steps=num_train_steps,
            num_warmup_steps=num_warmup_steps,
            optimizer_type='adamw')
        loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
        metrics = tf.keras.metrics.SparseCategoricalAccuracy(
            'accuracy', dtype=tf.float32)

    classifier_model.compile(optimizer=optimizer, loss=loss, metrics=[metrics])

    model_dir, tb_dir, checkpoint_dir = set_job_dirs()

    # Configure Keras callbacks
    callbacks = [tf.keras.callbacks.experimental.BackupAndRestore(backup_dir=checkpoint_dir)]
    callbacks.append(tf.keras.callbacks.TensorBoard(
            log_dir=tb_dir, update_freq='batch'))

    logging.info('Starting training ...')
    classifier_model.fit(
        x=train_dataset,
        validation_data=validation_dataset,
        steps_per_epoch=steps_per_epoch,
        epochs=FLAGS.epochs,
        validation_steps=validation_steps,
        callbacks=callbacks)

    # Save trained model
    logging.info('Training completed. Saving the trained model to: {}'.format(model_dir))
    classifier_model.save(model_dir)

if __name__ == '__main__':
    logging.set_verbosity(logging.INFO)
    app.run(main)


'''Unit tests for the base app'''

from django.test import TestCase
from nose.tools import assert_equal

from wtfhack.base.models import Repo, Language

class RepoTest(TestCase):

    def setUp(self):
        self.python = Language.objects.create(name='python',
                                    learn_url='http://www.diveintopython.net/toc/index.html')
        self.ruby = Language.objects.create(name='ruby',
                                    learn_url="http://tryruby.org/levels/1/challenges/0")

    def test_model(self):
        assert_equal(len(Repo.objects.all()), 0)
        repo = Repo.objects.create(full_name='ruby/ruby',
                                    language=self.ruby)
        assert_equal(len(Repo.objects.all()), 1)
        assert_equal(repo.language.name, 'ruby')

    def test_save_url(self):
        # repo is saved without specifying url
        repo = Repo.objects.create(full_name='ruby/ruby', language=self.ruby)
        # url should default to base url + full name
        assert_equal(repo.url, u'https://github.com/ruby/ruby')

    def test_unicode(self):
        repo = Repo.objects.create(full_name='sloria/usv', language=self.python)
        assert_equal(str(repo), "sloria/usv: python")

class LanguageTest(TestCase):

    def test_model(self):
        lang = Language.objects.create(name='python',
                                    learn_url='http://www.diveintopython.net/toc/index.html')
        assert_equal(len(Language.objects.all()), 1)
        assert_equal(lang.name, 'python')
    def test_all(self):
        python = Language.objects.create(name='python',
                                    learn_url='http://www.diveintopython.net/toc/index.html')
        ruby = Language.objects.create(name='ruby',
                                    learn_url="http://tryruby.org/levels/1/challenges/0")
        assert_equal(Language.all(), [u'python', u'ruby'])


import json

def fetch_json(inputfile, folder=None):
    """ Opens and loads JSONs """
    if not folder:
        folder = ""
    filepath = folder + inputfile
    with open(filepath) as f:
        data = json.load(f)

    return data

def write_json_file(data, filename="results.json"):
    """ Writes/creates a JSON file """
    with open(filename, 'w+') as outfile:
        json.dump(data, outfile)


import numpy as np

import _ckwrap
from ckwrap.result import CkwrapResult

from typing import Union, Tuple

L1_criteria = 0
L2_criteria = 1
L2Y_criteria = 2

def _ckcluster(
    x: np.ndarray,
    k: Union[int, Tuple[int, int]],
    y: np.ndarray,
    method: str,
    criteria: int,
) -> CkwrapResult:
    x = np.ascontiguousarray(x, dtype=np.float64)
    if x.ndim > 1:
        raise ValueError("'x' must be 1-dimensional.")

    if isinstance(k, int):
        k = k, k
    if k[0] < 1:
        raise ValueError("Minimum 'k' must be greater than 1.")
    if k[1] > np.unique(x).size:
        raise ValueError(
            "Max 'k' must be smaller than the number of unique 'x' values."
        )

    if y is None:
        y = np.array([1.0], dtype=np.float64)
    else:
        y = np.ascontiguousarray(y, dtype=np.float64)

    if y.ndim > 1:
        raise ValueError("'y' must be 1-dimensional.")

    if criteria == 2 and x.size != y.size:
        raise ValueError("Segs requires 'x' and 'y' to be same size.")

    if method not in ["linear", "loglinear", "quadratic"]:
        raise ValueError("Method must be one of 'linear', 'loglinear', 'quadratic'.")

    return _ckwrap.ckcluster(x, y, k[0], k[1], method, criteria)

def ckmeans(
    x: np.ndarray,
    k: Union[int, Tuple[int, int]] = (1, 9),
    weights: np.ndarray = None,
    method: str = "linear",
) -> CkwrapResult:
    """Returns the optimal k-means clustering result of `x` for k in range `k`.

    Uses ckmeans.1d.dp with `L2` criteria to minimize within-cluster sums of squared
    distances.
    Bayesian information criterion selects optimal `k` when `k` is passed as a range.
    Optimal weighted clustering can be performed by passing in `weights`, the
    default is equally weighted clustering.
    All `method`s return the same result and only differ in speed. From
    fastest to slowest the options are 'linear', 'loglinear' and 'quadratic'.

    Args:
        x: array, one dimensional
        k: single or range of k to test
        weights: weights for clustering, same shape as x
        method: options {'linear', 'loglinear', 'quadratic'}

    Returns:
        optimal clustering result
    """
    return _ckcluster(x, k, weights, method, L2_criteria)

def ckmedians(
    x: np.ndarray,
    k: Union[int, Tuple[int, int]] = (1, 9),
    method: str = "linear",
) -> CkwrapResult:
    """Returns the optimal k-medians clustering result of `x` for k in range `k`.

    Uses ckmeans.1d.dp with `L1` criteria to minimize within-clusters sum fo distances.
    Bayesian information criterion selects optimal `k` when `k` is passed as a range.
    All `method`s return the same result and only differ in speed. From
    fastest to slowest the options are 'linear', 'loglinear' and 'quadratic'.

    Args:
        x: array, one dimensional
        k: single or range of k to test
        method: options {'linear', 'loglinear', 'quadratic'}

    Returns:
        optimal clustering result
    """
    return _ckcluster(x, k, None, method, L1_criteria)

def cksegs(
    y: np.ndarray,
    k: Union[int, Tuple[int, int]] = (1, 9),
    x: np.ndarray = None,
    method: str = "quadratic",
) -> CkwrapResult:
    """Minimizes within-cluster sum of squared distance on `y`.

    Offers optimal piecewise constant approximation of `y` within clusters of `x`.
    Only `method` 'quadratic' guarantees optimality.
    Bayesian information criterion selects optimal `k` when `k` is passed as a range.

    Args:
        x: array, one dimensional
        k: single or range of k to test
        x: array, same shape as x, default is range(1, len(y) + 1)
        method: options {'quadratic', 'linear', 'loglinear'}

    Returns:
        optimal clustering result
    """
    if x is None:
        x = np.arange(1, y.size + 1)
    return _ckcluster(x, k, y, method, L2Y_criteria)


import argparse
import os
import pickle
import random
from typing import List, Sized, Union, Optional, Tuple

import numpy as np
import torch
from torchtext import vocab

def _check_truncate(vec, truncate, truncate_left=False):
    """
    Check that vector is truncated correctly.
    """
    if truncate is None:
        return vec
    if len(vec) <= truncate:
        return vec
    if truncate_left:
        return vec[-truncate:]
    else:
        return vec[:truncate]

def padded_tensor(
    items: List[Union[List[int], torch.LongTensor]],
    pad_idx: int = 0,
    left_padded: bool = False,
    max_len: Optional[int] = None,
) -> Tuple[torch.LongTensor, List[int]]:
    """
    Create a padded matrix from an uneven list of lists.

    Returns (padded, lengths), where padded is the padded matrix, and lengths
    is a list containing the lengths of each row.

    Matrix is right-padded (filled to the right) by default, but can be
    left padded if the flag is set to True.

    Matrix can also be placed on cuda automatically.

    :param list[iter[int]] items: List of items
    :param int pad_idx: the value to use for padding
    :param bool left_padded:
    :param int max_len: if None, the max length is the maximum item length

    :returns: (padded, lengths) tuple
    :rtype: (Tensor[int64], list[int])
    """

    # number of items
    n = len(items)
    # length of each item
    lens: List[int] = [len(item) for item in items]  # type: ignore
    # max in time dimension
    t = max(lens) if max_len is None else max_len

    # if input tensors are empty, we should expand to nulls
    t = max(t, 1)

    if isinstance(items[0], torch.Tensor):
        # keep type of input tensors, they may already be cuda ones
        output = items[0].new(n, t)  # type: ignore
    else:
        output = torch.LongTensor(n, t)  # type: ignore
    output.fill_(pad_idx)

    for i, (item, length) in enumerate(zip(items, lens)):
        if length == 0:
            # skip empty items
            continue
        if not isinstance(item, torch.Tensor):
            # put non-tensors into a tensor
            item = torch.LongTensor(item)  # type: ignore
        if left_padded:
            # place at end
            output[i, t - length:] = item
        else:
            # place at beginning
            output[i, :length] = item

    return output, lens

def padded_3d(
    tensors: List[torch.LongTensor],
    pad_idx: int = 0,
    left_padded: bool = False,
    max_len=None,
    dtype=torch.long,
):
    a = len(tensors)
    b = max(len(row) for row in tensors)  # type: ignore
    c = max(len(item) for row in tensors for item in row) if max_len is None else max_len  # type: ignore

    c = max(c, 1)

    output = torch.full((a, b, c), pad_idx, dtype=dtype)

    for i, row in enumerate(tensors):
        item: Sized
        idx = [i for i in range(row.size(0) - 1, -1, -1)]
        idx = torch.LongTensor(idx)
        for j, item in enumerate(row.index_select(0, idx)):  # type: ignore
            if len(item) == 0:
                continue
            if not isinstance(item, torch.Tensor):
                item = torch.Tensor(item, dtype=dtype)
            if left_padded:
                output[i, -j - 1, c - len(item):] = item
            else:
                output[i, -j - 1, : len(item)] = item

    return output

def build_vocab(lst_of_utterances, min_freq=3, unk_token='<UNK>', pad_token='<PAD>'):
    tok2idx = dict()
    tok2freq = dict()

    for u in lst_of_utterances:
        toks = u2toks(u)
        for tok in toks:
            if tok not in tok2freq:
                tok2freq[tok] = 1
            else:
                tok2freq[tok] += 1
    idx = 2
    for tok, freq in tok2freq.items():
        if freq >= min_freq:
            tok2idx[tok] = idx
            idx += 1
    tok2idx[pad_token] = 0
    tok2idx[unk_token] = 1
    return tok2idx

def build_embed(pretrained_embed_f, built_vocab):
    embs = vocab.Vectors(pretrained_embed_f)
    ret_embs = torch.zeros((len(built_vocab), embs.dim))
    for tok, idx in built_vocab.items():
        ret_embs[idx] = embs.get_vecs_by_tokens(tok, lower_case_backup=True)
    return [np.asarray(vec) for vec in ret_embs]

def read_fb_dialogues(f_path):
    dialogues = []
    with open(f_path, encoding='utf-8') as f:
        for line in f.readlines():
            # if not line.islower():
            #     raise RuntimeError('The corpus is not lower-cased!')
            item_arr = line.strip().split('\t')
            if len(item_arr) < 2:
                pass
            context = [u.strip() for u in ' '.join(item_arr[0].split()[1:]).split('__EOT__')]
            response = item_arr[1].strip()
            dialogues.append((context, response))
    return dialogues

def u2toks(u: str):
    return [tok.strip() for tok in u.split()]

def neg_sampling(dialogues, sample_from, built_dict, num_neg=1):
    utterances = []
    responses = []
    labels = []
    for context, response in dialogues:
        utterances.append([_check_truncate(vectorized(u2toks(u), built_dict), 50, True) for u in context])
        responses.append(_check_truncate(vectorized(u2toks(response), built_dict), 50))
        labels.append(1)

        for _ in range(num_neg):
            utterances.append([_check_truncate(vectorized(u2toks(u), built_dict), 50, True) for u in context])
            responses.append(_check_truncate(vectorized(u2toks(random.choice(sample_from)[1]), built_dict), 50))
            labels.append(0)
    return utterances, responses, labels

def vectorized(tok_arr, built_dict):
    return [built_dict.get(t, built_dict.get('<UNK>')) for t in tok_arr]

def dump_data(X_train_utterances, X_train_responses, y_train, f_name):
    X_train_utterances = padded_3d(
        [padded_tensor(utts, max_len=50, left_padded=True)[0] for utts in X_train_utterances],
        max_len=50,
        left_padded=True
    ).tolist()
    X_train_responses = padded_tensor(X_train_responses, max_len=50, left_padded=True)[0].tolist()

    pickle.dump((X_train_utterances, X_train_responses, y_train),
                open(os.path.join(args.out_dir, f_name), 'wb'))

def main(args):
    train_dialogues = read_fb_dialogues(os.path.join(args.data_dir, "train.txt"))
    dev_dialogues = read_fb_dialogues(os.path.join(args.data_dir, "valid.txt"))
    test_dialogues = read_fb_dialogues(os.path.join(args.data_dir, "test.txt"))
    all_utterances = [' '.join(c) + ' ' + r for c, r in train_dialogues + dev_dialogues + test_dialogues]
    built_dict = build_vocab(all_utterances)

    X_train_utterances, X_train_responses, y_train = neg_sampling(
        train_dialogues, train_dialogues, built_dict
    )
    X_dev_utterances, X_dev_responses, y_dev = neg_sampling(
        dev_dialogues, train_dialogues, built_dict, num_neg=9
    )
    X_test_utterances, X_test_responses, y_test = neg_sampling(
        test_dialogues, train_dialogues, built_dict, num_neg=9
    )
    dump_data(X_train_utterances, X_train_responses, y_train, 'train.pkl')
    dump_data(X_dev_utterances, X_dev_responses, y_dev, 'dev.pkl')
    dump_data(X_test_utterances, X_test_responses, y_test, 'test.pkl')

    embed = build_embed(args.embed_file, built_dict)
    pickle.dump((built_dict, embed), open(os.path.join(args.out_dir, 'vocab_and_embeddings.pkl'), 'wb'))

def check_args(args):
    if not os.path.exists(args.data_dir):
        raise ValueError("data_dir: {} does not exist!".format(args.data_dir))
    if not os.path.isfile(args.embed_file):
        raise ValueError("embed_file: {} does not exist!".format(args.embed_file))
    if not os.path.exists(args.out_dir):
        os.mkdir(args.out_dir)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--data_dir',
        required=True,
    )
    parser.add_argument(
        '--embed_file',
        required=True,
    )
    parser.add_argument(
        '--out_dir',
        required=True,
    )
    args = parser.parse_args()
    check_args(args)
    main(args)


import math
import random
import numpy as np
from Utils.utils import Util as util
from matplotlib import pyplot as plt

class Perceptron:
    def __init__(self, x_data, y_data, activation, hidden_layer):
        self.x_data = x_data
        self.y_data = y_data
        self.n_classes = np.unique(self.y_data, axis=0)
        self.attributes = x_data.shape[1]
        self.hidden_layer = hidden_layer
        self.output_layer = y_data.shape[1]
        self.epochs = 500
        self.realizations = 1
        self.precision = 10**(-7)
        self.train_size = 0.8
        self.activation = activation
        self.mse = []
        self.rmse = []

    def initWeigths(self, hidden_layer):
        params = {}
        a = -1.5
        b = 1.5
        params['w'] = (b - a) * np.random.random_sample((self.attributes+1, hidden_layer)) + a
        params['m'] = (b - a) * np.random.random_sample((hidden_layer+1, self.output_layer)) + a
        return params

    def updateEta(self, epoch):
        eta_i = 0.1
        eta_f = 0.05
        eta = eta_i * ((eta_f / eta_i) ** (epoch / self.epochs))
        self.eta = eta

    def function(self, u):
        if self.activation == 'logistic':
            y = 1.0/(1.0 + np.exp(-u))
        elif self.activation == 'tanh':
            y = (np.exp(u) - np.exp(-u))/(np.exp(u) + np.exp(-u))
        else:
            raise ValueError('Error in function!')
            y = 0
        return y

    def derivate(self, u):
        if self.activation == 'logistic':
            y_ = u * (1.0 - u)
        elif self.activation == 'tanh':
            y_ = 0.5 * (1.0 - (u * u))
        else:
            raise ValueError('Error in derivate!')
            y_ = 0
        return y_

    def predict(self, xi, params):
        w = params['w']
        m = params['m']

        H = np.dot(xi, w)
        H = self.function(H)
        H = np.concatenate(([-1], H), axis=None)
        Y = np.dot(H, m)

        return Y

    def train(self, x_train, y_train, hidden_layer):
        error_old = 0
        cont_epochs = 0
        mse_vector = []
        params = self.initWeigths(hidden_layer)
        w = params['w']
        m = params['m']
        while True:
            self.updateEta(cont_epochs)
            x_train, y_train = util.shuffleData(x_train, y_train)
            (p, _) = x_train.shape
            error_epoch = 0
            for k in range(p):
                x_k = x_train[k]
                H = np.dot(x_k, w)
                H = self.function(H)
                H_ = self.derivate(H)

                H = np.concatenate(([-1], H), axis=None)
                Y = np.dot(H, m)

                # Quadratic Error Calculation
                d = y_train[k]
                error = d - Y
                error_epoch += np.sum(error**2)

                # Output layer
                delta_output = (error).reshape(-1, 1)
                aux_output = (self.eta * delta_output)
                m += np.dot(H.reshape(-1, 1), aux_output.T)

                # Hidden layer
                delta_hidden = np.sum(np.dot(m, delta_output)) * H_
                aux_hidden = (self.eta * delta_hidden).reshape(-1, 1)
                w += np.dot(x_k.reshape(-1, 1), aux_hidden.T)

            mse = (0.5 * error_epoch) / p
            mse_vector.append(mse)

            if abs(error_epoch - error_old) <= self.precision:
                #print('Stop Precision: {} (Epochs {})'.format(abs(error_epoch - error_old), cont_epochs))
                break
            if cont_epochs >= self.epochs:
                #print('Stop Epochs: {}'.format(cont_epochs))
                break

            error_old = error_epoch
            cont_epochs += 1
        #util.plotErrors(mse_vector)
        params['w'] = w
        params['m'] = m
        return params

    def test(self, x_test, y_test, params):
        error_epoch = 0.0
        (p, _) = x_test.shape
        for k in range(p):
            x_k = x_test[k]
            y = self.predict(x_k, params)
            d = y_test[k]

            error = d - y
            error_epoch += error**2

        mse = error_epoch / p
        rmse = np.sqrt(mse)
        return mse, rmse

    def execute(self):
        x_data = util.insertBias(self.x_data)
        y_data = self.y_data

        for i in range(self.realizations):
            x_data_aux, y_data_aux = util.shuffleData(x_data, y_data)
            x_train, x_test, y_train, y_test = util.splitData(x_data_aux, y_data_aux, self.train_size)

            best_hidden_layer = self.hidden_layer
            params = self.train(x_train, y_train, best_hidden_layer)
            mse, rmse = self.test(x_test, y_test, params)

            self.mse.append(mse)
            self.rmse.append(rmse)

        print('{} Realizations'.format(self.realizations))
        print('MSE: {}'.format(np.mean(self.mse)))
        print('Std MSE: {}'.format(np.std(self.mse)))
        print('RMSE: {}'.format(np.mean(self.rmse)))
        print('Std RMSE: {}'.format(np.std(self.rmse)))

        #self.plotRegression(x_data, x_train, x_test, y_train, y_test, self.predict, params)

    def plotRegression(self, x_data, x_train, x_test, y_train, y_test, predict, params):
        x = x_data[:,1]
        y = [predict(np.array([-1, i]), params) for i in x]

        fig, ax = plt.subplots()
        plt.title('MLP Color Map')
        plt.xlabel('Eixo X')
        plt.ylabel('Eixo y')

        ax.scatter(x, y, label='Predict', color=[0.31, 0.31, 0.31])
        ax.scatter(x_train[:,1], y_train[:,0], label='Train Data', color=[0.00, 0.45, 0.74])
        ax.scatter(x_test[:,1], y_test[:,0], label='Test Data', color='green')

        ax.legend()
        ax.grid(True)
        plt.show()

        #fig.savefig('.\MultilayerPerceptronRegression\Results\sin_q10.png')


import csv
from collections import Counter
arch=open('appstore_games.csv','r')
csv_reader=csv.reader(arch,delimiter=',')
next(csv_reader)
rating={}
for linea in csv_reader:
    if linea[7]=="0" and "ES" in linea[12]:
        print(linea)
    rating[linea[2]] = (linea[6],linea[4])
mayor_rating=dict(Counter(rating).most_common(10))
print("Links a los iconos de los 10 juegos con más calificaciones de usuarios")
for i in mayor_rating:
    print("URL de juego: ",i,mayor_rating[i][1])
arch.close()


from setuptools import setup, find_packages

with open('README.rst', encoding='UTF-8') as f:
    readme = f.read()

setup(
    name='jfcli',
    version='0.0.3',
    description='CLI for JFrog Artifactory',
    long_description=readme,
    author='David',
    author_email='davidp@jfrog.com',
    install_requires=[],
    packages=find_packages('src'),
    package_dir={'': 'src'}
)


# %%

import pandas as pd
import numpy as np
import json
import math
import re

# %%

def to_array(data):
    return data.split('|')

def appendkey(data):
    data = str(data)
    return data if data.startswith('uuid:') else 'uuid:' + data

file_path1 = '/home/mahmud/Documents/All Factories12-05-2020.csv'
published = pd.read_csv(file_path1, converters={
    'address': json.loads,
    'workers': json.loads,
    'types': to_array,
    'memberships': to_array
})
published['uuid'] = published['uuid'].apply(appendkey)
published = pd.io.json.json_normalize(published.to_dict('records'))

def parseMember(tmp):
    if 'BGMEA' in tmp and 'BKMEA' in tmp:
        return 3
    if 'BGMEA' in tmp:
        return 1
    if 'BKMEA' in tmp:
        return 2
    else:
        return 4

address_cols = ['address_plot', 'address_ward', 'address_block',
                'address_floor', 'address_union', 'address_police', 'address_display',
                'address_village', 'address_district', 'address_postcode',
                'address_roadname', 'address_upazilla', 'address_postoffice',
                'address_roadnumber']
published['memberships'] = published['memberships'].apply(parseMember)
published = published[['name', 'uuid', 'types', 'memberships',
                       'workers_male', 'workers_female', 'workers_total'] + address_cols]

master = pd.read_excel(
    '/home/mahmud/Downloads/Combind Dataset for Research and Tech Team 28_May_2020.xlsx', sheet_name='Combind_3363')

master['uuid'] = master['uuid'].apply(appendkey)
merged = published.merge(master, left_on='uuid', right_on='uuid',
                         indicator=True, how='left')

print(merged[merged['_merge'] == 'left_only'][['uuid']])

published['Official Phone'] = merged['officialcontact:official_phone']
published['Official Contact Name'] = merged['official_contact_name']
published['Official Contact Email '] = merged['official_contact_email']
published['Official Contact Phone'] = merged['official_contact_phone']
published['Respondent Name'] = merged['respondent_name']
published['Respondent Phone'] = merged['respondent_phone']
published['Respondent Email'] = merged['respondent:respondent_email']
published['Market Type'] = merged['market_type']
published['Export Status Type'] = merged['export_status']
published['DB Serial'] = merged['DB_Serial'].astype(int)
published
# %%

onlyctg = '/home/mahmud/Documents/2020-05-27T11:07:37.451Z.csv'
all = '/home/mahmud/Documents/2020-05-27T12:25:03.893Z.csv'
all = pd.read_csv(all)
onlyctg = pd.read_csv(onlyctg)[['Key']]
ctg = onlyctg.merge(all, on='Key', how='left')

def getmembers(tmp):
    if tmp['BGMEA Exists'] and tmp['BKMEA Exists']:
        return 3
    if tmp['BGMEA Exists']:
        return 1
    if tmp['BKMEA Exists']:
        return 2
    else:
        return 4

ctg['DB Serial'] = ctg['Serial']
ctg['name'] = ctg['Factory Name']
ctg['address_display'] = ctg['Display Address']
ctg['uuid'] = ctg['Key']
ctg['types'] = ctg['Factory Type'].str.split(', ')
ctg['memberships'] = ctg.apply(getmembers, axis=1)
ctg['workers_male'] = ctg['Workers Male']
ctg['workers_female'] = ctg['Workers Female']
ctg['workers_total'] = ctg['Workers Total']
ctg['brands'] = ctg['Brands'].str.split(', ')

for col in ctg:
    if col.startswith('Address') and not col.endswith('Tags') and not col.endswith('Remarks'):
        ne_col_name = '_'.join([p.lower()
                                for p in col.split(' ', 1)]).replace(' ', '')
        ctg[ne_col_name] = ctg[col]

nctg = ctg[['name', 'uuid', 'types', 'memberships',
            'workers_male', 'workers_female', 'workers_total'] + address_cols]
nctg['Official Phone'] = ctg['Official Contact Phone']
nctg['Official Contact Phone'] = ctg['Official Contact Person Phone']
nctg['Official Contact Name'] = ctg['Official Contact Person Name']
nctg['Official Contact Email '] = ctg['Official Contact Person Email']
nctg['Respondent Name'] = ctg['Respondent Name']
nctg['Respondent Phone'] = ctg['Respondent Phone']
nctg['Respondent Email'] = ctg['Respondent Email']
nctg['Market Type'] = ctg['Market Type']
nctg['Export Status Type'] = ctg['Export Status Type']
nctg['DB Serial'] = ctg['DB Serial'].astype(int)
nctg

# %%
ng = '/home/mahmud/Downloads/R2_N.ganj 2nd Slot_195_MnE Review_20.2.20.xlsx'
ng = pd.read_excel(ng, sheet_name='N.Ganj 2nd Slot_195')
ngn = pd.DataFrame()
ngn['DB Serial'] = ng['serial']
ngn['name'] = ng['Factory Name']
ngn['address_display'] = ng['Address Display Address']
ngn['uuid'] = ng['Meta Instance ID']
ngn['types'] = ng['Factory Type'].str.split(', ')
ngn['memberships'] = ng.apply(getmembers, axis=1)
ngn['workers_male'] = ng['Workers Male']
ngn['workers_female'] = ng['Workers Female']
ngn['workers_total'] = ng['Workers Total']

ngn['Official Phone'] = ng['Official Contact Phone']
ngn['Official Contact Phone'] = ng['Official Contact Person Phone']
ngn['Official Contact Name'] = ng['Official Contact Person Name']
ngn['Official Contact Email '] = ng['Official Contact Person Email']
ngn['Respondent Name'] = ng['Respondent Name']
ngn['Respondent Phone'] = ng['Respondent Phone']
ngn['Respondent Email'] = ng['Respondent Email']
ngn['Market Type'] = ng['Market Type']
ngn['Export Status Type'] = ng['Export Status Export Status Type']
for col in ng:
    if col.startswith('Address') and not col.endswith('Tags') and not col.endswith('Remarks') and not 'Display' in col and not 'Section' in col and not 'Land Mark' in col:
        ne_col_name = '_'.join([p.lower()
                                for p in col.split(' ', 1)]).replace(' ', '')
        ngn[ne_col_name] = ng[col]

ngn

# %%
result = pd.concat([published, nctg, ngn])
result

# %%
coroactivepath = '/home/mahmud/Downloads/Final_Corona Map Data_11.5.20.csv'
corona = pd.read_csv(coroactivepath)
corona['uuid'] = corona['uuid'].apply(appendkey)
corona = corona['uuid'].to_list()

def isactive(id):
    return id in corona

result['COVID Active'] = result['uuid'].apply(isactive)
result

# %%

def strjoin(da):
    return "|".join(da)

result['types'] = result['types'].apply(strjoin)

output_path = "./outputs/brands.csv"
result.to_csv(output_path)

# %%

fmoDF = fmoDF.sort_values(by=['id'])
output_path = "./outputs/merged_data.csv"
fmoDF.to_csv(output_path)

# %%


import numpy as np
import matplotlib.pyplot as plt
from scipy.integrate import odeint
import pandas as pd

#%% SETEO DEL SISTEMA

## Parametros ----

mu_m1 = 1.848  # 1/h
K_S1 = 101.78  # g/l
mu_m2 = 0.502  # 1/h
K_S2 = 0.445  # g/l
Y_XS1 = 0.45998  # g/g
m_S1 = 0.0314  # g/(g.h)
Y_XS2 = 44.444  # g/g
m_S2 = 0.00000109  # g/(g.h)
K_1 = 0.00001075  # g/g
K_2 = 0.00000369  # g/(g.h)
K_3 = 0.0000000246  # g/g
K_4 = 0.000520  # g/(g.h)
S_m = 917.8
n = 2.011
C_S02 = 5.3  # g/l

parms = mu_m1, K_S1, mu_m2, K_S2, Y_XS1, m_S1, Y_XS2, m_S2, K_1, K_2, K_3, K_4, S_m, n, C_S02

## Condiciones iniciales ----

C_X0 = 1.00695  # g/l
C_S10 = 80  # g/l
C_S20 = 0.25  # g/l
C_P10 = 0  # g/l
C_P20 = 0  # g/l
V0 = 1.5  # l

Balances0 = [C_X0, C_S10, C_S20, C_P10, C_P20, V0]

t = list(range(0,102,2));t = np.array(t)

#%% PLANTEO E INTEGRACION NUMERICA DEL SET DE ECUACIONES DIFERENCIALES

def f(Balances, t, mu_m1, K_S1, mu_m2, K_S2, Y_XS1, m_S1, Y_XS2, m_S2, K_1, K_2, K_3, K_4, S_m, n, C_S02):

    C_X, C_S1, C_S2, C_P1, C_P2, V = Balances

    # mu

    mu = ((mu_m1*C_S1)/(K_S1+C_S1)) * ((mu_m2*C_S2)/(K_S2+C_S2)) * ( 1 - ((C_S1/C_S2)/(S_m))**n )

    # Caudal de alimentacion de glucosa

    F_1 = np.piecewise(t, [t<10, t>=10], [0, 0.005])

    # Concentracion de glucosa en la alimentacion

    C_S01 = np.piecewise(t, [t<40, t>=40], [300, 250])

    # Caudal de alimentacion de nitrogeno

    F_2 = np.piecewise(t, [t<10, (t>=10) & (t < 40), t>=40], [0, 0.005, 0])

    # Caudal total

    F_t = F_1 + F_2

    return [mu*C_X - ( (F_t/V) * C_X ),
           -(((mu*C_X)/(Y_XS1)) + m_S1*C_X) + ( ((F_1*C_S01)/V) - ((F_t*C_S1)/V) ),
           -(((mu*C_X)/(Y_XS2)) + m_S2*C_X) + ( ((F_2*C_S02)/V) - ((F_t*C_S2)/V) ),
           (K_1*mu*C_X + K_2*C_X) - ((F_t/V)*C_P1),
           (K_3*mu*C_X + K_4*C_X) - ((F_t/V)*C_P2),
           F_t]

columnas_salidas = 'Biomasa', 'Glucosa', 'Nitrogeno','Bicaverina','GA3','Volumen' # Me sirve para recordar el orden de salida

# Resuelvo los balances ----

resultados = odeint(f, Balances0, t, args=parms)

#%% POST PROCESSING

# Datos para plot tipo array ----

C_S2P = resultados[:,2]*100
C_P2P = resultados[:,4]*10

resultados_plot1 = np.copy(resultados)
resultados_plot1[:,2] = C_S2P
resultados_plot1[:,4] = C_P2P
resultados_plot1 = np.delete(resultados_plot1, (1,3,5), axis=1)

resultados_plot2 = np.copy(resultados[:,1])

# Conviertiendo datos para plot y salida tipo DataFrame ----

resultados_plot1_df = pd.DataFrame(resultados_plot1);resultados_plot1_df.columns = ['Biomasa','Nitrogeno','GA3']
resultados_plot1_df.insert(0,"Tiempo",t,True)
resultados_plot1_df.insert(4,"Glucosa",resultados_plot2,True)

#%% PLOT OBJETO DEL TIPO DATAFRAME

resultados_plot1_df.plot(x = "Tiempo", y = ["Biomasa","Nitrogeno","GA3"])
ax = resultados_plot1_df.plot(x = "Tiempo", secondary_y=["Glucosa"], mark_right=False)
ax.set_ylabel("Biomasa, Nitrogeno y GA3 [g/L]");
ax.right_ax.set_ylabel("Glucosa [g/L]");
ax.set_xlabel("Tiempo [h]");
ax.get_legend().set_bbox_to_anchor((1.39, 0.7))
ax.get_legend().set_title("Componentes\n")

figure = plt.gcf() # to get the current figure

figure.set_size_inches(8, 6)
figure.savefig("simulacion.png", dpi=100, bbox_inches='tight')

#%% PLOT OBJETO DEL TIPO SALIDA NUMPY

# Create Plot

fig, ax1 = plt.subplots()

ax1.set_xlabel('Tiempo [h]')
ax1.set_ylabel('Biomasa, GA3 y Nitrogeno [g/L]')
plot_1 = ax1.plot(t, resultados_plot1)
ax1.tick_params(axis ='y')

# Adding Twin Axes

ax2 = ax1.twinx()

ax2.set_ylabel('Glucosa [g/L]')
plot_2 = ax2.plot(t, resultados_plot2, color = 'magenta')
ax2.tick_params(axis ='y')

# Show plot

plt.show()

#%% SALIDA CSV

resultados_df = pd.DataFrame(resultados);resultados_df.columns = columnas_salidas;resultados_df.insert(0,"Tiempo",t,True)
resultados_df.to_csv('resultados.csv', index=False)

Simulacion Examen Final Bioprocesos I - Vpy/simulacion_FinalBiopI.py


# 0406.py
import cv2
import numpy as np

src = cv2.imread('../../img/spirit-week.jpg', cv2.IMREAD_GRAYSCALE)
dst = np.zeros(src.shape, dtype = src.dtype)

N = 128 # 8, 16, 32 - bigger numbers = more detail
height, width = src.shape # grayscale
## height, width, _ = src.shape # color

h = height // N
w = width // N
for i in range(N):
  for j in range(N):
    y = i * h
    x = j * w
    roi = src[y:y + h, x:x + w]
    dst[y:y + h, x:x + w] = cv2.mean(roi)[0] # grayscale
    # dst[y:y + h, x:x + w] = cv2.mean(roi)[0:3] # color

cv2.imshow('dst', dst)
cv2.waitKey()
cv2.destroyAllWindows()


#!/usr/bin/env python

import sys
import logging
import argparse
import time
import errno

import redis

from . import supervisor
from . import reporting
from . import locking

import crongod

logger = logging.getLogger(__name__)

def _get_log_level(name):
    level = getattr(logging, name.upper())
    if not isinstance(level, int):
        raise ValueError('Invalid log level: %s' % name)
    return level

def _build_lock_factory(config):
    lock_type = config.lock_type.lower()
    if lock_type == 'none':
        return locking.NoOpLockFactory()
    elif lock_type == 'file':
        return locking.FileLockFactory(
            directory=config.lock_prefix,
            blocking=config.lock_blocking)
    elif lock_type == 'redis':
        redis_client = redis.StrictRedis(
            host=config.redis_host,
            port=config.redis_port,
            db=config.redis_db,
            socket_timeout=5)
        return locking.RedisLockFactory(
            redis=redis_client,
            prefix=config.lock_prefix,
            blocking=config.lock_blocking)
    else:
        raise ValueError("Unknown lock type {}".format(lock_type))

def supervise_single_task():
    parser = argparse.ArgumentParser(
        description='crongod (%s)' % crongod.__version__,)

    parser.add_argument('cmd', help='the command to supervise')
    parser.add_argument('args', help='optional args to cmd', nargs=argparse.REMAINDER)
    parser.add_argument('--name', required=True)
    parser.add_argument('--version')
    parser.add_argument('--log-level', default='error', choices=('debug', 'info', 'warning', 'error', 'critial'))
    parser.add_argument('--timeout', default=None, type=int, help='command timeout in seconds')

    # Locking
    parser.add_argument('--lock-type', default='file', choices=('file', 'redis', 'none'))
    parser.add_argument('--lock-blocking', action='store_true', help='Enable lock blocking')
    parser.add_argument('--lock-prefix', default=None, help='The implementation dependant lock prefix')

    # Redis
    parser.add_argument('--redis-enable', action='store_true')
    parser.add_argument('--redis-host', default='logstash')
    parser.add_argument('--redis-port', default=6379, type=int)
    parser.add_argument('--redis-db', default=0, type=int)
    parser.add_argument('--redis-key', default='logstash_json')

    parser.set_defaults(lock_blocking=False)
    config = parser.parse_args()

    logging.basicConfig(level=_get_log_level(config.log_level))
    logger.debug(config)

    lock_factory = _build_lock_factory(config=config)
    lock = lock_factory.build(name=config.name)
    logger.debug('Acquiring lock %s', lock)
    if not lock.acquire():
        logger.debug('Lock %s already acquired and blocking is disabled', lock)
        sys.exit(errno.EWOULDBLOCK)

    try:
        task = supervisor.SupervisedTask(name=config.name, cmd=config.cmd, args=config.args, timeout=config.timeout)
        task.start()

        while task.supervise():
            time.sleep(1)
        context = task.build_context()
        if task.in_exceptional_state():
            print(task.build_error_message(context))
            status = 'ERROR'
        else:
            status = 'OK'

        sys.exit(task.returncode())
    finally:
        lock.release()

if __name__ == '__main__':
    supervise_single_task()


#!/usr/bin/python
#! coding:utf8
# 一个小爬虫 我的第一个python爬虫诞生啦

import requests
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
res = requests.get("http://www.sina.com.cn/")

res.encoding = 'utf-8'
f = file("/home/fly/Documents/res/xinlang","w+",1)
#f.Encoding = 'utf-8'
f.write(res.text)
f.flush()
f.close()
print(res.text)