'''
mostly taken from: https://github.com/karpathy/minGPT/
GPT model:
- the initial stem consists of a combination of token encoding and a positional encoding
- the meat of it is a uniform sequence of Transformer blocks
    - each Transformer is a sequential combination of a 1-hidden-layer MLP block and a self-attention block
    - all blocks feed into a central residual pathway similar to resnets
- the final decoder is a linear projection into a vanilla Softmax classifier
Mods:
- we use it as the encoder so it was slightly modified to avoid confusion with language modelling (GPT)
'''

import logging
import math
import sys

import torch
import torch.nn as nn
from torch.nn import functional as F

sys.path.insert(0, '.')  # nopep8
from utils.utils import get_obj_from_str, instantiate_from_config

logger = logging.getLogger(f'main.{__name__}')

class Config:
    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)

class SelfAttention(nn.Module):
    """
    A vanilla multi-head masked self-attention layer with a projection at the end.
    It is possible to use torch.nn.MultiheadAttention here but I am including an
    explicit implementation here to show that there is nothing too scary here.
    """

    def __init__(self, config):
        super().__init__()
        assert config.n_embd % config.n_head == 0
        # key, query, value projections for all heads
        self.key = nn.Linear(config.n_embd, config.n_embd)
        self.query = nn.Linear(config.n_embd, config.n_embd)
        self.value = nn.Linear(config.n_embd, config.n_embd)
        # regularization
        self.attn_drop = nn.Dropout(config.attn_pdrop)
        self.resid_drop = nn.Dropout(config.resid_pdrop)
        # output projection
        self.proj = nn.Linear(config.n_embd, config.n_embd)
        # # causal mask to ensure that attention is only applied to the left in the input sequence
        # mask = torch.tril(torch.ones(config.block_size,
        #                              config.block_size))
        # if hasattr(config, "n_unmasked"):
        #     mask[:config.n_unmasked, :config.n_unmasked] = 1
        # self.register_buffer("mask", mask.view(1, 1, config.block_size, config.block_size))
        self.n_head = config.n_head

    def forward(self, x):
        B, T, C = x.size()

        # calculate query, key, values for all heads in batch and move head forward to be the batch dim
        k = self.key(x).view(B, T, self.n_head, C // self.n_head).transpose(1, 2)  # (B, nh, T, hs)
        q = self.query(x).view(B, T, self.n_head, C // self.n_head).transpose(1, 2)  # (B, nh, T, hs)
        v = self.value(x).view(B, T, self.n_head, C // self.n_head).transpose(1, 2)  # (B, nh, T, hs)

        # self-attention; Self-attend: (B, nh, T, hs) x (B, nh, hs, T) -> (B, nh, T, T)
        att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))
        # att = att.masked_fill(self.mask[:, :, :T, :T] == 0, float('-inf'))
        att = F.softmax(att, dim=-1)
        y = self.attn_drop(att) @ v  # (B, nh, T, T) x (B, nh, T, hs) -> (B, nh, T, hs)
        y = y.transpose(1, 2).contiguous().view(B, T, C)  # re-assemble all head outputs side by side

        # output projection
        y = self.resid_drop(self.proj(y))

        return y

class Block(nn.Module):
    """ an unassuming Transformer block """

    def __init__(self, config):
        super().__init__()
        self.ln1 = nn.LayerNorm(config.n_embd)
        self.ln2 = nn.LayerNorm(config.n_embd)
        self.attn = SelfAttention(config)
        self.mlp = nn.Sequential(
            nn.Linear(config.n_embd, 4 * config.n_embd),
            nn.GELU(),  # nice
            nn.Linear(4 * config.n_embd, config.n_embd),
            nn.Dropout(config.resid_pdrop),
        )

    def forward(self, x):
        x = x + self.attn(self.ln1(x))
        x = x + self.mlp(self.ln2(x))
        return x

class NoPosEncoding(nn.Module):
    '''Does not apply any positional encoding'''
    def __init__(self, *args, **kwargs) -> None:
        super().__init__()

    def forward(self, x):
        return x

class ZeroInitPositionalEncoding(nn.Module):
    ''' Zero inited trainable pos embedding. It is just applied on the sequence, thus respects no priors. '''

    def __init__(self, block_shape, n_embd):
        super().__init__()
        self.block_shape = block_shape
        self.n_embd = n_embd
        self.pos_emb = nn.Parameter(torch.zeros(1, *block_shape, n_embd))

    def forward(self, token_embeddings):
        return token_embeddings + self.pos_emb

class PositionEmbeddingLearnedVisual(nn.Module):

    def __init__(self, block_shape, n_embd) -> None:
        super().__init__()
        self.block_shape = block_shape
        self.max_t, self.max_h, self.max_w = block_shape
        self.n_embd = n_embd
        # dividing n_embd almost evenly among each dimension; the remainer will be given to the time
        # dimension if `n_embd` is not divisible
        self.n_dims = len(block_shape)
        self.n_embd_t = self.n_embd_h = self.n_embd_w = self.n_embd // self.n_dims
        self.n_embd_t += self.n_embd % self.n_dims
        self.time_embed = nn.Embedding(self.max_t, self.n_embd_t)
        self.height_embed = nn.Embedding(self.max_h, self.n_embd_h)
        self.width_embed = nn.Embedding(self.max_w, self.n_embd_w)
        self.reset_parameters()

    def reset_parameters(self):
        nn.init.uniform_(self.time_embed.weight)
        nn.init.uniform_(self.height_embed.weight)
        nn.init.uniform_(self.width_embed.weight)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        '''
        Args:
            x (torch.Tensor): a batch of visual feature maps (B, t, h, w, d)
        Returns:
            torch.Tensor: x + pos
        '''
        return x + self.make_pos_emb(x)

    def make_pos_emb(self, x):
        B, t, h, w, d = x.shape
        t_i = torch.arange(t, device=x.device)
        w_i = torch.arange(w, device=x.device)
        h_i = torch.arange(h, device=x.device)
        # (t/w/h, D)
        t_emb = self.time_embed(t_i)
        w_emb = self.width_embed(w_i)
        h_emb = self.height_embed(h_i)
        # (t, w, h, d//3)
        t_emb = t_emb.view(t, 1, 1, self.n_embd_t).repeat(1, h, w, 1)
        w_emb = w_emb.view(1, 1, w, self.n_embd_w).repeat(t, h, 1, 1)
        h_emb = h_emb.view(1, h, 1, self.n_embd_h).repeat(t, 1, w, 1)
        # (t, w, h, d)
        pos = torch.cat([t_emb, w_emb, h_emb], dim=-1)
        # (B, t, w, h, d) -- same as x
        pos = pos.view(1, t, h, w, d).repeat(B, 1, 1, 1, 1)
        return pos

class PositionEmbeddingLearnedAudio(nn.Module):

    def __init__(self, block_shape, n_embd) -> None:
        super().__init__()
        self.block_shape = block_shape
        self.max_f, self.max_t = block_shape
        # dividing n_embd almost evenly among each dimension; the remainer will be given to the time
        # dimension if `n_embd` is not divisible: e.g. 512 / 3 -> [170, 170, 170+2]
        self.n_dims = len(block_shape)
        self.n_embd_t = self.n_embd_f = n_embd // self.n_dims
        self.n_embd_t += n_embd % self.n_dims
        self.freq_embed = nn.Embedding(self.max_f, self.n_embd_f)
        self.time_embed = nn.Embedding(self.max_t, self.n_embd_t)
        self.reset_parameters()

    def reset_parameters(self):
        nn.init.uniform_(self.freq_embed.weight)
        nn.init.uniform_(self.time_embed.weight)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        '''
        Args:
            x (torch.Tensor): a batch of spectrogram features maps (B, f, t, d)
        Returns:
            torch.Tensor: x + pos
        '''
        return x + self.make_pos_emb(x)

    def make_pos_emb(self, x):
        B, f, t, d = x.shape
        f_i = torch.arange(f, device=x.device)
        t_i = torch.arange(t, device=x.device)
        # (f/t, D)
        f_emb = self.freq_embed(f_i)
        t_emb = self.time_embed(t_i)
        # (f, t, d//2)
        f_emb = f_emb.view(f, 1, self.n_embd_f).repeat(1, t, 1)
        t_emb = t_emb.view(1, t, self.n_embd_t).repeat(f, 1, 1)
        # (f, t, d)
        pos = torch.cat([f_emb, t_emb], dim=-1)
        # (B, f, t, d)
        pos = pos.view(1, f, t, d).repeat(B, 1, 1, 1)
        return pos

class L2Normalize(nn.Module):

    def __init__(self, *args, **kwargs) -> None:
        super().__init__()

    def forward(self, x):
        eps = 1e-6 if isinstance(x, (torch.HalfTensor, torch.cuda.HalfTensor)) else 1e-12
        x = torch.nn.functional.normalize(x, p=2.0, dim=-1, eps=eps)
        return x

class Transformer(nn.Module):
    ''' the full transformer, with a context size of block_size '''

    def __init__(self, vis_pos_emb_module, aud_pos_emb_module, num_offset_cls,
                 visual_block_shape, audio_block_shape, pre_norm_cfg,
                 n_layer=12, n_head=8, n_embd=256, tok_pdrop=0., embd_pdrop=0., resid_pdrop=0., attn_pdrop=0.,
                 n_unmasked=0):
        super().__init__()
        config = Config(num_offset_cls=num_offset_cls,
                        audio_block_shape=audio_block_shape, visual_block_shape=visual_block_shape,
                        embd_pdrop=embd_pdrop, resid_pdrop=resid_pdrop, attn_pdrop=attn_pdrop,
                        tok_pdrop=tok_pdrop, n_layer=n_layer, n_head=n_head, n_embd=n_embd,
                        n_unmasked=n_unmasked)
        self.config = config
        # input embedding stem
        self.OFF_tok = nn.Parameter(torch.randn(1, 1, n_embd))
        self.MOD_tok = nn.Parameter(torch.randn(1, 1, n_embd))
        self.pre_lnorm_vis = instantiate_from_config(pre_norm_cfg)
        self.pre_lnorm_aud = instantiate_from_config(pre_norm_cfg)
        self.vis_pos_emb = instantiate_from_config(vis_pos_emb_module)
        self.aud_pos_emb = instantiate_from_config(aud_pos_emb_module)
        self.tok_drop_vis = nn.Dropout3d(tok_pdrop)
        self.tok_drop_aud = nn.Dropout2d(tok_pdrop)
        self.drop = nn.Dropout(config.embd_pdrop)
        # transformer
        self.blocks = nn.Sequential(*[Block(config) for _ in range(config.n_layer)])
        # heads
        self.ln_f = nn.LayerNorm(config.n_embd)
        self.off_head = nn.Linear(config.n_embd, config.num_offset_cls, bias=False)
        self.apply(self._init_weights)

    def _init_weights(self, module):
        if isinstance(module, (nn.Linear, nn.Embedding)):
            module.weight.data.normal_(mean=0.0, std=0.02)
            if isinstance(module, nn.Linear) and module.bias is not None:
                module.bias.data.zero_()
        elif isinstance(module, nn.LayerNorm):
            module.bias.data.zero_()
            module.weight.data.fill_(1.0)

    def forward(self, vis_embd: torch.Tensor, aud_embd: torch.Tensor, targets=None):
        B, Tv, Dv, H, W = vis_embd.shape
        B, Da, F, Ta = aud_embd.shape
        assert Da == Dv, f'Please define a bridge or fix {Da} vs {Dv}'

        # (B, Tv, H, W, Dv), (B, F, Ta, Da) <-
        vis_embd = vis_embd.permute(0, 1, 3, 4, 2).contiguous()
        aud_embd = aud_embd.permute(0, 2, 3, 1).contiguous()

        # making sure that both embeddings are normalized to the same base (can be configured as identity tho)
        vis_embd = self.pre_lnorm_vis(vis_embd)
        aud_embd = self.pre_lnorm_aud(aud_embd)

        # apply individual pos embeddings
        vis_embd = self.vis_pos_emb(vis_embd)
        aud_embd = self.aud_pos_emb(aud_embd)

        # apply token dropout
        vis_embd = self.tok_drop_vis(vis_embd)
        aud_embd = self.tok_drop_aud(aud_embd.transpose(2, 1)).transpose(2, 1)  # applying dropout on time-dim

        # (B, Nv, Dv), (B, Na, Da)
        vis_embd = vis_embd.view(B, Tv*H*W, Dv)
        aud_embd = aud_embd.view(B, F*Ta, Da)

        # broadcasting special tokens to the batch size
        off_tok = self.OFF_tok.expand(B, -1, -1)
        mod_tok = self.MOD_tok.expand(B, -1, -1)

        # (B, 2+Nv+1+Na, D)
        token_embeddings = torch.cat((off_tok, vis_embd, mod_tok, aud_embd), dim=1)

        x = self.drop(token_embeddings)

        x = self.blocks(x)
        x = self.ln_f(x)
        # this depends on the order in `token_embeddings`
        off_logits = self.off_head(x[:, 0, :])

        return off_logits

if __name__ == '__main__':
    pos_enc = FourierPositionEncoding(block_shape=[9, 27], n_embd=512)
    x = torch.rand(3, 9, 27, 512)
    print('input', x.shape)
    x = pos_enc(x)
    print(x.shape)

    pos_enc = FourierPositionEncoding(block_shape=[50, 7, 7], n_embd=512)
    x = torch.rand(3, 50, 7, 7, 512)
    print('input', x.shape)
    x = pos_enc(x)
    print(x.shape)

    pos_enc = PositionEmbeddingLearnedVisual([50, 7, 7], 512)
    x = torch.rand(3, 50, 7, 7, 512)
    print('x', x.shape)
    print('x.shape', pos_enc(x).shape)

    pos_enc = PositionEmbeddingLearnedAudio([9, 27], 512)
    x = torch.rand(3, 9, 27, 512)
    print('x', x.shape)
    print('x.shape', pos_enc(x).shape)

    ############################################################################
    # from omegaconf import OmegaConf
    # from time import time

    # cfg = OmegaConf.load('./configs/av_sync.yaml')
    # cfg.training.use_half_precision = True

    # device = torch.device('cuda:2')
    # torch.cuda.set_device(device)

    # model = instantiate_from_config(cfg.model.params.transformer)
    # model = model.to(device)

    # start_time = time()
    # for i in range(500):
    #     # vis = torch.rand(1, 50, 512, 7, 7, device=device)
    #     # aud = torch.rand(1, 512, 9, 27, device=device)
    #     vis = torch.rand(1, 50, 512, 1, 1, device=device)
    #     aud = torch.rand(1, 512, 1, 27, device=device)
    #     # cls_logits, off_logits, sync_logits, att = model(vis, aud)
    #     # inference in half precision
    #     with torch.autocast('cuda', enabled=cfg.training.use_half_precision):
    #         out = model(vis, aud)
    # print('Time:', round(time() - start_time, 3))
    ############################################################################


# Torch Imports
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torchvision
import math

from layers import *

class Var_ResEncoder(nn.Module):

    def __init__(self, l_dim, depth, strides, inplanes=4, input_channels=1):
        self.inplanes = inplanes
        super(Var_ResEncoder, self).__init__()

        self.res_enc_blocks = nn.Sequential(*[DoubleRes(BasicBlock, in_f, out_f, stride) for in_f, out_f, stride
                                        in zip(depth, depth[1:], strides)])

        #self.avgpool = nn.AvgPool2d(7, stride=1, padding=3)
        #self.fc = nn.Linear(1024, l_dim)
        #self.fc.weight.data.copy_(torch.eye(l_dim, 1024))
        #self.relu = nn.ReLU(inplace=True)

        self.mu_fc = nn.Linear(1024, l_dim)
        self.logvar_fc = nn.Linear(1024, l_dim)
        self.mu_fc.weight.data.copy_(torch.eye(l_dim, 1024))
        self.logvar_fc.weight.data.copy_(torch.eye(l_dim, 1024))

    def _make_encoding_layer(self, inplanes, planes, stride=2):
        return DoubleRes(BasicBlock, inplanes, planes, stride=stride)

    def reparameterize(self, mu, logvar):
        std = torch.exp(logvar/2)
        eps = torch.randn_like(std)
        return mu + eps * std

    def encode(self, x):
        x = self.res_enc_blocks(x)

        #x = self.avgpool(x)
        x = x.view(x.size(0), -1)
        #x = self.fc(x)
        #x = self.relu(x)

        return self.mu_fc(x), self.logvar_fc(x)

    def forward(self, x):
        mu, logvar = self.encode(x)
        z = self.reparameterize(mu, logvar)
        return z, mu, logvar

class ResDecoder(nn.Module):

    def __init__(self, l_dim, depth, inplanes=4, input_channels=1):
        super(ResDecoder, self).__init__()
        self.inplanes = inplanes
        self.fc = nn.Linear(l_dim, 1024)
        self.fc.weight.data.copy_(torch.eye(1024, l_dim))
        self.relu = nn.ReLU(inplace=True)
        self.sigmoid = nn.Sigmoid()

        self.res_dec_blocks = nn.Sequential(*[ConvTransposeLayer(in_f, out_f, out_f) for in_f, out_f
                                            in zip(depth, depth[1:])])

        self.double_res = DoubleRes(BasicBlock, depth[-1],
                                input_channels, stride=1)

        self.conv = nn.Conv2d(input_channels, input_channels,
                                kernel_size=3, stride=1,
                                padding=1)

    def _make_decoding_layer(self, inplanes, deconvplanes, resnetplanes):
        return ConvTransposeLayer(inplanes, deconvplanes, resnetplanes)

    def forward(self, x):
        x = self.fc(x)
        x = self.relu(x)
        x = torch.reshape(x, (-1, 64, 4, 4))
        x = self.res_dec_blocks(x)
        x = self.double_res(x)
        x = self.conv(x)

        return x

class Var_ResAutoEncoder(nn.Module):
    '''
        Convolutional AutoEncoder model.
        This model assumes the use of 1-channel images. If using 3-channels,
        modify each instance of [1] to [3].
    '''
    def __init__(self, enc_depth, dec_depth, strides, l_dim):
        super().__init__()
        self.l_dim        = l_dim                # Computed in setup_model.py
        self.enc_features = enc_depth            # [1] + [4, 8, 16, 32]
        self.dec_features = dec_depth  # [l_dim] + [32, 16, 8] + [1]
        self.input_channels = 1
        self.inplanes = 4
        self.strides = strides
        self.encoder = Var_ResEncoder(self.l_dim, self.enc_features,
                                   self.strides, self.inplanes,
                                   self.input_channels)
        self.decoder = ResDecoder(self.l_dim, self.dec_features, self.inplanes,
                                   self.input_channels)

    def forward(self, x):
        x, mu, logvar = self.encoder(x)
        x = self.decoder(x)
        return x, mu, logvar


import itertools

from oslo.config import cfg

from neutron.agent.linux import utils
from neutron.plugins.common import constants as qconstants
from neutron.services.loadbalancer import constants

PROTOCOL_MAP = {
    constants.PROTOCOL_TCP: 'tcp',
    constants.PROTOCOL_HTTP: 'http',
    constants.PROTOCOL_HTTPS: 'tcp',
}

POOL_PROTOCOL_MAP = {
    constants.PROTOCOL_TCP: 'http',
    constants.PROTOCOL_HTTP: 'http',
    constants.PROTOCOL_HTTPS: 'https',
}

BALANCE_MAP = {
    constants.LB_METHOD_ROUND_ROBIN: 'rr',
    constants.LB_METHOD_LEAST_CONNECTIONS: 'least_conn',
    constants.LB_METHOD_SOURCE_IP: 'ip_hash'
}

ACTIVE = qconstants.ACTIVE
INACTIVE = qconstants.INACTIVE

def save_config(conf_path, logical_config):
    """Convert a logical configuration to the SEnginx version."""
    protocol = logical_config['vip']['protocol']
    if not protocol:
        return

    data = []
    data.extend(_build_global(logical_config))

    # build protocol specified configs
    if PROTOCOL_MAP[protocol] == "http":
        data.extend(_build_http(logical_config))
    else:
        data.extend(_build_tcp(logical_config))

    utils.replace_file(conf_path, '\n'.join(data))

def _build_global(config):
    opts = [
            'user senginx %s;' % cfg.CONF.user_group,
            'worker_processes 1;',
            'error_log error.log;',
            'pid nginx.pid;',
            'events {',
            'worker_connections 10240;',
            '}',
            '',
            ]

    return opts

def _build_http(config):
    opts = [
            'http {',
            'include /usr/local/senginx/conf/mime.types;',
            'default_type /usr/local/senginx/conf/application/octet-stream;',
            'access_log http.access.log;',
            'sendfile on;',
            'keepalive_timeout 65;',
            ' ',
            ]

    opts.extend(_build_http_upstream(config));
    opts.extend(_build_http_server(config));

    opts.append('}')

    return opts

def _build_http_upstream(config):
    lb_method = config['pool']['lb_method']

    if not config['members']:
        return []

    opts = [
        'upstream %s {' % config['pool']['id'],
    ]

    if lb_method != constants.LB_METHOD_ROUND_ROBIN:
        opts.append('%s;' % BALANCE_MAP.get(lb_method))

    opts.append('')

    # add session persistence (if available)
    persist_opts = _get_session_persistence(config)
    opts.extend(persist_opts)

    # add the members
    for member in config['members']:
        if member['status'] in (ACTIVE, INACTIVE) and member['admin_state_up']:
            server = (('server %(address)s:%(protocol_port)s '
                       'weight=%(weight)s;') % member)
            opts.append(server)

    # add the first health_monitor (if available)
    health_opts = _get_server_health_option(config)
    opts.extend(health_opts)

    opts.append('}')
    opts.append('')

    return opts

def _build_http_server(config):
    pool_protocol = config['pool']['protocol']

    if not config['members']:
        return []

    opts = [
        'server {',
        'listen %s:%d;' % (
            _get_first_ip_from_port(config['vip']['port']),
            config['vip']['protocol_port']
        ),
        '',
        'location / {',
        'proxy_pass %s://%s;' %
        (POOL_PROTOCOL_MAP[pool_protocol], config['pool']['id']),
        '}',
    ]

    if config['healthmonitors']:
        opts.append('location /senginx-check-http-status {');
        opts.append('check_status csv;');
        opts.append('}');

    opts.append('}');
    opts.append('');

    return opts

def _build_tcp(config):
    opts = [
            'tcp {',
            'access_log tcp.access.log;',
            ' ',
            ]

    opts.extend(_build_tcp_upstream(config));
    opts.extend(_build_tcp_server(config));

    opts.append('}')

    return opts

def _build_tcp_upstream(config):
    lb_method = config['pool']['lb_method']

    if not config['members']:
        return []

    opts = [
        'upstream %s {' % config['pool']['id'],
    ]

    if lb_method == constants.LB_METHOD_SOURCE_IP:
        opts.append('%s;' % BALANCE_MAP.get(lb_method))

    opts.append('')

    # add session persistence (if available)
    #persist_opts = _get_session_persistence(config)
    #opts.extend(persist_opts)

    # add the members
    for member in config['members']:
        if member['status'] in (ACTIVE, INACTIVE) and member['admin_state_up']:
            server = (('server %(address)s:%(protocol_port)s') % member)
            if lb_method == constants.LB_METHOD_ROUND_ROBIN:
                server = server + ((' weight=%(weight)s;') % member)
            else:
                server = server + ';'
            opts.append(server)

    # add the first health_monitor (if available)
    health_opts = _get_server_health_option(config)
    opts.extend(health_opts)

    opts.append('}')
    opts.append('')

    return opts

def _build_tcp_server(config):
    if not config['members']:
        return []

    opts = [
        'server {',
        'listen %s:%d;' % (
            _get_first_ip_from_port(config['vip']['port']),
            config['vip']['protocol_port']
        ),
        '',
        'proxy_pass %s;' % config['pool']['id'],
        '}',
        ''
    ]

    return opts

def _get_first_ip_from_port(port):
    for fixed_ip in port['fixed_ips']:
        return fixed_ip['ip_address']

def _get_server_health_option(config):
    """return the first active health option."""
    for monitor in config['healthmonitors']:
        # not checking the status of healthmonitor for two reasons:
        # 1) status field is absent in HealthMonitor model
        # 2) only active HealthMonitors are fetched with
        # LoadBalancerCallbacks.get_logical_device
        if monitor['admin_state_up']:
            break
    else:
        return []

    opts = []

    delay = int(monitor['delay']) * 1000
    timeout = int(monitor['timeout']) * 1000

    if monitor['type'] == constants.HEALTH_MONITOR_HTTP:
        opts.append('check interval=%d fall=%d'
                    ' timeout=%d type=http;' %
                    (delay, monitor['max_retries'], timeout))
        opts.append('check_http_send "%(http_method)s %(url_path)s '
                    'HTTP/1.0\\r\\n\\r\\n";' % monitor)
        opts.append('check_http_expect_alive %s;' %
                    ' '.join(_expand_expected_codes(monitor['expected_codes'])))
    elif monitor['type'] == constants.HEALTH_MONITOR_HTTPS:
        opts.append('check interval=%d fall=%d'
                    ' timeout=%d type=ssl_hello;' %
                    (delay, monitor['max_retries'], timeout))
    elif monitor['type'] == constants.HEALTH_MONITOR_TCP:
        opts.append('check interval=%d fall=%d'
                    ' timeout=%d type=tcp;' %
                    (delay, monitor['max_retries'], timeout))

    return opts

def _get_session_persistence(config):
    persistence = config['vip'].get('session_persistence')
    if not persistence:
        return []

    opts = []
    if persistence['type'] == constants.SESSION_PERSISTENCE_SOURCE_IP:
        # XXX: no source ip persistence is availiable currently
        return opts
    elif persistence['type'] == constants.SESSION_PERSISTENCE_HTTP_COOKIE:
        opts.append('persistence insert_cookie cookie_name=senginx timeout=30;')
    elif (persistence['type'] == constants.SESSION_PERSISTENCE_APP_COOKIE and
          persistence.get('cookie_name')):
        opts.append('persistence insert_cookie cookie_name=senginx ' \
                'monitor_cookie=%s timeout=30;' %
                persistence['cookie_name'])

    return opts

def _expand_expected_codes(codes):
    """SEnginx does not support single response code,
    it only supports 2xx, 3xx, 4xx and 5xx.

    500 -> http_5xx
    200-204 -> http_2xx
    200, 203 -> http_2xx
    300, 304 -> http_3xx
    200-304 -> http_2xx http_3xx
    200, 304 -> http_2xx http_3xx
    ...
    """

    l_codes = []
    retval = []

    if '-' in codes:
        low, hi = codes.split('-')[:2]
        for i in range(int(low), int(hi) + 1):
            l_codes.append(str(i))
    else:
        l_codes = codes.replace(',', ' ').split(' ')

    for code in l_codes:
        code = code.strip()
        i_code = int(code)

        if i_code >= 200 and i_code < 300:
            retval.append('http_2xx')
        elif i_code >= 300 and i_code < 400:
            retval.append('http_3xx')
        elif i_code >= 400 and i_code < 500:
            retval.append('http_4xx')
        elif i_code >= 500 and i_code < 600:
            retval.append('http_5xx')

    return list(set(retval))


#!/usr/bin/env python
# coding: utf-8

# In[1]:

#encoding=utf-8
import pandas as pd
import numpy as np
import os
import pylab
import matplotlib.pyplot as plt
from scipy import stats, fftpack
import sklearn
import sklearn.metrics
from numpy import square, sqrt, array, arange, mean, var, linspace
from scipy import stats, fftpack
from sklearn import preprocessing

# In[2]:

os.chdir('D:/UNIVERSITY_STUDY/PHM/2nd_term/NASA PHM/IMS/2nd_test/2nd_test/')
file_chdir=os.getcwd()
set1=[]
for root,dirs,files in os.walk(file_chdir):
    for file in files:
        File=pd.read_csv(file,header=None,sep='\s+',engine='python')
        set1.append(File)

# In[3]:

set1[0]

# In[4]:

set_size = 984 #数据集大小

# In[5]:

set2=[]
plt.figure(figsize=(12,5))  #利用快速傅立叶变换将2号轴承时域转化为频域
x2 = np.linspace(0, 1, 12000)
i=0
for i in range(set_size):
    y2 = set1[i][:][1]
    yy = fftpack.fft(y2)
    yf1=abs(fftpack.fft(y2))/len(x2)           #归一化处理
    xf1 = np.arange(len(y2))   # 频率
    set2.append(yf1)
plt.subplot(223)
plt.plot(xf1,yf1,'r')
plt.title('FFT of Mixed wave(normalization)',fontsize=9,color='r')
plt.show()

# In[6]:

set2

# In[7]:

import math
Rs0=1
Rs1=[]
Rs2=[]
Rs3=[]
Rs4=[]
Rs5=[]

def RS_value(data_set, freq1, freq2):
    Rs_val=[]
    fengzi=0
    qq1=0
    qq2=0
    fengmu1=0
    fengmu2=0
    fengmu=0
    for j in range(set_size):
        f0_avg=np.mean(data_set[0])
        f1_avg=np.mean(data_set[j])
        for i in range(freq1,freq2):
            qq1=data_set[0][i]-f0_avg
            qq2=data_set[j][i]-f1_avg
            fengzi=np.abs(fengzi+qq1*qq2)
            fengmu1=fengmu1+qq1*qq1
            fengmu2=fengmu2+qq2*qq2
        fengmu=fengmu1*fengmu2
        Rs_val.append(fengzi/math.sqrt(fengmu1*fengmu2))
    return Rs_val

Rs1=RS_value(set2, 0, 2560)
Rs2=RS_value(set2, 2560, 5120)
Rs3=RS_value(set2, 5120, 7680)
Rs4=RS_value(set2, 7680, 10240)
Rs5=RS_value(set2, 0, 10240)

# In[8]:

#提取2号轴承均方根RMSE---------------1
i = 0
rmse = []
x = range(20480)
xs = range(set_size)
for i in range(set_size):
    y = set1[i][:][1]
    rmse.append(sqrt(sklearn.metrics.mean_squared_error(x,y)))
plt.plot(xs,rmse,'r')
plt.xlabel(u"Time domain-RMSE") #X轴标签
plt.ylabel("RMSE/g") #Y轴标签
plt.show()

#提取2号轴承方差VAR-------------2
i = 0
var1 = []
x = range(20480)
xs = range(set_size)
for i in range(set_size):
    y = set1[i][:][1]
    var1.append(var(y))
plt.plot(xs,var1,'r')
plt.xlabel(u"Time domain-VAR") #X轴标签
plt.ylabel("VAR") #Y轴标签
plt.show()

#提取2号轴承绝对均值-----------------------------3
i = 0
j = 0
mean1 = []
x = range(20480)
xs = range(set_size)
for i in range(set_size):
    y = set1[i][:][1]
    y1 = array([y]).T
    for j in range(20480):
        y1[j]=abs(y1[j])
    mean1.append(mean(y1))
plt.plot(xs,mean1,'r')
plt.xlabel(u"Time domain-mean") #X轴标签
plt.ylabel("mean/g") #Y轴标签
plt.show()

#提取2号轴承峰度-------------------------4
i = 0
j = 0
ku_2 = []
x = range(20480)
xs = range(set_size)
for i in range(set_size):
    y = set1[i][:][1]
    ku_2.append(stats.kurtosis(y))
plt.plot(xs,ku_2,'r')
plt.xlabel(u"Time domain-ku") #X轴标签
plt.ylabel("ku") #Y轴标签
plt.show()

#提取2号轴承偏度----------5
i = 0
j = 0
skew = []
x = range(20480)
xs = range(set_size)
for i in range(set_size):
    y = set1[i][:][1]
    s = pd.Series(y)
    skew.append(s.skew())
plt.plot(xs,skew,'r')
plt.xlabel(u"Time domain-skew") #X轴标签
plt.ylabel("skew") #Y轴标签
plt.show()

#提取2号轴承峰峰值----------------6
i = 0
j = 0
ptp_2 = []
x = range(20480)
xs = range(set_size)
for i in range(set_size):
    y = set1[i][:][1]
    ptp_2.append(max(y) - min(y))
plt.plot(xs,ptp_2,'r')
plt.xlabel(u"Time domain-ptp") #X轴标签
plt.ylabel("peak-to-peak value") #Y轴标签
plt.show()

# In[9]:

#装向量
set3 = [[0] * 6 for i in range(set_size)]
rmse=preprocessing.scale(rmse)
var1=preprocessing.scale(var1)
mean1=preprocessing.scale(mean1)
ku_2=preprocessing.scale(ku_2)
skew=preprocessing.scale(skew)
ptp_2=preprocessing.scale(ptp_2)
for i in range(set_size):
    set3[i][0]=rmse[i]
    set3[i][1]=var1[i]
    set3[i][2]=mean1[i]
    set3[i][3]=ku_2[i]
    set3[i][4]=skew[i]
    set3[i][5]=ptp_2[i]
print(set3)

# In[10]:

#把向量转化为Rs向量
Rst1=RS_value(set3, 0, 6)

plt.plot(xs,Rst1,'r')
plt.xlabel(u"Time domain-RSt") #X轴标签
plt.ylabel("Rst value") #Y轴标签
plt.show()
print(Rst1)

# In[11]:

#开始尝试进行小波包分解

import pywt
import pywt.data

mode = pywt.Modes.smooth
set4=[]

def plot_signal_decomp(data, w, title):
    """Decompose and plot a signal S.
    S = An + Dn + Dn-1 + ... + D1
    """
    w = pywt.Wavelet(w)#选取小波函数
    a = data
    ca = []#近似分量
    for i in range(3):
        (a, d) = pywt.dwt(a, w, mode)#进行3阶离散小波变换
        ca.append(a)

    rec_a = []

    for i, coeff in enumerate(ca):
        coeff_list = [coeff, None] + [None] * i
        rec_a.append(pywt.waverec(coeff_list, w))#重构
    set4.append(rec_a[2])
for i in range(set_size):
    plot_signal_decomp(set1[i][1], 'sym5', "DWT: Ecg sample - Symmlets5")
print(set4)

# In[12]:

print(len(set4[0]))

# In[13]:

#计算能量比
#分为8段：0-2560；2560-5120；5120-7680；7680-10240
#10240-12800;12800-15360;15360-17920;17920-20480
Et=[]
for i in range(set_size):
    E1=0
    for j in range(2560):
        E1=E1+np.abs(set4[i][j])*np.abs(set4[i][j])

    E2=0
    for j in range(2560,5120):
        E2=E2+np.abs(set4[i][j])*np.abs(set4[i][j])

    E3=0
    for j in range(5120,7680):
        E3=E3+np.abs(set4[i][j])*np.abs(set4[i][j])

    E4=0
    for j in range(7680,10240):
        E4=E4+np.abs(set4[i][j])*np.abs(set4[i][j])

    E5=0
    for j in range(10240,12800):
        E5=E5+np.abs(set4[i][j])*np.abs(set4[i][j])

    E6=0
    for j in range(12800,15360):
        E6=E6+np.abs(set4[i][j])*np.abs(set4[i][j])

    E7=0
    for j in range(15360,17920):
        E7=E7+np.abs(set4[i][j])*np.abs(set4[i][j])

    E8=0
    for j in range(17920,20480):
        E8=E8+np.abs(set4[i][j])*np.abs(set4[i][j])
    Ez=E1+E2+E3+E4+E5+E6+E7+E8
    Et.append([E1/Ez,E2/Ez,E3/Ez,E4/Ez,E5/Ez,E6/Ez,E7/Ez,E8/Ez])

print(Et)

# In[14]:

#构建特征集
feature= [[0] * 14 for i in range(set_size)]
for i in range(set_size):
    feature[i][0]=Rs1[i]
    feature[i][1]=Rs2[i]
    feature[i][2]=Rs3[i]
    feature[i][3]=Rs4[i]
    feature[i][4]=Rs5[i]
    feature[i][5]=Rst1[i]
    feature[i][6]=Et[i][0]
    feature[i][7]=Et[i][1]
    feature[i][8]=Et[i][2]
    feature[i][9]=Et[i][3]
    feature[i][10]=Et[i][4]
    feature[i][11]=Et[i][5]
    feature[i][12]=Et[i][6]
    feature[i][13]=Et[i][7]

# In[15]:

feature[0]

# In[16]:

#corr计算
Zcorr=[]
for j in range(14):
    fengmu=0
    fengzi=0
    qq1=0
    qq2=0
    corr=0
    ss1=[]
    #计算第一个特征的corr
    for i in range(set_size):
        ss1.append(feature[i][j])
    Fb=np.mean(ss1)
    Lb=(set_size-1)/2
    for i in range(set_size):
        fengzi=fengzi+np.abs((ss1[i]-Fb)*(i-Lb))
        qq1=qq1+(ss1[i]-Fb)*(ss1[i]-Fb)
        qq2=qq2+(i-Lb)*(i-Lb)
    fengmu=math.sqrt(qq1*qq2)
    corr=fengzi/fengmu
    Zcorr.append(corr)
print(Zcorr)

# In[17]:

#计算Mon

ZMon=[]
for j in range(14):
    ss2=[]
    Zeng=0
    Fu=0
    #计算第一个特征的Mon
    for i in range(set_size):
        ss2.append(feature[i][j])
    for i in range(set_size-1):
        if(ss2[i+1]-ss2[i]>0):
            Zeng=Zeng+1
        if(ss2[i+1]-ss2[i]<0):
            Fu=Fu+1
    Mon=np.abs((Zeng-Fu)/(set_size-1))
    ZMon.append(Mon)
print(ZMon)

# In[18]:

#计算敏感特性的标准Cri=（Corr+Mon）/2
ZCri=[]
for i in range(14):
    ZCri.append((Zcorr[i]+ZMon[i])/2)
print(ZCri)

# In[19]:

#故障标签
data_lable = []
for i in range(set_size):
    data_lable.append((1/(set_size-1))*i)
data_lable

# In[22]:

#最终特征集,基于敏感指标的大小排序，丢弃了Rs3和Rs5这两个特征
feature_fin = [[0] * 13 for i in range(set_size)]
for i in range(set_size):
    feature_fin[i][0] = feature[i][0]
    feature_fin[i][1] = feature[i][1]
    feature_fin[i][2] = feature[i][3]
    feature_fin[i][3] = feature[i][5]
    feature_fin[i][4] = feature[i][6]
    feature_fin[i][5] = feature[i][7]
    feature_fin[i][6] = feature[i][8]
    feature_fin[i][7] = feature[i][9]
    feature_fin[i][8] = feature[i][10]
    feature_fin[i][9] = feature[i][11]
    feature_fin[i][10] = feature[i][12]
    feature_fin[i][11] = feature[i][13]
    feature_fin[i][12] = data_lable[i]

# In[50]:

len(feature_fin)

# In[38]:

#将时间序列数据转化为有监督学习问题
def series_to_supervised(data, n_in=1, n_out=1, dropnan=True):
    n_vars = 1 if type(data) is list else data.shape[1]
    df = pd.DataFrame(data)
    cols, names = list(), list()
    # input sequence (t-n, ... t-1)
    for i in range(n_in, 0, -1):
        cols.append(df.shift(i))
        names += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)]
    # forecast sequence (t, t+1, ... t+n)
    for i in range(0, n_out):
        cols.append(df.shift(-i))
        if i == 0:
            names += [('var%d(t)' % (j+1)) for j in range(n_vars)]
        else:
            names += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)]
    # put it all together
    agg = pd.concat(cols, axis=1)
    agg.columns = names
    # drop rows with NaN values
    if dropnan:
        agg.dropna(inplace=True)
    return agg

#转换数据集
values = np.array(feature_fin)
re_feature = series_to_supervised(values)
# drop columns we don't want to predict
re_feature.drop(re_feature.columns[[13,14,15,16,17,18,19,20,21,22,23,24]], axis=1, inplace=True)

# In[51]:

re_feature

# 开始建立预测模型

# In[69]:

#取偶数点和奇数点分别作为训练集和测试集
values = re_feature.values
train_data = []#训练集
valid_data = []#验证集
test_data = []#测试集
for i in range(set_size-1):
    if i%3 == 0:
        valid_data.append(values[i])
    elif i%3 == 1:
        test_data.append(values[i])
    else:
        train_data.append(values[i])
train_data = np.array(train_data)
valid_data = np.array(valid_data)
test_data = np.array(test_data)
train_x, train_y = train_data[:, :-1], train_data[:, -1]
valid_x, valid_y = valid_data[:, :-1], valid_data[:, -1]
test_x, test_y = test_data[:, :-1], test_data[:, -1]
#将训练集和测试集转换成符合LSTM要求的数据格式,即 [样本，时间步，特征]
train_x = train_x.reshape((train_x.shape[0], 1, train_x.shape[1]))
valid_x = valid_x.reshape((valid_x.shape[0], 1, valid_x.shape[1]))
test_x = test_x.reshape((test_x.shape[0], 1, test_x.shape[1]))
print(train_x.shape, train_y.shape, test_x.shape, test_y.shape, valid_x.shape, valid_y.shape)

# In[70]:

#通过keras框架建立LSTM网络模型并训练
from keras.models import Sequential
from keras.layers import Dense, LSTM, Dropout

model = Sequential()
model.add(LSTM(50, activation='relu',input_shape=(train_x.shape[1], train_x.shape[2])))
model.add(Dense(1, activation='linear'))
model.compile(loss='mean_squared_error', optimizer='adam')
# fit network
LSTM = model.fit(train_x, train_y, epochs=100, batch_size=72, validation_data=(valid_x, valid_y), verbose=2, shuffle=False)
# plot history
plt.plot(LSTM.history['loss'], label='train')
plt.plot(LSTM.history['val_loss'], label='test')
plt.legend()
plt.show()

# In[71]:

#带入测试集进行预测
xl=range(328)
plt.figure(figsize=(24,8))
test_predict = model.predict(test_x)
plt.plot(xl,test_y,'r')
plt.plot(xl,test_predict,'b')
plt.show()

# In[ ]:


"""
MISCELLANEOUS_1

2020 01 15 JJH
"""
import os
import inspect
import numpy as np
import pandas as pd
from collections import OrderedDict
import activation_functions_module as afm
import neural_network_1 as nn_1
import matplotlib as plt

# Last time we made a single layer neural network. Refer neural_network_1.py for further comments.

class single_layer_3:
    def __init__(self, initial_input, layers, output_size, activation_function=afm.sigmoid):

        self.x = initial_input

        if IndexError:
            input_size = self.x.shape[0]
        else:
            input_size = self.x.shape[1]

        self.Weights = OrderedDict()
        self.Biases = OrderedDict()

        self.lst = []  # we need to use it again on product method.
        self.lst.append(input_size)

        for lyr in layers:
            self.lst.append(lyr)
        self.lst.append(output_size)

        for i in range(len(self.lst)-1):
            self.Weights['w'+str(i+1)] = np.random.randint(10, size=(self.lst[i], self.lst[i+1]))
            self.Biases['b'+str(i+1)] = np.random.randint(10, size=(1, self.lst[i+1]))
        self.activator = activation_function

    def product(self):
        product_iterand = self.x
        product_list = []

        for k in range(len(self.lst)-1):
            output_itr = self.activator(product_iterand.dot(self.Weights['w'+str(k+1)]) + self.Biases['b'+str(k+1)])
            product_iterand = output_itr
            product_list.append(output_itr)
        return output_itr, product_list

if __name__ == '__main__':
    np.random.seed(115)

    '''
before that, let us take a look at inspect module. this can be very useful  as we are going to write codes on basic IDLE.
'''
    #print(inspect.getmembers(afm, inspect.isfunction))
    """
[('arctan', <function arctan at 0x0000017C957F28B8>),
('identity', <function identity at 0x0000017C957F2948>),
('relu', <function relu at 0x0000017C86DCA0D8>),
('sigmoid', <function sigmoid at 0x0000017C86DCA318>),
('tanh', <function tanh at 0x0000017C957F2828>)]

> show all available functions from the targeted module. For that, you must previously import it.

"""
    #print(inspect.getmembers(nn_1, inspect.isclass))
    """
[('OrderedDict', <class 'collections.OrderedDict'>),
('single_layer_1', <class 'neural_network_1.single_layer_1'>),
('single_layer_2', <class 'neural_network_1.single_layer_2'>),
('single_layer_3', <class 'neural_network_1.single_layer_3'>)]

> show all available classes from the targeted module. We have imported neural_network_1 as nn_1.
"""
    #print(inspect.getdoc(nn_1.single_layer_3))
    """
    e.g.
initial_input = np.array([5, 7, 8 ,3, 4])
layers = [3, 5]
output_size = 3
> see document of targeted function.
    """
    #print(inspect.getsource(nn_1.single_layer_1))
    """
class single_layer_1:
    def __init__(self,
                 initial_input=np.array([1, 2]),
                 layer_1=np.random.randint(10, size=(2, 3)),
                 bias_1=np.random.randint(10, size=(1, 3)),
                 activation_function=afm.sigmoid):
        self.x = initial_input
        self.w = layer_1
        self.b = bias_1
        self.activator = activation_function  # now we can call these variables as much as we want.
        #print('x =', self.x, self.x.shape[0])
        #print('w =', self.w, self.w.shape)
        #print('b =', self.b, self.b.shape)

    def product(self):
        # output = self.x.dot(self.w) + self.b
        output = self.activator(self.x.dot(self.w) + self.b)
        return output
> show sourcecode of targeted function, or class, any other object.
    """
    print(inspect.signature(nn_1.single_layer_3))
    """
(initial_input, layers, output_size, activation_function=<function sigmoid at 0x00000252F432A318>)
> show arguments of targeted object.
    """
    #print(os.getcwd()) # C:\Users\Azerates\Documents\GitHub\STRZ_REPOSITORY_20191216
    #os.system('notepad')
    #files = os.listdir(os.curdir)  #files and directories
    #print(files)
    """
['.git',
'.gitattributes',
'.github',
'2_Layer_Network_Gradient_Descent.py',
'activation_functions_module.py',
'Apply_Gradient_Descent_To_Neural_Network.py',
'Apply_Gradient_Descent_To_Neural_Network_module.py',
'azerates20191122a.py',
'basic_neuron_practice.py',
'cointoss.py',
'forward_propagation_001.py',
'forward_propagation_002.py',
'neural_network_1.py',
'neural_network_2.py',
'README.md',
'__pycache__']

> source link : https://stackoverflow.com/questions/11968976/list-files-only-in-the-current-directory
we can now import files by this not using explorers
    """
    #for i in files:
      #  print(i)
    """
Same result, but aligned.

.git
.gitattributes
.github
2_Layer_Network_Gradient_Descent.py
activation_functions_module.py
Apply_Gradient_Descent_To_Neural_Network.py
Apply_Gradient_Descent_To_Neural_Network_module.py
azerates20191122a.py
basic_neuron_practice.py
cointoss.py
forward_propagation_001.py
forward_propagation_002.py
neural_network_1.py
neural_network_2.py
README.md
__pycache__
    """
    # print(inspect.getsource(nn_1)) # shows all source code. not recommended.
    # print(inspect.getsource(np))
    print('version 3 test')
    sl_input = np.array([0.1,0.2, 0.3, 0.4, 0.5, 0.4, 0.3, 0.2, 0.1])
    sl = single_layer_3(
                        initial_input=sl_input,
                        layers=[3, 4, 3, 2, 6, 4, 10, 4, 3, 5, 2],
                        output_size=10,
                        activation_function=afm.softmax)
    x = sl.product()[0]
    print( (x - np.mean(x)) / np.std(x) )  # z-centralisation
    print( (x - np.min(x)) / (np.max(x) - np.min(x))) # min-max normalization.
    # it seems, we may not need to add z-centralisation or min-max normalization
    print(x)
    print(np.sum(afm.softmax(sl_input))) # = 1.0
    print('\n')

    """How can we access files, which are not on this directory?
"""
    #print(inspect.getsource(os.path))
    print(os.path.relpath(
        'C:/Users/Azerates/PycharmProjects/ITWILL/SCRATCH_LECT_11', os.getcwd()
    )) #..\..\..\PycharmProjects\ITWILL\SCRATCH_LECT_11

    files = os.listdir(os.path.join('..', '..', '..','PycharmProjects', 'ITWILL', 'SCRATCH_LECT_11'))
    for i in files:
        print(i)
        """
azerates20191211a.py
breast-cancer-wisconsin-data.csv
iris.csv
KNN Classifier Reinvention azerates20191210c.py
Scikit Learn KNN azerates20191210a_knn.py
Scikit Learn Wisconsin Breast Cancer azerates20191210b_knn.py
__init__.py
        """

    iris_path = os.path.join('..', '..', '..','PycharmProjects', 'ITWILL', 'SCRATCH_LECT_11', 'iris.csv').replace('\\', '/')
    print('iris_path =',iris_path) # iris_path = ../../../PycharmProjects/ITWILL/SCRATCH_LECT_11/iris.csv

    colnames = [
        'sepal-length',
        'sepal-width',
        'petal-length',
        'petal-width',
        'Class'
    ]

    iris_path_data = pd.read_csv(iris_path, header=None, encoding='UTF-8', names=colnames)
    print(type(iris_path_data))
# refer from https://stackoverflow.com/questions/16476924/how-to-iterate-over-rows-in-a-dataframe-in-pandas
    for index, row in iris_path_data.iterrows():
        print(row[colnames[0]], row[colnames[1]], row[colnames[2]], row[colnames[3]], row[colnames[4]])
    """
result:
5.1 3.5 1.4 0.2 Iris-setosa
4.9 3.0 1.4 0.2 Iris-setosa
4.7 3.2 1.3 0.2 Iris-setosa
4.6 3.1 1.5 0.2 Iris-setosa
5.0 3.6 1.4 0.2 Iris-setosa
5.4 3.9 1.7 0.4 Iris-setosa
4.6 3.4 1.4 0.3 Iris-setosa
5.0 3.4 1.5 0.2 Iris-setosa
4.4 2.9 1.4 0.2 Iris-setosa
4.9 3.1 1.5 0.1 Iris-setosa
5.4 3.7 1.5 0.2 Iris-setosa
4.8 3.4 1.6 0.2 Iris-setosa
4.8 3.0 1.4 0.1 Iris-setosa
4.3 3.0 1.1 0.1 Iris-setosa
5.8 4.0 1.2 0.2 Iris-setosa
5.7 4.4 1.5 0.4 Iris-setosa
5.4 3.9 1.3 0.4 Iris-setosa
5.1 3.5 1.4 0.3 Iris-setosa
5.7 3.8 1.7 0.3 Iris-setosa
5.1 3.8 1.5 0.3 Iris-setosa
5.4 3.4 1.7 0.2 Iris-setosa
5.1 3.7 1.5 0.4 Iris-setosa
4.6 3.6 1.0 0.2 Iris-setosa
5.1 3.3 1.7 0.5 Iris-setosa
4.8 3.4 1.9 0.2 Iris-setosa
5.0 3.0 1.6 0.2 Iris-setosa
5.0 3.4 1.6 0.4 Iris-setosa
5.2 3.5 1.5 0.2 Iris-setosa
5.2 3.4 1.4 0.2 Iris-setosa
4.7 3.2 1.6 0.2 Iris-setosa
4.8 3.1 1.6 0.2 Iris-setosa
5.4 3.4 1.5 0.4 Iris-setosa
5.2 4.1 1.5 0.1 Iris-setosa
5.5 4.2 1.4 0.2 Iris-setosa
4.9 3.1 1.5 0.1 Iris-setosa
5.0 3.2 1.2 0.2 Iris-setosa
5.5 3.5 1.3 0.2 Iris-setosa
4.9 3.1 1.5 0.1 Iris-setosa
4.4 3.0 1.3 0.2 Iris-setosa
5.1 3.4 1.5 0.2 Iris-setosa
5.0 3.5 1.3 0.3 Iris-setosa
4.5 2.3 1.3 0.3 Iris-setosa
4.4 3.2 1.3 0.2 Iris-setosa
5.0 3.5 1.6 0.6 Iris-setosa
5.1 3.8 1.9 0.4 Iris-setosa
4.8 3.0 1.4 0.3 Iris-setosa
5.1 3.8 1.6 0.2 Iris-setosa
4.6 3.2 1.4 0.2 Iris-setosa
5.3 3.7 1.5 0.2 Iris-setosa
5.0 3.3 1.4 0.2 Iris-setosa
7.0 3.2 4.7 1.4 Iris-versicolor
6.4 3.2 4.5 1.5 Iris-versicolor
6.9 3.1 4.9 1.5 Iris-versicolor
5.5 2.3 4.0 1.3 Iris-versicolor
6.5 2.8 4.6 1.5 Iris-versicolor
5.7 2.8 4.5 1.3 Iris-versicolor
6.3 3.3 4.7 1.6 Iris-versicolor
4.9 2.4 3.3 1.0 Iris-versicolor
6.6 2.9 4.6 1.3 Iris-versicolor
5.2 2.7 3.9 1.4 Iris-versicolor
5.0 2.0 3.5 1.0 Iris-versicolor
5.9 3.0 4.2 1.5 Iris-versicolor
6.0 2.2 4.0 1.0 Iris-versicolor
6.1 2.9 4.7 1.4 Iris-versicolor
5.6 2.9 3.6 1.3 Iris-versicolor
6.7 3.1 4.4 1.4 Iris-versicolor
5.6 3.0 4.5 1.5 Iris-versicolor
5.8 2.7 4.1 1.0 Iris-versicolor
6.2 2.2 4.5 1.5 Iris-versicolor
5.6 2.5 3.9 1.1 Iris-versicolor
5.9 3.2 4.8 1.8 Iris-versicolor
6.1 2.8 4.0 1.3 Iris-versicolor
6.3 2.5 4.9 1.5 Iris-versicolor
6.1 2.8 4.7 1.2 Iris-versicolor
6.4 2.9 4.3 1.3 Iris-versicolor
6.6 3.0 4.4 1.4 Iris-versicolor
6.8 2.8 4.8 1.4 Iris-versicolor
6.7 3.0 5.0 1.7 Iris-versicolor
6.0 2.9 4.5 1.5 Iris-versicolor
5.7 2.6 3.5 1.0 Iris-versicolor
5.5 2.4 3.8 1.1 Iris-versicolor
5.5 2.4 3.7 1.0 Iris-versicolor
5.8 2.7 3.9 1.2 Iris-versicolor
6.0 2.7 5.1 1.6 Iris-versicolor
5.4 3.0 4.5 1.5 Iris-versicolor
6.0 3.4 4.5 1.6 Iris-versicolor
6.7 3.1 4.7 1.5 Iris-versicolor
6.3 2.3 4.4 1.3 Iris-versicolor
5.6 3.0 4.1 1.3 Iris-versicolor
5.5 2.5 4.0 1.3 Iris-versicolor
5.5 2.6 4.4 1.2 Iris-versicolor
6.1 3.0 4.6 1.4 Iris-versicolor
5.8 2.6 4.0 1.2 Iris-versicolor
5.0 2.3 3.3 1.0 Iris-versicolor
5.6 2.7 4.2 1.3 Iris-versicolor
5.7 3.0 4.2 1.2 Iris-versicolor
5.7 2.9 4.2 1.3 Iris-versicolor
6.2 2.9 4.3 1.3 Iris-versicolor
5.1 2.5 3.0 1.1 Iris-versicolor
5.7 2.8 4.1 1.3 Iris-versicolor
6.3 3.3 6.0 2.5 Iris-virginica
5.8 2.7 5.1 1.9 Iris-virginica
7.1 3.0 5.9 2.1 Iris-virginica
6.3 2.9 5.6 1.8 Iris-virginica
6.5 3.0 5.8 2.2 Iris-virginica
7.6 3.0 6.6 2.1 Iris-virginica
4.9 2.5 4.5 1.7 Iris-virginica
7.3 2.9 6.3 1.8 Iris-virginica
6.7 2.5 5.8 1.8 Iris-virginica
7.2 3.6 6.1 2.5 Iris-virginica
6.5 3.2 5.1 2.0 Iris-virginica
6.4 2.7 5.3 1.9 Iris-virginica
6.8 3.0 5.5 2.1 Iris-virginica
5.7 2.5 5.0 2.0 Iris-virginica
5.8 2.8 5.1 2.4 Iris-virginica
6.4 3.2 5.3 2.3 Iris-virginica
6.5 3.0 5.5 1.8 Iris-virginica
7.7 3.8 6.7 2.2 Iris-virginica
7.7 2.6 6.9 2.3 Iris-virginica
6.0 2.2 5.0 1.5 Iris-virginica
6.9 3.2 5.7 2.3 Iris-virginica
5.6 2.8 4.9 2.0 Iris-virginica
7.7 2.8 6.7 2.0 Iris-virginica
6.3 2.7 4.9 1.8 Iris-virginica
6.7 3.3 5.7 2.1 Iris-virginica
7.2 3.2 6.0 1.8 Iris-virginica
6.2 2.8 4.8 1.8 Iris-virginica
6.1 3.0 4.9 1.8 Iris-virginica
6.4 2.8 5.6 2.1 Iris-virginica
7.2 3.0 5.8 1.6 Iris-virginica
7.4 2.8 6.1 1.9 Iris-virginica
7.9 3.8 6.4 2.0 Iris-virginica
6.4 2.8 5.6 2.2 Iris-virginica
6.3 2.8 5.1 1.5 Iris-virginica
6.1 2.6 5.6 1.4 Iris-virginica
7.7 3.0 6.1 2.3 Iris-virginica
6.3 3.4 5.6 2.4 Iris-virginica
6.4 3.1 5.5 1.8 Iris-virginica
6.0 3.0 4.8 1.8 Iris-virginica
6.9 3.1 5.4 2.1 Iris-virginica
6.7 3.1 5.6 2.4 Iris-virginica
6.9 3.1 5.1 2.3 Iris-virginica
5.8 2.7 5.1 1.9 Iris-virginica
6.8 3.2 5.9 2.3 Iris-virginica
6.7 3.3 5.7 2.5 Iris-virginica
6.7 3.0 5.2 2.3 Iris-virginica
6.3 2.5 5.0 1.9 Iris-virginica
6.5 3.0 5.2 2.0 Iris-virginica
6.2 3.4 5.4 2.3 Iris-virginica
5.9 3.0 5.1 1.8 Iris-virginica
    """


import random
import torch
import numpy as np
from matplotlib import pyplot as plt
import umap
import hdbscan
from sklearn.cluster import KMeans

fingerpaths = []
with open("filtered_paths.txt", "r") as f:
    fingerpaths = [line.split(",")[0] for line in f.readlines()]
print("num of files:", len(fingerpaths))

random.shuffle(fingerpaths)
sample_of_paths = fingerpaths[:100000]
#sample_of_timeseries = []
sample_of_gt = []
for p in sample_of_paths:
    whole = torch.from_numpy(np.load(p))
    #print("whole shape", whole.shape) # torch.Size([1003])
    x = whole[:1000]
    #print("x shape", x.shape) # torch.Size([1000])
    gt = whole[1000:-1]
    #print("gt shape", gt.shape) # torch.Size([2])
    #sample_of_timeseries.append(x)
    sample_of_gt.append(gt)
#print(len(sample_of_timeseries))
#eg_x = sample_of_timeseries[0]
#print("x shape", eg_x.shape)
eg_gt = sample_of_gt[0]
print("gt shape", eg_gt.shape)

#whole_series_tensor = torch.stack(sample_of_timeseries)

kmeans = KMeans(n_clusters=10).fit(torch.stack(sample_of_gt))
print("kmeans labels", kmeans.labels_[:5])
predictions = kmeans.predict(torch.stack(sample_of_gt[:5]))
print("predictions:", predictions)
print(fingerpaths[0])
print(sample_of_gt[0])

rest_of_gt = []
for k in range(100000, len(fingerpaths)):
    whole = torch.from_numpy(np.load(fingerpaths[k]))
    gt = whole[1000:-1]
    rest_of_gt.append(gt)

print("rest_of_gt len", len(rest_of_gt))
rest_of_paths = fingerpaths[100000:]
print("rest_of_paths len", len(rest_of_paths))
rest_of_predictions = kmeans.predict(torch.stack(rest_of_gt))

with open ("clustered_paths.txt", "w") as f:
    for i in range(len(sample_of_paths)):
        line = f"{sample_of_paths[i]},{int(kmeans.labels_[i])}"
        f.write(line + "\n")

    for j in range(len(rest_of_gt)):
        line = f"{rest_of_paths[j]},{int(rest_of_predictions[j])}"
        f.write(line + "\n")

# whole_gt_tensor = torch.stack(sample_of_gt)
# fig, ax = plt.subplots(figsize=(10,10))
# ax.scatter(
#     whole_gt_tensor[:, 0],
#     whole_gt_tensor[:, 1]
# )
# plt.savefig("gt_scatter.png")

# reducer_big = umap.UMAP(n_components=64)
# embedding_big = reducer_big.fit_transform(whole_series_tensor)
# print("embedding_big shape", embedding_big.shape)

# # fig, ax = plt.subplots(figsize=(10,10))
# # ax.scatter(
# #     embedding[:, 0],
# #     embedding[:,1],
# #     s = 2
# # )
# # plt.savefig("umap_scatter.png")

# clusterer = hdbscan.HDBSCAN()
# clusterer.fit(embedding_big)
# cluster_labels = clusterer.labels_

# # do small embedding here
# reducer_small = umap.UMAP()
# embedding_small = reducer_small.fit_transform(whole_series_tensor)

# fig, ax = plt.subplots(figsize=(10,10))
# for label in np.unique(cluster_labels):
#     idx = np.argwhere(cluster_labels == label)
#     ax.scatter(
#         embedding_small[idx, 0],
#         embedding_small[idx, 1],
#         label=label
#     )
# plt.savefig("labelled_cluster.png")


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('accounting', '0002_donateamount_user'),
    ]

    operations = [
        migrations.CreateModel(
            name='DonateGoal',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('amount', models.DecimalField(max_digits=9, decimal_places=2)),
                ('created', models.DateTimeField(auto_now_add=True)),
                ('end', models.DateTimeField(null=True, blank=True)),
                ('description', models.TextField(null=True, blank=True)),
            ],
            options={
                'ordering': ('-created',),
            },
            bases=(models.Model,),
        ),
    ]


from django.shortcuts import render, redirect
from app.forms.business_form import BusinessForm
from app.models.buisness import Business
from app.models.neighborhood import Neighborhood
from django.http import JsonResponse

def buisness(request):
    data = Business.objects.all()
    if request.method == "POST":
        form = BusinessForm(request.POST)
        if form.is_valid():
            form.save()
            return redirect("buisness")
    else:
        form = BusinessForm()
    return render(request, "dashboard/buisness.html", {"form": form, "data": data})

def get_neighborhoods(request):
    region_id = request.GET.get("region_id")
    neighborhoods = Neighborhood.objects.filter(region_id=region_id).values(
        "id", "name"
    )
    return JsonResponse({"neighborhoods": list(neighborhoods)})


import openrouteservice
from openrouteservice import convert

import os
import json
'''
install ors library:
https://openrouteservice-py.readthedocs.io/en/latest/

get your api key:
https://openrouteservice.org/dev/#/home

paste it into key.txt (keep your key to yourself!)
'''
#print(os.getcwd())

routes = []

start = (8.34234,48.23424)
end = (8.34423,48.26424)
coords = (start, end)
keyfile = open('key.txt', 'r')
key = keyfile.read()
client = openrouteservice.Client(key=key) # Specify your personal API key
# decode_polyline needs the geometry only
geometry = client.directions(coords)['routes'][0]['geometry']
routes.append(geometry)

with open('route.geojson', 'w') as outfile:
    json.dump(geometry, outfile)

decoded = convert.decode_polyline(geometry)

print(decoded)


'''
Autonomous snake which should differentiate between pies and posions
Reinforcement learning with Q-Learn algorithm using mainly Keras as library

This file will give an array of actions, which the snake execute autonomously.

After research the best algorithm to use, seems to be SARSA, based on
http://cs229.stanford.edu/proj2016spr/report/060.pdf
Most of the Code can be found from:
https://github.com/korolvs/snake_nn/blob/master/nn_1.py

Author: Claudio
'''

# dependencies
import SnakeGame
import config
import random
import numpy as np
import tflearn
import math
from tflearn.layers.core import input_data, fully_connected
from tflearn.layers.estimator import regression
from statistics import mean
from collections import Counter

class SnakeNN:
    # Constructor
    def __init__(self, initial_games = 100, test_games = 100, goal_steps = 100, learning_rate = 1e-2, filename = 'snake_nn.tflearn'):
        self.initial_games = initial_games
        self.test_games = test_games
        self.goal_steps = goal_steps
        self.learning_rate = learning_rate
        self.vector_actions = [
                            [[-1,0], 0],
                            [[0,1], 1],
                            [[1,0], 2],
                            [[0,-1], 3],
                            ]
        self.filename = filename

    # Do 100 Initial games with random steps
    def initial_population(self):
        training_data = []

        for _ in range(self.initial_games):
            game = SnakeGame.SnakeGame()
            _, _, snake, _, _ = game.start()
            previous_observation = self.generate_observation(snake)

            for _ in range(self.goal_steps):
                action, game_action = self.generate_action(snake)
                done, _, snake, _, _ = game.step(game_action)

                if done:
                    training_data.append([self.add_action_to_observation(previous_observation, action), 0])
                    break

                else:
                    training_data.append([self.add_action_to_observation(previous_observation, action), 1])
                    previous_observation = self.generate_observation(snake)

        print(len(training_data))
        return training_data

    # Generate an action for the snake
    def generate_action(self, snake):
        action = random.randint(0,2) - 1
        return action, self.get_game_action(snake, action)

    # Get the action for a game
    def get_game_action(self, snake, action):
        snake_direction = self.get_snake_direction_vector(snake)
        new_direction = snake_direction

        if action == -1:
            new_direction = self.turn_left(snake_direction)

        elif action == 1:
            new_direction = self.turn_right(snake_direction)

        for pair in self.vector_actions:
            if pair[0] == new_direction.tolist():
                game_action = pair[1]

        return game_action

    # Generating an observation
    def generate_observation(self, snake):
        snake_direction = self.get_snake_direction_vector(snake)
        barrier_left = self.direction_blocked(snake, self.turn_left(snake_direction))
        barrier_right = self.direction_blocked(snake, self.turn_right(snake_direction))
        barrier_front = self.direction_blocked(snake, snake_direction)
        return np.array([int(barrier_left), int(barrier_front), int(barrier_right)])

    # add an action to an observation
    def add_action_to_observation(self, observation, action):
        return np.append([action], observation)

    # Get the direction of the action of the snake
    def get_snake_direction_vector(self, snake):
        return np.array(snake[0]) - np.array(snake[1])

    # Boolean if the direction of the snake is blocked
    def direction_blocked(self, snake, direction):
        point = np.array(snake[0]) + np.array(direction)
        return point.tolist() in snake[:-1] or point[0] == 0 or point[1] == 0 or point[0] == config.screen_heigth+1 or point[1] == config.screen_width+1

    # Turn the snake to the right of its direction
    def turn_right(self, vector):
        return np.array([vector[1], -vector[0]])

    # Turn the snake to the left of its direction
    def turn_left(self, vector):
        return np.array([-vector[1], vector[0]])

    # Model the neural network with tensorflowb
    def model(self):
        network = input_data(shape=[None, 4, 1], name= 'input')
        network = fully_connected(network, 1, activation= 'linear')
        network = regression(network, optimizer= 'adam', learning_rate= self.learning_rate, loss= 'mean_square', name= 'target')
        model = tflearn.DNN(network, tensorboard_dir= 'log')
        return model

    # Train the neural network model
    def train_model(self, training_data, model):
        X = np.array([i[0] for i in training_data]).reshape(-1,4,1)
        y = np.array([i[1] for i in training_data]).reshape(-1,1)
        model.fit(X, y, n_epoch = 1, shuffle= True, run_id= self.filename)
        model.save(self.filename)
        return model

    # Visualize the game
    def visuluaize_game(self, model):
        game = SnakeGame.SnakeGame(gui= True)
        _, _, snake, _, _ = game.start()
        previous_observation = self.generate_observation(snake)

        for _ in range(self.goal_steps):
            predictions = []

            for action in range(-1,2):
                predictions.append(model.predict(self.add_action_to_observation(previous_observation, action).reshape(-1,4,1)))

            action = np.argmax(np.array(predictions))
            game_action = self.get_game_action(snake, action -1)
            done, _, snake, _, _ = game.step(game_action)

            if done:
                break
            else:
                previous_observation = self.generate_observation(snake)

    # Test the neural network model
    def test_model(self, model):
        steps_arr = []

        for _ in range(self.test_games):
            steps = 0
            game_memory = []
            game = SnakeGame.SnakeGame()
            _, _, snake, _, _ = game.start()
            previous_observation = self.generate_observation(snake)

            for _ in range(self.goal_steps):
                predictions = []
                for action in range(-1,2):
                    predictions.append(model.predict(self.add_action_to_observation(previous_observation,action).reshape(-1,4,1)))
                    action = np.argmax(np.array(predictions))
                    game_action = self.get_game_action(snake, action -1)
                    done, _, snake, _, _ = game.step(game_action)
                    game_memory.append([previous_observation, action])

                    if done:
                        break

                    else:
                        previous_observation = self.generate_observation(snake)
                        steps += 1
                steps_arr.append(steps)

            print('Average steps:',mean(steps_arr))
            print(Counter(steps_arr))

    # Train model
    def train(self):
        training_data = self.initial_population()
        nn_model = self.model()
        nn_model = self.train_model(training_data, nn_model)
        self.test_model(nn_model)

    # Visualize the game
    def visualize(self):
        nn_model = self.model()
        nn_model.load(self.filename)
        self.visuluaize_game(nn_model)

    # Test the model
    def test(self):
        nn_model = self.model()
        nn_model.load(self.filename)
        self.test_model(nn_model)

# Start the process from terminal
if __name__ == "__main__":
    #SnakeNN().train()
    #SnakeNN().visualize()
    SnakeNN().test()


# Nhóm 30
# Ngô Duy Tân-N19DCCN165
# Trương Hoàng Sang- N19DCCN157
import numpy as np
import matplotlib.pyplot as plt
import math as mt
from scipy.spatial import distance_matrix

def calculo_distancia(m_distancia,camino,n_ciudades):
    c_dis = 0
    for c1 in range(0, n_ciudades):
        suma_actual = m_distancia[camino[c1], camino[c1 - 1]]
        c_dis = suma_actual + c_dis
    return c_dis

def enc_vecino(camino,p1,p2):
    vecino = np.array(camino)
    vecino[p1] = camino[p2]
    vecino[p2] = camino[p1]
    return vecino

ciudades = np.array([[1,7],[7,5],[7,1],[3,1],[4,3]])
n_ciudades = len(ciudades)
m_distancia = distance_matrix(ciudades,ciudades)
camino_0 = np.random.choice(np.arange(n_ciudades),n_ciudades,False)

# Inicializacion de Temperatura
lk = 10
tk = 0.1
rmin = 0.95
ra = 0
b = 1000

camino = np.array(camino_0)
camino_correcto = np.array(camino_0)
distancia_actual = calculo_distancia(m_distancia, camino, n_ciudades)
mut_anterior = np.random.choice(np.arange(n_ciudades), 1)
camino = np.array(camino_0)
while ra < rmin:
    # for c2 in range (0,3):
    int_aceptado = 0
    intento = 0
    for l in range(0, lk):
        # Selección de posición a mutar
        posicion_cambio = np.random.choice(np.arange(5), 1)
        # Evitar mutar el mísmo índice 2 veces seguidas
        if posicion_cambio == mut_anterior:
            l = l - 1
            continue
        else:
            mut_anterior = np.array(posicion_cambio)
            # Mutar
            if posicion_cambio == 5:
                nuevo_camino = np.array(enc_vecino(camino, int(posicion_cambio) - 2, int(posicion_cambio) - 1))
            else:
                nuevo_camino = np.array(enc_vecino(camino, posicion_cambio, int(posicion_cambio) - 1))
            # Cálculo nueva distancia
            nueva_distancia = calculo_distancia(m_distancia, nuevo_camino, n_ciudades)
            # Comparación distancias
            if nueva_distancia < distancia_actual:
                camino = np.array(nuevo_camino)
                distancia_actual = nueva_distancia
                int_aceptado = int_aceptado + 1
                camino_correcto = np.array(nuevo_camino)
            else:
                prob = mt.exp(-(nueva_distancia - distancia_actual) / tk)
                dss = np.random.random()
                if dss < prob:
                    camino = np.array(nuevo_camino)
                    distancia_actual = nueva_distancia
                    int_aceptado = int_aceptado + 1
            intento = intento + 1
        l = l + 1
    tk = b * tk
    ra = int_aceptado / intento

# Enfriamiento
cam_tom = []
cont = [0]
mut_anterior = np.random.choice(np.arange(5), 1)
distancia_actual = calculo_distancia(m_distancia,camino,n_ciudades)
#while tk > .00000000001:
for c2 in range (0,100):
    for l in range(0,lk):
        posicion_cambio = np.random.choice(np.arange(5), 1)
        if posicion_cambio == mut_anterior:
            l = l - 1
            continue
        else:
            mut_anterior = np.array(posicion_cambio)
            if posicion_cambio == 5:
                nuevo_camino = np.array(enc_vecino(camino, int(posicion_cambio) - 2, int(posicion_cambio) - 1))
            else:
                nuevo_camino = np.array(enc_vecino(camino, posicion_cambio, int(posicion_cambio) - 1))
                nueva_distancia = calculo_distancia(m_distancia, nuevo_camino, n_ciudades)
            if nueva_distancia < distancia_actual:
                camino = np.array(nuevo_camino)
                distancia_actual = nueva_distancia
                cam_tom.append(camino)
            else:
                if tk > .0000001:
                    prob = mt.exp(-(nueva_distancia - distancia_actual) / tk)
                    dss = np.random.random()
                    if dss < prob:
                        camino = np.array(nuevo_camino)
                        distancia_actual = nueva_distancia
                        tk = 1
        l = l + 1
    tk = tk * np.random.randint(.2, 1)

print('-------------------------------------')
print('Minimal distance found:')
print(distancia_actual)
print('-------------------------------------')
print('Order of the cities in the trajectory:')
print(camino)

# Cálculo promedio/desviación estandar
caminos_tomados = np.array(cam_tom)
c3 = 0
dd = []
for e1 in caminos_tomados:
    d = calculo_distancia(m_distancia, e1, n_ciudades)
    dd.append(d)

y = np.array(dd)
x = np.linspace(1, len(y), len(y))

vec_camino = []
for e2 in camino:
    vec_camino.append(ciudades[e2])
cc = np.array(vec_camino)
print('-------------------------------------')
print('Coordinates of the cities in the trajectory:')
print(cc)


import cv2 as cv #computer vision

img = cv.imread('C:\\Users\\buglow\\Desktop\\shapes.png')   #bringing the photo by path

                              # ====> Segmantation | pre-processing <====

imgGrey = cv.cvtColor(img, cv.COLOR_BGR2GRAY)                               #convert to gray scale
_, thrash = cv.threshold(imgGrey, 240, 255, cv.THRESH_BINARY)               #cnvt ->b&w (src, sabet, abyad, el method(if > sabet -->put abyad)
contours, _ = cv.findContours(thrash, cv.RETR_TREE, cv.CHAIN_APPROX_NONE)
                                                                            # Contours can be explained simply as a
                                                                            # curve joining all the continuous points
                                                                            # (along the boundary), having same color or intensity.

#view for clarification
cv.imshow("Original", img) #give window with the title shapes contain the image
cv.waitKey(0)              #wait any key
cv.destroyAllWindows()     #to destroy all after the key :B

cv.imshow("GRAY Scale", imgGrey) #give window with the title shapes contain the image
cv.waitKey(0)                    #wait any key
cv.destroyAllWindows()           #to destroy all after the key :B

for contour in contours:
                                # ====> Do the Job like a boss  <====

    epsilon = 0.01*cv.arcLength(contour, True)               #calc el epsilon value by kikking el coeff. f el arc-length of countour
    approx = cv.approxPolyDP(contour, epsilon, True)         #extracting points in a 2d array

    cv.drawContours(img, [approx], -1, (0, 150, 255), 6)     #draw on src contours with approx points, all, color, thickness

    x = approx.ravel()[0] -7 #ravel return flated array  wierdoo :P
    y = approx.ravel()[1] -7
#print(approx, len(approx))

                        #  ====> Take decision According to the number of points <=====

    if len(approx) == 3:
        cv.putText(img, "Triangle", (x, y), cv.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0))
    elif len(approx) == 4:
        x1, y1, w, h = cv.boundingRect(approx) #Xtract (first point) and (width and height)
        ratio = float(w)/h
        if 0.8 <= ratio <= 1.3:
            cv.putText(img, "Square", (x, y), cv.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0)) #src, text, set(point), fontType, fontSize, RGBcolor
        else:
            cv.putText(img, "Rectangle", (x, y), cv.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0))
    elif len(approx) == 5:
        cv.putText(img, "Pentagon", (x, y), cv.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0))
    elif len(approx) == 6:
        cv.putText(img, "Hexagon", (x, y), cv.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0))
    elif len(approx) == 10:
        cv.putText(img, "Star", (x, y), cv.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0))
    else:
        cv.putText(img, "Circle", (x, y), cv.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0))

cv.imshow("Final Result", img) #give window with the title shapes contain the image
cv.waitKey(0)                  #wait any key
cv.destroyAllWindows()         #to destroy all after the key :B
#----------------------------|
#                            |
# Done on 7:40 10/3/2020     |
# Paulo's Team:              |
#   Paulo Emil 2             |
#   Gamal Ashraf Gamal 2     |
#   Akram Diaa Abaas 2       |
#   Remon Wagde Marzouk 3    |
#----------------------------|


from flask import Flask, render_template, request
import requests
import json
import os

from dotenv import load_dotenv
load_dotenv()

TENOR_API_KEY = os.getenv("TENOR_API_KEY")

app = Flask(__name__)

# route the function
@app.route('/')
def index():
    """Return homepage."""
    query = request.args.get("search_gif")
    payload = {
        "q": query,
        "key": "3OGJ9M5CUDUK",
        "limit" : "10"
    }
    data = None
#     # sets the apikey and limit

# request GET api, receive api, returns searched gifs
    r = requests.get("https://api.tenor.com/v1/search", params=payload)
    if r.status_code == 200:
        # load the GIFs using the urls for the smaller GIF sizes
        data = r.json()
        data = data["results"]
    else:
        data = None
    return render_template("index.html", data=data)

    # get the top 10 GIFs for the search term
    # links to terminal
if __name__ == '__main__':
    app.run(debug=True)


import argparse
from collections import OrderedDict
import csv

def process_coordinates(reader):
    """
    Each row is single tab delimited string of three values
    """
    data = OrderedDict()
    total_rows = 0

    for raw_row in reader:
        total_rows += 1

        row = raw_row[0].split("\t")

        chromosome_name = row[0]
        start_position = row[1]
        end_position = row[2]

        key = f"{chromosome_name}{start_position}{end_position}"

        data[key] = [
            chromosome_name,
            start_position,
            end_position,
        ]

    return data, total_rows

def process_differential_coordinates(reader):
    """
    Each row is a list of 7 values
    """
    data = OrderedDict()
    total_rows = 0

    for row in reader:

        total_rows += 1

        chromosome_name = row[1]
        start_position = row[2]
        end_position = row[3]
        strand = row[4]
        padj = row[5]
        log2FoldChange = row[6]

        key = f"{chromosome_name}{start_position}{end_position}{strand}{padj}{log2FoldChange}"

        data[key] = [
            chromosome_name,
            start_position,
            end_position,
            strand,
            padj,
            log2FoldChange,
        ]

    return data, total_rows

def remove_duplicates(input_csv_path):
    """
    Opens a CSV file and returns back a dictionary of unique rows
    Columns should be: ID, chromosome_name, start_position, end_position, strand, padj, log2FoldChange
    """
    data = OrderedDict()
    total_rows = 0
    csv_type = ""

    # Open CSV, iterate over each row, only insert unique rows into dictionary
    with open(input_csv_path, "r") as csv_file:
        reader = csv.reader(csv_file)

        for row in reader:
            # Check if CSV is Coordinates or Differential Coordinates
            if row[0] == "\ufeff'ID'":
                csv_type = "diff"
                break
            else:
                # Coordinates CSV
                csv_type = "coor"
                break

        if csv_type == "diff":
            data, total_rows = process_differential_coordinates(reader)
        elif csv_type == "coor":
            data, total_rows = process_coordinates(reader)
        else:
            print(
                "Something bad happened! Columns don't match coordinate or differential coordinate format"
            )

    return data, total_rows, csv_type

def write_output_csv(output_csv_path, data, csv_type):
    """
    Accepts a dictionary of rows and writes a new CSV
    Columns should be: ID, chromosome_name, start_position, end_position, strand, padj, log2FoldChange
    """
    count = 1

    # Open a second output file and write only unique rows
    with open(output_csv_path, "w", newline="") as csv_file:
        writer = csv.writer(csv_file, delimiter="\t")

        # First row, header
        if csv_type == "diff":

            header = [
                "'ID'",
                "'chromosome_name'",
                "'start_position'",
                "'end_position'",
                "'strand'",
                "'padj'",
                "'log2FoldChange'",
            ]
            writer.writerow(header)

            # Prepend a unique id and write to csv
            for key, value in data.items():
                writer.writerow([count] + value)

                count += 1
        else:
            # Prepend a unique id and write to csv
            for key, value in data.items():
                writer.writerow(value)

                count += 1

    return count - 1

# python3 remove_duplicates.py input.csv output.csv
if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Remove duplicate rows from CSV file")
    parser.add_argument("input_csv_path", type=str, help="Path to source CSV")
    parser.add_argument("output_csv_path", type=str, help="Path to output CSV")
    args = parser.parse_args()

    unique_rows, total_rows, csv_type = remove_duplicates(args.input_csv_path)
    rows_written = write_output_csv(args.output_csv_path, unique_rows, csv_type)
    duplicates = total_rows - rows_written

    print(
        f"Looked at {total_rows} rows. Found {duplicates} duplicates. Found {rows_written} unique rows."
    )


import boto3
from src.config import YandexS3Config

def get_query_search(query_search: str = None):
    if query_search:
        return query_search.strip()

session = boto3.session.Session()
s3 = session.client(
    service_name='s3',
    endpoint_url='https://storage.yandexcloud.net',
    aws_access_key_id=YandexS3Config.AWS_ACCESS_KEY_ID,
    aws_secret_access_key=YandexS3Config.AWS_SECRET_ACCESS_KEY,
    region_name='ru-central1'
)


#!/usr/bin/env python
import sys
import setuptools
from setuptools import setup, find_packages

install_requires = [
    'six',
    'Django>=1.11',
    'psycopg2',
]

extras_require = {
    'dev': [
      'pytest>=2.9',
      'pytest-cov',
      'pytest-django',
      'coveralls',
      'flake8',
    ],
}

if int(setuptools.__version__.split(".", 1)[0]) < 18:
    assert "bdist_wheel" not in sys.argv, "setuptools 18 required for wheels."
    # For legacy setuptools + sdist.
    if sys.version_info[0:2] <= (2, 7):
        install_requires.append("backports.csv")
else:
    extras_require[":python_version<='2.7'"] = ["backports.csv"]

setup(name="opencivicdata",
      version='2.1.1',
      author="James Turk",
      author_email='james@openstates.org',
      license="BSD",
      description="python opencivicdata library",
      long_description="",
      url="",
      packages=find_packages(),
      include_package_data=True,
      install_requires=install_requires,
      extras_require=extras_require,
      platforms=["any"],
      classifiers=["Development Status :: 5 - Production/Stable",
                   "Intended Audience :: Developers",
                   "License :: OSI Approved :: BSD License",
                   "Natural Language :: English",
                   "Operating System :: OS Independent",
                   "Programming Language :: Python :: 2.7",
                   "Programming Language :: Python :: 3.5",
                   "Programming Language :: Python :: 3.6",
                   ],
      )


"""
seed the database with admins
"""

import sqlite3
conn = sqlite3.connect('hellrpgdb.db')
c = conn.cursor()

c.execute(
	"""CREATE TABLE user (
	id INTEGER PRIMARY KEY,
	username TEXT,
	password TEXT,
	last_played INTEGER
	);"""
)

conn.commit()
conn.close()
print("Your database was created")

# db = 'hellrpgdb.db'

# user = ["Andrew", "Super User", "Cheese", "Andrew.Kolansky@gmail.com" ]

# conn = sqlite3.connect(hellrpgdb.db)
# c = conn.cursor()

# print("Replacing old data")
# c.execute("DELTE FROM user")

# for user in USERS:
# 	c.execute("""INSERT INTO user("user_name",
# 	 "user_type", "password", "email") VALUES(?,?,?,?)
# 		""",(user[0],user[1], user[2], user[3], user[4]))

# conn.commit()
# c.close()

# print("Looks like we're all good")


from django.urls import path
from . import views

app_name = 'learn'
urlpatterns = [
    path('dash', views.dash, name='dash'),
    path('my-account/', views.my_account, name='my-account'),
    path('select-lesson/<int:level>', views.select_lesson, name='select-lesson'),
    path('lesson/<int:level>/<int:id>', views.lesson, name='lesson'),
    path('add-remove-homework/', views.add_remove_homework, name='add-remove-homework'),
    path('homework/', views.homework, name='homework'),
    path("check-username-exists/", views.ajax_check_username_exists, name='check-username-exists'),
    path('adjust-progress/', views.adjust_progress, name='adjust-progress'),
    path('no-cheating/', views.no_cheating, name='no-cheating'),

]


import time
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
from sklearn.metrics import roc_curve, auc
import matplotlib.pyplot as plt
from itertools import cycle

def get_labels(df, label):
    y_train = df[label]
    return y_train

def get_test_dataset():
    df_test = pd.read_csv("test.csv")
    df_test_labels = pd.read_csv("test_labels.csv")

    #ignoring the comments that have a label of -1;
    #per kaggle: value of -1 indicates it was not used for scoring;
    ignore_idx = df_test_labels.index[df_test_labels["toxic"] == -1].tolist()

    df_test_labels = df_test_labels[df_test_labels["toxic"] != -1]
    df_test = df_test.drop(ignore_idx)
    return df_test, df_test_labels

def main():
    labels = ["toxic", "severe_toxic", "obscene", "threat",
              "insult", "identity_hate"]
    accuracy = []
    df_train = pd.read_csv("train.csv")
    df_test, df_test_labels = get_test_dataset()

    X_train = df_train["comment_text"]
    X_test = df_test["comment_text"]

    vectorizer = TfidfVectorizer()

    train_vectors = vectorizer.fit_transform(X_train)
    test_vectors = vectorizer.transform(X_test)

    fpr = dict()
    tpr = dict()
    roc_auc = dict()

    for i, label in enumerate(labels):
        y_train = get_labels(df_train, label)

        clf = RandomForestClassifier(n_estimators=100)

        clf.fit(train_vectors,np.array(y_train))
        predict = clf.predict(test_vectors)

        y_test = get_labels(df_test_labels,label)
        fpr[i], tpr[i], _ = roc_curve(y_test,predict)
        roc_auc[i] = auc(fpr[i], tpr[i])

        acc_score = round(accuracy_score(y_test,predict),
                          2)
        accuracy.append(acc_score)

        print("Accuracy score on {}: {}"
              .format(label, acc_score))

    fpr["micro"], tpr["micro"], _ = roc_curve(y_test, predict)
    roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])

    all_fpr = np.unique(np.concatenate([fpr[i] for i in range(len(labels))]))

    # Then interpolate all ROC curves at this points
    mean_tpr = np.zeros_like(all_fpr)
    for i in range(len(labels)):
        mean_tpr += np.interp(all_fpr, fpr[i], tpr[i])

    # Finally average it and compute AUC
    mean_tpr /= len(labels)

    fpr["macro"] = all_fpr
    tpr["macro"] = mean_tpr
    roc_auc["macro"] = auc(fpr["macro"], tpr["macro"])

    plt.figure()
    lw = 2
    plt.plot(fpr["micro"], tpr["micro"],
             label='micro-average ROC curve (area = {0:0.2f})'
                   ''.format(roc_auc["micro"]),
             color='deeppink', linestyle=':', linewidth=4)

    plt.plot(fpr["macro"], tpr["macro"],
             label='macro-average ROC curve (area = {0:0.2f})'
                   ''.format(roc_auc["macro"]),
             color='navy', linestyle=':', linewidth=4)

    colors = cycle(['aqua', 'darkorange', 'cornflowerblue'])
    for i, color in zip(range(len(labels)), colors):
        plt.plot(fpr[i], tpr[i], color=color, lw=lw,
                 label='ROC curve of class {0} (area = {1:0.2f})'
                 ''.format(i, roc_auc[i]))

    plt.plot([0, 1], [0, 1], 'k--', lw=lw)
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')

    plt.legend(loc="lower right")
    plt.show()

##    plt.plot(labels, accuracy)
##    plt.xlabel("Toxicity")
##    plt.ylabel("Accuracy")
##    plt.show()

start = time.time()
main()
end = time.time()
print("Time: {}".format(end-start))