"""
@author:  Yuhao Cheng
@contact: yuhao.cheng[at]outlook.com
"""
#!!!!! ignore the warning messages
import warnings
warnings.filterwarnings('ignore')
import os
import pickle
import math
import torch
import time
import numpy as np
from PIL import Image
from collections import OrderedDict
import torchvision.transforms as T
import torchvision.transforms.functional as tf
from torch.utils.data import DataLoader

import logging
logger = logging.getLogger(__name__)

from pyanomaly.core.utils import AverageMeter, flow_batch_estimate, tensorboard_vis_images, vis_optical_flow, make_info_message, ParamSet
from pyanomaly.datatools.evaluate.utils import psnr_error

from pyanomaly.datatools.evaluate.utils import (
    simple_diff,
    find_max_patch,
    amc_score,
    calc_w
    )

from ..abstract.base_engine import BaseTrainer, BaseInference, BaseService

from ..engine_registry import ENGINE_REGISTRY

__all__ = ['MATrainer', 'AMCInference']

@ENGINE_REGISTRY.register()
class MATrainer(BaseTrainer):
    """
    G
    D_frame
    D_pattern
    AE_act
    AE_obj
    PatternNet
    """
    NAME = ["MA.TRAIN"]
    def custom_setup(self):
        # create loss meters
        self.loss_meter_G = AverageMeter(name='Loss_G')
        self.loss_meter_D = AverageMeter(name='Loss_D')

        self.optical = ParamSet(name='optical', size=self.config.DATASET.optical_size, output_format=self.config.DATASET.optical_format)
        # import ipdb; ipdb.set_trace()

    def train(self,current_step):
        # Pytorch [N, C, D, H, W]
        # initialize
        start = time.time()
        self.set_requires_grad(self.F, False)
        self.set_requires_grad(self.D, True)
        self.set_requires_grad(self.G, True)
        self.G.train()
        self.D.train()
        self.F.eval()
        writer = self.kwargs['writer_dict']['writer']
        global_steps = self.kwargs['writer_dict']['global_steps_{}'.format(self.kwargs['model_type'])]

        # get the data
        data, anno, meta = next(self._train_loader_iter)
        self.data_time.update(time.time() - start)

        # base on the D to get each frame
        # in this method, D = 2 and not change
        input_data = data[:, :, 0, :, :].cuda() # input(1-st) frame
        target = data[:, :, 1,:, :].cuda() # target(2-nd) frame

        # True Process =================Start===================
        #---------update optim_G ---------
        self.set_requires_grad(self.D, False)
        output_flow_G,  output_frame_G = self.G(input_data)
        gt_flow_esti_tensor = torch.cat([input_data, target], 1)
        flow_gt_vis, flow_gt  = flow_batch_estimate(self.F, gt_flow_esti_tensor, self.normalize.param['train'],
                                                    optical_size=self.config.DATASET.optical_size, output_format=self.config.DATASET.optical_format)
        fake_g = self.D(torch.cat([target, output_flow_G], dim=1))

        loss_g_adv = self.GANLoss(fake_g, True)
        loss_op = self.OpticalflowSqrtLoss(output_flow_G, flow_gt)
        loss_int = self.IntentsityLoss(output_frame_G, target)
        loss_gd = self.GradientLoss(output_frame_G, target)
        loss_g_all = self.loss_lamada['IntentsityLoss'] * loss_int + self.loss_lamada['GradientLoss'] * loss_gd + self.loss_lamada['OpticalflowSqrtLoss'] * loss_op + self.loss_lamada['GANLoss'] * loss_g_adv

        self.optimizer_G.zero_grad()
        loss_g_all.backward()
        self.optimizer_G.step()
        self.loss_meter_G.update(loss_g_all.detach())

        if self.config.TRAIN.adversarial.scheduler.use:
            self.optimizer_G_scheduler.step()

        #---------update optim_D ---------------
        self.set_requires_grad(self.D, True)
        self.optimizer_D.zero_grad()
        # import ipdb; ipdb.set_trace()
        real_d = self.D(torch.cat([target, flow_gt],dim=1))
        fake_d = self.D(torch.cat([target, output_flow_G.detach()], dim=1))
        loss_d_1 = self.GANLoss(real_d, True)
        loss_d_2 = self.GANLoss(fake_d, False)
        loss_d = (loss_d_1  + loss_d_2) * 0.5
        loss_d.backward()
        self.optimizer_D.step()
        if self.config.TRAIN.adversarial.scheduler.use:
            self.optimizer_D_scheduler.step()
        self.loss_meter_D.update(loss_d.detach())
        # ======================End==================

        self.batch_time.update(time.time() - start)

        if (current_step % self.steps.param['log'] == 0):
            msg = make_info_message(current_step, self.steps.param['max'], self.kwargs['model_type'], self.batch_time,
                                    self.config.TRAIN.batch_size, self.data_time, [self.loss_meter_G, self.loss_meter_D])
            logger.info(msg)

        writer.add_scalar('Train_loss_G', self.loss_meter_G.val, global_steps)
        writer.add_scalar('Train_loss_D', self.loss_meter_D.val, global_steps)

        if (current_step % self.steps.param['vis'] == 0):
            temp = vis_optical_flow(output_flow_G.detach(), output_format=self.config.DATASET.optical_format, output_size=(output_flow_G.shape[-2], output_flow_G.shape[-1]),
                                    normalize=self.normalize.param['train'])
            vis_objects = OrderedDict({
                'train_target_flow': flow_gt_vis.detach(),
                'train_output_flow_G': temp,
                'train_target_frame': target.detach(),
                'train_output_frame_G': output_frame_G.detach(),
            })
            tensorboard_vis_images(vis_objects, writer, global_steps, self.normalize.param['train'])
        global_steps += 1

        # reset start
        start = time.time()

        # self.saved_model = {'G':self.G, 'D':self.D}
        self.saved_model['G'] = self.G
        self.saved_model['D'] = self.D
        # self.saved_optimizer = {'optim_G': self.optimizer_G, 'optim_D': self.optimizer_D}
        self.saved_optimizer['optimizer_G'] = self.optimizer_G
        self.saved_optimizer['optimizer_D'] = self.optimizer_D
        # self.saved_loss = {'loss_G':self.loss_meter_G.val, 'loss_D':self.loss_meter_D.val}
        self.saved_loss['loss_G'] = self.loss_meter_G.val
        self.saved_loss['loss_D'] = self.loss_meter_D.val
        self.kwargs['writer_dict']['global_steps_{}'.format(self.kwargs['model_type'])] = global_steps

@ENGINE_REGISTRY.register()
class AMCInference(BaseInference):
    NAME = ["AMC.INFERENCE"]

    def inference(self):
        for h in self._hooks:
            h.inference()

@ENGINE_REGISTRY.register()
class AMCService(BaseService):
    def custom_setup(self):
        self.optical_format = self.config.DATASET.optical_format
        self.optical_szie = self.engine.config.DATASET.optical_size
        self.wf = 1.0
        self.wi = 1.0
        self.threshold = 0.0 # the threshold to judge whether the frame is the anomaly
        pass

    def get_clip_by_stride(self, video, stride=2):
        """Get the clip list by the stride
        """
        clip_list = []
        return clip_list

    def execute(self, data):
        output_dict = OrderedDict()
        # data.shape = [N,C,D,H,W], data is a whole vide, D=the length of the video
        clip_list = self.get_clip_by_stride(data) # the length of the length is the length of the video
        scores = np.empty(shape=(len(clip_list), ), dtype=np.float32)

        for index, clip in enumerate(clip_list):
            first_frame = clip[:, :, 0, :, :].cuda()
            second_frame = clip[:, :, 1, :, :].cuda()

            generated_flow, generated_frame = self.G(first_frame)
            gtFlowEstim = torch.cat([first_frame, second_frame], 1)
            _, gtFlow = flow_batch_estimate(self.F, gtFlowEstim, self.normalize.param['val'], output_format=self.optical_format, optical_size=self.optical_size)

            score, _, _ = amc_score(second_frame, generated_frame, gtFlow, generated_flow, self.wf, self.wi)
            score = score.tolist()
            scores[index] = score

        result_mask = scores.gt(self.threshold)
        output_dict['result_dict'] = result_mask

        return output_dict


"""
    lstm 模型训练。
    随机 shuffle + mutil-sample-dropout。
    收敛速度显著提升。
"""
import math
import random
import logging

from tqdm import tqdm

import pandas as pd
import numpy as np

import torch
from torch import nn
from torch.nn import utils, init
from torch.utils import data
import torch.optim as optim
import torch.nn.utils.rnn as rnn
import torch.nn.functional as F
from torch.optim.lr_scheduler import CosineAnnealingLR, StepLR
from torch.optim.optimizer import Optimizer, required

from sklearn.model_selection import StratifiedKFold

from utils import *

import os
import warnings
warnings.filterwarnings('ignore')
os.environ['CUDA_VISIBLE_DEVICES'] = '2'

# 自定义训练集
class SeqDataSet(data.Dataset):
    def __init__(self, multi_seqs, feas, labels, num_seqs, max_len, tag):
        self.multi_seqs = multi_seqs
        self.feas = feas
        self.labels = labels
        self.tag = tag

        self.num_seqs = num_seqs
        self.max_len = max_len

    def __getitem__(self, index):
        multi_seq = self.multi_seqs[index]
        fea = self.feas[index]
        label = self.labels[index]
        return multi_seq, fea, label

    def __len__(self):
        return len(self.labels)

    def collate_fn(self, batch_data):
        batch_data.sort(key=lambda data: data[0][0].shape[0], reverse=True)
        multi_seqs = [[] for i in range(self.num_seqs)]
        feas = []
        labels = []
        lens = []

        for data in batch_data:
            multi_seq = data[0]
            lens.append(min(multi_seq[0].shape[0], self.max_len))
            index = np.arange(multi_seq[0].shape[0])
            random.shuffle(index)
            index = index[:self.max_len]
            for i in range(self.num_seqs):
                multi_seqs[i].append(torch.LongTensor(multi_seq[i][index]))
            feas.append(data[1])
            labels.append(data[2])

        # multi_seqs [num_seqs, (batch_size, len)]
        # feas [batch_size, fea_size]
        # lens [batch_size]
        # labels [batch_size]
        multi_seqs = torch.stack([rnn.pad_sequence(seqs, batch_first=True, padding_value=0) for seqs in multi_seqs])
        # multi_seqs [num_seqs, batch_size, max_len]
        multi_seqs = multi_seqs.permute(1, 0, 2)
        # multi_seqs [batch_size, num_seqs, max_len]

        feas = torch.FloatTensor(feas)
        labels = torch.LongTensor(labels)
        lens = torch.IntTensor(lens)

        return multi_seqs, feas, lens, labels

# embedding 层，w2v向量转换和拼接
class embedNet(nn.Module):
    def __init__(self, embeddings):
        super(embedNet, self).__init__()
        embed_layers = [nn.Embedding.from_pretrained(embedding, padding_idx=0) for embedding in embeddings]
        self.num_seqs = len(embed_layers)
        self.embed_layers = nn.ModuleList(embed_layers)

    def forward(self, users_seqs):
        #users_seqs  [batch_size, num_seqs, max_len]
        users_seqs = users_seqs.permute(1, 0, 2)
        embeddings = [self.embed_layers[i](users_seqs[i]) for i in range(self.num_seqs)]
        embeddings = torch.cat(embeddings, dim=2)
        #embeddings [batch_size, max_len, embed_size]
        return embeddings

# lstm 层
class lstmNet(nn.Module):
    def __init__(self, input_size, hidden_size, num_layers, drop_out):
        super(lstmNet, self).__init__()
        self.LSTMLayer = nn.LSTM(input_size=input_size,
                                 hidden_size=hidden_size,
                                 num_layers=num_layers,
                                 dropout=drop_out,
                                 bidirectional=True,
                                 batch_first=True)
        self.init_params()

    def forward(self, embeddings, lens):
        # embeddings [batch_size, max_len, embed_size]
        x = rnn.pack_padded_sequence(embeddings, lens, batch_first=True)
        x, (h, c) = self.LSTMLayer(x)
        x, _ = rnn.pad_packed_sequence(x, batch_first=True, padding_value=0.0)
        # x [batch_size, max_len, 2 * hidden_size]
        x = torch.transpose(x, 1, 2)
        # x [batch_size, 2 * hidden_sizen, max_len]
        h = F.max_pool1d(x, x.shape[-1]).squeeze()
        # h [batch_size, 2 * hidden_size]
        return h

    def init_params(self):
        for layer in range(len(self.LSTMLayer.all_weights)):
            init.orthogonal_(self.LSTMLayer.all_weights[layer][0])
            init.orthogonal_(self.LSTMLayer.all_weights[layer][1])
            init.zeros_(self.LSTMLayer.all_weights[layer][2])
            init.zeros_(self.LSTMLayer.all_weights[layer][3])

# 分类全连接层
class classfiyNet(nn.Module):
    def __init__(self, input_size, num_labels, drop_out, num_drop):
        super(classfiyNet, self).__init__()
        self.fc1 = nn.Linear(input_size, input_size // 2)
        self.mutil_dropout1 = nn.ModuleList([
            nn.Dropout(drop_out) for _ in range(num_drop)
        ])
        self.fc2 = nn.Linear(input_size // 2, input_size // 4)
        self.mutil_dropout2 = nn.ModuleList([
            nn.Dropout(drop_out) for _ in range(num_drop)
        ])
        self.fc3 = nn.Linear(input_size // 4, num_labels)

    def forward(self, x):
        h = self.fc1(x)
        h = F.leaky_relu(h, inplace=True)
        if self.training:
            hs = []
            # hs num_drop
            for dropout in self.mutil_dropout1:
                hs.append(F.leaky_relu(self.fc2(dropout(h)), inplace=True))

            ys = []
            # ys num_drop^2
            for dropout in self.mutil_dropout2:
                for h in hs:
                    ys.append(self.fc3(dropout(h)))

            return ys
        else:
            h = self.fc2(h)
            h = F.leaky_relu(h, inplace=True)
            y = self.fc3(h)

            return y

    def init_params(self):
        for name, w in self.fc1.named_parameters():
            if 'weight' in name and w.dim() > 1:
                init.xavier_normal_(w)
            else:
                init.zeros_(w)
        for name, w in self.fc2.named_parameters():
            if 'weight' in name and w.dim() > 1:
                init.xavier_normal_(w)
            else:
                init.zeros_(w)
        for name, w in self.fc3.named_parameters():
            if 'weight' in name and w.dim() > 1:
                init.xavier_normal_(w)
            else:
                init.zeros_(w)

# 完整的网路结构
class Net(nn.Module):
    def __init__(self, embed_size, fea_size, hidden_size, num_layers, drop_out, num_drop):
        super(Net, self).__init__()
        self.lstm_layer = lstmNet(embed_size, hidden_size, num_layers, drop_out)
        self.fc_feas = nn.Sequential(
            nn.Linear(fea_size, hidden_size),
            nn.LeakyReLU(inplace=True),
        )
        self.fc_embed = nn.Sequential(
            nn.Linear(embed_size, hidden_size),
            nn.LeakyReLU(inplace=True),
        )
        self.init_params()
        self.fc_output = classfiyNet(4 * hidden_size, 20, drop_out, num_drop)

    def forward(self, embeddings, feas, lens):
        # lens [batch_size]
        # feas [batch_size, fea_size]
        # embeddings [batch_size, max_len, embed_size]
        lstm_output = self.lstm_layer(embeddings, lens)
        # lstm_output [batch_size, 2 * hidden_size]
        embeddings = embeddings.permute(0, 2, 1)
        embed_output = F.max_pool1d(embeddings, embeddings.shape[-1]).squeeze()
        embed_output = self.fc_embed(embed_output)
        # embed_output [batch_size, hidden_size]
        feas_output = self.fc_feas(feas)
        # feas_output [batch_size, hidden_size]
        h = torch.cat([embed_output, lstm_output, feas_output], dim=1)
        # h [batch_size, 4 * hidden_size]
        y = self.fc_output(h)
        # y  num_drop * [batch_size, 20]
        return y

    def init_params(self):
        for name, w in self.fc_feas.named_parameters():
            if 'weight' in name and w.dim() > 1:
                init.xavier_normal_(w)
            else:
                init.zeros_(w)
        for name, w in self.fc_embed.named_parameters():
            if 'weight' in name and w.dim() > 1:
                init.xavier_normal_(w)
            else:
                init.zeros_(w)

def train(params):
    logger = get_logger('{}.log'.format(params['task']), '{}_logger'.format(params['task']))
    logger.info('start {}'.format(params['task']))

    set_all_seed(params['seed'])

    for key, value in params.items():
        logger.info('{} : {}'.format(key, value))

    logger.info('loading seqs, feas and w2v embeddings ...')
    train_val_data, sub_data, embeddings, embed_size, fea_size = load_data(params['cols'], params['embed_dir'], params['seqs_file'], params['feas_file'])

    logger.info('embed_size : {} | fea_size : {}'.format(embed_size, fea_size))
    batch_size = params['batch_size']
    sub_dataset = SeqDataSet(sub_data['seqs'],
                             sub_data['feas'],
                             sub_data['users'], len(params['cols']), params['max_len'], 'sub')
    sub_loader = data.DataLoader(sub_dataset, batch_size * 10, shuffle=False, collate_fn=sub_dataset.collate_fn, pin_memory=True)

    sub = np.zeros(shape=(sub_data['num'], 20))
    sub = pd.DataFrame(sub, index=sub_data['users'])

    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=params['seed'])

    for i, (train_idx, val_idx) in enumerate(skf.split(train_val_data['feas'], train_val_data['labels'])):
        logger.info('------------------------------------------{} fold------------------------------------------'.format(i))
        train_dataset = SeqDataSet(train_val_data['seqs'][train_idx],
                                   train_val_data['feas'][train_idx],
                                   train_val_data['labels'][train_idx], len(params['cols']), params['max_len'], 'train')
        train_loader = data.DataLoader(train_dataset, batch_size, shuffle=True, collate_fn=train_dataset.collate_fn, pin_memory=True)

        val_dataset = SeqDataSet(train_val_data['seqs'][val_idx],
                                 train_val_data['feas'][val_idx],
                                 train_val_data['labels'][val_idx], len(params['cols']), params['max_len'], 'val')
        val_loader = data.DataLoader(val_dataset, batch_size * 10, shuffle=False, collate_fn=val_dataset.collate_fn, pin_memory=True)

        logger.info('train samples : {} | val samples : {} | sub samples : {}'.format(len(train_idx), len(val_idx), sub_data['num']))
        logger.info('loading net ...')

        embed_net = embedNet(embeddings).cuda()
        net = Net(embed_size, fea_size, params['hidden_size'], params['num_layers'], params['drop_out'], params['num_drop']).cuda()

        #optimizer = Ranger(params=net.parameters(), lr=params['lr'])
        optimizer = optim.AdamW(params=net.parameters(), lr=params['lr'])
        scheduler = StepLR(optimizer, step_size=2, gamma=params['gamma'])
        #scheduler = CosineAnnealingLR(optimizer, T_max=params['num_epochs'])
        loss_func = CrossEntropyLabelSmooth(20, params['label_smooth'])
        #loss_func = nn.CrossEntropyLoss()

        earlystop = EarlyStopping(params['early_stop_round'], logger, params['task'] + str(i))

        for epoch in range(params['num_epochs']):
            train_loss, val_loss = 0.0, 0.0
            train_age_acc, val_age_acc = 0.0, 0.0
            train_gender_acc, val_gender_acc = 0.0, 0.0
            train_acc, val_acc = 0.0, 0.0

            n, m = 0, 0
            lr_now = scheduler.get_last_lr()[0]
            logger.info('--> [Epoch {:02d}/{:02d}] lr = {:.7f}'.format(epoch, params['num_epochs'], lr_now))

            # 训练模型
            net.train()
            for seqs, feas, lens, labels in tqdm(train_loader, desc='[Epoch {:02d}/{:02d}] Train'.format(epoch, params['num_epochs'])):
                seqs = seqs.cuda()
                feas = feas.cuda()
                lens = lens.cuda()
                labels = labels.cuda()

                logits = net(embed_net(seqs), feas, lens)
                loss = sum([loss_func(logit, labels) for logit in logits]) / len(logits)
                loss.backward()
                optimizer.step()
                optimizer.zero_grad()
                logits = sum(logits) / len(logits)

                train_loss += loss.detach() * labels.shape[0]
                train_age_acc += (logits.argmax(dim=1).detach() % 10 == labels % 10).sum()
                train_gender_acc += (logits.argmax(dim=1).detach() // 10 == labels // 10).sum()

                n += lens.shape[0]

            scheduler.step()

            train_loss = (train_loss / n).item()
            train_age_acc = (train_age_acc / n).item()
            train_gender_acc = (train_gender_acc / n).item()
            train_acc = train_age_acc + train_gender_acc

            # 预测验证集
            net.eval()
            with torch.no_grad():
                for seqs, feas, lens, labels in tqdm(val_loader, desc='[Epoch {:02d}/{:02d}]  Val '.format(epoch, params['num_epochs'])):
                    seqs = seqs.cuda()
                    feas = feas.cuda()
                    lens = lens.cuda()
                    labels = labels.cuda()

                    logits = net(embed_net(seqs), feas, lens)
                    loss = loss_func(logits, labels)

                    val_loss += loss.detach() * labels.shape[0]
                    val_age_acc += (logits.argmax(dim=1) % 10 == labels % 10).sum()
                    val_gender_acc += (logits.argmax(dim=1).detach() // 10 == labels // 10).sum()

                    m += lens.shape[0]

                val_loss = (val_loss / m).item()
                val_age_acc = (val_age_acc / m).item()
                val_gender_acc = (val_gender_acc / m).item()
                val_acc = val_age_acc + val_gender_acc

            logger.info('train_loss {:.5f} | train_gender_acc {:.5f} | train_age_acc {:.5f} | train_acc {:.5f} | val_loss {:.5f} | val_gender_acc {:.5f} | val_age_acc {:.5f} | val_acc {:.5f}'
                        .format(train_loss, train_gender_acc, train_age_acc, train_acc, val_loss, val_gender_acc, val_age_acc, val_acc))

            # 早停
            earlystop(val_loss, val_acc, net)
            if earlystop.early_stop:
                break

        break
        net.load_state_dict(torch.load('{}_checkpoint.pt'.format(params['task']+str(i))))
        logger.info('predicting sub ...')
        net.eval()
        with torch.no_grad():
            for it in range(10):
                probs = []
                users = []
                for seqs, feas, lens, ids in tqdm(sub_loader, desc='predict_{}'.format(it)):
                    seqs = seqs.cuda()
                    feas = feas.cuda()
                    lens = lens.cuda()

                    logits = net(embed_net(seqs), feas, lens)
                    logits = F.softmax(logits, dim=1)

                    probs.append(logits)
                    users.append(ids)

                probs = torch.cat(probs).cpu().numpy()
                users = torch.cat(users).numpy()
                sub += pd.DataFrame(probs, users)
            sub = sub / 10

    return sub

if __name__ == '__main__':

    params = {
        'seed' : 2020,
        'task' : 'lstm1',
        'embed_dir' : './data/w2v',
        'seqs_file' : './data/seqs.pkl',
        'feas_file' : './data/feas.pkl',
        'cols' : ['time', 'click', 'creative', 'ad', 'product', 'ader', 'industry', 'category'],
        'batch_size' : 256,
        'max_len' : 256,
        'hidden_size' : 256,
        'num_layers' : 2,
        'drop_out' : 0.2,
        'num_drop' : 2,
        'label_smooth' : 0.1,
        'num_epochs' : 50,
        'early_stop_round' : 3,
        'lr' : 1e-3,
        'gamma' : 0.75,
    }

    sub = train(params)
    sub_split = pd.DataFrame(np.zeros((sub.shape[0], 12)), index=sub.index)
    sub_split.loc[:,0] = sub.loc[:,:9].sum(axis=1)
    sub_split.loc[:,1] = sub.loc[:,10:].sum(axis=1)
    for i in range(10):
        sub_split.loc[:,i+2] = sub.loc[:,[i,i+10]].sum(axis=1)
    sub_split.to_pickle('./torch_{}_sub.pkl'.format(params['task']))


import numpy as np
from numpy.fft import fft, ifft
import pandas as pd
pd.set_option('mode.chained_assignment', None)
import re
import os
from os.path import join
#import matplotlib.pyplot as plt
from data_processing import process_one_csv_file,preprocess,init_class_to_label_dict
# fea_cols = [
#     'Speed', 'lpf_spd_gps',  # 1
#     'AccX', 'AccY', 'AccZ',  # 4
#     'GraX', 'GraY', 'GraZ',  # 7
#     'MagX', 'MagY', 'MagZ',  # 10
#     'GyrX', 'GyrY', 'GyrZ',  # 13
#     'x_accel', 'y_accel', 'z_accel',  # 只去除重力加速度 #16
#     'lpf_x_accel', 'lpf_y_accel', 'lpf_z_accel',  # 只去除重力加速度 #19
#     'AccHX', 'AccHY', 'AccHZ',  # 去除竖直分量的加速度 #22
#     'lpf_AccHX', 'lpf_AccHY', 'lpf_AccHZ',  # 低通滤波后的水平面加速度 #25
#     'AccHX_abs','AccHY_abs','AccHZ_abs' #地球坐标系下的手机加速度 28
#     ]
fea_cols = [
        'Speed',
        'lpf_spd_gps',  #1
        'AccX',
        'AccY',
        'AccZ',  #4
        'GraX',
        'GraY',
        'GraZ',  #7
        'MagX',
        'MagY',
        'MagZ',  #10
        'GyrX',
        'GyrY',
        'GyrZ',  #13
        'x_accel',
        'y_accel',
        'z_accel',  #只去除重力加速度 #16
        'lpf_x_accel',
        'lpf_y_accel',
        'lpf_z_accel',  #只去除重力加速度lpf #19
        'AccHX',
        'AccHY',
        'AccHZ',  #去除竖直分量的加速度 #22
        'lpf_AccHX',
        'lpf_AccHY',
        'lpf_AccHZ',  #低通滤波后的水平面加速度 #25
        'spd_est',
        'lpf_spd_est', #27  由'lpf_AccHX','lpf_AccHY','lpf_AccHZ'预估的速度
         ]
fea_cols = ['Label',
            'AccX','AccY','AccZ',  # 4
            'GraX','GraY','GraZ',  # 7
            'MagX','MagY','MagZ',  # 10
            'GyrX','GyrY','GyrZ',  # 13

            ]
class_lst = ['bus', 'subway', 'others','CHSR','elevetor']
gb_cols = ['Time']
class_cols = ['Trans']
from sklearn import preprocessing
def init_enc(label_to_class_dict):
    """
    #enc.transform([[0],[1],[2]]).toarray()
    # array([[1., 0., 0.],
    #        [0., 1., 0.],
    #        [0., 0., 1.]])
    """
    enc = preprocessing.OneHotEncoder()
    tmp = np.array(list(label_to_class_dict.keys()))
    tmp = tmp.reshape([tmp.size, 1])
    enc.fit(tmp)
    return enc
def generate():
    root = '/media/ltelab/D/wangsixian/Huawei_Project'
    save_path = join(root, 'data_final/')
    # 11111111111111111111

    csv_root_path_list = ['data/2_processed_csv_data/2_cleaned_data_yss/line_labeled_csv',
                          'data/2_processed_csv_data/2_cleaned_data_yss',
                          ]

    for csv_root_path in csv_root_path_list:
        csv_list = [csv_root_path + "/" + file for file in os.listdir(csv_root_path) if file.endswith('.csv')]

        merge_dict = {'bus': 'bus', 'subway': 'subway', 'others': 'others', 'walk': 'others', 'wait': 'others',
                      'subway_in_out': 'others', 'unknown': 'others', 'subway_walk': 'others', 'subway_wait': 'others',
                      'bus_wait': 'others', 'CHSR': 'CHSR', 'elevetor': 'elevetor'}
        # 针对第二次数据同一个csv文件中包含不同交通方式的处理
        for filename in csv_list:
            fileprefix = filename.split('/')[-1].split('.')[0]
            print(filename, "!!!")
            # print(fileprefix ,"!!!!!")
            df = pd.read_csv(filename)
            df['Trans'] = df.apply(lambda x: merge_dict[x['Trans']], axis=1)  # 合并标签成3类 subway，bus，others

            for trans, subdf in df.groupby('Trans'):  # subdf中Trans相同，但是是不同采样段的
                # if trans is 'bus' or 'subway' we also need to add the `Line` label  2019-09-26 11:45:27

                subdf['idx'] = subdf.index
                subdf.index = range(subdf.shape[0])
                subdf['newsample'] = subdf['idx'].diff() != 1
                subdf['studyIndex'] = subdf['newsample'].cumsum()  # 重置studyindex

                for studyIndex, sdf in subdf.groupby('studyIndex'):
                    sdf.index = range(sdf.shape[0])
                    if trans == 'subway' or trans == 'bus':
                        if 'Line' in sdf:
                            line = int(sdf['Line'].unique()[0])
                        else:
                            line = 'no'
                        name = trans + '_05-' + fileprefix + '-' + "{:0>2d}".format(studyIndex) + '_line_' + str(line)
                    else:
                        name = trans + '_05-' + fileprefix + '-' + "{:0>2d}".format(studyIndex)
                    if os.path.exists(join(save_path, str(name) + "_X.npy")):
                        x = np.load(join(save_path, str(name) + "_X.npy"))
                        continue
                    # print(save_path,"!")
                    sdf['name'] = name

                    success = preprocess(sdf,
                                         time_step,
                                         enc,
                                         fea_cols,
                                         gb_cols,
                                         class_cols,
                                         save_path=save_path)
                    if success:
                        dfnames.append(name)
                        print('ndarray appended')

    # 22222222222222222222

    # csv_file_path = join(root,'data/testdata_2019.07/')
    # csv_file_path = join(root ,'data/2_processed_csv_data/2_cleaned_data_yss/')
    # csv_file_path = r"C:\Users\wsx\Desktop\todo-no-label-dataset-18.11.28_nk"
    csv_file_path_list = [
                          r'/media/ltelab/D/wangsixian/Huawei_Project/data/2_processed_csv_data/3_new_data',
                          r"/media/ltelab/D/wangsixian/Huawei_Project/data/HSRdata",
                          r"/media/ltelab/D/wangsixian/Huawei_Project/data/testdata_2019.07",
                          r"/media/ltelab/D/wangsixian/Huawei_Project/data/testdata_2019.10",
                          r"/media/ltelab/D/wangsixian/Huawei_Project/data/elevetor_data"]
    for csv_file_path in csv_file_path_list:
        filelist = [csv_file_path + "/" + file for file in os.listdir(csv_file_path) if file.endswith('.csv')]
        for file in filelist:
            process_one_csv_file(file, save_path)
if __name__=='__main__':
    class_to_label_dict, label_to_class_dict = init_class_to_label_dict(class_lst)
    enc = init_enc(label_to_class_dict)
    time_step = 30
    dfnames = []
    root = '/media/ltelab/D/wangsixian/Huawei_Project'
    save_path = join(root, 'data_final')

    #process_one_csv_file("/media/ltelab/D/wangsixian/Huawei_Project/data/testdata_2019.10/others_normal.csv",save_path)
    # generate samplelist
    generate()
    for file in os.listdir(save_path):
        if file.endswith('_X.npy'):
            if os.path.exists(join(save_path, 'sampleList.npy')):
                file_list = np.load(join(save_path, 'sampleList.npy'))
                file_list = np.hstack((file_list, np.array(file[:-6])))
                np.save(join(save_path, 'sampleList.npy'), file_list)
            else:
                np.save(join(save_path, 'sampleList.npy'), np.array(file[:-6]))
    print('End!')


import requests
from bs4 import BeautifulSoup
from mail import my_mail

def jhw(url, origin_price):
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')
    item_title = soup.find('h2', id='item_title').get_text()
    try:
        rmb_price = soup.find_all('span', color='orange')[1].get_text()
        text = '商品名{}\n原价是{}\n现价是{}\n便宜了{}'.format(item_title, origin_price, rmb_price, origin_price - int(rmb_price))
        if origin_price > int(rmb_price):
            mail_fyx = my_mail(text, '1101022351@qq.com')
            mail_fyx.run()

    except Exception:
        # print(Exception)
        mail_fyx = my_mail('有错误了', '1101022351@qq.com')
        mail_fyx.run()

if __name__ == '__main__':
    # jhw('http://surugaya.masadora.net/product/detail/602002954001', 88)
    jhw('http://surugaya.masadora.net/product/detail/120045357001', 23)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Applicable_To',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('Department', models.CharField(max_length=100)),
                ('Session', models.CharField(max_length=10)),
                ('Year', models.CharField(max_length=15, choices=[(b'1st', b'1st'), (b'2nd', b'2nd'), (b'3rd', b'3rd'), (b'4th', b'4th'), (b'5th', b'5th'), (b'M.Tech-1st year', b'M.Tech-1st year'), (b'M.Tech-2nd year', b'M.Tech-2nd year'), (b'PhD', b'PhD')])),
            ],
        ),
        migrations.CreateModel(
            name='Application',
            fields=[
                ('App_id', models.AutoField(serialize=False, primary_key=True)),
                ('Submission_date', models.DateTimeField()),
                ('Submission_status', models.CharField(max_length=10, choices=[(b'Approved', b'Approved'), (b'Pending', b'Pending'), (b'Rejected', b'Rejected')])),
            ],
        ),
        migrations.CreateModel(
            name='Details',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('Scholarship_period', models.CharField(max_length=15)),
                ('Amount', models.IntegerField()),
                ('Reference', models.URLField()),
                ('Selection_criteria', models.CharField(max_length=250)),
                ('Deadline', models.DateTimeField()),
            ],
        ),
        migrations.CreateModel(
            name='Sanction_List',
            fields=[
                ('Sanction_id', models.AutoField(serialize=False, primary_key=True)),
                ('Amount', models.IntegerField()),
            ],
        ),
        migrations.CreateModel(
            name='Scholarships',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('Name', models.CharField(max_length=100)),
                ('Sponsoring_org', models.CharField(max_length=100)),
                ('Type', models.CharField(max_length=30, choices=[(b'Central Scholarship', b'Central Scholarship'), (b'State-Level Post Metric Scholarship', b'State-Level Post Metric Scholarship'), (b'Endowment Scholarship instituted at IIT', b'Endowment Scholarship instituted at IIT'), (b'Endowment Scholarship instituted at BHU', b'Endowment Scholarship instituted at BHU')])),
                ('Target_Group', models.CharField(max_length=10)),
            ],
        ),
        migrations.CreateModel(
            name='Students',
            fields=[
                ('Roll_Number', models.IntegerField(serialize=False, primary_key=True)),
                ('Full_Name', models.CharField(max_length=120, null=True, blank=True)),
                ('Semester', models.CharField(max_length=10, choices=[(b'1st', b'1st'), (b'2nd', b'2nd'), (b'3rd', b'3rd'), (b'4th', b'4th'), (b'5th', b'5th'), (b'6th', b'6th'), (b'7th', b'7th'), (b'8th', b'8th'), (b'9th', b'9th'), (b'10th', b'10th')])),
                ('Degree', models.CharField(max_length=10, choices=[(b'B.Tech', b'B.Tech'), (b'IDD', b'IDD'), (b'M.Tech', b'M.Tech'), (b'PhD', b'PhD')])),
                ('Department', models.CharField(max_length=120)),
                ('Mobile_Number', models.IntegerField()),
                ('Hostel', models.CharField(max_length=120)),
                ('Email', models.EmailField(max_length=120)),
                ('Password', models.CharField(max_length=120)),
            ],
        ),
        migrations.AddField(
            model_name='sanction_list',
            name='Scholarship_name',
            field=models.ForeignKey(to='RegistrationForScholarship.Scholarships'),
        ),
        migrations.AddField(
            model_name='sanction_list',
            name='Student',
            field=models.ForeignKey(to='RegistrationForScholarship.Students'),
        ),
        migrations.AddField(
            model_name='details',
            name='Scholarship',
            field=models.OneToOneField(to='RegistrationForScholarship.Scholarships'),
        ),
        migrations.AddField(
            model_name='application',
            name='Student',
            field=models.ForeignKey(to='RegistrationForScholarship.Students'),
        ),
        migrations.AddField(
            model_name='applicable_to',
            name='Scholarship',
            field=models.ForeignKey(to='RegistrationForScholarship.Scholarships'),
        ),
    ]


"""OpenAPI specific loader behavior."""
import json

import pytest

from schemathesis.specs.openapi import loaders
from schemathesis.specs.openapi.schemas import OpenApi30, SwaggerV20

def test_openapi_asgi_loader(fastapi_app, run_asgi_test):
    # When an ASGI app is loaded via `from_asgi`
    schema = loaders.from_asgi("/openapi.json", fastapi_app)
    strategy = schema["/users"]["GET"].as_strategy()
    # Then it should successfully make calls via `call_asgi`
    run_asgi_test(strategy)

def test_openapi_wsgi_loader(flask_app, run_wsgi_test):
    # When a WSGI app is loaded via `from_wsgi`
    schema = loaders.from_wsgi("/schema.yaml", flask_app)
    strategy = schema["/success"]["GET"].as_strategy()
    # Then it should successfully make calls via `call_wsgi`
    run_wsgi_test(strategy)

@pytest.mark.parametrize(
    "version, expected",
    (
        ("20", SwaggerV20),
        ("30", OpenApi30),
    ),
)
def test_force_open_api_version(version, expected):
    schema = {
        # Invalid schema, but it happens in real applications
        "swagger": "2.0",
        "openapi": "3.0.0",
    }
    loaded = loaders.from_dict(schema, force_schema_version=version, validate_schema=False)
    assert isinstance(loaded, expected)

def test_number_deserializing(testdir):
    # When numbers in a schema are written in scientific notation but without a dot
    # (achieved by dumping the schema with json.dumps)
    schema = {
        "openapi": "3.0.2",
        "info": {"title": "Test", "description": "Test", "version": "0.1.0"},
        "paths": {
            "/teapot": {
                "get": {
                    "summary": "Test",
                    "parameters": [
                        {
                            "name": "key",
                            "in": "query",
                            "required": True,
                            "schema": {"type": "number", "multipleOf": 0.00001},
                        }
                    ],
                    "responses": {"200": {"description": "OK"}},
                }
            }
        },
    }

    schema_path = testdir.makefile(".yaml", schema=json.dumps(schema))
    # Then yaml loader should parse them without schema validation errors
    parsed = loaders.from_path(str(schema_path))
    # and the value should be a number
    value = parsed.raw_schema["paths"]["/teapot"]["get"]["parameters"][0]["schema"]["multipleOf"]
    assert isinstance(value, float)

def test_unsupported_type():
    # When Schemathesis can't detect the Open API spec version
    with pytest.raises(ValueError, match="^Unsupported schema type$"):
        # Then it raises an error
        loaders.from_dict({})


from decouple import config
import gdown
import zipfile

#1 Descargar wav
url = config("RAW_DATA")
name_raw = config("FOLDER_NAME")
output = config("RAW_FOLDER")+name_raw
gdown.download(url, output, quiet=False)
# Descomprimir
with zipfile.ZipFile(output, 'r') as zip_ref:
    zip_ref.extractall(config("RAW_FOLDER"))

#2 Descargar csv
url = config("CSV_DATA")
name_raw = config("CSV_FILE")
output = config("CSV_FOLDER")+name_raw
gdown.download(url, output, quiet=False)
# Descomprimir
with zipfile.ZipFile(output, 'r') as zip_ref:
    zip_ref.extractall(config("CSV_FOLDER"))


import importlib

from socket         import error as socket_error

class Server:
    def __init__(self, client_adapter, join_mode="auto"):
        self.client = client_adapter
        self.client.server = self
        self.join_mode = join_mode

        # config parameters
        self.max_games = 4

        # game modules
        self.games = []

    #### game functions ####
    def setgames(self, game_list):
        # stop old games
        self.stop_games()
        self.load_games(game_list)

    def find_game_by_id(self, game_id):
        return self.games[game_id]

    def count_games(self):
        return len(self.games)

    def stop_games(self):
        if self.count_games() > 0:
            for game in self.games:
                game.stop()
        self.games = []

    def find_autojoin_game(self):
        return self.games[0]

    def maybe_join_games(self):
        if self.join_mode == "auto":
            for _, player in self.find_all_players():
                player.set_game(self.find_autojoin_game())

    def filter_games_list(self, games_list):
        if self.join_mode == "auto":
            return game_list[:1]
        else:
            return game_list

    def load_games(self, game_list):
        # start new games
        game_list = self.filter_games_list(games_list)
        for i, game_name in enumerate(game_list):
            module     = importlib.import_module("ghoust.games." + game_name)
            game_class = getattr(module, game_name)
            game       = game_class(i)
            game.setup()
            self.games.append(game)
        self.maybe_join_games()

    def stop(self):
        for game in self.games:
            game.stop()
        self.client.stop()

    def start(self):
        for i in range(3):
            try:
                self.client.connect()
                break
            except socket_error as e:
                print("socket.error: [{}] {}".format(e.errno, e.strerror))
                if i == 2:
                    raise e
                print("retrying after 10s")
                time.sleep(10)

        self.client.publish("GHOUST/server/status", "ACTIVE")
        self.client.start()


import json

def get_stored_number():
    filename = "number.json"

    try:
        with open(filename) as file:
            number = json.load(file)
    except FileNotFoundError:
        return None
    else:
        return number

def get_new_number():
    prompt = "Please input your favorite number: "
    filename = "number.json"

    number = input(prompt)
    with open(filename, 'w') as file:
        json.dump(number, file)
    return number

def show_favorite_number():
    number = get_stored_number()
    if number:
        print("I konw your favorite number. It's " + number)
    else:
        number = get_new_number()
        print("I will show your favorite number to you.")

show_favorite_number()


#!/usr/bin/python3

from flask import Flask, jsonify, request, make_response, render_template
from datetime import datetime

from blueprint.usuario3 import usuario

app = Flask(__name__)
app.register_blueprint(usuario)

# 127.0.0.1:5000
@app.route('/')
def home():
    return render_template('index.html', title='Home')

if __name__ == '__main__':
    app.run(debug=True)


# coding: utf-8

"""
    Cloudera Manager API

Cloudera Manager API v33

Introduced in Cloudera Manager 6.3.0

Cloudera Product Documentation

    OpenAPI spec version: 6.3.0

    Generated by: https://github.com/swagger-api/swagger-codegen.git
"""

from pprint import pformat
from six import iteritems
import re

class ApiRollingUpgradeServicesArgs(object):
    """
    NOTE: This class is auto generated by the swagger code generator program.
    Do not edit the class manually.
    """

    """
    Attributes:
      swagger_types (dict): The key is attribute name
                            and the value is attribute type.
      attribute_map (dict): The key is attribute name
                            and the value is json key in definition.
    """
    swagger_types = {
        'upgrade_from_cdh_version': 'str',
        'upgrade_to_cdh_version': 'str',
        'slave_batch_size': 'float',
        'sleep_seconds': 'float',
        'slave_fail_count_threshold': 'float',
        'upgrade_service_names': 'list[str]'
    }

    attribute_map = {
        'upgrade_from_cdh_version': 'upgradeFromCdhVersion',
        'upgrade_to_cdh_version': 'upgradeToCdhVersion',
        'slave_batch_size': 'slaveBatchSize',
        'sleep_seconds': 'sleepSeconds',
        'slave_fail_count_threshold': 'slaveFailCountThreshold',
        'upgrade_service_names': 'upgradeServiceNames'
    }

    def __init__(self, upgrade_from_cdh_version=None, upgrade_to_cdh_version=None, slave_batch_size=None, sleep_seconds=None, slave_fail_count_threshold=None, upgrade_service_names=None):
        """
        ApiRollingUpgradeServicesArgs - a model defined in Swagger
        """

        self._upgrade_from_cdh_version = None
        self._upgrade_to_cdh_version = None
        self._slave_batch_size = None
        self._sleep_seconds = None
        self._slave_fail_count_threshold = None
        self._upgrade_service_names = None

        if upgrade_from_cdh_version is not None:
          self.upgrade_from_cdh_version = upgrade_from_cdh_version
        if upgrade_to_cdh_version is not None:
          self.upgrade_to_cdh_version = upgrade_to_cdh_version
        if slave_batch_size is not None:
          self.slave_batch_size = slave_batch_size
        if sleep_seconds is not None:
          self.sleep_seconds = sleep_seconds
        if slave_fail_count_threshold is not None:
          self.slave_fail_count_threshold = slave_fail_count_threshold
        if upgrade_service_names is not None:
          self.upgrade_service_names = upgrade_service_names

    @property
    def upgrade_from_cdh_version(self):
        """
        Gets the upgrade_from_cdh_version of this ApiRollingUpgradeServicesArgs.
        Current CDH Version of the services. Example versions are: \"5.1.0\", \"5.2.2\" or \"5.4.0\"

        :return: The upgrade_from_cdh_version of this ApiRollingUpgradeServicesArgs.
        :rtype: str
        """
        return self._upgrade_from_cdh_version

    @upgrade_from_cdh_version.setter
    def upgrade_from_cdh_version(self, upgrade_from_cdh_version):
        """
        Sets the upgrade_from_cdh_version of this ApiRollingUpgradeServicesArgs.
        Current CDH Version of the services. Example versions are: \"5.1.0\", \"5.2.2\" or \"5.4.0\"

        :param upgrade_from_cdh_version: The upgrade_from_cdh_version of this ApiRollingUpgradeServicesArgs.
        :type: str
        """

        self._upgrade_from_cdh_version = upgrade_from_cdh_version

    @property
    def upgrade_to_cdh_version(self):
        """
        Gets the upgrade_to_cdh_version of this ApiRollingUpgradeServicesArgs.
        Target CDH Version for the services. The CDH version should already be present and activated on the nodes. Example versions are: \"5.1.0\", \"5.2.2\" or \"5.4.0\"

        :return: The upgrade_to_cdh_version of this ApiRollingUpgradeServicesArgs.
        :rtype: str
        """
        return self._upgrade_to_cdh_version

    @upgrade_to_cdh_version.setter
    def upgrade_to_cdh_version(self, upgrade_to_cdh_version):
        """
        Sets the upgrade_to_cdh_version of this ApiRollingUpgradeServicesArgs.
        Target CDH Version for the services. The CDH version should already be present and activated on the nodes. Example versions are: \"5.1.0\", \"5.2.2\" or \"5.4.0\"

        :param upgrade_to_cdh_version: The upgrade_to_cdh_version of this ApiRollingUpgradeServicesArgs.
        :type: str
        """

        self._upgrade_to_cdh_version = upgrade_to_cdh_version

    @property
    def slave_batch_size(self):
        """
        Gets the slave_batch_size of this ApiRollingUpgradeServicesArgs.
        Number of hosts with slave roles to upgrade at a time. Must be greater than zero. Default is 1.

        :return: The slave_batch_size of this ApiRollingUpgradeServicesArgs.
        :rtype: float
        """
        return self._slave_batch_size

    @slave_batch_size.setter
    def slave_batch_size(self, slave_batch_size):
        """
        Sets the slave_batch_size of this ApiRollingUpgradeServicesArgs.
        Number of hosts with slave roles to upgrade at a time. Must be greater than zero. Default is 1.

        :param slave_batch_size: The slave_batch_size of this ApiRollingUpgradeServicesArgs.
        :type: float
        """

        self._slave_batch_size = slave_batch_size

    @property
    def sleep_seconds(self):
        """
        Gets the sleep_seconds of this ApiRollingUpgradeServicesArgs.
        Number of seconds to sleep between restarts of slave host batches.  Must be greater than or equal to 0. Default is 0.

        :return: The sleep_seconds of this ApiRollingUpgradeServicesArgs.
        :rtype: float
        """
        return self._sleep_seconds

    @sleep_seconds.setter
    def sleep_seconds(self, sleep_seconds):
        """
        Sets the sleep_seconds of this ApiRollingUpgradeServicesArgs.
        Number of seconds to sleep between restarts of slave host batches.  Must be greater than or equal to 0. Default is 0.

        :param sleep_seconds: The sleep_seconds of this ApiRollingUpgradeServicesArgs.
        :type: float
        """

        self._sleep_seconds = sleep_seconds

    @property
    def slave_fail_count_threshold(self):
        """
        Gets the slave_fail_count_threshold of this ApiRollingUpgradeServicesArgs.
        The threshold for number of slave host batches that are allowed to fail to restart before the entire command is considered failed.  Must be greater than or equal to 0. Default is 0.
 This argument is for ADVANCED users only.

        :return: The slave_fail_count_threshold of this ApiRollingUpgradeServicesArgs.
        :rtype: float
        """
        return self._slave_fail_count_threshold

    @slave_fail_count_threshold.setter
    def slave_fail_count_threshold(self, slave_fail_count_threshold):
        """
        Sets the slave_fail_count_threshold of this ApiRollingUpgradeServicesArgs.
        The threshold for number of slave host batches that are allowed to fail to restart before the entire command is considered failed.  Must be greater than or equal to 0. Default is 0.
 This argument is for ADVANCED users only.

        :param slave_fail_count_threshold: The slave_fail_count_threshold of this ApiRollingUpgradeServicesArgs.
        :type: float
        """

        self._slave_fail_count_threshold = slave_fail_count_threshold

    @property
    def upgrade_service_names(self):
        """
        Gets the upgrade_service_names of this ApiRollingUpgradeServicesArgs.
        List of services to upgrade. Only the services that support rolling upgrade should be included.

        :return: The upgrade_service_names of this ApiRollingUpgradeServicesArgs.
        :rtype: list[str]
        """
        return self._upgrade_service_names

    @upgrade_service_names.setter
    def upgrade_service_names(self, upgrade_service_names):
        """
        Sets the upgrade_service_names of this ApiRollingUpgradeServicesArgs.
        List of services to upgrade. Only the services that support rolling upgrade should be included.

        :param upgrade_service_names: The upgrade_service_names of this ApiRollingUpgradeServicesArgs.
        :type: list[str]
        """

        self._upgrade_service_names = upgrade_service_names

    def to_dict(self):
        """
        Returns the model properties as a dict
        """
        result = {}

        for attr, _ in iteritems(self.swagger_types):
            value = getattr(self, attr)
            if isinstance(value, list):
                result[attr] = list(map(
                    lambda x: x.to_dict() if hasattr(x, "to_dict") else x,
                    value
                ))
            elif hasattr(value, "to_dict"):
                result[attr] = value.to_dict()
            elif isinstance(value, dict):
                result[attr] = dict(map(
                    lambda item: (item[0], item[1].to_dict())
                    if hasattr(item[1], "to_dict") else item,
                    value.items()
                ))
            else:
                result[attr] = value

        return result

    def to_str(self):
        """
        Returns the string representation of the model
        """
        return pformat(self.to_dict())

    def __repr__(self):
        """
        For `print` and `pprint`
        """
        return self.to_str()

    def __eq__(self, other):
        """
        Returns true if both objects are equal
        """
        if not isinstance(other, ApiRollingUpgradeServicesArgs):
            return False

        return self.__dict__ == other.__dict__

    def __ne__(self, other):
        """
        Returns true if both objects are not equal
        """
        return not self == other


'''A dynamic content management system using jQuery and Python'''
VERSION = (0, 9, 0, 'alpha', 1)

import os
import sys

from .utils.version import get_version

# This list is updated by the views.appsite.appsite handler
empty_choice = ('','-----------------')

__version__ = version = get_version(VERSION)
__license__ = "BSD"
__author__ = "Luca Sbardella"
__contact__ = "luca.sbardella@gmail.com"
__homepage__ = "http://djpcms.com/"
__docformat__ = "restructuredtext"

PACKAGE_DIR = os.path.dirname(os.path.abspath(__file__))
LIBRARY_NAME = os.path.basename(PACKAGE_DIR)
SOFTWARE_NAME = LIBRARY_NAME + ' ' +  __version__

def DEFAULT_JAVASCRIPT(*extra):
    js = ['djpcms/djpcms.js',
          'djpcms/showdown.js']
    js.extend(extra)
    return js

LOGGING_SAMPLE = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format': '%(asctime)s | (p=%(process)s,t=%(thread)s)\
 | %(levelname)s | %(name)s | %(message)s'
        },
        'simple': {
            'format': '%(asctime)s %(levelname)s %(message)s',
            'datefmt': '%Y-%m-%d %H:%M:%S'
        },
    },
    'handlers': {
        'simple_console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'simple'
        },
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'verbose'
        }
    },
    'loggers': {
        'djpcms.request':{
            'level': 'ERROR',
            'propagate': True,
        }
    }
}

class Renderer(object):
    '''A mixin for all classes which render into string or bytes.

.. attribute:: description

    An optional description of the renderer.

    Default ``None``
'''
    description = None

    def render(self, request=None, **kwargs):
        '''render ``self`` as a string or bytes.'''
        raise NotImplementedError()

    def content_type(self):
        '''Content Type for this renderer'''
        return 'text/plain'

    def media(self, request):
        '''It returns an instance of :class:`djpcms.media.Media` or ``None``.
It should be overwritten by derived classes.'''
        return None

def is_renderer(obj):
    return isinstance(obj, Renderer)


# -*- coding: utf-8 -*-
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

plt.rcParams['figure.figsize'] = (10, 8)
plt.rcParams['font.size'] = 12
plt.rcParams['lines.linewidth'] = 2

#Histograms:

x = np.genfromtxt('build/randxor_1.txt', unpack=True)
y = np.genfromtxt('build/randxor_2.txt', unpack=True)

plt.subplot(2, 1, 1)
plt.hist(x)
plt.title(r"$(a_1, b_1, c_1) = (11, 1, 7)$")

plt.subplot(2, 1, 2)
plt.hist(y)
plt.title(r"$(a_2, b_2, c_ 2) = (11, 4, 7)$")

plt.tight_layout()

plt.savefig('build/randxor_hist.pdf')
plt.clf()

# Correlation:

x=np.resize(x, (32767, 2)) #turns the one-dimensional arrays into two-dimensional ones
y=np.resize(x, (32767, 2)) #to create pairs of successive numbers

plt.plot(x[:, 0], x[:, 1], 'k.') #plots first array column against second array column
plt.title(u"Korrelation für $(a_1, b_1, c_1) = (11, 1, 7)$")
plt.savefig('build/randxor_corr_1.pdf')

plt.plot(y[:, 0], y[:, 1], 'k.')
plt.title(u" Korrelation für $(a_2, b_2, c_ 2) = (11, 4, 7)$")
plt.savefig('build/randxor_corr_2.pdf')

plt.clf()

# Recursion rate:

#b, c and sequence length have been saved to file in three columns in that order
#using x and y for b and c, and z for recursion rate
x, y, z = np.genfromtxt('build/recursion.txt', unpack=True)

fig = plt.figure()
ax = fig.gca(projection='3d')

bottom = np.zeros_like(z)
width = depth = 0.4

ax.bar3d(x, y, bottom, width, depth, z)
ax.set_title('Rekursionsrate')

plt.savefig('build/randxor_rec.pdf')


#!/usr/bin/python
# -*- coding: UTF-8 -*-

from xml.dom.minidom import parse
import xml.dom.minidom
import codecs, json, os, re
import hyperparams as hp
from util import is_Chinese

def loadOneXml(xml_file, xml2json, reload):
    """
    parser xml file to json format and save in 'temp/'

    :param xml_file: the source xml file
    :param xml2json:save xml to json format
    :param reload: if reload is True, the save file will be constructed newly

    :return text: string, the parsered data
    :return PlatoNamedEntityUIMA: list, the parsered data
    :return PlatoRelationUIMA: list, the parsered data
    """
    # parser text, Sentence, BaseToken, PlatoNamedEntityUIMA, PlatoRelationUIMA from xml file
    Sentence, BaseToken, PlatoNamedEntityUIMA, PlatoRelationUIMA = [], [], [], []

    # parser
    DOMTree = xml.dom.minidom.parse(xml_file)
    xmi = DOMTree.documentElement

    # get text, Sentence, BaseToken, PlatoNamedEntityUIMA, PlatoRelationUIMA
    nodelist = xmi.getElementsByTagName("cas:Sofa")
    text = nodelist[0].getAttribute("sofaString")
    for textspan_Sentence in xmi.getElementsByTagName("textspan:Sentence"):
        Sentence.append([textspan_Sentence.getAttribute("begin"), textspan_Sentence.getAttribute("end")])
    for syntax_BaseToken in xmi.getElementsByTagName("syntax:BaseToken"):
        BaseToken.append([syntax_BaseToken.getAttribute("begin"), syntax_BaseToken.getAttribute("end")])
    for typesystem_PlatoNamedEntityUIMA in xmi.getElementsByTagName("typesystem:PlatoNamedEntityUIMA"):
        PlatoNamedEntityUIMA.append([typesystem_PlatoNamedEntityUIMA.getAttribute("begin"), typesystem_PlatoNamedEntityUIMA.getAttribute("end"),
                                     typesystem_PlatoNamedEntityUIMA.getAttribute("semanticTag"), typesystem_PlatoNamedEntityUIMA.getAttribute("xmi:id")])
    for typesystem_PlatoRelationUIMA in xmi.getElementsByTagName("typesystem:PlatoRelationUIMA"):
        PlatoRelationUIMA.append([typesystem_PlatoRelationUIMA.getAttribute("entFrom"), typesystem_PlatoRelationUIMA.getAttribute("entTo"),
                                 typesystem_PlatoRelationUIMA.getAttribute("semanticTag")])

    # save data
    if reload == True:
        if os.path.exists(xml2json): os.remove(xml2json)
        with codecs.open(xml2json, 'a+', encoding='utf-8') as fw:
            fw.write(json.dumps(text)+'\n'+json.dumps(PlatoNamedEntityUIMA)+'\n'+json.dumps(PlatoRelationUIMA)+'\n')
            fw.close()

    NamedEntity = modifyTag(PlatoNamedEntityUIMA)

    return text, NamedEntity, PlatoRelationUIMA

def modifyTag(PlatoNamedEntityUIMA):
    results = []
    for item in PlatoNamedEntityUIMA:
        result = item
        entity = item[2]
        if entity == '不良反应-诊断' or entity == '不良反应-疾病':
            result[2] = '不良反应-引发疾病'
        elif entity == '成分-复方-成分-含量' or entity == '成分-复方-成分':
            result[2] = '成分-复方成分'
        elif entity == '注意事项-人群' or entity == '注意事项-疾病相关':
            result = []
        elif entity == '用法用量-每日剂量-低值' or entity == '用法用量-每日剂量-高值':
            result[2] = '用法用量-每日剂量'
        elif entity == '用法用量-每次剂量-低值' or entity == '用法用量-每次剂量-高值':
            result[2] = '用法用量-每次剂量'
        elif entity == '用法用量-疾病' or entity == '用法用量-疾病状态-低值':
            result = []
        elif entity == '用法用量-给药频次-低值' or entity == '用法用量-给药频次-高值':
            result[2] = '用法用量-给药频次'
        elif entity == '用法用量-起始剂量-低值' or entity == '用法用量-起始剂量-高值':
            result[2] = '用法用量-起始剂量'
        elif entity == '用法用量-疾病-低值' or entity == '用法用量-疾病-高值':
            result[2] = '用法用量-疾病'
        elif entity == '药物禁忌-人群':
            result = []
        elif entity == '适应症-症状' or entity == '适应症-诊断根据':
            result = []
        else:
            pass

        if result != []:
            results.append(result)
    return results

def getType(source_dir, type_file, reload=False):
    """
    store all types and relevant counts of NER and relation and save in 'temp/'

    :param source_dir: the directory that store xml files
    :param type_file:save NER types and relation types
    :param reload: if reload is True, the save file will be constructed newly

    :return NERType: dictionary, the tag types of NER types
    :return relationType: dictionary, the tag types of relation types
    """
    NERType, relationType = {}, {}

    for xml_file in os.listdir(source_dir):
        text, NamedEntity, Relation = loadOneXml(source_dir+xml_file, hp.xml2json, reload)
        for ner in NamedEntity:
            begin, end, type = int(ner[0]), int(ner[1]), ner[2]
            NERType[type] = 1 if type not in NERType else NERType[type] + 1
        for rela in Relation:
            ent_from, ent_to, type = rela[0], rela[1], rela[2]
            relationType[type] = 1 if type not in relationType else relationType[type] + 1

    # save data
    if reload == True:
        if os.path.exists(type_file): os.remove(type_file)
        with codecs.open(type_file, 'w', 'utf-8') as fw:
            fw.write(json.dumps(NERType)+'\n'+json.dumps(relationType)+'\n')
            fw.close()
    # a = (sorted(NERType.items(), key=lambda item:item[1], reverse=True))
    # b = []
    # for i in range(len(a)):
    #     b.append(a[i][0])
    # print(b)
    return NERType, relationType

def xml2NER(source_dir, ner_file, reload=False):
    """
    transfer json format to the format of NER inputs

    :param source_dir: the directory that store xml files
    :param ner_file:save processed NER format data
    :param reload: if reload is True, the save file will be constructed newly

    :return S: list, the processed and split sentences
    :return T: list, correspond tags of the processed and split sentences
    """
    text2tag, tag2label = hp.text2tag, hp.tag2label
    sentences, tags = [], []
    for xml_file in os.listdir(source_dir):
        text, NamedEntity, Relation = loadOneXml(source_dir+xml_file, hp.xml2json, reload)
        sentence, tag = [], []
        for t in text:
            sentence.append(t)
        for i in range(len(sentence)):
            tag.append('O')
        for ner in NamedEntity:
            begin, end, type = int(ner[0]), int(ner[1]), ner[2]
            end = end-1
            # store as "BIESO" in tags
            if begin == end:
                tag[begin] = "S-"+ner[2]
            elif begin + 1 == end:
                tag[begin] = "B-"+ner[2]
                tag[end] = "E-" + ner[2]
            elif begin + 2 == end:
                tag[begin] = "B-" + ner[2]
                tag[begin+1] = "I-" + ner[2]
                tag[end] = "E-" + ner[2]
            else:
                tag[begin] = "B-"+ner[2]
                for i in range(begin+1, end-1):
                    tag[i] = "I-" + ner[2]
                if text[end] in ['，', ',', '。', '：', ':', '；', ';', '、', '）', ' ', '\n']:
                    tag[end-1] = "E-" + ner[2]
                else:
                    tag[end - 1] = "I-" + ner[2]
                    tag[end] = "E-" + ner[2]

        sentences.extend(sentence)
        tags.extend(tag)
    print("处理前句子长度：{} ； 标注长度：{}".format(len(sentences), len(tags)))
    sentences, tags = processSentences(sentences, tags)
    print("处理后句子长度：{} ； 标注长度：{}".format(len(sentences), len(tags)))

    # split sentences
    S,T = splitSentence(sentences, tags)

    #save
    if reload == True:
        if os.path.exists(ner_file): os.remove(ner_file)
        with codecs.open(ner_file, 'a+', encoding='utf-8') as fw:
            for (sent, tag) in zip(S, T):
                for (char, t) in zip(sent, tag):
                    fw.write(char + ' ' + t +'\n')
                fw.write('\n')
            fw.close()

    return S, T

def processSentences(sentences, tags):
    """
    process data, delete trashy chars and corresponding tags

    :param sentences: all the sentences
    :param tags:correspond tags of sentences

    :return sentences: the processed sentences
    :return tags: list, correspond tags of the processed sentences
    """
   # sentences = re.split('(。|！|\!|\.|？|\?)', paragraph)  # 保留分割符
    # delete '【' and '】' and their middles
    # while('【' in sentences and '】' in sentences):
    #     begin = sentences.index('【')
    #     end = sentences.index('】')
    #     if end > begin and end-begin < 15:
    #         for i in range(begin, end+1):
    #             sentences.pop(begin)
    #             tags.pop(begin)
    #     elif end < begin:
    #         sentences.pop(end)
    #         tags.pop(end)
    #     else:
    #         print("{}-{} 有多余的【】".format(begin, end))
    #         break

    # delete '\r', '\n', '?', ' ' in sentences
    list = ['\r', '\n', '?', ' ', '①', '②', '']
    for c in list:
        while (c in sentences):
            index = sentences.index(c)
            sentences.pop(index)
            tags.pop(index)

    # delete item numbers, include '（1）' and '1' and '1.' and '1、', there is "。" before them
    start = 0
    while('（' in sentences and '）' in sentences):
        try:
            begin = sentences.index('（', start)
            end = sentences.index('）', start)
            if end > begin and end - begin == 2 or end - begin == 3 and sentences[begin + 1].isdigit() and sentences[begin-1] == '。':
                for i in range(begin, end + 1):
                    sentences.pop(begin)
                    tags.pop(begin)
            else:
                start = end + 1
        except:
            break

    i = 0
    while(i+2 < len(sentences)):
        if sentences[i].isdigit() and sentences[i-1] == '。' and sentences[i+1] in ['.', '．','、'] and is_Chinese(sentences[i+2]):
            for j in range(2):
                sentences.pop(i)
                tags.pop(i)
        elif sentences[i].isdigit() and sentences[i-1] == '。' and is_Chinese(sentences[i+1]):
            sentences.pop(i)
            tags.pop(i)
        else:
            i += 1

    return sentences, tags

def splitSentence(sentences, tags):
    """
    split sentenses by "。" and save in S,T to yield train data and test data

    :param sentences: all the processed sentences
    :param tags:correspond tags of  processed sentences

    :return S: list, the split sentences
    :return T: list, correspond tags of the split sentences
    """
    S, T = [], []
    start = 0
    for i in range(len(sentences)):
        if i - start < 60:
            if sentences[i] in ['。']:
                sent, tag = [], []
                for j in range(start, i+1):
                    sent.append(sentences[j])
                    tag.append(tags[j])
                start = i + 1
                S.append(sent)
                T.append(tag)
        else:
            if sentences[i] in ['。', '，']:
                sent, tag = [], []
                for j in range(start, i + 1):
                    sent.append(sentences[j])
                    tag.append(tags[j])
                start = i + 1
                S.append(sent)
                T.append(tag)
    return S, T

# NERType, relationType = getType(hp.source_dir, hp.type_file, reload=True)
# print(NERType)
# print(relationType)

#_, _ = xml2NER(hp.source_dir, hp.ner_file, reload=True)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import gridspec
import random
from sklearn.metrics import mean_squared_error
from sklearn import preprocessing
from sys import maxsize
import copy

def DMD_YRR_prep(YRR_path, control = 'None'):

    '''

    DMD_YRR_prep

        @brief A function that prepares matrices for DMD calculations

        @param <str> YRR_path: Path to a YRR dataframe

                                    Need to have following attributes:

                                    - YSET_I_R
                                    - YSETResults
                                    - a1_combo,...,t39_combo

                                    Note: Each UniqueID should have exactly 3 responses - 1 intake and 2 corresponding first retakes.

        @return <List>  <np.ndarray> Y1  - Intake Matrix
                        <np.ndarray> R1  - Retake 1 Matrix
                        <np.ndarray> R2  - Retake 2 Matrix
                        <int>        sub - number of control rows added

    '''

    ldict = {}

    # -------- Read Data --------
    df_c= pd.read_csv(YRR_path)
    df_c.drop('Unnamed: 0', axis = 1, inplace = True)
    Y1_idx=range(0,df_c.shape[0],3)
    R1_idx=range(1,df_c.shape[0],3)
    R2_idx=range(2,df_c.shape[0],3)

    var_n = ['Y1', 'R1', 'R2']

    newlist = []
    for i in range(9):
        newlist.append(i)
    for i in range(16, 39):
        newlist.append(i)

    # -------- Prepare Dataframe for each timestamp --------
    # *** Might need to adjust if additional controls are introduced!! ***
    for var in var_n:
        read_data = """import numpy as np
import pandas as pd
import copy
df_{0}=df_c.iloc[{0}_idx,:]
dfr = copy.deepcopy(df_{0}.loc[:, 'RiskFactor'])

if control != 'None':
    if control == 'Ethnicity':
        dfi = copy.deepcopy(df_{0}.loc[:,'Ethnicity_Asian':'Ethnicity_Other'])
    else:
        dfi = copy.deepcopy(df_{0}.loc[:, '{1}'])

df_{0} = df_{0}.loc[:,'a1_combo':'t39_combo']
df_{0} = pd.concat([df_{0}, dfr], axis=1)

if control != 'None':
    df_{0} = pd.concat([df_{0}, dfi], axis=1)

{0} = df_{0}.convert_objects(convert_numeric=True).values
{0} = {0}.transpose()
""".format(var, control)
        exec(read_data,locals(),ldict)

    Y1 = ldict['Y1']
    R1 = ldict['R1']
    R2 = ldict['R2']

    # -------- One-hot coding for each control accordingly --------
    # *** Might need to adjust if additional controls are introduced!! ***
    if control != 'None':

        m = int(Y1.shape[0]-1)
        n = int(Y1.shape[1])
        for i in range(Y1.shape[1]):
            if R1[m,i]!=Y1[m,i]:
                R1[m,i] = Y1[m,i]
            if R2[m,i]!=Y1[m,i]:
                R2[m,i] = Y1[m,i]

        List_Num = []
        for i in range(Y1.shape[1]):
            List_Num.append(Y1[40,i])
            List_Num.append(R1[40,i])
            List_Num.append(R2[40,i])
        List_Num = list(dict.fromkeys(List_Num))
        Numpy_Num = np.array(List_Num)
        Numpy_Num = np.sort(Numpy_Num)

        for var in var_n:
            onehot = """
Add_Matrix_{0} = np.zeros((len(List_Num),n))

if control == 'Ethnicity':
    sub = 5

else:
    if control == 'GRYD_Zone':
        for i in range({0}.shape[1]):
            if {0}[{0}.shape[0]-1,i]<=6.0:
                k = {0}[-1,i]-1
                Add_Matrix_{0}[int(k),i] = 1.0
            elif {0}[{0}.shape[0]-1,i]>6.0 and {0}[{0}.shape[0]-1,i]<=16.0:
                k = {0}[-1,i]-2
                Add_Matrix_{0}[int(k),i] = 1.0
            else:
                k = {0}[-1,i]-3
                Add_Matrix_{0}[int(k),i] = 1.0
        {0} = np.delete({0},-1,axis=0)
        {0} = np.vstack([{0},Add_Matrix_{0}])
        sub = 21

    else:
        min_val = np.amin({0}[m,:])
        max_val = np.amax({0}[m,:])
        Add_Matrix = np.zeros((int(max_val-min_val+1),n))
        for i in range({0}.shape[1]):
            k = int({0}[-1,i]-min_val)
            Add_Matrix[k,i] = 1.0
        {0} = np.delete({0},-1,axis=0)
        {0} = np.vstack([{0},Add_Matrix])
        sub = int(max_val-min_val+1)""".format(var)
            exec(onehot,locals(),ldict)
        sub = ldict['sub']
    else:
        sub = 0

    # -------- Standardization --------

    for var in var_n:
        std = """
for i in range({0}.shape[1]):
    {0}[37][i] += 1
    {0}[38][i] += 1

for i in newlist:
    for j in range({0}.shape[1]):
        {0}[i][j] = ({0}[i][j] - 1) / 4

for i in range({0}.shape[1]):
    {0}[39][i] /= 9""".format(var)
        exec(std,locals(),ldict)

    Y1 = ldict['Y1']
    R1 = ldict['R1']
    R2 = ldict['R2']

    return [Y1, R1, R2, int(sub)]

class DMD:

    def __init__(self, X0, X1, sub = 0, train = 1.0, test = 0, control = 'None', pred = 'Test', name = "DMD"):

        '''

        DMD

            @brief Dynamic Mode Decomposition that supports DMDc

            @param  <np.ndarray> X0               - Matrix at timestamp 0
                    <np.ndarray> X1               - Matrix at timestamp 1
                    <int>        sub = 0          - Number of control variables added, default is 0
                    <float>      train = 1.0      - Proportion of training set, value should be between 0 and 1, default is 1
                    <float>      test = 0         - Proportion of the testing set, value should between 0 and 1, default is 0
                    <str>        control = 'None' - Name of control variable, default is 'None'
                    <str>        pred = 'Test'    - Dataset to predict on, value should be either 'Test' or 'Whole', default is 'Test'

            @return <List>  <np.ndarray> A_X0_X1  - Transition Matrix approximated by DMD
                            <float>      MSE      - Mean square error of the test result

        '''

        self.name = name

        if train + test != 1:
            if train == 1 and test != 0:
                train = 1 - test

        # Test-train Split
        Index = list(range(X0.shape[1]))
        random.shuffle(Index)
        Train_X = np.empty((X0.shape[0], 0))
        Test_X = np.empty((X0.shape[0], 0))
        Train_Y = np.empty((X1.shape[0] - sub, 0))
        Test_Y = np.empty((X1.shape[0] - sub, 0))
        for i in range(int(round(len(Index) * train))):
            Train_X = np.hstack((Train_X, X0[:, Index[i]].reshape(-1, 1)))
            if sub != 0:
                Train_Y = np.hstack((Train_Y, X1[:-sub, Index[i]].reshape(-1, 1)))
            else:
                Train_Y = np.hstack((Train_Y, X1[:, Index[i]].reshape(-1, 1)))
        for i in range(int(round(len(Index) * train)), len(Index)):
            Test_X = np.hstack((Test_X, X0[:, Index[i]].reshape(-1, 1)))
            if sub != 0:
                Test_Y = np.hstack((Test_Y, X1[:-sub, Index[i]].reshape(-1, 1)))
            else:
                Test_Y = np.hstack((Test_Y, X1[:, Index[i]].reshape(-1, 1)))

        # SVD for X0
        U_X0, Sig_X0, V_X0 = np.linalg.svd(Train_X, full_matrices=False)
        U_X0_T = U_X0.conjugate().transpose()
        V_X0_T = V_X0.conjugate().transpose()
        Sig_inv_X0 = np.zeros((X0.shape[0], X0.shape[0]))
        for i in range(X0.shape[0]):
            for j in range(X0.shape[0]):
                if i == j:
                    Sig_inv_X0[i][j] = 1 / Sig_X0[i]

        # Build up the DMD A matrix for X0 and X1
        A_step1 = np.dot(Train_Y, V_X0_T)
        A_step2 = np.dot(A_step1, Sig_inv_X0)
        A_X0_X1 = np.dot(A_step2, U_X0_T)

        if pred == 'Whole':
            Pred = np.dot(A_X0_X1, X0)
            if sub != 0:
                MSE = mean_squared_error(X1[:-sub, :], Pred)
            else: MSE = mean_squared_error(X1, Pred)
        else:
            Pred = np.dot(A_X0_X1, Test_X)
            MSE = mean_squared_error(Test_Y, Pred)

        self.A = A_X0_X1
        self.control = control
        self.MSE = MSE
        self.RF = A_X0_X1[39,:39]
        if control != 'None':
            self.eigval, self.eigvec = np.linalg.eig(A_X0_X1[:, :-sub])
        else:
            self.eigval, self.eigvec = np.linalg.eig(A_X0_X1)
        self.w_domeigvec = self.eigvec[:,0].real[:39]
        self.eiglog = np.log(self.eigval)[:39]
        self.label = {'a1':'a1: I try to be nice to other people because I care about their feelings.',
                      'a2':'a2: I get very angry and "lose my temper" (yell or get mad).',
                      'a3':'a3: I do as I am told.',
                      'a4':'a4: I try to scare people to get what I want.',
                      'a5':'a5: I am accused of not telling the truth or cheating.',
                      'a6':'a6: I take things that are not mine from home, school, or elsewhere.',
                      'b7':'b7: When I go out, I tell my parents or guardians where I am going or leave them a note.',
                      'b8':'b8: My parents or guardians know where I am when I am not at home or at school.',
                      'b9':'b9: My parents or guardians know who I am with, when I am not at home or at school.',
                      'c10':'c10: Did you fail to go on to the next grade in school or fail a course in school?',
                      'c11':'c11: Did you get suspended, expelled or transferred to another school for disciplinary reasons?',
                      'c12':'c12: Did you "go out" on a date with a boyfriend or girlfriend for the very first time?',
                      'c13':'c13: Did you break up with a boyfriend or girlfriend or did he or she break up with you?',
                      'c14':'c14: Did you have a big fight or problem with a friend?',
                      'c15':'c15: Did you start hanging out with a new group of friends?',
                      'c16':'c16: Did anyone you were close too die or get seriously injured?',
                      'de17':'de17: Sometimes I like to do something a little dangerous just for the fun of it.',
                      'de18':'de18: I sometimes find it exciting to do things that might get me in trouble.',
                      'de19':'de19: I often do things without stopping to think if I will get in trouble for it.',
                      'de20':'de20: I like to have fun when I can, even if I will get into trouble for it later.',
                      'f21':'f21: It is okay for me to lie (or not tell the truth) if it will keep my friends from getting in trouble with \n       parents, teachers or police.',
                      'f22':'f22: It is okay for me to lie (or not tell the truth) to someone if it will keep me from getting into trouble \n       with him or her.',
                      'f23':'f23: It is okay to steal something from someone who is rich and can easily replace it.',
                      'f24':'f24: It is okay to take little things from a store without paying for them because stores make so much money \n       that it won\'t hurt them.',
                      'f25':'f25: It is okay to beat people up if they hit me first.',
                      'f26':'f26: It is okay to beat people up if I do it to stand up for myself.',
                      'g27':'g27: If your friends told you not to do something because it was wrong, would you listen to them?',
                      'g28':'g28: If your friends told you not to do something because it was against the law, would you listen to them?',
                      'g29':'g29: If your friends were getting you into trouble at home, would you still hang out with them?',
                      'g30':'g30: If your friends were getting you into trouble at school, would you still hang out with them?',
                      'g31':'g31: If your friends were getting you into trouble with the police, would you still hang out with them?',
                      'h32':'h32: How many of your friends have skipped school without an excuse?',
                      'h33':'h33: How many of your friends have stolen something?',
                      'h34':'h34: How many of your friends have attacked someone with a weapon (like a knife or a gun)?',
                      'h35':'h35: How many of your friends have sold marijuana or other illegal drugs?',
                      'h36':'h36: How many of your friends have used cigarettes, tobacco or alcohol or marijuana or other illegal drugs?',
                      'h37':'h37: How many of your friends have belonged to a gang?',
                      't38':'t38: How many people in your family think that you will join a gang?',
                      't39':'t39: How many people in your family are gang members?'}

    def plot_importance(self, absolute = False):
        #make bar plot of the last row of A to visualize
        index=np.zeros(39)
        for i in range (39):
            index[i]=i
        if absolute == False:
            plt.bar(index,self.RF)
        else:
            plt.bar(index,abs(self.RF))
        plt.title('{}: Entries in Last Row of A'.format(self.name))
        plt.ylabel('Entry Values')
        plt.xticks(np.arange(min(index), max(index)+1, 1.0),self.label.keys(), rotation='vertical')
        plt.show()

    def plot_change(self, absolute = False):
        index=np.arange(39)
        if absolute == False:
            plt.bar(index,self.w_domeigvec)
        else:
            plt.bar(index,abs(self.w_domeigvec))
        plt.title('{}: Dominant Eigenvector Entries'.format(self.name))
        plt.xticks(np.arange(min(index), max(index)+1, 1.0),self.label.keys(),rotation='vertical')
        plt.show()

    def plot_eiglog(self):
        # plot of log of eigenvalues
        plt.scatter(self.eiglog.real,self.eiglog.imag)
        plt.title('{}: Log(eigenvalues)'.format(self.name))
        plt.xlabel('Real(Growth)')
        plt.ylabel('Imaginary(Frequency)')
        plt.show()

    def plot_change_importance(self, absolute = True, text = True, color = 'rainbow_r', xlim = [-2,2], ylim = [-4,4]):

        font = {'size': 10}
        plt.rc('font', **font)

        fig_size= plt.rcParams["figure.figsize"]
        fig_size[0] = 14
        fig_size[1] = 8

        plt.suptitle('{}: Change vs Importance'.format(self.name), size = 20)

        if absolute is False:
            scaled_ch = self.w_domeigvec
            scaled_im = self.RF
        else:
            scaled_ch = preprocessing.scale(abs(self.w_domeigvec))
            scaled_im = preprocessing.scale(abs(self.RF))

        colors = []
        c_dict = {'a':0,'b':1,'c':2,'d':3,'f':4,'g':5,'h':6,'t':7}
        for c in sorted(self.label.keys()):
            colors.append(c_dict[c[0]])
        colors = np.array(colors)

        # set up subplot grid
        gridspec.GridSpec(1,5)

        # large subplot
        if text is True:
            plt.subplot2grid((1,5), (0,0), colspan=2)

        plt.scatter(scaled_ch, scaled_im,c=colors, cmap=color, s=100)

        for i, txt in enumerate(sorted(self.label.keys())):
            plt.annotate(txt, (scaled_ch[i]+0.045, scaled_im[i]-0.05))

        if absolute is True:
            plt.xlim(xlim[0],xlim[1])
            plt.ylim(ylim[0],ylim[1])
        plt.axhline(0, color='red')
        plt.axvline(0, color='red')
        plt.xlabel('Standardized Weight of Entries in Dominant Eigenvector \n Change')
        plt.ylabel('Importance \n Standardized Weight of Entries in Last Row of A Matrix')

        if text is True:
            # small subplot
            plt.subplot2grid((1,5), (0,2), colspan = 3)
            # Remove the plot frame lines. They are unnecessary here.
            plt.gca().spines['top'].set_visible(False)
            plt.gca().spines['bottom'].set_visible(False)
            plt.gca().spines['right'].set_visible(False)
            plt.gca().spines['left'].set_visible(False)
            plt.axis('off')
            Text = '\n'.join(sorted(self.label.values()))
            plt.text(0,1,Text,horizontalalignment='left',verticalalignment='top')
        plt.savefig('{}_ch_im.png'.format(self.name))
        plt.show()


#-*- coding:utf-8 -*-
#!/usr/bin/env python
# @Auther: BoZheng

import os, sys
import csv
import random
import numpy
from sklearn.neighbors import KNeighborsClassifier
from itertools import islice

def loadData(data_path):

    toInt = lambda x: int(int(x) > 0)

    csvfile = open(data_path + "train.csv", "rb")
    reader = csv.reader(csvfile)
    trainData = [map(toInt, row[1:]) for row in islice(reader, 1, None)]
    csvfile.close()

    csvfile = open(data_path + "train.csv", "rb")
    reader = csv.reader(csvfile)
    trainLabel = [int(row[0]) for row in islice(reader, 1, None)]
    csvfile.close()

    csvfile = open(data_path + "test.csv", "rb")
    reader = csv.reader(csvfile)
    testData = [map(toInt, row) for row in islice(reader, 1, None)]
    csvfile.close()

    return (trainData, trainLabel, testData)

def KNNsolver(trainData, trainLabel, testData):
    solver = KNeighborsClassifier(n_neighbors = 5)
    solver.fit(trainData, trainLabel)
    result = solver.predict(testData)
    return result

def saveResult(result):
    csvfile = open("result.csv", "wb")
    writer = csv.writer(csvfile)
    writer.writerow(["ImageId", "Label"])
    for i, v in enumerate(result):
        writer.writerow([i+1, v])
    csvfile.close()

if __name__ == '__main__':
    trainData, trainLabel, testData = loadData('./data/')
    result = KNNsolver(trainData, trainLabel, testData)
    saveResult(result)