from __future__ import division
import cv2
import os
import numpy as np
import scipy
import pickle
import matplotlib.pyplot as plt
from itertools import islice

LIMIT = None

data_dictionary = {'0': 0, '1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '7': 7,
                   '8': 8, '9': 9, 'a': 10, 'b': 11,
                   'c': 12,
                   'd': 13, 'e': 14, 'f': 15, 'g': 16, 'h': 17, 'k': 18,
                   'l': 19,
                   'm': 20, 'n': 21, 'p': 22, 'q': 23,
                   'r': 24,
                   's': 25, 't': 26, 'u': 27, 'w': 28, 'x': 29, 'y': 30,
                   'z': 31}

GESTURE_FOLDER = 'gestures/'

def return_data():
    X = []
    y = []
    features = []

    for filename in os.listdir(os.path.join(GESTURE_FOLDER)):
        for file in os.listdir(os.path.join(GESTURE_FOLDER, filename)):
            if file.endswith('.jpg'):
                full_path = os.path.join(GESTURE_FOLDER, filename, file)
                X.append(full_path)
                y.append(data_dictionary[filename])

    for i in range(len(X)):
        img = plt.imread(X[i])
        features.append(img)

    features = np.array(features).astype('float32')
    labels = np.array(y).astype('float32')

    with open("features_numbers_letters", "wb") as f:
        pickle.dump(features, f, protocol=4)
    with open("labels_numbers_letters", "wb") as f:
        pickle.dump(labels, f, protocol=4)

return_data()


from sklearn.metrics import *
from rdflib import Graph, Namespace
import rdflib
import copy
from typing import List, Dict, Tuple
import pandas as pd
import numpy as np
import torch
import os
import random
import json
import time
import logging
from datetime import timedelta

class TrainingConfig(object):
    """
    BiLSTM模型的训练参数
    """
    batch_size = 64
    # 学习速率
    lr = 0.001
    epoches = 50
    print_step = 10

class LSTMConfig(object):
    """
    BiLSTM模型中LSTM模块的参数
    """
    emb_size = 128  # 词向量的维数
    hidden_size = 128  # lstm隐向量的维数

class BertTrainingConfig(object):
    """
    BiLSTM模型的训练参数
    """
    batch_size = 64
    # 学习速率
    lr = 0.00001
    other_lr = 0.0001
    epoches = 50
    print_step = 5

class BERTLSTMConfig(object):
    """
    BERT+BiLSTM模型中LSTM模块的参数
    """
    emb_size = 768  # 词向量的维数
    hidden_size = 512  # lstm隐向量的维数

def score_evalution(answers, predictions):
    """
    用avr_F1评价预测结果
    @param answers: {'0':'asd|sdf', '1':'qwe|wer' }
    @param predictions: {'0':'asd|sdf', '1':'qwe|wer' }
    @return: avr_F1
    """
    avr_F1 = 0
    for index, answer in answers.items():
        prediction = predictions[index]
        answer_list = answer.split('|')
        answer_set = set()
        for item in answer_list:
            answer_set.add(item)

        prediction_list = prediction.split('|')
        prediction_set = set()
        for item in prediction_list:
            prediction_set.add(item)

        intersection_set = answer_set.intersection(prediction_set)

        A = len(answer_set)
        G = len(prediction_set)
        if G==0 or len(intersection_set) == 0:
            avr_F1 += 0
            continue
        P = len(intersection_set)/(A * 1.0)
        R = len(intersection_set)/(G * 1.0)
        avr_F1 += (2 * P * R)/(P + R)
    avr_F1 /= len(answers)
    return avr_F1

def cal_lstm_crf_loss(crf_scores, targets, tag2id):
    """计算双向LSTM-CRF模型的损失
    该损失函数的计算可以参考:https://arxiv.org/pdf/1603.01360.pdf
    """
    targets_copy = copy.deepcopy(targets)
    pad_id = tag2id.get('<pad>')
    start_id = tag2id.get('<start>')
    end_id = tag2id.get('<end>')

    device = crf_scores.device

    # targets_copy:[B, L] crf_scores:[B, L, T, T]
    batch_size, max_len = targets_copy.size()
    target_size = len(tag2id)

    # mask = 1 - ((targets_copy == pad_id) + (targets_copy == end_id))  # [B, L]
    mask = (targets_copy != pad_id)
    lengths = mask.sum(dim=1)
    targets_copy = indexed(targets_copy, target_size, start_id)

    # # 计算Golden scores方法１
    # import pdb
    # pdb.set_trace()
    targets_copy = targets_copy.masked_select(mask)  # [real_L]

    flatten_scores = crf_scores.masked_select(
        mask.view(batch_size, max_len, 1, 1).expand_as(crf_scores)
    ).view(-1, target_size*target_size).contiguous()

    golden_scores = flatten_scores.gather(
        dim=1, index=targets_copy.unsqueeze(1)).sum()

    # 计算golden_scores方法２：利用pack_padded_sequence函数
    # targets_copy[targets_copy == end_id] = pad_id
    # scores_at_targets = torch.gather(
    #     crf_scores.view(batch_size, max_len, -1), 2, targets_copy.unsqueeze(2)).squeeze(2)
    # scores_at_targets, _ = pack_padded_sequence(
    #     scores_at_targets, lengths-1, batch_first=True
    # )
    # golden_scores = scores_at_targets.sum()

    # 计算all path scores
    # scores_upto_t[i, j]表示第i个句子的第t个词被标注为j标记的所有t时刻事前的所有子路径的分数之和
    scores_upto_t = torch.zeros(batch_size, target_size).to(device)
    for t in range(max_len):
        # 当前时刻 有效的batch_size（因为有些序列比较短)
        batch_size_t = (lengths > t).sum().item()
        if t == 0:
            scores_upto_t[:batch_size_t] = crf_scores[:batch_size_t,
                                                      t, start_id, :]
        else:
            # We add scores at current timestep to scores accumulated up to previous
            # timestep, and log-sum-exp Remember, the cur_tag of the previous
            # timestep is the prev_tag of this timestep
            # So, broadcast prev. timestep's cur_tag scores
            # along cur. timestep's cur_tag dimension
            scores_upto_t[:batch_size_t] = torch.logsumexp(
                crf_scores[:batch_size_t, t, :, :] +
                scores_upto_t[:batch_size_t].unsqueeze(2),
                dim=1
            )
    all_path_scores = scores_upto_t[:, end_id].sum()

    # 训练大约两个epoch loss变成负数，从数学的角度上来说，loss = -logP
    loss = (all_path_scores - golden_scores) / batch_size
    return loss

def tensorized(batch, maps):
    """
    BiLSTM训练使用
    @param batch:
    @param maps:
    @return:
    """
    PAD = maps.get('<pad>')
    UNK = maps.get('<unk>')

    max_len = len(batch[0])
    batch_size = len(batch)

    batch_tensor = torch.ones(batch_size, max_len).long() * PAD
    for i, l in enumerate(batch):
        for j, e in enumerate(l):
            batch_tensor[i][j] = maps.get(e, UNK)
    # batch各个元素的长度
    lengths = [len(l) for l in batch]

    return batch_tensor, lengths

def tensorized_bert(batch, maps):
    """
    BiLSTM训练使用
    @param batch:
    @param maps:
    @return:
    """
    PAD = maps.get('[PAD]')
    UNK = maps.get('[UNK]')

    max_len = len(batch[0])
    batch_size = len(batch)

    batch_tensor = torch.ones(batch_size, max_len).long() * PAD
    for i, l in enumerate(batch):
        for j, e in enumerate(l):
            batch_tensor[i][j] = maps.get(e, UNK)
    # batch各个元素的长度
    lengths = [len(l) for l in batch]

    return batch_tensor, lengths

def sort_by_lengths(word_lists, tag_lists):
    pairs = list(zip(word_lists, tag_lists))
    indices = sorted(range(len(pairs)),
                     key=lambda k: len(pairs[k][0]),
                     reverse=True)
    pairs = [pairs[i] for i in indices]
    # pairs.sort(key=lambda pair: len(pair[0]), reverse=True)

    word_lists, tag_lists = list(zip(*pairs))

    return word_lists, tag_lists, indices

def indexed(targets, tagset_size, start_id):
    """将targets中的数转化为在[T*T]大小序列中的索引,T是标注的种类"""
    batch_size, max_len = targets.size()
    for col in range(max_len-1, 0, -1):
        targets[:, col] += (targets[:, col-1] * tagset_size)
    targets[:, 0] += (start_id * tagset_size)
    return targets

def get_operator(question):
    """
    得到句子的约束算子，只针对min\max的算子，对算子为‘=’或者空的情况均返回None
    @param question: 用户问题（str）
    @return: 约束算子 和 约束属性
    """
    operater, obj = None, None
    if '最多' in question:
        operater, obj = 'max', '流量'
    elif '最少' in question:
        operater, obj = 'min', '流量'
    elif '最便宜的' in question or '最实惠的' in question:
        operater, obj = 'min', '价格'
    elif '最贵' in question:
        operater, obj = 'max', '价格'
    return operater, obj

def parse_triples_file(file_path):
    """
    解析triples.txt为triples.rdf
    """
    triples = []
    # read raw_triples
    with open(file_path, 'r', encoding='utf-8') as f:
        for line in f:
            line = line.strip()
            line = line.replace(' ', '')
            line_block = line.split('	')
            t = []
            flag = True
            for block in line_block:
                # 调整三元组格式与训练文件一致
                block = block.replace('档位介绍表', '档位介绍')
                # 三元组改成小写
                block = block.lower()
                if block.find('http://yunxiaomi.com/kbqa') != -1:
                    _ = block.find('_')
                    t.append('http://yunxiaomi.com/'+block[_+1: -1])
                else:
                    block = block[1:-1]
                    if '|' in block:
                        blocks = block.split('|')
                        for b in blocks:
                            flag = False
                            copy_t = copy.deepcopy(t)
                            copy_t.append('http://yunxiaomi.com/'+b)
                            triples.append(copy_t)
                    else:
                        t.append('http://yunxiaomi.com/'+block)
            if flag:
                triples.append(t)
    # transfer to RDF file
    graph = Graph()
    for triple in triples:
        if triple is None:
            print('None')
            continue
        graph.add((rdflib.term.URIRef(u'{}'.format(triple[0])),
                   rdflib.term.URIRef(u'{}'.format(triple[1])),
                   rdflib.term.URIRef(u'{}'.format(triple[2]))))
    graph.serialize('../data/process_data/triples.rdf', format='n3')
    graph.close()

def to_sparql(entity: str, main_property: str, sub_properties: List[Tuple] = None):
    """
    将实体信息等转化成sparql语句
    :param entity: str
    :param main_property: str
    :param sub_properties: List[Tuple[key, value], ...]
    :return: str
    """
    if sub_properties is None:
        sub_properties = []
    prefix = '<http://yunxiaomi.com/{}>'
    # relations第0个元素存放主要关系， 之后存放次要关系
    relations, conditions = [], ''
    # ”档位介绍-取消方式“ 类场景
    if main_property.find('-') != -1:
        relations.append(main_property.split('-')[0])
        relations.append(main_property.split('-')[1])
    # “生效规则” 类场景
    else:
        relations.append(main_property)
    # 填充关系到sparql
    for i, r in enumerate(relations):
        condition = ''
        if i == 0:
            condition = prefix.format(entity) + ' ' + prefix.format(relations[i])
            if len(relations) > 1:
                condition += ' ?instance'
            else:
                condition += ' ?ans'
        elif i == 1:
            condition += '?instance ' + prefix.format(relations[i]) + ' ?ans'
        # else:
        #     condition += '?instance ' + prefix.format(relations[i]) + ' ' + prefix.format(relations[i])
        if len(relations)-1 != i or (len(relations) > 1 and len(sub_properties) > 0):
            condition += '. '
        conditions += condition
    idx = 0
    if len(relations) > 1:
        for key, value in sub_properties:
            condition = '?instance ' + prefix.format(key) + ' ' + prefix.format(value)
            if len(sub_properties) - 1 > idx:
                condition += '. '
            conditions += condition
            idx += 1
    s = """select ?ans where {%s}""" % (conditions, )
    return s

def make_dataset(data_path, target_file, label_file, train=True):
    if isinstance(data_path, list):
        df = pd.DataFrame(pd.read_excel(data_path[0]))
        for i in range(1, len(data_path)):
            df = pd.concat([df, pd.DataFrame(pd.read_excel(data_path[i]))], ignore_index=True)
    else:
        df = pd.DataFrame(pd.read_excel(data_path))
    # # entity_map
    # with open('../data/file/entity_map.json', 'r', encoding='utf-8') as f:
    #     entity_mapping = json.load(f)
    # 标签和id的映射
    ans_label2id = {}
    prop_label2id = {}
    entity_label2id = {}
    # 开通方式、条件
    binary_label2id = {'档位介绍-开通方式': 0, '档位介绍-开通条件': 1}
    # 标注数据xlxs转化成txt
    if train:
        df = df.loc[:, ['用户问题', '答案类型', '属性名', '实体', '答案', '有效率']]
    # 未标注数据xlxs转化成txt
    else:
        length = []
        columns = df.columns
        for column in columns:
            length.append(len(str(df.iloc[1].at[column])))
        max_id = np.argmax(length)
        # df = df.loc[:, ['query']]
        df = df.loc[:, [columns[max_id]]]
    # 转换数据
    with open(target_file, 'w', encoding='utf-8') as f:
        for i in range(len(df)):
            line = df.loc[i]
            line = list(line)
            for idx in range(len(line)):
                # 大写字母改成小写
                line[idx] = str(line[idx]).strip().lower()
            if train:
                # 答案类型
                if line[1] not in ans_label2id:
                    ans_label2id[line[1]] = len(ans_label2id)
                # 属性名
                sub_blocks = line[2].split('|')
                for j, sub_b in enumerate(sub_blocks):
                    # 把这几类统一变成”其他“类
                    # if sub_b in ('适用app', '生效规则', '叠加规则', '封顶规则'):
                    #     sub_b = '其他'
                    #     sub_blocks[j] = sub_b
                    if sub_b not in prop_label2id:
                        prop_label2id[sub_b] = len(prop_label2id)
                # line[2] = '|'.join(list(set(sub_blocks)))
                # # 实体
                # if line[3] in entity_mapping:
                #     line[3] = entity_mapping[line[3]]
                if line[3] not in entity_label2id:
                    entity_label2id[line[3]] = len(entity_label2id)
            f.write('\t'.join(line)+'\n')
    # 整理标签和id的映射
    if train:
        if label_file is not None:
            with open(label_file, 'w', encoding='utf-8') as f:
                json.dump({'ans_type': ans_label2id, 'main_property': prop_label2id, 'entity': entity_label2id,
                           'binary_type': binary_label2id},
                          fp=f,
                          ensure_ascii=False, indent=2)

def make_dataset_for_binary(data_path, target_file):
    """
    为二分类任务制作数据集
    :param data_path:
    :param target_file:
    :return:
    """
    if isinstance(data_path, list):
        df = pd.DataFrame(pd.read_excel(data_path[0]))
        for i in range(1, len(data_path)):
            df = pd.concat([df, pd.DataFrame(pd.read_excel(data_path[i]))], ignore_index=True)
    else:
        df = pd.DataFrame(pd.read_excel(data_path))
    # 标注数据xlxs转化成txt
    df = df.loc[:, ['用户问题', '答案类型', '属性名', '实体', '答案', '有效率']]
    method_count, condition_text = 0, []
    # 转换数据
    with open(target_file, 'w', encoding='utf-8') as f:
        for i in range(len(df)):
            line = df.loc[i]
            if str(line['用户问题']) in ('20元20g还可以办理吗', ):
                continue
            props = str(line['属性名'])
            line = [str(line['用户问题']), '', str(line['有效率'])]
            if '开通方式' in props and '开通条件' not in props:
                line[1] = '档位介绍-开通方式'
                f.write('\t'.join(line)+'\n')
                method_count += 1
            elif '开通方式' not in props and '开通条件' in props:
                line[1] = '档位介绍-开通条件'
                f.write('\t'.join(line)+'\n')
                condition_text.append(line)
        # 中和结果,重采样
        if method_count > len(condition_text):
            for i in range(method_count - len(condition_text)):
                line = random.choice(condition_text)
                f.write('\t'.join(line)+'\n')

def label_to_multi_hot(max_len, label_ids):
    """
    转化成multi-hot编码
    :param max_len: multi-hot编码的最大长度
    :param label_ids:
    :return:
    """
    multi_hot = [0] * max_len
    for idx in label_ids:
        multi_hot[idx] = 1
    return multi_hot

def logits_to_multi_hot(data: torch.Tensor, ans_pred: torch.Tensor, label_hub, threshold=0.5):
    """
    logits转化成multi-hot编码
    1 运用规则： 当并列句中，属性名肯定有两个，而其他句子中属性名只有一个
    :return:
    """
    if data.is_cuda:
        data = data.detach().data.cpu().numpy()
    else:
        data = data.detach().numpy()
    result = []
    for i in range(data.shape[0]):
        temp = [0] * data.shape[1]
        if label_hub.ans_id2label[ans_pred[i].item()] == '并列句':
            # 获取分数最高的两个属性
            max_idx = np.argmax(data[i], axis=-1)
            data[i][max_idx] = -1e5
            temp[max_idx] = 1
            max_idx = np.argmax(data[i], axis=-1)
            temp[max_idx] = 1
        else:
            # 获取分数最高的1个属性
            max_idx = np.argmax(data[i], axis=-1)
            temp[max_idx] = 1
        result.append(temp)
    return np.array(result)

def logits_to_multi_hot_old_version(data: torch.Tensor, threshold=0.5):
    """
    logits转化成multi-hot编码【没有考虑答案类型的旧版本】
    :return:
    """
    if data.is_cuda:
        data = data.detach().data.cpu().numpy()
    else:
        data = data.numpy()
    result = []
    for i in range(data.shape[0]):
        result.append([1 if v >= threshold else 0 for v in list(data[i])])
    return np.array(result)

def remove_stop_words(sentence: str, stopwords: List):
    """
    移除停用词
    :param sentence:
    :param stopwords:
    :return:
    """
    for word in stopwords:
        sentence = sentence.replace(word, '')
    return sentence

def load_model(model_path, map_location=None):
    print('模型加载中...')
    model = torch.load(model_path, map_location=map_location)
    return model

def save_model(model, model_path, model_name, debug='1'):
    # debug模式下，需要删除之前同名的模型
    if debug == '1':
        for file in os.listdir(model_path):
            # 删除同一个模型
            if model_name[:-20] in file:
                os.remove(os.path.join(model_path, file))
    print('模型保存中...')
    torch.save(model, os.path.join(model_path, model_name))

def setup_seed(seed):
    """
    确定随机数
    :param seed: 种子
    :return:
    """
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    random.seed(seed)
    np.random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)  # 为了禁止hash随机化，使得实验可复现。

def split_train_dev(data_size: int, radio=0.8):
    """
    从标注数据中随机划分训练与验证集，返回标注数据中的行号列表
    :param data_size:
    :param radio:
    :return: [[id1, id2, id3...], [idx,... ]]
    """
    line_ids = [i for i in range(data_size)]
    random.shuffle(line_ids)
    train_ids = line_ids[:int(data_size*radio)]
    dev_ids = line_ids[int(data_size*radio):]
    with open('../data/file/train_dev_ids.json', 'w', encoding='utf-8') as f:
        json.dump({'train_ids': train_ids, 'dev_ids': dev_ids}, f, ensure_ascii=False, indent=2)

def split_labeled_data(source_file, train_file, dev_file):
    """
    根据训练与验证的ID，写入文件
    :param source_file:
    :param train_file:
    :param dev_file:
    :return:
    """
    train_dev_ids = json.load(open('../data/file/train_dev_ids.json', 'r', encoding='utf-8'))
    train_dev_ids['dev_ids'] = set(train_dev_ids['dev_ids'])
    train_data, dev_data = [], []
    with open(source_file, 'r', encoding='utf-8') as f:
        for i, line in enumerate(f):
            if i in train_dev_ids['dev_ids']:
                dev_data.append(line.strip())
            else:
                train_data.append(line.strip())
    with open(train_file, 'w', encoding='utf-8') as f:
        f.write('\n'.join(train_data))
    with open(dev_file, 'w', encoding='utf-8') as f:
        f.write('\n'.join(dev_data))

def get_time_str():
    """
    返回当前时间戳字符串
    :return:
    """
    return time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime(time.time()))

def get_time_dif(start_time):
    """
    获取已经使用的时间
    :param start_time:
    :return:
    """
    end_time = time.time()
    time_dif = end_time - start_time
    return timedelta(seconds=int(round(time_dif)))

def get_labels(ans_pred: torch.Tensor, prop_pred: np.ndarray, entity_pred: torch.Tensor, label_hub):
    """
    根据标签ID返回标签名称
    :param ans_pred:
    :param prop_pred:
    :param entity_pred:
    :param label_hub:
    :return:
    """
    ans_labels, prop_labels, entity_labels = [], [], []
    for ans_id in ans_pred:
        ans_labels.append(label_hub.ans_id2label[ans_id.item()])
    for entity_id in entity_pred:
        entity_labels.append(label_hub.entity_id2label[entity_id.item()])
    for line in prop_pred:
        temp = []
        for i in range(len(line)):
            if line[i] == 1:
                temp.append(label_hub.prop_id2label[i])
        prop_labels.append(temp)
    return ans_labels, prop_labels, entity_labels

def fetch_error_cases(pred_data: Dict, gold_data: Dict, question: Dict):
    """
    获取预测错误的样本
    :param pred_data:
    :param gold_data:
    :return:
    """
    time_str = get_time_str()
    time_str = time_str.replace(':', '：')
    count = 1
    with open('../data/results/error_case_{}.txt'.format(time_str), 'w', encoding='utf-8') as f:
        for idx, res in pred_data['result'].items():
            ans = gold_data[idx]
            ans = set(ans.split('|'))
            if '' in ans:
                ans.remove('')
            res = set(res.split('|'))
            if ans != res:
                f.write('第{}个错误结果\n'.format(count))
                f.write('问题编号为: {}, 问题为：{}\n'.format(idx, question[idx]))
                f.write('正确答案：{}\n'.format(ans))
                f.write('预测答案：{}\n'.format(res))
                f.write('预测中间结果：{}\n'.format(pred_data['model_result'][idx]))
                f.write('【问题分析】：【】\n')
                f.write('\n')
                count += 1

def k_fold_data(data: List[Dict], k=5, batch_size=32, seed=1, collate_fn='1'):
    """
    data即位DataSet类中的data。将data分成k份，然后组成训练验证集
    :param data:
    :param k:
    :param batch_size:
    :param seed: 随机数种子
    :collate_fn:
    :return:
    """
    print('K折数据划分中...')
    from data import BertDataset, bert_collate_fn, binary_collate_fn
    from torch.utils.data import DataLoader
    temp_data = copy.deepcopy(data)
    random.seed(seed)
    random.shuffle(temp_data)
    block_len = len(temp_data)//k
    data_blocks = [temp_data[i*block_len: (i+1)*block_len] for i in range(k)]
    train_dev_tuples = []
    for i, block in enumerate(data_blocks):
        train, dev = [], block
        for _ in range(k):
            if _ != i:
                train += data_blocks[_]
        train_set = BertDataset(train_file=None, tokenizer=None, label_hub=None, init=False)
        dev_set = BertDataset(train_file=None, tokenizer=None, label_hub=None, init=False)
        train_set.data = train
        dev_set.data = dev
        if collate_fn == '1':
            train_dev_tuples.append((DataLoader(train_set, batch_size=batch_size, shuffle=True, collate_fn=bert_collate_fn),
                                     DataLoader(dev_set, batch_size=batch_size, shuffle=True, collate_fn=bert_collate_fn)))
        else:
            train_dev_tuples.append(
                (DataLoader(train_set, batch_size=batch_size, shuffle=True, collate_fn=binary_collate_fn),
                 DataLoader(dev_set, batch_size=batch_size, shuffle=True, collate_fn=binary_collate_fn)))
    return train_dev_tuples

def filter_sub_properties(sub_properties: List[Tuple], entity_label: str, ans_label: str):
    """
    过滤 sub_properties
    去掉 实体中流量或价钱被预测成约束的情况 或 去掉重复的约束属性名
    :param sub_properties:
    :param entity_label:
    :param ans_label:
    :return:
    """
    sub_properties_filter = []
    sub_property_set = set()
    for sub_property in sub_properties:
        if sub_property[0] not in sub_property_set:
            sub_property_set.add(sub_property[0])
        # 重复的约束属性名而且答案类型是属性值
        elif ans_label == '属性值':
            continue
        if sub_property[0] == '子业务':
            sub_properties_filter.append(sub_property)
        elif sub_property[1] in entity_label:
            if entity_label == '1元5gb流量券':  # 该实体中 1元 是约束条件
                sub_properties_filter.append(sub_property)
            else:
                continue
        else:
            sub_properties_filter.append(sub_property)
    return sub_properties_filter

def reverse_prop(prop_label: List[str]):
    """
    置换标签，在查找答案为空的时候使用
    :param prop_label:
    :return:
    """
    changed = False
    for i, prop in enumerate(prop_label):
        # 假设以下条件独立
        if '方式' in prop:
            prop_label[i] = prop.replace('方式', '条件')
            changed = True
        elif '条件' in prop:
            prop_label[i] = prop.replace('条件', '方式')
            changed = True
        elif '档位介绍-有效期规则' == prop:
            prop_label[i] = '生效规则'
            changed = True
        elif '生效规则' == prop:
            prop_label[i] = '档位介绍-有效期规则'
            changed = True
    return changed

def reverse_entity(entity_label: str):
    """
    置换属性名，在查找答案为空的时候使用
    :param entity_label:
    :return:
    """
    if entity_label == '嗨购月包':
        return True, '嗨购产品'
    elif entity_label == '嗨购产品':
        return True, '嗨购月包'
    if entity_label == '北京移动plus会员权益卡':
        return True, '北京移动plus会员'
    elif entity_label == '北京移动plus会员':
        return True, '北京移动plus会员权益卡'
    else:
        return False, ''

def rm_symbol(sentence):
    import re
    return re.sub(',|\.|，|。|？|\?|！|!|：', '', sentence)

def judge_cancel(question):
    """
    判断“取消”属性名
    :param question:
    :return:
    """
    import re
    neg_word, pos_word = ['不要', '不想', '取消', '不需要', '不用'], ['办理', '需要', '开通']
    sentence_blocks = re.split(',|\.|：|，|。', question)
    for block in sentence_blocks:
        for n in neg_word:
            if n in block:
                for p in pos_word:
                    if p in block:
                        return True
    return False

def rules_to_judge_prop(question):
    """
    规则判断属性名：适用app、叠加规则、封顶规则、档位介绍-带宽、档位介绍-有效期规则、生效规则
    :param question:
    :return:
    """
    # 适用app：当前只需要判断是否包含“app”和“适用”关键字
    if 'app' in question and '适用' in question:
        return '适用app'
    # 生效规则：判断包含“生效”并带疑问关键字
    if '生效' in question and ('么' in question or '吗' in question or '啥' in question):
        return '生效规则'
    # 叠加规则：包含“可以叠加”关键字。注意与“叠加包”区别
    if '可以叠加' in question and '叠加包' not in question:
        return '叠加规则'
    # 封顶规则：包含“限速”或“上限”关键字，不能包含“解除”、”恢复“关键字
    if ('限速' in question or '上限' in question) and ('解除' not in question and '恢复' not in question):
        return '封顶规则'
    # 有效期规则
    if ('到期' in question or '有效期' in question) and ('办理' not in question and '取消' not in question and '关闭' not in question):
        return '档位介绍-有效期规则'
    # 带宽
    if '通带宽' in question or '网速' in question:
        return '档位介绍-带宽'
    return None

def rules_to_judge_entity(question, predict_entity):
    """
    规则来判断实体
    :param question:
    :param predict_entity
    :return:
    """
    if predict_entity == '畅享套餐':
        if '升档' in question:
            return '新畅享套餐升档优惠活动'
        elif '促销' in question or '78元无限流量套餐' in question:
            return '畅享套餐促销优惠活动'
        elif '新全球通' in question:
            return '新全球通畅享套餐'
        elif '首充活动' in question:
            return '畅享套餐首充活动'
    elif predict_entity == '移动王卡':
        if '惠享合约' in question:
            return '移动王卡惠享合约'
    elif predict_entity == '5g畅享套餐':
        if '合约版' in question:
            return '5g畅享套餐合约版'
    elif predict_entity == '30元5gb包':
        if '半价' in question:
            return '30元5gb半价体验版'
    elif predict_entity == '移动花卡':
        if '新春升级' in question:
            return '移动花卡新春升级版'
    elif predict_entity == '随心看会员':
        if '合约版' in question:
            return '随心看会员合约版'
    elif predict_entity == '北京移动plus会员':
        if '权益卡' in question:
            return '北京移动plus会员权益卡'
    elif predict_entity == '5g智享套餐':
        if '合约版' in question:
            return '5g智享套餐合约版'
        elif '家庭版' in question:
            return '5g智享套餐家庭版'
    elif predict_entity == '全国亲情网':
        if '亲情网免费' in question:
            return '全国亲情网功能费优惠活动'
    elif predict_entity == '精灵卡':
        if '首充' in question and '优惠' in question:
            return '精灵卡首充优惠活动'
    # 合并的样本外的样本
    else:
        if '无忧' in question:
            return '流量无忧包'
    return None

class MyLogger(object):
    def __init__(self, log_file, debug='0'):
        self.ch = logging.StreamHandler()
        self.formatter = logging.Formatter("%(asctime)s - %(message)s")
        self.fh = logging.FileHandler(log_file, mode='w')
        self.logger = logging.getLogger()
        self.debug = debug
        self.init()

    def init(self):
        self.logger.setLevel(logging.INFO)
        # 输出到文件
        self.fh.setLevel(logging.INFO)
        self.fh.setFormatter(self.formatter)
        # 输出到控制台
        self.ch.setLevel(logging.INFO)
        self.ch.setFormatter(self.formatter)
        self.logger.addHandler(self.ch)
        self.logger.addHandler(self.fh)

    def log(self, message):
        if self.debug == '1':
            self.logger.info(message)

def generate_false_label(predict_file):
    """
    根据预测的结果反向生成同名伪标签xlsx
    :param predict_file
    :return
    """
    result_path = '../data/results/'
    with open(result_path+predict_file, 'r', encoding='utf-8') as f:
        predict_dict = json.load(f)
    predict_results = []
    for i, predict_entity in predict_dict['model_result'].items():
        instance = []
        for key, value in predict_entity.items():
            if key == 'main_property':
                instance.append('|'.join(value))
            elif key == 'sub_properties':
                cons_prop, cons_value = [], []
                for pair in value:
                    cons_prop.append(pair[0])
                    cons_value.append(str(pair[1]))
                instance.append('|'.join(cons_prop))
                instance.append('|'.join(cons_value))
            else:
                if key == 'operator' and value == 'other':
                    instance.append('')
                else:
                    instance.append(value)
        instance.append(predict_dict['result'][i])
        predict_results.append(instance)
    header = ['用户问题', '答案类型', '实体', '属性名', '约束算子', '约束属性名', '约束属性值', '答案']
    pd.DataFrame(predict_results, index=None, columns=header).to_excel(result_path+predict_file[:-5]+'.xlsx', index=False)
    print('预测结果写入xlsx成功')

def view_gpu_info(gpu_id: int):
    import pynvml
    pynvml.nvmlInit()
    handle = pynvml.nvmlDeviceGetHandleByIndex(gpu_id)
    mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle)
    # 这里是字节bytes，所以要想得到以兆M为单位就需要除以1024**2
    print('当前为第{}号显卡，总显存数：{}MB；显存使用数：{}MB；显存剩余数：{}MB'.format(gpu_id, mem_info.total/1024**2, mem_info.used/1024**2, mem_info.free/1024**2))  # 第二块显卡总的显存大小
    return mem_info.total/1024**2, mem_info.used/1024**2, mem_info.free/1024**2

def max_count(data: dict, k=1):
    """
    对于存在"标签"->"个数"的字典data，得到最大数量的标签
    :param data:
    :param k: 前k个统计
    :return:
    """
    max_count_, la = 0, None
    second_count_, la2 = 0, None
    for lab, c in data.items():
        if c > max_count_:
            max_count_ = c
            la = lab
    # 只有在标签个数至少为2时才有第二多的标签，否则直接返回空结果
    if k == 2 and len(data) > 1:
        for lab, c in data.items():
            if c > second_count_ and lab != la:
                second_count_ = c
                la2 = lab
    if k == 1:
        return la, max_count_
    else:
        return la, max_count_, la2, second_count_

def vote_integration(ans_logits: List[torch.Tensor],
                     prop_logits: List[torch.Tensor],
                     entity_logits: List[torch.Tensor],
                     label_hub,
                     batch_size: int):
    """
    模型投票融合
    :param ans_logits:
    :param prop_logits:
    :param entity_logits:
    :param label_hub:
    :param batch_size:
    :return: 最终的标签
    """
    vote_method = 2
    mid_results = {'ans': [], 'prop': [], 'entity': []}
    for i in range(batch_size):
        mid_results['ans'].append({})
        mid_results['prop'].append({})
        mid_results['entity'].append({})
    ans_results, prop_results, entity_results = [], [], []
    for i in range(len(ans_logits)):
        ans_pred = ans_logits[i].data.cpu().argmax(dim=-1)
        prop_pred = logits_to_multi_hot(prop_logits[i], ans_pred, label_hub)
        entity_pred = entity_logits[i].data.cpu().argmax(dim=-1)
        ans_labels, prop_labels, entity_labels = get_labels(ans_pred, prop_pred, entity_pred, label_hub)
        for j in range(batch_size):
            if ans_labels[j] not in mid_results['ans'][j]:
                mid_results['ans'][j][ans_labels[j]] = 0
            mid_results['ans'][j][ans_labels[j]] += 1
            # prop投票1: key是模型预测的直接结果
            if vote_method == 1:
                prop_label = tuple(sorted(prop_labels[j]))
                if prop_label not in mid_results['prop'][j]:
                    mid_results['prop'][j][prop_label] = 0
                mid_results['prop'][j][prop_label] += 1
            # prop投票2: key是模型预测的每个个体标签
            else:
                prop_label = prop_labels[j]
                for lab in prop_label:
                    if lab not in mid_results['prop'][j]:
                        mid_results['prop'][j][lab] = 0
                    mid_results['prop'][j][lab] += 1
            if entity_labels[j] not in mid_results['entity'][j]:
                mid_results['entity'][j][entity_labels[j]] = 0
            mid_results['entity'][j][entity_labels[j]] += 1
    for instance in mid_results['ans']:
        ans_results.append(max_count(instance)[0])
    if vote_method == 1:
        for i, instance in enumerate(mid_results['prop']):
            prop_results.append(max_count(instance)[0])
    else:
        for i, instance in enumerate(mid_results['prop']):
            ans = ans_results[i]
            la, max_count_, la2, second_count_ = max_count(instance, k=2)
            if ans == '并列句':
                if second_count_ <= len(ans_logits)//3:
                    print('-------------------出现了不应该发生的情况-------------------')
                    ans_results[i] = '属性值'
                    prop_results.append([la])
                else:
                    prop_results.append([la, la2])
            else:
                prop_results.append([la])
                if second_count_ > len(ans_logits)//2:
                    prop_results[-1].append(la2)
    for instance in mid_results['entity']:
        entity_results.append(max_count(instance)[0])
    return ans_results, prop_results, entity_results

def average_integration(ans_logits: List[torch.Tensor],
                        prop_logits: List[torch.Tensor],
                        entity_logits: List[torch.Tensor],
                        label_hub):
    """
    模型平均融合
    :param ans_logits:
    :param prop_logits:
    :param entity_logits:
    :param label_hub:
    :return: 最终的标签
    """
    local_ans_logits, local_prop_logits, local_entity_logits = torch.stack(ans_logits).mean(dim=0), torch.stack(prop_logits).mean(dim=0), torch.stack(entity_logits).mean(dim=0)
    ans_pred = local_ans_logits.data.cpu().argmax(dim=-1)
    prop_pred = logits_to_multi_hot(local_prop_logits, ans_pred, label_hub)
    entity_pred = local_entity_logits.data.cpu().argmax(dim=-1)
    ans_labels, prop_labels, entity_labels = get_labels(ans_pred, prop_pred, entity_pred, label_hub)
    return ans_labels, prop_labels, entity_labels

def binary_average_integration(logits: List[torch.Tensor], label_hub):
    """
    二进制模型的平均融合
    :param logits:
    :param label_hub:
    :return:
    """
    local_logits = torch.stack(logits).mean(dim=0)
    pred = local_logits.data.cpu().argmax(dim=-1)
    labels = []
    for label_id in pred:
        labels.append(label_hub.binary_id2label[label_id.item()])
    return labels

def tqdm_with_debug(data, debug=None):
    """
    将tqdm加入debug模式
    :param data:
    :param debug:
    :return:
    """
    from tqdm import tqdm
    if debug == '1' or debug is True:
        if isinstance(data, enumerate):
            temp_data = list(data)
        else:
            temp_data = data
        len_data = len(temp_data)
        ans = tqdm(temp_data, total=len_data)
        return ans
    else:
        return data

# def entity_map():
#     a = {'新畅享套餐升档优惠活动': '畅享套餐', '畅享套餐促销优惠活动': '畅享套餐', '畅享套餐促销优惠': '畅享套餐', '新全球通畅享套餐': '畅享套餐', '畅享套餐首充活动': '畅享套餐',
#          '移动王卡惠享合约': '移动王卡', '5g畅享套餐合约版': '5g畅享套餐', '30元5gb半价体验版': '30元5gb包', '移动花卡新春升级版': '移动花卡', '随心看会员合约版': '随心看会员', '北京移动plus会员权益卡': '北京移动plus会员',
#          '5g智享套餐合约版': '5g智享套餐', '5g智享套餐家庭版': '5g智享套餐',
#          '全国亲情网功能费优惠活动': '全国亲情网', '精灵卡首充优惠活动': '精灵卡'}
#     with open('../data/file/entity_map.json', 'w', encoding='utf-8') as f:
#         json.dump(a, f, ensure_ascii=False, indent=2)

if __name__ == '__main__':
    # parse_triples_file('../data/raw_data/triples.txt')
    # print(to_sparql(entity='视频会员通用流量月包', main_property='档位介绍-上线时间', sub_properties={'价格': '70', '子业务': '优酷会员'}))
    # make_dataset(['../data/raw_data/train_augment_few_nlpcda.xlsx',
    #               '../data/raw_data/train_augment_simbert.xlsx',
    #               '../data/raw_data/train_augment_synonyms.xlsx'],
    #              target_file='../data/dataset/augment3.txt',
    #              label_file=None,
    #              train=True)
    make_dataset_for_binary(['../data/raw_data/train_denoised.xlsx'],
                            target_file='../data/dataset/binary_labeled.txt')
    make_dataset_for_binary(['../data/raw_data/train_augment_few_nlpcda.xlsx',
                             '../data/raw_data/train_augment_simbert.xlsx',
                             '../data/raw_data/train_augment_synonyms.xlsx'],
                            target_file='../data/dataset/binary_augment3.txt')
    # make_dataset(['../data/raw_data/test2_denoised.xlsx'],
    #              target_file='../data/dataset/cls_unlabeled2.txt',
    #              label_file=None,
    #              train=False)
    # split_train_dev(5000)
    # split_labeled_data(source_file='../data/dataset/cls_labeled.txt',
    #                    train_file='../data/dataset/cls_train.txt',
    #                    dev_file='../data/dataset/cls_dev.txt')
    # ------------------获取错误答案详情---------------
    # with open('../data/results/ans_dev.json', 'r', encoding='utf-8') as f:
    #     pred = json.load(f)
    # with open('../data/dataset/cls_dev.txt', 'r', encoding='utf-8') as f:
    #     answer, question = {}, {}
    #     for i, line in enumerate(f):
    #         ans = line.strip().split('\t')[-1]
    #         answer[str(i)] = ans
    #         question[str(i)] = line.strip().split('\t')[0]
    # fetch_error_cases(pred, answer, question)
    # generate_false_label('ensemble_bert_aug2_use_efficiency_2021-07-20-08-29-22_seed1_fi0_gpu2080_be81_0.95411990.pth.json')
    # make_dataset(['../data/raw_data/test2_denoised.xlsx'],
    #              target_file='../data/dataset/cls_unlabeled.txt',
    #              label_file=None,
    #              train=False)
    # entity_map()


import os
import time
import psutil
import urllib.request as urllib2
import smtplib
import schedule
from sys import *
from email import encoders
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart

def is_connected():
    try:
        urllib2.urlopen('http://216.58.192.142',timeout = 1)
        return True
    except urllib2.URLError as err:
        return False

def MailSender(filename,time):
    try:
        fromaddr = "shubhamborse0911@gmail.com"
        toaddr = "borse911shubham@gmail.com"

        msg = MIMEMultipart()

        msg['From'] = fromaddr

        msg['To'] = toaddr

        body = """
        Hello %s,
        Welcome to Marvellous Infosystems.
        Please find attached document which contains Log Running Process.
        Log file is created at : %s

        This is auto generated mail.

        Thanks & Regards,
        Shubham Shyam Borse
        Marvellous Infosystems
          """%(toaddr,time)

        Subject = """
        Marvellous Infosystems Process log generated at : %s
        """%(time)

        msg['Subject'] = Subject

        msg.attach(MIMEText(body,'plan'))

        attachment = open(filename,'rb')

        p = MIMEBase('application', 'octet-stream')

        p.se_payload((attachment).read())

        encoders.encode_base64(p)

        p.add_header('Content-Disposition', "attachment; filename = %s" % filename)

        msg.attach(p)

        s = smtplib.SMTP('smtp.gmail.com',587)

        s.starttls()

        s.login(fromaddr,"cgxp qnya iogr hgpv")

        text = msg.as_string()

        s.sendmail(fromaddr,toaddr,text)

        s.quit()

        print("Log file succesfully sent through Mail")

    except Exception as E:
        print ("Unable to send mail.",E)

def ProcessLog(log_dir = 'Marvellous'):
    listprocess = []

    if not os.path.exists(log_dir):
        try:
            os.mkdir(log_dir)
        except:
            pass

    separator = "-" * 80
    log_path = os.path.join(log_dir,"Marvellous%s.txt"%(time.ctime()))
    f = open(log_path,'w')
    f.write(separator+"\n")
    f.write("Marvellous Infosystems Process Logger : "+time.ctime() + "\n")
    f.write(separator +"\n")
    f.write("\n")

    for proc in psutil.process_iter():
        try:
            pinfo = proc.as_dict(attrs = ['pid','name','username'])
            vms = proc.memory_info().vms/(1024*1024)
            pinfo['vms'] = vms
            listprocess.append(pinfo);
        except (psutil.NoSuchProcess,psutil.AccessDenied, psutil.ZombieProcess):
            pass

    for element in listprocess:
        f.write("%s\n"% element)

    print("Log file is Succesfully genrated at location %s" % (log_path))

    connected = is_connected()

    if connected:
        startTime = time.time()
        MailSender(log_path,time.ctime())
        endTime = time.time()

        print('Took %s Second to send mail'%(endTime - startTime))
    else:
        print("There is no internet Connection")

def main():
    print("---- Marvellous Infosystems by Shubham Borse ----")

    print("Application name : " +argv[0])

    if(len(argv)!= 2):
        print("Error: Invalid number of arguments")
        exit()

    if (argv[1]== "-h" or (argv[1] == "-H")):
        print("This Script is used to Log record of running processes")
        exit()

    if (argv[1]== "-u") or (argv[1]== "-U"):
        print("Usage :ApplicationName AbsolutePath_of_Directory")
        exit()

    try:
        schedule.every(int(argv[1])).seconds.do(ProcessLog)
        while True:
            schedule.run_pending()
            time.sleep(1)
    except ValueError as E:
        print("Error : Invalid Datatype of input",E)

    except Exception as E:
        print("Error : Invalid input",E)

if __name__ == "__main__" :
    main()


from __future__ import print_function
import os
from os.path import join
import torch
import time
import numpy as np
from math import log10
from visualization import save_tensor_image
from networks.utils import classvsr, flow

def test(testloader, model, criterion, opt):
    psnrs_folder = []

    with torch.no_grad():
        for iteration, batch in enumerate(testloader, 1):
            # input, targetdeblur, targetsr
            if not opt.isReal:
                LQ_all = batch[0].to(opt.device)
                GT_all = batch[1].to(opt.device)
                names = batch[2]
                dir_name = batch[3]
            else:
                LQ_all = batch[0].to(opt.device)
                names = batch[1]
                dir_name = batch[2]

            # Backward
            start_time = time.perf_counter()  # -------------------------begin to deal with the video

            n, t, c, h, w = LQ_all.size()

            SR_list = []

            for i in range(t):

                if i == 0:
                    lq = torch.stack((LQ_all[:, i], LQ_all[:, i], LQ_all[:, i], LQ_all[:, i+1], LQ_all[:, i+2]), dim=1)
                elif i == 1:
                    lq = torch.stack((LQ_all[:, i-1], LQ_all[:, i-1], LQ_all[:, i], LQ_all[:, i+1], LQ_all[:, i+2]), dim=1)
                elif i == t - 2:
                    lq = torch.stack((LQ_all[:, i-2], LQ_all[:, i-1], LQ_all[:, i], LQ_all[:, i+1], LQ_all[:, i+1]), dim=1)
                elif i == t - 1:
                    lq = torch.stack((LQ_all[:, i-2], LQ_all[:, i-1], LQ_all[:, i], LQ_all[:, i], LQ_all[:, i]), dim=1)
                else:
                    lq = LQ_all[:, i - 2 : i + 3]

                lr = []
                n, t, c, h, w = lq.size()

                for i in range(t):
                    LR = lq[:, i].squeeze().permute(1, 2, 0)

                    lr_list, num_h, num_w, H, W = classvsr.crop_cpu(LR, opt.Crop, opt.Step, opt.device)

                    lr.append(lr_list)

                for i in range(len(lr)):
                    lr[i] = torch.stack(lr[i]).permute(0, 3, 1, 2)

                lr = torch.stack(lr, 1)

                A, B, C = [], [], []

                for i in range(len(lr)):
                    LQ = lr[i].unsqueeze(0)

                    dis_flow1 = flow.dis_flow(LQ[: ,1].squeeze(), LQ[: ,2].squeeze())
                    dis_flow2 = flow.dis_flow(LQ[:, 3].squeeze(), LQ[:, 2].squeeze())

                    if (dis_flow1 < opt.Threshold and dis_flow2 < opt.Threshold) :

                        A.append(i)

                    else:

                        dis_flow3 = flow.dis_flow(LQ[:, 0].squeeze(), LQ[:, 1].squeeze())
                        dis_flow4 = flow.dis_flow(LQ[:, 3].squeeze(), LQ[:, 4].squeeze())

                        if (dis_flow3 < opt.Threshold and dis_flow4 < opt.Threshold):
                            B.append(i)
                        else:
                            C.append(i)

                n, t, c, h, w = lr.size()

                if A != []:
                    lq_A = torch.stack([lr[i] for i in A])
                    output_A = model(lq_A, 'A')

                if B != []:
                    lq_B = torch.stack([lr[i] for i in B])
                    output_B = model(lq_B, 'B')

                if C != []:
                    lq_C = torch.stack([lr[i] for i in C])
                    output_C = model(lq_C, 'C')

                output = []
                for i in range(len(lr)):
                    if i in A:
                        output.append(output_A[A.index(i)])
                    elif i in B:
                        output.append(output_B[B.index(i)])
                    else:
                        output.append(output_C[C.index(i)])

                output = torch.stack(output, 0)

                sr = []

                for i in range(n):
                   sr.append(output[i].permute(1, 2, 0))

                sr = classvsr.combine(sr, num_h, num_w, H, W, opt.Crop, opt.Step, 4)

                n, t, c, h, w = LQ_all.size()

                SR_ref = sr[0:h*4, 0:w*4].permute(2, 0, 1).unsqueeze(0)

                SR_list.append(SR_ref)

            SR_ref = torch.stack(SR_list, dim=1)

            ################### save image ###################
            if opt.isSave:
                SR_ref_clipped = torch.clamp(SR_ref, 0, 1)

                video_name = dir_name[0].split('/')[-1]  # name for test video

                root = join(opt.save_path, video_name)

                if not os.path.exists(root):
                    os.makedirs(root)

                for i in range(t):
                    frame_index = iteration - 1 + i
                    frame_name = names[frame_index][0]
                    path = os.path.join(root, frame_name)
                    save_tensor_image(SR_ref_clipped[:, i], path)

            for i in range(t):
                ################### calculate psnr ###################
                if not opt.isReal:
                    mse = criterion(SR_ref[:,i], GT_all[:,i])
                    if mse == float("inf"):
                        mse = 1

                    psnr = 10 * log10(1 / mse)
                    psnrs_folder.append(psnr)
                    opt.psnrs_all.append(psnr)
                else:
                    psnrs_folder.append(0)
                    opt.psnrs_all.append(0)

        torch.cuda.synchronize()  # wait for CPU & GPU time syn
        evalation_time = time.perf_counter() - start_time  # ---------finish all the videos
        print("Folder Avg. PSNR:{:4f} Time: {:4f}".format(np.mean(psnrs_folder), evalation_time / t))
        return np.mean(psnrs_folder)

def model_test(testloader, model, opt):
    model = model.to(opt.device)
    criterion = torch.nn.MSELoss(reduction='mean')
    criterion = criterion.to(opt.device)
    psnr = test(testloader, model, criterion, opt)
    return psnr


import numpy as np
from keras.preprocessing import image

test_image = image.load_img('prediction_test_images/random_cancer_img_3.png',target_size=(64,64))
test_image = image.img_to_array(test_image)
test_image = np.expand_dims(test_image, axis=0)
result = classifer.predict(test_image)
if result[0][0] > 0.5:
    print ("Possible Normal tissue")
else:
    print ("Possible Cancer tissue")


#!/usr/bin/env python3
import time
import os
import re
import json
from itertools import groupby
from decor import to_snake, to_camel
from functions import arg, method, create_comments
from header_detection import subtypes_autodetection

not_cpp_post_mod       = ['=delete', '=0', '=default']
post_class_modifiers   = ['final', ]
generic_types          = [  'char', 'unsigned char', 'bool', 'short', 'unsigned short', 'int', 'unsigned int', 'long', 'unsigned long', 'float', 'double', 'long long',
                            'int8_t', 'uint8_t', 'int16_t', 'uint16_t', 'int32_t', 'uint32_t', 'int64_t', 'uint64_t']
class parent:
    type           = 'public'
    name           = ''
    template_types = []
    def __init__(self, **kwargs):
        self.__dict__.update(kwargs)
    def __str__(self):
        return self.type+' '+self.name+' <'+str(self.template_types)+'>'

def fields(line):
    line=line.replace('\n','')
    line=line.replace(';','; ')
    line=re.sub('\s+',' ', line)
    args=line.split('; ')
    ret=[]
    for a in args:
        p=a.split(' ')
        kv=p[-1].split('=')
        pm=None
        if 'static' in p:
            pm='static'
            p.remove('static')
        t=' '.join(p[:-1])
        if len(kv)==1:
            ret.append(arg(pre_modifier=pm, type=t, name=kv[0]))
        elif len(kv)==2:
            ret.append(arg(pre_modifier=pm, type=t, name=kv[0], value=kv[1]))
    del ret[-1]
    return ret

def add_sg(var, setters, getters, snake_case):
    sgetters=[]
    for v in var:
        if not snake_case:
            setter=to_camel("set_"+v.name)
            getter=to_camel("get_"+v.name)
        else:
            setter=to_snake("set_"+v.name)
            getter=to_snake("get_"+v.name)
        if getters:
            sgetters.append(method( return_type=v.type,
                                    name=getter,
                                    args=None,
                                    post_modifier="const",
                                    body="return "+v.name+";",
                                    hint='getter'))
        if setters:
            t=v.type
            if v.pre_modifier!='const':
                 pm=None
                 if t.find('static ')!=-1:
                     t=t.replace('static ','')
                     pm='static'
            if t in generic_types or '*':
                args=[arg(type=t, name='_'+v.name), ]
            elif '*' in t:
                args=[arg(type=t, name='_'+v.name), ]
            else:
                args=[arg(pre_modifier="const ", type=t+'&', name='_'+v.name), ]
            sgetters.append(method( return_type='void',
                                    name=setter,
                                    args=args,
                                    pre_modifier=pm,
                                    post_modifier=None,
                                    body=v.name+" = _"+v.name+";",
                                    hint='setter'))
    return sgetters

class class_:
    template_types     = []
    parents            = []
    del_comments       = False
    name               = ''
    post_modifier      = ''

    public_methods     = []
    public_comments    = []

    protected_methods  = []
    protected_comments = []

    private_methods    = []
    private_comments   = []

    protected_fields   = []
    private_fields     = []
    set                = []
    get                = []
    snake_case         = True
    gen_test           = True
    pre_class          = ''
    post_class_hpp     = ''
    post_class_cpp     = ''

    def __init__(self, **kwargs):
        self.public_methods=[]
        self.protected_methods=[]
        self.private_methods=[]
        self.__dict__.update(kwargs)
        if 'template_types' in kwargs:
            self.template_types=[]
            if isinstance(kwargs['template_types'], str):
                self.template_types=kwargs['template_types'].split(' ')
        if 'parents' in kwargs:
            if isinstance(kwargs['parents'], str):
                s=kwargs['parents'].split(',')
                self.parents=[]
                for d in s:
                    self.parents.append(parent(name=d))
        if 'private_fields' in kwargs:
            if isinstance(kwargs['private_fields'], str):
                self.private_fields=fields(kwargs['private_fields'])
        if 'protected_fields' in kwargs:
            if isinstance(kwargs['protected_fields'], str):
                self.protected_fields=fields(kwargs['protected_fields'])
        ss=False
        gg=False
        if 'private' in self.set:
            ss=True
        if 'private' in self.get:
            gg=True
        self.public_methods.extend(add_sg(self.private_fields, ss, gg, self.snake_case))
        if 'protected' in self.set:
            ss=True
        if 'protected' in self.get:
            gg=True
        self.public_methods.extend(add_sg(self.protected_fields, ss, gg, self.snake_case))
        templated_class=self.name
        if self.template_types:
            templated_class=templated_class+'<'+', '.join(self.template_types)+'>'
        for m in self.public_methods:
            m.class_name=templated_class
        for m in self.protected_methods:
            m.class_name=templated_class
        for m in self.private_methods:
            m.class_name=templated_class
        if not self.del_comments:
            self.comment_methods()

    def comment_methods(self):
        self.public_comments    = create_comments(self.public_methods)
        self.protected_comments = create_comments(self.protected_methods)
        self.private_comments   = create_comments(self.private_methods)

    def uncomment_methods(self):
        self.public_comments    = []
        self.protected_comments = []
        self.private_comments   = []

    # vd=0 - not virtual
    # vd=1 - virtual
    # vd=2 - pure virtual
    # vd=3 - virtual protected
    # Constructors:
    # dd=0 - custom constructor (delete default)
    # dd=1 - deleted constructor
    # dd=2 - default constructor
    def basic_class_content(self, dd=0, dc=0, dm=0, vd=0, custom=None):
        templated_class=self.name
        if self.template_types and  isinstance(self.template_types,list):
            templated_class=templated_class+'<'+', '.join(self.template_types)+'>'

        default=method(name=self.name)
        if dd==0:
            default.post_modifier="noexcept"
        elif dd==1:
            default.post_modifier="=delete"
        elif dd==2:
            default.post_modifier="=default"
        self.public_methods.append(default)

        op=method(return_type=templated_class, name="operator=", hint='copy', args=[arg(pre_modifier='const', type=self.name+"&"),])
        copyc=method(name=self.name, hint='copy', args=[arg(pre_modifier="const", type=self.name+"&"),])
        if dc==0:
            copyc.post_modifier="noexcept"
            op.post_modifier="noexcept"
        elif dc==1:
            copyc.post_modifier="=delete"
            op.post_modifier="=delete"
        elif dc==2:
            copyc.post_modifier="=default"
            op.post_modifier="=default"
        self.public_methods.append(op)
        self.public_methods.append(copyc)

        mop=method(return_type=templated_class+'&', name="operator=", hint='move', args=[arg(pre_modifier='const', type=self.name+"&&"),])
        movec=method(name=self.name, hint='move', args=[arg(pre_modifier="const", type=self.name+"&&"),])
        if dm==0:
            movec.post_modifier="noexcept"
            mop.post_modifier="noexcept"
        elif dm==1:
            movec.post_modifier="=delete"
            mop.post_modifier="=delete"
        elif dm==2:
            movec.post_modifier="=default"
            mop.post_modifier="=default"
        self.public_methods.append(mop)
        self.public_methods.append(movec)

        if custom:
            for constructor_params in custom:
                self.public_methods.append(method(name=self.name, args=constructor_params, class_name=self.name))
        if   vd==0:
            self.public_methods.append(method(name='~'+self.name))
        elif vd==1:
            self.public_methods.append(method(pre_modifier='virtual', name='~'+self.name))
        elif vd==2:
            self.public_methods.append(method(pre_modifier='virtual', name='~'+self.name, post_modifier='=0'))
        elif vd==3:
            self.protected_methods.append(method(name='~'+self.name, pre_modifier='virtual'))
        for m in self.public_methods:
            m.class_name=templated_class
        for m in self.protected_methods:
            m.class_name=templated_class

    def decl(self, ts=' '*4):
        ret=''
        if self.pre_class and isinstance(self.pre_class, str):
            ret+=self.pre_class
        if self.template_types:
            ret+='template <class '+', class '.join(self.template_types)+'>\n'
        ret+="class "+self.name
        if self.parents:
            ret+=' : '
            i=1
            for p in self.parents:
                ret+=(p.type+' '+p.name)
                if p.template_types:
                    ret+=('<'+', '.join(p.template_types)+'>')
                if i<len(self.parents):
                    ret+=', '
                i=i+1
        if self.post_modifier:
            ret+=self.post_modifier
        ret+='\n{\n'+ts+'public:\n'+ts*2
        if self.public_methods:
            i=1
            for p in self.public_methods:
                if self.public_comments and i in self.public_comments:
                    ret+=self.public_comments[i].replace('\n','\n'+ts*2)
                ret+=(p.decl().replace('\n','\n'+ts*2)+'\n'+ts*2)
                ret+=('\n'+2*ts)
                i=i+1
        if self.protected_methods or self.protected_fields:
            ret+=('\n'+ts+'protected:\n'+ts*2)
            if self.protected_methods:
                i=1
                for p in self.protected_methods:
                    if self.protected_comments and i in self.protected_comments:
                        ret+=self.protected_comments[i].replace('\n','\n'+ts*2)
                    ret+=(p.decl().replace('\n','\n'+ts*2)+'\n'+ts*2)
                    ret+=('\n'+ts*2)
                    i=i+1
            if self.protected_fields:
                max_len=0
                for p in self.protected_fields:
                    max_len=max(max_len, len(str(p)))
                for p in self.protected_fields:
                    ret+=(str(p)+';'+' '*(max_len-len(str(p))+1)+'//!< \n'+ts*2)
        ret+=('\n'+ts+'private:\n'+ts*2)
        if self.private_methods:
            i=1
            for p in self.private_methods:
                if self.private_comments and i in self.private_comments:
                    ret+=self.private_comments[i].replace('\n','\n'+ts*2)
                i=i+1
                ret+=(p.decl().replace('\n','\n'+ts*2)+'\n'+ts*2)
                ret+=('\n'+ts*2)
        if self.private_fields:
            max_len=0
            for p in self.private_fields:
                max_len=max(max_len, len(str(p)))
            for p in self.private_fields:
                ret+=(str(p)+';'+' '*(max_len-len(str(p))+1)+'//!< \n'+ts*2)
        ret+='\n};\n'
        if self.post_class_hpp and isinstance(self.post_class_hpp, str):
            ret+=self.post_class_hpp
        return ret

    def impl(self, ts=' '*4):
        spp=''
        cpp=''
        cppt=''
        methods=self.public_methods+self.protected_methods+self.private_methods
        class_fields=self.private_fields+self.protected_fields
        templated_class=self.name
        if self.template_types:
            templated_class=templated_class+'<'+', '.join(self.template_types)+'>'
        for v in class_fields:
            if v.pre_modifier=='static':
                if self.template_types and  isinstance(self.template_types,list):
                    spp+="template <class "+', class '.join(self.template_types)+'>\n'
                spp+=('static '+v.type+' '+templated_class+'::'+v.name+';\n')
        for v in methods:
            if v.pre_modifier=='static':
                if self.template_types and  isinstance(self.template_types,list):
                    spp+="template <class "+', class '.join(self.template_types)+'>\n'
                spp+=v.impl()+'\n'
        for m in methods:
            if m.hint=='setter' or m.hint=='getter' or m.pre_modifier=='static':
                continue
            if (m.post_modifier and m.post_modifier not in not_cpp_post_mod) or not m.post_modifier:
                is_template=False
                a=''
                if self.template_types and  isinstance(self.template_types,list):
                    a="template <class "+', class '.join(self.template_types)+'>\n'
                if m.template_args:
                    is_template=True
                    cppt+=a
                    cppt+=(m.impl(class_fields=class_fields)+'\n\n')
                else:
                    cpp+=a
                    cpp+=(m.impl(class_fields=class_fields)+'\n\n')
        if self.template_types and  isinstance(self.template_types,list):
            cppt=cppt+cpp
            cpp=''
        else:
            cpp=spp+cpp
            spp=''
        return (spp, cpp+self.post_class_cpp, cppt)

    def autodetect (self):
        types=[]
        f=self.private_fields + self.protected_fields
        for v in f:
            types.append(v.type)
        m=self.private_methods + self.protected_methods + self.public_methods
        autodetected=[]
        for v in m:
            autodetected.extend(v.autodetected)
        autodetected.extend(subtypes_autodetection(types))
        headers = [el for el, _ in groupby(sorted(autodetected))]
        return headers

    def save(self, namespace, directory, user, email, tester_name=None, tester_email=None):
        b=self.decl()
        impl=self.impl()
        if self.template_types:
            b+=('\n#include "'+self.name+'_impl.hpp"')
        elif impl[2]:
            b+=('\n'+impl[2])
        a=namespace(namespace, b)
        a=includes(None, c.autodetect(), None)+a
        a=header(self.name, user, email)+a
        f = open(os.path.join(directory, 'include', self.name+'.hpp'), 'w')
        f.write(a)
        f.close()
        if not self.template_types:
            f = open(os.path.join(directory, 'src', self.name+'.cpp'), 'w')
            f.write(impl[0]+impl[1])
            f.close()
        if self.test==True:
            ts=' '*4
            f = open(os.path.join(directory, 'test', self.name+'.cpp'), 'w')
            tester='#include "'+self.name+'.hpp'+'"\n'
            tester+='#include <iostream>\n\n'
            tester+='using namespace std;\n\n'
            tester+='int main(int argc, char** argv)\n{\n'+ts
            constructors=[c for c in self.public_methods if c.name==self.name]
            i=1
            # Test all constructors
            dyns=[]
            for c in constructors:
                if c.post_modifier!='=delete':
                    tester+=c.name+' test'+str(i)
                    if c.args and c.args.type!=self.name+"&" and c.args.type!=self.name+"&&":
                        tester+=('('+str(c.args)+')')
                        dyns.append(c.args)
                    tester+=(';\n'+ts)
                    i=i+1
            i=1
            for d in dyns:
                tester+=self.name+'* dyn_test'+str(i)+' = new '+self.name
                if d and d.type!=self.name+"&" and d.type!=self.name+"&&":
                    tester+=('('+str(d)+')')
                tester+=(';\n'+ts)
                i=i+1
            tester+='\n'+ts
            i=1
            for d in dyns:
                tester+='delete dyn_test'+str(i)
                tester+=(';\n'+ts)
                i=i+1

            tester+='\nreturn 0;\n}\n'
            if not tester_name:
                f.write(header(self.name, user, email)+tester)
            else:
                f.write(header(self.name, tester_name, tester_email)+tester)
            f.close()
        else:
            f = open(os.path.join(directory, 'include', self.name+'_impl.hpp'), 'w')
            f.write(impl[2]+'\n'+self.post_class_cpp)
            f.close()

def virtuals(line):
    l=funcs(line)
    for func in l:
        func.pre_modifier='virtual'
    return l

def consts(line):
    l=funcs(line)
    for func in l:
        func.pre_modifier='const'
    return l

# TODO: Change maintainer to somebody from developers
# TODO: Add simple variable checking to method body
# TODO: Add to copy constructor copying of containers if they're exist in class
# TODO: Add loggers to methods
# TODO: Add automatic test creation
# TODO: Add common programming patterns
# TODO: Add automatic client(C++)-server(django) generation


#Import Libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

#Import Dataset
dataset = pd.read_csv("/content/drive/MyDrive/Mall_Customers.csv")
x = dataset.iloc[:, [3,4]].values

#Elbow method to find the optimal number of clusters
from sklearn.cluster import KMeans
wcss = []
for i in range (1, 11):
    kmeans = KMeans(n_clusters = i, init = "k-means++", max_iter = 300, n_init = 10, random_state = 0)
    kmeans.fit(x)
    wcss.append(kmeans.inertia_)
plt.plot(range(1,11), wcss)
plt.title("Metodo del codo")
plt.xlabel("Numero de Clusters")
plt.ylabel("WCSS(K)")
plt.show()

#Apply the k-means method to segment the data set
kmeans = KMeans(n_clusters = 5, init="k-means++", max_iter = 300, n_init = 10, random_state = 0)
y_kmeans = kmeans.fit_predict(x)

#Visualizacion de los clusters
plt.scatter(x[y_kmeans == 0, 0],x[y_kmeans == 0, 1], s = 100, c = "red", label = "Cluster 1 ")
plt.scatter(x[y_kmeans == 1, 0],x[y_kmeans == 1, 1], s = 100, c = "blue", label = "Cluster 2 ")
plt.scatter(x[y_kmeans == 2, 0],x[y_kmeans == 2, 1], s = 100, c = "green", label = "Cluster 3 ")
plt.scatter(x[y_kmeans == 3, 0],x[y_kmeans == 3, 1], s = 100, c = "orange", label = "Cluster 4 ")
plt.scatter(x[y_kmeans == 4, 0],x[y_kmeans == 4, 1], s = 100, c = "magenta", label = "Cluster 5 ")
plt.scatter(kmeans.cluster_centers_[:,0], kmeans.cluster_centers_[:,1], s = 300, c = "yellow", label = "Baricentros")
plt.title("Customer Cluster")
plt.xlabel("Annual Income (en miles de $)")
plt.ylabel("Spending Score (1-100)")
plt.legend()
plt.show()


import databroker as db
import numpy as np
from matplotlib import pyplot as plt

from databroker_extractor import common as c_db, common as c_dt, common as c_io, common as c_plot

if __name__ == '__main__':
    detector = 'xray_eye3_image'

    # This may be used if the configuration is not set via ~/.config. See the following documentation for more details:
    # http://nsls-ii.github.io/databroker/configuration.html
    '''
    from metadatastore.mds import MDSRO  # metadata store read-only
    from filestore.fs import FileStoreRO  # file store read-only
    from databroker import db # Broker, DataBroker as db, get_images, get_table, get_events, get_fields

    # This an example. You'll need to know your local configuration.
    mds = MDSRO({'host': 'localhost',
                 'port': 27018,
                 'database': 'datastore',
                 'timezone': 'US/Eastern'})

    # This an example. You'll need to know your local configuration.
    fs = FileStoreRO(
        config={
            'host': 'localhost',
            'port': 27018,
            'database': 'filestore',
        },
        root_map={'/': 'W:'}
    )

    import filestore.api as fsapi
    from filestore.handlers import NpyHandler, SingleTiffHandler, AreaDetectorTiffHandler
    # fsapi.register_handler('tiff', SingleTiffHandler)
    fsapi.register_handler('tiff', AreaDetectorTiffHandler)

    db = Broker(mds, fs)
    db.get_images(db['58732824'], detector)
    '''
    # db.DataBroker.fs.root_map = {'/': 'W:\\'}
    # scan_id = -1
    # scan_id = '31a8fc'

    harmonics_scan = True
    fiber_scan = False
    pinhole_scan = False
    list_scans = False

    clim = (0, 255)
    cmap = 'jet'
    plt.rcParams['image.cmap'] = cmap
    enable_log_correction = False

    if harmonics_scan:
        # Harmonics scan:
        scan_ids = [
            # center vertical mbs on ID cone: 0.6x1.2 (pbs) 17585 (scan#) date: 02/28/2017
            # ID harmonic: 5th energy: 9.65keV gap: 6.640
            # Ti foil, elm: 50pC, .1s .05x.05,.1x.4
            #     100pC, .1s .2x.4
            # ('1eff511d', 'ivu_gap'),
            # ('8f6a6004', 'ivu_gap'),
            # ('1f1422b4', 'ivu_gap'),
            # ('7949f1b0', 'dcm_b'),
            # ('daeb15e3', 'dcm_b'),
            # ('6edfa33a', 'dcm_b'),
            # center vertical mbs on ID cone: 0.6x1.2 (pbs) .4x.1 (mbs)  17599 (scan#)
            # ID harmonic: 7th energy: 9.75keV gap: 5.240 can't scan gap at 9.65keV...
            # Ti foil, elm: 50pC, .1s .05x.05,.1x.4
            #     100pC, .1s .2x.4
            # ('74798cb6', 'ivu_gap'),
            # ('dc2b5045', 'ivu_gap'),
            # ('c4f95268', 'ivu_gap'),
            # ('b2365717', 'dcm_b'),
            # ('ac99ddd0', 'dcm_b'),
            # ('b808a890', 'dcm_b'),
            # center vertical mbs on ID cone: 0.6x1.2 (pbs) .4x.1 (mbs) 17610 (scan#)
            # gap scan: #17613
            # ID harmonic: 7th energy: 9.65keV gap: 5.2017
            # Ti foil, elm: 50pC, .1s .05x.05,.1x.4
            #     100pC, .1s .2x.4
            # ('e23fb7a1', 'dcm_b'),
            # ('afc6da9e', 'dcm_b'),
            # ('e71b8b5f', 'dcm_b'),
            # 2017-03-18 (elevation studies):
            ('809d548e', 'dcm_b'),
        ]

        # save = True
        save = False

        for i, scan_id in enumerate(scan_ids):
            chx.get_and_plot(scan_id[0], save=save, field=scan_id[1], idx=i, is_vs_energy=True)

    if fiber_scan:
        first_slice = 1000
        second_slice = 1250
        third_slice = 1500

        # Dark field:
        scan_id_dark_field = '12738c63'
        scan_dark_field, t_dark_field = c_db.get_scan(scan_id_dark_field)
        images_dark_field = db.get_images(scan_dark_field, detector)
        mean_dark_field = np.mean(images_dark_field[0], axis=0)
        print('     ID: {}  Number of images: {}'.format(scan_id_dark_field, len(images_dark_field[0])))
        print('     Min: {}  Max: {}\n'.format(mean_dark_field.min(), mean_dark_field.max()))

        plt.imshow(mean_dark_field, clim=clim, cmap=cmap)
        plt.savefig('mean_dark_field_{}.png'.format(scan_id_dark_field))
        c_plot.clear_plt()

        shape = mean_dark_field.shape
        print('Shape of mean_dark_field: {}'.format(shape))

        c_io.save_data_numpy(data=mean_dark_field, name='mean_dark_field_{}.dat'.format(scan_id_dark_field))
        c_plot.save_raw_image(data=mean_dark_field, name='mean_dark_field_{}.tif'.format(scan_id_dark_field))

        fig = plt.figure()
        ax = fig.add_subplot(111)
        ax.set_xlim((0, shape[0]))
        ax.grid()
        ax.plot(mean_dark_field[:, first_slice], label='pixel={}'.format(first_slice))
        ax.plot(mean_dark_field[:, second_slice], label='pixel={}'.format(second_slice))
        ax.plot(mean_dark_field[:, third_slice], label='pixel={}'.format(third_slice))
        ax.legend()
        fig.savefig('mean_dark_field_slices_{}.png'.format(scan_id_dark_field))
        c_plot.clear_plt()

        # Fiber in:
        scan_id_fiber_in = '58732824'
        scan_fiber_in, t_fiber_in = c_db.get_scan(scan_id_fiber_in)
        images_fiber_in = db.get_images(scan_fiber_in, detector)
        mean_fiber_in = np.mean(images_fiber_in[0], axis=0)
        print('     ID: {}  Number of images: {}'.format(scan_id_fiber_in, len(images_fiber_in[0])))
        print('     Min: {}  Max: {}\n'.format(mean_fiber_in.min(), mean_fiber_in.max()))

        plt.imshow(mean_fiber_in, clim=clim, cmap=cmap)
        plt.savefig('mean_fiber_in_{}.png'.format(scan_id_fiber_in))
        c_plot.clear_plt()

        # Fiber out:
        scan_id_fiber_out = '190ae619'
        scan_fiber_out, t_fiber_out = c_db.get_scan(scan_id_fiber_out)
        images_fiber_out = db.get_images(scan_fiber_out, detector)
        mean_fiber_out = np.mean(images_fiber_out[0], axis=0)
        print('     ID: {}  Number of images: {}'.format(scan_id_fiber_out, len(images_fiber_out[0])))
        print('     Min: {}  Max: {}\n'.format(mean_fiber_out.min(), mean_fiber_out.max()))

        plt.imshow(mean_fiber_out, clim=clim, cmap=cmap)
        plt.savefig('mean_fiber_out_{}.png'.format(scan_id_fiber_out))
        c_plot.clear_plt()

        # Diff fiber in and dark field:
        mean_diff_fiber_in = mean_fiber_in - mean_dark_field
        print('Min: {}  Max: {}\n'.format(mean_diff_fiber_in.min(), mean_diff_fiber_in.max()))
        plt.imshow(mean_diff_fiber_in, clim=clim, cmap=cmap)
        plt.savefig('mean_fiber_in_minus_mean_dark_field.png')
        c_plot.clear_plt()

        shape = mean_diff_fiber_in.shape
        print('Shape of mean_diff_fiber_in: {}'.format(shape))

        c_io.save_data_numpy(data=mean_diff_fiber_in, name='mean_fiber_in_minus_mean_dark_field.dat')
        c_plot.save_raw_image(data=mean_diff_fiber_in, name='mean_fiber_in_minus_mean_dark_field.tif')

        fig = plt.figure()
        ax = fig.add_subplot(111)
        ax.set_xlim((0, shape[0]))
        ax.grid()
        ax.plot(mean_diff_fiber_in[:, first_slice], label='pixel={}'.format(first_slice))
        ax.plot(mean_diff_fiber_in[:, second_slice], label='pixel={}'.format(second_slice))
        ax.plot(mean_diff_fiber_in[:, third_slice], label='pixel={}'.format(third_slice))
        ax.legend()
        fig.savefig('mean_fiber_in_minus_mean_dark_field_slices.png')
        c_plot.clear_plt()

        # Diff fiber in and dark field:
        mean_diff_fiber_out = mean_fiber_out - mean_dark_field
        print('Min: {}  Max: {}\n'.format(mean_diff_fiber_out.min(), mean_diff_fiber_out.max()))
        plt.imshow(mean_diff_fiber_out, clim=clim, cmap=cmap)
        plt.savefig('mean_fiber_out_minus_mean_dark_field.png')
        c_plot.clear_plt()

        shape = mean_diff_fiber_out.shape
        print('Shape of mean_diff_fiber_out: {}'.format(shape))

        c_io.save_data_numpy(data=mean_diff_fiber_out, name='mean_fiber_out_minus_mean_dark_field.dat')
        c_plot.save_raw_image(data=mean_diff_fiber_out, name='mean_fiber_out_minus_mean_dark_field.tif')

        fig = plt.figure()
        ax = fig.add_subplot(111)
        ax.set_xlim((0, shape[0]))
        ax.grid()
        ax.plot(mean_diff_fiber_out[:, first_slice], label='pixel={}'.format(first_slice))
        ax.plot(mean_diff_fiber_out[:, second_slice], label='pixel={}'.format(second_slice))
        ax.plot(mean_diff_fiber_out[:, third_slice], label='pixel={}'.format(third_slice))
        ax.legend()
        fig.savefig('mean_fiber_out_minus_mean_dark_field_slices.png')
        c_plot.clear_plt()

        # Fiber in - fiber out:
        mean_fiber_in_minus_out = mean_diff_fiber_in - mean_diff_fiber_out
        print('Min: {}  Max: {}\n'.format(mean_fiber_in_minus_out.min(), mean_fiber_in_minus_out.max()))
        plt.imshow(mean_fiber_in_minus_out, clim=clim, cmap=cmap)
        plt.savefig('mean_fiber_in_minus_out.png')
        c_plot.clear_plt()

        # Fiber in / fiber out:
        where_zero = np.where(mean_diff_fiber_out == 0)
        mean_diff_fiber_out[where_zero] = 1
        mean_fiber_in_div_out = mean_diff_fiber_in / mean_diff_fiber_out
        print('Min: {}  Max: {}\n'.format(mean_fiber_in_div_out.min(), mean_fiber_in_div_out.max()))
        max_in = mean_diff_fiber_in.max()
        max_out = mean_diff_fiber_out.max()
        print('Max in: {}  Max out: {}'.format(max_in, max_out))
        plt.imshow(mean_fiber_in_div_out, clim=max_in / max_out, cmap=cmap)
        plt.savefig('mean_fiber_in_div_out.png')
        c_plot.clear_plt()

    if pinhole_scan:
        # Dark field:
        start_time = c_dt.current_timestamp()

        scan_id_dark_field = 'a9a0687c'
        scan_dark_field, t_dark_field = c_db.get_scan(scan_id_dark_field)
        images_dark_field = db.get_images(scan_dark_field, detector)
        mean_dark_field = np.mean(images_dark_field[0], axis=0)

        # Left and right parts have different chips, so need to normalize left and right parts:
        '''
        np.mean(mean_dark_field[:, :mean_dark_field.shape[1]/2])
        Out[7]: 2.1016548665364585

        In [8]: np.mean(mean_dark_field[:, mean_dark_field.shape[1]/2:])
        Out[8]: 2.8157670403773496
        '''

        '''
        left_mean = np.mean(mean_dark_field[:, :mean_dark_field.shape[1] / 2])
        right_mean = np.mean(mean_dark_field[:, mean_dark_field.shape[1] / 2:])
        if left_mean > right_mean:
            mean_dark_field[:, mean_dark_field.shape[1] / 2:] = mean_dark_field[:,
                                                                mean_dark_field.shape[1] / 2:] * left_mean / right_mean
        else:
            mean_dark_field[:, :mean_dark_field.shape[1] / 2] = mean_dark_field[:,
                                                                :mean_dark_field.shape[1] / 2] * right_mean / left_mean
        '''
        log_mean_dark_field = np.log10(mean_dark_field)
        if enable_log_correction:
            neg_idx = np.where(log_mean_dark_field <= 0)
            log_mean_dark_field[neg_idx] = 0.0
        print('     ID: {}  Number of images: {}'.format(scan_id_dark_field, len(images_dark_field[0])))
        print('     Min: {}  Max: {}'.format(mean_dark_field.min(), mean_dark_field.max()))

        plt.imshow(mean_dark_field, clim=clim, cmap=cmap)
        # plt.imshow(log_mean_dark_field, cmap=cmap)
        plt.savefig('mean_pinhole_dark_field_{}.png'.format(scan_id_dark_field))
        c_plot.clear_plt()

        # plt.imshow(mean_dark_field, clim=clim, cmap=cmap)
        plt.imshow(log_mean_dark_field, cmap=cmap)
        plt.savefig('mean_pinhole_dark_field_{}_log.png'.format(scan_id_dark_field))
        c_plot.clear_plt()

        print('Duration: {:.3f} sec\n'.format(c_dt.current_timestamp() - start_time))

        # Pinhole:
        start_time = c_dt.current_timestamp()

        scan_id_pinhole = 'afe3cf59'
        scan_pinhole, t_pinhole = c_db.get_scan(scan_id_pinhole)
        images_pinhole = db.get_images(scan_pinhole, detector)
        mean_pinhole = np.mean(images_pinhole[0], axis=0)
        mean_pinhole_orig = np.copy(mean_pinhole)
        mean_pinhole /= mean_dark_field

        '''
        if left_mean > right_mean:
            mean_pinhole[:, mean_pinhole.shape[1] / 2:] = mean_pinhole[:,
                                                          mean_pinhole.shape[1] / 2:] * left_mean / right_mean
        else:
            mean_pinhole[:, :mean_pinhole.shape[1] / 2] = mean_pinhole[:,
                                                          :mean_pinhole.shape[1] / 2] * right_mean / left_mean
        '''
        log_mean_pinhole = np.log10(mean_pinhole)
        if enable_log_correction:
            neg_idx = np.where(log_mean_pinhole <= 0)
            log_mean_pinhole[neg_idx] = 0.0
        print('     ID: {}  Number of images: {}'.format(scan_id_pinhole, len(images_pinhole[0])))
        print('     Min: {}  Max: {}'.format(mean_pinhole.min(), mean_pinhole.max()))

        plt.imshow(mean_pinhole, clim=clim, cmap=cmap)
        # plt.imshow(log_mean_pinhole, cmap=cmap)
        plt.savefig('mean_pinhole_{}.png'.format(scan_id_pinhole))
        c_plot.clear_plt()

        # plt.imshow(mean_pinhole, clim=clim, cmap=cmap)
        plt.imshow(log_mean_pinhole, cmap=cmap)
        plt.savefig('mean_pinhole_{}_log.png'.format(scan_id_pinhole))
        c_plot.clear_plt()

        plt.imshow(mean_pinhole_orig, clim=clim, cmap=cmap)
        plt.savefig('mean_pinhole_{}_orig.png'.format(scan_id_pinhole))
        c_plot.clear_plt()

        plt.imshow(np.log10(mean_pinhole_orig), cmap=cmap)
        plt.savefig('mean_pinhole_{}_orig_log.png'.format(scan_id_pinhole))
        c_plot.clear_plt()

        print('Duration: {:.3f} sec\n'.format(c_dt.current_timestamp() - start_time))

        # # Diff pinhole and dark field:
        # # mean_diff_pinhole = mean_pinhole - mean_dark_field
        # mean_diff_pinhole = mean_pinhole
        # log_mean_diff_pinhole = np.log10(mean_diff_pinhole)
        # if enable_log_correction:
        #     neg_idx = np.where(log_mean_diff_pinhole <= 0)
        #     log_mean_diff_pinhole[neg_idx] = 0.0
        # print('Min: {}  Max: {}\n'.format(log_mean_diff_pinhole.min(), log_mean_diff_pinhole.max()))
        # # plt.imshow(mean_diff_pinhole, clim=clim, cmap=cmap)
        # plt.imshow(log_mean_diff_pinhole, cmap=cmap)
        # plt.savefig('mean_pinhole_minus_mean_dark_field_log.png')
        # _clear_plt()

    if list_scans:
        scans_list = c_db.get_scans_list(keyword='Chubar')
        print('Scans list: {}'.format(len(scans_list)))
        for s in scans_list:
            print(s)

    print('Done')


import time, random, os
import pickle
import torch
import numpy as np
from tqdm import tqdm
from sklearn.svm import LinearSVC
from sklearn.metrics import accuracy_score
from sklearn.model_selection import GridSearchCV
from torch.nn.functional import conv2d, max_pool2d

from snn import DEFAULT_NETWORK_PARAMS, SpikingConv, SpikingPool, SpikingInput
from utils import DebugPrint, spike_to_time, load_encoded_MNIST

"""
This file is used to save CSNN outputs for analysis purpose.
"""

class SNN:
    """
    Spiking convolutional neural network model with IF neurons that can fire at most once.
    """
    def __init__(self, input_shape, params=DEFAULT_NETWORK_PARAMS):

        """
        All layers must have the same voltage convention for VDSP
        Initially, neurons have a potential of v_rest (= neuron has not fired)
        When firing, the potential of a neuron is set to v_reset (= neuron has fired)
        """
        self.v_rest = params["v_rest"]
        self.v_reset = params["v_reset"]

        self.input_layer = SpikingInput(input_shape, v_reset=self.v_reset, v_rest=self.v_rest)
        print(self.input_layer.output_shape)

        conv1 = SpikingConv(input_shape,
            out_channels=params["conv_out_channels"], kernel_size=params["conv_kernel_size"],
            stride=params["conv_stride"], padding=params["conv_padding"], nb_winners=params["conv_nb_winners"],
            firing_threshold=params["conv_firing_thr"], inhibition_radius=params["conv_inhib_radius"],
            vdsp_lr=params["conv_vdsp_lr"], adaptive_lr=params["conv_adaptive_lr"],
            vdsp_max_lr=params["conv_vdsp_max_lr"], update_lr_cnt=params["conv_update_lr_cnt"],
            vdsp_dep_factor=params["conv_vdsp_dep_factor"], v_reset=self.v_reset, v_rest=self.v_rest
        )
        print(conv1.output_shape)

        pool1 = SpikingPool(conv1.output_shape,
            kernel_size=params["pool_kernel_size"], stride=params["pool_stride"],
            padding=params["pool_padding"], v_reset=self.v_reset, v_rest=self.v_rest
        )
        print(pool1.output_shape)

        self.conv_layers = [conv1]
        self.pool_layers = [pool1]

        self.trainable_layers = self.conv_layers
        self.output_shape = self.pool_layers[-1].output_shape
        self.output_size = np.prod(self.output_shape)
        self.recorded_sum_spks = []

    def save_weights(self, layer, filename):
        np.save(filename, self.conv_layers[layer].weights)

    def reset(self):
        self.input_layer.reset()
        for layer in self.conv_layers:
            layer.reset()
        for layer in self.pool_layers:
            layer.reset()

    def __call__(self, x, train_layer=None):
        self.reset()
        self.input_layer.init(x)
        nb_timesteps = x.shape[0]
        output_conv = np.zeros((nb_timesteps,) + self.conv_layers[0].output_shape, dtype=np.uint8)
        output_spikes = np.zeros((nb_timesteps,) + self.output_shape, dtype=np.uint8)
        tot_spks = 0
        for t in range(nb_timesteps):
            spk, pot = self.input_layer(x[t])
            tot_spks += spk.sum()
            spk, pot = self.conv_layers[0](spk, pot, train=(train_layer==0))
            tot_spks += spk.sum()
            output_conv[t] = spk.astype(np.uint8)
            spk, pot = self.pool_layers[0](spk)
            tot_spks += spk.sum()
            output_spikes[t] = spk.astype(np.uint8)
        if train_layer == None: self.recorded_sum_spks.append(tot_spks)
        return output_conv, output_spikes

def main(
    seed=1,                            # To control the randomness
    nb_timesteps=15,                   # Number of spike bins
    epochs=1,                          # Number of epochs per layer
    convergence_rate=0.01,             # Stop training when learning convergence reaches this rate
    max_train_samples=-1,              # Maximum number of samples used for training per layer
    snn_params=DEFAULT_NETWORK_PARAMS, # Parameters of the SNN
    extended_print=False               # Print all messages
):
    # Custom printing
    printd = DebugPrint(extended_print)

    # Seed everything
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.deterministic = True

    # Load dataset encoded as spikes with a temporal coding
    X_train, X_test, y_train, y_test = load_encoded_MNIST(nb_timesteps=nb_timesteps)

    # Init SNN
    input_shape = X_train[0][0].shape
    snn = SNN(input_shape, snn_params)

    printd(f"Input shape : {X_train[0].shape} ({np.prod(X_train[0].shape)} values)")
    printd(f"Output shape : {snn.output_shape} ({snn.output_size} values)")
    printd(f"Mean spikes count per input : {X_train.mean(0).sum()}")

    CHECKPOINTS = [0, 100, 200, 300, 400, 500, 600]
    curr_checkpoint = 0
    train_cnt = 0

    for x,y in zip(tqdm(X_train), y_train):

        snn(x, train_layer=0)

        train_cnt += 1

        if curr_checkpoint < len(CHECKPOINTS) and train_cnt >= CHECKPOINTS[curr_checkpoint]:

            NB_TO_SAVE = 5000
            cnt = 0
            conv_train = np.zeros((NB_TO_SAVE,) + snn.conv_layers[0].output_shape, dtype=np.uint8)
            pool_train = np.zeros((NB_TO_SAVE,) + snn.output_shape, dtype=np.uint8)
            for i, (x, y) in enumerate(zip(tqdm(X_train), y_train)):
                spk_conv, spk_out = snn(x)
                if spk_out.sum() == 0: printd("[WARNING] No output spike recorded.")
                conv_train[i] = spk_conv.sum(0)
                pool_train[i] = spk_out.sum(0)
                cnt += 1
                if cnt >= NB_TO_SAVE:
                    break

            np.save(f"logs/output_conv_train_{CHECKPOINTS[curr_checkpoint]}_training_samples", conv_train)
            np.save(f"logs/output_pool_train_{CHECKPOINTS[curr_checkpoint]}_training_samples", pool_train)

            snn.save_weights(0, f"logs/weights_{train_cnt}_training_samples")
            curr_checkpoint += 1

        if snn.trainable_layers[0].get_learning_convergence() < convergence_rate:
            break

    snn.save_weights(0, f"logs/weights_{train_cnt}_training_samples")

    NB_TO_SAVE = 5000
    cnt = 0
    conv_train = np.zeros((NB_TO_SAVE,) + snn.conv_layers[0].output_shape, dtype=np.uint8)
    pool_train = np.zeros((NB_TO_SAVE,) + snn.output_shape, dtype=np.uint8)
    for i, (x, y) in enumerate(zip(tqdm(X_train), y_train)):
        spk_conv, spk_out = snn(x)
        if spk_out.sum() == 0: printd("[WARNING] No output spike recorded.")
        conv_train[i] = spk_conv.sum(0)
        pool_train[i] = spk_out.sum(0)
        cnt += 1
        if cnt >= NB_TO_SAVE:
            break

    np.save(f"logs/output_conv_train_{train_cnt}_training_samples", conv_train)
    np.save(f"logs/output_pool_train_{train_cnt}_training_samples", pool_train)
    np.save("logs/output_y_train", y_train[0:NB_TO_SAVE])

##################################################################################################
##################################################################################################

if __name__ == "__main__":
    main()


"""This module provides methods to build models for
protein families. The method here is very simple.
"""

import XGram.Utils

from XGram import Model, Generator

from XGram.Model import Rule

import Bio.Alphabet.IUPAC

import sys, string, re

def addEmissionMatch( grammar, column, generator_chain,
                      t, nt, pnt):

    ## emission from match state
    grammar.addRule( Rule( start = ( nt, ),
                           end = (t, pnt),
                           rate = ("prob", 1.0 ) ))

    ## insert chain for this state
    chain = Model.Chain()
    generator_chain.buildGrammar(chain)
    chain.setTerminals( (t,) )
    chain.setUpdatePolicy( "rev" )
    grammar.addChain( chain )

def buildProteinFamily( data,
                        tree_model = None,
                        name = "proteinfamily",
                        local = False):
    """build protein family for alignment in data.

    tree_model: model used to estimate tree. If it is not given,
        the tree is assumed to be part of the alignment file.

    1. A tree is estimated data using a default model.
    2. Each state is assigned to be match or indel state
        (50% aligned?) -> could use a grammar for this?
        What about the parametric gap model?
    3. equilibrium frequencies are determined via EM
    """

    mali = XGram.Utils.Mali()

    mali.readFromFile( open(data), format = "stockholm" )

    model = Model.Model()
    alphabet = Model.Alphabet()
    grammar = Model.Grammar()

    start_state = "start"
    end_state = ""

    previous_state = "col0"
    previous_t = "t_%s" % previous_state
    previous_nt = "nt_%s" % previous_state
    previous_pnt = "nt_%s*" % previous_state

    ## the chain generator to use
    generator_chain = Generator.Chain( \
         alphabet = Bio.Alphabet.IUPAC.IUPACProtein.letters.lower(),
         )

    columns = mali.getColumns()

    ## add transition form start to first state
    grammar.addRule( Rule( start = ( start_state, ),
                           end = ( previous_nt, ),
                           rate = ("prob", 1.0 ) ))

    addEmissionMatch( grammar, columns[0], generator_chain,
                      previous_t, previous_nt, previous_pnt )

    for c in range( 1, len(columns) ):

        ## add transition from start state for local alignments
        if local:
            grammar.addRule( Rule( start = ( start_state, ),
                                   end = ( current_nt, ),
                                   rate = ("prob", 1.0 ) ))

        column = columns[c]
        current_state = "col%i" % c
        current_t = "t_%s" % current_state
        current_nt = "nt_%s" % current_state
        current_pnt = "nt_%s*" % current_state

        ## transition from previous to this state
        grammar.addRule( Rule( start = ( previous_pnt, ),
                               end = ( current_nt, ),
                               rate = ("prob", 1.0 ) ))

        addEmissionMatch( grammar, column, generator_chain,
                          current_t, current_nt, current_pnt)

        ## add transition to end state for local alignments
        if local:
            grammar.addRule( Rule( start = ( current_pnt, ),
                                   end = ( end_state, ),
                                   rate = ("prob", 1.0 ) ))

        previous_t = current_t
        previous_nt = current_nt
        previous_pnt = current_pnt

    ## add final transition , if local alignment
    if not local:
        grammar.addRule( Rule( start = ( current_pnt, ),
                               end = ( end_state, ),
                               rate = ("prob", 1.0 ) ))

    grammar.setUpdateRules(True)
    grammar.setUpdateRates(True)
    grammar.setName( name )

    generator_alphabet = Generator.AlphabetProtein()
    generator_alphabet.buildGrammar(alphabet)

    model.setAlphabet( alphabet )
    model.setGrammar( grammar )

    return model


'''
Tests for flask-api
'''
import os
import json
import pytest
import main

@pytest.fixture
def client():
    main.APP.config['TESTING'] = True
    client = main.APP.test_client()

    yield client

def test_home(client):
    response = client.get('/')
    response_copy = {
        "status": 200,
        "message": "You successfully hit the /home route"
    }
    assert response.status_code == 200
    assert response.json == response_copy

def test_data(client):
    body = {
        "test": "test_body"
    }
    response_copy = {
        "status": 200,
        "message": "You successfully hit the /data route"
    }
    response = client.post('/data',
                           data=body,
                           content_type='application/json')
    assert response.status_code == 200
    response_data = response.json
    assert response_data == response_copy


import logging

from bokeh.plotting import figure, output_file, show

from case_study.units.adaptations.AccurateControllerArmatureAdaptation import AccurateControllerArmatureAdaptation
from case_study.units.adaptations.StepDelay import DecoupleAdaptation
from case_study.units.de_based.DriverControllerStatechartFMU_CTInOut import DriverControllerStatechartFMU_CTInOut
from case_study.units.de_based.EnvironmentStatechartFMU_CTInOut import EnvironmentStatechartFMU_CTInOut
from master.GaussSeidelMaster import GaussSeidelMaster
from case_study.units.adaptations.MultiRateAdaptation_Power_Window_Obstacle import MultiRateAdaptation_Power_Window_Obstacle

NUM_RTOL = 1e-08
NUM_ATOL = 1e-08

l = logging.getLogger()
l.setLevel(logging.DEBUG)

START_STEP_SIZE = 0.01
MAX_STEP_SIZE = 0.01
MIN_STEPS_BEFORE_INCREASE_STEP_SIZE = 10
STEP_SIZE_INCREASE_RATE = 10 # e.g, 10%

FMU_START_RATE = 10
STOP_TIME = 6;

environment = EnvironmentStatechartFMU_CTInOut("env", NUM_RTOL, NUM_ATOL)

controller = DriverControllerStatechartFMU_CTInOut("controller", NUM_RTOL, NUM_ATOL)

plant = MultiRateAdaptation_Power_Window_Obstacle("plant", NUM_RTOL, NUM_ATOL, FMU_START_RATE, START_STEP_SIZE)

armature_threshold = 5 # This was the initial threshold, but did not work.
adapt_armature = AccurateControllerArmatureAdaptation("arm_adapt", NUM_RTOL, NUM_ATOL, armature_threshold, True)

delay_up = DecoupleAdaptation("delay_up")
delay_down = DecoupleAdaptation("delay_down")

units = [
         environment,                   # 0
         controller,                    # 1
         plant,                         # 2
         adapt_armature,                # 3
         delay_up,                      # 4
         delay_down                     # 5
         ]

order = [
         4,
         5,
         0,
         2,
         3,
         1
         ]

initial_state = [
                 None,
                 None,
                 None,
                 {adapt_armature.out_obj: 0.0},
                 {delay_up.out_v: 0.0},
                 {delay_down.out_v: 0.0},
                 ]

def coupling_controller(y):
    envOut = y[0]
    adaptArmOut = y[3]
    return {controller.in_dup : envOut[environment.out_up],
            controller.in_ddown : envOut[environment.out_down],
            controller.in_obj : adaptArmOut[adapt_armature.out_obj]}

def coupling_plant(y):
    dupOut = y[4]
    ddownOut = y[5]

    return {plant.in_power_up : dupOut[delay_up.out_v],
            plant.in_power_down : ddownOut[delay_down.out_v]
            }

def coupling_adapt_armature(y):
    pOut = y[2]
    return {adapt_armature.armature_current: pOut[plant.out_power_i]}

def coupling_delay_up(y):
    cOut = y[1]
    return {delay_up.in_v: cOut[controller.out_up]} if cOut != None else None

def coupling_delay_down(y):
    cOut = y[1]
    return {delay_down.in_v: cOut[controller.out_down]} if cOut != None else None

couplings = [
             lambda y: None,
             coupling_controller,
             coupling_plant,
             coupling_adapt_armature,
             coupling_delay_up,
             coupling_delay_down
             ]

trace_i = [0.0]
trace_omega = [0.0]
trace_x = [0.0]
trace_F = [0.0]
times = [0.0]

def plot(t, s, y):
    pOut = y[2]

    times.append(t)
    trace_i.append(pOut[plant.out_power_i])
    trace_x.append(pOut[plant.out_window_x])
    trace_F.append(pOut[plant.out_obs_F])

master = GaussSeidelMaster()

master.simulate(order, units, couplings, initial_state,
                    START_STEP_SIZE, MAX_STEP_SIZE, STEP_SIZE_INCREASE_RATE, MIN_STEPS_BEFORE_INCREASE_STEP_SIZE,
                    STOP_TIME, plot)

color_pallete = [
                "#e41a1c",
                "#377eb8",
                "#4daf4a",
                "#984ea3",
                "#ff7f00",
                "#ffff33",
                "#a65628",
                "#f781bf"
                 ]

output_file("./results.html", title="Results")
p = figure(title="Plot", x_axis_label='time', y_axis_label='')
p.line(x=times, y=trace_omega, legend="trace_omega", color=color_pallete[0])
p.line(x=times, y=trace_i, legend="trace_i", color=color_pallete[1])
show(p)

output_file("./results_x.html", title="Results")
p = figure(title="Plot", x_axis_label='time', y_axis_label='')
p.line(x=times, y=trace_x, legend="trace_x", color=color_pallete[2])
show(p)

output_file("./results_F.html", title="Results")
p = figure(title="Plot", x_axis_label='time', y_axis_label='')
p.line(x=times, y=trace_F, legend="trace_F", color=color_pallete[3])
show(p)