from pathlib import Path

from django.core.exceptions import SuspiciousFileOperation
from django.template import Origin, TemplateDoesNotExist
from django.template.loaders.filesystem import Loader as FilesystemLoader
from django.utils._os import safe_join

from . import themes

class Loader(FilesystemLoader):
    def get_template_sources(self, template_name):
        template_path = template_name.split(";")
        if (
            len(template_path) == 3
            and template_path[0] == "themes"
            and template_path[1] in themes
        ):
            theme_info = themes[template_path[1]]
            try:
                name = safe_join(theme_info["path"] / "templates", template_path[2])
            except SuspiciousFileOperation:
                return

            yield Origin(
                name=name,
                template_name=template_name,
                loader=self,
            )


from models import Note
from django.contrib.auth.models import User
from random import randrange
from sys import stdout
from django.utils.timezone import now
from django.utils.encoding import smart_unicode

def gen_random_string_en(dicts_array, size):
    string = ""
    dicts_size = len(dicts_array)

    while size > len(string):
        string += dicts_array[randrange(dicts_size)] + " "

    return string

def gen_random_string_zh(dicts_array, size):
    string = ""
    dicts_size = len(dicts_array)

    while size > len(string):
        string += smart_unicode(dicts_array[randrange(dicts_size)])
        if 0 == (len(string) % randrange(2, 5)):
            string += " "

    return string

def gen(count, dicts, get_string_function):
    user = User.objects.all()[0]
    date_now = now()
    total = count

    while 0 < count:
        stdout.write("%d\r" % (total - count))
        stdout.flush()
        count -= 1
        note = Note()
        note.user = user
        note.pub_date = date_now
        note.title = get_string_function(dicts, randrange(20, 100))
        note.body = get_string_function(dicts, randrange(500, 1000))
        note.save()

def gen_en(count):
    f = open("dict.txt", "r")
    dicts = [line.strip() for line in f]
    f.close()

    gen(count, dicts, gen_random_string_en)

def gen_zh(count):
    f = open("zh_dict.txt", "r")
    dicts = [line.strip() for line in f]
    f.close()

    gen(count, dicts, gen_random_string_zh)


# 参考地址：https://github.com/PaddlePaddle/PGL/blob/main/examples/gcn/train.py
# 模型的保存与加载：https://www.paddlepaddle.org.cn/documentation/docs/zh/tutorial/quick_start/save_model/save_model.html
import paddle
import argparse
from preprocess import load
from GCN import GCN
from GAT import GAT
from GCNII import GCNII
from pgl.utils.logger import log
from paddle.optimizer import Adam
def train(node_index, node_label, gnn_model, graph, criterion, optim):
    gnn_model.train()
    pred = gnn_model(graph, graph.node_feat["feat"])
    pred = paddle.gather(pred, node_index)
    loss = criterion(pred, node_label)
    loss.backward()
    acc = paddle.metric.accuracy(input=pred, label=node_label, k=1)
    optim.step()
    optim.clear_grad()
    return loss, acc
@paddle.no_grad()
def eval(node_index, node_label, gnn_model, graph, criterion):
    gnn_model.eval()
    pred = gnn_model(graph, graph.node_feat["feat"])
    pred = paddle.gather(pred, node_index)
    loss = criterion(pred, node_label)
    acc = paddle.metric.accuracy(input=pred, label=node_label, k=1)
    return loss, acc
def main(args):
    # 初始化数组
    dur = []
    best_test = []
    cal_val_acc = []
    cal_test_acc = []
    cal_val_loss = []
    cal_test_loss = []
    # step1. 数据加载
    dataset = load()
    # dataset.graph.tensor()
    graph = dataset.graph
    train_index = dataset.train_index
    train_label = dataset.train_label
    val_index = dataset.valid_index
    val_label = dataset.valid_label
    test_index = dataset.test_index
    # step2. 损失函数配置
    criterion = paddle.nn.loss.CrossEntropyLoss()
    # step3. 模型配置
    if args.model_name == 'GCN':
        gnn_model = GCN(input_size=graph.node_feat["feat"].shape[1],
                                num_class=dataset.num_classes,
                                num_layers=args.num_layers,
                                dropout=0.5,
                                hidden_size=128)
    if args.model_name == 'GAT':
        gnn_model = GAT(input_size=graph.node_feat["feat"].shape[1],
                        num_class=dataset.num_classes,
                        num_layers=args.num_layers,
                        feat_drop=0.6,
                        attn_drop=0.6,
                        num_heads=8,
                        hidden_size=64)
    if args.model_name == 'GCNII':
        gnn_model = GCNII(input_size=graph.node_feat["feat"].shape[1],
                        num_class=dataset.num_classes,
                        num_layers=args.num_layers,
                        )
    # step4. 优化方法配置
    optim = Adam(
            learning_rate=args.lr,
            parameters=gnn_model.parameters(),
            weight_decay=0.0005)
    # step4. 模型训练
    gnn_model.train()
    for epoch in range(args.epoch):
        train_loss, train_acc = train(train_index, train_label, gnn_model,
                                          graph, criterion, optim)

        val_loss, val_acc = eval(val_index, val_label, gnn_model, graph,
                                    criterion)
        cal_val_acc.append(val_acc.numpy())
        cal_val_loss.append(val_loss.numpy())
        layer_state_dict = gnn_model.state_dict()
        paddle.save(layer_state_dict,"work/GCN.pdparams")
        # paddle.save(gnn_model.weight,"work/GCN.weight.pdtensor")
        log.info(f"epoch{epoch}:Model:val Accuracy:{val_acc.numpy()}   Loss:{val_loss.numpy()}")
        if val_acc.numpy()>0.71:
            print(val_acc.numpy())
            break
    # log.info("Runs %s: Model: GCN Best Test Accuracy: %f" %
    #              (run, cal_test_acc[np.argmin(cal_val_loss)]))
    # best_test.append(cal_test_acc[np.argmin(cal_val_loss)])
    # log.info("Average Speed %s sec/ epoch" % (np.mean(dur)))
    # log.info("Dataset: %s Best Test Accuracy: %f ( stddev: %f )" %
    #          (args.dataset, np.mean(best_test), np.std(best_test)))

if __name__=="__main__":
    parser = argparse.ArgumentParser(
        description='Benchmarking Citation Network')
    parser.add_argument("--model_name", type=str,default='GCN', help="model_name")
    parser.add_argument("--num_layers",type=int,default=1,help="layers number")
    parser.add_argument("--lr",type=float,default=0.01,help="learning_rate")
    parser.add_argument("--epoch", type=int, default=200, help="Epoch")
    parser.add_argument("--runs", type=int, default=10, help="runs")
    parser.add_argument(
        "--feature_pre_normalize",
        type=bool,
        default=True,
        help="pre_normalize feature")
    args = parser.parse_args()
    log.info(args)
    main(args = args)


import json
from unified.core.actions import Actions
from documents.google_docs import util
from documents.google_docs.entities.google_docs_document import GoogledocsDocument

class GoogledocsAction(Actions):

    def create_document_from_text(self, context, payload):
        '''Create a  file with text.'''

        access_token = util.get_access_token(context["headers"])
        url = "https://docs.googleapis.com/v1/documents"
        file = GoogledocsDocument(**payload)
        create_file = {
            "title": file.name
        }
        response = util.rest("post", url, access_token, create_file)

        if response.ok:
            file_id = json.loads(response.text)['documentId']
            # updating content in docs
            update_content_response = self.create_content_in_file(file_id, file.content, access_token)

            if update_content_response.ok:
                # move file to given folder id
                if file.folder_id:
                    move_file = self.move_file(access_token, file_id, file.folder_id)
                update_content_obj = json.loads(update_content_response.text)
                update_content_obj['id'] = file_id

                return json.loads(update_content_obj), update_content_response.status_code

        return json.loads(response.text), response.status_code

    def create_content_in_file(self, file_id, file_content, access_token):
        '''create a file content'''

        url = f"https://docs.googleapis.com/v1/documents/{file_id}:batchUpdate"
        request_body = {
            "requests": [{"insertText": {
                "text": file_content,
                "endOfSegmentLocation": {"segmentId": ""}}}]}
        return util.rest("POST", url, access_token, request_body)

    def move_file(self, access_token, file_id, folder_id):
        '''Move a file from one folder to another.'''

        url = f"https://www.googleapis.com/drive/v2/files/{file_id}/parents"
        request_body = {
            "id": folder_id
        }
        response = util.rest("POST", url, access_token, request_body)

        return response

    def create_document_from_template(self, context, payload):
        '''Creates a new doc based on an existing one'''

        access_token = util.get_access_token(context["headers"])
        file = GoogledocsDocument(**payload)
        url = f"https://www.googleapis.com/drive/v2/files/{file.template_id}/copy"
        file_body = {
            "title": file.name
        }

        response = util.rest("POST", url, access_token, file_body)

        if response.ok:
            templete = json.loads(response.text)
            file_id = templete.get('id')
            # Moving file to given folder
            if file.folder_id:
                move_file_response = self.move_file(access_token, file_id, file.folder_id)
                if not move_file_response.ok:
                    return json.loads(move_file_response.text), move_file_response.status_code

            return json.loads(response.text), response.status_code

    def append_text_to_document(self, context, payload):
        '''Appends text to an existing document.'''

        file = GoogledocsDocument(**payload)
        access_token = util.get_access_token(context["headers"])
        data = self.create_content_in_file(file.document_id, file.text_to_append, access_token)
        return json.loads(data.text), data.status_code


# -*- coding: utf-8 -*-
"""
Created on Wed May 20 13:02:06 2020

@author: srinivasan.c
"""

from keras.preprocessing.image import ImageDataGenerator, array_to_img,img_to_array,load_img
import tensorflow as tf

datagen = ImageDataGenerator(
        rotation_range=180,
        width_shift_range = 0.2,
        height_shift_range = 0.2,
        shear_range = 0.2,
        zoom_range = 0.2,
        horizontal_flip = True,
        fill_mode = 'nearest'
        )
img = load_img('D:\python\data\image\dog1.jpg')
x = img_to_array(img)
x = x.reshape((1,) + x.shape)

file_name = 'dog_' #filename prefix
i = 1
format_type = 'jpg' # format type for generated image
save_dir = 'D:/python/CNN/test_data/dog/'
for batch in datagen.flow(x, batch_size = 1,
                           save_to_dir = save_dir, save_prefix = 'dog',
                           save_format = format_type):
    i += 1
    if i > 70:
        break

import os
os.getcwd()
collection = save_dir
for i, filename in enumerate(os.listdir(collection)):
    i+=1
    os.rename(save_dir + filename, save_dir + file_name + str(i) + "."+format_type)


import pandas as pd
import numpy as np
import numpy.random as rd
import sklearn
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler

def pca_trafo(pose_feats,comp):
    n_comp = comp
    pca_trafo = PCA(n_components=n_comp)
    data = pose_feats

    pca_data = pca_trafo.fit_transform(data)
    pca_inv_data = pca_trafo.inverse_transform(pca_data)

    """plt.figure(figsize = (10,6.5));
    plt.semilogy(pca_trafo.explained_variance_ratio_, '--o');
    plt.xlabel('principal component', fontsize = 20);
    plt.ylabel('explained variance', fontsize = 20);
    plt.tick_params(axis='both', which='major', labelsize=10);
    plt.tick_params(axis='both', which='minor', labelsize=10);
    plt.xlim([0, 66]);"""

    z_scaler = StandardScaler()
    z_data = z_scaler.fit_transform(data)
    pca_trafo2 = PCA().fit(z_data);

    fig, ax1 = plt.subplots(figsize = (10,6.5))
    ax1.set_xlabel('principal component', fontsize = 10);

    plt.legend(loc=(0.01, 0.075) ,fontsize = 10);

    ax2 = ax1.twinx()
    ax2.semilogy(pca_trafo.explained_variance_ratio_.cumsum(), '--go', label = 'cumulative explained variance ratio');
    for tl in ax2.get_yticklabels():
        tl.set_color('g')

    ax2.tick_params(axis='both', which='minor', labelsize=10);
    ax2.tick_params(axis='both', which='minor', labelsize=10);
    plt.xlim([0, 66]);
    plt.legend(loc=(0.01, 0),fontsize = 18);

    fig = plt.figure(figsize=(12, 10))
    sns.heatmap(pca_trafo.inverse_transform(np.eye(n_comp)), cmap="hot", cbar=False)
    plt.ylabel('principal component', fontsize=20);
    plt.xlabel('original feature index', fontsize=20);
    plt.tick_params(axis='both', which='major', labelsize=10);
    plt.tick_params(axis='both', which='minor', labelsize=10);

    fig = plt.figure(figsize=(12, 10))
    plt.plot(pca_inv_data.mean(axis=0), '--o', label = 'mean')
    plt.plot(np.square(pca_inv_data.std(axis=0)), '--o', label = 'variance')
    plt.legend(loc='lower right')
    plt.ylabel('feature contribution', fontsize=20);
    plt.xlabel('feature index', fontsize=20);
    plt.tick_params(axis='both', which='major', labelsize=10);
    plt.tick_params(axis='both', which='minor', labelsize=10);
    plt.xlim([0, 66])
    plt.legend(loc='lower left', fontsize=10)
    plt.show()
    return fig


import math
import os
import pickle
import re
import string
import time

from data.models import Data
from nltk import sent_tokenize, stem, word_tokenize
from nltk.corpus import stopwords
from nltk.probability import FreqDist
from nltk.stem import PorterStemmer, SnowballStemmer
from sklearn.feature_extraction.text import TfidfVectorizer

INPUT_FILES = "./input/Cranfield"

# Clear text

def remove_special_character(text):
    processed_text = text.lower()
    processed_text = processed_text.replace("’", "'")
    processed_text = processed_text.replace("“", '"')
    processed_text = processed_text.replace("”", '"')

    non_words = re.compile(r"[^A-Za-z']+")
    processed_text = re.sub(non_words, ' ', processed_text)

    return processed_text

def remove_stopwords(text):  # step 1
    stop_words = set(stopwords.words('english'))
    # xóa stopwords
    words = [
        w for w in text.split(" ")
        if w not in stop_words
    ]
    return ' '.join(words)

def remove_punctuation(text):  # step 2
    words = [
        char for char in text.split(" ")
        if char not in string.punctuation
    ]
    return " ".join(words)

def remove_stem(text):  # step 3
    porter = PorterStemmer()
    token_words = word_tokenize(text)
    words = [
        porter.stem(word) for word in token_words
    ]
    return " ".join(words)

def clear_text(text):
    # processing
    text = text.lower()
    text = remove_stopwords(text)
    text = remove_special_character(text)
    text = remove_punctuation(text)
    text = remove_stem(text)

    return text

# Read data

def get_text_from_file(filename):
    with open(filename, encoding='cp1252', mode='r') as f:
        text = f.read()
    f.close()
    return text

def read_data():
    data = []
    arr_file = []
    for doc_file in os.listdir(INPUT_FILES):
        filename = os.path.join(INPUT_FILES, doc_file)
        text = get_text_from_file(filename)
        if len(text) == 0:
            continue
        arr_file.append(doc_file.split(".")[0])
        data.append(clear_text(text))
    return data, arr_file

def read_data_from_database():
    data = []
    arr_file = []
    allFiles = Data.objects.all()
    for file in allFiles:
        text = file.text
        words = get_words_from_text(text)
        if len(text) == 0:
            continue
        arr_file.append(file.id)
        data.append(clear_text(text))
    return data, arr_file

# build index, tf_idf arrays

def build_tf_idf_sklearn(data):
    tfidfVectorizer = TfidfVectorizer(
        analyzer='word', ngram_range=(1, 6), min_df=0.01, sublinear_tf=True,
        use_idf=True, smooth_idf=True
    )

    tfidf_matrix = tfidfVectorizer.fit_transform(data)
    feature_names = tfidfVectorizer.get_feature_names()

    # init
    tfidf_scores = dict()
    for index in feature_names:
        tfidf_scores[index] = []
    for index in range(len(data)):
        feature_index = tfidf_matrix[index, :].nonzero()[1]
        for x in feature_index:
            tfidf_scores[feature_names[x]].append(
                [index, tfidf_matrix[index, x]])
    return tfidfVectorizer, tfidf_scores

def open_queries():
    result = dict()
    for i in open("./input/query.txt").readlines():
        t = i.split('\t')
        result[t[0]] = t[1]
    return result

def get_data_ground_truth():
    path = os.path.join('./input', 'RES')
    data = dict()

    for file in os.listdir(path):
        filename = os.path.join(path, file)
        text = get_text_from_file(filename)
        text = text.rstrip('\n')
        cutLine = text.split('\n')
        for index, line in enumerate(cutLine):
            # cutTab[1] chua can quan tam toi do chua can dung
            cutTab = line.split('\t')
            cutSpace = cutTab[0].split(" ")
            if cutSpace[0] not in data.keys():
                data[cutSpace[0]] = [cutSpace[1]]
            else:
                data[cutSpace[0]].append(cutSpace[1])
    return data

def get_relevant_ranking_for_query(query, tfidfVectorizer, tfidf_scores, feature_names, arr_file):
    # clear query
    query = clear_text(query)
    query = " ".join(
        [
            word for word in query.split(" ")
            if word in feature_names
        ]
    )

    # compute tf_idf
    tfidf_matrix = tfidfVectorizer.fit_transform([query])
    feature_index = tfidf_matrix[0, :].nonzero()[1]

    # get vocal in query
    feature_names_query = tfidfVectorizer.get_feature_names()

    # check word in vocal
    query_tfidf_scores = dict()

    for x in feature_index:
        if feature_names_query[x] in feature_names:
            if feature_names_query[x] not in query_tfidf_scores.keys():
                query_tfidf_scores[feature_names_query[x]] = [
                    tfidf_matrix[0, x]]
            else:
                query_tfidf_scores[feature_names_query[x]].append(
                    tfidf_matrix[0, x])

    # find q length
    q_length = 0

    relevant_between_words = dict()
    # compute relevant query and data_train
    relevant_between_words = {
        word: [
            [
                item[0],
                item[1] * query_tfidf_scores[word][0]
            ] for item in tfidf_scores[word]
        ] for word in query_tfidf_scores.keys()
    }

    for key, value in query_tfidf_scores.items():
        q_length += math.pow(value[0], 2)
    q_length = math.sqrt(q_length)

    q_score = dict()
    for _, value in relevant_between_words.items():
        for i in value:
            if i[0] not in q_score.keys():
                q_score[i[0]] = i[1]
            else:
                q_score[i[0]] += i[1]
    for key in q_score.keys():
        q_score[key] = q_score[key] / (q_length + 0.01)

    # rank
    q = sorted(q_score.items(), key=lambda item: item[1], reverse=True)

    x_retrieved = []
    for i in q:
        x_retrieved.append(arr_file[i[0]])
    return x_retrieved

def get_Average_Precision(x_retrieved, relevant_docs):
    # find R_Precision value
    validation_result = {'R': [], 'P': []}
    c = 0
    for i in range(len(relevant_docs)):
        if x_retrieved[i] in relevant_docs:
            c += 1
        validation_result['R'].append((c / len(relevant_docs)))
        validation_result['P'].append((c / (i + 1)))
    return sum(validation_result['P']) / len(validation_result['P'])

def train():
    try:
        pkl_file = open(os.path.join('.\input', 'data',
                                     'train_in_database.pickle'), 'rb')
        arr_file = pickle.load(pkl_file)
        tfidfVectorizer = pickle.load(pkl_file)
        tfidf_scores = pickle.load(pkl_file)
        pkl_file.close()
    except:
        data, arr_file = read_data_from_database()
        tfidfVectorizer, tfidf_scores = build_tf_idf_sklearn(data)
        with open(os.path.join('.\input', 'data', 'train_in_database.pickle'), mode='wb') as f:
            pickle.dump(arr_file, f)
            pickle.dump(tfidfVectorizer, f)
            pickle.dump(tfidf_scores, f)
        f.close()
    return arr_file, tfidfVectorizer, tfidf_scores

def search_in_database(query, arr_file, tfidfVectorizer, tfidf_scores):
    feature_names = tfidfVectorizer.get_feature_names()
    return get_relevant_ranking_for_query(
        query, tfidfVectorizer, tfidf_scores, feature_names, arr_file
    )

def main():
    try:
        pkl_file = open(os.path.join('.\input', 'data', 'train.pickle'), 'rb')
        arr_file = pickle.load(pkl_file)
        tfidfVectorizer = pickle.load(pkl_file)
        tfidf_scores = pickle.load(pkl_file)
        pkl_file.close()
    except:
        data, arr_file = read_data()
        tfidfVectorizer, tfidf_scores = build_tf_idf_sklearn(data)
        with open(os.path.join('.\input', 'data', 'train.pickle'), mode='wb') as f:
            pickle.dump(arr_file, f)
            pickle.dump(tfidfVectorizer, f)
            pickle.dump(tfidf_scores, f)
        f.close()

    feature_names = tfidfVectorizer.get_feature_names()
    queries = open_queries()
    list_of_x_retrieved = dict()
    for key, query in queries.items():
        list_of_x_retrieved[key] = get_relevant_ranking_for_query(
            query, tfidfVectorizer, tfidf_scores, feature_names, arr_file
        )

     # Bắt đầu đánh giá mô hình.
    data_ground_truth = get_data_ground_truth()
    Average_precision_of_all_x_retrieved = \
        {
            key: get_Average_Precision(value, data_ground_truth[key])
            for key, value in list_of_x_retrieved.items()
        }
    MAP = 0
    for key, value in Average_precision_of_all_x_retrieved.items():
        MAP += value
    MAP = MAP / len(Average_precision_of_all_x_retrieved)
    print("MAP:", MAP)

if __name__ == '__main__':
    t0 = time.clock()
    main()
    t1 = time.clock() - t0
    print("Time elapsed: ", t1)


## Title:       send.py
## Author:      Jeroen Venema
## Created:     25/10/2022
## Last update: 11/09/2023

## syntax
## send.py FILENAME <PORT> <BAUDRATE>
##

## Modinfo:
## 25/10/2022 initial version
## 10/09/2023 Script converts binary file to Intel Hex during transmission.
##            Using defaults as constants.
## 11/09/2023 Wait time variable introduced for handling PC serial drivers with low buffer memory

DEFAULT_START_ADDRESS = 0x40000
DEFAULT_SERIAL_PORT   = 'COM11'
DEFAULT_BAUDRATE      = 115200
DEFAULT_LINE_WAITTIME = 0       ## A value of +/- 0.003 Helps PC serial drivers with low buffer memory

def errorexit(message):
  print(message)
  print('Press ENTER to continue')
  input()
  exit()
  return

import sys
import time
import os.path
import tempfile
try:
  import serial
except ModuleNotFoundError:
  errorexit('Please install the \'pyserial\' module with pip')
try:
  from intelhex import IntelHex
except ModuleNotFoundError:
  errorexit('Please install the \'intelhex\' module with pip')

if len(sys.argv) == 1 or len(sys.argv) >4:
  sys.exit('Usage: send.py FILENAME <PORT> <BAUDRATE>')

if not os.path.isfile(sys.argv[1]):
  sys.exit(f'Error: file \'{sys.argv[1]}\' not found')

if len(sys.argv) == 2:
  serialport = DEFAULT_SERIAL_PORT

if len(sys.argv) >= 3:
  serialport = sys.argv[2]

if len(sys.argv) == 4:
  baudrate = int(sys.argv[3])
else:
  baudrate = DEFAULT_BAUDRATE

nativehexfile = ((sys.argv[1])[-3:] == 'hex') or ((sys.argv[1])[-4:] == 'ihex')

# report parameters used
print(f'Sending \'{sys.argv[1]}\' ', end="")
if nativehexfile: print('as native hex file')
else:
  print('in Intel Hex format')
  print(f'Using start address 0x{DEFAULT_START_ADDRESS:x}')
print(f'Using serial port {serialport}')
print(f'Using Baudrate {baudrate}')

if nativehexfile:
  file = open(sys.argv[1], "r")
  content = file.readlines()
else:
  # Instantiate ihex object and load binary file to it, write out as ihex format to temp file
  ihex = IntelHex()
  file = tempfile.TemporaryFile("w+t")
  ihex.loadbin(sys.argv[1], offset=DEFAULT_START_ADDRESS)
  ihex.write_hex_file(file)
  file.seek(0)

try:
  with serial.Serial(serialport, baudrate,rtscts=False,dsrdtr=False,timeout=None) as ser:
    print('Opening serial port...')
    time.sleep(1)
    print('Sending data...')

    if nativehexfile:
      for line in content:
        ser.write(str(line).encode('ascii'))
        time.sleep(DEFAULT_LINE_WAITTIME)
    else:
      for line in file:
        ser.write(str(line).encode('ascii'))
        time.sleep(DEFAULT_LINE_WAITTIME)

except serial.SerialException:
  errorexit('Error: serial port unavailable')

file.close()


import math
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from voicefixer.vocoder.config import Config

# From xin wang of nii
class SineGen(torch.nn.Module):
    """Definition of sine generator
    SineGen(samp_rate, harmonic_num = 0,
            sine_amp = 0.1, noise_std = 0.003,
            voiced_threshold = 0,
            flag_for_pulse=False)

    samp_rate: sampling rate in Hz
    harmonic_num: number of harmonic overtones (default 0)
    sine_amp: amplitude of sine-wavefrom (default 0.1)
    noise_std: std of Gaussian noise (default 0.003)
    voiced_thoreshold: F0 threshold for U/V classification (default 0)
    flag_for_pulse: this SinGen is used inside PulseGen (default False)

    Note: when flag_for_pulse is True, the first time step of a voiced
        segment is always sin(np.pi) or cos(0)
    """

    def __init__(
        self,
        samp_rate=24000,
        harmonic_num=0,
        sine_amp=0.1,
        noise_std=0.003,
        voiced_threshold=0,
        flag_for_pulse=False,
    ):
        super(SineGen, self).__init__()
        self.sine_amp = sine_amp
        self.noise_std = noise_std
        self.harmonic_num = harmonic_num
        self.dim = self.harmonic_num + 1
        self.sampling_rate = samp_rate
        self.voiced_threshold = voiced_threshold
        self.flag_for_pulse = flag_for_pulse

    def _f02uv(self, f0):
        # generate uv signal
        uv = torch.ones_like(f0)
        uv = uv * (f0 > self.voiced_threshold)
        return uv

    def _f02sine(self, f0_values):
        """f0_values: (batchsize, length, dim)
        where dim indicates fundamental tone and overtones
        """
        # convert to F0 in rad. The interger part n can be ignored
        # because 2 * np.pi * n doesn't affect phase
        rad_values = (f0_values / self.sampling_rate) % 1

        # initial phase noise (no noise for fundamental component)
        rand_ini = torch.rand(
            f0_values.shape[0], f0_values.shape[2], device=f0_values.device
        )
        rand_ini[:, 0] = 0
        rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini

        # instantanouse phase sine[t] = sin(2*pi \sum_i=1 ^{t} rad)
        if not self.flag_for_pulse:
            # for normal case

            # To prevent torch.cumsum numerical overflow,
            # it is necessary to add -1 whenever \sum_k=1^n rad_value_k > 1.
            # Buffer tmp_over_one_idx indicates the time step to add -1.
            # This will not change F0 of sine because (x-1) * 2*pi = x *2*pi
            tmp_over_one = torch.cumsum(rad_values, 1) % 1
            tmp_over_one_idx = (tmp_over_one[:, 1:, :] - tmp_over_one[:, :-1, :]) < 0
            cumsum_shift = torch.zeros_like(rad_values)
            cumsum_shift[:, 1:, :] = tmp_over_one_idx * -1.0

            sines = torch.sin(
                torch.cumsum(rad_values + cumsum_shift, dim=1) * 2 * np.pi
            )
        else:
            # If necessary, make sure that the first time step of every
            # voiced segments is sin(pi) or cos(0)
            # This is used for pulse-train generation

            # identify the last time step in unvoiced segments
            uv = self._f02uv(f0_values)
            uv_1 = torch.roll(uv, shifts=-1, dims=1)
            uv_1[:, -1, :] = 1
            u_loc = (uv < 1) * (uv_1 > 0)

            # get the instantanouse phase
            tmp_cumsum = torch.cumsum(rad_values, dim=1)
            # different batch needs to be processed differently
            for idx in range(f0_values.shape[0]):
                temp_sum = tmp_cumsum[idx, u_loc[idx, :, 0], :]
                temp_sum[1:, :] = temp_sum[1:, :] - temp_sum[0:-1, :]
                # stores the accumulation of i.phase within
                # each voiced segments
                tmp_cumsum[idx, :, :] = 0
                tmp_cumsum[idx, u_loc[idx, :, 0], :] = temp_sum

            # rad_values - tmp_cumsum: remove the accumulation of i.phase
            # within the previous voiced segment.
            i_phase = torch.cumsum(rad_values - tmp_cumsum, dim=1)

            # get the sines
            sines = torch.cos(i_phase * 2 * np.pi)
        return sines

    def forward(self, f0):
        """sine_tensor, uv = forward(f0)
        input F0: tensor(batchsize=1, length, dim=1)
                  f0 for unvoiced steps should be 0
        output sine_tensor: tensor(batchsize=1, length, dim)
        output uv: tensor(batchsize=1, length, 1)
        """

        with torch.no_grad():
            f0_buf = torch.zeros(f0.shape[0], f0.shape[1], self.dim, device=f0.device)
            # fundamental component
            f0_buf[:, :, 0] = f0[:, :, 0]
            for idx in np.arange(self.harmonic_num):
                # idx + 2: the (idx+1)-th overtone, (idx+2)-th harmonic
                f0_buf[:, :, idx + 1] = f0_buf[:, :, 0] * (idx + 2)

            # generate sine waveforms
            sine_waves = self._f02sine(f0_buf) * self.sine_amp

            # generate uv signal
            # uv = torch.ones(f0.shape)
            # uv = uv * (f0 > self.voiced_threshold)
            uv = self._f02uv(f0)

            # noise: for unvoiced should be similar to sine_amp
            #        std = self.sine_amp/3 -> max value ~ self.sine_amp
            # .       for voiced regions is self.noise_std
            noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3
            noise = noise_amp * torch.randn_like(sine_waves)

            # first: set the unvoiced part to 0 by uv
            # then: additive noise
            sine_waves = sine_waves * uv + noise
        return sine_waves, uv, noise

class LowpassBlur(nn.Module):
    """perform low pass filter after upsampling for anti-aliasing"""

    def __init__(self, channels=128, filt_size=3, pad_type="reflect", pad_off=0):
        super(LowpassBlur, self).__init__()
        self.filt_size = filt_size
        self.pad_off = pad_off
        self.pad_sizes = [
            int(1.0 * (filt_size - 1) / 2),
            int(np.ceil(1.0 * (filt_size - 1) / 2)),
        ]
        self.pad_sizes = [pad_size + pad_off for pad_size in self.pad_sizes]
        self.off = 0
        self.channels = channels

        if self.filt_size == 1:
            a = np.array(
                [
                    1.0,
                ]
            )
        elif self.filt_size == 2:
            a = np.array([1.0, 1.0])
        elif self.filt_size == 3:
            a = np.array([1.0, 2.0, 1.0])
        elif self.filt_size == 4:
            a = np.array([1.0, 3.0, 3.0, 1.0])
        elif self.filt_size == 5:
            a = np.array([1.0, 4.0, 6.0, 4.0, 1.0])
        elif self.filt_size == 6:
            a = np.array([1.0, 5.0, 10.0, 10.0, 5.0, 1.0])
        elif self.filt_size == 7:
            a = np.array([1.0, 6.0, 15.0, 20.0, 15.0, 6.0, 1.0])

        filt = torch.Tensor(a)
        filt = filt / torch.sum(filt)
        self.register_buffer("filt", filt[None, None, :].repeat((self.channels, 1, 1)))

        self.pad = get_pad_layer_1d(pad_type)(self.pad_sizes)

    def forward(self, inp):
        if self.filt_size == 1:
            return inp
        return F.conv1d(self.pad(inp), self.filt, groups=inp.shape[1])

def get_pad_layer_1d(pad_type):
    if pad_type in ["refl", "reflect"]:
        PadLayer = nn.ReflectionPad1d
    elif pad_type in ["repl", "replicate"]:
        PadLayer = nn.ReplicationPad1d
    elif pad_type == "zero":
        PadLayer = nn.ZeroPad1d
    else:
        print("Pad type [%s] not recognized" % pad_type)
    return PadLayer

class MovingAverageSmooth(torch.nn.Conv1d):
    def __init__(self, channels, window_len=3):
        """Initialize Conv1d module."""
        super(MovingAverageSmooth, self).__init__(
            in_channels=channels,
            out_channels=channels,
            kernel_size=1,
            groups=channels,
            bias=False,
        )

        torch.nn.init.constant_(self.weight, 1.0 / window_len)
        for p in self.parameters():
            p.requires_grad = False

    def forward(self, data):
        return super(MovingAverageSmooth, self).forward(data)

class Conv1d(torch.nn.Conv1d):
    """Conv1d module with customized initialization."""

    def __init__(self, *args, **kwargs):
        """Initialize Conv1d module."""
        super(Conv1d, self).__init__(*args, **kwargs)

    def reset_parameters(self):
        """Reset parameters."""
        torch.nn.init.kaiming_normal_(self.weight, nonlinearity="relu")
        if self.bias is not None:
            torch.nn.init.constant_(self.bias, 0.0)

class Stretch2d(torch.nn.Module):
    """Stretch2d module."""

    def __init__(self, x_scale, y_scale, mode="nearest"):
        """Initialize Stretch2d module.
        Args:
            x_scale (int): X scaling factor (Time axis in spectrogram).
            y_scale (int): Y scaling factor (Frequency axis in spectrogram).
            mode (str): Interpolation mode.
        """
        super(Stretch2d, self).__init__()
        self.x_scale = x_scale
        self.y_scale = y_scale
        self.mode = mode

    def forward(self, x):
        """Calculate forward propagation.
        Args:
            x (Tensor): Input tensor (B, C, F, T).
        Returns:
            Tensor: Interpolated tensor (B, C, F * y_scale, T * x_scale),
        """
        return F.interpolate(
            x, scale_factor=(self.y_scale, self.x_scale), mode=self.mode
        )

class Conv2d(torch.nn.Conv2d):
    """Conv2d module with customized initialization."""

    def __init__(self, *args, **kwargs):
        """Initialize Conv2d module."""
        super(Conv2d, self).__init__(*args, **kwargs)

    def reset_parameters(self):
        """Reset parameters."""
        self.weight.data.fill_(1.0 / np.prod(self.kernel_size))
        if self.bias is not None:
            torch.nn.init.constant_(self.bias, 0.0)

class UpsampleNetwork(torch.nn.Module):
    """Upsampling network module."""

    def __init__(
        self,
        upsample_scales,
        nonlinear_activation=None,
        nonlinear_activation_params={},
        interpolate_mode="nearest",
        freq_axis_kernel_size=1,
        use_causal_conv=False,
    ):
        """Initialize upsampling network module.
        Args:
            upsample_scales (list): List of upsampling scales.
            nonlinear_activation (str): Activation function name.
            nonlinear_activation_params (dict): Arguments for specified activation function.
            interpolate_mode (str): Interpolation mode.
            freq_axis_kernel_size (int): Kernel size in the direction of frequency axis.
        """
        super(UpsampleNetwork, self).__init__()
        self.use_causal_conv = use_causal_conv
        self.up_layers = torch.nn.ModuleList()
        for scale in upsample_scales:
            # interpolation layer
            stretch = Stretch2d(scale, 1, interpolate_mode)
            self.up_layers += [stretch]

            # conv layer
            assert (
                freq_axis_kernel_size - 1
            ) % 2 == 0, "Not support even number freq axis kernel size."
            freq_axis_padding = (freq_axis_kernel_size - 1) // 2
            kernel_size = (freq_axis_kernel_size, scale * 2 + 1)
            if use_causal_conv:
                padding = (freq_axis_padding, scale * 2)
            else:
                padding = (freq_axis_padding, scale)
            conv = Conv2d(1, 1, kernel_size=kernel_size, padding=padding, bias=False)
            self.up_layers += [conv]

            # nonlinear
            if nonlinear_activation is not None:
                nonlinear = getattr(torch.nn, nonlinear_activation)(
                    **nonlinear_activation_params
                )
                self.up_layers += [nonlinear]

    def forward(self, c):
        """Calculate forward propagation.
        Args:
            c : Input tensor (B, C, T).
        Returns:
            Tensor: Upsampled tensor (B, C, T'), where T' = T * prod(upsample_scales).
        """
        c = c.unsqueeze(1)  # (B, 1, C, T)
        for f in self.up_layers:
            if self.use_causal_conv and isinstance(f, Conv2d):
                c = f(c)[..., : c.size(-1)]
            else:
                c = f(c)
        return c.squeeze(1)  # (B, C, T')

class ConvInUpsampleNetwork(torch.nn.Module):
    """Convolution + upsampling network module."""

    def __init__(
        self,
        upsample_scales=[3, 4, 5, 5],
        nonlinear_activation="ReLU",
        nonlinear_activation_params={},
        interpolate_mode="nearest",
        freq_axis_kernel_size=1,
        aux_channels=80,
        aux_context_window=0,
        use_causal_conv=False,
    ):
        """Initialize convolution + upsampling network module.
        Args:
            upsample_scales (list): List of upsampling scales.
            nonlinear_activation (str): Activation function name.
            nonlinear_activation_params (dict): Arguments for specified activation function.
            mode (str): Interpolation mode.
            freq_axis_kernel_size (int): Kernel size in the direction of frequency axis.
            aux_channels (int): Number of channels of pre-convolutional layer.
            aux_context_window (int): Context window size of the pre-convolutional layer.
            use_causal_conv (bool): Whether to use causal structure.
        """
        super(ConvInUpsampleNetwork, self).__init__()
        self.aux_context_window = aux_context_window
        self.use_causal_conv = use_causal_conv and aux_context_window > 0
        # To capture wide-context information in conditional features
        kernel_size = (
            aux_context_window + 1 if use_causal_conv else 2 * aux_context_window + 1
        )
        # NOTE(kan-bayashi): Here do not use padding because the input is already padded
        self.conv_in = Conv1d(
            aux_channels, aux_channels, kernel_size=kernel_size, bias=False
        )
        self.upsample = UpsampleNetwork(
            upsample_scales=upsample_scales,
            nonlinear_activation=nonlinear_activation,
            nonlinear_activation_params=nonlinear_activation_params,
            interpolate_mode=interpolate_mode,
            freq_axis_kernel_size=freq_axis_kernel_size,
            use_causal_conv=use_causal_conv,
        )

    def forward(self, c):
        """Calculate forward propagation.
        Args:
            c : Input tensor (B, C, T').
        Returns:
            Tensor: Upsampled tensor (B, C, T),
                where T = (T' - aux_context_window * 2) * prod(upsample_scales).
        Note:
            The length of inputs considers the context window size.
        """
        c_ = self.conv_in(c)
        c = c_[:, :, : -self.aux_context_window] if self.use_causal_conv else c_
        return self.upsample(c)

class DownsampleNet(nn.Module):
    def __init__(self, input_size, output_size, upsample_factor, hp=None, index=0):
        super(DownsampleNet, self).__init__()
        self.input_size = input_size
        self.output_size = output_size
        self.upsample_factor = upsample_factor
        self.skip_conv = nn.Conv1d(input_size, output_size, kernel_size=1)
        self.index = index
        layer = nn.Conv1d(
            input_size,
            output_size,
            kernel_size=upsample_factor * 2,
            stride=upsample_factor,
            padding=upsample_factor // 2 + upsample_factor % 2,
        )

        self.layer = nn.utils.weight_norm(layer)

    def forward(self, inputs):
        B, C, T = inputs.size()
        res = inputs[:, :, :: self.upsample_factor]
        skip = self.skip_conv(res)

        outputs = self.layer(inputs)
        outputs = outputs + skip

        return outputs

class UpsampleNet(nn.Module):
    def __init__(self, input_size, output_size, upsample_factor, hp=None, index=0):

        super(UpsampleNet, self).__init__()
        self.up_type = Config.up_type
        self.use_smooth = Config.use_smooth
        self.use_drop = Config.use_drop
        self.input_size = input_size
        self.output_size = output_size
        self.upsample_factor = upsample_factor
        self.skip_conv = nn.Conv1d(input_size, output_size, kernel_size=1)
        self.index = index
        if self.use_smooth:
            window_lens = [5, 5, 4, 3]
            self.window_len = window_lens[index]

        if self.up_type != "pn" or self.index < 3:
            # if self.up_type != "pn":
            layer = nn.ConvTranspose1d(
                input_size,
                output_size,
                upsample_factor * 2,
                upsample_factor,
                padding=upsample_factor // 2 + upsample_factor % 2,
                output_padding=upsample_factor % 2,
            )
            self.layer = nn.utils.weight_norm(layer)
        else:
            self.layer = nn.Sequential(
                nn.ReflectionPad1d(1),
                nn.utils.weight_norm(
                    nn.Conv1d(input_size, output_size * upsample_factor, kernel_size=3)
                ),
                nn.LeakyReLU(),
                nn.ReflectionPad1d(1),
                nn.utils.weight_norm(
                    nn.Conv1d(
                        output_size * upsample_factor,
                        output_size * upsample_factor,
                        kernel_size=3,
                    )
                ),
                nn.LeakyReLU(),
                nn.ReflectionPad1d(1),
                nn.utils.weight_norm(
                    nn.Conv1d(
                        output_size * upsample_factor,
                        output_size * upsample_factor,
                        kernel_size=3,
                    )
                ),
                nn.LeakyReLU(),
            )

        if hp is not None:
            self.org = Config.up_org
            self.no_skip = Config.no_skip
        else:
            self.org = False
            self.no_skip = True

        if self.use_smooth:
            self.mas = nn.Sequential(
                # LowpassBlur(output_size, self.window_len),
                MovingAverageSmooth(output_size, self.window_len),
                # MovingAverageSmooth(output_size, self.window_len),
            )

    def forward(self, inputs):

        if not self.org:
            inputs = inputs + torch.sin(inputs)
            B, C, T = inputs.size()
            res = inputs.repeat(1, self.upsample_factor, 1).view(B, C, -1)
            skip = self.skip_conv(res)
            if self.up_type == "repeat":
                return skip

        outputs = self.layer(inputs)
        if self.up_type == "pn" and self.index > 2:
            B, c, l = outputs.size()
            outputs = outputs.view(B, -1, l * self.upsample_factor)

        if self.no_skip:
            return outputs

        if not self.org:
            outputs = outputs + skip

        if self.use_smooth:
            outputs = self.mas(outputs)

        if self.use_drop:
            outputs = F.dropout(outputs, p=0.05)

        return outputs

class ResStack(nn.Module):
    def __init__(self, channel, kernel_size=3, resstack_depth=4, hp=None):
        super(ResStack, self).__init__()

        self.use_wn = Config.use_wn
        self.use_shift_scale = Config.use_shift_scale
        self.channel = channel

        def get_padding(kernel_size, dilation=1):
            return int((kernel_size * dilation - dilation) / 2)

        if self.use_shift_scale:
            self.scale_conv = nn.utils.weight_norm(
                nn.Conv1d(
                    channel, 2 * channel, kernel_size=kernel_size, dilation=1, padding=1
                )
            )

        if not self.use_wn:
            self.layers = nn.ModuleList(
                [
                    nn.Sequential(
                        nn.LeakyReLU(),
                        nn.utils.weight_norm(
                            nn.Conv1d(
                                channel,
                                channel,
                                kernel_size=kernel_size,
                                dilation=3 ** (i % 10),
                                padding=get_padding(kernel_size, 3 ** (i % 10)),
                            )
                        ),
                        nn.LeakyReLU(),
                        nn.utils.weight_norm(
                            nn.Conv1d(
                                channel,
                                channel,
                                kernel_size=kernel_size,
                                dilation=1,
                                padding=get_padding(kernel_size, 1),
                            )
                        ),
                    )
                    for i in range(resstack_depth)
                ]
            )
        else:
            self.wn = WaveNet(
                in_channels=channel,
                out_channels=channel,
                cin_channels=-1,
                num_layers=resstack_depth,
                residual_channels=channel,
                gate_channels=channel,
                skip_channels=channel,
                # kernel_size=5,
                # dilation_rate=3,
                causal=False,
                use_downup=False,
            )

    def forward(self, x):
        if not self.use_wn:
            for layer in self.layers:
                x = x + layer(x)
        else:
            x = self.wn(x)

        if self.use_shift_scale:
            m_s = self.scale_conv(x)
            m_s = m_s[:, :, :-1]

            m, s = torch.split(m_s, self.channel, dim=1)
            s = F.softplus(s)

            x = m + s * x[:, :, 1:]  # key!!!
            x = F.pad(x, pad=(1, 0), mode="constant", value=0)

        return x

class WaveNet(nn.Module):
    def __init__(
        self,
        in_channels=1,
        out_channels=1,
        num_layers=10,
        residual_channels=64,
        gate_channels=64,
        skip_channels=64,
        kernel_size=3,
        dilation_rate=2,
        cin_channels=80,
        hp=None,
        causal=False,
        use_downup=False,
    ):
        super(WaveNet, self).__init__()

        self.in_channels = in_channels
        self.causal = causal
        self.num_layers = num_layers
        self.out_channels = out_channels
        self.gate_channels = gate_channels
        self.residual_channels = residual_channels
        self.skip_channels = skip_channels
        self.cin_channels = cin_channels
        self.kernel_size = kernel_size
        self.use_downup = use_downup

        self.front_conv = nn.Sequential(
            nn.Conv1d(
                in_channels=self.in_channels,
                out_channels=self.residual_channels,
                kernel_size=3,
                padding=1,
            ),
            nn.ReLU(),
        )
        if self.use_downup:
            self.downup_conv = nn.Sequential(
                nn.Conv1d(
                    in_channels=self.residual_channels,
                    out_channels=self.residual_channels,
                    kernel_size=3,
                    stride=2,
                    padding=1,
                ),
                nn.ReLU(),
                nn.Conv1d(
                    in_channels=self.residual_channels,
                    out_channels=self.residual_channels,
                    kernel_size=3,
                    stride=2,
                    padding=1,
                ),
                nn.ReLU(),
                UpsampleNet(self.residual_channels, self.residual_channels, 4, hp),
            )

        self.res_blocks = nn.ModuleList()
        for n in range(self.num_layers):
            self.res_blocks.append(
                ResBlock(
                    self.residual_channels,
                    self.gate_channels,
                    self.skip_channels,
                    self.kernel_size,
                    dilation=dilation_rate**n,
                    cin_channels=self.cin_channels,
                    local_conditioning=(self.cin_channels > 0),
                    causal=self.causal,
                    mode="SAME",
                )
            )
        self.final_conv = nn.Sequential(
            nn.ReLU(),
            Conv(self.skip_channels, self.skip_channels, 1, causal=self.causal),
            nn.ReLU(),
            Conv(self.skip_channels, self.out_channels, 1, causal=self.causal),
        )

    def forward(self, x, c=None):
        return self.wavenet(x, c)

    def wavenet(self, tensor, c=None):

        h = self.front_conv(tensor)
        if self.use_downup:
            h = self.downup_conv(h)
        skip = 0
        for i, f in enumerate(self.res_blocks):
            h, s = f(h, c)
            skip += s
        out = self.final_conv(skip)
        return out

    def receptive_field_size(self):
        num_dir = 1 if self.causal else 2
        dilations = [2 ** (i % self.num_layers) for i in range(self.num_layers)]
        return (
            num_dir * (self.kernel_size - 1) * sum(dilations)
            + 1
            + (self.front_channels - 1)
        )

    def remove_weight_norm(self):
        for f in self.res_blocks:
            f.remove_weight_norm()

class Conv(nn.Module):
    def __init__(
        self,
        in_channels,
        out_channels,
        kernel_size,
        dilation=1,
        causal=False,
        mode="SAME",
    ):
        super(Conv, self).__init__()

        self.causal = causal
        self.mode = mode
        if self.causal and self.mode == "SAME":
            self.padding = dilation * (kernel_size - 1)
        elif self.mode == "SAME":
            self.padding = dilation * (kernel_size - 1) // 2
        else:
            self.padding = 0
        self.conv = nn.Conv1d(
            in_channels,
            out_channels,
            kernel_size,
            dilation=dilation,
            padding=self.padding,
        )
        self.conv = nn.utils.weight_norm(self.conv)
        nn.init.kaiming_normal_(self.conv.weight)

    def forward(self, tensor):
        out = self.conv(tensor)
        if self.causal and self.padding is not 0:
            out = out[:, :, : -self.padding]
        return out

    def remove_weight_norm(self):
        nn.utils.remove_weight_norm(self.conv)

class ResBlock(nn.Module):
    def __init__(
        self,
        in_channels,
        out_channels,
        skip_channels,
        kernel_size,
        dilation,
        cin_channels=None,
        local_conditioning=True,
        causal=False,
        mode="SAME",
    ):
        super(ResBlock, self).__init__()
        self.causal = causal
        self.local_conditioning = local_conditioning
        self.cin_channels = cin_channels
        self.mode = mode

        self.filter_conv = Conv(
            in_channels, out_channels, kernel_size, dilation, causal, mode
        )
        self.gate_conv = Conv(
            in_channels, out_channels, kernel_size, dilation, causal, mode
        )
        self.res_conv = nn.Conv1d(out_channels, in_channels, kernel_size=1)
        self.skip_conv = nn.Conv1d(out_channels, skip_channels, kernel_size=1)
        self.res_conv = nn.utils.weight_norm(self.res_conv)
        self.skip_conv = nn.utils.weight_norm(self.skip_conv)

        if self.local_conditioning:
            self.filter_conv_c = nn.Conv1d(cin_channels, out_channels, kernel_size=1)
            self.gate_conv_c = nn.Conv1d(cin_channels, out_channels, kernel_size=1)
            self.filter_conv_c = nn.utils.weight_norm(self.filter_conv_c)
            self.gate_conv_c = nn.utils.weight_norm(self.gate_conv_c)

    def forward(self, tensor, c=None):
        h_filter = self.filter_conv(tensor)
        h_gate = self.gate_conv(tensor)

        if self.local_conditioning:
            h_filter += self.filter_conv_c(c)
            h_gate += self.gate_conv_c(c)

        out = torch.tanh(h_filter) * torch.sigmoid(h_gate)

        res = self.res_conv(out)
        skip = self.skip_conv(out)
        if self.mode == "SAME":
            return (tensor + res) * math.sqrt(0.5), skip
        else:
            return (tensor[:, :, 1:] + res) * math.sqrt(0.5), skip

    def remove_weight_norm(self):
        self.filter_conv.remove_weight_norm()
        self.gate_conv.remove_weight_norm()
        nn.utils.remove_weight_norm(self.res_conv)
        nn.utils.remove_weight_norm(self.skip_conv)
        nn.utils.remove_weight_norm(self.filter_conv_c)
        nn.utils.remove_weight_norm(self.gate_conv_c)

@torch.jit.script
def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
    n_channels_int = n_channels[0]
    in_act = input_a + input_b
    t_act = torch.tanh(in_act[:, :n_channels_int])
    s_act = torch.sigmoid(in_act[:, n_channels_int:])
    acts = t_act * s_act
    return acts

@torch.jit.script
def fused_res_skip(tensor, res_skip, n_channels):
    n_channels_int = n_channels[0]
    res = res_skip[:, :n_channels_int]
    skip = res_skip[:, n_channels_int:]
    return (tensor + res), skip

class ResStack2D(nn.Module):
    def __init__(self, channels=16, kernel_size=3, resstack_depth=4, hp=None):
        super(ResStack2D, self).__init__()
        channels = 16
        kernel_size = 3
        resstack_depth = 2
        self.channels = channels

        def get_padding(kernel_size, dilation=1):
            return int((kernel_size * dilation - dilation) / 2)

        self.layers = nn.ModuleList(
            [
                nn.Sequential(
                    nn.LeakyReLU(),
                    nn.utils.weight_norm(
                        nn.Conv2d(
                            1,
                            self.channels,
                            kernel_size,
                            dilation=(1, 3 ** (i)),
                            padding=(1, get_padding(kernel_size, 3 ** (i))),
                        )
                    ),
                    nn.LeakyReLU(),
                    nn.utils.weight_norm(
                        nn.Conv2d(
                            self.channels,
                            self.channels,
                            kernel_size,
                            dilation=(1, 3 ** (i)),
                            padding=(1, get_padding(kernel_size, 3 ** (i))),
                        )
                    ),
                    nn.LeakyReLU(),
                    nn.utils.weight_norm(nn.Conv2d(self.channels, 1, kernel_size=1)),
                )
                for i in range(resstack_depth)
            ]
        )

    def forward(self, tensor):
        x = tensor.unsqueeze(1)
        for layer in self.layers:
            x = x + layer(x)
        x = x.squeeze(1)

        return x

class FiLM(nn.Module):
    """
    feature-wise linear modulation
    """

    def __init__(self, input_dim, attribute_dim):
        super().__init__()
        self.input_dim = input_dim
        self.generator = nn.Conv1d(
            attribute_dim, input_dim * 2, kernel_size=3, padding=1
        )

    def forward(self, x, c):
        """
        x: (B, input_dim, seq)
        c: (B, attribute_dim, seq)
        """
        c = self.generator(c)
        m, s = torch.split(c, self.input_dim, dim=1)

        return x * s + m

class FiLMConv1d(nn.Module):
    """
    Conv1d with FiLMs in between
    """

    def __init__(self, in_size, out_size, attribute_dim, ins_norm=True, loop=1):
        super().__init__()
        self.loop = loop
        self.mlps = nn.ModuleList(
            [nn.Conv1d(in_size, out_size, kernel_size=3, padding=1)]
            + [
                nn.Conv1d(out_size, out_size, kernel_size=3, padding=1)
                for i in range(loop - 1)
            ]
        )
        self.films = nn.ModuleList([FiLM(out_size, attribute_dim) for i in range(loop)])
        self.ins_norm = ins_norm
        if self.ins_norm:
            self.norm = nn.InstanceNorm1d(attribute_dim)

    def forward(self, x, c):
        """
        x: (B, input_dim, seq)
        c: (B, attribute_dim, seq)
        """
        if self.ins_norm:
            c = self.norm(c)
        for i in range(self.loop):
            x = self.mlps[i](x)
            x = F.relu(x)
            x = self.films[i](x, c)

        return x


#!/usr/bin/python3
# -*- coding: utf-8 -*-

import sys
import os
import math
import yaml
import csv
import cv2
import shutil
import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets,layers,models,losses,optimizers,Model
from params import *

def inception(inputs,filters):
    a = layers.Conv2D(filters,(1,1),activation='relu',padding='same')(inputs)
    b = layers.Conv2D(filters,(1,1),activation='relu',padding='same')(inputs)
    b = layers.Conv2D(filters,(3,3),activation='relu',padding='same')(b)
    #c = layers.Conv2D(filters,(1,1),activation='relu',padding='same')(inputs)
    #c = layers.Conv2D(filters,(3,3),activation='relu',padding='same')(c)
    #c = layers.Conv2D(filters,(3,3),activation='relu',padding='same')(c)
    #d = layers.Conv2D(filters,(1,1),activation='relu',padding='same')(inputs)
    #d = layers.Conv2D(filters,(3,3),activation='relu',padding='same')(d)
    #d = layers.Conv2D(filters,(3,3),activation='relu',padding='same')(d)
    #d = layers.Conv2D(filters,(3,3),activation='relu',padding='same')(d)
    #e = layers.MaxPooling2D((3,3),strides=(1,1),padding='same')(inputs)
    #e = layers.Conv2D(filters,(1,1),activation='relu',padding='same')(e)
    #return layers.Concatenate(axis=3)([a,b,c,d,e])
    #return layers.Concatenate(axis=3)([a,b,c,e])
    return layers.Concatenate(axis=3)([a,b])

def reduction(inputs,f):
    a = layers.Conv2D(f,(3,3),strides=(2,2),activation='relu')(inputs)
    return a

def create(cutout_size,depth,filters,rate):
    if depth:
        inputs = layers.Input(shape=(cutout_size,cutout_size,4))
    else:
        inputs = layers.Input(shape=(cutout_size,cutout_size,3))
    a = inception(inputs,filters)
    a = reduction(a,filters)
    a = inception(a,filters)
    a = reduction(a,filters)
    a = inception(a,filters)
    a = reduction(a,filters)
    a = inception(a,filters)
    a = reduction(a,filters)
    a = inception(a,filters)
    a = reduction(a,filters)
    a = inception(a,filters)
    a = layers.Flatten()(a)
    outputs = layers.Dense(5)(a)
    model = Model(inputs=inputs,outputs=outputs)
    model.compile(
        optimizer=optimizers.Nadam(learning_rate=rate),
        loss='mean_absolute_error',
        metrics=['acc'])
    model.summary()
    return model

def infer(model,inputs):
    return model.predict(tf.expand_dims(inputs,0),steps=1)[0]

def load_dataset(depth,path,csv_name):
    inputs = []
    outputs = []
    instances = []
    with open(csv_name,newline='') as file:
        for row in csv.reader(file):
            r = 0
            name = row[r]
            r += 1
            center = (float(row[r]),float(row[r + 1]))
            r += 2
            screen = (float(row[r]),float(row[r + 1]))
            r += 2
            ndc = (float(row[r]),float(row[r + 1]),float(row[r + 2]))
            r += 3
            head_pos = (float(row[r]),float(row[r + 1]),float(row[r + 2]))
            r += 3
            head_dir = (float(row[r]),float(row[r + 1]))
            r += 2
            light_dir = (float(row[r]),float(row[r + 1]))
            r += 2
            light_color = (float(row[r]),float(row[r + 1]),float(row[r + 2]))
            r += 3
            ambient_color = (float(row[r]),float(row[r + 1]),float(row[r + 2]))
            r += 3
            skin_color = (float(row[r]),float(row[r + 1]),float(row[r + 2]))
            r += 3
            if depth:
                image = np.multiply(cv2.imread(path + name,cv2.IMREAD_UNCHANGED).astype(np.float32),1.0 / 255.0)
            else:
                image = np.multiply(cv2.imread(path + name).astype(np.float32),1.0 / 255.0)
            inputs.append(image)
            dx = screen[0] - center[0]
            dy = screen[1] - center[1]
            output = [dx,dy,ndc[2],head_dir[0],head_dir[1]]
            outputs.append(output)
            instances.append((name,center,screen,ndc,head_pos,head_dir,light_dir,light_color,ambient_color,skin_color))
    return (np.array(inputs),np.array(outputs),instances)

def train(model,dataset,epochs,batch_size):
    n = int(0.2 * dataset[0].shape[0])
    x_train = dataset[0][n:]
    y_train = dataset[1][n:]
    x_val = dataset[0][:n]
    y_val = dataset[1][:n]
    history = model.fit(
        x=x_train,
        y=y_train,
        validation_data=[x_val,y_val],
        epochs=epochs,
        batch_size=batch_size,
        verbose=2,
    )

def test(model,dataset,frame_width,frame_height,cutout_size):
    errors = []
    error_names = ['screen.x','screen.y','ndc.x','ndc.y','ndc.z','head_pos.x','head_pos.y','head_pos.z','head_dir.y','head_dir.p','skin.r','skin.g','skin.b']

    mxx = 2.7990382
    myy = 3.7320509
    mzz = -1.0020020
    mzw = -1.0
    mwz = -0.2002002
    mww = 0.0

    for i in range(0,len(dataset[0])):
        print('    {} / {}'.format(i,len(dataset[0]) - 1))
        inference = infer(model,dataset[0][i])
        dx = inference[0]
        dy = inference[1]
        ndcz = inference[2]
        heady = inference[3]
        headp = inference[4]
        screenx = dataset[2][i][1][0] + dx
        screeny = dataset[2][i][1][1] + dy
        escreenx = dataset[2][i][2][0] - screenx
        escreeny = dataset[2][i][2][1] - screeny
        ndcx = 2.0 * screenx / frame_width - 1.0
        ndcy = 1.0 - 2.0 * screeny / frame_height
        endcx = dataset[2][i][3][0] - ndcx
        endcy = dataset[2][i][3][1] - ndcy
        endcz = dataset[2][i][3][2] - ndcz
        z = (mzw - mww * ndcz) / (mwz * ndcz - mzz)
        homw = mwz * z + mww
        x = homw * ndcx / mxx
        y = homw * ndcy / myy
        ex = dataset[2][i][4][0] - x
        ey = dataset[2][i][4][1] - y
        ez = dataset[2][i][4][2] - z
        eheady = dataset[2][i][5][0] - heady
        eheadp = dataset[2][i][5][1] - headp
        errors.append((escreenx,escreeny,endcx,endcy,endcz,ex,ey,ez,eheady,eheadp))
    total = len(errors)
    n = len(errors[0])
    avg = []
    for i in range(0,n):
        avg.append(0.0)
    for error in errors:
        for i in range(0,n):
            avg[i] += error[i]
    for i in range(0,n):
        avg[i] /= total
    dif = []
    for i in range(0,n):
        dif.append(0.0)
    for error in errors:
        for i in range(0,n):
            d = error[i] - avg[i]
            dif[i] += d * d
    almost_total = total - 1.0
    for i in range(0,n):
        dif[i] /= almost_total
    stddev = []
    for i in range(0,n):
        stddev.append(math.sqrt(dif[i]))
    print('statistics:')
    for i in range(0,n):
        print('    {:>16}: {:>6.3f} +/- {:>6.3f}'.format(error_names[i],avg[i],stddev[i]))

if __name__ == '__main__':
    params = Params('./params.yaml')

    weights_name = './pose.h5'

    path1data = './data1/'
    csv1data = './data1/files.csv'
    path1test = './test1/'
    csv1test = './test1/files.csv'

    if len(sys.argv) < 2:
        print('usage:')
        print('')
        print('    python3 pose.py <command>')
        print('')
        print('where command is:')
        print('    train       - start training')
        print('    train more  - improve training')
        print('    test        - numerically test the network')
        exit(-1)

    if sys.argv[1] == 'train':
        print('creating model...')
        model = create(params.cutout_size,params.depth,params.pose_filters,params.pose_rate)
        print('loading data1 dataset...')
        dataset = load_dataset(params.depth,path1data,csv1data)
        if (len(sys.argv) > 2) and (sys.argv[2] == 'more'):
            print('loading old weights...')
            model.load_weights(weights_name)
        print('training...')
        train(model,dataset,params.pose_epochs,params.pose_batch_size)
        print('saving new weights...')
        model.save_weights(weights_name)

    elif sys.argv[1] == 'test':
        print('creating model...')
        model = create(params.cutout_size,params.depth,params.pose_filters,params.pose_rate)
        print('loading weights...')
        model.load_weights(weights_name)
        print('loading test0 dataset...')
        dataset = load_dataset(params.depth,path1test,csv1test)
        print('measuring statistics...')
        test(model,dataset,params.frame_width,params.frame_height,params.cutout_size)


from django.urls import path
from main import views

app_name = 'main'
urlpatterns = [
    path('', views.home, name='home'),
    path('music/', views.music_home, name='music_home'),
    path('adminlinks/', views.admin_links, name="admin_links"),
    path('dbjson/', views.dbjson, name="dbjson"),
]


import os,argparse,socket,sys
from itertools import count
import pandas as pd
import numpy as np
import AGNLCLib
import matplotlib
is_turgon = socket.gethostname() == 'turgon'
# Dont try to show graphs unless a terminal is attached and
# we're on a n appropriate server
if (sys.stdout.isatty() is False) or (is_turgon is False):
    matplotlib.use('Agg')

# setup global variables for use in the data pipeline (these can be overridden in environment)
HOMEDIR = os.environ['HOME']
TESTEXT = os.environ.get('TESTEXT','')
#json files for project configuration
PROJECTDIR = os.environ.get('PROJECTDIR','{}/git/AS5599_project'.format(HOMEDIR))
CONFIGDIR = os.environ.get('CONFIGDIR','{}/git/AS5599_project/config'.format(HOMEDIR))

# objects we have data for are all subdirs of the project dir, removing the code dir
AGN_NAMES = [ agn.name for agn in os.scandir(PROJECTDIR) if agn.is_dir()
              and agn.name not in ['code','config'] and agn.name[0] != '.']

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("agn", type=str,
                        help="AGN for light curve calibration",
                        choices=AGN_NAMES)
    parser.add_argument("fltr", type=str,
                        help="Filter band for calibration",
                        choices=["u","B","g","V","r","i","z","all"])
    parser.add_argument('-s', '--show-outliers',
                        action='store_true',
                        help='Create plot of calibration model showing \
unclipped outliers which have been included')
    parser.add_argument('-d', '--delta',type=np.float64,
                        help='Delta value used for calibration, here used in the point\
density function to detect meaningful outliers. Otherwise median cadence used')
    parser.add_argument('-x', '--no-snr',
                        action='store_true',
                        help='Do not print out SNR for filter bands (\
included  by default)')
    parser.add_argument('-v', '--verbose',
                        action='store_true',
                        help='Extra output logging for calculations \
such as number of values removed by clipping')
    args=parser.parse_args()

    if args.fltr == 'all':
        args.fltr = None
    noprint=np.logical_not(args.verbose)
    # Create lightcurve model for this AGN
    model = AGNLCLib.AGNLCModel(PROJECTDIR,CONFIGDIR,args.agn)
    periods = [kk for kk in model.config().observation_params()['periods'].keys()]

    if args.show_outliers:
        for pp in periods:
            AGNLCLib.CalibrationOutlierPlot(model,pp,fltr=args.fltr,add_model=True,
                                     overwrite=True,noprint=noprint,delta=args.delta)

    if args.no_snr is False:
        for pp in periods:
            AGNLCLib.CalibrationSNR(model,pp,fltr=args.fltr,noprint=noprint)


""" Tasks file for making things easier to test """
import shutil
from pathlib import Path
from invoke.tasks import task

@task()
def lint(ctx):
    """
    Linting step.

    :param ctx: Context
    :type ctx: Context
    """
    ctx.run('pylint *')

@task()
def build(ctx):
    """
    Build step.

    :param ctx: Context
    :type ctx: Context
    """
    ctx.run('pyinstaller -F --uac-admin --icon="./resources/ToGif.ico" --clean main.py')
    dist_dir = Path('ImageToGif')
    dist_file = dist_dir.joinpath('main.exe')
    new_file = dist_dir.joinpath('ImageToGif.exe')
    if new_file.exists():
        new_file.unlink()
    dist_file.rename(new_file)
    readme = Path('readme.txt')
    shutil.copy2(readme, dist_dir.joinpath(readme))


from io import StringIO
import os
import pytest

from mne.utils import (set_config, get_config, get_config_path,
                       set_memmap_min_size, _get_stim_channel, sys_info)

def test_config(tmpdir):
    """Test mne-python config file support."""
    tempdir = str(tmpdir)
    key = '_MNE_PYTHON_CONFIG_TESTING'
    value = '123456'
    value2 = '123'
    old_val = os.getenv(key, None)
    os.environ[key] = value
    assert (get_config(key) == value)
    del os.environ[key]
    # catch the warning about it being a non-standard config key
    assert (len(get_config('')) > 10)  # tuple of valid keys
    with pytest.warns(RuntimeWarning, match='non-standard'):
        set_config(key, None, home_dir=tempdir, set_env=False)
    assert (get_config(key, home_dir=tempdir) is None)
    pytest.raises(KeyError, get_config, key, raise_error=True)
    assert (key not in os.environ)
    with pytest.warns(RuntimeWarning, match='non-standard'):
        set_config(key, value, home_dir=tempdir, set_env=True)
    assert (key in os.environ)
    assert (get_config(key, home_dir=tempdir) == value)
    with pytest.warns(RuntimeWarning, match='non-standard'):
        set_config(key, None, home_dir=tempdir, set_env=True)
    assert (key not in os.environ)
    with pytest.warns(RuntimeWarning, match='non-standard'):
        set_config(key, None, home_dir=tempdir, set_env=True)
    assert (key not in os.environ)
    if old_val is not None:
        os.environ[key] = old_val
    # Check if get_config with key=None returns all config
    key = 'MNE_PYTHON_TESTING_KEY'
    assert key not in get_config(home_dir=tempdir)
    with pytest.warns(RuntimeWarning, match='non-standard'):
        set_config(key, value, home_dir=tempdir)
    assert get_config(home_dir=tempdir)[key] == value
    old_val = os.environ.get(key)
    try:  # os.environ should take precedence over config file
        os.environ[key] = value2
        assert get_config(home_dir=tempdir)[key] == value2
    finally:  # reset os.environ
        if old_val is None:
            os.environ.pop(key, None)
        else:
            os.environ[key] = old_val
    # Check what happens when we use a corrupted file
    json_fname = get_config_path(home_dir=tempdir)
    with open(json_fname, 'w') as fid:
        fid.write('foo{}')
    with pytest.warns(RuntimeWarning, match='not a valid JSON'):
        assert key not in get_config(home_dir=tempdir)
    with pytest.warns(RuntimeWarning, match='non-standard'):
        pytest.raises(RuntimeError, set_config, key, 'true', home_dir=tempdir)

    # degenerate conditions
    pytest.raises(ValueError, set_memmap_min_size, 1)
    pytest.raises(ValueError, set_memmap_min_size, 'foo')
    pytest.raises(TypeError, get_config, 1)
    pytest.raises(TypeError, set_config, 1)
    pytest.raises(TypeError, set_config, 'foo', 1)
    pytest.raises(TypeError, _get_stim_channel, 1, None)
    pytest.raises(TypeError, _get_stim_channel, [1], None)

def test_sys_info():
    """Test info-showing utility."""
    out = StringIO()
    sys_info(fid=out)
    out = out.getvalue()
    assert ('numpy:' in out)


import json
import boto3
from botocore.exceptions import ClientError

def update_thing_event(props):
    type = str(props['Type'])
    enable = True if str(props['Enable']).lower() == 'true' else False

    print('update thing event: requested', type, enable)

    try:
        client = boto3.client('iot')
        response = client.update_event_configurations(
            eventConfigurations={
                type: {
                    'Enabled': enable
                }
            }
        )
        print('update thing event: success', response)
        return type + '-' + str(enable)
    except Exception as e:
        print("update thing event: fail", e)
        return None

def disable_thing_event(id):
    print('disable thing event: requested')

    try:
        client = boto3.client('iot')
        response = client.update_event_configurations(
            eventConfigurations={
                type: {
                    'Enabled': False
                }
            }
        )
        print('update thing event: success', response)
        return 'ok'
    except Exception as e:
        print("update thing event: fail", e)
        return None

def handle(event, context):
    print('event-->', json.dumps(event))

    request_type = event['RequestType']
    physical_id = None

    if request_type == 'Create':
        physical_id = update_thing_event(event['ResourceProperties'])
    elif request_type == 'Delete':
        physical_id = event['PhysicalResourceId']
    elif request_type == 'Update':
        physical_id = update_thing_event(event['ResourceProperties'])

    if physical_id != None:
        return { 'PhysicalResourceId': physical_id }
    else:
        raise Exception('Fail to handl iot-event-message')


from django.urls import path

from myapp import views
app_name = 'myapp'
urlpatterns = [
    path(r'', views.index, name='index'),
    path(r'about/', views.about, name='about'),
    path(r'<int:id>', views.details, name='detail'),
    path(r'courses/', views.courses, name='courses'),
    path(r'place_order/', views.place_order, name='place_order'),
    path(r'courses/<int:cour_id>/', views.course_detail, name='course_detail')
]


# -*- coding: utf-8 -*-
"""
Created on Mon Aug  9 16:36:21 2021

@author: 20210595
"""

# -*- coding: utf-8 -*-
"""
Created on Sun Jul 25 00:58:52 2021

@author: 20210595
"""

# from . import space_autogenerated
# positionsClassifiers = space_autogenerated.positionsClassifiers
# positionsPreprocessingTechniques = space_autogenerated.positionsPreprocessingTechniques
# positions = space_autogenerated.positions
# float_int_positions = space_autogenerated.float_int_positions
from space_autogenerated import positionsPreprocessingTechniques, positionsModels, positions, float_int_positions

#Pipeline
from sklearn.pipeline import make_pipeline, Pipeline

#pre-processing techniques

from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import Normalizer

# statsmodels
from statsmodels.tsa.ar_model import AutoReg
from statsmodels.tsa.arima.model import ARIMA
from statsmodels.tsa.statespace.sarimax import SARIMAX
from statsmodels.tsa.holtwinters import ExponentialSmoothing
from statsmodels.tsa.holtwinters import Holt
from statsmodels.tsa.holtwinters import SimpleExpSmoothing
from statsmodels.tsa.statespace.exponential_smoothing import ExponentialSmoothing
from statsmodels.tsa.exponential_smoothing.ets import ETSModel
from statsmodels.tsa.statespace.dynamic_factor import DynamicFactor
from statsmodels.tsa.statespace.dynamic_factor_mq import DynamicFactorMQ
from statsmodels.tsa.vector_ar.var_model import VAR
from statsmodels.tsa.statespace.varmax import VARMAX
from statsmodels.tsa.vector_ar.svar_model import SVAR
from statsmodels.tsa.vector_ar.vecm import VECM
from statsmodels.tsa.statespace.structural import UnobservedComponents

#example_int_to_string = int_to_string(3, rbf = 0, kernel2 = 1)

class ChoosePipeline(object):
    def __init__(self, positionsModels = positionsModels, positions = positions, position_int_float = float_int_positions, positionsPreprocessingTechniques= positionsPreprocessingTechniques):
        self.position_int_float = position_int_float
        self.positions = positions
        self.positionsPreprocessingTechniques = positionsPreprocessingTechniques
        self.positionsModels = positionsModels

    def __call__(self, x, serie):

        #NewSerie = values.reshape((len(values), 1))
        _positions = self.positions
        build_vec = self._build_vector(x)
        list_index_techniques_to_use = []
        for i in self.positionsPreprocessingTechniques:
            if x[i] > 15:
                list_index_techniques_to_use.append(i)

        lista_index_final_prepross = []
        for i in range(len(positionsPreprocessingTechniques)):
            for j in list_index_techniques_to_use:
                if positionsPreprocessingTechniques[i]==j:
                    lista_index_final_prepross.append(i)

        preprocessPipeline = self._preprossessing_techniques(lista_index_final_prepross, build_vec)
        #return preprocessPipeline

        valueIndicesModels = [x[i] for i in self.positionsModels]
        max_value = max(valueIndicesModels)
        max_index = valueIndicesModels.index(max_value)
        indexModel = self.positionsModels[max_index]

        #print(positionModelNew)
        # valueIndicesModels = [x[i] for i in self.positionsModels]
        # max_value = max(valueIndicesModels)
        # max_index = valueIndicesModels.index(max_value)
        # indexModel = self.positionsModels[max_index]
        # #The last index is the classifier
        # list_index_techniques_to_use.append(indexModel)
        # lista_index_final = []
        # for i in range(len(_positions)):
        #     for j in list_index_techniques_to_use:
        #         if positions[i][1]==j:
        #             lista_index_final.append(i)
        try:
            PipelineTSFinal = self._techniques(indexModel, build_vec, preprocessPipeline, serie=serie)
        except:
            PipelineTSFinal = 'invalid'
        # PipelineToUse = self._techniques(indexModel, build_vec, preprocessPipeline, serie=serie)
        return preprocessPipeline, PipelineTSFinal
        # return PipelineTSFinal

    def _build_vector(self, x):
        _vector_int_float = x.copy()
        _vector_pos_copy = self.position_int_float.copy()
        for i in range(len(_vector_int_float)):
            if _vector_pos_copy[i][1] == 'int':
                _vector_int_float[i] = int(round(_vector_int_float[i]))
        return _vector_int_float

    def _preprossessing_techniques(self, lista_index_final_prepross, build_vec):
        x=build_vec
        estimators = [('SimpleImputer', SimpleImputer(strategy = self._int_to_string(round(x[1]), mean=0, median=1, most_frequent=2),
                                                      fill_value=x[2]))
                      ]
        new_estimators = [estimators[i] for i in lista_index_final_prepross]
        preprossPipeline = Pipeline(new_estimators)
        return preprossPipeline

    def _techniques(self, indexModel, build_vec, preprocessPipeline, serie):
        x=build_vec
        if indexModel == self.positions[len(self.positionsPreprocessingTechniques)][1]:
            pipelineModelTS = Pipeline([('AutoReg', AutoReg(preprocessPipeline.fit(serie).transform(serie),
                                          lags=round(x[4]),
                                          trend=self._int_to_string(round(x[5]), n=0, c=1, t=2, ct=3)
                                          ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+1][1]:
            pipelineModelTS = Pipeline([('ARIMA', ARIMA(preprocessPipeline.fit(serie).transform(serie),
                                      order=(round(x[7]),round(x[8]),round(x[9])),
                                      trend=self._int_to_string(round(x[10]), n=0, c=1, t=2, ct=3),
                                      enforce_stationarity=self._int_to_bool(round(x[11])),
                                      enforce_invertibility=self._int_to_bool(round(x[12])),
                                      concentrate_scale=self._int_to_bool(round(x[13]))
                                      ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+2][1]:
            pipelineModelTS = Pipeline([('SARIMAX', SARIMAX(preprocessPipeline.fit(serie).transform(serie),
                                          order=(round(x[15]),round(x[16]),round(x[17])),
                                          trend=self._int_to_string(round(x[18]), n=0, c=1, t=2, ct=3),
                                          enforce_stationarity=self._int_to_bool(round(x[19])),
                                          enforce_invertibility=self._int_to_bool(round(x[20]))
                                          ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+3][1]:
            pipelineModelTS = Pipeline([('ExponentialSmoothing', ExponentialSmoothing(preprocessPipeline.fit(serie).transform(serie),
                                                                  trend=self._int_to_string(round(x[22]), add=0, mul=1, additive=2, multiplicative=3),
                                                                  damped_trend=self._int_to_bool(round(x[23])),
                                                                  seasonal=self._int_to_string(round(x[24]), add=0, mul=1, additive=2, multiplicative=3),
                                                                  initialization_method=self._int_to_string(round(x[25]), estimated=0, heuristic=1, known=2),
                                                                  initial_level=x[26],
                                                                  initial_trend=x[27]
                                                                  ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+4][1]:
            pipelineModelTS = Pipeline([('Holt', Holt(preprocessPipeline.fit(serie).transform(serie),
                                    exponential=self._int_to_bool(round(x[29])),
                                    damped_trend=self._int_to_bool(round(x[30])),
                                    initialization_method=self._int_to_string(round(x[31]), estimated=0, heuristic=1, known=2),
                                    initial_level=x[32],
                                    initial_trend=x[33]
                                    ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+5][1]:
            pipelineModelTS = Pipeline([('SimpleExpSmoothing', SimpleExpSmoothing(preprocessPipeline.fit(serie).transform(serie),
                                                                initialization_method=self._int_to_string(round(x[35]), estimated=0, heuristic=1, known=2),
                                                                initial_level=x[36]
                                                                ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+6][1]:
            pipelineModelTS = Pipeline([('ETSModel', ETSModel(preprocessPipeline.fit(serie).transform(serie),
                                            error=self._int_to_string(round(x[38]), add=0, mul=1),
                                            trend=self._int_to_string(round(x[39]), add=0, mul=1),
                                            damped_trend=self._int_to_bool(round(x[40])),
                                            seasonal=self._int_to_string(round(x[41]), add=0, mul=1),
                                            seasonal_periods=round(x[42]),
                                            initialization_method=self._int_to_string(round(x[43]), estimated=0, heuristic=1, known=2),
                                            initial_level=x[44],
                                            initial_trend=x[45]
                                            ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+7][1]:
            pipelineModelTS = Pipeline([('DynamicFactor', DynamicFactor(preprocessPipeline.fit(serie).transform(serie),
                                                      k_factors=round(x[47]),
                                                      factor_order=round(x[48]),
                                                      error_cov_type=self._int_to_string(round(x[49]), scalar=0, diagonal=1, unstructured=2)
                                                      ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+8][1]:
            pipelineModelTS = Pipeline([('DynamicFactorMQ', DynamicFactorMQ(preprocessPipeline.fit(serie).transform(serie),
                                                          k_endog_monthly =round(x[51]),
                                                          factors=round(x[52]),
                                                          factor_orders=round(x[53]),
                                                          factor_multiplicities=round(x[54]),
                                                          idiosyncratic_ar1=self._int_to_bool(round(x[55])),
                                                          standardize=self._int_to_bool(round(x[56])),
                                                          init_t0=self._int_to_bool(round(x[57]))
                                                          ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+9][1]:
            pipelineModelTS = Pipeline([('VAR', VAR(preprocessPipeline.fit(serie).transform(serie)))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+10][1]:
            pipelineModelTS = Pipeline([('VARMAX', VARMAX(preprocessPipeline.fit(serie).transform(serie),
                                        order=(round(x[60]),round(x[61])),
                                        trend=self._int_to_string(round(x[62]), n=0, c=1, t=2, ct=3),
                                        error_cov_type=self._int_to_string(round(x[63]), diagonal=0, unstructured=1),
                                        measurement_error=self._int_to_bool(round(x[64])),
                                        enforce_stationarity=self._int_to_bool(round(x[65])),
                                        enforce_invertibility=self._int_to_bool(round(x[66]))
                                        ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+11][1]:
            pipelineModelTS = Pipeline([('SVAR', SVAR(preprocessPipeline.fit(serie).transform(serie),
                                    svar_typestr = self._int_to_string(round(x[68]), A=0, B=1, AB=2),
                                    ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+12][1]:
            pipelineModelTS = Pipeline([('VECM', VECM(preprocessPipeline.fit(serie).transform(serie),
                                    k_ar_diff=round(x[70]),
                                    coint_rank=round(x[71]),
                                    deterministic=self._int_to_string(round(x[72]), nc=0, co=1, ci=2, lo=3, li=4)
                                    ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+13][1]:
            pipelineModelTS = Pipeline([('UnobservedComponents', UnobservedComponents(preprocessPipeline.fit(serie).transform(serie),
                                                                    level=self._int_to_bool(round(x[74])),
                                                                    trend=self._int_to_bool(round(x[75])),
                                                                    seasonal=round(x[76]),
                                                                    cycle=self._int_to_bool(round(x[77])),
                                                                    autoregressive=round(x[78]),
                                                                    irregular=self._int_to_bool(round(x[79])),
                                                                    stochastic_level=self._int_to_bool(round(x[80])),
                                                                    stochastic_trend=self._int_to_bool(round(x[81])),
                                                                    stochastic_seasonal=self._int_to_bool(round(x[82])),
                                                                    stochastic_cycle=self._int_to_bool(round(x[83])),
                                                                    damped_cycle=self._int_to_bool(round(x[84])),
                                                                    use_exact_diffuse=self._int_to_bool(round(x[85]))
                                                                    ))])
        return pipelineModelTS

    def _int_to_string(self, value, **kwargs):
        for element in kwargs:
            if kwargs[element] == value:
                return element

    def _int_to_bool(self, value):
            if value == 1:
                return True
            else:
                return False

# from numpy import genfromtxt
# examples = genfromtxt('prueba.csv', delimiter=',')
# from pandas import read_csv
# import pandas as pd
# series = read_csv('ibm-common-stock-closing-prices.csv', header=0, index_col=0)
# NewSerie = series.values
# prueba = ChoosePipeline()
# # x_va = prueba(examples[6], NewSerie)

# nueva = list()
# for i in examples:
#     x_va = prueba(i, NewSerie)
#     nueva.append(x_va)

if __name__=='__main__':
    from numpy import genfromtxt
    examples = genfromtxt('prueba.csv', delimiter=',')
    from pandas import read_csv
    import pandas as pd
    series = read_csv('ibm-common-stock-closing-prices.csv', header=0, index_col=0)
    NewSerie = series.values
    prueba = ChoosePipeline()
    #x_va = prueba(examples[1], NewSerie)

    nueva = list()
    for i in examples:
        x_va = prueba(i, NewSerie)
        nueva.append(x_va)

# from sklearn import datasets

# # import some data to play with
# iris = datasets.load_iris()
# X = iris.data  # we only take the first two features.
# y = iris.target

# def _build_vector(x):
#     _vector_int_float = x.copy()
#     _vector_pos_copy = float_int_positions.copy()
#     for i in range(len(_vector_int_float)):
#         if _vector_pos_copy[i][1] == 'int':
#             _vector_int_float[i] = int(round(_vector_int_float[i]))
#     return _vector_int_float

# x = examples[1]
# build_vec = _build_vector(x)
# list_index_techniques_to_use = []
# for i in positionsPreprocessingTechniques:
#     if round(x[i]) == 1:
#         list_index_techniques_to_use.append(i)
# valueIndicesClassifiers = [x[i] for i in positionsClassifiers]
# max_value = max(valueIndicesClassifiers)
# max_index = valueIndicesClassifiers.index(max_value)
# indexClassifier = positionsClassifiers[max_index]
# #The last index is the classifier
# list_index_techniques_to_use.append(indexClassifier)

# auxiliar = []
# for i in range(len(positions)):
#     for j in list_index_techniques_to_use:
#         if positions[i][1]==j:
#             auxiliar.append(i)