#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Apr 22 10:36:47 2020

@author: jacquelinedowling
"""

#Fixed capacities unmet demand
#6 year test period in chunks

##===========================================
#Import stuff
##===========================================
from __future__ import division
import os
import sys
import copy
import numpy as np

import pickle
import numpy as np
from numpy import genfromtxt
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import matplotlib.ticker as ticker

import datetime
from matplotlib.dates import DayLocator, MonthLocator, HourLocator, AutoDateLocator, DateFormatter, drange
from matplotlib.dates import MO, TU, WE, TH, FR, SA, SU, WeekdayLocator
from numpy import arange
from matplotlib.ticker import ScalarFormatter
from matplotlib.ticker import FormatStrFormatter

import matplotlib.cm as cm
import matplotlib.mlab as mlab
import os
import re
import matplotlib.patches as mpatches

import matplotlib.ticker as mtick

#UPDATE TO 2018!!!!

pgp_q = 'pink'
batt_q = 'purple'

#pgp_q = '#ffb3ff'
#batt_q = '#8c1aff'
#dem_c = 'black'

#pgp_q = '#ffb3ff'
#batt_q = '#8c1aff'
#pgp_q = 'pink'
#batt_q = 'xkcd:violet'

##===========================================
#Read in Base Case: PGP Batteries, Wind Solar
##===========================================

pickle_in = open('/Users/jacquelinedowling/MEM_Nov2019/SEM-1.2/Output_Data/fixedcaps_yrs/fixedcaps_yrs_1yrs_2018_2018_tst1980.pickle', 'rb')
#pickle_in = open('/Users/jacquelinedowling/Documents/SEM-1.1_20190114/Output_Data/PGPtest5/PGPtest5_WindSolarPGPBatt_2015.pickle','rb')
base = pickle.load(pickle_in)
info = base[0]
inputs = base[1]
results = base[2]

#def get_pickles():
#
#    unsorted_pickles = [] #collection of all pickle files for triple year runs for ex
#    startyrs_list =[]
#
#    path = "/Users/jacquelinedowling/MEM_Nov2019/SEM-1.2/Output_Data/fixedcaps_2018/"
#    for filename in os.listdir(path):
#        if re.match("fixedcaps_2018_SolarWindPGPbatt2018_\d+.pickle",filename):
#            pickle_in = open(str(path)+ filename,"rb")
#            base = pickle.load(pickle_in)
#            inputs = base[1]
#            start_year = int(inputs['START_YEAR'])
#
#            unsorted_pickles.append(base)
#            startyrs_list.append(start_year)
#    sorted_pickles = [i for _,i in sorted(zip(startyrs_list, unsorted_pickles))]
#
#    return sorted_pickles

def get_group(cap_per):

    unsorted_pickles = [] #collection of all pickle files for triple year runs for ex
    startyrs_list =[]

    path = "/Users/jacquelinedowling/MEM_Nov2019/SEM-1.2/Output_Data/fixedcaps_chunks/"
#    fixedcaps_chunks_1yrcaps_2018_2018_chunk2
    for filename in os.listdir(path):
        if re.match("fixedcaps_chunks_"+str(cap_per)+"yrcaps_\d+_2018_chunk\d+.pickle",filename):
            pickle_in = open(str(path)+ filename,"rb")
            base = pickle.load(pickle_in)
            inputs = base[1]
            start_year = int(inputs['START_YEAR'])

            unsorted_pickles.append(base)
            startyrs_list.append(start_year)
    sorted_pickles = [i for _,i in sorted(zip(startyrs_list, unsorted_pickles))]

    return sorted_pickles

#Example here
double_years = get_group(2)

def get_met_demand(cap_per):
    print(cap_per)
    pickles = get_group(cap_per)
    x_years = []
    y_values = []

    for i in range(0, len(pickles)):
        info = pickles[i][0]
        inputs = pickles[i][1]
        results = pickles[i][2]

        #demand - unmet)/demand will give you the % demand met
        #diff = sum(inputs["DEMAND_SERIES"]) - sum(results['DISPATCH_UNMET_DEMAND'])
        #pcent_met = (diff / sum(inputs["DEMAND_SERIES"]))*100

        h_unmet = sum(results['DISPATCH_UNMET_DEMAND'])

        #There will be two x values (start and end year). We'll duplicate the y value such that each x value matches a y value.
        x_years.append(inputs['START_YEAR'])
        y_values.append(h_unmet)
        x_years.append(inputs['END_YEAR']+1) #the plus one is because the run ends at the end of this year
#        y_values.append(pcent_met)

    return y_values

#======================================================================================================
# Fixed capacity plotting
#======================================================================================================
#=======================================================
# Figure size settings
#=======================================================

plt.rcParams.update({'axes.titlesize': 'large'})
plt.rcParams.update({'axes.labelsize': 'large'})

import matplotlib.pylab as pylab
params = {'legend.fontsize': 'large',
          'figure.figsize': (10, 7),
         'axes.labelsize': 'large',
         'axes.titlesize':'x-large',
         'xtick.labelsize':'large',
         'ytick.labelsize':'large'}
pylab.rcParams.update(params)

### Create lists for the plot
##materials = ['Aluminum', 'Copper', 'Steel']
#cap_pers = ['1-yr', '2-yr', '3-yr', '4-yr', '5-yr', '6-yr']
##x_pos = np.arange(len(materials))
#x_pos = np.arange(len(cap_pers))
##CTEs = [aluminum_mean, copper_mean, steel_mean]
##error = [aluminum_std, copper_std, steel_std]
#CTEs = []
#error = []
#for i in range(1,7):
#    demands = get_met_demand(i)
#    CTEs.append(np.mean(demands))
#    error.append(np.std(demands))
#
##($\degree C^{-1}$)
##
### Build the plot
#fig, ax = plt.subplots()
#ax.bar(x_pos, CTEs, yerr=error, align='center', alpha=0.5, ecolor='black', capsize=10)
#ax.set_ylabel('Demand met in each year')
#ax.set_xticks(x_pos)
#ax.set_xticklabels(cap_pers)
#ax.set_ylim(99.7,100.1)
##ax.set_title('Coefficent of Thermal Expansion (CTE) of Three Metals')
#ax.yaxis.grid(True)
##
### Save the figure and show
#plt.tight_layout()
#plt.savefig('bar_plot_with_error_bars.pdf')
#plt.show()
#

data = [get_met_demand(1),get_met_demand(2),get_met_demand(3),
            get_met_demand(4),get_met_demand(5),get_met_demand(6)]

#  Make boxplot
# rectangular box plot
def boxplot(ax,listOfLists,xlabel,ylabel, xticks=None):
    bplot = ax.boxplot(listOfLists,
                       vert=True,   # vertical box aligmnent
                       patch_artist=True,   # fill with color
                       whis=1000000000
                       ) # make whiskers cover full range

    numCats = len(listOfLists)

    # fill with colors
    colors = ['chocolate','peru','burlywood','wheat','blanchedalmond','oldlace']

    for i in range(numCats):
        bplot['boxes'][i].set_facecolor( colors[np.mod(i,numCats)] )
        bplot['medians'][i].set_color( 'black' )

    # adding horizontal grid lines
    ax.yaxis.grid(True)
    if xticks is not None:
        ax1.set_xticks([y+1 for y in range(numCats)], xticks)
        print(xticks)
        #print(xticks)
        #ticks = list(ax1.get_xticks())
        #print(ticks)
        #for i, label in enumerate(xticks):
        #    ticks[i].set_text(label)
        #plt.setp(ax1, xticks)
    else:
        ax.set_xticks([y+1 for y in range(numCats)],)
#    ax.set_xticks('1yr: 2018', '1yr: 2018','1yr: 2018','1yr: 2018','1yr: 2018','1yr: 2018',)
    ax.set_xlabel(xlabel)
    ax.set_ylabel(ylabel)
#    ax.yaxis.set_major_formatter(mtick.PercentFormatter(decimals=2))

    #ax.set_ylim([0,1.1*np.max(listOfLists)])
#    ylimit = 1.1*max(map(lambda x: max(x), listOfLists))
#    ax.set_ylim(top=100)

fig2, ax2 = plt.subplots()
boxplot(ax2,data, "Simulation period used to determine fixed capacities",
        "Unmet demand in 6-yr test periods across 1980-2018 (hours)",
        xticks=['1yr: 2018', '2yr: 2018','3yr: 2018','4yr: 2018','5yr: 2018','6yr: 2018'])

plt.xticks(range(1,7), labels=['1-yr:\n2018-2018', '2-yr:\n2017-2018','3-yr:\n2016-2018','4-yr:\n2015-2018','5-yr:\n2014-2018','6-yr:\n2013-2018'])
## Save the figure and show

fig2.text(.09, 0.94, 'b)', size='large')

plt.tight_layout()
plt.savefig('si/SI_fixedcaps_boxplot.pdf')
plt.show()

#
#colorlist = ('red', 'gold', 'lime', 'aqua', 'blue', 'darkviolet')
###take in the data
##x,y = get_met_demand(1)
##
#fig = plt.figure()
#ax1 = plt.subplot2grid((2, 1), (0, 0), colspan=1, rowspan=1)
#
##linelist = []
##avglist = []
##for i in range(1,7):
##    y = get_met_demand(i)
##    line = ax1.plot(x, y, '-', color=colorlist[i-1], linewidth=1.6)
##    print(i, np.average(y), np.std(y))
##    avglist.append(np.average(y))
##    linelist.append(line)
#
##
#
##ax1 = df['myvar'].plot(kind='bar')
#ax1.yaxis.set_major_formatter(mtick.PercentFormatter(decimals=2))
#
#
#ax1.set_xlim(1980, 1987)
##ax1.set_ylim(99.98,100)
#ax1.set_ylabel("Demand met in each year")
##ax1.set_title("Fixed capacities.\nUnmet demand = $10/kWh")
#
##periods = ['2018-2018', '2017-2018', '2016-2018','2015-2018','2014-2018','2013-2018']
##
##handlelist=[]
##for i in range(1,7):
##    patch = mpatches.Patch(color=colorlist[i-1], label=str(i)+'-yr: '+str(periods[i-1]))
##    handlelist.append(patch)
##
##plt.legend(loc='right',bbox_to_anchor=(1.45, .5),handles=[handlelist[5],handlelist[4],handlelist[3],handlelist[2],handlelist[1],handlelist[0]])
#
#
#pos1 = ax1.get_position() # get the original position
#left, width = 0.1 , .9
#bottom, height = 0.1, .9
#right = left + width
#top = bottom + height
## axes coordinates are 0,0 is bottom left and 1,1 is upper right
#p = mpatches.Rectangle(
#    (left, bottom), width, height,
#    fill=False, transform=ax1.transAxes, clip_on=False, color='white')
#ax1.add_patch(p)
#ax1.text(left, bottom, 'Fixed capacities based\non the 2018 base case.', size = 'large',
#        horizontalalignment='left',
#        verticalalignment='bottom',
#        transform=ax1.transAxes)
#
#p = mpatches.Rectangle(
#    (left, bottom), width, height,
#    fill=False, transform=ax1.transAxes, clip_on=False, color='white')
#ax1.add_patch(p)
##ax1.text(1.2*(left+right), 0.5*(bottom+1.5*top), 'Fixed capacities based\non results from these \nsimulation periods:', size = 'large',
##        horizontalalignment='center',
##        verticalalignment='bottom',
##        transform=ax1.transAxes)
#
##ax2 = plt.subplot2grid((2, 1), (1, 0), colspan=1, rowspan=1)
##xlist = [1,2,3,4,5,6]
##ax2.plot(xlist, avglist)
##ax2.set_title('Average demand met for each simulation period')
##
#
#plt.tight_layout()
##fig.text(.155, 0.94, 'a)', size='large')
##fig.text(.155, 0.46, 'b)', size='large')
#plt.savefig('fixedcaps_2018.pdf', bbox_inches='tight')
#plt.show()


import os
import subprocess
import shlex
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('--dataRoot', type=str, help='data root')
parser.add_argument('--imageSize', type=int, help='data root')
args = parser.parse_args()
data_root = args.dataRoot
image_size = args.imageSize

classes = ['boxing', 'handclapping', 'handwaving', 'jogging', 'running', 'walking']

frame_rate = 25

for _, label in enumerate(classes):
    print('---')
    print(label)

    for vid in os.listdir(os.path.join(data_root,'raw', label)):
        print(vid)
        fname = vid.replace('_uncomp.avi', '')
        try:
            os.makedirs('%s/processed/%s/%s' % (data_root, label, fname))
        except OSError:
            pass
        subprocess.call(shlex.split('ffmpeg -i %s/raw/%s/%s -r %d -f image2 -s %dx%d  %s/processed/%s/%s/image-%%03d_%dx%d.png' % (data_root, label, vid, frame_rate, image_size, image_size, data_root, label, fname, image_size, image_size)))


import os
import sys

import keras

from dataset import *

import pandas as pd
import numpy as np

from keras.layers import Input, Flatten, Dense
from keras.layers import concatenate, Dropout, BatchNormalization
from keras.models import Model
from keras import optimizers
from keras.constraints import max_norm

from keras.utils import to_categorical
from keras.callbacks import EarlyStopping, ModelCheckpoint

from sklearn.model_selection import train_test_split

if __name__ == "__main__":

    if len(sys.argv) != 2:
        print("usage: ", sys.argv[0], " rowstoread (if -1 read alla file) ")
        exit(1)

    numofrows = int(sys.argv[1])

    remote_data = "./"
    data_files = [remote_data + f for f in os.listdir(remote_data) \
            if "Doublets" in f and f.endswith("h5")]
    print("I will use : ", data_files)

    data = None
    if numofrows < 0:
        data = pd.read_hdf(data_files[0])
    else:
        data = pd.read_hdf(data_files[0], stop=numofrows)

    print(data.head())

    print("File read done")

    true = (data["label"]==1.0)
    fake = (data["label"]==-1.0)
    print("Number of doublets before sampling: %d"%len(data))

    sig = data[true]
    bkg = data[fake].sample(n=len(sig))
    data = pd.concat([sig,bkg]).sample(frac=1.0)

    print("New number of doublets: %d"%len(data))

    X = data[featureLabs].values
    Y = data["label"] == 1.0

    print("Features thata will be used: ")
    print(featureLabs)

    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.5, random_state=42)

    print('Training Features Shape:', X_train.shape)
    print('Training Labels Shape:', Y_train.shape)
    print('Testing Features Shape:', X_test.shape)
    print('Testing Labels Shape:', Y_test.shape)

    features   = Input(shape=(len(featureLabs),), name='hit_input')
    b_norm = BatchNormalization()(features)
    dense = Dense(64, activation='relu', kernel_constraint=max_norm(0.1), name='dense_1')(features)
    dense = Dense(64, activation='relu', kernel_constraint=max_norm(0.1), name='dense_2')(dense)
    drop = Dropout(0.5)(dense)
    dense = Dense(64, activation='relu', kernel_constraint=max_norm(0.1), name='dense_3')(drop)
    dense = Dense(64, activation='relu', kernel_constraint=max_norm(0.1), name='dense_4')(dense)
    b_norm = BatchNormalization()(dense)
    pred = Dense(2, activation='relu', kernel_constraint=max_norm(0.1), name='output')(b_norm)

    model = Model(inputs=features, outputs=pred)
    model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

    callbacks = [EarlyStopping(monitor='val_loss', patience=50),
            ModelCheckpoint("best_model.h5",save_best_only="True")]

    Y_train_onehot = to_categorical(Y_train)

    print(Y_train)
    print(Y_train_onehot)

    X_train = X_train/np.max(X_train,axis=0)

    history = model.fit(X_train, Y_train_onehot, \
            batch_size = 1024 , epochs=100, shuffle=True, \
            validation_split=0.15, callbacks=callbacks, verbose=1)


from PySide import QtCore
from PySide import QtGui
from shiboken import wrapInstance

import maya.OpenMayaUI as omui

def maya_main_window():
    """
    Return the Maya main window widget as a Python object
    """
    main_window_ptr = omui.MQtUtil.mainWindow()
    return wrapInstance(long(main_window_ptr), QtGui.QWidget)

class TestDialog(QtGui.QDialog):

    def __init__(self, parent=maya_main_window()):
        super(TestDialog, self).__init__(parent)

        self.setWindowTitle("Test Dialog")
        self.setMinimumWidth(200)
        self.setWindowFlags(self.windowFlags() ^ QtCore.Qt.WindowContextHelpButtonHint)

        self.create_widgets()
        self.create_layouts()
        self.create_connections()

    def create_widgets(self):
        self.lineedit = QtGui.QLineEdit()
        self.checkbox1 = QtGui.QCheckBox()
        self.checkbox2 = QtGui.QCheckBox()
        self.ok_btn = QtGui.QPushButton("OK")
        self.cancel_btn = QtGui.QPushButton("Cancel")

    def create_layouts(self):
        form_layout = QtGui.QFormLayout()
        form_layout.addRow("Name:", self.lineedit)
        form_layout.addRow("Hidden:", self.checkbox1)
        form_layout.addRow("Locked:", self.checkbox2)

        button_layout = QtGui.QHBoxLayout()
        button_layout.addStretch()
        button_layout.addWidget(self.ok_btn)
        button_layout.addWidget(self.cancel_btn)

        main_layout = QtGui.QVBoxLayout(self)
        main_layout.addLayout(form_layout)
        main_layout.addLayout(button_layout)

    def create_connections(self):
        self.lineedit.editingFinished.connect(self.print_hello_name)

        self.checkbox1.toggled.connect(self.print_is_hidden)

        self.cancel_btn.clicked.connect(self.close)

    def print_hello_name(self):
        name = self.lineedit.text()
        print("Hello {0}!".format(name))

    def print_is_hidden(self):
        hidden = self.checkbox1.isChecked()
        if hidden:
            print("Hidden")
        else:
            print("Visible")
'''

if __name__ == "__main__":

    test_dialog = TestDialog()
    test_dialog.show()

'''

def initializeUI():
    test_dialog = TestDialog()
    test_dialog.show()

initializeUI()


import numpy as np

from keras import datasets as dt
from keras.datasets import fashion_mnist
import matplotlib.pyplot as plt
from tensorflow.python.keras.models import load_model

# get K samples of each class, label with the max class with maximal similarity to the target
def get_samples(x, y, K=1):
    indices = [(y == i).nonzero()[0][:K] for i in np.unique(y)]
    indices = np.array(indices).flatten()
    return x[indices], y[indices]

# a simple validation for a single sample. Extend this to get some overall results for the whole test set.
# Return, e.g., accuracy
def test_model(model, x, y, x_test, y_test, class_reprezentatives):
    x0_sample, y0_sample = get_samples(x, y, class_reprezentatives)

    classes_count = len(np.unique(y))
    confusion_matrix = np.zeros((classes_count, classes_count))
    correct = 0
    for i, example in enumerate(x_test):
        # create an array of example repetitions, of len(y)
        x0_test = np.repeat(example[np.newaxis, :, :, :], len(y0_sample), axis=0)
        # compare the examples array with x0_sample
        pred_list = model.predict([x0_test, x0_sample])
        # get the index of maximum similarity
        pred = np.argmax(pred_list)

        real_class = y_test[i]
        predicted_class = y0_sample[pred]
        confusion_matrix[real_class][predicted_class] += 1
        if real_class == predicted_class:
            correct += 1

    accuracy = correct / len(x_test)
    # normalize the matrix
    confusion_matrix /= confusion_matrix.max()
    return accuracy, confusion_matrix

def plot_confusion_matrix(confusion_matrix, title):
    grid_ticks = range(confusion_matrix.shape[0])
    im = plt.imshow(confusion_matrix)
    plt.colorbar(im)
    plt.title(title)
    plt.yticks(grid_ticks, grid_ticks)
    plt.xticks(grid_ticks, grid_ticks)
    plt.xlabel("Actual class")
    plt.ylabel("Predicted class")
    plt.savefig("{}.png".format(title.lower().replace(" ", "_")))
    plt.show()

class Dataset:
    def __init__(self, data, test_len=100):
        ((self.x, self.y), (self.x_test, self.y_test)) = data
        self.x = np.expand_dims(self.x, axis=3)
        self.x_test = np.expand_dims(self.x_test, axis=3)[:test_len]
        self.y_test = self.y_test[:test_len]

def main():
    # load saved model
    model_siam = load_model("models/siam.model")

    # load datasets
    fashion = Dataset(fashion_mnist.load_data(), 500)
    digits = Dataset(dt.mnist.load_data(path='mnist.npz'), 500)

    # combine data
    both_data = (
        (np.concatenate((fashion.x[:, :, :, 0], digits.x[:, :, :,  0])), np.concatenate((fashion.y, digits.y + 10))),
        (np.concatenate((fashion.x_test[:, :, :,  0], digits.x_test[:, :, :,  0])), np.concatenate((fashion.y_test, digits.y_test + 10)))
    )

    # combinated datasets dataset
    combo = Dataset(both_data, 1000)

    # test fashion first:
    accuracy, confusion_matrix = test_model(model_siam, combo.x, combo.y, fashion.x_test, fashion.y_test, 1)
    print("fashion accuracy: {}".format(accuracy))
    # fashion accuracy: 0.742
    plot_confusion_matrix(confusion_matrix, "Fashion confusion matrix")

    # test digits:
    accuracy, confusion_matrix = test_model(model_siam, combo.x, combo.y, digits.x_test, digits.y_test + 10, 1)
    print("digits accuracy: {}".format(accuracy))
    # fashion accuracy: 0.126
    plot_confusion_matrix(confusion_matrix, "Digits confusion matrix")

    # as we see, the digits accuracy is pretty bad. However we can try raising the number of representatives:

    # plot the relation between number of representatives and accuracy
    # use only first 100 for better time performance
    accuracies = []
    for representatives_cnt in range(1, 14, 2):
        acc, _ = test_model(model_siam, combo.x, combo.y, digits.x_test[:100], digits.y_test[:100] + 10, representatives_cnt)
        accuracies.append(acc)

    print(accuracies)
    # [0.18, 0.44, 0.36, 0.36, 0.42, 0.39, 0.41]
    plt.plot(range(1, 14, 2), accuracies)
    plt.xlabel("Number of representatives")
    plt.ylabel("Accuracy")
    plt.title("Representatives-accuracy relation")
    plt.savefig("representatives_accuracy_relation.png")
    plt.show()
    # as we can see, the accuracy rises significantly.

    # lets find out how similar are classes
    # all we need to do is to add the confusion matrix over its diagonal
    # meaning we are going to add numbers with indexes ie: (1,2), (2,1)
    _, cm = test_model(model_siam, combo.x, combo.y, combo.x_test, combo.y_test, 3)
    for i in range(cm.shape[0]):
        for j in range(i, cm.shape[1]):
            cm[i][j] += cm[j][i]
            cm[j][i] = 0

    cm /= cm.max()
    plot_confusion_matrix(cm, "Classes similarity")
    # there are few classes, that our network considers quite similar:
    # fashion: (0, 3) = (tshirt, dress), (2, 6) = (pullover, shirt),
    #          (4, 6) = (coat, shirt)
    # digits: (17, 13) = 7, 3
    # mix: (5, 15) = (sandal, 5)

if __name__ == '__main__':
    main()


from operator import attrgetter
import json
from xml.etree.ElementTree import Element, SubElement, tostring
import yaml

class DataRepresentator(object):
    """class for converting iterables to specific data format"""

    def get_instance_attr(self, instance, attr):
        """method for getting attr from instance"""
        attr = attrgetter(attr[0])(instance)
        return str(attr()) if callable(attr) else str(attr)

    def get_instance_attr_pair(self, instance, attr):
        """method for getting attr verb. name attr value pair from instance"""
        attr_value = self.get_instance_attr(instance, attr)
        return str(attr[-1]), str(attr_value)

    def get_csv(self, iterable, attrs):
        """method for generating csv repr of data"""
        result = ""
        for index, item in iterable:
            item_result = ",".join([self.get_instance_attr(
                item, attr) for attr in attrs])
            result += "{0}\n".format(item_result)
        return result

    def get_json(self, iterable, attrs):
        """method for generating json repr of data"""
        data_list = []
        for index, item in iterable:
            item_dict = {}
            for attr in attrs:
                key, value = self.get_instance_attr_pair(item, attr)
                item_dict.update({key: value})
            data_list.append(item_dict)
        return json.dumps(data_list)

    def get_xml(self, iterable, attrs):
        """method for getting xml repr of data"""
        root = Element("root")
        for index, item in iterable:
            for attr in attrs:
                fname, value = self.get_instance_attr_pair(item, attr)
                child = SubElement(root, fname.replace(" ", "_"))
                child.text = value
        return str(tostring(root))[2:-1]

    def get_yaml(self, iterable, attrs):
        """method for generating yaml repr of data"""
        # data_list = []
        # for index, item in iterable:
        #     item_dict = {}
        #     for attr in attrs:
        #         key, value = self.get_instance_attr_pair(item, attr)
        #         item_dict.update({key: value})
        #     data_list.append(item_dict)
        return yaml.dump(json.loads(self.get_json(iterable, attrs)))


#Standard modules
import time
import matplotlib.pyplot as plt
import os
import numpy as np
import multiprocessing
import sys
import queue
import ctypes

import importlib

#RASH modules
import masterboard
import config

import Trajectory_Via as trj

import parser
import kinematics
import plotter as pltr
import simple_walk
import simple_trot
import creep
import idle_trot

import input_coordinates

class InputTask:
    def __init__(self, trajectory_queue, message_queue, shared_position, standard_input, masterboard_started_event):
        self.trajectory_queue = trajectory_queue
        self.message_queue = message_queue
        self.standard_input = standard_input

        self.shared_position = shared_position

        self.masterboard_started_event = masterboard_started_event

        self._running = True

        self._mode = "Walk"

    def run(self):
        # Priority settings for "InputTask" process
        #print(f"Input task started with pid: {os.getpid()} and priority: {os.sched_getparam(os.getpid())}")

        pid = os.getpid()
        sched = os.SCHED_FIFO
        param = os.sched_param(50)

        os.sched_setscheduler(pid, sched, param)

        #print(f"Input task with pid: {os.getpid()} changed to priority: {os.sched_getparam(os.getpid())}\n")

        sys.stdin = os.fdopen(self.standard_input)

        print()
        print(f"Robot is currently in {self._mode} mode")

        #Make sure master board has recorded a position before we try to read it
        self.masterboard_started_event.wait()

        with self.shared_position.get_lock():
                initial_position = self.shared_position[:]

        while(self._running):
            try:
                input_string = input(">")

                joint_variables, input_message = parser.interpret_input(input_string, self._mode, initial_position)

            except EOFError:
                self.terminate()
                continue

            if input_message == "simple walk":
                if self._mode != "Walk":
                    print(f"Currently in {self._mode}, change mode.")
                    continue
                initial_position = self.walk(initial_position, joint_variables)

            elif input_message == "simple trot":
                initial_position = self.trot(initial_position, joint_variables, self._mode)

            elif input_message == "Creep":
                if self._mode != "Idle":
                    print(f"Currently in {self._mode}, change mode.")
                initial_position = self.creep(initial_position, joint_variables)

            elif input_message == "Send":
                if self._mode == "Walk":
                    zero_config = kinematics.walk_zero_config
                    elbows = kinematics.walk_elbow
                    joint_polarities = kinematics.walk_joint_polarities
                else:
                    #return None, "InputError"

                    zero_config = kinematics.idle_zero_config
                    elbows = kinematics.idle_elbow
                    joint_polarities = kinematics.idle_joint_polarities

                importlib.reload(input_coordinates)

                coordinates = np.array(input_coordinates.coordinates, dtype=float)
                velocity = np.array(input_coordinates.velocity, dtype=float)
                acceleration = np.array(input_coordinates.acceleration, dtype=float)

                points = kinematics.simple_walk_controller(coordinates, initial_position, zero_config, joint_polarities, elbows)

                if None in points:
                    continue

                initial_position = self.que_trajectory(initial_position, points, velocity, acceleration)

            elif input_message != "":
                initial_position = self.handle_input(input_message, initial_position)

            elif input_message == "InputError":
                continue

            else:
                initial_position = self.que_trajectory(initial_position, joint_variables)

        sys.stdin.close()

        #print(f"Input task with pid: {os.getpid()} terminated")

    def terminate(self):
        self._running = False

    def handle_input(self, input_message, initial_position):

        if input_message == "Stand":
            initial_position = self.stand(self._mode, initial_position)

        elif input_message == "Home":
            initial_position = self.home(self._mode, initial_position)

        elif input_message == "Current":
            print(initial_position)

        elif input_message == "Walk" or input_message == "Idle":
            initial_position = self.change_mode(input_message, initial_position)

        elif input_message == "Exit":
            self.message_queue.put(input_message)
        else:
            self.message_queue.put(input_message)

        return initial_position

    def creep(self, initial_position, num_cycles):
        zero_config = kinematics.idle_zero_config
        elbows = kinematics.idle_elbow
        joint_polarities = kinematics.idle_joint_polarities

        home = kinematics.simple_walk_controller(creep.home, initial_position, zero_config, joint_polarities, elbows)
        initial_position = self.que_trajectory(initial_position, home)

        for i in range(num_cycles):
            #print("Walk cycle")

            HL_lowering = kinematics.simple_walk_controller(creep.HL_lowering, initial_position, zero_config, joint_polarities, elbows)
            initial_position = self.que_trajectory(initial_position, HL_lowering, k_v = creep.velocity, k_a = creep.acceleration)

            HL_lifting = kinematics.simple_walk_controller(creep.HL_lifting, initial_position, zero_config, joint_polarities, elbows)
            initial_position = self.que_trajectory(initial_position, HL_lifting, k_v = creep.velocity, k_a = creep.acceleration)

            HL_home = kinematics.simple_walk_controller(creep.HL_home, initial_position, zero_config, joint_polarities, elbows)
            initial_position = self.que_trajectory(initial_position, HL_home, k_v = creep.velocity, k_a = creep.acceleration)

            HR_lowering = kinematics.simple_walk_controller(creep.HR_lowering, initial_position, zero_config, joint_polarities, elbows)
            initial_position = self.que_trajectory(initial_position, HR_lowering, k_v = creep.velocity, k_a = creep.acceleration)

            HR_lifting = kinematics.simple_walk_controller(creep.HR_lifting, initial_position, zero_config, joint_polarities, elbows)
            initial_position = self.que_trajectory(initial_position, HR_lifting, k_v = creep.velocity, k_a = creep.acceleration)

            HR_home = kinematics.simple_walk_controller(creep.HR_home, initial_position, zero_config, joint_polarities, elbows)
            initial_position = self.que_trajectory(initial_position, HR_home, k_v = creep.velocity, k_a = creep.acceleration)

            FR_lowering = kinematics.simple_walk_controller(creep.FR_lowering, initial_position, zero_config, joint_polarities, elbows)
            initial_position = self.que_trajectory(initial_position, FR_lowering, k_v = creep.velocity, k_a = creep.acceleration)

            FR_lifting = kinematics.simple_walk_controller(creep.FR_lifting, initial_position, zero_config, joint_polarities, elbows)
            initial_position = self.que_trajectory(initial_position, FR_lifting, k_v = creep.velocity, k_a = creep.acceleration)

            FR_home = kinematics.simple_walk_controller(creep.FR_home, initial_position, zero_config, joint_polarities, elbows)
            initial_position = self.que_trajectory(initial_position, FR_home, k_v = creep.velocity, k_a = creep.acceleration)

            FL_lowering = kinematics.simple_walk_controller(creep.FL_lowering, initial_position, zero_config, joint_polarities, elbows)
            initial_position = self.que_trajectory(initial_position, FL_lowering, k_v = creep.velocity, k_a = creep.acceleration)

            FL_lifting = kinematics.simple_walk_controller(creep.FL_lifting, initial_position, zero_config, joint_polarities, elbows)
            initial_position = self.que_trajectory(initial_position, FL_lifting, k_v = creep.velocity, k_a = creep.acceleration)

            FL_home = kinematics.simple_walk_controller(creep.FL_home, initial_position, zero_config, joint_polarities, elbows)
            initial_position = self.que_trajectory(initial_position, FL_home, k_v = creep.velocity, k_a = creep.acceleration)

            home = kinematics.simple_walk_controller(creep.home, initial_position, zero_config, joint_polarities, elbows)
            initial_position = self.que_trajectory(initial_position, home, k_v = creep.velocity, k_a = creep.acceleration)

        home = kinematics.simple_walk_controller(simple_walk.home, initial_position, zero_config, joint_polarities, elbows)
        initial_position = self.que_trajectory(initial_position, home)

        return initial_position

    def walk(self, initial_position, num_cycles):
        importlib.reload(simple_walk)

        home = kinematics.simple_walk_controller(simple_walk.home, initial_position)
        initial_position = self.que_trajectory(initial_position, home)
        acceleration = simple_walk.acceleration

        for i in range(num_cycles):
            #print("Walk cycle")

            down = kinematics.simple_walk_controller(simple_walk.left_leg_down, initial_position)
            back = kinematics.simple_walk_controller(simple_walk.left_leg_back, initial_position)

            forward_to_back = np.vstack((down,back))

            initial_position = self.que_trajectory(initial_position, forward_to_back, k_v = simple_walk.forward_to_back_velocity, k_a = acceleration)

            up = kinematics.simple_walk_controller(simple_walk.left_leg_up, initial_position)
            forward = kinematics.simple_walk_controller(simple_walk.left_leg_forward, initial_position)

            back_to_forward = np.vstack((up, forward))

            initial_position = self.que_trajectory(initial_position, back_to_forward, k_v = simple_walk.back_to_forward_velocity, k_a = acceleration)

            #print(down, back, up, forward)

        home = kinematics.simple_walk_controller(simple_walk.home, initial_position)
        initial_position = self.que_trajectory(initial_position, home)

        return initial_position

    def trot(self, initial_position, num_cycles, mode):
        # Trot settings for Idle mode

        if mode == "Idle":
            importlib.reload(idle_trot) #Enables live update of varibles when testing trot settings

            zero_config = kinematics.idle_zero_config
            elbows = kinematics.idle_elbow
            joint_polarities = kinematics.idle_joint_polarities

            front_left_up_coordinates = idle_trot.HR_FL
            front_right_up_coordinates = idle_trot.HL_FR
            home_coordinates = idle_trot.home

            FL_up_velocity = idle_trot.HR_FL_velocity
            FR_up_velocity = idle_trot.HL_FR_velocity

            k_a = idle_trot.acceleration

        # Trot settings for Walk mode
        elif mode == "Walk":
            importlib.reload(simple_trot) #Enables live update of varibles when testing trot settings

            zero_config = kinematics.walk_zero_config
            elbows = kinematics.walk_elbow
            joint_polarities = kinematics.walk_joint_polarities

            front_left_up_coordinates = simple_trot.front_left_up
            front_right_up_coordinates = simple_trot.front_right_up
            home_coordinates = simple_trot.home

            FL_up_velocity = simple_trot.velocity
            FR_up_velocity = simple_trot.velocity

            k_a = simple_trot.acceleration

        # Calculate angles and que trajectory
        home = kinematics.simple_walk_controller(home_coordinates, initial_position, zero_config, joint_polarities, elbows)
        initial_position = self.que_trajectory(initial_position, home)

        for i in range(num_cycles):

            front_left_up = kinematics.simple_walk_controller(front_left_up_coordinates, initial_position, zero_config, joint_polarities, elbows)

            initial_position = self.que_trajectory(initial_position, front_left_up, k_v = FL_up_velocity, k_a = k_a)

            front_right_up = kinematics.simple_walk_controller(front_right_up_coordinates, initial_position, zero_config, joint_polarities, elbows)

            initial_position = self.que_trajectory(initial_position, front_right_up, k_v = FR_up_velocity, k_a = k_a)

        home = kinematics.simple_walk_controller(home_coordinates, initial_position, zero_config, joint_polarities, elbows)
        initial_position = self.que_trajectory(initial_position, home)

        return initial_position

    def stand(self, mode, initial_position):
        if mode == "Walk":
            points = config.walk_stand

        elif mode == "Idle":
            points = config.idle_stand

        initial_position = self.que_trajectory(initial_position, points = points, k_v = config.up_speed, k_a = config.up_acceleration)

        return initial_position

    def home(self, mode, initial_position):
        if mode == "Walk":
            points = config.walk_home

        elif mode == "Idle":
            points = config.idle_home

        initial_position = self.que_trajectory(initial_position, points = points, k_v = config.down_speed, k_a = config.down_acceleration)

        return initial_position

    def change_mode(self, new_mode, initial_position):
        if self._mode == new_mode:
            return initial_position

        initial_position = self.home(self._mode, initial_position)
        self._mode = new_mode
        initial_position = self.home(self._mode, initial_position)

        print()
        print(f"Robot is now in {self._mode} mode")

        return initial_position

    def que_trajectory(self, initial_position, points, k_v=config.default_speed, k_a=config.default_acceleration):
        trajectory = trj.Trajectory(points=[initial_position, points], k_v=k_v, k_a=k_a)
        new_initial_position = trajectory.q_f

        if trajectory.is_valid():
            self.trajectory_queue.put(trajectory)

        return new_initial_position

class MasterBoardTask:
    def __init__(self, trajectory_queue, message_queue, shared_position, masterboard_started_event):
        #Local variables
        self._running = True
        self._terminating = False

        self.shared_position = shared_position

        self.trajectory_queue = trajectory_queue
        self.message_queue = message_queue

        self.masterboard_started_event = masterboard_started_event

        self.debug_plot = True

        self.balance_mode = False

        #self.plot = plotter.Plotter(f"Via points test | kp = {config.kp}, kd = {config.kd}, current saturation = {config.current_saturation}")

        #for i in range(config.N_SLAVES_CONTROLLED * 2):
        #    self.plot.create_plot(f"Joint {i}, Current Reference", "Current [A]")
        #    self.plot.create_plot(f"Joint {i}, Position Reference", "Position [rad]")
        #    self.plot.create_plot(f"Joint {i}, Position Reading", "Position [rad]")
        #    self.plot.create_plot(f"Joint {i}, Position Error", "position [rad]")

        self.master_board = masterboard.MasterBoard()

        if self.debug_plot:
            self.plotter = pltr.Plotter(f"Via points test | kp = {config.kp}, kd = {config.kd}, current saturation = {config.current_saturation}")

            for i in range(config.N_SLAVES_CONTROLLED * 2):
                self.plotter.create_plot(f"Joint {i}, Position Reference", "Position [rad]")
                self.plotter.create_plot(f"Joint {i}, Position Reading", "Position [rad]")
                self.plotter.create_plot(f"Joint {i}, Velocity Reference", "Velocity [rad/s]")
                self.plotter.create_plot(f"Joint {i}, Velocity Reading", "Velocity [rad/s]")

                #self.plotter.create_plot(f"Joint {i}, Position Error", "position [rad]")

                #self.plotter.create_plot(f"Joint {i}, Current Reference", "Current [A]")

                #self.plotter.create_plot(f"Joint {i}, Velocity Error", "Velocity [rad/s]")

    def run(self):
        # Priority settings for "MasterBoardTask"
        print(f"Master board task started with pid: {os.getpid()} and priority: {os.sched_getparam(os.getpid())}")

        pid = os.getpid() # Identify process ID
        sched = os.SCHED_FIFO # FIFO real time scheduler
        param = os.sched_param(90) # Process priority

        os.sched_setscheduler(pid, sched, param) # Update priority settings

        print(f"Master board task with pid: {os.getpid()} changed to priority: {os.sched_getparam(os.getpid())}\n")

        # Local variables
        dt = config.dt
        timeout = False
        current_time = 0.0
        cpt = 0

        # Troubleshooting arrays (Normally not used)
        plot_duration = 1000

        plot_array_size = int(plot_duration / dt)

        time_per_timestep_array = np.zeros(plot_array_size + 1)
        cycle_time_array = np.zeros(plot_array_size + 1)
        ic_time_array = np.zeros(plot_array_size + 1)

        # Trajectory and Queue initiation
        #self.trajectory_queue_init()

        q, q_prime = self.master_board.get_state()

        with self.shared_position.get_lock():
            self.shared_position[:] = q

        self.masterboard_started_event.set()

        trajectory = None

        # Timer initiation
        cycle_time = 0
        program_start = time.perf_counter()
        timestep_start = time.perf_counter()
        timestep_end = time.perf_counter()
        cycle_start = time.perf_counter()

        self.prev_t = 0

        self.prev_att_err = 0

        self.acc_error = np.zeros(2)

        self.kp = 200 #200
        self.kd = 3.5   #2
        self.ki = 6000 #3000

        # Communication with MasterBoard
        while(self._running and not timeout):
            balance_mode = self.balance_mode

            #Shared position. This allows InputTask to read the current position from a shared array
            current_position = self.master_board.get_state()[0]
            with self.shared_position.get_lock():
                self.shared_position[:] = current_position

            if balance_mode:
                if not self.message_queue.empty():
                    input_message = self.message_queue.get(block=False)
                    self.handle_input(input_message, q)

                attitude = self.master_board.get_attitude()

                if self.prev_t == 0:
                    self.prev_t = current_time

                q, q_prime = self.balance(attitude, current_position, current_time)

                timeout = self.master_board.set_reference(q, q_prime)

            elif trajectory == None:

                if not self.message_queue.empty():
                    input_message = self.message_queue.get(block=False)
                    self.handle_input(input_message, q)

                # No trajectory, try to get new from queue.
                elif not self.trajectory_queue.empty():
                    trajectory = self.trajectory_queue.get(block=False) # Check queue
                    initial_trajectory_time = time.perf_counter()

                else:
                    if self._terminating:
                        self._running = False

            else:

                # Follow current trajectory

                time_before_initial_condition = time.perf_counter()

                current_trajectory_time = time.perf_counter() - initial_trajectory_time

                q, q_prime = trajectory.step(current_trajectory_time)

                ic_time_array[cpt] =  time.perf_counter() - time_before_initial_condition

                timeout = self.master_board.set_reference(q, q_prime)

                '''
                for i in range(len(q)):
                    if i < 2:
                        self.plotter.add_data(f"Joint {i}, Position Set Point", q[i])
                        self.plotter.add_data(f"Joint {i}, Velocity Set Point", q_prime[i])

                        #pos_reading, vel_reading = self.master_board.get_state()

                        #self.plotter.add_data(f"Joint {i}, Position Reading", pos_reading[i])
                        #self.plotter.add_data(f"Joint {i}, Velocity Reading", vel_reading[i])

                        #self.plotter.add_data(f"Joint {i}, Position Error", pos_reading[i] - q[i])
                        #self.plotter.add_data(f"Joint {i}, Velocity Error", vel_reading[i] - q_prime[i])

                '''
                if trajectory.is_done():
                    trajectory = None

            # Update current reference to each joint.
            if self.debug_plot:
                timeout = self.master_board.track_reference(self.plotter)
                self.plotter.add_time(current_time)
            else:
                timeout = self.master_board.track_reference(None)
            #self.plot.add_time(current_time)

            # Cycle time is time spent calculating trajectory
            cycle_time = time.perf_counter() - cycle_start
            cycle_time_array[cpt] = cycle_time

            if cycle_time >= dt:
                cycle_time = dt

            # Sleep MasterBoard for 1ms.
            time.sleep(dt - cycle_time) # ideally: dt - cycle_time = 1ms
            cycle_start = time.perf_counter() # Start time for next cycle

            timestep_end = cycle_start
            time_per_timestep_array[cpt] = timestep_end - timestep_start
            timestep_start = cycle_start

            current_time = current_time + dt

            if cpt < plot_array_size:
                cpt = cpt + 1

        self.master_board.terminate()

        #Reset process priority
        sched = os.SCHED_OTHER
        param = os.sched_param(os.sched_get_priority_min(sched))
        os.sched_setscheduler(pid, sched, param)

        print(f"Master board task with pid: {os.getpid()} changed to priority: {os.sched_getparam(os.getpid())}")

        plt.plot(time_per_timestep_array[0:cpt - 1])
        plt.show()
        plt.plot(cycle_time_array[0:cpt - 1])
        plt.show()
        plt.plot(ic_time_array[0:cpt - 1])
        plt.show()

        if self.debug_plot:
            #for i in range(config.N_SLAVES_CONTROLLED * 2):
            #    self.plotter.set_num_cols(3)

            #   self.plotter.create_axis([f"Joint {2*i}, Position Reference", f"Joint {2*i}, Position Reading"])

            #    self.plotter.create_axis([f"Joint {2*i}, Velocity Reference"])

            #    self.plotter.create_axis([f"Joint {2*i}, Velocity Reading"])

            #    self.plotter.create_axis([f"Joint {2*i+1}, Position Reference", f"Joint {2*i+1}, Position Reading"])

            #    self.plotter.create_axis([f"Joint {2*i+1}, Velocity Reference"])

            #    self.plotter.create_axis([f"Joint {2*i+1}, Velocity Reading"])

                #self.plotter.create_axis([
                #   f"Joint {2*i}, Velocity Set Point",
                #    f"Joint {2*i+1}, Velocity Set Point"
                #])

            self.plotter.create_axis([f"Joint {0}, Position Reference", f"Joint {0}, Position Reading"])
            self.plotter.create_axis([f"Joint {2}, Position Reference", f"Joint {2}, Position Reading"])
            self.plotter.create_axis([f"Joint {4}, Position Reference", f"Joint {4}, Position Reading"])
            self.plotter.create_axis([f"Joint {6}, Position Reference", f"Joint {6}, Position Reading"])
            self.plotter.create_axis([f"Joint {1}, Position Reference", f"Joint {1}, Position Reading"])
            self.plotter.create_axis([f"Joint {3}, Position Reference", f"Joint {3}, Position Reading"])
            self.plotter.create_axis([f"Joint {5}, Position Reference", f"Joint {5}, Position Reading"])
            self.plotter.create_axis([f"Joint {7}, Position Reference", f"Joint {7}, Position Reading"])

            self.plotter.set_num_cols(4)
            self.plotter.plot(grid=True)

    def trajectory_queue_init(self):
        '''
        For å få lavere cycle time ved første trajectory
        '''
        trajectory = trj.Trajectory([0,0,0,0,0,0,0,0])
        self.trajectory_queue.put(trajectory)
        self.trajectory_queue.get()
        trajectory.set_initial_conditions(0, 0)

    def handle_input(self, input_message, q):
        if input_message == "Exit":
            print("Should exit")
            self._running = False

        elif input_message == "Position":
            self.print_current_position(q)

        elif input_message == "Balance":
            if self.balance_mode:
                self.balance_mode = False
            else:
                self.balance_mode = True
                self.prev_t = 0
                self.acc_error = np.zeros(2)

    def print_current_position(self, current_pos_ref):
        #Prints the current reference and position of all the motors
        #Meant to be called by the InputTask through the use of synchronized bools
        current_position = self.master_board.get_state()[0]
        current_actual_position = self.master_board.get_actual_state()[0]

        np.set_printoptions(suppress=True)

        # Display position in radians
        print("\n--------------------------POSITION [RAD]----------------------------------\n")
        print(f"Setpoint:\n{current_pos_ref.round(5).tolist()}\n")
        print(f"Current Postition (get_state):\n{current_position.round(5).tolist()}\n")
        print(f"Error = Current Position - Setpoint:\n{(current_position - current_pos_ref).round(6).tolist()}\n")
        print(f"\nActual Position (get_actual_state):\n{current_actual_position.round(5).tolist()}\n")

        #Display position in degress
        print("\n-------------------------POSITION [DEG]-----------------------------------\n")
        print(f"Setpoint ={np.degrees(current_pos_ref).round(5).tolist()}")
        print(f"Current Postition (get_state):\n{np.degrees(current_position).round(5).tolist()}")
        print(f"Error = Current Position - Setpoint:\n{np.degrees(current_position - current_pos_ref).round(5).tolist()}\n")
        print(f"\nActual Position (get_actual_state):\n{np.degrees(current_actual_position).round(5).tolist()}")
        print("\n--------------------------------------------------------------------------\n")

    def balance(self, attitude, current_position, t):
        att_ref = np.array([0.0,0.0])
        att_read = attitude[:2]

        att_err = att_ref - att_read

        if t != 0 and t != self.prev_t:
            att_err_prime = (att_err - self.prev_att_err) / (t - self.prev_t)
        else:
            att_err_prime = 0

        self.acc_error = self.acc_error + (self.ki * att_err * (t - self.prev_t))

        delta_y = self.kp * att_err + self.kd * att_err_prime + self.acc_error

        self.prev_att_err = att_err
        self.prev_t = t

        delta_y = - delta_y

        coordinates = np.array([
            [0, 100 + delta_y[1]/2 + delta_y[0]/2],
            [0, 100 + delta_y[1]/2 - delta_y[0]/2],
            [0, 100 - delta_y[1]/2 + delta_y[0]/2],
            [0, 100 - delta_y[1]/2 - delta_y[0]/2]
        ])

        q = kinematics.simple_walk_controller(coordinates, current_position, zero_config=kinematics.idle_zero_config, joint_polarities=kinematics.idle_joint_polarities, elbows=kinematics.idle_elbow)

        if None in q:
            return current_position, np.zeros(8)

        return q, np.zeros(8)

    def terminate(self):
        print(f"Master board task with pid: {os.getpid()} terminated")
        self._running = False

masterboard_started_event = multiprocessing.Event()

shared_position = multiprocessing.Array(ctypes.c_double, 8)

standard_input = sys.stdin.fileno()

trajectory_queue = multiprocessing.Queue()
message_queue = multiprocessing.Queue()

input_task = InputTask(trajectory_queue, message_queue, shared_position, standard_input, masterboard_started_event)
master_board_task = MasterBoardTask(trajectory_queue, message_queue, shared_position, masterboard_started_event)

input_thread = multiprocessing.Process(target = input_task.run, daemon = True)
input_thread.start()

master_board_task.run()

os._exit(0)


import pandas
import numpy as np
import torch
from kmeans_pytorch import kmeans
import os
import cv2
import glob

# topcon_candidates = pandas.read_csv('/home/ubuntu/data/Workspace/Soobin/data/resolution/topcon_candidate_higher_res.csv')

# total_cnt = len(topcon_candidates['file_path'])
# length = int(total_cnt/20)
# img_path = '/home/ubuntu/data/OCT/Seoul/'

def create_tempfeatures(idx):
    # temp = np.zeros((length,256*256*3+1),dtype='float32')
    if idx == 19:
        st_idx = length*idx
        end_idx = total_cnt
    else:
        st_idx = length*idx
        end_idx = length*(idx+1)
    print(st_idx,"  ",end_idx)
    transformation = np.zeros((end_idx-st_idx,20000*3),dtype='float32')

    for i, idx in enumerate (list(range(st_idx,end_idx))):
        path = topcon_candidates['file_path'][idx]
        path = '/'.join(path.split('/')[4:])
        im = cv2.imread(img_path + path)
        im = np.array(cv2.resize(im,dsize=(200,100)),dtype='float32')
        # h, edges = np.histogramdd(im.reshape(-1,3),8,normed=True,range=[(0,255),(0,255),(0,255)])
        transform=np.fft.fft2(im)
        transformation[i] = np.abs(transform.flatten())

    res = torch.from_numpy(transformation)  # to use gpu
    return res

csv = []
def topcon_imwrite(cluster, idxx, idx):
    if idxx == 19:
        st_idx = length*idxx
        end_idx = total_cnt
    else:
        st_idx = length*idxx
        end_idx = length*(idxx+1)
    print(st_idx,"  ",end_idx)

    path = topcon_candidates['file_path'][st_idx+idx]
    path = '/'.join(path.split('/')[4:])
    path = img_path+path
    im = cv2.imread(path)

    name = topcon_candidates['file_name'][st_idx+idx]

    item = []
    item.append(path)
    item.append(name)
    item.append(str(cluster))
    csv.append(item)

    savepath ='./topcon_candidates_5class_higher/attempt'+str(idxx)+'/img'
    savepath = savepath+str(cluster)+'/'

    os.makedirs(savepath,exist_ok=True)
    cv2.imwrite(savepath+name,im)

def delete_existing_img(path):
    dirs = glob.glob(path+'*')
    directory = [dire for dire in dirs if not '.jpg' in dire]
    directory = [dire for dire in directory if not '.csv' in dire]
    print(directory)
    image = [img for img in dirs if 'jpg' in img]

    print(len(image))
    for dire in directory:
        directory_img = glob.glob(dire+'/*/*')
        for img in directory_img:
            search_img = img.split('/')[10]
            for imgg in image:
                if search_img in imgg:
                    os.remove(imgg)
                    image.remove(imgg)
    print(len(image))
    return

def delete_existing_folder():
    del_attempt_img = [(0,0),(9,4),(12,2),(14,4),(18,3),(19,3)]
    import shutil
    for (attempt, img) in del_attempt_img:
        del_path = '/home/ubuntu/data/Workspace/Soobin/data/topcon_wide_higher_resolution/attempt'+str(attempt)+'/img'+str(img)
        print(del_path)
        shutil.rmtree(del_path)

def coexist_multiclass(to_class):
    path='/home/ubuntu/data/Workspace/Soobin/wide_total/'
    path = glob.glob(path+'*')
    tot_cnt = len(path)
    print(tot_cnt)
    length = int(tot_cnt/50)
    for attempt in range(24,50):
        csv=[]
        if attempt == 49:
            st_idx = length*attempt
            end_idx = tot_cnt
        else:
            st_idx = length*attempt
            end_idx = length*(attempt+1)
        print(st_idx,"  ",end_idx)
        transformation = np.zeros((end_idx-st_idx,20000*3),dtype='float32')

        for i, idx in enumerate (list(range(st_idx,end_idx))):
            im = cv2.imread(path[idx])
            im = np.array(cv2.resize(im,dsize=(200,100)),dtype='float32')
            # h, edges = np.histogramdd(im.reshape(-1,3),8,normed=True,range=[(0,255),(0,255),(0,255)])
            transform=np.fft.fft2(im)
            transformation[i] = np.abs(transform.flatten())
        res = torch.from_numpy(transformation)
        print("start clustering")
        labels,_ = kmeans(X=res, num_clusters=to_class, distance='euclidean', device=torch.device('cuda:1'))
        cluster_map = pandas.DataFrame()
        cluster_map['cluster'] = labels

        for i, idx in enumerate (list(range(st_idx,end_idx))):
            im_path = path[idx]
            im = cv2.imread(im_path)
            name = path[idx].split('/')[7]
            cluster = cluster_map['cluster'][i]
            savepath ='/home/ubuntu/data/Workspace/Soobin/attempt'+str(attempt)+'/img'
            savepath = savepath+str(cluster)+'/'
            os.makedirs(savepath,exist_ok=True)
            cv2.imwrite(savepath+name,im)
            # item = []
            # item.append(path)
            # item.append(name)
            # item.append(str(cluster))
            # csv.append(item)
        # csv_file = pandas.DataFrame(csv, columns=['file_path', 'file_name','cluster'])
        # csv_file.to_csv('/home/ubuntu/data/Workspace/Soobin/attempt%d/'%(attempt)+'cluster.csv', index=False, encoding='cp949')
    return

coexist_multiclass(6)
# path='/home/ubuntu/data/Workspace/Soobin/wide_total/'
# path = glob.glob(path+'*')
# # print(path)
# for idx, p in enumerate(path):
#     name = path[idx].split('/')[8]
#     print(name)
#     # if 'wide_total' in name:
#     #     split = path[idx].split('/')[:7]
#     #     rename_path = '/'.join(split)
#     #     rename = name[14:]
#     #     print(rename)
#     #     os.rename(path[idx],rename_path+'/'+rename)

def to_csv(path):
    for f in path:
        item = []
        name = f.split('/')[10][:-4]
        parms = name.split('_')
        code = parms[0]
        year = parms[2][:4]
        month= parms[2][4:6]
        date = parms[2][6:]
        item.append('OCT')
        item.append('Seoul')
        item.append(year)
        item.append(month)
        item.append(date)
        item.append(code)
        item.append(name)
        csv.append(item)
        print(item)
    csv_file = pandas.DataFrame(csv, columns=['root', 'hospital','year', 'month', 'date', 'code', 'file'])
    csv_file.to_csv(savepath+'wide_higher_resolution.csv', index=False, encoding='cp949')
    return

def move_result():
    path = '/home/ubuntu/data/Workspace/Soobin/data/topcon_wide_higher_resolution/'
    files = glob.glob(path+'*/*/*')
    print(files[:30])
    savepath ='/home/ubuntu/data/Workspace/Soobin/wide_higher_resolution/data/'
    for f in files:
        im = cv2.imread(f)
        name = f.split('/')[10][:-4]
        cv2.imwrite(savepath+name,im)
        # os.remove(f)
    return

    # length = len(path)
    # transformation = np.zeros((len(path),20000*3),dtype='float32')

    # for p in path:
    #     im = cv2.imread(p)
    #     im = np.array(cv2.resize(im,dsize=(200,100)),dtype='float32')
    #     # h, edges = np.histogramdd(im.reshape(-1,3),8,normed=True,range=[(0,255),(0,255),(0,255)])
    #     transform=np.fft.fft2(im)
    #     transformation[i] = np.abs(transform.flatten())

    # res = torch.from_numpy(transformation)  # to use gpu
    # print("start clustering")
    # labels,_ = kmeans(X=topcon, num_clusters=to_class, distance='euclidean', device=torch.device('cuda:0'))
    # cluster_map = pandas.DataFrame()
    # cluster_map['cluster'] = labels

    # for i in range (0,len(cluster_map['cluster'])):
    #         topcon_imwrite(cluster_map['cluster'][i],j, i)

def tot_dat():
    for j in range(0,20):
        topcon = create_tempfeatures(j)
        print("start clustering")
        labels,_ = kmeans(X=topcon, num_clusters=5, distance='euclidean', device=torch.device('cuda:0'))
        cluster_map = pandas.DataFrame()
        cluster_map['cluster'] = labels
        for i in range (0,len(cluster_map['cluster'])):
            topcon_imwrite(cluster_map['cluster'][i],j, i)
        csv_file = pandas.DataFrame(csv, columns=['file_path', 'file_name','cluster'])
        csv=[]
        csv_file.to_csv('./topcon_candidates_5class_higher/attempt%d/'%(j)+'cluster.csv', index=False, encoding='cp949')
        print("one attempt done!")


# -*- coding:utf-8 -*-
"""
Author:
    Weichen Shen,wcshen1994@163.com

Reference:
    [1] Wide & Deep Learning for Recommender Systems(https://arxiv.org/pdf/1606.07792.pdf)
"""

from tensorflow.python.keras.layers import Dense, Embedding, Concatenate, Flatten, add
from tensorflow.python.keras.models import Model
from tensorflow.python.keras.initializers import RandomNormal
from tensorflow.python.keras.regularizers import l2
from ..layers import PredictionLayer, MLP
from ..utils import get_input

def WDL(deep_feature_dim_dict, wide_feature_dim_dict, embedding_size=8, hidden_size=[128, 128], l2_reg_linear=1e-5, l2_reg_embedding=1e-5, l2_reg_deep=0,
        init_std=0.0001, seed=1024, keep_prob=1, activation='relu', final_activation='sigmoid',
        ):
    """Instantiates the Wide&Deep Learning architecture.

    :param deep_feature_dim_dict: dict,to indicate sparse field and dense field in deep part like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']}
    :param wide_feature_dim_dict: dict,to indicate sparse field and dense field in wide part like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']}
    :param embedding_size: positive integer,sparse feature embedding_size
    :param hidden_size: list,list of positive integer or empty list, the layer number and units in each layer of deep net
    :param l2_reg_linear: float. L2 regularizer strength applied to wide part
    :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
    :param l2_reg_deep: float. L2 regularizer strength applied to deep net
    :param init_std: float,to use as the initialize std of embedding vector
    :param seed: integer ,to use as random seed.
    :param keep_prob: float in (0,1]. keep_prob used in deep net
    :param activation: Activation function to use in deep net
    :param final_activation: str,output activation,usually ``'sigmoid'`` or ``'linear'``
    :return: A Keras model instance.
    """
    if not isinstance(deep_feature_dim_dict,
                      dict) or "sparse" not in deep_feature_dim_dict or "dense" not in deep_feature_dim_dict:
        raise ValueError(
            "feature_dim must be a dict like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_5',]}")

    sparse_input, dense_input, bias_sparse_input, bias_dense_input = get_input(
        deep_feature_dim_dict, wide_feature_dim_dict)
    sparse_embedding, wide_linear_embedding = get_embeddings(
        deep_feature_dim_dict, wide_feature_dim_dict, embedding_size, init_std, seed, l2_reg_embedding, l2_reg_linear)

    embed_list = [sparse_embedding[i](sparse_input[i])
                  for i in range(len(sparse_input))]
    deep_input = Concatenate()(embed_list)
    deep_input = Flatten()(deep_input)
    if len(dense_input) > 0:
        deep_input = Concatenate()([deep_input]+dense_input)

    deep_out = MLP(hidden_size, activation, l2_reg_deep, keep_prob,
                   False, seed)(deep_input)
    deep_logit = Dense(1, use_bias=False, activation=None)(deep_out)
    final_logit = deep_logit
    if len(wide_feature_dim_dict['dense']) + len(wide_feature_dim_dict['sparse']) > 0:
        if len(wide_feature_dim_dict['sparse']) > 0:
            bias_embed_list = [wide_linear_embedding[i](
                bias_sparse_input[i]) for i in range(len(bias_sparse_input))]
            linear_term = add(bias_embed_list) if len(
                bias_embed_list) > 1 else bias_embed_list[0]
            final_logit = add([final_logit, linear_term])
        if len(wide_feature_dim_dict['dense']) > 0:
            wide_dense_term = Dense(1, use_bias=False, activation=None)(Concatenate()(
                bias_dense_input) if len(bias_dense_input) > 1 else bias_dense_input[0])
            final_logit = add([final_logit, wide_dense_term])

    output = PredictionLayer(final_activation)(final_logit)
    model = Model(inputs=sparse_input + dense_input +
                  bias_sparse_input + bias_dense_input, outputs=output)
    return model

def get_embeddings(deep_feature_dim_dict, wide_feature_dim_dict, embedding_size, init_std, seed, l2_rev_V, l2_reg_w):
    sparse_embedding = [Embedding(deep_feature_dim_dict["sparse"][feat], embedding_size,
                                  embeddings_initializer=RandomNormal(
                                      mean=0.0, stddev=init_std, seed=seed),
                                  embeddings_regularizer=l2(l2_rev_V),
                                  name='sparse_emb_' + str(i) + '-' + feat) for i, feat in
                        enumerate(deep_feature_dim_dict["sparse"])]
    linear_embedding = [Embedding(wide_feature_dim_dict["sparse"][feat], 1,
                                  embeddings_initializer=RandomNormal(mean=0.0, stddev=init_std,
                                                                      seed=seed), embeddings_regularizer=l2(l2_reg_w),
                                  name='linear_emb_' + str(i) + '-' + feat) for
                        i, feat in enumerate(wide_feature_dim_dict["sparse"])]

    return sparse_embedding, linear_embedding


import contextlib
import ctypes
import doctest
import imp
import os
import re
import shutil
import struct
import sys
import tempfile
import uuid
import unittest
if sys.hexversion <= 0x03030000:
    from mock import patch
else:
    from unittest.mock import patch
import warnings
from xml.etree import cElementTree as ET

import numpy as np
import pkg_resources

from glymur import Jp2k
from glymur.lib import openjp2 as opj2
import glymur

try:
    data_root = os.environ['OPJ_DATA_ROOT']
except KeyError:
    data_root = None
except:
    raise

# Doc tests should be run as well.
def load_tests(loader, tests, ignore):
    tests.addTests(doctest.DocTestSuite('glymur.jp2k'))
    return tests

@contextlib.contextmanager
def chdir(dirname=None):
    curdir = os.getcwd()
    try:
        if dirname is not None:
            os.chdir(dirname)
        yield
    finally:
        os.chdir(curdir)

class TestJp2k(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        # Setup a JP2 file with a bad XML box.  We only need to do this once
        # per class rather than once per test.
        jp2file = pkg_resources.resource_filename(glymur.__name__,
                                                  "data/nemo.jp2")
        with tempfile.NamedTemporaryFile(suffix='.jp2', delete=False) as tfile:
            cls._bad_xml_file = tfile.name
            with open(jp2file, 'rb') as ifile:
                # Everything up until the jp2c box.
                buffer = ifile.read(77)
                tfile.write(buffer)

                # Write the xml box with bad xml
                # Length = 28, id is 'xml '.
                buffer = struct.pack('>I4s', int(28), b'xml ')
                tfile.write(buffer)

                buffer = '<test>this is a test'
                buffer = buffer.encode()
                tfile.write(buffer)

                # Get the rest of the input file.
                buffer = ifile.read()
                tfile.write(buffer)
                tfile.flush()

    @classmethod
    def tearDownClass(cls):
        os.unlink(cls._bad_xml_file)

    def setUp(self):
        self.jp2file = pkg_resources.resource_filename(glymur.__name__,
                                                       "data/nemo.jp2")

    def tearDown(self):
        pass

    @unittest.skipIf(sys.hexversion < 0x03020000,
                     "Uses features introduced in 3.2.")
    def test_invalid_xml_box_warning(self):
        # Should be able to recover from xml box with bad xml.
        # Just verify that a warning is issued on 3.3+
        with self.assertWarns(UserWarning) as cw:
            jp2k = Jp2k(self._bad_xml_file)

    def test_reduce_max(self):
        # Verify that reduce=-1 gets us the lowest resolution image
        j = Jp2k(self.jp2file)
        thumbnail1 = j.read(reduce=-1)
        thumbnail2 = j.read(reduce=5)
        np.testing.assert_array_equal(thumbnail1, thumbnail2)
        self.assertEqual(thumbnail1.shape, (46, 81, 3))

    def test_invalid_xml_box(self):
        # Should be able to recover from xml box with bad xml.
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            jp2k = Jp2k(self._bad_xml_file)

        self.assertEqual(jp2k.box[3].id, 'xml ')
        self.assertEqual(jp2k.box[3].offset, 77)
        self.assertEqual(jp2k.box[3].length, 28)
        self.assertIsNone(jp2k.box[3].xml)

    def test_bad_area_parameter(self):
        # Verify that we error out appropriately if given a bad area parameter.
        j = Jp2k(self.jp2file)
        with self.assertRaises(IOError):
            # Start corner must be >= 0
            d = j.read(area=(-1, -1, 1, 1))
        with self.assertRaises(IOError):
            # End corner must be > 0
            d = j.read(area=(10, 10, 0, 0))
        with self.assertRaises(IOError):
            # End corner must be >= start corner
            d = j.read(area=(10, 10, 8, 8))

    def test_reduce_too_high(self):
        # Verify that we error out appropriately if not given a JPEG 2000 file.
        j = Jp2k(self.jp2file)
        with self.assertRaises(IOError):
            d = j.read(reduce=6)

    def test_not_JPEG2000(self):
        # Verify that we error out appropriately if not given a JPEG 2000 file.
        filename = pkg_resources.resource_filename(glymur.__name__, "jp2k.py")
        with self.assertRaises(IOError):
            jp2k = Jp2k(filename)

    def test_file_not_present(self):
        # Verify that we error out appropriately if not given an existing file
        # at all.
        if sys.hexversion < 0x03030000:
            error = OSError
        else:
            error = IOError
        with self.assertRaises(error):
            filename = 'this file does not actually exist on the file system.'
            jp2k = Jp2k(filename)

    def test_nemo_tile(self):
        # Issue 134, trouble reading first nemo tile.
        j = Jp2k(self.jp2file)
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            tiledata = j.read(tile=0)
        subsetdata = j.read(area=(0, 0, 512, 512))
        np.testing.assert_array_equal(tiledata, subsetdata)

    def test_write_cprl(self):
        # Issue 17
        j = Jp2k(self.jp2file)
        expdata = j.read(reduce=2)
        with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
            ofile = Jp2k(tfile.name, 'wb')
            ofile.write(expdata, prog='CPRL')
            actdata = ofile.read()
            np.testing.assert_array_equal(actdata, expdata)

            c = ofile.get_codestream()
            self.assertEqual(c.segment[2].SPcod[0], glymur.core.CPRL)

    def test_jp2_boxes(self):
        # Verify the boxes of a JP2 file.
        jp2k = Jp2k(self.jp2file)

        # top-level boxes
        self.assertEqual(len(jp2k.box), 6)

        self.assertEqual(jp2k.box[0].id, 'jP  ')
        self.assertEqual(jp2k.box[0].offset, 0)
        self.assertEqual(jp2k.box[0].length, 12)
        self.assertEqual(jp2k.box[0].longname, 'JPEG 2000 Signature')

        self.assertEqual(jp2k.box[1].id, 'ftyp')
        self.assertEqual(jp2k.box[1].offset, 12)
        self.assertEqual(jp2k.box[1].length, 20)
        self.assertEqual(jp2k.box[1].longname, 'File Type')

        self.assertEqual(jp2k.box[2].id, 'jp2h')
        self.assertEqual(jp2k.box[2].offset, 32)
        self.assertEqual(jp2k.box[2].length, 45)
        self.assertEqual(jp2k.box[2].longname, 'JP2 Header')

        self.assertEqual(jp2k.box[3].id, 'uuid')
        self.assertEqual(jp2k.box[3].offset, 77)
        self.assertEqual(jp2k.box[3].length, 638)

        self.assertEqual(jp2k.box[4].id, 'uuid')
        self.assertEqual(jp2k.box[4].offset, 715)
        self.assertEqual(jp2k.box[4].length, 2412)

        self.assertEqual(jp2k.box[5].id, 'jp2c')
        self.assertEqual(jp2k.box[5].offset, 3127)
        self.assertEqual(jp2k.box[5].length, 1133427)

        # jp2h super box
        self.assertEqual(len(jp2k.box[2].box), 2)

        self.assertEqual(jp2k.box[2].box[0].id, 'ihdr')
        self.assertEqual(jp2k.box[2].box[0].offset, 40)
        self.assertEqual(jp2k.box[2].box[0].length, 22)
        self.assertEqual(jp2k.box[2].box[0].longname, 'Image Header')
        self.assertEqual(jp2k.box[2].box[0].height, 1456)
        self.assertEqual(jp2k.box[2].box[0].width, 2592)
        self.assertEqual(jp2k.box[2].box[0].num_components, 3)
        self.assertEqual(jp2k.box[2].box[0].bits_per_component, 8)
        self.assertEqual(jp2k.box[2].box[0].signed, False)
        self.assertEqual(jp2k.box[2].box[0].compression, 7)
        self.assertEqual(jp2k.box[2].box[0].cspace_unknown, False)
        self.assertEqual(jp2k.box[2].box[0].ip_provided, False)

        self.assertEqual(jp2k.box[2].box[1].id, 'colr')
        self.assertEqual(jp2k.box[2].box[1].offset, 62)
        self.assertEqual(jp2k.box[2].box[1].length, 15)
        self.assertEqual(jp2k.box[2].box[1].longname, 'Colour Specification')
        self.assertEqual(jp2k.box[2].box[1].precedence, 0)
        self.assertEqual(jp2k.box[2].box[1].approximation, 0)
        self.assertEqual(jp2k.box[2].box[1].colorspace, glymur.core.SRGB)
        self.assertIsNone(jp2k.box[2].box[1].icc_profile)

    @unittest.skipIf(data_root is None,
                     "OPJ_DATA_ROOT environment variable not set")
    def test_j2k_box(self):
        # Verify that a J2K file has no boxes.
        filename = os.path.join(data_root, 'input/conformance/p0_01.j2k')
        jp2k = Jp2k(filename)
        self.assertEqual(len(jp2k.box), 0)

    def test_64bit_XL_field(self):
        # Verify that boxes with the XL field are properly read.
        # Don't have such a file on hand, so we create one.  Copy our example
        # file, but making the codestream have a 64-bit XL field.
        with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
            with open(self.jp2file, 'rb') as ifile:
                # Everything up until the jp2c box.
                buffer = ifile.read(3127)
                tfile.write(buffer)

                # The L field must be 1 in order to signal the presence of the
                # XL field.  The actual length of the jp2c box increased by 8
                # (8 bytes for the XL field).
                L = 1
                T = b'jp2c'
                XL = 1133427 + 8
                buffer = struct.pack('>I4sQ', int(L), T, XL)
                tfile.write(buffer)

                # Get the rest of the input file (minus the 8 bytes for L and
                # T.
                ifile.seek(8, 1)
                buffer = ifile.read()
                tfile.write(buffer)
                tfile.flush()

            jp2k = Jp2k(tfile.name)

            self.assertEqual(jp2k.box[5].id, 'jp2c')
            self.assertEqual(jp2k.box[5].offset, 3127)
            self.assertEqual(jp2k.box[5].length, 1133427 + 8)

    def test_L_is_zero(self):
        # Verify that boxes with the L field as zero are correctly read.
        # This should only happen in the last box of a JPEG 2000 file.
        # Our example image has its last box at byte 588458.
        baseline_jp2 = Jp2k(self.jp2file)
        with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
            with open(self.jp2file, 'rb') as ifile:
                # Everything up until the jp2c box.
                buffer = ifile.read(588458)
                tfile.write(buffer)

                L = 0
                T = b'uuid'
                buffer = struct.pack('>I4s', int(L), T)
                tfile.write(buffer)

                # Get the rest of the input file (minus the 8 bytes for L and
                # T.
                ifile.seek(8, 1)
                buffer = ifile.read()
                tfile.write(buffer)
                tfile.flush()

            new_jp2 = Jp2k(tfile.name)

            # The top level boxes in each file should match.
            for j in range(len(baseline_jp2.box)):
                self.assertEqual(new_jp2.box[j].id,
                                 baseline_jp2.box[j].id)
                self.assertEqual(new_jp2.box[j].offset,
                                 baseline_jp2.box[j].offset)
                self.assertEqual(new_jp2.box[j].length,
                                 baseline_jp2.box[j].length)

    @unittest.skipIf(data_root is None,
                     "OPJ_DATA_ROOT environment variable not set")
    def test_read_differing_subsamples(self):
        # Verify that we error out appropriately if we use the read method
        # on an image with differing subsamples
        #
        # Issue 86.
        filename = os.path.join(data_root, 'input/conformance/p0_05.j2k')
        j = Jp2k(filename)
        with self.assertRaises(IOError):
            j.read()

    @unittest.skipIf(data_root is None,
                     "OPJ_DATA_ROOT environment variable not set")
    def test_empty_box_with_j2k(self):
        # Verify that the list of boxes in a J2C/J2K file is present, but
        # empty.
        filename = os.path.join(data_root, 'input/conformance/p0_05.j2k')
        j = Jp2k(filename)
        self.assertEqual(j.box, [])

    def test_code_block_height_different_than_width(self):
        # Verify that we can set a code block size where height does not equal
        # width.
        data = np.zeros((128, 128), dtype=np.uint8)
        with tempfile.NamedTemporaryFile(suffix='.j2k') as tfile:
            j = Jp2k(tfile.name, 'wb')

            # The code block dimensions are given as rows x columns.
            j.write(data, cbsize=(16, 32))

            c = j.get_codestream()

            # Code block size is reported as XY in the codestream.
            self.assertEqual(tuple(c.segment[2].SPcod[5:7]), (3, 2))

    def test_negative_too_many_dimensions(self):
        # OpenJP2 only allows 2D or 3D images.
        with tempfile.NamedTemporaryFile(suffix='.j2k') as tfile:
            j = Jp2k(tfile.name, 'wb')
            with self.assertRaises(IOError) as ce:
                data = np.zeros((128, 128, 2, 2), dtype=np.uint8)
                j.write(data)

    def test_unrecognized_jp2_colorspace(self):
        # We only allow RGB and GRAYSCALE.
        with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
            j = Jp2k(tfile.name, 'wb')
            with self.assertRaises(IOError) as ce:
                data = np.zeros((128, 128, 3), dtype=np.uint8)
                j.write(data, colorspace='cmyk')

    def test_2D_rgb(self):
        # RGB must have at least 3 components.
        with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
            j = Jp2k(tfile.name, 'wb')
            with self.assertRaises(IOError) as ce:
                data = np.zeros((128, 128, 2), dtype=np.uint8)
                j.write(data, colorspace='rgb')

    def test_colorspace_with_j2k(self):
        # Specifying a colorspace with J2K does not make sense.
        with tempfile.NamedTemporaryFile(suffix='.j2k') as tfile:
            j = Jp2k(tfile.name, 'wb')
            with self.assertRaises(IOError) as ce:
                data = np.zeros((128, 128, 3), dtype=np.uint8)
                j.write(data, colorspace='rgb')

    def test_specify_rgb(self):
        with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
            j = Jp2k(tfile.name, 'wb')
            data = np.zeros((128, 128, 3), dtype=np.uint8)
            j.write(data, colorspace='rgb')
            self.assertEqual(j.box[2].box[1].colorspace, glymur.core.SRGB)

    def test_specify_gray(self):
        with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
            j = Jp2k(tfile.name, 'wb')
            data = np.zeros((128, 128), dtype=np.uint8)
            j.write(data, colorspace='gray')
            self.assertEqual(j.box[2].box[1].colorspace,
                             glymur.core.GREYSCALE)

    def test_specify_grey(self):
        with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
            j = Jp2k(tfile.name, 'wb')
            data = np.zeros((128, 128), dtype=np.uint8)
            j.write(data, colorspace='grey')
            self.assertEqual(j.box[2].box[1].colorspace,
                             glymur.core.GREYSCALE)

    def test_grey_with_extra_component(self):
        with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
            j = Jp2k(tfile.name, 'wb')
            data = np.zeros((128, 128, 2), dtype=np.uint8)
            j.write(data)
            self.assertEqual(j.box[2].box[0].height, 128)
            self.assertEqual(j.box[2].box[0].width, 128)
            self.assertEqual(j.box[2].box[0].num_components, 2)
            self.assertEqual(j.box[2].box[1].colorspace,
                             glymur.core.GREYSCALE)

    def test_grey_with_two_extra_components(self):
        with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
            j = Jp2k(tfile.name, 'wb')
            data = np.zeros((128, 128, 3), dtype=np.uint8)
            j.write(data, colorspace='gray')
            self.assertEqual(j.box[2].box[0].height, 128)
            self.assertEqual(j.box[2].box[0].width, 128)
            self.assertEqual(j.box[2].box[0].num_components, 3)
            self.assertEqual(j.box[2].box[1].colorspace,
                             glymur.core.GREYSCALE)

    def test_rgb_with_extra_component(self):
        with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
            j = Jp2k(tfile.name, 'wb')
            data = np.zeros((128, 128, 4), dtype=np.uint8)
            j.write(data)
            self.assertEqual(j.box[2].box[0].height, 128)
            self.assertEqual(j.box[2].box[0].width, 128)
            self.assertEqual(j.box[2].box[0].num_components, 4)
            self.assertEqual(j.box[2].box[1].colorspace, glymur.core.SRGB)

    def test_specify_ycc(self):
        # We don't support writing YCC at the moment.
        with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
            j = Jp2k(tfile.name, 'wb')
            with self.assertRaises(IOError) as ce:
                data = np.zeros((128, 128, 3), dtype=np.uint8)
                j.write(data, colorspace='ycc')

    def test_uinf_ulst_url_boxes(self):
        # Verify that we can read UINF, ULST, and URL boxes.  I don't have
        # easy access to such a file, and there's no such file in the
        # openjpeg repository, so I'll fake one.
        with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
            with open(self.jp2file, 'rb') as ifile:
                # Everything up until the jp2c box.
                buffer = ifile.read(77)
                tfile.write(buffer)

                # Write the UINF superbox
                # Length = 50, id is uinf.
                buffer = struct.pack('>I4s', int(50), b'uinf')
                tfile.write(buffer)

                # Write the ULST box.
                # Length is 26, 1 UUID, hard code that UUID as zeros.
                buffer = struct.pack('>I4sHIIII', int(26), b'ulst', int(1),
                                     int(0), int(0), int(0), int(0))
                tfile.write(buffer)

                # Write the URL box.
                # Length is 16, version is one byte, flag is 3 bytes, url
                # is the rest.
                buffer = struct.pack('>I4sBBBB',
                                     int(16), b'url ',
                                     int(0), int(0), int(0), int(0))
                tfile.write(buffer)
                buffer = struct.pack('>ssss', b'a', b'b', b'c', b'd')
                tfile.write(buffer)

                # Get the rest of the input file.
                buffer = ifile.read()
                tfile.write(buffer)
                tfile.flush()

            jp2k = Jp2k(tfile.name)

            self.assertEqual(jp2k.box[3].id, 'uinf')
            self.assertEqual(jp2k.box[3].offset, 77)
            self.assertEqual(jp2k.box[3].length, 50)

            self.assertEqual(jp2k.box[3].box[0].id, 'ulst')
            self.assertEqual(jp2k.box[3].box[0].offset, 85)
            self.assertEqual(jp2k.box[3].box[0].length, 26)
            ulst = []
            ulst.append(uuid.UUID('00000000-0000-0000-0000-000000000000'))
            self.assertEqual(jp2k.box[3].box[0].ulst, ulst)

            self.assertEqual(jp2k.box[3].box[1].id, 'url ')
            self.assertEqual(jp2k.box[3].box[1].offset, 111)
            self.assertEqual(jp2k.box[3].box[1].length, 16)
            self.assertEqual(jp2k.box[3].box[1].version, 0)
            self.assertEqual(jp2k.box[3].box[1].flag, (0, 0, 0))
            self.assertEqual(jp2k.box[3].box[1].URL, 'abcd')

    def test_xml_box_with_trailing_nulls(self):
        # ElementTree does not like trailing null chars after valid XML
        # text.
        with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
            with open(self.jp2file, 'rb') as ifile:
                # Everything up until the jp2c box.
                buffer = ifile.read(77)
                tfile.write(buffer)

                # Write the xml box
                # Length = 36, id is 'xml '.
                buffer = struct.pack('>I4s', int(36), b'xml ')
                tfile.write(buffer)

                buffer = '<test>this is a test</test>' + chr(0)
                buffer = buffer.encode()
                tfile.write(buffer)

                # Get the rest of the input file.
                buffer = ifile.read()
                tfile.write(buffer)
                tfile.flush()

            jp2k = Jp2k(tfile.name)

            self.assertEqual(jp2k.box[3].id, 'xml ')
            self.assertEqual(jp2k.box[3].offset, 77)
            self.assertEqual(jp2k.box[3].length, 36)

    def test_asoc_label_box(self):
        # Construct a fake file with an asoc and a label box, as
        # OpenJPEG doesn't have such a file.
        data = Jp2k(self.jp2file).read(reduce=3)
        with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
            j = Jp2k(tfile.name, 'wb')
            j.write(data)

            with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile2:

                # Offset of the codestream is where we start.
                buffer = tfile.read(77)
                tfile2.write(buffer)

                # read the rest of the file, it's the codestream.
                codestream = tfile.read()

                # Write the asoc superbox.
                # Length = 36, id is 'asoc'.
                buffer = struct.pack('>I4s', int(56), b'asoc')
                tfile2.write(buffer)

                # Write the contained label box
                buffer = struct.pack('>I4s', int(13), b'lbl ')
                tfile2.write(buffer)
                tfile2.write('label'.encode())

                # Write the xml box
                # Length = 36, id is 'xml '.
                buffer = struct.pack('>I4s', int(35), b'xml ')
                tfile2.write(buffer)

                buffer = '<test>this is a test</test>'
                buffer = buffer.encode()
                tfile2.write(buffer)

                # Now append the codestream.
                tfile2.write(codestream)
                tfile2.flush()

                jasoc = Jp2k(tfile2.name)
                self.assertEqual(jasoc.box[3].id, 'asoc')
                self.assertEqual(jasoc.box[3].box[0].id, 'lbl ')
                self.assertEqual(jasoc.box[3].box[0].label, 'label')
                self.assertEqual(jasoc.box[3].box[1].id, 'xml ')

    def test_openjpeg_library_message(self):
        # Verify the error message produced by the openjpeg library.
        # This will confirm that the error callback mechanism is working.
        with open(self.jp2file, 'rb') as fp:
            data = fp.read()
            with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
                # Codestream starts at byte 3127. SIZ marker at 3137.
                # COD marker at 3186.  Subsampling at 3180.
                tfile.write(data[0:3179])

                # Make the DY bytes of the SIZ segment zero.  That means that
                # a subsampling factor is zero, which is illegal.
                tfile.write(b'\x00')
                tfile.write(data[3180:3182])
                tfile.write(b'\x00')
                tfile.write(data[3184:3186])
                tfile.write(b'\x00')

                tfile.write(data[3186:])
                tfile.flush()
                with warnings.catch_warnings():
                    warnings.simplefilter("ignore")
                    j = Jp2k(tfile.name)
                regexp = re.compile(r'''OpenJPEG\slibrary\serror:\s+
                                        Invalid\svalues\sfor\scomp\s=\s0\s+
                                        :\sdx=1\sdy=0''', re.VERBOSE)
                if sys.hexversion < 0x03020000:
                    with self.assertRaisesRegexp(IOError, regexp) as ce:
                        d = j.read(reduce=3)
                else:
                    with self.assertRaisesRegex(IOError, regexp) as ce:
                        d = j.read(reduce=3)

    @unittest.skipIf(sys.hexversion < 0x03020000,
                     "Uses features introduced in 3.2.")
    def test_config_file_via_environ(self):
        """Verify that we can read a configuration file set via environ var."""
        with tempfile.TemporaryDirectory() as tdir:
            filename = os.path.join(tdir, 'glymurrc')
            with open(filename, 'wb') as tfile:
                tfile.write('[library]\n'.encode())
                line = 'openjp2: {0}\n'.format(glymur._OPENJP2._name)
                tfile.write(line.encode())
                tfile.flush()
                with patch.dict('os.environ', {'GLYMURCONFIGDIR': tdir}):
                    imp.reload(glymur)
                    j = Jp2k(self.jp2file)

    @unittest.skipIf(sys.hexversion < 0x03020000,
                     "Uses features introduced in 3.2.")
    def test_config_file_via_environ_is_wrong(self):
        # A non-existant library location should be rejected.
        with tempfile.TemporaryDirectory() as tdir:
            fname = os.path.join(tdir, 'glymurrc')
            with open(fname, 'w') as fp:
                with tempfile.NamedTemporaryFile(suffix='.dylib') as tfile:
                    fp.write('[library]\n')
                    fp.write('openjp2: {0}.not.there\n'.format(tfile.name))
                    fp.flush()
                    with patch.dict('os.environ', {'GLYMURCONFIGDIR': tdir}):
                        # Misconfigured new configuration file should
                        # be rejected.
                        with self.assertWarns(UserWarning) as cw:
                            imp.reload(glymur)

    @unittest.skipIf(sys.hexversion < 0x03020000,
                     "Uses features introduced in 3.2.")
    def test_missing_config_file_via_environ(self):
        # Verify that we error out properly if the configuration file
        # specified via environment variable is not found.
        with tempfile.TemporaryDirectory() as tdir:
            with patch.dict('os.environ', {'GLYMURCONFIGDIR': tdir}):
                # Misconfigured new configuration file should
                # be rejected.
                with self.assertWarns(UserWarning) as cw:
                    imp.reload(glymur)

    @unittest.skipIf(sys.hexversion < 0x03020000,
                     "Uses features introduced in 3.2.")
    def test_home_dir_missing_config_dir(self):
        # Verify no exception is raised if $HOME is missing .glymur directory.
        with tempfile.TemporaryDirectory() as tdir:
            with patch.dict('os.environ', {'HOME': tdir}):
                # Misconfigured new configuration file should
                # be rejected.
                with self.assertWarns(UserWarning) as cw:
                    imp.reload(glymur)

    def test_xmp_attribute(self):
        # Verify that we can read the XMP packet in our shipping example file.
        j = Jp2k(self.jp2file)
        xmp = j.box[4].data
        ns0 = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}'
        ns1 = '{http://ns.adobe.com/xap/1.0/}'
        name = '{0}RDF/{0}Description'.format(ns0)
        elt = xmp.find(name)
        attr_value = elt.attrib['{0}CreatorTool'.format(ns1)]
        self.assertEqual(attr_value, 'glymur')

    def test_unrecognized_exif_tag(self):
        # An unrecognized exif tag should be handled gracefully.
        with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
            shutil.copyfile(self.jp2file, tfile.name)

            # The Exif UUID starts at byte 77.  There are 8 bytes for the L and
            # T fields, then 16 bytes for the UUID identifier, then 6 exif
            # header bytes, then 8 bytes for the TIFF header, then 2 bytes
            # the the Image IFD number of tags, where we finally find the first
            # tag, "Make" (271).  We'll corrupt it by changing it into 171,
            # which does not correspond to any known Exif Image tag.
            with open(tfile.name, 'r+b') as fp:
                fp.seek(117)
                buffer = struct.pack('<H', int(171))
                fp.write(buffer)

            # Verify that a warning is issued, but only on python3.
            # On python2, just suppress the warning.
            if sys.hexversion < 0x03030000:
                with warnings.catch_warnings():
                    warnings.simplefilter("ignore")
                    j = Jp2k(tfile.name)
            else:
                with self.assertWarns(UserWarning) as cw:
                    j = Jp2k(tfile.name)

            exif = j.box[3].data
            # Were the tag == 271, 'Make' would be in the keys instead.
            self.assertTrue(171 in exif['Image'].keys())
            self.assertFalse('Make' in exif['Image'].keys())

if __name__ == "__main__":
    unittest.main()


from django.db import models
from django.db import IntegrityError
from django.db.models import Q

from common import errors

class Slide(models.Model):
    '''滑动记录表'''
    STYPES = (
        ('like', '喜欢'),
        ('superlike', '超级喜欢'),
        ('dislike', '不喜欢'),
    )
    uid = models.IntegerField(verbose_name='滑动用户')
    sid = models.IntegerField(verbose_name='被滑动的用户')
    slide_type = models.CharField(max_length=10, choices=STYPES, verbose_name='滑动类型')
    slide_time = models.DateTimeField(auto_now_add=True, verbose_name='滑动时间')

    class Meta:
        # 设置联合唯一，不能对同一个用户进行两次操作
        unique_together = ['uid', 'sid']
        db_table = 'slide'

    @classmethod
    def slide(cls, uid, sid, slide_type):
        try:
            cls.objects.create(uid=uid, sid=sid, slide_type=slide_type)
        except IntegrityError:
            # 抛出重复滑动异常
            raise errors.RepeatSlideErr

    @classmethod
    def is_liked(cls, uid, sid):
        slide = Slide.objects.filter(uid=uid, sid=sid).first()
        if not slide:
            return None
        elif slide.slide_type in ['like', 'superlike']:
            return True
        elif slide.slide_type == 'dislike':
            return False

class Friend(models.Model):
    '''好友表'''
    uid1 = models.IntegerField(verbose_name='uid1')
    uid2 = models.IntegerField(verbose_name='uid2')

    class Meta:
        unique_together = ['uid1', 'uid2']
        db_table = 'friend'

    @classmethod
    def make_friend(cls, uid1, uid2):
        uid1, uid2 = (uid1, uid2) if uid1 < uid2 else (uid2, uid1)
        cls.objects.create(uid1=uid1, uid2=uid2)

    @classmethod
    def broken(cls, uid1, uid2):
        uid1, uid2 = (uid1, uid2) if uid1 < uid2 else (uid2, uid1)
        cls.objects.filter(uid1=uid1, uid2=uid2).delete()

    @classmethod
    def find_friends(cls, uid):
        condition = Q(uid1=uid) | Q(uid2=uid)
        friend_list = Friend.objects.filter(condition)

        fid_list = []
        for f in friend_list:
            if f.uid1 == uid:
                fid_list.append(f.uid2)
            else:
                fid_list.append(f.uid1)

        return fid_list


from collections import OrderedDict

# import seaborn as sns
import torch

import models
from datasets import (Chest_few_shot, CropDisease_few_shot, EuroSAT_few_shot,
                      ISIC_few_shot, miniImageNet_few_shot)

def load_checkpoint(model, load_path, device):
    '''
    Load model and optimizer from load path
    Return the epoch to continue the checkpoint
    '''
    state = torch.load(load_path, map_location=torch.device(device))['state']
    clf_state = OrderedDict()
    state_keys = list(state.keys())
    for _, key in enumerate(state_keys):
        if "feature." in key:
            # an architecture model has attribute 'feature', load architecture
            # feature to backbone by casting name from 'feature.trunk.xx' to 'trunk.xx'
            newkey = key.replace("feature.", "")
            state[newkey] = state.pop(key)
        elif "classifier." in key:
            newkey = key.replace("classifier.", "")
            clf_state[newkey] = state.pop(key)
        else:
            state.pop(key)
    model.load_state_dict(state)

def load_checkpoint2(model, load_path, device):
    '''
    Load model and optimizer from load path
    Return the epoch to continue the checkpoint
    '''
    sd = torch.load(load_path, map_location=torch.device(device))
    model.load_state_dict(sd['model'])

    return sd['epoch']

if torch.cuda.is_available():
    dev = "cuda:0"
else:
    dev = "cpu"
device = torch.device(dev)

dataset_class_list = [miniImageNet_few_shot, EuroSAT_few_shot,
                      CropDisease_few_shot, Chest_few_shot, ISIC_few_shot]
dataset_names_list = ['miniImageNet', 'EuroSAT',
                      'CropDisease', 'ChestX', 'ISIC']

# dataset_class_list = [miniImageNet_few_shot,
#                       EuroSAT_few_shot, CropDisease_few_shot]
# dataset_names_list = ['miniImageNet', 'EuroSAT',  'CropDisease']

dataloader_list = []
for i, dataset_class in enumerate(dataset_class_list):
    transform = dataset_class.TransformLoader(
        224).get_composed_transform(aug=True)
    transform_test = dataset_class.TransformLoader(
        224).get_composed_transform(aug=False)
    split = 'datasets/split_seed_1/{0}_unlabeled_20.csv'.format(
        dataset_names_list[i])
    if dataset_names_list[i] == 'miniImageNet':
        split = None
    dataset = dataset_class.SimpleDataset(
        transform, split=split)
    loader = torch.utils.data.DataLoader(dataset, batch_size=256,
                                         num_workers=0,
                                         shuffle=True, drop_last=True)
    dataloader_list.append(loader)

vanilla_models = []
baseline_model = models.ResNet10()
load_checkpoint(
    baseline_model, 'logs/AdaBN/teacher_miniImageNet/399.tar', device)
baseline_model.eval()
vanilla_models.append(baseline_model)

for name in dataset_names_list[1:]:
    vanilla_models.append(models.ResNet10())
    load_checkpoint2(vanilla_models[-1],
                     'logs/vanilla/{0}/checkpoint_best.pkl'.format(name), device)
    vanilla_models[-1].eval()

label_dataset = []
base_features = []
bms_feature = []
with torch.no_grad():
    for i, loader in enumerate(dataloader_list):
        # loader_iter = iter(loader)
        # x, _ = loader_iter.next()
        for x, _ in loader:
            base_features += baseline_model(x)
            bms_feature += vanilla_models[i](x)
            label_dataset += [dataset_names_list[i]]*len(x)
            # break

    base_features = torch.stack(base_features)
    bms_feature = torch.stack(bms_feature)

    torch.save(base_features, './lab/tsne/t-sne_base_features.pt')
    torch.save(bms_feature, './lab/tsne/t-sne_bms_feature.pt')
    with open('./lab/tsne/t-sne_label_dataset.txt', 'w') as filehandle:
        for listitem in label_dataset:
            filehandle.write('%s\n' % listitem)