import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision
import torchvision.transforms as transforms
import pickle
from torch.utils import data as data2
import torch.optim as optim
import numpy as np
from torch.autograd import Variable
import os, sys
import copy
import math
import time

from distill_network import *
import PIL.Image as Image
import utils
import logging
import logging.config

init_opt = 'similar_init'
# config logging in python code
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

# create a file handler
handler = logging.FileHandler('distill_main_{}.log'.format(init_opt))# set RotatingileHandler and use handler.doRollover() to create a new logging file each time)
handler.setLevel(logging.INFO)

# create a logging format
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)

# add the file handler to the logger
logger.addHandler(handler)

# time.clock() => time.process_time(). Deprecated in Python 3.3 and will be removed from 3.8
try:
    clock = time.process_time
except AttributeError:
    clock = time.clock
# later shall use `clock()`

# define distill loss
class DistillLoss(nn.Module):

    def __init__(self):
        super(DistillLoss,self).__init__()

    def forward(self, input, target):

        # TODO implement the distill loss here

        y_shape = y.size()[1]
        x_added_dim = x.unsqueeze(1)
        x_stacked_along_dimension1 = x_added_dim.repeat(1,NUM_WORDS,1)
        diff = torch.sum((y - x_stacked_along_dimension1)**2,2)
        totloss = torch.sum(torch.sum(torch.sum(diff)))
        return totloss

# parameters for slide-learn
logger.info("set params")

batch_size=23
num_epochs = 200
learning_rate = 0.001
log_interval = 10
save_interval  = 200
seq_len = 75

distill = True

# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# def load_dataloaders():

#     # load data
#     batch_size=23
#     [train_ids, train_labels, test_ids, test_labels] = pickle.load(open('slide_6_10.pkl', 'rb'))
#     # print("loading data")
#     # print(len(train_ids), len(train_labels), len(test_ids)) # 1173 1173 253
#     training_dataset = Dataset(train_ids, train_labels)
#     X_train = training_dataset.get_X()
#     Y_train = training_dataset.get_y()
#     # print("example Dataset")
#     # print(training_dataset[0][0].size()) # tuple e.g. (torch.Size([1, 6, 10, 75]), 0)
#     # from iCaRL print(X_train.shape, Y_train.shape, X_test.shape) #(50000, 3, 32, 32) (50000,) (10000, 3, 32, 32)
#     test_dataset = Dataset(test_ids, test_labels)
#     X_test = test_dataset.get_X()
#     Y_test = test_dataset.get_y()
#     # print(X_train.shape)
#     # print(len(X_train))
#     # print(Y_train[0])
#     # print(Y_train.shape)
#     # print(X_train.shape, Y_train.shape, X_test.shape)
#     train_loader = data2.DataLoader(training_dataset, batch_size=batch_size)
#     test_loader = data2.DataLoader(test_dataset, batch_size=batch_size)

#     return train_loader, test_loader

# def slide_learn(iteration):

#     dirName = 'slide_info_' + str(iteration)
#     try:
#         # Create target Directory
#         os.mkdir(dirName)
#     except FileExistsError:
#         print('Re-writing the touch')
#     dirName = dirName + '/'

#     train_loader, test_loader = load_dataloaders()

#     model = CNN_LSTM(seq_len).to(device)

#     # optimizer
#     optimizer = optim.Adam(model.parameters(), lr=learning_rate)

#     '''
#     def get_total_loss(networks):
#     frcnn_xe_loss = tf.reduce_mean([net.compute_frcnn_crossentropy_loss() for net in networks])
#     tf.summary.scalar('loss/frcnn/class', frcnn_xe_loss)
#     frcnn_bbox_loss = tf.reduce_mean([net.compute_frcnn_bbox_loss() for net in networks])
#     tf.summary.scalar('loss/frcnn/bbox', frcnn_bbox_loss)
#     l2_loss = tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
#     tf.summary.scalar('loss/weight_decay', l2_loss)
#     total_loss = frcnn_xe_loss + frcnn_bbox_loss + l2_loss

#     if args.distillation:
#         distillation_xe_loss = tf.reduce_mean([net.compute_distillation_crossentropy_loss() for net in networks])
#         tf.summary.scalar('loss/distillation/class', distillation_xe_loss)
#         distillation_bbox_loss = tf.reduce_mean([net.compute_distillation_bbox_loss() for net in networks])
#         tf.summary.scalar('loss/distillation/bbox', distillation_bbox_loss)
#         total_loss += distillation_xe_loss + distillation_bbox_loss
#     tf.summary.scalar('loss/total', total_loss)
#     return total_loss
#     '''

#     if distill:
#         criterion = nn.CrossEntropyLoss() + DistillLoss()

#     else:
#         criterion = nn.CrossEntropyLoss()

#     # set initial hidden state
#     (h_ini, c_ini) = (torch.zeros(2, batch_size, 50).to(device) , torch.zeros(2, batch_size, 50).to(device))

#     epoch_lists = []
#     model.train()
#     for epoch in range(1, num_epochs + 1):
#         for batch_idx, (data, target) in enumerate(train_loader):

#             data = np.expand_dims(data, axis=1)
#             data = torch.FloatTensor(data)
#             #print(target.size())
#             data, target = data.to(device), target.to(device)

#             data, target = Variable(data), Variable(target)
#             #print('Data size:', data.size())

#             optimizer.zero_grad()

#             # init hidden states
#             model.init_hidden(h_ini, c_ini)

#             output = model(data)
#             #print(target.size(), output.size())

#             loss = F.nll_loss(output, target)
#             loss.backward()
#             optimizer.step()
#             #if batch_idx % log_interval == 0:
#         epoch_lists.append(loss.item())
#         if epoch % save_interval == 0:
#             torch.save(model.state_dict(), dirName + 'model_epoch_' + str(epoch) +'.ckpt')

#     # save epochs
#     pickle.dump(epoch_lists, open(dirName + 'loss.pkl', 'wb'))

#     ## check for accuracy

#     results = []
#     model.eval()
#     test_loss = 0
#     correct = 0
#     for data, target in train_loader:

#         data = np.expand_dims(data, axis=1)
#         data = torch.FloatTensor(data)
#         data, target = data.to(device), target.to(device)
#         data, target = Variable(data, volatile=True), Variable(target)
#         output = model(data)
#         test_loss += criterion(
#             output, target).item()  # sum up batch loss
#         pred = output.data.max(
#             1, keepdim=True)[1]  # get the index of the max log-probability
#         correct += pred.eq(target.data.view_as(pred)).long().cpu().sum()

#     test_loss /= len(train_loader.dataset)
#     results.append( 100.0 * correct.item() / len(train_loader.dataset) )

#     test_loss = 0
#     correct = 0
#     for data, target in test_loader:

#         data = np.expand_dims(data, axis=1)
#         data = torch.FloatTensor(data)
#         data, target = data.to(device), target.to(device)
#         data, target = Variable(data, volatile=True), Variable(target)
#         output = model(data)
#         test_loss += criterion(
#             output, target).item()  # sum up batch loss
#         pred = output.data.max(
#             1, keepdim=True)[1]  # get the index of the max log-probability
#         correct += pred.eq(target.data.view_as(pred)).long().cpu().sum()

#     test_loss /= len(test_loader.dataset)

#     results.append( 100.0 * correct / len(test_loader.dataset) )

#     pickle.dump(results, open(dirName + 'results.pkl', 'wb'))

# slide_learn(400)

###########
# Distillation network
###########

def load_datasets():

    # load data
    [train_ids, train_labels, test_ids, test_labels] = pickle.load(open('slide_6_10_{}_classwise.pkl'.format(init_opt), 'rb'))
    # print("loading data")
    # print(len(train_ids), len(train_labels), len(test_ids)) # 1173 1173 253
    # interpret the data class by class
    logger.info("train_ids {} {}".format(len(train_ids), train_ids))
    X_train = []
    Y_train = []
    X_test = []
    Y_test = []
    for i in range(len(train_ids)):
        print("prepare data for class %d" %(i))
        training_dataset = Dataset(train_ids[i], train_labels[i])
        X_train.append(training_dataset.get_X())
        Y_train.append(training_dataset.get_y())
        # print("example Dataset")
        # print(training_dataset[0][0].size()) # tuple e.g. (torch.Size([1, 6, 10, 75]), 0)
        # from iCaRL print(X_train.shape, Y_train.shape, X_test.shape) #(50000, 3, 32, 32) (50000,) (10000, 3, 32, 32)
        test_dataset = Dataset(test_ids[i], test_labels[i])
        X_test.append(test_dataset.get_X())
        Y_test.append(test_dataset.get_y())
        # print(X_train.shape)
        # print(len(X_train))
        # print(Y_train[0])
        # print(Y_train.sh?ape)
    X_train = np.array(X_train)
    X_test = np.array(X_test)
    Y_train = np.array(Y_train)
    Y_test = np.array(Y_test)
    # print(X_train.shape, Y_train.shape, X_test.shape, Y_test.shape) # (23, 50, 6, 10, 75) (23, 50) (23, 12, 6, 10, 75) (23, 12)
    logger.info("load datasets {} {} {} {}".format(X_train.shape, Y_train.shape, X_test.shape, Y_test.shape))
    X_val = 0
    Y_val = 0
    return X_train, Y_train, X_val, Y_val, X_test, Y_test

# parameters for distill-learning

iteration = 400
iteration_finetune = 300

lr = 0.001
schedules = range(50, iteration, 50)
gamma = 0.5
momentum = 0.9
decay = 0.0001
batchsize=23
num_class = 23
init_num_class = 3 # so that the rest 20 classes can be evenly distributed over periods
cur_num_class = init_num_class
new_num_class = 5
# np.random.seed(100)
# class_order = np.random.permutation(num_class)
class_order = np.arange(num_class)
logger.info("class order {}".format(class_order))

memory_K = 300 # (50 per class * 6 classes) #2000 # TODO: tune later
T = 2
dist_ratio = 0.5
stop_acc = 0.998

period_train = (num_class-init_num_class)//new_num_class
memory_size = memory_K//new_num_class # how many samples to load each time
class_old = np.array([], dtype=int)
memory_images = np.zeros(shape=(0, memory_size, 1, 6, 10, 75), dtype = np.uint8)
memory_labels = np.zeros(shape=(0, memory_size), dtype=int)
acc_nvld_basic = np.zeros((period_train))
acc_nvld_finetune = np.zeros((period_train))
crossentropy = nn.CrossEntropyLoss()

# get feature dim
# skip since we are not using pre-trained resnet
input_size = (1, 6, 10, 75)
# newshape=(-1,)+input_size "(-1, 1, 6, 10)"

# load the dataset
images_train, labels_train, images_val, labels_val, images_test, labels_test = load_datasets()
num_train_per_class = images_train.shape[1]
# logger.info("num_train_per_class %d " % num_train_per_class)
# get model state
model_path = 'model/model_slide_e2e_{}_{}.pth'.format(init_opt,cur_num_class+new_num_class)
flag_model = os.path.exists(model_path)

# gradually increase cur_num_class
for period in range(period_train):
    logger.info('-----------------------------------')
    logger.info('period = %d' % (period))
    if period == 0:
        class_novel = class_order[:(cur_num_class+new_num_class)]
    else:
        class_novel = class_order[cur_num_class:(cur_num_class+new_num_class)]
    logger.info("new classes: {}".format(class_novel))
    # organize the dataset for this period
    # TODO: instead of select sequentially, use representative classes for the init training
    images_new_train = np.reshape(images_train[class_novel], newshape=(-1, 1, 6, 10, 75)) # one more dim on time compared to orginal img
    labels_new_train = np.reshape(labels_train[class_novel], newshape=(-1))
    images_new_test = np.reshape(images_test[class_novel], newshape=(-1, 1, 6, 10, 75))
    labels_new_test = np.reshape(labels_test[class_novel], newshape=(-1))

    num_class_old = class_old.shape[0]

    if period == 0:
        # use the init data to train
        images_combined_train = images_new_train
        labels_combined_train = labels_new_train
        # create the model
        net = Network(num_class=(cur_num_class+new_num_class), num_new_class=0)
        net.to(device)

    else:
        # combine the representatives in memory and newly added data to train
        images_combined_train = np.concatenate((images_new_train, np.reshape(memory_images, newshape=(-1, 1, 6, 10, 75))), axis=0)
        labels_combined_train = np.concatenate((labels_new_train, np.reshape(memory_labels, newshape=(-1))), axis=0)
        # create a new model and make use of previously trained model by state_dict
        net = Network(num_class=num_class_old, num_new_class=new_num_class)
        # logger.info("num_class_old %d, new_num_class %d" %(num_class_old, new_num_class))
        if period == 1:
            last_net = Network(num_class=num_class_old, num_new_class=0)
        else:
            last_net = Network(num_class=num_class_old-new_num_class, num_new_class=new_num_class)
        last_net.load_state_dict(torch.load(model_path))
        last_net.eval()
        pretrained_dict = last_net.state_dict()
        net_dict = net.state_dict()

        # print("last_net dict")
        # print([key for key, value in pretrained_dict.items()])
        # print("net_dict")
        # print([key for key, value in net_dict.items()])

        # filter the pretrained dict, drop the last linear weight for net (and subnet if it exists)
        inherit_layers = ['cnn', 'lstm'] # drop the last linear layer
        pretrained_dict = {k:v for k, v in pretrained_dict.items() if any(layer in k for layer in inherit_layers)}
        # print("filtered pretrained_dict")
        # print([key for key, value in pretrained_dict.items()])

        # 1. filter out unnecessary keys
        pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in net_dict}
        # 2. overwrite entries in the existing state dict
        net_dict.update(pretrained_dict)
        # 3. load the new state dict
        net.load_state_dict(net_dict)
        net.to(device)

    # reorganize the datasets
    images_nvld_test = images_test[np.concatenate((class_old, class_novel), axis=0)]
    images_nvld_test = np.reshape(images_nvld_test, newshape= (-1, 1, 6, 10, 75))
    labels_nvld_test = labels_test[np.concatenate((class_old, class_novel), axis=0)]
    labels_nvld_test = np.reshape(labels_nvld_test, newshape=(-1))

    # skip data augmentation part

    logger.info("training size = %d"%(labels_combined_train.shape[0]))

    # training
    lrc = lr
    logger.info("current lr = %f" % (lrc))
    acc_training = []
    softmax = nn.Softmax(dim=1).to(device)

    if net.subnet:
        net_old = net.subnet
    else:
        net_old = copy.deepcopy(net)

    # training over iterations/ bactches
    for iter in range(iteration):

        # learning rate
        if iter in schedules:
            lrc *= gamma
            # print("type iter", type(iter)) # int
            logger.info("iteration {}, current lr= {}".format(iter, lrc))

        # optimizer
        #optimizer = optim.Adam(net.parameters(), lr=lrc, momentum=momentum, \
        #   weight_decay=decay, nesterov=True)
        optimizer = optim.Adam(net.parameters(), lr=lrc)

        # train
        # print("begin train")
        # print(labels_combined_train.shape[0])
        idx_train = np.random.permutation(labels_combined_train.shape[0])
        # print("idx_train")
        # print(idx_train)
        # raise ValueError("stop here")
        loss_avg = 0
        loss_cls_avg = 0
        loss_dist_avg = 0
        acc_avg = 0
        num_exp = 0
        tstart = time.process_time()
        batchnum_train = math.floor(labels_combined_train.shape[0]/batchsize)
        correct = 0
        results = []
        # load model
        if period == 0 and flag_model:
            logger.info("load model %s for period 0" % model_path)
            net.load_state_dict(torch.load(model_path))
            net = net.to(device)
            # not the case for CNN-LSTM unless u load the model before
            # TODO use the state_dict to load partial model for either first model or subnet
        # print("batchnum_train", batchnum_train)
        # print("idx_train", idx_train)
        net.train()
        for batch in range(batchnum_train):
            if period == 0 and flag_model: # no need training
                num_exp = 1
                break

            if batch == batchnum_train - 1:
                idx = idx_train[batch * batchsize:] # last batch, take all the rest
            else:
                idx = idx_train[batch * batchsize : (batch+1)*batchsize]
            bs = len(idx) # same as batchsize unless it is the last batch
            img = images_combined_train[idx]
            lab = labels_combined_train[idx]

            # TODO: change onehot encoding to current number of classes only
            # lab_onehot = utils.one_hot(lab, num_class)
            lab_onehot = utils.one_hot(lab, (cur_num_class+new_num_class))

            # transform
            # TODO double check the data format
            # see Network: batch_size, timesteps, C, H, W, sequence_size = x.size() requires 6D
            img = np.expand_dims(img, axis=1)
            img = torch.FloatTensor(img).to(device)
            # print("img input example", img.shape) # torch.Size([23, 1, 1, 6, 10, 75])
            # print(img)
            lab_onehot = torch.from_numpy(lab_onehot).float().to(device)

            optimizer.zero_grad()

            # init hidden space for lstm
            (h_ini, c_ini) = (torch.zeros(2, bs, 50).to(device) , torch.zeros(2, bs, 50).to(device))
            net.init_hidden(h_ini, c_ini)

            output = net(img)
            # print("output")
            # print(output)
            # try to use nll loss, same as slide_learn
            lab = torch.from_numpy(lab).to(device)
            # print("lab")
            # print(lab)

            # raise ValueError("stop here to check")

#            indices = torch.LongTensor(np.concatenate((class_old, class_novel), axis=0))
#            indices = indices.to(device)
#            # if period > 0:
#            #     # print the variables for debugging
#            #     print("lab")
#            #     print(lab)
#            #     print("lab_onehot", lab_onehot.size())
#            #     print(lab_onehot)
#            #     print("output", output.size())
#            #     print(output)
#            #
#            #     print("indicesi", indices.size())
#            #     print(indices)
#            prob_cls = softmax(torch.index_select(output, 1, indices))
#            lab_onehot = torch.index_select(lab_onehot, 1, indices)
#            # if period >0:
#            #     print("prob_cls", prob_cls.size(), "lab_onehot", lab_onehot.size())
#            #     print("combined old & new indices")
#            #     print(indices)

            loss_cls = F.nll_loss(output, lab)
            # loss_cls = F.binary_cross_entropy(prob_cls, lab_onehot) # TODO: binary? # classification loss
            # print("batch", batch, "classification loss ", loss_cls)
            # distllation loss for only old class data !
            if period > 0:

                indices = torch.LongTensor(class_old).to(device)
                dist = torch.index_select(output, 1, indices)
                dist = softmax(dist/T)
                # same initialization procedure as net
                (h_ini, c_ini) = (torch.zeros(2, bs, 50).to(device), torch.zeros(2, bs, 50).to(device))
                net_old.init_hidden(h_ini, c_ini)
                output_old = net_old(img)
                ouput_old = torch.index_select(output_old, 1, indices)
                lab_dist = softmax(Variable(output_old, requires_grad=False))

                # # TODO: RH: what is the difference, one is class_old, one is enumerate(lab) if la in class_old
                # if not flag_dist_all: # TODO haven't set the flag before
                #     # only for old class data
                #     indices = [id for id, la in enumerate(lab) if la in class_old]
                #     indices = torch.LongTensor(indices)
                #     indices = indices.cuda()
                #     dist = torch.index_select(dist, 0, indices)
                #     lab_dist = torch.index_select(lab_dist, 0, indices)

                # TODO：should only calculate distill loss for those of labels of old classes

                # print("dist example", dist.size())
                # print(dist)
                # print("lab_dist example", lab_dist.size())
                # print(lab_dist)
                loss_dist = F.binary_cross_entropy(dist, lab_dist)  # distill loss
            else:
                loss_dist = 0
            # print("batch", batch, "distillation loss", loss_dist)
            loss = loss_cls + dist_ratio * loss_dist

            loss_avg += loss.item()
            loss_cls_avg += loss_cls.item()
            if period == 0:
                loss_dist_avg += 0
            else:
                loss_dist_avg += loss_dist.item()

            # acc = np.sum(np.equal(np.argmax(prob_cls.cpu().data.numpy(), axis=-1), np.argmax(lab_onehot.cpu().data.numpy(), axis=-1)))
            # acc_avg += acc
            # num_exp += np.shape(lab)[0]

            pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability
            acc = pred.eq(lab.data.view_as(pred)).long().cpu().sum()
            acc_avg += acc
            num_exp += bs
            # In PyTorch, we need to set the gradients to zero before starting to do backpropragation
            # because PyTorch accumulates the gradients on subsequent backward passes.

            loss.backward()
            optimizer.step()

            # # add random noise to gradients / weights, not implemented now
            # if gredient_noise_ratio > 0:
            #     for p in net.parameters():
            #         p.data.sub_(gredient_noise_ratio * lrc * torch.from_numpy(
            #             (np.random.random(np.shape(p.data.cpu().data.numpy())) - 0.5)*2).float().cuda())

        loss_avg /= num_exp
        loss_cls_avg /= num_exp
        loss_dist_avg /= num_exp
        acc_avg /= num_exp
        acc_training.append(acc_avg)
        tend = time.process_time()
        tcost = tend - tstart

        logger.info('Training Period: %d \t Iter: %d \t time = %.1f \t loss = %.6f \t acc = %.4f' % (period, iter, tcost, loss_avg, acc_avg))

        # test all (novel + old) classes based on logists
        if period > -1:
            idx_test = np.random.permutation(labels_nvld_test.shape[0])
            loss_avg = 0
            acc_avg = 0
            num_exp = 0
            tstart = time.process_time()

            batchnum_test = math.ceil(labels_nvld_test.shape[0] / batchsize)
            for bi in range(batchnum_test):
                if bi == batchnum_test - 1:
                    idx = idx_test[bi * batchsize:]
                else:
                    idx = idx_test[bi * batchsize:(bi + 1) * batchsize]
                bs = len(idx)
                img = images_nvld_test[idx]
                lab = labels_nvld_test[idx]
                lab_onehot = utils.one_hot(lab, num_class)

                # normalization
                img = np.expand_dims(img, axis=1)
                img = torch.FloatTensor(img).to(device)

                # init hidden space for lstm
                (h_ini, c_ini) = (torch.zeros(2, bs, 50).to(device) , torch.zeros(2, bs, 50).to(device))
                net.init_hidden(h_ini, c_ini)

                output = net(img)
                indices = torch.LongTensor(np.concatenate((class_old, class_novel), axis=0))
                indices = indices.to(device)
                output = torch.index_select(output, 1, indices)
                output = softmax(output)
                output = output.cpu().data.numpy()
                lab_onehot = lab_onehot[:, np.concatenate((class_old, class_novel), axis=0)]

                acc = np.sum(np.equal(np.argmax(output, axis=-1), np.argmax(lab_onehot, axis=-1)))
                acc_avg += acc
                num_exp += np.shape(lab)[0]

            acc_avg /= num_exp
            tend = time.process_time()
            tcost = tend - tstart
            logger.info('Testing novel+old Period: %d \t Iter: %d \t time = %.1f \t\t\t\t\t\t acc = %.4f' % (period, iter, tcost, acc_avg))
            acc_nvld_basic[period] = acc_avg

        # raise ValueError("stop here to check")

        if len(acc_training)>90 and acc_training[-1]>stop_acc and acc_training[-5]>stop_acc:
            logger.info('training loss converged')
            break

    # raise ValueError("stop here to check")

    # save model for each period, update model_path
    #if period == 0 and (not flag_model):
    model_path = 'model/model_slide_e2e_{}_{}.pth'.format(init_opt,cur_num_class+new_num_class)
    logger.info('save model: %s' % model_path)
    torch.save(net.state_dict(), model_path)

    cur_num_class += new_num_class

   # balanced finetune

    net_old = copy.deepcopy(net)

    # finetune
    if period>0:
        # lrc = lr
        lrc = lr*0.1 # small learning rate for finetune
        print('current lr = %f' % (lrc))
        softmax = nn.Softmax(dim=-1).cuda()

        acc_finetune_train = []
        for iter in range(iteration_finetune):

            # learning rate
            if iter in schedules:
                lrc *= gamma
                print('current lr = %f'%(lrc))

            # criterion = nn.CrossEntropyLoss()
            optimizer = torch.optim.SGD(net.parameters(), lr=lrc, momentum=momentum,
                                        weight_decay=decay, nesterov=True)

            # finetune train
            # RH: different from the training mainly at the random permutation step?
            idx_finetune_novel = np.random.permutation(labels_novel_train.shape[0])
            idx_finetune_novel = idx_finetune_novel[:memory_size*num_class_novel]
            idx_finetune_old = np.arange(start=labels_novel_train.shape[0], stop=labels_combined_train.shape[0])
            idx_finetune = np.concatenate((idx_finetune_novel, idx_finetune_old), axis=0)
            np.random.shuffle(idx_finetune)

            loss_avg = 0
            acc_avg = 0
            num_exp = 0
            tstart = time.clock()

            batchnum_train = math.ceil(idx_finetune.shape[0] // batchsize)
            for bi in range(batchnum_train):
                if bi == batchnum_train - 1:
                    idx = idx_finetune[bi * batchsize:]
                else:
                    idx = idx_finetune[bi * batchsize:(bi + 1) * batchsize]
                img = images_combined_train[idx]
                lab = labels_combined_train[idx]
                lab_onehot = utils.one_hot(lab, num_class)

                # transform
                if flag_augmentation_e2e == False:
                    img = utils.img_transform(img, 'train')
                img = torch.from_numpy(img).float()
                img = img.cuda()
                lab_onehot = torch.from_numpy(lab_onehot)
                lab_onehot = lab_onehot.float()
                lab_onehot = lab_onehot.cuda()

                # print("Outside: input size", img.size(), "output_size", lab.size())
                output = net.forward(img)

                # classification loss
                indices = torch.LongTensor(np.concatenate((class_old, class_novel), axis=0))
                indices = indices.cuda()
                prob_cls = torch.index_select(output, 1, indices)
                prob_cls = softmax(prob_cls)
                lab_onehot = torch.index_select(lab_onehot, 1, indices)
                loss_cls = F.binary_cross_entropy(prob_cls, lab_onehot)

                # distillation loss for all classes (maybe the author only distillates for novel classes)
                if period>0:
                    indices = torch.LongTensor(np.concatenate((class_old, class_novel), axis=0))
                    indices = indices.cuda()
                    dist = torch.index_select(output, 1, indices)
                    dist = softmax(dist/T)

                    output_old = net_old.forward(img)
                    output_old = torch.index_select(output_old, 1, indices)
                    lab_dist = Variable(output_old, requires_grad=False)
                    lab_dist = softmax(lab_dist/T)

                    loss_dist = F.binary_cross_entropy(dist, lab_dist)
                else:
                    loss_dist = 0

                loss = loss_cls + dist_ratio*loss_dist

                loss_avg += loss.item()
                acc = np.sum(np.equal(np.argmax(prob_cls.cpu().data.numpy(), axis=-1), np.argmax(lab_onehot.cpu().data.numpy(), axis=-1)))
                acc_avg += acc
                num_exp += np.shape(lab)[0]

                optimizer.zero_grad()
                loss.backward()
                optimizer.step()

                # add random noise to gradients / weights
                if gredient_noise_ratio > 0:
                    for p in net.parameters():
                        p.data.sub_(gredient_noise_ratio * lrc * torch.from_numpy(
                            (np.random.random(np.shape(p.data.cpu().data.numpy())) - 0.5) * 2).float().cuda())

            loss_avg /= num_exp
            acc_avg /= num_exp
            acc_finetune_train.append(acc_avg)
            tend = time.clock()
            tcost = tend - tstart
            print('Finetune Training Iter: %d \t time = %.1f \t loss = %.6f \t acc = %.4f'%(iter, tcost, loss_avg, acc_avg))

            # test all (novel + old) classes based on logists
            if period > 0:
                # images_nvld_test = images_test[np.concatenate((class_old, class_novel), axis=0)]
                # images_nvld_test = np.reshape(images_nvld_test, newshape=(-1, 3, 32, 32))
                # labels_nvld_test = labels_test[np.concatenate((class_old, class_novel), axis=0)]
                # labels_nvld_test = np.reshape(labels_nvld_test, newshape=(-1))
                idx_test = np.random.permutation(labels_nvld_test.shape[0])
                loss_avg = 0
                acc_avg = 0
                num_exp = 0
                tstart = time.clock()

                batchnum_test = math.ceil(labels_nvld_test.shape[0] // batchsize)
                for bi in range(batchnum_test):
                    if bi == batchnum_test - 1:
                        idx = idx_test[bi * batchsize:]
                    else:
                        idx = idx_test[bi * batchsize:(bi + 1) * batchsize]
                    img = images_nvld_test[idx]
                    lab = labels_nvld_test[idx]
                    lab_onehot = utils.one_hot(lab, num_class)

                    if flag_augmentation_e2e == False:  # old transform
                        img = utils.img_transform(img, 'test')
                    img = torch.from_numpy(img).float()
                    img = img.cuda()

                    # # normalization
                    output = net.forward(img)
                    indices = torch.LongTensor(np.concatenate((class_old, class_novel), axis=0))
                    indices = indices.cuda()
                    output = torch.index_select(output, 1, indices)
                    output = softmax(output)
                    output = output.cpu().data.numpy()
                    lab_onehot = lab_onehot[:, np.concatenate((class_old, class_novel), axis=0)]

                    acc = np.sum(np.equal(np.argmax(output, axis=-1), np.argmax(lab_onehot, axis=-1)))
                    acc_avg += acc
                    num_exp += np.shape(lab)[0]

                acc_avg /= num_exp
                tend = time.clock()
                tcost = tend - tstart
                print('Finetune Testing novel+old Period: %d \t Iter: %d \t time = %.1f \t\t\t\t\t\t acc = %.4f' % (period, iter, tcost, acc_avg))
                acc_nvld_finetune[period] = acc_avg

            if len(acc_finetune_train) > 20 and acc_finetune_train[-1] > stop_acc and acc_finetune_train[-5] > stop_acc:
                print('training loss converged')
                break

    if period>0:
        print('------------------- result ------------------------')
        print('Period: %d, basic acc = %.4f, finetune acc = %.4f' % (period, acc_nvld_basic[period], acc_nvld_finetune[period]))
        print('---------------------------------------------------')

    if period == period_train-1:
        print('------------------- ave result ------------------------')
        print('basic acc = %.4f, finetune acc = %.4f' % (np.mean(acc_nvld_basic[1:], keepdims=False), np.mean(acc_nvld_finetune[1:], keepdims=False)))
        print('---------------------------------------------------')

    # new after fine-tune training

    # memory management [random select]
    # reduce old memory
    memory_size = memory_K // (num_class_novel + num_class_old)
    if period >0:
        memory_images = memory_images[:, :memory_size]
        memory_labels = memory_labels[:, :memory_size]
    # add new memory
    memory_new_images = np.zeros((num_class_novel, memory_size, 3, 32, 32), dtype=np.uint8)
    memory_new_labels = np.zeros((num_class_novel, memory_size), dtype=np.int32)
    # random selection
    for k in range(num_class_novel):

        img = images_train[class_novel[k], np.random.permutation(500)[:memory_size]]
        memory_new_images[k] = img
        memory_new_labels[k] = np.tile(class_novel[k], (memory_size))

    # herding
    # to do

    if period > 0:
        memory_images = np.concatenate((memory_images, memory_new_images), axis=0)
        memory_labels = np.concatenate((memory_labels, memory_new_labels), axis=0)
    else:
        memory_images = memory_new_images
        memory_labels = memory_new_labels

    class_old = np.append(class_old, class_novel, axis=0)
    # TODO update the memory
    logger.info("update the memory")

# memory management [random select]
    # reduce old memory
    memory_size = min(num_train_per_class, memory_K // (new_num_class + num_class_old))
    if period >0:
        memory_images = memory_images[:, :memory_size]
        memory_labels = memory_labels[:, :memory_size]
    # add new memory
    memory_new_images = np.zeros((new_num_class, memory_size, 1, 6, 10, 75), dtype=np.uint8)
    memory_new_labels = np.zeros((new_num_class, memory_size), dtype=np.int32)
    # random selection
    for k in range(new_num_class):

        img = images_train[class_novel[k], np.random.permutation(num_train_per_class)[:memory_size]]
        img = np.expand_dims(img, axis=1) # broadcast array from shape (51, 6, 10, 75) to shape (51, 1, 6, 10, 75)
        memory_new_images[k] = img
        memory_new_labels[k] = np.tile(class_novel[k], (memory_size))

    # herding
    # to do

    if period > 0:
        memory_images = np.concatenate((memory_images, memory_new_images), axis=0)
        memory_labels = np.concatenate((memory_labels, memory_new_labels), axis=0)
    else:
        memory_images = memory_new_images
        memory_labels = memory_new_labels

    class_old = np.append(class_old, class_novel, axis=0)


#!/usr/bin/env python3
# MIT License
#
# Copyright (c) 2020 FABRIC Testbed
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
#
# Author: Komal Thareja (kthare10@renci.org)
from datetime import datetime
from typing import List

from fabric.actor.core.common.constants import Constants

"""
Implements various classes for reading and loading the information from config file
"""

class GlobalConfig:
    """
    Represent Global configuration
    """
    def __init__(self, *, config: dict):
        self.runtime = {}
        if Constants.config_section_runtime in config:
            for prop in config[Constants.config_section_runtime]:
                for key, value in prop.items():
                    self.runtime[key] = value
        self.logging = {}
        if Constants.config_logging_section in config:
            for prop in config[Constants.config_logging_section]:
                for key, value in prop.items():
                    self.logging[key] = value
        self.oauth = {}
        if Constants.config_section_o_auth in config:
            for prop in config[Constants.config_section_o_auth]:
                for key, value in prop.items():
                    self.oauth[key] = value
        self.database = {}
        if Constants.config_section_database in config:
            for prop in config[Constants.config_section_database]:
                for key, value in prop.items():
                    self.database[key] = value
        self.container = {}
        if Constants.config_section_container in config:
            for prop in config[Constants.config_section_container]:
                for key, value in prop.items():
                    self.container[key] = value
        self.time = {}
        if Constants.config_section_time in config:
            for prop in config[Constants.config_section_time]:
                for key, value in prop.items():
                    self.time[key] = value

        self.neo4j = {}
        if Constants.config_section_neo4j in config:
            self.neo4j = config[Constants.config_section_neo4j]

        self.pdp = {}
        if Constants.config_section_pdp in config:
            self.pdp = config[Constants.config_section_pdp]

    def get_runtime(self) -> dict:
        """
        Return runtime config
        """
        return self.runtime

    def get_logging(self) -> dict:
        """
        Return logging config
        """
        return self.logging

    def get_oauth(self) -> dict:
        """
        Return oauth config
        """
        return self.oauth

    def get_database(self) -> dict:
        """
        Return database config
        """
        return self.database

    def get_container(self) -> dict:
        """
        Return container config
        """
        return self.container

    def get_time(self) -> dict:
        """
        Return time config
        """
        return self.time

    def get_neo4j_config(self) -> dict:
        """
        Return neo4j config
        """
        return self.neo4j

    def get_pdp_config(self) -> dict:
        """
        Return PDP config
        """
        return self.pdp

class HandlerConfig:
    """
    Represents Handler Config
    """
    def __init__(self, *, config: list):
        self.module_name = None
        self.class_name = None
        self.properties = {}
        for prop in config:
            for key, value in prop.items():
                if key == 'module':
                    self.module_name = value
                elif key == 'class_name':
                    self.class_name = value
                elif key == 'properties':
                    for p in value:
                        for k, v in p.items():
                            self.properties[k] = str(v)

    def get_module_name(self) -> str:
        """
        Return Module Name for Handler
        """
        return self.module_name

    def get_class_name(self) -> str:
        """
        Return Class Name for Handler
        """
        return self.class_name

    def get_properties(self) -> dict:
        """
        Return Handler Properties
        """
        return self.properties

class Attribute:
    """
    Represents Attributes
    """
    def __init__(self, *, config: list):
        self.key = None
        self.type = None
        self.value = None
        for prop in config:
            for key, value in prop.items():
                if key == 'key':
                    self.key = value
                elif key == 'type':
                    self.type = value
                elif key == 'value':
                    self.value = value

    def get_type(self) -> str:
        """
        Return Attribute Type
        """
        return self.type

    def get_key(self) -> str:
        """
        Return Attribute Key
        """
        return self.key

    def get_value(self) -> str:
        """
        Return Attribute Value
        """
        return self.value

class PoolConfig:
    """
    Represents Pool Config
    """
    def __init__(self, *, config: list):
        self.description = None
        self.attributes = []
        self.type = None
        self.label = None
        self.units = None
        now = datetime.utcnow()
        self.start = now
        self.end = now.replace(year=now.year + 20)
        self.properties = {}
        self.factory_module = None
        self.factory_class = None
        self.handler = None
        for prop in config:
            for key, value in prop.items():
                if key == 'type':
                    self.type = value
                elif key == 'label':
                    self.label = value
                elif key == 'description':
                    self.description = value
                elif key == 'units':
                    self.units = int(value)
                elif key == 'start':
                    self.start = value
                elif key == 'end':
                    self.end = value
                elif key == 'properties':
                    for p in value:
                        for k, v in p.items():
                            self.properties[k] = str(v)
                elif key == 'factory.class':
                    self.factory_class = value
                elif key == 'factory.module':
                    self.factory_module = value
                elif key == 'handler':
                    self.handler = HandlerConfig(config=value)
                elif key == 'attributes':
                    for a in value:
                        self.attributes.append(Attribute(config=a['attribute']))

    def get_type(self) -> str:
        """
        Return Pool Type
        """
        return self.type

    def get_label(self) -> str:
        """
        Return Pool Label
        """
        return self.label

    def get_description(self) -> str:
        """
        Return Pool Description
        """
        return self.description

    def get_units(self) -> int:
        """
        Return Pool Units
        """
        return self.units

    def get_start(self) -> datetime:
        """
        Return Start lease time
        """
        return self.start

    def get_end(self) -> datetime:
        """
        Return End lease time
        """
        return self.end

    def get_handler(self) -> HandlerConfig:
        """
        Return Handler Config
        """
        return self.handler

    def get_properties(self) -> dict:
        """
        Return Properties
        """
        return self.properties

    def get_factory_class(self) -> str:
        """
        Return Factory class
        """
        return self.factory_class

    def get_factory_module(self) -> str:
        """
        Return Factory Module
        """
        return self.factory_module

    def get_attributes(self) -> List[Attribute]:
        """
        Return Attributes
        """
        return self.attributes

class ControlConfig:
    """
    Represents Control config
    """
    def __init__(self, *, config: list):
        self.type = None
        self.module_name = None
        self.class_name = None
        for prop in config:
            for key, value in prop.items():
                if key == 'type':
                    self.type = value
                elif key == 'module':
                    self.module_name = value
                elif key == 'class':
                    self.class_name = value

    def get_type(self) -> str:
        """
        Return type
        """
        return self.type

    def get_module_name(self) -> str:
        """
        Return Module name
        """
        return self.module_name

    def get_class_name(self) -> str:
        """
        Return Class Name
        """
        return self.class_name

class ResourceConfig:
    """
    Represents resource config
    """
    def __init__(self, *, config: list):
        self.description = None
        self.attributes = []
        self.type = None
        self.label = None
        self.properties = {}
        self.resource_module = None
        self.resource_class = None
        self.handler = None
        for prop in config:
            for key, value in prop.items():
                if key == 'type':
                    self.type = value
                elif key == 'label':
                    self.label = value
                elif key == 'description':
                    self.description = value
                elif key == 'properties':
                    for p in value:
                        for k, v in p.items():
                            self.properties[k] = str(v)
                elif key == 'resource_class':
                    self.resource_class = value
                elif key == 'resource_module':
                    self.factory_module = value
                elif key == 'handler':
                    self.handler = HandlerConfig(config=value)
                elif key == 'control':
                    self.control = ControlConfig(config=value)
                elif key == 'attributes':
                    for a in value:
                        self.attributes.append(Attribute(config=a['attribute']))

    def get_type(self) -> str:
        """
        Return type
        """
        return self.type

    def get_label(self) -> str:
        """
        Return label
        """
        return self.label

    def get_description(self) -> str:
        """
        Return description
        """
        return self.description

    def get_handler(self) -> HandlerConfig:
        """
        Return handler config
        """
        return self.handler

    def get_control(self) -> ControlConfig:
        """
        Return control config
        """
        return self.control

    def get_properties(self) -> dict:
        """
        Return properties
        """
        return self.properties

    def get_resource_class(self) -> str:
        """
        Return resource class
        """
        return self.resource_class

    def get_resource_module(self) -> str:
        """
        Return resource module
        """
        return self.resource_module

    def get_attributes(self) -> List[Attribute]:
        """
        Return resource attributes
        """
        return self.attributes

class PolicyConfig:
    """
    Represents Policy config
    """
    def __init__(self, *, config: list):
        self.module_name = None
        self.class_name = None
        self.properties = {}
        for prop in config:
            for key, value in prop.items():
                if key == 'module':
                    self.module_name = value
                elif key == 'class':
                    self.class_name = value
                elif key == 'properties':
                    for p in value:
                        for k, v in p.items():
                            self.properties[k] = str(v)

    def get_module_name(self) -> str:
        """
        Return module name
        """
        return self.module_name

    def get_class_name(self) -> str:
        """
        Return class name
        """
        return self.class_name

    def get_properties(self) -> dict:
        """
        Return properties
        """
        return self.properties

class ActorConfig:
    """
    Represents Actor Config
    """
    def __init__(self, *, config: list):
        self.policy = None
        self.description = None
        self.pools = []
        self.resources = []
        self.controls = []
        self.type = None
        self.name = None
        self.guid = None
        self.kafka_topic = None
        self.policy = None
        self.substrate_file = None
        for prop in config:
            for key, value in prop.items():
                if key == 'type':
                    self.type = value
                elif key == 'name':
                    self.name = value
                elif key == 'guid':
                    self.guid = value
                elif key == 'description':
                    self.description = value
                elif key == 'kafka-topic':
                    self.kafka_topic = value
                elif key == 'substrate.file':
                    self.substrate_file = value
                elif key == 'pools':
                    for p in value:
                        self.pools.append(PoolConfig(config=p['pool']))
                elif key == 'controls':
                    for c in value:
                        self.controls.append(ControlConfig(config=c['control']))
                elif key == 'policy':
                    self.policy = PolicyConfig(config=value)
                elif key == 'resources':
                    for p in value:
                        self.resources.append(ResourceConfig(config=p['resource']))

    def get_type(self) -> str:
        """
        Return Actor Type
        """
        return self.type

    def get_name(self) -> str:
        """
        Return Actor Name
        """
        return self.name

    def get_guid(self) -> str:
        """
        Return Actor Guid
        """
        return self.guid

    def get_description(self) -> str:
        """
        Return Actor Description
        """
        return self.description

    def get_kafka_topic(self) -> str:
        """
        Return Kafka topic
        """
        return self.kafka_topic

    def get_pools(self) -> List[PoolConfig]:
        """
        Return Pools
        """
        return self.pools

    def get_controls(self) -> List[ControlConfig]:
        """
        Return Controls
        """
        return self.controls

    def get_policy(self) -> PolicyConfig:
        """
        Return Policy
        """
        return self.policy

    def get_resources(self) -> List[ResourceConfig]:
        """
        Return resources
        """
        return self.resources

    def get_substrate_file(self) -> str:
        """
        Return Substrate file
        """
        return self.substrate_file

class Peer:
    """
    Represent Peer Config
    """
    def __init__(self, *, config: list):
        self.name = None
        self.type = None
        self.guid = None
        self.kafka_topic = None
        self.delegation = None
        for prop in config:
            for key, value in prop.items():
                if key == 'name':
                    self.name = value
                elif key == 'type':
                    self.type = value
                elif key == 'guid':
                    self.guid = value
                elif key == 'kafka-topic':
                    self.kafka_topic = value
                elif key == 'delegation':
                    self.delegation = value

    def get_name(self) -> str:
        """
        Return Name
        """
        return self.name

    def get_type(self) -> str:
        """
        Return Type
        """
        return self.type

    def get_guid(self) -> str:
        """
        Return Guid
        """
        return self.guid

    def get_kafka_topic(self) -> str:
        """
        Return Kafka Topic
        """
        return self.kafka_topic

    def get_delegation(self) -> str:
        """
        Return delegation
        """
        return self.delegation

class Configuration:
    """
    Represent Configuration read from file
    """
    def __init__(self, *, config: dict):
        self.global_config = GlobalConfig(config=config)
        self.actor = ActorConfig(config=config['actor'])
        self.peers = []
        if 'peers' in config:
            for e in config['peers']:
                self.peers.append(Peer(config=e['peer']))

    def get_global_config(self) -> GlobalConfig:
        """
        Return Global Config
        """
        return self.global_config

    def get_runtime_config(self) -> dict:
        """
        Return Runtime Config
        """
        if self.global_config is not None:
            return self.global_config.get_runtime()
        return None

    def get_oauth_config(self) -> dict:
        """
        Return OAuth Config
        """
        if self.global_config is not None:
            return self.global_config.get_oauth()
        return None

    def get_actor(self) -> ActorConfig:
        """
        Return Actor Config
        """
        return self.actor

    def get_peers(self) -> List[Peer]:
        """
        Return Peer Config
        """
        return self.peers


from flask_restful import Resource, request
import pandas as pd, json, os

raw_file = 'api/static/data/raw.csv'
final_file = 'api/static/data/final.csv'

class Filter(Resource):
    def get(self):
        if os.path.isfile(final_file):
            data = pd.read_csv(final_file)
        else:
            data = pd.read_csv(raw_file)

        if 'case_id' in data.columns and 'task' in data.columns and 'timestamp' in data.columns:
            # Semua event yang unik
            all_event = list(set(data['task']))

            # List dari semua id
            id_list = list(set(data['case_id']))
            # Mengambil data per id
            data_per_id = {}
            for id in id_list:
                if id not in data_per_id:
                    data_per_id[id] = data[data['case_id'] == id]
            # List start event dan end event
            start_list = []
            end_list = []
            for key, dt in data_per_id.items():
                for x in range(len(dt.index)):
                    if x == 0:
                        start_list.append(dt['task'].iloc[x])
                    elif dt['case_id'].iloc[x-1] != dt['case_id'].iloc[x]:
                        start_list.append(dt['task'].iloc[x])
                    elif x == len(dt.index)-1:
                        end_list.append(dt['task'].iloc[x])
                    elif dt['case_id'].iloc[x+1] != dt['case_id'].iloc[x]:
                        end_list.append(dt['task'].iloc[x])
            start_list = list(set(start_list))
            end_list = list(set(end_list))

            return json.dumps(
                {
                    'data': {
                        'start': start_list,
                        'end': end_list,
                        'all': all_event
                    },
                    'message': 'Succesfully load the data',
                    'status': 'success'
                }
            )

        else:
            return json.dumps(
                {
                    'data':'',
                    'message': 'Data is not finished',
                    'status': 'error'
                }
            )
        pass

    def post(self):
        args = request.get_json(force=True)
        if args != None:
            if os.path.isfile(final_file):
                data = pd.read_csv(final_file)
            else:
                data = pd.read_csv(raw_file)

            try:
                start = args['data']['start']

                # Kolom untuk sort data ulang
                data['Sort'] = list([x for x in range(0,len(data['case_id']))])

                # List dari semua id
                id_list = data['case_id'].unique()

                # Mengambil data per id
                data_per_id = dict()
                for id in id_list:
                    if id not in data_per_id:
                        data_per_id[id] = data[data['case_id'] == id]

                # List of selected start event
                list_of_data = []
                for key, dt in data_per_id.items():
                    for ev in start:
                        if dt['task'].iloc[0] == ev:
                            list_of_data.append(dt)

                data = pd.concat(list_of_data)
                data.sort_values('Sort', inplace=True)
                data.drop('Sort', axis=1, inplace=True)
                data.to_csv(final_file, index=False)

                return json.dumps(
                    {
                        'data': '',
                        'message': 'Success filtering start event',
                        'status': 'success'
                    }
                )
            except KeyError as ke:
                print(ke)

            try:
                end = args['data']['end']

                # Kolom untuk sort data ulang
                data['Sort'] = list([x for x in range(0,len(data['case_id']))])

                # List dari semua id
                id_list = data['case_id'].unique()

                # Mengambil data per id
                data_per_id = dict()
                for id in id_list:
                    if id not in data_per_id:
                        data_per_id[id] = data[data['case_id'] == id]

                # List selected end event
                list_of_data = []
                for key, dt in data_per_id.items():
                    for ev in end:
                        if dt['task'].iloc[-1] == ev:
                            list_of_data.append(dt)

                data = pd.concat(list_of_data)
                data.sort_values('Sort', inplace=True)
                data.drop('Sort', axis=1, inplace=True)
                data.to_csv(final_file, index=False)

                return json.dumps(
                    {
                        'data': '',
                        'message': 'Success filtering end event',
                        'status': 'success'
                    }
                )
            except KeyError as ke:
                print(ke)

            try:
                all = args['data']['all']

                # Kolom untuk sort data ulang
                data['Sort'] = list([x for x in range(0,len(data['case_id']))])

                list_data = []
                for idx, ev in enumerate(data['task']):
                    for evt in all:
                        if ev == evt:
                            list_data.append(data.iloc[idx])

                data = pd.DataFrame(list_data)
                data.sort_values('Sort', inplace=True)
                data.drop('Sort', axis=1, inplace=True)
                data.to_csv(final_file, index=False)

                return json.dumps(
                    {
                        'data': '',
                        'message': 'Success filtering all event',
                        'status': 'success'
                    }
                )
            except KeyError as ke:
                print(ke)
        else:
            return json.dumps(
                {
                    'data': '',
                    'message': 'Failed filtering the data',
                    'status': 'error'
                }
            )


# coding:UTF-8
import time
import argparse
import cv2
from glob import glob

import numpy as np
import matplotlib.pyplot as plt

import chainer
from chainer import optimizers
from chainer import cuda
from chainer import serializers

from net import Model
from load_train import Train_DataLoader
from load_test import Test_DataLoader
from image_read import Image_Read

'''----------------------------'''
SEGMENTATION_PATH_DATA = '../../Annotated-SDD/image/train/GLAY/*.png'
SEGMENTATION_PATH_TEST = '../../Annotated-SDD/image/test/GLAY/*.png'
'''----------------------------'''

def main():
    '''Parameter Settings'''
    parser = argparse.ArgumentParser()
    parser.add_argument('--epoch', type=int, default=100, help='number of epoch')
    parser.add_argument('--batch', type=int, default=10, help='number of batch')
    parser.add_argument('--sequence', type=int, default=10, help='number of sequence')
    parser.add_argument('--relate_coordinate', type=int, default=9, help='number of relate_coordinate')
    parser.add_argument('--learning_rate', type=float, default=0.001, help='number of learning rate')
    parser.add_argument('--i_size', type=int, default=290, help='number of input layer unit')
    parser.add_argument('--h_size', type=int, default=128, help='number of hidden layer unit')
    parser.add_argument('--o_size', type=int, default=2, help='number of output layer unit')
    parser.add_argument('--GPU_ID', type=int, default=0, help='number of GPU')
    args = parser.parse_args()
    train(args)

def train(args):
    # train image path
    image_data = glob(SEGMENTATION_PATH_DATA)
    image_data.sort()
    # test image path
    image_test = glob(SEGMENTATION_PATH_TEST)
    image_test.sort()

    # data load
    data_train = Train_DataLoader(args.sequence)
    data_test = Test_DataLoader(args.sequence)

    # Define model
    model = Model(i_size=args.i_size, h_size=args.h_size, o_size=args.o_size, train=True)
    cuda.get_device_from_id(args.GPU_ID).use()
    model.to_gpu()

    # optimizer
    optimizer = optimizers.RMSprop(lr=args.learning_rate)
    optimizer.setup(model)

    # initial loss
    train_loss = []
    test_loss = []

    # iteration
    n_iter = 0
    n_iter_test = 0

    for epoch in range(1, args.epoch + 1):
        print('epoch', epoch)
        start = time.time()

        '''TRAIN'''
        print("TRAIN")
        # -5 to delete extra data
        for data_iter in range(0, len(data_train.all_data) - 5, args.batch):
            # initial loss
            loss_train = 0

            x = []  # input
            t = []  # target
            image_append = []   # list image pixels

            # list scenes
            tel_train = []

            n_iter += 1
            model.cleargrads()
            model.reset_state()
            for batch_ in range(args.batch):
                list_random = np.random.randint(0, len(data_train.list))
                # image load
                data_image_train = cv2.imread(image_data[list_random], 0)
                data_image_train //= 10
                image_append.append(data_image_train)
                # coordinate match the scene
                trajectory_random = np.random.randint(0, len(data_train.list_coordinate[list_random]))
                trajectory = data_train.list_coordinate[list_random][trajectory_random]
                # Since -1 is necessary for the target data
                index = np.random.randint(0, len(trajectory) - args.sequence - 1)
                x.append(trajectory[index:index + args.sequence, :])
                t.append(trajectory[index + 1:index + 1 + args.sequence, :])
                tel_train.append(list_random)

            x = np.asarray(x, dtype=np.float32)
            t = np.asarray(t, dtype=np.float32)

            x_copy = np.copy(x[:, :10, :])

            # Calculation of movement amount
            x = np.diff(x, axis=1)
            t = np.diff(t, axis=1)

            for sequence in range(args.relate_coordinate):
                image_coordinate_image = []

                xy = x[:, sequence, :]          # input
                training = t[:, sequence, :]    # test

                # Absolute coordinates of target
                xy_center = np.copy(x_copy[:, sequence, :])

                # calculation of target environment
                for k in range(args.batch):
                    image_coordinate = Image_Read(image_append[k], xy_center[k], tel_train[k], train=True)
                    image_coordinate_image.append(image_coordinate.img)
                # target centered (100 x 100 x CHANNEL x BATCH_SIZE)
                image_coordinate_image = np.array(image_coordinate_image, dtype=np.float32)
                image_coordinate_image_ = chainer.Variable(cuda.to_gpu(image_coordinate_image))

                # learning GPU
                xy = chainer.Variable(cuda.to_gpu(xy))
                training = chainer.Variable(cuda.to_gpu(training))

                loss_train += model(image_coordinate_image_, xy, training, train=True)

            print("iteration:", n_iter, "loss:", loss_train)
            loss_train.backward()
            optimizer.update()

        end = time.time()
        f = open('loss.txt', 'a')
        print(("epoch:{}, loss:{}, time:{}".format(epoch, loss_train.data, end - start)), file=f)
        f.close()

        # save epoch
        if epoch % 20 == 0:
            # Move to CPU once to prevent GPU dependence problems during testing
            model.to_cpu()
            serializers.save_npz('./Train/' + str(epoch) + '.model', model)
            serializers.save_npz('./Train/' + str(epoch) + '.state', optimizer)
            model.to_gpu()

        '''TEST'''
        print("TEST")
        # -7 to delete extra data
        for test_iter in range(0, len(data_test.all_data) - 7, args.batch):
            # initial loss
            loss_test = 0

            x = []  # input
            t = []  # target
            image_append_test = []

            # list scenes
            tel_test = []
            n_iter_test += 1
            model.cleargrads()
            model.reset_state()
            for batch_test in range(args.batch):
                list_random_test = np.random.randint(0, len(data_test.list))
                # image load
                data_image_test = cv2.imread(image_test[list_random_test], 0)
                data_image_test //= 10
                image_append_test.append(data_image_test)
                # coordinate match the scene
                trajectory_random = np.random.randint(0, len(data_test.list_coordinate[list_random_test]))
                trajectory_test = data_test.list_coordinate[list_random_test][trajectory_random]
                # Since -1 is necessary for the target data
                index_test = np.random.randint(0, len(trajectory_test) - args.sequence - 1)
                x.append(trajectory_test[index_test:index_test + args.sequence, :])
                t.append(trajectory_test[index_test + 1:index_test + 1 + args.sequence, :])
                tel_test.append(list_random_test)

            x = np.asarray(x, dtype=np.float32)
            t = np.asarray(t, dtype=np.float32)

            x_copy = np.copy(x[:, :10, :])

            # Calculation of movement amount
            x = np.diff(x, axis=1)
            t = np.diff(t, axis=1)

            for sequence in range(args.relate_coordinate):
                image_coordinate_image = []

                xy = x[:, sequence, :]
                training = t[:, sequence, :]

                # Absolute coordinates of target
                xy_center = np.copy(x_copy[:, sequence, :])

                # calculation of target environment
                for k in range(args.batch):
                    image_coordinate = Image_Read(image_append_test[k], xy_center[k], tel_test[k], train=False)
                    image_coordinate_image.append(image_coordinate.img)
                # target centered (100 x 100 x CHANNEL x BATCH_SIZE)
                image_coordinate_image = np.array(image_coordinate_image, dtype=np.float32)
                image_coordinate_image_ = chainer.Variable(cuda.to_gpu(image_coordinate_image))

                # learning GPU
                xy = chainer.Variable(cuda.to_gpu(xy))
                training = chainer.Variable(cuda.to_gpu(training))

                loss_test += model(image_coordinate_image_, xy, training, train=True)

            print("iteration:", n_iter_test, "loss:", loss_test)

        # figure
        train_loss.append(loss_train.data)
        test_loss.append(loss_test.data)
        plt.plot(train_loss, label="train_loss")
        plt.plot(test_loss, label="test_loss")
        plt.yscale('log')
        plt.legend()
        plt.grid(True)
        plt.title("loss")
        plt.xlabel("epoch")
        plt.ylabel("loss")
        plt.savefig("Fig/fig_loss.png")
        plt.clf()

if __name__ == '__main__':
    main()


from django.contrib.auth.mixins import LoginRequiredMixin
from django.shortcuts import render, get_object_or_404, redirect
from django.urls import resolve
from django.views import View
from django.views.decorators.csrf import csrf_exempt

from rest_framework import status
from rest_framework.authentication import SessionAuthentication, BasicAuthentication
from rest_framework.decorators import api_view, authentication_classes, permission_classes
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework.parsers import JSONParser
from django.http import Http404
from rest_framework.views import APIView

from django.http import JsonResponse, HttpResponse
from onlineapp.models import *
from onlineapp.forms import *
from onlineapp.college_serializers import *

class CollegeView(LoginRequiredMixin,View):

    authentication_classes = (SessionAuthentication, BasicAuthentication)
    permission_classes = (IsAuthenticated,)

    login_url = '/login/'

    def get(self,request,*args,**kwargs):

        user_permissions = [permission['name'] for permission in request.user.user_permissions.values('name')]
        print(user_permissions)

        if kwargs:
            college = get_object_or_404(College,**kwargs)
            # students = Student.objects.filter(college_id=kwargs['id']).values('id','name','mocktest1__total').order_by('-mocktest1__total')
            students = college.student_set.order_by('-mocktest1__total')
            return render(request,
                          'onlineapp/studentdetails.html',
                          {'students_list': students,
                                'college': college,
                           'user_permissions':user_permissions,})

        colleges=College.objects.all()
        c = College.objects.values_list('name', 'acronym', 'id')
        return render(request,
                      template_name='onlineapp/allcolleges.html',
                      context={
                          'college':colleges,
                          'colleges_list': c,
                          'user_permissions': user_permissions,
                      })

class AddCollegeView(LoginRequiredMixin,View):
    authentication_classes = (SessionAuthentication, BasicAuthentication)
    permission_classes = (IsAuthenticated,)
    login_url = '/login/'

    def get(self,request,*args,**kwargs):

        user_permissions = [permission['name'] for permission in request.user.user_permissions.values('name')]
        print(user_permissions)

        form=AddCollege()
        header = "Add College"
        if kwargs:
            college=College.objects.get(**kwargs)
            form=AddCollege(instance=college)
            header = "Edit College"
        return render(request,'onlineapp/form_addcollege.html',{'form':form,'header':header,'user_permissions':user_permissions,})

    def post(self,request,**kwargs):
        if resolve(request.path_info).url_name=='delete_college':
            College.objects.get(id=kwargs.get('id')).delete()
            return redirect('get_all_colleges')

        if resolve(request.path_info).url_name=='edit_college':
            college=College.objects.get(pk=kwargs.get('id'))
            form=AddCollege(request.POST,instance=college)
            if form.is_valid():
                form.save()
                return redirect('get_all_colleges')

        form = AddCollege(request.POST)
        if form.is_valid():
            form.save()
            return redirect('get_all_colleges')

class AddStudentView(LoginRequiredMixin,View):
    authentication_classes = (SessionAuthentication, BasicAuthentication)
    permission_classes = (IsAuthenticated,)
    login_url = '/login/'
    def get(self,request,*args,**kwargs):

        user_permissions = [permission['name'] for permission in request.user.user_permissions.values('name')]
        print(user_permissions)

        form=AddStudent()
        form1=AddMarks()
        header="Add Student"
        if kwargs.get('id'):
            student=Student.objects.get(id=kwargs.get('id'))
            marks=MockTest1.objects.get(student=student)
            form=AddStudent(instance=student)
            form1=AddMarks(instance=marks)
            header = "Edit Student"
        return render(request,'onlineapp/form_addstudent.html',{'form':form,'form1':form1,'header':header,'user_permissions':user_permissions,})
        pass

    def post(self,request,**kwargs):
        if resolve(request.path_info).url_name=='delete_student':
            student=Student.objects.get(id=kwargs.get('id'))
            try:
                marks=MockTest1.objects.get(student=student)
                marks.delete()
            except Exception as e:
                pass
            student.delete()
            return redirect('student_details',id=kwargs.get('clg_id'))

        if resolve(request.path_info).url_name=='edit_student':
            student=Student.objects.get(id=kwargs.get('id'))
            form = AddStudent(request.POST,instance=student)
            if form.is_valid():
                try:
                    marks=MockTest1.objects.get(student=student)
                    form1=AddMarks(request.POST,instance=marks)
                    marks.total = sum([int(request.POST['problem' + str(i)]) for i in range(1,5)])
                    if form1.is_valid():
                        form1.save()
                        form.save()
                except Exception as e:
                    pass
            return redirect('student_details', id=kwargs.get('clg_id'))

        student=Student(college=College.objects.get(id=kwargs.get('clg_id')))
        form=AddStudent(instance=student,data=request.POST)
        if form.is_valid():
            form.save()
            marks=MockTest1(student=Student.objects.get(id=student.id))
            form1=AddMarks(instance=marks,data=request.POST)
            marks.total = sum([int(request.POST['problem' + str(i)]) for i in range(1,5)])
            form1.save()
            return redirect('student_details', id=kwargs.get('clg_id'))
    pass

# @api_view(['GET', 'POST'])
# def college_serializeView(request):
#     if request.method == 'GET':
#         colleges = College.objects.all()
#         serializer = CollegeSerializer(colleges, many=True)
#         return JSONResponse(serializer.data)
#
#     elif request.method == 'POST':
#         serializer = CollegeSerializer(data=request.data)
#         if serializer.is_valid():
#             serializer.save()
#             return JSONResponse(serializer.data, status=status.HTTP_201_CREATED)
#         return JSONResponse(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

@csrf_exempt
@api_view(['GET','POST'])
@authentication_classes((SessionAuthentication, BasicAuthentication))
@permission_classes((IsAuthenticated,))
def college_serializeView(request):
    if request.method == 'GET':
        colleges = College.objects.all()
        serializer = CollegeSerializer(colleges, many=True)
        return JsonResponse(serializer.data, safe=False)

    elif request.method == 'POST':
        data = JSONParser().parse(request)
        serializer = CollegeSerializer(data=data)
        if serializer.is_valid():
            serializer.save()
            return JsonResponse(serializer.data, status=201)
        return JsonResponse(serializer.errors, status=400)

@csrf_exempt
@api_view(['GET','PUT','DELETE'])
@authentication_classes((SessionAuthentication, BasicAuthentication))
@permission_classes((IsAuthenticated,))
def college_modify_serializeView(request,**kwargs):
    try:
        college = College.objects.get(**kwargs)
    except College.DoesNotExist:
        return HttpResponse(status=404)

    if request.method == 'GET':
        serializer = CollegeSerializer(college)
        return JsonResponse(serializer.data)

    elif request.method == 'PUT':
        data = JSONParser().parse(request)
        serializer = CollegeSerializer(college, data=data)
        if serializer.is_valid():
            serializer.save()
            return JsonResponse(serializer.data)
        return JsonResponse(serializer.errors, status=400)

    elif request.method == 'DELETE':
        college.delete()
        return HttpResponse(status=204)

class students_serializeView(APIView):
    authentication_classes = (SessionAuthentication, BasicAuthentication)
    permission_classes = (IsAuthenticated,)
    def get(self, request,*args, **kwargs):
        colleges = College.objects.get(id=kwargs['cid'])
        students = Student.objects.filter(college=colleges)
        serializer = StudentSerializer(students, many=True)
        return JsonResponse(serializer.data, safe=False)

    def post(self, request,*args, **kwargs):
        college = College.objects.get(id=kwargs['cid'])
        student=Student(college=college)
        serializer=StudentSerializer(student,data=request.data)
        if serializer.is_valid():
            serializer.save()
        else:
            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

        data = serializer.data
        marks=MockTest1(student=student)
        serializer_marks=MockTestSerializer(marks,data=request.data["mocktest1"])
        if serializer_marks.is_valid():
            serializer_marks.save()
            data.update({"mocktest1":serializer_marks.data})
            return Response(data, status=status.HTTP_201_CREATED)
        else:
            return Response(serializer_marks.errors, status=status.HTTP_400_BAD_REQUEST)

class students_details_serializeView(APIView):
    authentication_classes = (SessionAuthentication, BasicAuthentication)
    permission_classes = (IsAuthenticated,)
    def get(self, request,*args, **kwargs):
        student = Student.objects.filter(id=kwargs['id'])
        serializer = StudentSerializer(student, many=True)
        return JsonResponse(serializer.data, safe=False)

    def put(self, request,*args, **kwargs):
        student=Student.objects.get(id=kwargs['id'])
        serializer=StudentSerializer(student,data=request.data)
        if serializer.is_valid():
            serializer.save()
        else:
            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

        data = serializer.data
        marks=MockTest1.objects.get(student=student)
        serializer_marks=MockTestSerializer(marks,data=request.data["mocktest1"])
        if serializer_marks.is_valid():
            serializer_marks.save()
            data.update({"mocktest1":serializer_marks.data})
            return Response(data, status=status.HTTP_201_CREATED)
        else:
            return Response(serializer_marks.errors, status=status.HTTP_400_BAD_REQUEST)

    def delete(self,request,*args,**kwargs):
        student = Student.objects.filter(id=kwargs['id']).delete()
        return HttpResponse(status=204)


# File of the class Utils, containing various helper functions
# File: utils.py
# Author: Atharva Kulkarni

import pandas as pd
import numpy as np
from gensim.models import KeyedVectors
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import nltk
nltk.download('vader_lexicon')
from nltk.sentiment.vader import SentimentIntensityAnalyzer

class Utils():
    """" Class containing various helper functions """

    # -------------------------------------------- Class Constructor --------------------------------------------
    def __init__(self):
        """ Class Constructor """
        self.tokenizer = Tokenizer(oov_token='[oov]')

    # -------------------------------------------- Function to read data --------------------------------------------

    def read_data(self, path, columns=[]):
        """ Function to read data
        @param path (str): path to the data to be read.
        @columns (list): List of columns to be read.
        return data (pandas.DataFrame): a dataframe of the data read.
        """
        delimiter = path.split(".")[-1]

        if delimiter == "csv":
            if not columns:
                data = pd.read_csv(path, sep=",")
                return data
            else:
                data = pd.read_csv(path, sep=",", usecols=columns)
                return data

        elif delimiter == "tsv":
            if not columns:
                data = pd.read_csv(path, sep="\t")
                return data
            else:
                data = pd.read_csv(path, sep="\t", usecols=columns)
                return data

        else:
            return None

    # -------------------------------------------- Function read dictionary --------------------------------------------

    def get_dict(self, path, key_column, value_column):
        """ Function to read a file into dictionary.
        @param path (str): path to file.
        return dict: created dictionary.
        """
        data = pd.read_csv(path)
        return dict(zip(data[key_column].values.tolist(), data[value_column].values.tolist()))

    # -------------------------------------------- Function to tokenize and pad input text --------------------------------------------

    def tokenize_and_pad(self, corpus, maxlen=100, padding_type='post', truncating_type='post', mode="train"):
        """ Function to prepare text for model input (tokenize and pad).
        @param corpus (list): the corpus to prepare.
        @param maxlen (int): max allowed length of input texts.
        @param padding_type (str): padding type (post or pre).
        @param truncating_type (str): truncating type (post or pre).
        @mode (str): specify train or test mode.
        """
        if mode == "train":
            self.tokenizer.fit_on_texts(corpus)

        corpus = self.tokenizer.texts_to_sequences(corpus)
        corpus = np.asarray(pad_sequences(corpus,
                                          padding=padding_type,
                                          truncating=truncating_type,
                                          maxlen=maxlen))
        return corpus, self.tokenizer

    # -------------------------------------------- Function to read embeddings --------------------------------------------

    def get_embedding_matrix(self, path, vocab, top=50000):
        """ Function to get the word embedding matrix.
        @param path (str): path to the word embeddings file.
        @param vocab (list): list of corpus vocab.
        @return embedding_matrix (np.array): embedding matrix.
        """
        if path.split(".")[-1] == "bin":
            embedding_model = KeyedVectors.load_word2vec_format(path, binary=True, limit=top)
        else:
            embedding_model = KeyedVectors.load_word2vec_format(path, binary=False, limit=top)

        embeddin_model_vocab =[word for word in embedding_model.vocab.keys()]
        final_vocab = list(set(vocab) | set(embeddin_model_vocab))

        embedding_matrix = np.zeros((len(final_vocab), 300))
        cnt = 0
        for index in range(len(final_vocab)):
            if final_vocab[index] in embeddin_model_vocab:
                vec = embedding_model.wv[final_vocab[index]]
                if vec is not None:
                    embedding_matrix[index] = vec
            else:
                cnt = cnt + 1
                continue
        print("zero embeddings: ", cnt)
        return embedding_matrix

     # -------------------------------------------- Function to create age bins --------------------------------------------

    def categorize_age(self, age):
        if age >0 and age <=25:
            return 0
        elif age >25 and age <=40:
            return 1
        elif age >40 and age <=60:
            return 2
        elif age >60:
            return 3

    # -------------------------------------------- Function to create age bins --------------------------------------------

    def categorize_income(self, income):
        if income >0 and income <=25000:
            return 0
        elif income >25000 and income <=75000:
            return 1
        elif income >75000:
            return 2

    # ----------------------------------- Function to calculate weight of each word -----------------------------------

    def get_word_weights(self, essay, feature):
        ''' Function to calculate weight of each word according to the specified feature
        @param essay (pd.Series) list of essays.
        @param feature (pd.Series) The target variable (gold_empthy_bin or gold_distres_bin).
        @return weight (dict) The dictionary contribution of each word to the feature passed.
        '''
        dictionary1 = {}   #to count the number of times a word contributes to feature=1
        dictionary0 = {}   #to count the number of times a word contributes to feature=0
        words=[]
        for i in range(0,len(essay)):   #loop to find both counts(feature=1, feature=0) for each word
            words = essay[i].split()
            for word in words:
                if word.isalpha():
                    if word in dictionary1.keys():
                        if feature[i] == 1:
                            dictionary1[word] = dictionary1[word] + 1
                        else :
                            dictionary0[word] = dictionary0[word] + 1
                    else:
                        if feature[i] == 1:
                            dictionary1[word] = 1
                            dictionary0[word] = 0
                        else:
                            dictionary0[word] = 1
                            dictionary1[word] = 0

        weight = {}  #to store weight of each word
        for i in dictionary0:
            weight[i]=((dictionary1[i]-dictionary0[i])/(dictionary1[i]+dictionary0[i]))

        return weight

    # ----------------------------------- Function to calculate score for each Essay -----------------------------------

    def get_essay_empathy_distress_scores(self, essay, word_weights, transform='original'):
        ''' Function to calculate score for each Essay.
        @param essay (list) list of essays.
        @param word_weights (dict) Tthe dictionary obtained from the wordWeight function
        @return essay_weight (list): list of weights for each essay.
        '''
        essay_weight = []
        for i in range(0, len(essay)):
            weight = 0
            for word in essay[i].split():
                if word in word_weights.keys():
                    weight = weight + float(word_weights[word])
            essay_weight.append(weight)
        if transform == "original":
            return np.reshape(essay_weight, (len(essay_weight), 1))
        elif transform == "tan-inverse":
            np.arctan(np.reshape(essay_weight, (len(essay_weight), 1)))

    # ----------------------------------- Function to get word emotion and vad scores -----------------------------------

    def get_word_scores(self, sentiment='anger'):
        path = "/content/gdrive/My Drive/WASSA-2021-Shared-Task/resources/NRC-resources/"+sentiment+"-scores.txt"
        word_scores = {}
        with open(path) as f:
            data = f.readlines()
            for row in data:
                row = row.split("\t")
                row[1] = row[1].split("\n")
                word_scores[row[0]] = row[1][0]
        f.close()
        return word_scores

    # -------------------------------------------- Function to get essay emotion and vad scores --------------------------------------------

    def get_essay_nrc_scores(self, essay, nrc_features, normalize=False):
        word_scores_list = []
        for element in nrc_features:
            word_scores = self.get_word_scores(element)
            word_scores_list.append(word_scores)
        essay_scores = np.zeros((len(essay), len(word_scores_list)))

        for i in range(len(essay)):
            for j in range(len(word_scores_list)):
                score = 0
                cnt = 0
                for word in essay[i].split():
                    if word in word_scores_list[j].keys():
                        cnt = cnt = 1
                        score = score + float(word_scores_list[j].get(word))
                essay_scores[i][j] = score

        return essay_scores