import torch
import torch.nn as nn
import torch.nn.functional as F
import time
import torchvision
from torchvision import transforms
import numpy as np
import random
import scipy.io as sio
from sklearn.preprocessing import MinMaxScaler

#MNIST precomputed min_max score
# Pre-computed min and max values (after applying GCN) from train data per class
mnist_min_max = [(-0.8826567065619495, 9.001545489292527),
                   (-0.6661464580883915, 20.108062262467364),
                   (-0.7820454743183202, 11.665100841080346),
                   (-0.7645772083211267, 12.895051191467457),
                   (-0.7253923114302238, 12.683235701611533),
                   (-0.7698501867861425, 13.103278415430502),
                   (-0.778418217980696, 10.457837397569108),
                   (-0.7129780970522351, 12.057777597673047),
                   (-0.8280402650205075, 10.581538445782988),
                   (-0.7369959242164307, 10.697039838804978)]

#CIFAR precomputed in min_max sore
cifar_min_max =    [(-28.94083453598571, 13.802961825439636),
                   (-6.681770233365245, 9.158067708230273),
                   (-34.924463588638204, 14.419298165027628),
                   (-10.599172931391799, 11.093187820377565),
                   (-11.945022995801637, 10.628045447867583),
                   (-9.691969487694928, 8.948326776180823),
                   (-9.174940012342555, 13.847014686472365),
                   (-6.876682005899029, 12.282371383343161),
                   (-15.603507135507172, 15.2464923804279),
                   (-6.132882973622672, 8.046098172351265)]

def generate_data(normal_class = 4,
                  transductive= True,
                  flatten = True,
                  GCN = False,
                  resize = None,
                  dataset = "MNIST"):
    if dataset in ["cardio","thyroid","lympho"]:
        return generate_tabular_dataset(dataset = dataset)
    if transductive:
        dataset = generate_transductive_dataset(normal_class = normal_class,
                                                data_dir= '../dataset',
                                                flatten = flatten,
                                                GCN = GCN ,
                                                resize = resize,
                                                dataset= dataset)
        return generate_numpy_data(dataset)
    else:
        trainset, testset = generate_disjoint_dataset(normal_class = normal_class,
                                                      data_dir= '../dataset',
                                                      flatten = flatten,
                                                      GCN = GCN,
                                                      resize= resize,
                                                      dataset =dataset)
        return generate_numpy_data(trainset), generate_numpy_data(testset)

def global_contrast_normalization(x: torch.tensor, scale='l2'):
    """
    Apply global contrast normalization to tensor, i.e. subtract
    mean across features (pixels) and normalize by scale,
    which is either the standard deviation, L1- or L2-norm across
    features (pixels).
    Note this is a *per sample* normalization globally across
    features (and not across the dataset).
    """

    assert scale in ('l1', 'l2')
    n_features = int(np.prod(x.shape))
    mean = torch.mean(x)  # mean over all features (pixels) per sample
    x -= mean
    if scale == 'l1':
        x_scale = torch.mean(torch.abs(x))
    if scale == 'l2':
        x_scale = torch.sqrt(torch.sum(x ** 2)) / n_features
    x /= x_scale
    return x

def get_target_label_idx(labels, targets):
    """
    Get the indices of labels that are included in targets.
    :param labels: array of labels
    :param targets: list/tuple of target labels
    :return: list with indices of target labels
    """
    return np.argwhere(np.isin(labels, targets)).flatten().tolist()

def generate_downsampled_indices(dataset,
                                 normal_class,
                                 down_sample_rate = 0.1):
    targets = torch.tensor(dataset.targets)
    idx = np.arange(len(targets))
    # Get indices to keep
    idx_to_keep = targets[idx]== normal_class
    down_sampled_idx = targets[idx] != normal_class
    # Nomial idex consists only with 1 label
    # Abnormal idx consists of all other labels
    nomial_idx = idx[idx_to_keep]
    abnormal_idx = idx[down_sampled_idx]
    m = nomial_idx.shape[0]
    np.random.seed(4321)
    abnormal_idx = np.random.choice(abnormal_idx,
                                    size= int(down_sample_rate * m),
                                    replace = False)
    overall_idx  = np.append(nomial_idx, abnormal_idx, axis = 0)
    random.seed(4321)
    random.shuffle(overall_idx)
    return overall_idx

def generate_disjoint_dataset(normal_class,
                              data_dir= 'dataset',
                              flatten = True,
                              GCN = False,
                              resize = None,
                              dataset = "MNIST"):
    if resize != None:
        transform_lst = [transforms.Resize(resize), transforms.ToTensor()]
    else:
        transform_lst = [transforms.ToTensor()]
    if GCN:
        if dataset == "MNIST":
            transform_lst.extend([transforms.Lambda(lambda x: global_contrast_normalization(x, scale='l1')),
                                        transforms.Normalize([mnist_min_max[normal_class][0]],
                                                            [mnist_min_max[normal_class][1] - mnist_min_max[normal_class][0]])])
        elif dataset == "CIFAR10":
            transform_lst.extend([transforms.Lambda(lambda x: global_contrast_normalization(x, scale='l1')),
                                        transforms.Normalize([cifar_min_max[normal_class][0]] * 3,
                                                            [cifar_min_max[normal_class][1] - cifar_min_max[normal_class][0]]*3)])
    if flatten:
        transform_lst.extend([lambda x: x.numpy().flatten()])
    transform = transforms.Compose(transform_lst)
    if dataset == "MNIST":
        train_set = torchvision.datasets.MNIST(data_dir,
                                               train=True,
                                               download=True,
                                               transform = transform)
        test_set = torchvision.datasets.MNIST(data_dir,
                                              train=False,
                                              download=True,
                                              transform = transform)
    elif dataset == "CIFAR10":
        train_set = torchvision.datasets.CIFAR10(data_dir,
                                                 train=True,
                                                 download=True,
                                                 transform = transform)
        test_set = torchvision.datasets.CIFAR10(data_dir,
                                                train=False,
                                                download=True,
                                                transform = transform)
    #Subset the training data
    targets = torch.tensor(train_set.targets)
    idx = get_target_label_idx(targets.clone().data.cpu().numpy(), normal_class)
    #Subset of the sample
    train_set.data = train_set.data[idx]
    train_set.targets = torch.tensor(train_set.targets)[idx]
    test_set.targets = torch.tensor(test_set.targets)
    train_set = relabel_dataset(normal_class, train_set)
    test_set = relabel_dataset(normal_class, test_set)

    return train_set, test_set

def generate_transductive_dataset(normal_class,
                                  data_dir= 'dataset',
                                  flatten = True,
                                  GCN = False ,
                                  resize= None,
                                  dataset = "MNIST"):
    if resize != None:
        transform_lst = [transforms.Resize(resize), transforms.ToTensor()]
    else:
        transform_lst = [transforms.ToTensor()]
    if GCN:
        if dataset == "MNIST":
            transform_lst.extend([transforms.Lambda(lambda x: global_contrast_normalization(x, scale='l1')),
                                        transforms.Normalize([mnist_min_max[normal_class][0]],
                                                            [mnist_min_max[normal_class][1] - mnist_min_max[normal_class][0]])])
        elif dataset == "CIFAR10":
            transform_lst.extend([transforms.Lambda(lambda x: global_contrast_normalization(x, scale='l1')),
                                        transforms.Normalize([cifar_min_max[normal_class][0]] * 3,
                                                            [cifar_min_max[normal_class][1] - cifar_min_max[normal_class][0]]*3)])
    if flatten:
        transform_lst.extend([lambda x: x.numpy().flatten()])
    transform = transforms.Compose(transform_lst)
    if dataset == "MNIST":
        dataset = torchvision.datasets.MNIST(data_dir,
                                             train=True,
                                             download=True,
                                             transform = transform)
    elif dataset == "CIFAR10":
        dataset = torchvision.datasets.CIFAR10(data_dir,
                                               train=True,
                                               download=True,
                                               transform = transform)
    #Downsample the dataset with the normal class
    idx = generate_downsampled_indices(dataset, normal_class)
    #Subset of the sample
    dataset.data = dataset.data[idx]
    dataset.targets = torch.tensor(dataset.targets)[idx]
    #Replacing the label, normal class = 0, abnormal class = 1
    return relabel_dataset(normal_class, dataset)

def relabel_dataset(normal_class, dataset):
    for i in range(len(dataset)):
        if dataset.targets[i] == normal_class:
            dataset.targets[i] = 0
        else:
            dataset.targets[i] = 1
    return dataset

def generate_numpy_data(dataset):
    X = []
    y = []
    for i in range(len(dataset)):
        if type(dataset[i][0]) == np.ndarray:
            X.append(dataset[i][0])
        else:
            X.append(dataset[i][0].detach().cpu().numpy())
        y.append(dataset[i][1])
    return np.array(X), np.array(y)

def generate_tabular_dataset(dataset = "cardio"):
    try:
        data = sio.loadmat('../dataset/tabular/' + dataset + '.mat')
    except:
        print("dataset not found, need to have data in the correct directory")
        return None
    scaler = MinMaxScaler()
    features = data['X']
    scaler.fit(features)
    features = scaler.transform(features)
    labels = data['y']
    count = 0
    for i in labels:
        if i == 1:
            count += 1
    print("percentage of anomaly: %.3f" % (count / labels.shape[0]))
    return features.astype('float32'), labels


import torch as th
import torch.nn as nn
import torchvision.models as models
from torchnet.meter import AUCMeter

import numpy as np

from math import ceil
import random

import argparse
from os.path import exists
import sys

from tqdm import tqdm

train_ratio = 0.75

def get_vgg16_modified() -> nn.Module:
    vgg16 = models.vgg16()

    vgg16.classifier[-1] = nn.Linear(4096, 1)
    vgg16.classifier.add_module("7", nn.Sigmoid())
    return vgg16

# 1000 images en 224 * 224 * 3 ~= 42Go
def train_vgg16():
    # Create argument parser
    parser = argparse.ArgumentParser("Train VGG16 Main")
    parser.add_argument("-d", "--data-path", type=str, required=True, dest="data_path")
    parser.add_argument("-l", "--label-path", type=str, required=True, dest="label_path")
    parser.add_argument("-o", "--output-model-path", type=str, required=True, dest="output_model_path")

    # Parse args
    args = parser.parse_args()

    # Get args
    data_path = args.data_path
    label_path = args.label_path
    output_model_path = args.output_model_path

    # Test if numpy data and labels exist
    if not exists(data_path):
        raise FileNotFoundError("Numpy data file doesn't exist ({}) !".format(data_path))
    if not exists(label_path):
        raise FileNotFoundError("Numpy label file doesn't exist ({}) !".format(label_path))

    print("Loading VGG16 model...")
    # Create modified VGG16 model
    vgg16 = get_vgg16_modified()

    # Define loss function
    loss_fn = nn.MSELoss()

    # Pass to cuda model and loss function
    vgg16.cuda()
    loss_fn.cuda()

    # Create optimizer
    optim = th.optim.SGD(vgg16.parameters(), lr=1e-4)

    # Epoch number and batch size
    nb_epoch = 3
    batch_size = 32

    print("Loading data...")
    # Load data
    data = np.load(data_path)
    labels = np.load(label_path)

    # Separate eval data
    nb_split = int(data.shape[0] * train_ratio)
    data = data[:nb_split]
    labels = labels[:nb_split]

    print("Shuffle data...")
    # Shuffle data
    # https://stackoverflow.com/questions/6127503/shuffle-array-in-c
    # to prevent memory allocation
    for i in tqdm(range(data.shape[0] - 1)):
        j = i + random.randint(0, sys.maxsize) // (sys.maxsize // (data.shape[0] - i) + 1)

        data[i, :, :, :], data[j, :, :, :] = data[j, :, :, :], data[i, :, :, :]
        labels[i], labels[j] = labels[j], labels[i]

    # Compute batch number
    nb_batch = ceil(data.shape[0] / batch_size)

    print("Starting training...")
    # Train loop
    for e in range(nb_epoch):

        sum_loss = 0

        # Batch loop
        for i_b in tqdm(range(nb_batch)):
            # Get batch indexes
            i_min = i_b * batch_size
            i_max = (i_b + 1) * batch_size
            i_max = i_max if i_max < data.shape[0] else data.shape[0]

            # Slice data to get batch
            batch = data[i_min:i_max, :, :, :]
            batch = th.tensor(batch).cuda().float() / 255.

            # And labels
            batch_label = th.tensor(labels[i_min:i_max]).cuda().float()

            # Forward
            out = vgg16(batch).squeeze(1)

            # Compute loss
            loss = loss_fn(out, batch_label)

            # Backward
            loss.backward()

            # Upgrade weights
            optim.step()

            sum_loss += loss.item()

        print("Epoch {}, loss = {}".format(e, sum_loss / nb_batch))

    th.save(vgg16.state_dict(), output_model_path)

def test_vgg16():
    parser = argparse.ArgumentParser("Test VGG16 Main")
    parser.add_argument("-d", "--data-path", type=str, required=True, dest="data_path")
    parser.add_argument("-l", "--label-path", type=str, required=True, dest="label_path")
    parser.add_argument("-m", "--model-path", type=str, required=True, dest="model_path")

    args = parser.parse_args()

    data_path = args.data_path
    label_path = args.label_path
    model_path = args.model_path

    # Test if numpy data and labels exist
    if not exists(data_path):
        raise FileNotFoundError("Numpy data file doesn't exist ({}) !".format(data_path))
    if not exists(label_path):
        raise FileNotFoundError("Numpy label file doesn't exist ({}) !".format(label_path))
    # Test if model save file exist
    if not exists(model_path):
        raise FileNotFoundError("Model state dict file doesn't exist ({}) !".format(model_path))

    print("Load model...")
    # Load model
    vgg16 = get_vgg16_modified()
    vgg16.load_state_dict(th.load(model_path))
    vgg16.cuda()
    vgg16.eval()

    # Create AUC Meter
    auc_meter = AUCMeter()

    print("Load data...")
    # Load data
    data = np.load(data_path)
    labels = np.load(label_path)

    # Split eval
    nb_split = int(data.shape[0] * train_ratio)
    data = data[nb_split:]
    labels = labels[nb_split:]

    batch_size = 32
    nb_batch = ceil(data.shape[0] / batch_size)

    # Loop on eval data
    for i_b in tqdm(range(nb_batch)):
        # Get batch indexes
        i_min = i_b * batch_size
        i_max = (i_b + 1) * batch_size
        i_max = i_max if i_max < data.shape[0] else data.shape[0]

        # Slice data to get batch
        batch = data[i_min:i_max, :, :, :]
        batch = batch.transpose(0, 3, 1, 2)
        batch = th.tensor(batch).cuda().float() / 255.

        # And labels
        batch_label = th.tensor(labels[i_min:i_max]).cuda().float()

        # Forward - Inférence
        out = vgg16(batch).squeeze(1)

        # Update metric
        auc_meter.add(out.cpu().detach(), batch_label.cpu().detach())

    print("AUC value = {}".format(auc_meter.value()[0]))

if __name__ == "__main__":
    train_vgg16()
    #test_vgg16()


"""Helper functions for working with datasets represented as a pandas.DataFrame"""
from __future__ import annotations

import pathlib

import numpy as np
import pandas as pd

def get_dataset_csv_filename(data_dir_name: str, timenow: str) -> str:
    """Get name of csv file representing dataset.

    This function is called by
    :func:`vak.prep.frame_classification.dataset_df.get_dataset_csv_path`.

    Parameters
    ----------
    data_dir_name : str
        Name of directory specified as parameter ``data_dir``
        when calling :func:`vak.core.prep.prep`.
        This becomes the "prefix" of the csv filename.
    timenow : str
        Timestamp.
        This becomes the "suffix" of the csv filename.

    Returns
    -------
    dataset_csv_filename : str
        String, in the form f"{data_dir_name}_prep_{timenow}.csv"
    """
    return f"{data_dir_name}_prep_{timenow}.csv"

def get_dataset_csv_path(
    dataset_path: pathlib.Path, data_dir_name: str, timenow: str
) -> pathlib.Path:
    """Returns the path that should be used to save
    a pandas DataFrame representing a dataset
    to a csv file.

    Parameters
    ----------
    dataset_path : str, pathlib.Path
        Path to directory that represents dataset.
    data_dir_name : str
        Name of directory specified as parameter ``data_dir``
        when calling :func:`vak.core.prep.prep`.
        This becomes the "prefix" of the csv filename.
    timenow : str
        Timestamp.
        This becomes the "suffix" of the csv filename.

    Returns
    -------
    dataset_csv_path : pathlib.Path
        Path that is used when saving ``dataset_df`` as a csv file
        in the root of the dataset directory, ``dataset_path``.
    """
    dataset_csv_filename = get_dataset_csv_filename(data_dir_name, timenow)
    dataset_csv_path = dataset_path / dataset_csv_filename
    return dataset_csv_path

def add_split_col(df: pd.DataFrame, split: str) -> pd.DataFrame:
    """Add a 'split' column to a pandas DataFrame.

    Used by :func:`vak.prep`
    to assign an entire dataset to the same split,
    e.g. 'train' or 'predict'.
    All rows in the 'split' column will have the value specified.

    Parameters
    ----------
    df : pandas.DataFrame
        A dataframe that represents a dataset.
    split : str
        A string that will be assigned to every row
        in the added "split" column.
        One of {'train', 'val', 'test', 'predict'}.
    """
    if split not in {"train", "val", "test", "predict"}:
        raise ValueError(
            f"value for split should be one of {{'train', 'val', 'test', 'predict'}}, but was '{split}'"
        )
    split_col = np.asarray([split for _ in range(len(df))], dtype="object")
    df["split"] = split_col
    return df


import requests
from bs4 import BeautifulSoup
import re
from threading import Thread

class Extractor(Thread):
    def __init__(self, url:str, callback):
        super().__init__()
        self.url = url
        self.callback = callback
        self.mod_title = ""
        self.authors = ""

    def run(self):
        try:
            soup = self.get_html(self.url)
            self.mod_title = self.get_mod_title(soup)
            self.authors = self.get_authors(soup)
        except:
            self.mod_title = ""
            self.authors = ""
        self.callback(self.mod_title, self.authors)

    def get_html(self, url:str):
        try:
            response = requests.get(url)
            if response.status_code == 200:
                soup = BeautifulSoup(response.text, 'html.parser')
                return soup
            else:
                print("Failed to retrieve the web page. Status code:", response.status_code)
                return None
        except Exception as e:
            print("An error occurred:", str(e))
            return None

    def get_mod_title(self, soup):
        page_title = ""
        page_title_h1 = soup.find('h1', id="PageTitle")

        if page_title_h1:
            page_title = page_title_h1.contents[0].get_text().replace("\t", "").replace("\n", "")
            print(page_title)
        return page_title

    def get_authors(self, soup):
        result = ""
        meta_description = soup.find('meta', attrs={'name': 'description'})
        if meta_description:
            description_content = meta_description.get('content')
            pattern = r'submitted by (.+)$'
            match = re.search(pattern, description_content)

            if match:
                result = match.group(1)
            else:
                print("Authors not found in the text.")
        return result.replace(" and", ",")


from PyQt5 import QtCore, QtGui, QtWidgets
import sys
import socket_wrapper as sw

from socket_wrapper.utils import time_stamp, unzip_folder
from Qt_thread_aux import ClientConnection as cc

# TODO(Jerry): July 24th, 2019
#  Standardize all messages. Right now, the titles are all different.

client = sw.Client()
current_row = -1

class Ui_frmClient(object):
    def __init__(self, frmClientTerminal):
        self.auto_reconn = True

        self.connection = cc.ClientConnectionThread(client)
        self.connection.sig.connect(self.update_messages)

        self.setupUi(frmClientTerminal)
        self.retranslateUi(frmClientTerminal)
        self.clicked_binding(frmClientTerminal)
        self.menu_actions()

    def setupUi(self, frmClientTerminal):
        frmClientTerminal.setObjectName("frmClientTerminal")
        frmClientTerminal.resize(823, 464)
        self.centralwidget = QtWidgets.QWidget(frmClientTerminal)
        self.centralwidget.setObjectName("centralwidget")
        self.layoutWidget = QtWidgets.QWidget(self.centralwidget)
        self.layoutWidget.setGeometry(QtCore.QRect(10, 0, 641, 22))
        self.layoutWidget.setObjectName("layoutWidget")
        self.horizontalLayout_11 = QtWidgets.QHBoxLayout(self.layoutWidget)
        self.horizontalLayout_11.setContentsMargins(0, 0, 0, 0)
        self.horizontalLayout_11.setObjectName("horizontalLayout_11")
        self.lblListOfConnection = QtWidgets.QLabel(self.layoutWidget)
        font = QtGui.QFont()
        font.setFamily("Myriad Pro")
        font.setPointSize(12)
        self.lblListOfConnection.setFont(font)
        self.lblListOfConnection.setObjectName("lblListOfConnection")
        self.horizontalLayout_11.addWidget(self.lblListOfConnection)
        self.line = QtWidgets.QFrame(self.centralwidget)
        self.line.setGeometry(QtCore.QRect(90, 265, 721, 31))
        self.line.setFrameShape(QtWidgets.QFrame.HLine)
        self.line.setFrameShadow(QtWidgets.QFrame.Sunken)
        self.line.setObjectName("line")
        self.layoutWidget_2 = QtWidgets.QWidget(self.centralwidget)
        self.layoutWidget_2.setGeometry(QtCore.QRect(570, 290, 241, 131))
        self.layoutWidget_2.setObjectName("layoutWidget_2")
        self.verticalLayout_9 = QtWidgets.QVBoxLayout(self.layoutWidget_2)
        self.verticalLayout_9.setContentsMargins(0, 0, 0, 0)
        self.verticalLayout_9.setObjectName("verticalLayout_9")
        self.btnStartClient = QtWidgets.QPushButton(self.layoutWidget_2)
        font = QtGui.QFont()
        font.setFamily("Myriad Pro")
        font.setPointSize(22)
        self.btnStartClient.setFont(font)
        self.btnStartClient.setAutoFillBackground(False)
        self.btnStartClient.setObjectName("btnStartClient")
        self.verticalLayout_9.addWidget(self.btnStartClient)
        self.verticalLayout_10 = QtWidgets.QVBoxLayout()
        self.verticalLayout_10.setObjectName("verticalLayout_10")
        self.horizontalLayout_12 = QtWidgets.QHBoxLayout()
        self.horizontalLayout_12.setObjectName("horizontalLayout_12")
        self.lblInputIP = QtWidgets.QLabel(self.layoutWidget_2)
        self.lblInputIP.setObjectName("lblInputIP")
        self.horizontalLayout_12.addWidget(self.lblInputIP)
        spacerItem = QtWidgets.QSpacerItem(38, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum)
        self.horizontalLayout_12.addItem(spacerItem)
        self.linInputIP = QtWidgets.QLineEdit(self.layoutWidget_2)
        self.linInputIP.setObjectName("linInputIP")
        self.horizontalLayout_12.addWidget(self.linInputIP)
        self.verticalLayout_10.addLayout(self.horizontalLayout_12)
        self.horizontalLayout_13 = QtWidgets.QHBoxLayout()
        self.horizontalLayout_13.setObjectName("horizontalLayout_13")
        self.lblPort = QtWidgets.QLabel(self.layoutWidget_2)
        self.lblPort.setObjectName("lblPort")
        self.horizontalLayout_13.addWidget(self.lblPort)
        spacerItem1 = QtWidgets.QSpacerItem(28, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum)
        self.horizontalLayout_13.addItem(spacerItem1)
        self.linPort = QtWidgets.QLineEdit(self.layoutWidget_2)
        self.linPort.setObjectName("linPort")
        self.horizontalLayout_13.addWidget(self.linPort)
        self.verticalLayout_10.addLayout(self.horizontalLayout_13)
        self.verticalLayout_9.addLayout(self.verticalLayout_10)
        self.lblStatusLog = QtWidgets.QLabel(self.centralwidget)
        self.lblStatusLog.setGeometry(QtCore.QRect(10, 265, 101, 21))
        font = QtGui.QFont()
        font.setFamily("Myriad Pro")
        font.setPointSize(12)
        font.setBold(False)
        font.setWeight(50)
        self.lblStatusLog.setFont(font)
        self.lblStatusLog.setObjectName("lblStatusLog")
        self.txtStatusUpdate = QtWidgets.QTextEdit(self.centralwidget)
        self.txtStatusUpdate.setGeometry(QtCore.QRect(10, 290, 541, 131))
        font = QtGui.QFont()
        font.setFamily("Myriad Pro")
        font.setPointSize(10)
        self.txtStatusUpdate.setFont(font)
        self.txtStatusUpdate.setReadOnly(True)
        self.txtStatusUpdate.setObjectName("txtStatusUpdate")
        self.lstTransferredFiles = QtWidgets.QListWidget(self.centralwidget)
        self.lstTransferredFiles.setGeometry(QtCore.QRect(12, 32, 541, 231))
        self.lstTransferredFiles.setSelectionBehavior(QtWidgets.QAbstractItemView.SelectRows)
        self.lstTransferredFiles.setObjectName("lstTransferredFiles")
        self.btnUnzip = QtWidgets.QPushButton(self.centralwidget)
        self.btnUnzip.setGeometry(QtCore.QRect(568, 32, 241, 121))
        font = QtGui.QFont()
        font.setFamily("Myriad Pro")
        font.setPointSize(22)
        self.btnUnzip.setFont(font)
        self.btnUnzip.setAutoFillBackground(False)
        self.btnUnzip.setObjectName("btnUnzip")
        self.linOutput = QtWidgets.QLineEdit(self.centralwidget)
        self.linOutput.setGeometry(QtCore.QRect(630, 160, 181, 20))
        self.linOutput.setObjectName("linOutput")
        self.lblOuput = QtWidgets.QLabel(self.centralwidget)
        self.lblOuput.setGeometry(QtCore.QRect(570, 160, 51, 16))
        self.lblOuput.setObjectName("lblOuput")
        self.widget = QtWidgets.QWidget(self.centralwidget)
        self.widget.setGeometry(QtCore.QRect(640, 200, 171, 51))
        self.widget.setObjectName("widget")
        self.verticalLayout = QtWidgets.QVBoxLayout(self.widget)
        self.verticalLayout.setContentsMargins(0, 0, 0, 0)
        self.verticalLayout.setObjectName("verticalLayout")
        self.lblLocationResults = QtWidgets.QLabel(self.widget)
        self.lblLocationResults.setObjectName("lblLocationResults")
        self.verticalLayout.addWidget(self.lblLocationResults)
        self.lblTimeResults = QtWidgets.QLabel(self.widget)
        self.lblTimeResults.setObjectName("lblTimeResults")
        self.verticalLayout.addWidget(self.lblTimeResults)
        self.widget1 = QtWidgets.QWidget(self.centralwidget)
        self.widget1.setGeometry(QtCore.QRect(570, 200, 58, 51))
        self.widget1.setObjectName("widget1")
        self.verticalLayout_2 = QtWidgets.QVBoxLayout(self.widget1)
        self.verticalLayout_2.setContentsMargins(0, 0, 0, 0)
        self.verticalLayout_2.setObjectName("verticalLayout_2")
        self.lblLocation = QtWidgets.QLabel(self.widget1)
        font = QtGui.QFont()
        font.setPointSize(10)
        self.lblLocation.setFont(font)
        self.lblLocation.setObjectName("lblLocation")
        self.verticalLayout_2.addWidget(self.lblLocation)
        self.lblTime = QtWidgets.QLabel(self.widget1)
        font = QtGui.QFont()
        font.setPointSize(10)
        self.lblTime.setFont(font)
        self.lblTime.setObjectName("lblTime")
        self.verticalLayout_2.addWidget(self.lblTime)
        frmClientTerminal.setCentralWidget(self.centralwidget)
        self.menubar = QtWidgets.QMenuBar(frmClientTerminal)
        self.menubar.setGeometry(QtCore.QRect(0, 0, 823, 21))
        self.menubar.setObjectName("menubar")
        self.menuFile = QtWidgets.QMenu(self.menubar)
        self.menuFile.setObjectName("menuFile")
        self.menuSettings = QtWidgets.QMenu(self.menubar)
        self.menuSettings.setObjectName("menuSettings")
        frmClientTerminal.setMenuBar(self.menubar)
        self.statusbar = QtWidgets.QStatusBar(frmClientTerminal)
        self.statusbar.setObjectName("statusbar")
        frmClientTerminal.setStatusBar(self.statusbar)
        self.actionAuto_Reconnect = QtWidgets.QAction(frmClientTerminal)
        self.actionAuto_Reconnect.setObjectName("actionAuto_Reconnect")
        self.actionTimeout = QtWidgets.QAction(frmClientTerminal)
        self.actionTimeout.setObjectName("actionTimeout")
        self.actionReconnect_Time = QtWidgets.QAction(frmClientTerminal)
        self.actionReconnect_Time.setObjectName("actionReconnect_Time")
        self.actionHelp = QtWidgets.QAction(frmClientTerminal)
        self.actionHelp.setObjectName("actionHelp")
        self.actionClose = QtWidgets.QAction(frmClientTerminal)
        self.actionClose.setObjectName("actionClose")
        self.actionReset = QtWidgets.QAction(frmClientTerminal)
        self.actionReset.setObjectName("actionReset")
        self.actionSave_Directory = QtWidgets.QAction(frmClientTerminal)
        self.actionSave_Directory.setObjectName("actionSave_Directory")
        self.actionSave_Log = QtWidgets.QAction(frmClientTerminal)
        self.actionSave_Log.setObjectName("actionSave_Log")
        self.actionAuto_Unzip = QtWidgets.QAction(frmClientTerminal)
        self.actionAuto_Unzip.setObjectName("actionAuto_Unzip")
        self.menuFile.addAction(self.actionSave_Log)
        self.menuFile.addAction(self.actionClose)
        self.menuFile.addAction(self.actionReset)
        self.menuFile.addAction(self.actionHelp)
        self.menuSettings.addAction(self.actionAuto_Reconnect)
        self.menuSettings.addAction(self.actionAuto_Unzip)
        self.menuSettings.addAction(self.actionReconnect_Time)
        self.menuSettings.addAction(self.actionTimeout)
        self.menuSettings.addAction(self.actionSave_Directory)
        self.menubar.addAction(self.menuFile.menuAction())
        self.menubar.addAction(self.menuSettings.menuAction())

        self.retranslateUi(frmClientTerminal)
        QtCore.QMetaObject.connectSlotsByName(frmClientTerminal)

    def retranslateUi(self, frmClientTerminal):
        _translate = QtCore.QCoreApplication.translate
        frmClientTerminal.setWindowTitle(_translate("frmClientTerminal", "Client Terminal"))
        self.lblListOfConnection.setText(_translate("frmClientTerminal", "Transferred Files"))
        self.btnStartClient.setText(_translate("frmClientTerminal", "START"))
        self.lblInputIP.setText(_translate("frmClientTerminal", "IP Address"))
        self.linInputIP.setText(_translate("frmClientTerminal", "192.168.1.118"))
        self.lblPort.setText(_translate("frmClientTerminal", "Port Number"))
        self.linPort.setText(_translate("frmClientTerminal", "8000"))
        self.lblStatusLog.setText(_translate("frmClientTerminal", "Status Log:"))
        self.btnUnzip.setText(_translate("frmClientTerminal", "UNPACT"))
        self.lblOuput.setText(_translate("frmClientTerminal", "Output:"))
        self.lblLocationResults.setText(_translate("frmClientTerminal", "None"))
        self.lblTimeResults.setText(_translate("frmClientTerminal", "None"))
        self.lblLocation.setText(_translate("frmClientTerminal", "Location: "))
        self.lblTime.setText(_translate("frmClientTerminal", "Time: "))
        self.menuFile.setTitle(_translate("frmClientTerminal", "File"))
        self.menuSettings.setTitle(_translate("frmClientTerminal", "Settings"))
        self.actionAuto_Reconnect.setText(_translate("frmClientTerminal", "Auto Reconnect"))
        self.actionTimeout.setText(_translate("frmClientTerminal", "Timeout"))
        self.actionReconnect_Time.setText(_translate("frmClientTerminal", "Reconnect Time"))
        self.actionHelp.setText(_translate("frmClientTerminal", "Help"))
        self.actionClose.setText(_translate("frmClientTerminal", "Close"))
        self.actionReset.setText(_translate("frmClientTerminal", "Reset"))
        self.actionSave_Directory.setText(_translate("frmClientTerminal", "Save Location"))
        self.actionSave_Log.setText(_translate("frmClientTerminal", "Save Log"))
        self.actionAuto_Unzip.setText(_translate("frmClientTerminal", "Auto Unzip"))

    def clicked_binding(self, frmClientTerminal):
        frmClientTerminal.closeEvent = self.close_gui
        self.lstTransferredFiles.clicked.connect(self.file_details)
        self.btnStartClient.clicked.connect(self.set_client_for_ui)
        self.btnUnzip.clicked.connect(self.unpack)

    def menu_actions(self):
        self.actionAuto_Reconnect.triggered.connect(self.auto_reconnect)
        self.actionSave_Directory.triggered.connect(self.change_save_location)
        # self.actionReset.triggered.hconnect(partial(self.reset_client, True))

    def auto_reconnect(self):
        box = QtWidgets.QMessageBox()
        box.setText('Do you want the client to auto reconnect?')
        box.setWindowTitle('Set Auto Connect')
        box.setStandardButtons(QtWidgets.QMessageBox.Yes |
                               QtWidgets.QMessageBox.No)
        box.setIcon(QtWidgets.QMessageBox.Question)

        retval = box.exec_()
        if retval == QtWidgets.QMessageBox.Yes:
            self.auto_reconn = True
        else:
            self.auto_reconn = False

    def file_details(self):
        global current_row
        current_row = self.lstTransferredFiles.currentRow()
        details = client.get_list_of_file(current_row)
        self.lblLocationResults.setText(details.get_location())
        self.lblTimeResults.setText(details.get_time())
        self.linOutput.setText('C:/Users/user/Desktop/ZYNC/save/temp/')

    def change_save_location(self):
        dir_name = QtWidgets.QFileDialog.getExistingDirectory(
            None, 'Select a Directory')

        if dir_name:
            self.txtStatusUpdate.append(
                'Save location changed to: ' + dir_name)

    def close_gui(self, e):
        global client

        client = None
        return self.connection.end()

    def update_messages(self):
        global client

        if not self.connection.get_messages():
            return 1
        for message in self.connection.get_messages():
            splt_message = message.split()
            if splt_message[0] == 'RESET':
                self.txtStatusUpdate.append(time_stamp(
                    2, dates=False) + 'Connection Lost')
                self.connection.pause_communication()
                self.set_client_for_ui()
                break
            elif splt_message[0] == 'BUTTON':
                self.btnStartClient.setText(splt_message[1])
                continue
            elif splt_message[0] == 'FILE':
                row = self.lstTransferredFiles.currentRow()
                self.lstTransferredFiles.addItem(splt_message[1])
                continue

            self.txtStatusUpdate.append(message)
        self.connection.set_messages()

    def set_client_for_ui(self):
        global client

        self.txtStatusUpdate.append(sw.time_stamp(
            dates=False) + 'Initializing client, do not close window...')
        try:
            input_ip = self.linInputIP.text()
            input_port = int(self.linPort.text())
            self.connection.set_ip_port(input_ip, input_port)
        except ValueError:
            self.txtStatusUpdate.append(
                sw.time_stamp(dates=False) + 'Error: Bad Input')
            return 1

        self.btnStartClient.setDisabled(True)
        self.btnStartClient.setText('CONNECTING')
        self.linInputIP.setDisabled(True)
        self.linPort.setDisabled(True)

        self.connection.start_connection()
        self.connection.resume_connection()

    def unpack(self):
        if current_row == -1:
            return 1
        self.txtStatusUpdate.append(time_stamp(dates=False) +'Unpacking target')
        output = self.linOutput.text()
        unzip_folder(self.lblLocationResults.text(), output=output)
        self.txtStatusUpdate.append(time_stamp(dates=False) +'Target unpacked')
        return 0

if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    ClientTerminal = QtWidgets.QMainWindow()
    ui = Ui_frmClient(ClientTerminal)

    ClientTerminal.show()
    sys.exit(app.exec_())


from airflow import DAG
from datetime import datetime
from airflow.utils.dates import days_ago
from airflow_db_logger.operators import AirflowDBLoggerCleanupOperator

default_args = {"owner": "tester", "start_date": days_ago(0), "retries": 0}

dag = DAG(
    "db-log-cleanup",
    default_args=default_args,
    description="Test base airflow db logger cleanup",
    schedule_interval=None,
    catchup=False,
)

with dag:
    AirflowDBLoggerCleanupOperator(
        task_id="db_log_cleanup",
        up_to=datetime.now(),
        since=None,
    )

if __name__ == "__main__":
    dag.clear()
    dag.schedule_interval = None
    dag.run()


import os, json, sys
import numpy as np
import pandas as pd
from numpy.random import randn
from numpy.random import seed
from numpy import mean
from numpy import var
from math import sqrt
import pickle
from scipy import stats
from scipy.stats import ttest_1samp, ttest_ind, ttest_rel
import statistics
import matplotlib.pyplot as plt
import logging, socket
import statsmodels.api as sm
from scipy.stats import mannwhitneyu
from statsmodels.sandbox.regression.predstd import wls_prediction_std

# class Stat_Group():
#################################################
# Logging configuration
#################################################
computername = socket.gethostname()
if computername == "BigBang":
    mstfile = "G:\\Unity\\MST_JSAM\\analyse_csvs\\Data_Rogens\\MST\\17_TimQueißertREST1fertig.csv"
    dirname = ".\\Data_Rogens\\Results"
if computername == "XenonBang":
    dirname = "G:\\Programming\\MST_JSAM\\analyse_csvs\\Data_Rogens\\Results"
if computername == "Laptop-LittleBang":
    dirname = "D:\\Programming\\MST_JSAM\\analyse_csvs\\Data_Rogens\\Results"

logfilename = os.path.join(dirname, "results.log")
logger_stat = logging.getLogger("")
logger_stat.setLevel(logging.DEBUG)

formatter = logging.Formatter('%(asctime)s :: %(module)s :: %(levelname)s :: %(name)s :: %(message)s')
file_handler_stat = logging.FileHandler(logfilename, 'w+')
file_handler_stat.setLevel(logging.DEBUG)
file_handler_stat.setFormatter(formatter)

c_handler = logging.StreamHandler(sys.stdout)
c_handler.setLevel(logging.DEBUG)
c_handler.setFormatter(formatter)

logger_stat.addHandler(file_handler_stat)
logger_stat.addHandler(c_handler)
logger_stat.debug("entering stats ....")

class Statistic_Exp_Dir():
    """ takes an directory with multiple files and
        performes the statistik with the information in these files
        """
    def __init__(self, datadir, resultsdir, output_filename = "stats_output.txt"): #experiment_name = 'MST', groups = [], key = "cor_seqsum_lpn", level = "pn", paradigma = 0, is_independent = False):
        self.datadir = datadir
        self.resultsdir = resultsdir
        self.output_filename = output_filename
        self.all_exp = []
        self.groups = [] # a list of the groups... each group consists of a list of experiement classes
        self.print_output = []
        self.is_print_to_std = True
        self.first_write = True
        self.first_write
        # self.experiment_name = experiment_name
        # self.key = key
        # self.is_independent = is_independent
        # self.paradigma = paradigma
        # self.level = level
#        self.test_group_differences_ttest(key, self.is_independent, self.paradigma, self.level)
    def perform_all_available_analyses(self):
        self.read_exp_files_from_dir()
        self.create_groups()
        self.check_data_consistency()
        self.was_there_learning_in_each_experiment()
        self.estimate_q(is_q_fake_abs = True)
        self.estimate_phi()
        self.estimate_anova()

    def filter_experiments(self, experiment_names = [], days= [], vpns =[]):
        """ filter the experiments according to the given parameter
            return a list of the corresponding experiements"""
        exps = []
        for group in self.groups:
            for exp in group:
                is_exp = False
                is_day = False
                is_vpn = False
                if len(experiment_names) == 0:
                    is_exp = True
                else:
                    if exp.experiment_name in experiment_names:
                        is_exp = True
                if len(days) == 0:
                    is_day = True
                else:
                    if exp.day in days:
                        is_day = True
                if len(vpns) == 0:
                    is_vpn = True
                else:
                    if exp.vpn in vpns:
                        is_vpn = True
                if (is_exp and is_day and is_vpn):
                    exps.append(exp)

        return exps

    def estimate_anova(self):
        """ Wyombs 2012
        We collected three behavioral variables during training: the time between key presses (i.e.,
        the vector of inter-key intervals), movement time (MT), and error. MT is the time elapsed
        from the initial to final key press. Error was scored as any trial not produced in the correct
        order as well as those trials not completed within the 8 s time limit. To test for learning, we
        entered the MT data for each subject, sequence, and session into a repeated-measures
        ANOVA (with subject treated as a random factor). To test for differences in error over
        training, we combined error for each frequent sequence and entered them for each subject
        and session using a repeated-measures ANOVA. """
        self.myprint("performing an ANOVA")

    # def create_one_csv(self):
    #     """ creates one big csv for further analysis with R"""
    #     self.all_exp = []
    #     self.groups = []
    #     self.read_exp_files_from_dir()
    #     self.create_groups()
    #     self.check_data_consistency()
    #     # create a dataframe with all important variables
    #     columnnames = [
    #         'experiment_name',
    #         'vpn',                                         # die Versuchspersonennummer
    #         'day',                                                               # DER Trainingstag
    #         'paradigma',                                    # falls an einem Tag unterschiedliche Interventionen erfolgten (MST_21 vs. MST_22 vs. MST_23)
    #         'sequence_length',
    #         'filename',
    #         'root_dir',
    #         'data_dir',
    #         'is_delete_first',                            #! ein wesentlicher Unterschied ist noch, dass beim MST der erste ipi nicht geloescht wird
    #         'all_ipi_lsln',
    #         'cor_ipi_lsln',
    #         'err_ipi_lsln',
    #         'all_hits_lsln',
    #         'all_ipi_lblsln',
    #         'cor_ipi_lblsln',
    #         'err_ipi_lblsln',
    #         'all_hits_lblsln',
    #         'all_ipi_lplsln',
    #         'cor_ipi_lplsln',
    #         'err_ipi_lplsln',
    #         'all_hits_lplsln',
    #         'all_ipi_lplblsln',
    #         'cor_ipi_lplblsln',
    #         'err_ipi_lplblsln',
    #         'all_hits_lplblsln',
    #         'all_seqsum_lpn',
    #         'all_seqsum_lplbn',
    #         'all_seqtimesum_lplsn',
    #         'all_seqtimesum_lplblsn',
    #         'cor_seqsum_lpn',
    #         'cor_seqsum_lplbn',
    #         'cor_seqtimesum_lplsn',
    #         'cor_seqtimesum_lplblsn',
    #         'err_seqsum_lpn',
    #         'err_seqsum_lplbn',
    #         'err_seqtimesum_lplsn',
    #         'err_seqtimesum_lplblsn',
    #         'all_seqtimesum_slope_lpn',
    #         'all_seqtimesum_to_max_slope_lpn',
    #         'cor_seqtimesum_slope_lpn',
    #         'cor_seqtimesum_to_max_slope_lpn',
    #         'err_seqtimesum_slope_lpn',
    #         'err_seqtimesum_to_max_slope_lpn',
    #         'all_seqtimesum_per_block_slope_lpn',
    #         'all_seqtimesum_per_block_to_max_slope_lpn',
    #         'cor_seqtimesum_per_block_slope_lpn',
    #         'cor_seqtimesum_per_block_to_max_slope_lpn',
    #         'err_seqtimesum_per_block_slope_lpn',
    #         'err_seqtimesum_per_block_to_max_slope_lpn',
    #         'all_seqnum_per_block_slope_lpn',
    #         'cor_seqnum_per_block_slope_lpn',
    #         'err_seqnum_per_block_slope_lpn',
    #         'net_A',
    #         'net_C',
    #         'net_c',
    #         'net_ipi',
    #         'net_is_estimate_clustering',
    #         'net_k',
    #         'net_kappa',
    #         'net_m',
    #         'net_my2',
    #         'net_phi',
    #         'net_phi_real',
    #         'net_phi_fake_list',
    #         'net_q_real',
    #         'net_q_real_t',
    #         'net_q_real_p',
    #         'net_q_fake_list',
    #         'net_g_real',
    #         'net_g_fake_list',
    #         'net_is_adapt_communities_across_trials',
    #         'net_is_estimate_Q',
    #         'net_num_random_Q',
    #         'net_resolution_parameter'
    #     ]

    #     columnnames = [
    #         'experiment_name',
    #         'vpn',                                         # die Versuchspersonennummer
    #         'day',                                                               # DER Trainingstag
    #         'paradigma',                                    # falls an einem Tag unterschiedliche Interventionen erfolgten (MST_21 vs. MST_22 vs. MST_23)
    #         'sequence_length',
    #         'filename',
    #         'root_dir',
    #         'data_dir',
    #         'is_delete_first',                            #! ein wesentlicher Unterschied ist noch, dass beim MST der erste ipi nicht geloescht wird
    #         'all_ipi_lsln',
    #         'cor_ipi_lsln',
    #         'err_ipi_lsln',
    #         'all_hits_lsln',
    #         'all_ipi_lblsln',
    #         'cor_ipi_lblsln',
    #         'err_ipi_lblsln',
    #         'all_hits_lblsln',
    #         'all_ipi_lplsln',
    #         'cor_ipi_lplsln',
    #         'err_ipi_lplsln',
    #         'all_hits_lplsln',
    #         'all_ipi_lplblsln',
    #         'cor_ipi_lplblsln',
    #         'err_ipi_lplblsln',
    #         'all_hits_lplblsln',
    #         'all_seqsum_lpn',
    #         'all_seqsum_lplbn',
    #         'all_seqtimesum_lplsn',
    #         'all_seqtimesum_lplblsn',
    #         'cor_seqsum_lpn',
    #         'cor_seqsum_lplbn',
    #         'cor_seqtimesum_lplsn',
    #         'cor_seqtimesum_lplblsn',
    #         'err_seqsum_lpn',
    #         'err_seqsum_lplbn',
    #         'err_seqtimesum_lplsn',
    #         'err_seqtimesum_lplblsn',
    #         'all_seqtimesum_slope_lpn',
    #         'all_seqtimesum_to_max_slope_lpn',
    #         'cor_seqtimesum_slope_lpn',
    #         'cor_seqtimesum_to_max_slope_lpn',
    #         'err_seqtimesum_slope_lpn',
    #         'err_seqtimesum_to_max_slope_lpn',
    #         'all_seqtimesum_per_block_slope_lpn',
    #         'all_seqtimesum_per_block_to_max_slope_lpn',
    #         'cor_seqtimesum_per_block_slope_lpn',
    #         'cor_seqtimesum_per_block_to_max_slope_lpn',
    #         'err_seqtimesum_per_block_slope_lpn',
    #         'err_seqtimesum_per_block_to_max_slope_lpn',
    #         'all_seqnum_per_block_slope_lpn',
    #         'cor_seqnum_per_block_slope_lpn',
    #         'err_seqnum_per_block_slope_lpn',
    #         'net_A',
    #         'net_C',
    #         'net_c',
    #         'net_ipi',
    #         'net_is_estimate_clustering',
    #         'net_k',
    #         'net_kappa',
    #         'net_m',
    #         'net_my2',
    #         'net_phi',
    #         'net_phi_real',
    #         'net_phi_fake_list',
    #         'net_q_real',
    #         'net_q_real_t',
    #         'net_q_real_p',
    #         'net_q_fake_list',
    #         'net_g_real',
    #         'net_g_fake_list',
    #         'net_is_adapt_communities_across_trials',
    #         'net_is_estimate_Q',
    #         'net_num_random_Q',
    #         'net_resolution_parameter'
    #     ]
    #     df = pd.DataFrame(columns=columnnames)
    #     # perform an loop to fill all experiments to it
    #     for i in range(5):
    #         df.loc[i, 'lib'] = 'name' + str(i)#

    def read_exp_files_from_dir(self):
        """ read all files from one directory and append them to self.all_exp
        """
        files = [os.path.join(self.datadir, f) for f in os.listdir(self.datadir) if os.path.isfile(os.path.join(self.datadir, f))]
        for file in files:
            with open(file, 'rb') as fp:
                self.all_exp.append(pickle.load(fp))

    def create_groups(self):
        """ group the experiment data according to the performed experiment
        """
        # estimate the number of group
        self.experiment_names = []
        for exp in self.all_exp:
            if not(exp.experiment_name in self.experiment_names):
                self.experiment_names.append(exp.experiment_name)
        self.num_groups = len(self.experiment_names)

        for i in range(self.num_groups):
            self.groups.append([])
        for exp in self.all_exp:
            g = self.groups[self.experiment_names.index(exp.experiment_name)]
            g.append(exp)

        self.myprint(f"found {self.num_groups} groups in the given directory")
        #logger_stat.debug(tmp)
        for gr in range(self.num_groups):
            self.myprint(f"Group {gr} with name = {self.experiment_names[gr]} ({len(self.groups[gr])} files)")
            exp = self.groups[gr][0]
            self.myprint(f"number of Paradigma: {len(exp.cor_seqsum_lpn)}")
            self.myprint(f"... with number of blocks:", end=" ")
            for i in range(len(exp.cor_seqsum_lpn)):
                self.myprint(f"{len(exp.all_seqsum_lplbn[i])}", end=" ")
            self.myprint(f" ")

    def check_data_consistency(self):
        """ checking that all experiment of a given type have the same number
             paradigma and blocks
        """

        #logger_stat.debug(tmp)
        self.myprint("checking dataconsistency ...")
        self.myprint("checking that every experiment file of a given type and day has the same number of paradigma")
        for gr in range(self.num_groups):
            subj_counter = 0
            self.myprint("____________________________________")
            self.myprint(f"analysing Group {self.experiment_names[gr]}")
            for exp in self.groups[gr]:
                num_p = len(exp.cor_seqsum_lpn)
                num_b = []
                for i in range(num_p):
                    num_b.append(exp.all_seqsum_lplbn[i])
                if subj_counter>0:

                    if (not(num_p==num_p_old) or not(num_b==num_b_old)):
                        # in SEQ and SRTT there are always the same number of sequences per block
                        if (self.experiment_names[gr]=="SEQ" or self.experiment_names[gr]=="SRTT"):
                            self.myprint("DIFFERENCE")
                        if (self.experiment_names[gr]=="MST"):
                            if not(len(num_b)==len(num_b_old)):
                                self.myprint("DIFFERENCE")
                        #self.myprint(f"difference detected in group{gr} subj{subj_counter}  num_p = {num_p} num_bloecke = {num_b} .... to")
                        #self.myprint(f"difference detected in group{gr} subj{subj_counter-1}  num_p = {num_p_old} num_bloecke = {num_b_old}")
                else:
                    num_p_old = num_p
                    num_b_old = num_b
                self.myprint(f"subj{subj_counter: >4} file={exp.filename: >10} num_p = {num_p} num_bloecke = {num_b:} .... to")

                subj_counter +=1

        self.myprint("checking dataconsistency finished")
        self.myprint("______________________________________________")

    def myprint(self, mystring, end = "\n"):
        if not(end=="\n"):
            mystring.strip("\n")
        if self.is_print_to_std:
            print(mystring, end = end)

        # try:
        #     last_char = self.print_output[-1][-1]
        # except:
        #     last_char = "\n"
        # if last_char == "\n":
        self.print_output.append(mystring + end)
        # else:
        #     self.print_output[-1] = self.print_output[-1] + mystring + end

        if self.first_write:
            with open(os.path.join(self.resultsdir, self.output_filename), "w") as f:
                f.write(mystring)
            self.first_write = False
        else:

            with open(os.path.join(self.resultsdir, self.output_filename), "a") as f:
                f.write(mystring)
                f.write(end)

    """We collected three behavioral variables during training: the time between key presses (i.e.,
    the vector of inter-key intervals), movement time (MT), and error. MT is the time elapsed
    from the initial to final key press. Error was scored as any trial not produced in the correct
    order as well as those trials not completed within the 8 s time limit. To test for learning, we
    entered the MT data for each subject, sequence, and session into a repeated-measures
    ANOVA (with subject treated as a random factor). To test for differences in error over
    training, we combined error for each frequent sequence and entered them for each subject
    and session using a repeated-measures ANOVA. For all statistical tests, we set a probability
    threshold of P < 0.05 for the rejection of the null hypothesis.
        x #      phi
                    phi_real':             self.phi_real,
                    phi_fake_list
        n #     'phi_real_slope':       phi_real_slope,
        y #     'q_real':               self.q_real,
        y #     'q_real_t':             self.q_real_t,
        y #     'q_real_p':             self.q_real_p,
        y #     'q_fake_list':          self.q_fake_list,
        n #     'q_fake_list_mean':     sum(self.q_fake_list)/len(self.q_fake_list),
        y #     'g_real':               self.tolist_ck(self.g_real),
        y #     'g_fake_list':          self.tolist_ck(self.g_fake_list), # arrays verschachtelt in einer Liste
        y #     'A':                    self.A.tolist()
        C
        ipi
        k
        kappa
        m
        my2
            # }
    """

    def estimate_q(self, is_q_fake_abs = True):
        """ is_q_fake_abs whether the difference between shuffeled an real Q is
            used or whether the absolute value will be used
        """
        self.myprint(" ")
        self.myprint("__________________________________________________________")
        self.myprint("________________________Q_________________________________")
        self.myprint("__________________________________________________________")
        self.myprint("We quantified chunking within each sequence by the optimized modularity Qmulti–trial of the")
        self.myprint("sequence networks. Modularity in this case measures the separability between clusters of")
        self.myprint("IKIs. Higher values of Q indicate a greater ease in separating chunks.")
        self.myprint("___________MST_______")
        exp_list = self.filter_experiments(experiment_names=["MST"], days=[1], vpns=[])
        self.group_q(exp_list, "MST Day 1", q_fake_abs = is_q_fake_abs)
        exp_list = self.filter_experiments(experiment_names=["MST"], days=[2], vpns=[])
        self.group_q(exp_list, "MST Day 2", q_fake_abs = is_q_fake_abs)
        self.myprint("___________SRTT_______")
        exp_list = self.filter_experiments(experiment_names=["SRTT"], days=[1], vpns=[])
        self.group_q(exp_list, "SRTT Day 1", q_fake_abs = is_q_fake_abs)
        exp_list = self.filter_experiments(experiment_names=["SRTT"], days=[2], vpns=[])
        self.group_q(exp_list, "SRTT Day 2", q_fake_abs = is_q_fake_abs)
        self.myprint("___________SEQ_______")
        exp_list = self.filter_experiments(experiment_names=["SEQ"], days=[1], vpns=[])
        self.group_q(exp_list, "SEQ Day 1", q_fake_abs = is_q_fake_abs)
        exp_list = self.filter_experiments(experiment_names=["SEQ"], days=[2], vpns=[])
        self.group_q(exp_list, "SEQ Day 2", q_fake_abs = is_q_fake_abs)
        self.myprint("end estimation group q")

    def group_q(self,exp_list, desc, q_fake_abs = False):
        """ in case of q_fake_abs = True then the abs difference will be computed
            after z-score correction
            otherwise the simple difference will be used (this is not really correcte becaus different distributions)
        """
        if not exp_list:
            self.myprint("not experiments for {desc}")
            return
        subj_q = [] # a list
        subj_q_fake = []
        subj_q_fake_z = []
        subj_q_abs = []
        subj_q_z = []
        print(len(exp_list))
        for exp in exp_list:
            subj_q_abs.append(abs(exp.net.Q_list[0] - statistics.mean(exp.net.Q_list[1:])))
            subj_q.append(exp.net.Q_list[0])
            subj_q_fake.append(statistics.mean(exp.net.Q_list[1:]))
            stderr=statistics.stdev(exp.net.Q_list[1:])/sqrt(len(exp.net.Q_list[1:]))
            subj_q_z.append((exp.net.Q_list[0]-statistics.mean(exp.net.Q_list[1:]))/stderr)
            # fuege jeden normalisieren fake wert in Form eines Z Wertes nun ein
            for f in exp.net.Q_list[1:]:
                subj_q_fake_z.append((f-statistics.mean(exp.net.Q_list[1:]))/stderr)
        if q_fake_abs:
            self.myprint(f"Subject z-score for real Q ")
            self.myprint(f"{str(subj_q_z)}")
            self.myprint(f"with mean = {statistics.mean(subj_q_z)}")
            self.myprint(exp.net.print_Q_parts())
            G1 = [abs(elem) for elem in subj_q_z]
            G2 = [abs(elem) for elem in subj_q_fake_z]
            #G2 = abs(subj_q_fake_z)
            #t, p = stats.ttest_ind(G1, G2)
            m = [statistics.mean(G1), statistics.mean(G2)]
            std = [statistics.stdev(G1), statistics.stdev(G2)]
            #self.print_pt_2g(key=desc, t=t,p=p, mymean = m, std=std)
            # data1 = [0.873, 2.817, 0.121, -0.945, -0.055, -1.436, 0.360, -1.478, -1.637, -1.869]
            # data2 = [1.142, -0.432, -0.938, -0.729, -0.846, -0.157, 0.500, 1.183, -1.075, -0.169]
            t, p = mannwhitneyu(G1, G2)

            self.print_pt_2g(key=desc, t=t,p=p, mymean = m, std=std)

#             G1 = np.array(abs(subj_q_z))
#             G2 = np.array(abs(subj_q_fake_z))
# #            G1 = np.array(subj_q_abs)
# #            G1 = subj_q_abs
# #            self.myprint(str(G1))
#             t, p = stats.ttest_1samp(G1,0.0)
#             m = G1.mean()
#             #m = statistics.mean(G1)
#             std = G1.std()
# #           std = statistics.stdev(G1)
#             self.print_pt_1g(desc, t,p, m, std)

        if not q_fake_abs:
            G1 = subj_q
            G2 = subj_q_fake
            t, p = stats.ttest_rel(G1, G2)
            m = [statistics.mean(G1), statistics.mean(G2)]
            std = [statistics.stdev(G1), statistics.stdev(G2)]
            self.print_pt_2g(key=desc, t=t,p=p, mymean = m, std=std)

    def estimate_phi(self):
        pass

    def was_there_learning_in_each_experiment(self):
        self.myprint("Question 1: Was there learning?")
        self.myprint("1.1. is there a linear increase in the number of correct sequences per block per paradigma (linear Regression)?")

        for group_idx in range(len(self.groups)):
            exp_name = self.experiment_names[group_idx]
            if exp_name == "MST":
                self.myprint("___________________________________________________")
                self.myprint("START____MST______________________________________")
                self.myprint("MST")
                self.myprint("1.1. is there a linear increase in the number of correct sequences per block per paradigma (linear Regression)?")
                self.myprint(".... for day 1...")
                exp_list = self.filter_experiments(experiment_names=["MST"], days=[1], vpns=[])
                self.estimate_linear_regression(exp_list, "cor_seqsum_lplbn", 0, mean_last_dim = False, num_blocks=10)
                self.myprint(".... for day 2...")
                exp_list = self.filter_experiments(experiment_names=["MST"], days=[2], vpns=[])
                self.estimate_linear_regression(exp_list, "cor_seqsum_lplbn", 0, mean_last_dim = False, num_blocks=10)
                self.myprint("1.1. is there a linear decrease in the Time to perform an sequence per block per paradigma (linear Regression)?")
                self.myprint(".... for day 1...")
                exp_list = self.filter_experiments(experiment_names=["MST"], days=[1], vpns=[])
                self.estimate_linear_regression(exp_list, "cor_seqtimesum_lplblsn", 0, mean_last_dim = True, num_blocks=10)
                self.myprint(".... for day 2...")
                exp_list = self.filter_experiments(experiment_names=["MST"], days=[2], vpns=[])
                self.estimate_linear_regression(exp_list, "cor_seqtimesum_lplblsn", 0, mean_last_dim = True, num_blocks=10)
                self.myprint("now use the slope of each subject and perform a t-test for the group")
                exp_list = self.filter_experiments(experiment_names=["MST"], days=[1], vpns=[])
                self.ttest_one_group(exp_list, "cor_seqtimesum_slope_lpn", 0)
                exp_list = self.filter_experiments(experiment_names=["MST"], days=[2], vpns=[])
                self.ttest_one_group(exp_list, "cor_seqtimesum_slope_lpn", 0)
                self.myprint("END____MST______________________________________")
                self.myprint("___________________________________________________")
                self.myprint(" ")
            if exp_name == "SEQ":
                self.myprint("___________________________________________________")
                self.myprint("START____SEQ______________________________________")
                self.myprint("SEQ")
                # self.myprint("1.1. is there a linear decrease in the number of correct sequences per block per paradigma (linear Regression)?")
                # self.myprint(".... for day 1...")
                # exp_list = self.filter_experiments(experiment_names=["SEQ"], days=[1], vpns=[])
                # self.estimate_linear_regression(exp_list, "cor_seqsum_lplbn", 0)
                # self.myprint(".... for day 2...")
                # exp_list = self.filter_experiments(experiment_names=["SEQ"], days=[2], vpns=[])
                # self.estimate_linear_regression(exp_list, "cor_seqsum_lplbn", 0)
                self.myprint("1.1. is there a linear decrease in the Time to perform an sequence per block per paradigma (linear Regression)?")
                self.myprint(".... for day 1...")
                exp_list = self.filter_experiments(experiment_names=["SEQ"], days=[1], vpns=[])
                print([e.filename for e in exp_list])
                self.estimate_linear_regression(exp_list, "cor_seqtimesum_lplblsn", 0, mean_last_dim = True, num_blocks=10)
                self.myprint(".... for day 2...")
                exp_list = self.filter_experiments(experiment_names=["SEQ"], days=[2], vpns=[])
                print([e.filename for e in exp_list])
                self.estimate_linear_regression(exp_list, "cor_seqtimesum_lplblsn", 0, mean_last_dim = True, num_blocks=10)
                self.myprint("now use the slope of each subject and perform a t-test for the group")
                exp_list = self.filter_experiments(experiment_names=["SEQ"], days=[1], vpns=[])
                self.ttest_one_group(exp_list, "cor_seqtimesum_slope_lpn", 0)
                exp_list = self.filter_experiments(experiment_names=["SEQ"], days=[2], vpns=[])
                self.ttest_one_group(exp_list, "cor_seqtimesum_slope_lpn", 0)
                self.myprint("END____SEQ______________________________________")
                self.myprint("___________________________________________________")
                self.myprint(" ")
            if exp_name == "SRTT":
                self.myprint("___________________________________________________")
                self.myprint("START____SRTT______________________________________")
                self.myprint("SRTT")
                self.myprint("1.1. is there a linear decrease in the Time to perform an sequence per block per paradigma (linear Regression)?")
                self.myprint(".... for day 1...")
                exp_list = self.filter_experiments(experiment_names=["SRTT"], days=[1], vpns=[])
                self.estimate_linear_regression(exp_list, "cor_seqtimesum_lplblsn", 0, mean_last_dim = True, num_blocks=6)
                self.myprint(".... for day 2...")
                exp_list = self.filter_experiments(experiment_names=["SRTT"], days=[2], vpns=[])
                self.estimate_linear_regression(exp_list, "cor_seqtimesum_lplblsn", 0, mean_last_dim = True, num_blocks=6)
                self.myprint("1.1. is there a linear decrease in the Time to perform an sequence per block per paradigma (linear Regression) ALLL?")
                self.myprint(".... for day 1...")
                exp_list = self.filter_experiments(experiment_names=["SRTT"], days=[1], vpns=[])
                self.estimate_linear_regression(exp_list, "all_seqtimesum_lplblsn", 0, mean_last_dim = True, num_blocks=6)
                self.myprint(".... for day 2...")
                exp_list = self.filter_experiments(experiment_names=["SRTT"], days=[2], vpns=[])
                self.estimate_linear_regression(exp_list, "all_seqtimesum_lplblsn", 0, mean_last_dim = True, num_blocks=6)

                self.myprint("now use the slope of each subject and perform a t-test for the group")
                exp_list = self.filter_experiments(experiment_names=["SRTT"], days=[1], vpns=[])
                self.ttest_one_group(exp_list, "cor_seqtimesum_slope_lpn", 0)
                exp_list = self.filter_experiments(experiment_names=["SRTT"], days=[2], vpns=[])
                self.ttest_one_group(exp_list, "cor_seqtimesum_slope_lpn", 0)
                self.myprint("END____SRTT______________________________________")
                self.myprint("_________________________________________________")
                self.myprint(" ")

    def ttest_one_group(self, exp_list, key, paradigma, mean_last_dim = False, is_independent = True):
        # reduce a list of lists to a list by averaging
        subj_values = [] # a list
        for exp in exp_list:
            v = getattr(exp,key)
            t = v[paradigma][0]

            if mean_last_dim:
                t = self.list_of_list_to_list(t)
            subj_values.append(t)

        G1 = np.array(subj_values)

        # G1 = self.list_of_list_to_list(data[0])
        # G2 = self.list_of_list_to_list(data[1])
        self.myprint(str(G1))
        t, p = stats.ttest_1samp(G1,0.0)
        m = G1.mean()
        #m = statistics.mean(G1)
        std = G1.std()
#        std = statistics.stdev(G1)

        self.print_pt_1g(key, t,p, m, std)

    def print_pt_1g(self,key, t,p, mymean, std):
        self.myprint(f"{key} t-test different from 0  p={p} t={t}, m={mymean}, std={std}")

    def estimate_linear_regression(self, exp_list, key, paradigma, mean_last_dim = False, num_blocks=6):
        """ performing linear regression with statsmodels"""
        subj_values = [] # a list

        for exp in exp_list:
            v = getattr(exp,key)
            t = v[paradigma]
            print(f"the data of file {exp.filename}  and parameter={key}... {t}")
            if mean_last_dim:
                t = self.list_of_list_to_list(t)
            if len(t)==(num_blocks-1):
                t.append(mean(t))
            subj_values.append(t)
            # print(len(t))
            # print(exp.experiment_name)
            # print(exp.filename)

        X = np.array(subj_values)
        print(X)
        #Xm = np.mean(X,axis=0)
        Xm = np.sum(X,axis=0)
        Xm =sm.add_constant(Xm, prepend=False)
        print(Xm)
        y = np.arange(0,Xm.shape[0],1)
        mod = sm.OLS(y,Xm)
        self.myprint("Regression across the mean of all subjects for each blocks ...")
        self.myprint(str(Xm))
        res = mod.fit()
        self.myprint(res.summary().as_text())

    def test_group_differences_ttest(self, key, is_independent, paradigma, level):
        # ich laufe ueber die Buchstaben der levelbeschreibung und ziehe die richtigen Daten heraus
        values = self.get_target_values_by_key(key)
        for i in range(len(level)):
            if level[i]== 'p':
                values = self.filter_target_values_by_paradigma(values, paradigma)
            if level[i]=='n':
                self.test_group_differences_two_groups(key, values, is_independent=is_independent)

#        d = self.get_target_values_by_key_and_level(key, paradigma, level)
 #       if len(d)==2:

    def get_target_values_by_key(self, key):
        """ get the target attributes out of the experiment objects
            and put these into a list for each group
        """
        target_val = []
        for group in self.groups:
            subject_list = []
            for subj_exp in group.subj_exp_list:
                exp_value = getattr(subj_exp, key)
                subject_list.append(exp_value)
            target_val.append(subject_list)
        return target_val

    def filter_target_values_by_paradigma(self,values, paradigma):
        new_val = []
        for attribute_list in values:
            new_attribute_list = []
            for attribute in attribute_list:
                new_attribute_list.append(attribute[paradigma])
            new_val.append(new_attribute_list)
        return new_val

    def list_of_list_to_list(self, input_list):
        # if input_list is a list of list then it will be transformed to a list
        # by averaging the second dimension
        if any(isinstance(el, list) for el in input_list):
            input_list = [statistics.mean(f) for f in input_list]
        return input_list

    def test_group_differences_two_groups(self, key, data, is_independent=True):
        # reduce a list of lists to a list by averaging
        G1 = self.list_of_list_to_list(data[0])
        G2 = self.list_of_list_to_list(data[1])

        if is_independent:
            t, p = stats.ttest_ind(G1, G2)
        else:
            t, p = stats.ttest_rel(G1, G2)
        m = [statistics.mean(G1), statistics.mean(G2)]
        std = [statistics.stdev(G1), statistics.stdev(G2)]

        self.print_pt_2g(key=key, t=t,p=p, mymean = m, std=std)

    def get_target_values_by_key_level_1(self, key, paradigma):
        # extracts from the dictionaries of all groups the value
        # with key = key
        # returns a list with groups, the group list consists of a list with the key elements
        target_val = []
        for group in self.groups:
            subject_list = []
            for subj_exp in group.subj_exp_list:
                exp_value = getattr(subj_exp, key)[paradigma]
                print(exp_value)
                subject_list.append(exp_value)
            target_val.append(subject_list)
        return target_val

    def print_pt_2g(self, key, t, p, mymean=0, std=0):
        mymean = [float(m) for m in mymean]
        std = [float(s) for s in std]
        self.myprint(f"{key} p = {p:.7}  with t = {t:.3}  (mean = {mymean[0]:.3} +- {std[0]:.4}  vs. {mymean[1]:.3} +- {std[1]:.4}")

    def show_group_differences(self, key):
        data = self.get_target_values_by_key(key)
        data = np.asarray(data)
        print(f"Group Results of {key}")
        df = pd.DataFrame(data.T, columns = [self._ids[0], self._ids[1]])
        print(df.head(30))

    def plot_one_group_sequence(self, key):
        data = self.get_target_values_by_key(key)
        #print(data)
        #print("---")
        data = data[0]
        #print(np.asarray(data))
        #print(data)
        #print(data.shape)
        for subj in data:
            plt.plot(subj)

        plt.show()
        # plt.plot( 'x', 'y1', data=data, marker='o', markerfacecolor='blue', markersize=12, color='skyblue', linewidth=4)
        # plt.plot( 'x', 'y2', data=df, marker='', color='olive', linewidth=2)
        # plt.plot( 'x', 'y3', data=df, marker='', color='olive', linewidth=2, linestyle='dashed', label="toto")
        # plt.legend()

if __name__ == "__main__":
    # seed random number generator
    experiment_name = 'MST'
    experiment_name = 'ASTEROID'
    _ids = ["MST_G1_", "MST_G2_"]
    _ids = ["ASTEROID_G1_", "ASTEROID_G2_"]
    my_stat = Statistic(experiment = experiment_name, group_list= [], data_path = ".\\Data_python", _ids = _ids)
    #my_stat.test_group_differences_ttest('corrsq_slope')
    my_stat.test_group_differences_ttest('success_per_block_slope', is_independent=False)
    my_stat.test_group_differences_ttest('abs_success', is_independent=False)
    my_stat.show_group_differences('abs_success')
    my_stat.show_group_differences('success_per_block_slope')


from itertools import permutations
from collections import deque
from copy import deepcopy

def solution(expression):
    answer = 0
    operand = ['+','-','*']
    oprlist = list(permutations(operand, 3))

    originalqueue = deque([])
    tmp = ''
    for e in expression:
        if e.isdigit():
            tmp += e
        else:
            originalqueue.append(tmp)
            originalqueue.append(e)
            tmp = ''
    originalqueue.append(tmp)

    for opr in oprlist:
        expqueue = deepcopy(originalqueue)
        for o in opr:
            newqueue = deque([])
            while expqueue:
                now = expqueue.popleft()
                if now != o:
                    newqueue.append(now)
                else:
                    beforenum = newqueue.pop()
                    afternum = expqueue.popleft()
                    newqueue.append(str(eval(beforenum+o+afternum)))
            expqueue = deepcopy(newqueue)
        res = abs(int(newqueue[0]))
        if answer < res:
            answer = res

    return answer

print(solution("100-200*300-500+20"	))


"""
Written by Ben Bowes, Oct., 2018

This script reads all the bootstrap forecast performance result files, plots histograms, and calculates averages.
t-tests are done to compute p-values and confidence intervals are computed
"""

import pandas as pd
import numpy as np
import os
import matplotlib.pyplot as plt
import matplotlib
from scipy import stats

matplotlib.rcParams.update({'font.size': 8})

well_list = ["043", "125", "129", "153", "155", "170", "175"]
# well_list = ["043"]

for well in well_list:  # loop through all wells
    # specify folder locations
    out_folder = "C:/Users/Ben Bowes/PycharmProjects/Tensorflow/rnn_lstm_comparison_results_fcst/mmps" + well

    if not os.path.exists(out_folder):
        os.makedirs(out_folder)
        print("created new directory: %s" % out_folder)

    rnn_full_results_folder = "C:/Users/Ben Bowes/PycharmProjects/Tensorflow/Rivanna_results_fcst/mmps" + well +\
                              "_results_full_bootstrap_fcst_rnn/"
    lstm_full_results_folder = "C:/Users/Ben Bowes/PycharmProjects/Tensorflow/Rivanna_results_fcst/mmps" + well +\
                               "_results_full_bootstrap_fcst_lstm/"
    rnn_storms_results_folder = "C:/Users/Ben Bowes/PycharmProjects/Tensorflow/Rivanna_results_fcst/mmps" + well +\
                                "_results_storm_bootstrap_fcst_rnn/"
    lstm_storms_results_folder = "C:/Users/Ben Bowes/PycharmProjects/Tensorflow/Rivanna_results_fcst/mmps" + well +\
                                 "_results_storm_bootstrap_fcst_lstm/"

    folder_list = [rnn_full_results_folder, lstm_full_results_folder, rnn_storms_results_folder,
                   lstm_storms_results_folder]

    rmse_df_list = []
    nse_df_list = []
    mae_df_list = []
    rmse_storms_df_list = []
    nse_storms_df_list = []
    mae_storms_df_list = []

    for folder in folder_list:
        folder_name1 = folder.split("/")[6].split("_")[2]
        folder_name2 = folder.split("/")[6].split("_")[5]
        folder_name = folder_name1 + "_" + folder_name2
        print(folder_name)

        rmse_t1_list, rmse_t9_list, rmse_t18_list = [], [], []
        nse_t1_list, nse_t9_list, nse_t18_list = [], [], []
        mae_t1_list, mae_t9_list, mae_t18_list = [], [], []

        rmse_storms_t1_list, rmse_storms_t9_list, rmse_storms_t18_list = [], [], []
        nse_storms_t1_list, nse_storms_t9_list, nse_storms_t18_list = [], [], []
        mae_storms_t1_list, mae_storms_t9_list, mae_storms_t18_list = [], [], []

        count = 0
        for file in os.listdir(folder):  # extract forecast data
            if count % 100 == 0:
                print(folder, "count is", count)
            data = folder + file
            if file.endswith("_RMSE.csv"):
                # print(file)
                rmse_df = pd.read_csv(data)
                rmse_t1, rmse_t9, rmse_t18 = rmse_df[["0"]].iloc[0], rmse_df[["0"]].iloc[8], rmse_df[["0"]].iloc[17]
                rmse_t1_list.append(rmse_t1[0])
                rmse_t9_list.append(rmse_t9[0])
                rmse_t18_list.append(rmse_t18[0])
            if file.endswith("_NSE.csv"):
                nse_df = pd.read_csv(data)
                nse_t1, nse_t9, nse_t18 = nse_df[["0"]].iloc[0], nse_df[["0"]].iloc[8], nse_df[["0"]].iloc[17]
                nse_t1_list.append(nse_t1[0])
                nse_t9_list.append(nse_t9[0])
                nse_t18_list.append(nse_t18[0])
            if file.endswith("_MAE.csv"):
                mae_df = pd.read_csv(data)
                mae_t1, mae_t9, mae_t18 = mae_df[["0"]].iloc[0], mae_df[["0"]].iloc[8], mae_df[["0"]].iloc[17]
                mae_t1_list.append(mae_t1[0])
                mae_t9_list.append(mae_t9[0])
                mae_t18_list.append(mae_t18[0])
            count += 1

        # write extracted data to data frames
        folder_RMSE_df = pd.DataFrame([rmse_t1_list, rmse_t9_list, rmse_t18_list]).transpose()
        folder_RMSE_df.columns = [(folder_name + "_t+1"), (folder_name + "_t+9"), (folder_name + "_t+18")]
        # print("folder rmse df", folder_RMSE_df.head())
        folder_NSE_df = pd.DataFrame([nse_t1_list, nse_t9_list, nse_t18_list]).transpose()
        folder_NSE_df.columns = [(folder_name + "_t+1"), (folder_name + "_t+9"), (folder_name + "_t+18")]
        folder_MAE_df = pd.DataFrame([mae_t1_list, mae_t9_list, mae_t18_list]).transpose()
        folder_MAE_df.columns = [(folder_name + "_t+1"), (folder_name + "_t+9"), (folder_name + "_t+18")]

        # append folder dataframes to lists
        rmse_df_list.append(folder_RMSE_df)
        nse_df_list.append(folder_NSE_df)
        mae_df_list.append(folder_MAE_df)

    # concat data to well dfs
    rmse_df = pd.concat(rmse_df_list, axis=1)
    rmse_df = rmse_df[:1000]
    nse_df = pd.concat(nse_df_list, axis=1)
    nse_df = nse_df[:1000]
    mae_df = pd.concat(mae_df_list, axis=1)
    mae_df = mae_df[:1000]

    # save well dfs
    rmse_df.to_csv(os.path.join(out_folder, "rmse_df.csv"), index=False)
    nse_df.to_csv(os.path.join(out_folder, "nse_df.csv"), index=False)
    mae_df.to_csv(os.path.join(out_folder, "mae_df.csv"), index=False)

    # plot histograms of RMSE for individual well, need to get same axis values
    col_list = rmse_df.columns

    plt.figure(1, figsize=(6, 9))
    for i in range(0, len(col_list), 1):
        ax = plt.subplot(4, 3, i+1)
        rmse_df.hist(ax=ax, column=col_list[i], bins=15, grid=False, color='k')
        bs_type = col_list[i].split("_")[0]
        model_type = col_list[i].split("_")[1]
        ax.set_title("")
        # ax.set_xlim(0, 0.25)
        # ax.set_ylim(0, 400)
        if i % 3 == 0:
            ax.set_ylabel(bs_type + " " + model_type)
        if i < 3:
            ax.set_title(col_list[i].split("_")[2])

    # for i in range(0, len(col_list), 1):
    #     if i == 0:
    #         ax = plt.subplot(6, 3, i+1)
    #     else:
    #         ax = plt.subplot(6, 3, i+1, sharex=ax)
    #     rmse_df.hist(ax=ax, column=col_list[i], bins=15)
    #     bs_type = col_list[i].split("_")[0]
    #     model_type = col_list[i].split("_")[1]
    #     ax.set_title("")
    #     if i % 3 == 0:
    #         ax.set_ylabel(bs_type + " " + model_type)
    #     if i < 3:
    #         ax.set_title(col_list[i].split("_")[2])
    plt.tight_layout()
    plt.gcf().text(0.5, 0.05, "RMSE (m)")
    plt.subplots_adjust(bottom=0.1)
    # plt.show()
    plt.savefig(os.path.join(out_folder, "rmse_hists.png"), dpi=300)
    plt.close()

    # perform t-tests
    rnn_full_storm_tvalues, rnn_full_storm_pvalues = [], []
    lstm_full_storm_tvalues, lstm_full_storm_pvalues = [], []
    rnn_lstm_full_tvalues, rnn_lstm_full_pvalues = [], []
    rnn_lstm_storm_tvalues, rnn_lstm_storm_pvalues = [], []

    rnn_full_storm_t1_t, rnn_full_storm_t1_p = stats.ttest_ind(rmse_df["full_rnn_t+1"], rmse_df["storm_rnn_t+1"])
    rnn_full_storm_t9_t, rnn_full_storm_t9_p = stats.ttest_ind(rmse_df["full_rnn_t+9"], rmse_df["storm_rnn_t+9"])
    rnn_full_storm_t18_t, rnn_full_storm_t18_p = stats.ttest_ind(rmse_df["full_rnn_t+18"], rmse_df["storm_rnn_t+18"])
    rnn_full_storm_tvalues.append([rnn_full_storm_t1_t, rnn_full_storm_t9_t, rnn_full_storm_t18_t])
    rnn_full_storm_pvalues.append([rnn_full_storm_t1_p, rnn_full_storm_t9_p, rnn_full_storm_t18_p])

    lstm_full_storm_t1_t, lstm_full_storm_t1_p = stats.ttest_ind(rmse_df["full_lstm_t+1"], rmse_df["storm_lstm_t+1"])
    lstm_full_storm_t9_t, lstm_full_storm_t9_p = stats.ttest_ind(rmse_df["full_lstm_t+9"], rmse_df["storm_lstm_t+9"])
    lstm_full_storm_t18_t, lstm_full_storm_t18_p = stats.ttest_ind(rmse_df["full_lstm_t+18"],rmse_df["storm_lstm_t+18"])
    lstm_full_storm_tvalues.append([lstm_full_storm_t1_t, lstm_full_storm_t9_t, lstm_full_storm_t18_t])
    lstm_full_storm_pvalues.append([lstm_full_storm_t1_p, lstm_full_storm_t9_p, lstm_full_storm_t18_p])

    rnn_lstm_full_t1_t, rnn_lstm_full_t1_p = stats.ttest_ind(rmse_df["full_rnn_t+1"], rmse_df["full_lstm_t+1"])
    rnn_lstm_full_t9_t, rnn_lstm_full_t9_p = stats.ttest_ind(rmse_df["full_rnn_t+9"], rmse_df["full_lstm_t+9"])
    rnn_lstm_full_t18_t, rnn_lstm_full_t18_p = stats.ttest_ind(rmse_df["full_rnn_t+18"], rmse_df["full_lstm_t+18"])
    rnn_lstm_full_tvalues.append([rnn_lstm_full_t1_t, rnn_lstm_full_t9_t, rnn_lstm_full_t18_t])
    rnn_lstm_full_pvalues.append([rnn_lstm_full_t1_p, rnn_lstm_full_t9_p, rnn_lstm_full_t18_p])

    rnn_lstm_storm_t1_t, rnn_lstm_storm_t1_p = stats.ttest_ind(rmse_df["storm_rnn_t+1"], rmse_df["storm_lstm_t+1"])
    rnn_lstm_storm_t9_t, rnn_lstm_storm_t9_p = stats.ttest_ind(rmse_df["storm_rnn_t+9"], rmse_df["storm_lstm_t+9"])
    rnn_lstm_storm_t18_t, rnn_lstm_storm_t18_p = stats.ttest_ind(rmse_df["storm_rnn_t+18"], rmse_df["storm_lstm_t+18"])
    rnn_lstm_storm_tvalues.append([rnn_lstm_storm_t1_t, rnn_lstm_storm_t9_t, rnn_lstm_storm_t18_t])
    rnn_lstm_storm_pvalues.append([rnn_lstm_storm_t1_p, rnn_lstm_storm_t9_p, rnn_lstm_storm_t18_p])

    # save t-test results to dataframe
    ttest_cols = ["rnn_full_storm_t", "rnn_full_storm_p", "lstm_full_storm_t", "lstm_full_storm_p",
                  "rnn_lstm_full_t", "rnn_lstm_full_p", "rnn_lstm_storm_t", "rnn_lstm_storm_p"]

    ttest_df = pd.DataFrame([rnn_full_storm_tvalues[0], rnn_full_storm_pvalues[0],
                             lstm_full_storm_tvalues[0], lstm_full_storm_pvalues[0],
                             rnn_lstm_full_tvalues[0], rnn_lstm_full_pvalues[0],
                             rnn_lstm_storm_tvalues[0], rnn_lstm_storm_pvalues[0]]).transpose()
    ttest_df.columns = ttest_cols
    ttest_df["forecast"] = ["t+1", "t+9", "t+18"]
    ttest_df = ttest_df.set_index("forecast")

    ttest_df.to_csv(os.path.join(out_folder, "ttest.csv"))

    # calculate means
    mean_list = []
    for i in col_list:
        col_mean = rmse_df[i].mean()
        mean_list.append(col_mean)

    mean_df = pd.DataFrame(mean_list).transpose()
    mean_df.columns = col_list

    mean_df.to_csv(os.path.join(out_folder, "means.csv"), index=False)

    # calculate confidence intervals
    upper_ci_list = []
    lower_ci_list = []
    for i in col_list:
        col_ci = stats.t.interval(0.95, len(rmse_df[i]) - 1, loc=np.mean(rmse_df[i]), scale=stats.sem(rmse_df[i]))
        upper_ci_list.append(col_ci[1])
        lower_ci_list.append(col_ci[0])

    # calculate error
    # rnn_rmse_t1_err = rnn_rmse_t1_mean - rnn_rmse_t1_ci[0]

    # save CIs to df
    ci_df = pd.DataFrame([lower_ci_list, upper_ci_list], columns=col_list, index=["lower", "upper"])
    ci_df.to_csv(os.path.join(out_folder, "CIs.csv"))


from typing import List

class Solution:
    def cellsInRange(self, s: str) -> List[str]:
        cells = s.split(':')
        col1 = ord(cells[0][0])
        row1 = int(cells[0][1])
        col2 = ord(cells[1][0])
        row2 = int(cells[1][1])
        output = []
        for i in range(col2-col1+1):
            for j in range(row2-row1+1):
                output.append(chr(col1+i)+''+str(row1+j))
        return output

input = ["K1:L2", "A1:F1"]
output = [["K1", "K2", "L1", "L2"], ["A1", "B1", "C1", "D1", "E1", "F1"]]
for i in range(len(output)):
    r = Solution().cellsInRange(input[i])
    if str(r) != str(output[i]):
        raise Exception('Failed: ' + str(input[i]) + ' ---- Got: ' + str(r) + ' !== ' + str(output[i]))
    print('Passed input: ' + str(input[i]))


# -*- coding: utf-8 -*-

import argparse
import contextlib
import getpass
import os
import sys
import urllib.parse

import filelock

from ..core._imperative_rt import PersistentCache as _PersistentCache
from ..logger import get_logger
from ..version import __version__, git_version

class PersistentCacheOnServer(_PersistentCache):
    def __init__(self):
        super().__init__()
        cache_type = os.getenv("MGE_FASTRUN_CACHE_TYPE")
        if cache_type not in ("FILE", "MEMORY"):
            try:
                redis_config = self.get_redis_config()
            except Exception as exc:
                get_logger().error(
                    "failed to connect to cache server {!r}; try fallback to "
                    "in-file cache".format(exc)
                )
            else:
                if redis_config is not None:
                    self.add_config(
                        "redis",
                        redis_config,
                        "fastrun use redis cache",
                        "failed to connect to cache server",
                    )
        if cache_type != "MEMORY":
            path = self.get_cache_file(self.get_cache_dir())
            self.add_config(
                "in-file",
                {"path": path},
                "fastrun use in-file cache in {}".format(path),
                "failed to create cache file in {}".format(path),
            )
        self.add_config(
            "in-memory",
            {},
            "fastrun use in-memory cache",
            "failed to create in-memory cache",
        )

    def get_cache_dir(self):
        cache_dir = os.getenv("MGE_FASTRUN_CACHE_DIR")
        if not cache_dir:
            from ..hub.hub import _get_megengine_home

            cache_dir = os.path.expanduser(
                os.path.join(_get_megengine_home(), "persistent_cache")
            )
        os.makedirs(cache_dir, exist_ok=True)
        return cache_dir

    def get_cache_file(self, cache_dir):
        cache_file = os.path.join(cache_dir, "cache.bin")
        with open(cache_file, "a"):
            pass
        return cache_file

    @contextlib.contextmanager
    def lock_cache_file(self, cache_dir):
        lock_file = os.path.join(cache_dir, "cache.lock")
        with filelock.FileLock(lock_file):
            yield

    def get_redis_config(self):
        url = os.getenv("MGE_FASTRUN_CACHE_URL")
        if url is None:
            return None
        assert sys.platform != "win32", "redis cache on windows not tested"
        prefix = "mgbcache:{}:MGB{}:GIT:{}::".format(
            getpass.getuser(), __version__, git_version
        )
        parse_result = urllib.parse.urlparse(url)
        assert not parse_result.username, "redis conn with username unsupported"
        if parse_result.scheme == "redis":
            assert parse_result.hostname and parse_result.port, "invalid url"
            assert not parse_result.path
            config = {
                "hostname": parse_result.hostname,
                "port": str(parse_result.port),
            }
        elif parse_result.scheme == "redis+socket":
            assert not (parse_result.hostname or parse_result.port)
            assert parse_result.path
            config = {
                "unixsocket": parse_result.path,
            }
        else:
            assert False, "unsupported scheme"
        if parse_result.password is not None:
            config["password"] = parse_result.password
        config["prefix"] = prefix
        return config

    def flush(self):
        if self.config is not None and self.config.type == "in-file":
            with self.lock_cache_file(self.get_cache_dir()):
                super().flush()

def _clean():
    nr_del = PersistentCacheOnServer().clean()
    if nr_del is not None:
        print("{} cache entries deleted".format(nr_del))

def main():
    parser = argparse.ArgumentParser(description="manage persistent cache")
    subp = parser.add_subparsers(description="action to be performed", dest="cmd")
    subp.required = True
    subp_clean = subp.add_parser("clean", help="clean all the cache of current user")
    subp_clean.set_defaults(action=_clean)
    args = parser.parse_args()
    args.action()

if __name__ == "__main__":
    main()


from pymongo import MongoClient
from datetime import datetime
START = datetime(2016,10,1)
END = datetime(2016,11,1)
client=MongoClient('10.8.8.111',27017)
db = client.userValue
collection=db.cache
a = collection.aggregate([{'$match':{'201610':{'$exists':True}}}])
num_gao=0
num_vip=0
for i in a:
    num_gao=num_gao+1
    if(i['201610']['vip']==True):
        num_vip=num_vip+1
print(num_gao)
print(num_vip)


from bs4 import BeautifulSoup
from urllib.parse import urlsplit
from os.path import basename
import pabu_modules.aass as aass
import pabu_modules.logger as log
import re, errno, urllib.request, os, requests, shutil

# function to specifically download the index file
def download_index(url, site_directory, mobile_device):
	site_directory = re.sub(r'\/$', '', site_directory)
	link_file_path = site_directory + '/index.html'

	create_directory(site_directory)
	download_from_url(url, link_file_path, mobile_device)
	return True

def download_asset(asset_url, file_path, mobile_device):
	download_from_url(asset_url, file_path, mobile_device)

# function to download a single file and save it to a specified fiepath
def download_from_url(url, file_path, mobile_device):
	log.log("Downloading URL | \"" + url + "\"", "url_download_attempt")
	if mobile_device == "false":
		ua = 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:55.0) Gecko/20100101 Firefox/55.0'
	else:
		ua = 'Mozilla/5.0 (Linux; <Android Version>; <Build Tag etc.>) AppleWebKit/<WebKit Rev> (KHTML, like Gecko) Chrome/<Chrome Rev> Mobile Safari/<WebKit Rev>'
	# set user agent to prevent 403 errors
	opener = urllib.request.build_opener()
	opener.addheaders = [('User-agent', ua)]
	urllib.request.install_opener(opener)
	# try and download asset
	try:
		urllib.request.urlretrieve(url, file_path)
	# raise errors
	except urllib.error.HTTPError as e:
		if e.code == 404:
			log.log("404 Error - File not found | \"" + url + "\"", "error")
			return False
		elif e.code == 500:
			log.log("500 Error - Internal Server Error | \"" + url + "\"", "error")
			return False
		else:
			raise e
	except urllib.error.URLError as eUrl:
		log.log("URL Error -  | \"" + url + "\"", "URL-Error")
		return False
	except Exception as ex:
		raise ex

	return True

def determine_file_path(asset_url, site_directory):
	folder = site_directory + aass.determine_storage_location(asset_url)
	filename = urlsplit(asset_url).path
	file_path = folder + '\\' + basename(filename)
	create_directory(folder)
	return file_path

# function to create a directory
def create_directory(directory):
	# try and make new directory
	try:
		os.makedirs(directory)
	# raise error if error not errno.EExist
	except OSError as e:
		if e.errno != errno.EEXIST:
			raise

	return True

def find_assets(file_path):
	# parse the HTML with BeautifulSoup package
	data = open(file_path, 'r', encoding="utf8")
	print ("Making soup")
	soup = BeautifulSoup(data, "html.parser")

	# store all link tags in variable
	print ("Getting the ingredients")
	asset_tags = soup.find_all("link")

	# loop through all script tags and push to asset_tags variable (array)
	for a in soup.find_all("script"):
		asset_tags.append(a)

	# loop through all img tags and push to asset_tags variable (array)
	for a in soup.find_all("img"):
		asset_tags.append(a)

	# loop through all div tags and push to asset_tags variable (array)
	for a in soup.find_all("div"):
		asset_tags.append(a)

	# loop through all source tags and push to asset_tags variable (array)
	for a in soup.find_all("source"):
		asset_tags.append(a)

	return asset_tags

def find_asset_urls(asset_tags):
	# empyt array to store downloaded asset hrefs, this is checked to ensure assets are not downloaded more than once
	downloaded_assets = []
	asset_urls = []
	# loop through all stored tags in asset_tags variable
	for asset in asset_tags:
		# loop through attributes and match regex for any known and desired filetype
		for attr in asset.attrs:
			# set href variable to false because setting it in the else statement caused errors
			href = False
			# store attribute value in variable
			val = str(asset.get(attr))
			# check if attribute value contains known, desirable file extensions (using regex)
			if re.search(r"\.(ico|png|webP|jpg|jpeg|gif|bmp|js|css|scss|sass|woff|svg|json|pdf|txt)", val):
				href = val
				log.log('Found Asset | [' + attr + '] = ' + href, 'found_assets')
			# if href not equal to false
			if href != False:
				# strip all unneccesary chars from href
				old_ref = href
				href = aass.purify_url_string(href)
				log.log('Confirmed Asset Purified | [' + old_ref + '] = ' + href, 'confirmed_assets--purified')
				# add href to downloaded_assets
				asset_urls.append(href)

	return asset_urls

def build_url(asset_url, base_url):
	log.log("Received Asset URL | \"" + asset_url + "\"", "asset_url_builder")
	if re.search(r"^\/\/", asset_url):
		return 'https:' + asset_url
	elif re.search(r"^\/", asset_url):
		return base_url + re.sub(r"^\/", '', asset_url)
	elif re.search(r"^http", asset_url):
		return asset_url
	elif re.search(r"\.\.\/", asset_url):
		return base_url + re.sub(r"\.\.\/", '', asset_url)
	elif re.search(r"^[a-z]", asset_url) and asset_url[0:3] != "http":
		return base_url + re.sub(r"^\/", '', asset_url)
	else:
		return "https://"+asset_url

def grab_images_from_css(filepath):
	css_asset_links = []
	file = open(filepath, "r", encoding="utf8")
	images = re.findall('url\(([^)]+)\)', file.read())
	file.close()
	for link in images:
		if len(re.findall(";base64,", link)) == 0 and link not in css_asset_links:
			pure_link = aass.purify_url_string(link)
			css_asset_links.append(pure_link)

	return css_asset_links

def update_link(old_link, new_link, site_directory, file):
	# log message to console
	new_link = new_link.replace('\\', '/')
	site_directory = site_directory.replace('\\', '/')
	log.log("Saving Asset | \"" + old_link + "\" [as] \"" + new_link + "\"", "asset_storage")
	if site_directory[-1:] == "/":
		site_directory = site_directory[:-1]

	# try opening file with read permissions
	try:
		index = open(file, 'r', encoding="utf8")
	# raise errors
	except Exception as e:
		raise e
	# store file content and replace original asset link with local equivalent
	new_content = index.read().replace(old_link, new_link)
	new_content = new_content.replace(site_directory, '')
	# close index file
	index.close()
	# reopen with write permissions
	index = open(file, 'w', encoding="utf8")
	# replace file content with new version
	index.write(new_content)
	# close file
	index.close()
	return True

# function to generate dev enviroment from template folder
def generate_dev_enviroment(config):
	# set variables based on config file
	client_directory = config["client_directory"]
	test_id = config["test_id"]
	test_directory = client_directory + test_id
	site_directory = test_directory + '/site'
	code_template = config["enviroment_template"]
	link_file_path = site_directory + '/index.html'

	# try opening file with read permissions
	try:
		index = open(link_file_path, 'r', encoding="utf8")
	# raise errors
	except Exception as e:
		raise e
	# store file content and replace original asset link with local equivalent
	new_content = index.read().replace('','