import numpy as np
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
from torch.optim import Adam, SGD
import torchvision
from torchvision import transforms
import torch
import pandas as pd
from sklearn.model_selection import train_test_split
import torch.nn.functional as F
import time
import wfdb
import ast
import random
import os.path
import AE_Models as Models
from torch.autograd import Variable
from sklearn.preprocessing import MinMaxScaler
import sys
# Set path variables to load the PTB-XL dataset and its scaler
cwd = os.path.dirname(os.path.abspath(__file__))
cwd = os.path.dirname(cwd)
mlb_path = os.path.join(cwd, "mlb.pkl")
scaler_path = os.path.join(cwd)
ptb_path = os.path.join(cwd, "ptb-xl-a-large-publicly-available-electrocardiography-dataset-1.0.1/")
output_path = os.path.join(cwd, "ptb-xl-a-large-publicly-available-electrocardiography-dataset-1.0.1", "output/")
path_ecg_synthetic = os.path.join(cwd, "..", "ECG-Synthetic")
import utils

batchsize = 64
test_batches = 1500
epoch = 500
lr = 0.001
prev_loss = 999
diverge_tresh = 1.1
lr_adapt = 0.5
model = 'TCN' #Set Model to 'TCN' or 'CNN'
save_model = 0

class PTB_XL(Dataset):
    def __init__(self, stage=None):
        self.stage = stage
        if self.stage == 'train':
            global X_train
            global y_train
            self.y_train = y_train
            self.X_train = X_train
        if self.stage == 'val':
            global y_val
            global X_val
            self.y_val = y_val
            self.X_val = X_val
        if self.stage == 'test':
            global y_test
            global X_test
            self.y_test = y_test
            self.X_test = X_test

    def __len__(self):
        if self.stage == 'train':
            return len(self.y_train)
        if self.stage == 'val':
            return len(self.y_val)
        if self.stage == 'test':
            return len(self.y_test)

    def __getitem__(self, idx):
        if self.stage == 'train':
            sample = self.X_train[idx].transpose((1, 0)), self.y_train[idx]
        if self.stage == 'val':
            sample = self.X_val[idx].transpose((1, 0)), self.y_val[idx]
        if self.stage == 'test':
            sample = self.X_test[idx].transpose((1, 0)), self.y_test[idx]
            #if idx == 50:
            #    print("max: ",np.max(self.X_test[idx]))
            #    print("min: ",np.min(self.X_test[idx]))
        return sample

def str_to_number(label):
    a = np.zeros(5)
    if not label:
        return a
    for i in label:
        if i == 'NORM':
            a[0] = 1
        if i == 'MI':
            a[1] = 1
        if i == 'STTC':
            a[2] = 1
        if i == 'HYP':
            a[3] = 1
        if i == 'CD':
            a[4] = 1
    return a

def init():
    AE_test_dataset = PTB_XL('test')
    global AE_PTB_testdata
    AE_PTB_testdata = torch.utils.data.DataLoader(AE_test_dataset, batch_size=batchsize, shuffle=True)

class Config:
    csv_path = ''
    seed = 2021
    device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
    attn_state_path = path_ecg_synthetic + '/attn.pth'
    lstm_state_path = path_ecg_synthetic + '/lstm.pth'
    cnn_state_path = path_ecg_synthetic + '/cnn.pth'

    attn_logs = path_ecg_synthetic + '/attn.csv'
    lstm_logs = path_ecg_synthetic + '/lstm.csv'
    cnn_logs = path_ecg_synthetic + '/cnn.csv'

    train_csv_path = path_ecg_synthetic + '/mitbih_with_syntetic_train.csv'
    test_csv_path = path_ecg_synthetic + '/mitbih_with_syntetic_test.csv'

def random_num():
    tensor = torch.tensor((), dtype=torch.float32)
    data = tensor.new_full((12, 1000), 0.1)
    for b in range(12):
        #for a in range(1000):
        data[b] = torch.tensor((np.random.normal(0, 6, 1000)), dtype=torch.float32)
    #data = data.expand(12, 1000)
    return data

def newshape(x):
    tensor = torch.tensor((), dtype=torch.float32)
    data = tensor.new_full((1, 1000), 0.1)#
    for a in range(65):
        data[0][a] = x[0][0]
    for a, value in enumerate(x[0]):
        data[0][65 + 5 * a] = value
        data[0][65 + 5 * a + 1] = value
        data[0][65 + 5 * a + 2] = value
        data[0][65 + 5 * a + 3] = value
        data[0][65 + 5 * a + 4] = value
    data = data.expand(12, 1000)
    return data

class ECGDataset(Dataset):

    def __init__(self, df):
        self.df = df
        self.data_columns = self.df.columns[:-2].tolist()

    def __getitem__(self, idx):
        signal = self.df.loc[idx, self.data_columns].astype('float32')
        signal = torch.FloatTensor(np.array([signal.values]))

        #if idx == 0:
        #    print("ECG: ", signal)
        signal = newshape(signal) # reshape to match input of PTB-XL
        #signal = random_num()
        #if idx == 0:
        #    print("ECG: ", signal)
        target = torch.LongTensor(np.array(self.df.loc[idx, 'class']))
        return signal, target

    def __len__(self):
        return len(self.df)

def get_dataloader(phase: str, batch_size: int = 64) -> DataLoader:
    '''
    Dataset and DataLoader.
    Parameters:
        pahse: training or validation phase.
        batch_size: data per iteration.
    Returns:
        data generator
    '''
    df = pd.read_csv(config.train_csv_path)
    print("Total Dataset size: ", df.shape)
    train_df1, ae_df = train_test_split(
        df, test_size=0.1, random_state=config.seed, stratify=df['label']
    )
    print(train_df1.shape)
    train_df1 = train_df1.reset_index(drop=True)
    train_df, val_df = train_test_split(
        train_df1, test_size=0.15, random_state=config.seed, stratify=train_df1['label']
    )
    #print(df.shape)
    train_df, val_df = train_df.reset_index(drop=True), val_df.reset_index(drop=True)
    ae_df = ae_df.reset_index(drop=True)
    if phase == 'ae':
        df = ae_df
        print("Dataset AE size: ", df.shape)
    else:
        df = train_df if phase == 'train' else val_df
    dataset = ECGDataset(df)
    dataloader = DataLoader(dataset=dataset, batch_size=batch_size, num_workers=4)
    return dataloader

def test_PTB(loss_last_10_ptb, e):
    batch_nr = 0
    loss_ptb_epoch = 0
    with torch.no_grad():
        for b, batch in enumerate(AE_PTB_testdata):
            x_train, label_train = batch
            x_train, label_train = x_train.double().to(device), label_train.double().to(device)
            x_train_2 = client_ptb(x_train)
            output_train_enc = encode(x_train_2)
            output_train = decode(output_train_enc)
            loss_train = error_AE(x_train_2, output_train)  # calculates cross-entropy loss
            loss_ptb_epoch += loss_train
            batch_nr += 1
            loss_last_10_ptb.append(loss_train)

        if e == 0:
            print("x_train_ptb: ", x_train.shape)
        loss_10 = 0
        if e % 10 == 0:
            for loss in loss_last_10_ptb:
                loss_10 += loss / batch_nr
            #print("epoch: {}, train-loss_10_ptb: {:.6f}".format(e, loss_10 / 10))
            final_loss = loss_10 / 10
            loss_last_10_ptb.clear()

        print("epoch: {}, train-loss_PTB: {:.6f}".format(e, loss_ptb_epoch / batch_nr))
        data = loss_ptb_epoch / batch_nr
        return loss_last_10_ptb, data

def start_training():
    """
    function, which does the training process for the autoencoder and seves the weights at the end of every epoch,
    if no divergenvce occurs

    --> adaptive learning rate, depending on divergences --> in case of divergence, results of the epoch are dismissed,
    and the training of this epoch restarts with a smaller learning rate

    """
    prevloss = prev_loss
    learnrate = lr###
    optimizerencode = Adam(encode.parameters(), lr=learnrate)###
    optimizerdecode = Adam(decode.parameters(), lr=learnrate)###
    loss_last_10 = []
    loss_last_10_ptb = []
    total_time = time.time()
    data = []
    datalist= []
    """
    phases = ['train', 'val', 'ae']
    dataloaders = {
        phase: get_dataloader(phase, batchsize) for phase in phases
    }
    """

    time_train_epoch = time.time()
    for e in range(epoch):
        #x_test, label_test = x_tester, label_tester
        #x_test, label_test = x_test.to(device), label_test.to(device)
        loss_train_epoch = 0
        batch_nr = 0
        if e == 1:
            time_train_epoch = time.time() - time_train_epoch
            print("estimated_time_total: ", time_train_epoch*epoch/60, " min")
        for b, batch in enumerate(AE_PTB_testdata): #dataloaders['ae']):AE_PTB_testdata):#dataloaders['ae'])
                x_train, label_train = batch
                #x_train = datalist[b]
                #print("x_train: ", x_train)
                x_train, label_train = x_train.double().to(device), label_train.double().to(device)
                #print("x_train shape: ", x_train.shape)

                x_train_2 = client_ptb(x_train)
                #if b == 0:
                #    print("input AE mean ", torch.mean(x_train_2).item())
                #    print("input AE var: ", torch.var(x_train_2).item())
                #x_train = x_train.reshape(-1, 128)
                #print("output client: ",x_train_2.size())
                optimizerencode.zero_grad()
                optimizerdecode.zero_grad()
                if (e == 1 and b == 1):
                    print("shape: ", x_train_2.shape)
                output_train_enc = encode(x_train_2)
                if (e == 1 and b == 1):
                    print("encoded shape: ", output_train_enc.shape)
                #print(output_train_enc.shape)
                output_train = decode(output_train_enc)
                #if b == 0:
                #    print("output AE mean: ", torch.mean(output_train).item())
                #    print("output AE var: ", torch.var(output_train).item())
                #if b == 0:
                #    print(output_train)
                #print("output decoder: ",output_train.size())
                loss_train = error_AE(x_train_2, output_train)  # calculates cross-entropy loss
                #print("loss_train", loss_train)
                loss_train.backward()

                loss_train_epoch += loss_train

                optimizerencode.step()
                optimizerdecode.step()
                batch_nr += 1
                loss_last_10.append(loss_train)

        if e == 0:
            print("x_train: ", x_train.shape)

        print("epoch: {}, train-loss: {:.6f}".format(e, loss_train_epoch / batch_nr))

        if (e % 300 == 0):
            print("output encoder: ", output_train_enc.size())
        #if e == 99:
        #    print(loss_last_10)
        loss_10 = 0
        if e % 10 == 0:
            for loss in loss_last_10:
                loss_10 += loss / batch_nr
            #print("epoch: {}, train-loss_10: {:.6f}".format(e, loss_10 / 10))
            final_loss = loss_10 / 10
            loss_last_10.clear()

        #loss_last_10_ptb, loss_train_epoch_ptb = test_PTB(loss_last_10_ptb, e)

        #if loss_train_epoch_ptb < 0.3: #with same dataset : 0.174317
            #4 Layers: 0.217885 (24), 0.194269 (48), same dataset: 0.132442
            #3 Layers: 0.187612 (24), 0.182876 (48)
            #3Layers same Dataset: 0.128
            #Saved 51 epochs: 0.216848
            #print("saved")
        if model == 'TCN' and save_model == 1:
            torch.save(encode.state_dict(), "convencoder_TCN.pth")
            torch.save(decode.state_dict(), "convdecoder_TCN.pth")
        if model == 'CNN' and save_model == 1:
            torch.save(encode.state_dict(), "../client/convencoder_medical.pth")
            torch.save(decode.state_dict(), "../server/convdecoder_medical.pth")

    print("total time:", time.time()-total_time)

def start_training3():
    """
    function, which does the training process for the autoencoder and seves the weights at the end of every epoch,
    if no divergenvce occurs

    --> adaptive learning rate, depending on divergences --> in case of divergence, results of the epoch are dismissed,
    and the training of this epoch restarts with a smaller learning rate

    """
    prevloss = prev_loss
    learnrate = lr###
    optimizerencode = Adam(encode.parameters(), lr=learnrate)###
    optimizerdecode = Adam(decode.parameters(), lr=learnrate)###
    loss_last_10 = []
    loss_last_10_ptb = []
    total_time = time.time()
    data = []
    datalist= []
    for a in range(1):
        for b in range(64):
            data.append(random_num())
        datalist.append(data)
    print(data[0])
    print(len(datalist))

    phases = ['train', 'val', 'ae']
    dataloaders = {
        phase: get_dataloader(phase, batchsize) for phase in phases
    }
    for b, batch in enumerate(AE_PTB_testdata):
        pass

    time_train_epoch = time.time()
    for e in range(epoch):
        #x_test, label_test = x_tester, label_tester
        #x_test, label_test = x_test.to(device), label_test.to(device)
        loss_train_epoch = 0
        batch_nr = 0
        if e == 1:
            time_train_epoch = time.time() - time_train_epoch
            print("estimated_time_total: ", time_train_epoch*epoch/60, " min")
        for b, batch in enumerate(AE_PTB_testdata):#dataloaders['ae']):
                x_train, label_train = batch
                #x_train = datalist[b]
                #print("x_train: ", x_train)
                x_train, label_train = x_train.double().to(device), label_train.double().to(device)
                if len(x_train) != 64:
                    break
                #print("x_train shape: ", x_train.shape)

                x_train_2 = get_grad(x_train, label_train)
                #x_train = x_train.reshape(-1, 128)
                #print("output client: ",x_train_2.size())
                optimizerencode.zero_grad()
                optimizerdecode.zero_grad()
                output_train_enc = encode(x_train_2)
                #print(output_train_enc.shape)
                output_train = decode(output_train_enc)
                #if b == 0:
                #    print(output_train)
                #print("output decoder: ",output_train.size())
                loss_train = error_AE(x_train_2, output_train)  # calculates cross-entropy loss
                #print("loss_train", loss_train)
                loss_train.backward()

                loss_train_epoch += loss_train

                optimizerencode.step()
                optimizerdecode.step()
                batch_nr += 1
                loss_last_10.append(loss_train)

        if e == 0:
            print("x_train: ", x_train.shape)

        print("epoch: {}, train-loss: {:.6f}".format(e, loss_train_epoch / batch_nr))

        if (e % 300 == 0):
            print("output encoder: ", output_train_enc.size())
        #if e == 99:
        #    print(loss_last_10)
        loss_10 = 0
        if e % 10 == 0:
            for loss in loss_last_10:
                loss_10 += loss / batch_nr
            #print("epoch: {}, train-loss_10: {:.6f}".format(e, loss_10 / 10))
            final_loss = loss_10 / 10
            loss_last_10.clear()

        loss_last_10_ptb, loss_train_epoch_ptb = test_PTB(loss_last_10_ptb, e)

        #if loss_train_epoch_ptb < 0.3: #with same dataset : 0.174317
            #4 Layers: 0.217885 (24), 0.194269 (48), same dataset: 0.132442
            #3 Layers: 0.187612 (24), 0.182876 (48)
            #3Layers same Dataset: 0.128
            #Saved 51 epochs: 0.216848
            #print("saved")
            #torch.save(encode.state_dict(), "../client/convencoder_medical.pth")
            #encode2 = Encode64()
            #encode2.load_state_dict(torch.load("./convencoder_medical.pth"))
            #encode2.eval()
            #torch.save(decode.state_dict(), "../server/convdecoder_medical.pth")
            #prevloss = loss_test

    print("total time:", time.time()-total_time)

def get_grad(x_train, label_train):
    #label_train = get_label_batch()
    #label_train = label_train.double().to(device)
    server_optimizer.zero_grad()

    # Client part
    activations = client_ptb(x_train)
    server_inputs = activations.detach().clone()

    # Server part
    server_inputs = Variable(server_inputs, requires_grad=True)
    outputs = server_ptb(server_inputs)
    # print("output shape", outputs.shape)
    # print("label", label_train.shape)
    loss = error(outputs, label_train)
    loss.backward()
    return server_inputs.grad

def get_label_batch():
    tensor = torch.tensor((), dtype=torch.int32)
    data = tensor.new_full((64, 5), 0)  #
    for a in range(64):
        for b in range(5):
            data[a][b] = torch.tensor((np.random.randint(0, 2)))
    #print(data)
    return data

def start_training_grad():
    """
    function, which does the training process for the autoencoder and seves the weights at the end of every epoch,
    if no divergenvce occurs

    --> adaptive learning rate, depending on divergences --> in case of divergence, results of the epoch are dismissed,
    and the training of this epoch restarts with a smaller learning rate

    """
    prevloss = prev_loss
    learnrate = lr###
    client_optimizer = Adam(client_ptb.parameters(), lr=learnrate)###
    server_optimizer = Adam(server_ptb.parameters(), lr=learnrate)###

    phases = ['train', 'val', 'ae']
    dataloaders = {
        phase: get_dataloader(phase, batchsize) for phase in phases
    }

    for e in range(epoch):
        #x_test, label_test = x_tester, label_tester
        #x_test, label_test = x_test.to(device), label_test.to(device)
        loss_epoch = 0
        loss_train = 0
        batch_nr = 0
        loss_minmax_total = 0
        for b, batch in enumerate(AE_PTB_testdata):#dataloaders['ae']):
                x_train, label_train = batch
                x_train = x_train.double().to(device)
                #label_train = get_label_batch()
                label_train = label_train.double().to(device)
                #print(x_train.shape)
                #print(len(x_train))
                if len(x_train) != 64:
                    break

                #x_train = x_train.reshape(-1, 128)
                #print("output client: ",x_train.size())

                #client_optimizer.zero_grad()
                server_optimizer.zero_grad()

                # Client part
                activations = client_ptb(x_train)
                server_inputs = activations.detach().clone()

                # Server part
                server_inputs = Variable(server_inputs, requires_grad=True)
                outputs = server_ptb(server_inputs)
                #print("output shape", outputs.shape)
                #print("label", label_train.shape)
                loss = error(outputs, label_train)
                loss_epoch += loss
                loss.backward()
                server_inputs_AE = server_inputs.grad
                minmax_inputs = server_inputs_AE.detach().clone().cpu()
                server_optimizer.step()
                #if b == 0:
                    #print("pre AE: ", server_inputs_AE[5])

                loss_train_inc, grad_post = AE_grad(grad_preprocessing(server_inputs_AE.detach().clone().cpu()), b)
                loss_train += loss_train_inc
                #print("post:",grad_post.shape)
                #print("pre:",minmax_inputs.shape)
                loss_minmax = error_MinMax(minmax_inputs, grad_post)
                loss_minmax_total += loss_minmax
                batch_nr +=1

                #running_loss += loss.item()
                #total_samples += labels.shape[0]
                #print("grad: ", server_inputs.grad)

                # Client part
                #activations.backward(output_decode_grad)
                #client_optimizer.step()
                #output_train_enc = encode(x_train)
                #output_train = decode(output_train_enc)
                #print("output decoder: ",output_train.size())
                #loss_train = error(x_train, output_train)  # calculates cross-entropy loss
                #loss_train.backward()

                #optimizerencode.step()
                #optimizerdecode.step()

        print("epoch: {}, train-loss: {:.6f}".format(e + 1, loss_epoch/batch_nr))
        print("epoch: {}, train-loss_AE: {:.6f}".format(e + 1, loss_train/batch_nr))
        print("epoch: {}, train-minmax_AE: {:.6f}".format(e + 1, loss_minmax_total/batch_nr*1000000))

       # if loss_test <= prevloss*diverge_tresh:
        torch.save(encode_grad.state_dict(), "../server/grad_encoder_medical.pth")
            #encode2 = Encode64()
            #encode2.load_state_dict(torch.load("./convencoder_medical.pth"))
            #encode2.eval()

        torch.save(decode_grad.state_dict(), "../client/grad_decoder_medical.pth")
            #prevloss = loss_test

def grad_preprocessing(grad):
    grad_new = grad.numpy()
    for a in range(64):
        grad_new[a] = scaler.fit_transform(grad[a])
    return grad_new

def grad_postprocessing(grad):
    grad_new = grad.numpy()
    for a in range(64):
        #scaler.fit(grad[a])
        grad_new[a] = scaler.inverse_transform(grad[a])
    grad_new = torch.DoubleTensor(grad_new)
    return grad_new

def AE_grad(server_inputs, b):
    optimizerencode_grad.zero_grad()
    optimizerdecode_grad.zero_grad()

    server_inputs = torch.DoubleTensor(server_inputs).to(device)

    output_encode_grad = encode_grad(server_inputs)
    output_decode_grad = decode_grad(output_encode_grad)

    grad_post = grad_postprocessing(output_decode_grad.detach().clone().cpu())

    loss_train = error_AE(server_inputs, output_decode_grad)  # calculates cross-entropy loss
    loss_train.backward()

    optimizerencode_grad.step()
    optimizerdecode_grad.step()

    return loss_train, grad_post

def main():
    """
    initialize device, client model, encoder and decoder and starts the training process
    """
    global config
    config = Config()

    global X_train, X_val, y_val, y_train, y_test, X_test
    sampling_frequency = 100
    datafolder = ptb_path
    task = 'superdiagnostic'
    outputfolder = mlb_path

    # Load PTB-XL data
    data, raw_labels = utils.load_dataset(datafolder, sampling_frequency)
    # Preprocess label data
    labels = utils.compute_label_aggregations(raw_labels, datafolder, task)
    # Select relevant data and convert to one-hot
    data, labels, Y, _ = utils.select_data(data, labels, task, min_samples=0, outputfolder=outputfolder)
    input_shape = data[0].shape
    print(input_shape)

    # 1-9 for training
    X_train = data[labels.strat_fold < 10]
    y_train = Y[labels.strat_fold < 10]
    # 10 for validation
    X_val = data[labels.strat_fold == 10]
    y_val = Y[labels.strat_fold == 10]

    X_test = data[labels.strat_fold == 10]
    y_test = Y[labels.strat_fold == 10]

    num_classes = 5  # <=== number of classes in the finetuning dataset
    input_shape = [1000, 12]  # <=== shape of samples, [None, 12] in case of different lengths

    print(X_train.shape, y_train.shape, X_val.shape, y_val.shape, X_test.shape, y_test.shape)

    import pickle

    standard_scaler = pickle.load(open(scaler_path + '/standard_scaler.pkl', "rb"))

    X_train = utils.apply_standardizer(X_train, standard_scaler)
    X_val = utils.apply_standardizer(X_val, standard_scaler)
    X_test = utils.apply_standardizer(X_test, standard_scaler)

    init()

    global device
    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')  # cuda:0
    seed = 0
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True

    global error
    #error = nn.MSELoss()
    error = nn.BCELoss()

    global error_AE
    error_AE = nn.MSELoss()

    global error_MinMax
    error_MinMax = nn.MSELoss()
    #error_AE = nn.CrossEntropyLoss()

    global client_ptb, server_ptb, encode, decode, decode_grad, encode_grad
    if model == 'TCN':
        client_ptb = Models.Small_TCN_5_Client(5, 12).double().to(device)
        server_ptb = Models.Small_TCN_5_Server(5, 12).double().to(device)
        encode = Models.EncodeTCN().double().to(device)
        decode = Models.DecodeTCN().double().to(device)
        encode_grad = Models.Grad_Encoder().double().to(device)
        decode_grad = Models.Grad_Decoder().double().to(device)
    else:
        client_ptb = Models.Client_PTB().double().to(device)
        server_ptb = Models.Server_PTB().double().to(device)
        encode = Models.Encode192().double().to(device)
        decode = Models.Decode192().double().to(device)
        encode_grad = Models.Grad_Encoder().double().to(device)
        decode_grad = Models.Grad_Decoder().double().to(device)

    global scaler
    scaler = MinMaxScaler()

    global optimizerencode_grad
    optimizerencode_grad = Adam(encode_grad.parameters(), lr=0.001)###
    global optimizerdecode_grad
    optimizerdecode_grad = Adam(decode_grad.parameters(), lr=0.001)###
    global server_optimizer
    server_optimizer = Adam(server_ptb.parameters(), lr=0.001)  ###

    start_training()
    #start_training_grad()

if __name__ == '__main__':
    main()


import torch
import torch.nn as nn
import torchvision.models as models

# model saved in /uac/gds/yuewang/.cache/torch/checkpoints/vgg16-397923af.pth
class EncoderCNN(nn.Module):
    def __init__(self, img_ext_model):
        """Load the pretrained ResNet-152 and replace top fc layer."""
        super(EncoderCNN, self).__init__()
        self.img_ext_model = img_ext_model
        if img_ext_model == 'resnet':
            resnet = models.resnet152(pretrained=True)
            modules = list(resnet.children())[:-1]  # delete the last fc layer.
            self.img_ext = nn.Sequential(*modules)
        elif self.img_ext_model == 'complex_resnet':
            resnet = models.resnet101(pretrained=True)
            modules = list(resnet.children())[:-2]  # delete the last fc layer.
            self.img_ext = nn.Sequential(*modules)
        elif img_ext_model == 'vgg':
            vgg = models.vgg16(pretrained=True)
            modules = list(vgg.children())[:-1]  # delete the last fc layer.
            self.img_ext = nn.Sequential(*modules)

    def forward(self, images):
        """Extract feature vectors from input images."""
        with torch.no_grad():
            features = self.img_ext(images)

        if self.img_ext_model == 'resnet':
            features = features.reshape(features.size(0), -1)
        elif self.img_ext_model == 'complex_resnet':
            features = features.view([features.size(0), 2048, -1]).permute([0, 2, 1])  # [batch_size, 49, 2048]
        elif self.img_ext_model == 'vgg':
            features = features.view([features.size(0), 512, -1]).permute([0, 2, 1])  # [batch_size, 49, 512]
        return features


import random
import sys
import itertools

def MR(n, k):
    """Implementation of the Miller-Rabin primality test.
        Inputs:
            - integer n, we will ensure it is greater than two and odd.
            - integer k, number rounds of testing to perform

        Pseudocode from
    https://en.wikipedia.org/wiki/Miller–Rabin_primality_test#Example.

    Returns False if definitely composite, True if probably prime.
    """

    # First test to ensure n > 2 and n odd
    if n == 1:
        return False
    if n == 2:
        return True
    if (n % 2 == 0):
        return False
    if n == 3:
        return True

    # Now find s and d
    s, d = get_s_d(n)

    #print(f"N:\n{n}\nn-1 = (2**{s})*{d}")

    for _ in range(k):
        b = random.randint(2, n-2)
        #print(f"B: {b}")
        x = pow(b, d, n)
        for _ in range(s):
            y = pow(x, 2, n)
            if y == 1 and x != 1 and (x != n-1):
                return False
            x = y
        if y != 1:
            return False

    return True

def get_s_d(n):
    n_m_1 = n-1
    s = 0
    old_d = n_m_1
    old_r = 0

    # Implementation of do-while loop
    while True:
        d, r = divmod(old_d, 2)
        if r != 0:
            break
        s += 1
        old_d = d
        old_r = r

    return s, old_d

if __name__ == "__main__":
    num_tests = int(sys.stdin.readline().rstrip('\n'))
    for i in range(num_tests):
        line = sys.stdin.readline()
        # Convert line to list of possible integers
        line = list(line.rstrip('\n'))

        subsets = []
        for i in range(1, len(line)+1):
            subsets.extend(itertools.permutations(line, r=i))

        possible_ints = [int(''.join(e)) for e in subsets]

        # Strip out duplicates
        possible_ints = list(set(possible_ints))

        #print(len(possible_ints))

        num_primes = 0
        for e in possible_ints:
            if MR(e, 20):
                num_primes += 1

        print(f"{num_primes}")


import requests
import chardet
import json
import pandas as pd
import get_save

# 爬取信息
url = "https://c.m.163.com/ug/api/wuhan/app/data/list-total?t=318749413808"
headers = {
    "user-agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/83.0.4103.116 Safari/537.36"}
req = requests.get(url, headers=headers)
req.encoding = chardet.detect(req.content)["encoding"]
data_json = json.loads(req.text)
china_history_data = data_json["data"]["chinaDayList"]
# print(china_history_data)
# 获取当天日期
china_date = pd.DataFrame(china_history_data)["date"]
# print(info)
# 获取每天总计数据
# 获取每天总计数据（浓缩）
china_total_data = pd.DataFrame([i["total"] for i in china_history_data])
# print(china_total_data.head())
# 获取每天实时数据（浓缩）
china_today_data = pd.DataFrame([i["today"] for i in china_history_data])
# print(china_today_data)

# 更改列名
china_total_data.columns = ["total_" + i for i in china_today_data.columns]
china_today_data.columns = ["today_" + i for i in china_today_data.columns]
# print(china_total_data)
# 合并
china_data = pd.concat([china_date, china_total_data, china_today_data], axis=1)
get_save.save_data(china_data, "china_data")
print(china_data)


import numpy as np
# import pandas as pd
import h5py
import argparse
import os

# def seq2instance(data, P, Q):
#     num_step, dims, nodes = data.shape
#     num_sample = num_step - P - Q + 1
#     x = np.zeros(shape = (num_sample, P, dims, nodes))
#     y = np.zeros(shape = (num_sample, Q, dims, nodes))
#     for i in range(num_sample):
#         x[i] = data[i : i + P, :, :]
#         y[i] = data[i + P : i + P + Q, :, :]
#     return x, y
#
# def loadTrafficData(args):
#     # Traffic
#     if args.traffic_file.endswith('npy'):
#         Traffic = np.load(args.traffic_file)
#     else:
#         f = h5py.File(args.traffic_file, 'r')
#         Traffic = f['data'].value
#
#     T_trend, len_trend, len_test = args.T_trend, args.len_trend, args.len_test
#
#     if len_trend > 0:
#         number_of_skip_hours = T_trend * len_trend
#     else:
#         print("wrong")
#     print('number_of_skip_hours:', number_of_skip_hours)
#
#     Traffic = Traffic[number_of_skip_hours - args.P - args.Q + 1:len(Traffic)]
#
#
#     Traffic = np.reshape(Traffic, (-1, Traffic.shape[1], Traffic.shape[2] * Traffic.shape[3]))
#     Traffic = np.transpose(Traffic, (0, 2, 1))
#
#     allX, allY = seq2instance(Traffic, args.P, args.Q)
#     num_step = allX.shape[0]
#     train_steps = round(args.train_ratio * (num_step - len_test))
#     test_steps = len_test
#     val_steps = num_step - train_steps - test_steps
#     trainX, trainY = allX[:train_steps], allY[:train_steps]
#     valX, valY = allX[train_steps:train_steps + val_steps], allY[train_steps:train_steps + val_steps]
#     testX, testY = allX[-test_steps:], allY[-test_steps:]
#
#     return (trainX, trainY, valX, valY, testX, testY)

def loadTrafficData(args):
    # Traffic
    if args.traffic_file.endswith('npy'):
        all_data = np.load(args.traffic_file)
        max, min = np.max(all_data), np.min(all_data)
    else:
        df = h5py.File(args.traffic_file, 'r')
        all_data = df['data'][:]
        max, min = np.max(all_data), np.min(all_data)

    len_total = len(all_data)
    T_closeness, T_period, T_trend = 1, args.T, args.T * 7

    if args.len_trend > 0:
        number_of_skip_hours = T_trend * args.len_trend
    elif args.len_period > 0:
        number_of_skip_hours = T_period * args.len_period
    elif args.len_closeness > 0:
        number_of_skip_hours = T_closeness * args.len_closeness
    else:
        print("wrong")
    print('number_of_skip_hours:', number_of_skip_hours)

    Y = all_data[number_of_skip_hours:len_total]
    len_train = round((len(Y) - args.len_test) * args.train_ratio)
    len_val = len(Y) - len_train - args.len_test

    if args.len_closeness > 0:
        X_closeness = all_data[number_of_skip_hours - T_closeness:len_total - T_closeness]
        for i in range(args.len_closeness - 1):
            X_closeness = np.concatenate(
                (X_closeness, all_data[number_of_skip_hours - T_closeness * (2 + i):len_total - T_closeness * (2 + i)]),
                axis=1)
    if args.len_period > 0:
        X_period = all_data[number_of_skip_hours - T_period:len_total - T_period]
        for i in range(args.len_period - 1):
            X_period = np.concatenate(
                (X_period, all_data[number_of_skip_hours - T_period * (2 + i):len_total - T_period * (2 + i)]), axis=1)
    if args.len_trend > 0:
        X_trend = all_data[number_of_skip_hours - T_trend:len_total - T_trend]
        for i in range(args.len_trend - 1):
            X_trend = np.concatenate(
                (X_trend, all_data[number_of_skip_hours - T_trend * (2 + i):len_total - T_trend * (2 + i)]), axis=1)

    X_closeness_train = X_closeness[:len_train]
    X_period_train = X_period[:len_train]
    X_trend_train = X_trend[:len_train]

    X_closeness_val = X_closeness[len_train:len_train + len_val]
    X_period_val = X_period[len_train:len_train + len_val]
    X_trend_val = X_trend[len_train:len_train + len_val]

    X_closeness_test = X_closeness[-args.len_test:]
    X_period_test = X_period[-args.len_test:]
    X_trend_test = X_trend[-args.len_test:]

    trainX = np.concatenate([X_closeness_train, X_period_train, X_trend_train], axis=1)
    valX = np.concatenate([X_closeness_val, X_period_val, X_trend_val], axis=1)
    testX = np.concatenate([X_closeness_test, X_period_test, X_trend_test], axis=1)

    trainY = Y[:len_train]
    valY = Y[len_train:len_train + len_val]
    testY = Y[-args.len_test:]

    # batch_size, seq, H * W, 2
    trainX = trainX.reshape(trainX.shape[0], trainX.shape[1], -1)
    trainX = trainX.reshape(trainX.shape[0], -1, 2, trainX.shape[-1])
    trainX = np.transpose(trainX, (0, 1, 3, 2))
    trainY = trainY.reshape(trainY.shape[0], trainY.shape[1], -1)
    trainY = np.expand_dims(np.transpose(trainY, (0, 2, 1)), 1)

    valX = valX.reshape(valX.shape[0], valX.shape[1], -1)
    valX = valX.reshape(valX.shape[0], -1, 2, valX.shape[-1])
    valX = np.transpose(valX, (0, 1, 3, 2))
    valY = valY.reshape(valY.shape[0], valY.shape[1], -1)
    valY = np.expand_dims(np.transpose(valY, (0, 2, 1)), 1)

    testX = testX.reshape(testX.shape[0], testX.shape[1], -1)
    testX = testX.reshape(testX.shape[0], -1, 2, testX.shape[-1])
    testX = np.transpose(testX, (0, 1, 3, 2))
    testY = testY.reshape(testY.shape[0], testY.shape[1], -1)
    testY = np.expand_dims(np.transpose(testY, (0, 2, 1)), 1)

    return trainX, trainY, valX, valY, testX, testY

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--dataset', default='BikeNYC', help='traffic file load path')
    parser.add_argument('--train_ratio', type=float, default=0.9, help='training set [default : 0.9]')
    parser.add_argument('--T', type=int, default=48, help='num of time intervals in one day = 48')
    parser.add_argument('--T_trend', type=int, default=336, help='num of time intervals in one day * num of day intervals in trend = 48 * 7')
    parser.add_argument('--len_closeness', type=int, default=3, help='len of closeness sequence = 3')
    parser.add_argument('--len_period', type=int, default=4, help='len of period sequence = 4')
    parser.add_argument('--len_trend', type=int, default=4, help='len of trend sequence = 4')
    parser.add_argument('--len_test', type=int, default=960, help='num of test samples = 960')
    parser.add_argument('--P', type=int, default=11, help='history steps')
    parser.add_argument('--Q', type=int, default=1, help='prediction steps')
    args = parser.parse_args()
    if args.dataset == 'BikeNYC':
        args.traffic_file = 'data/BikeNYC/bike_flow.npy'
        args.save_path = 'data/BikeNYC'
    elif args.dataset == 'TaxiNYC':
        args.traffic_file = 'data/TaxiNYC/taxi_flow.npy'
        args.save_path = 'data/TaxiNYC'
    elif args.dataset == 'TaxiBJ':
        args.traffic_file = 'data/TaxiBJ/BJ13_M32x32_T30_InOut.h5'
        args.save_path = 'data/TaxiBJ'
    else:
        raise ValueError('dataset does not exist')

    trainX, trainY, valX, valY, testX, testY = loadTrafficData(args)

    np.savez(os.path.join(args.save_path, 'train.npz'), x=trainX, y=trainY)
    np.savez(os.path.join(args.save_path, 'val.npz'), x=valX, y=valY)
    np.savez(os.path.join(args.save_path, 'test.npz'), x=testX, y=testY)


#!/usr/bin/env python3
#lock

import json
from base64 import b64encode, b64decode
import argparse
from Crypto.PublicKey import RSA
from Crypto.Cipher import AES, PKCS1_OAEP
from Crypto.Random import get_random_bytes
from ecdsa import SigningKey
import sys
from os import walk
from os.path import isfile, join

parser = argparse.ArgumentParser()
parser.add_argument('-d','--directory',required = True)
parser.add_argument('-p','--publicKeyPath',required = True)
parser.add_argument('-r','--privateKeyPath',required = True)
parser.add_argument('-s','--subject',required = True)
args = parser.parse_args()

directory = args.directory
publicKeyPath = args.publicKeyPath
privateKeyPath = args.privateKeyPath
subject = args.subject

pub_in = open(publicKeyPath, "r")

pub_subject = ""

for line in pub_in:
    if line.strip() == "subject:":
        pub_subject = pub_in.readline().strip()
        break

pub_in.close()

if pub_subject != subject:
    print("Subjects do not match! Aborting...")
    sys.exit()

pub_in = open(publicKeyPath, "r")

public_key = ""

for line in pub_in:
    if line.strip() == "subject:":
        break
    else:
        public_key += line

pub_in.close()

#Get and encrypt AES key

aes_key = get_random_bytes(16) #256 bit

rsa_key = RSA.import_key(public_key)
cipher_rsa = PKCS1_OAEP.new(rsa_key)
enc_aes_key = cipher_rsa.encrypt(aes_key)

#We will write the encrypted AES key to keyfile

keyfile_out = open("keyfile", "wb")
keyfile_out.write(enc_aes_key)
keyfile_out.close()

#Get private key

priv_in = open(privateKeyPath, "r")

private_key = ""

for line in priv_in:
    if line.strip() == "type:":
        break
    else:
        private_key += line

priv_in.close()

#Sign with the locker's private key

keyfile_in = open("keyfile", "rb")
keyfile = keyfile_in.read()
keyfile_in.close()

sk = SigningKey.from_pem(private_key)
signature = sk.sign(keyfile)

open("keyfile.sig", "wb").write(signature)

#lock all files and files in sub directories in the specified directory
for path, subdirs, files in walk(directory):
    for name in files:
        read_file = open(join(path, name), "rb")
        text = read_file.read()
        read_file.close()

        cipher = AES.new(aes_key, AES.MODE_GCM)
        ciphertext, mac = cipher.encrypt_and_digest(text)

        json_k = ['nonce', 'ciphertext', 'mac']
        json_v = [b64encode(cipher.nonce).decode('utf-8'), b64encode(ciphertext).decode('utf-8'), b64encode(mac).decode('utf-8')]
        cipher_json = json.dumps(dict(zip(json_k, json_v)))

        write_file = open(join(path, name), 'w')
        write_file.write(str(cipher_json))
        write_file.close()

#lock all files and files in sub directories in the specified directory
for path, subdirs, files in walk(directory):
    for name in files:
        read_file = open(join(path, name), "r")
        cipher_content = read_file.read()
        cipher_json = json.loads(cipher_content)
        cipher_json["nonce"] = b64decode(cipher_json["nonce"])
        cipher_json["ciphertext"] = b64decode(cipher_json["ciphertext"])
        cipher_json["mac"] = b64decode(cipher_json["mac"])

        cipher = AES.new(aes_key, AES.MODE_GCM, cipher_json["nonce"])
        plaintext = cipher.decrypt_and_verify(cipher_json["ciphertext"], cipher_json["mac"])


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Time    : 2021/4/9 下午2:31
# @Author  : MaybeShewill-CV
# @Site    : https://github.com/MaybeShewill-CV/image-classification-tensorflow
# @File    : verify_repvgg_conv.py
# @IDE: PyCharm
"""

"""
import argparse
import time
import json

import tensorflow as tf
import numpy as np
import tqdm

from cls_model_zoo import cnn_basenet
from cls_model_zoo import loss
from local_utils import config_utils

PADDING = 'VALID'
PHASE = 'test'
INPUT_TENSOR_SIZE = [224, 224]
INPUT_CHANNELS = 3
OUTPUT_CHANNELS = 64
STRIDE = 2

class RepVggTest(cnn_basenet.CNNBaseModel):
    """

    """
    def __init__(self, phase, cfg):
        """

        :param phase:
        :param cfg:
        """
        super(RepVggTest, self).__init__()
        self._phase = phase
        self._cfg = cfg

        self._is_training = self._is_net_for_training()

    def _is_net_for_training(self):
        """
        if the net is used for training or not
        :return:
        """
        if isinstance(self._phase, tf.Tensor):
            phase = self._phase
        else:
            phase = tf.constant(self._phase, dtype=tf.string)

        return tf.equal(phase, tf.constant('train', dtype=tf.string))

    @classmethod
    def _fuse_conv_bn_weights(cls, conv_kernel, bn_moving_avg_mean, bn_moving_avg_var, bn_gamma, bn_beta, eps=1e-7):
        """

        :param conv_kernel:
        :param bn_moving_avg_mean:
        :param bn_moving_avg_var:
        :param bn_gamma:
        :param bn_beta:
        :return:
        """
        std = np.sqrt(bn_moving_avg_var + eps)
        t = (bn_gamma / std).reshape(1, 1, 1, -1)
        output_kernel = conv_kernel * t
        output_bias = bn_beta - bn_moving_avg_mean * bn_gamma / std

        return output_kernel, output_bias

    @classmethod
    def _pad_1x1_kernel_to_3x3_kernel(cls, conv_1x1_kernel):
        """

        :param conv_1x1_kernel:
        :return:
        """
        h, w, in_channels, output_channels = conv_1x1_kernel.shape
        output = []
        for in_index in range(in_channels):
            output_c = []
            for out_index in range(output_channels):
                output_c.append(
                    np.pad(
                        conv_1x1_kernel[:, :, in_index, out_index],
                        [[1, 1], [1, 1]],
                        mode='constant',
                        constant_values=0.0
                    )
                )
            output.append(output_c)

        # return np.array(output, dtype=np.float32).reshape((3, 3, in_channels, output_channels))
        return np.array(output, dtype=np.float32).transpose((2, 3, 0, 1))

    @classmethod
    def _make_bn_input_layer_bn_equal_conv_kernel(cls, input_channels):
        """

        :param input_channels:
        :return:
        """
        kernel_value = np.zeros((input_channels, input_channels, 3, 3), dtype=np.float32)
        for i in range(input_channels):
            kernel_value[i, i % input_channels, 1, 1] = 1

        return kernel_value.transpose((2, 3, 0, 1))
        # return kernel_value.reshape((3, 3, input_channels, input_channels))

    def _fuse_conv_block_params(self, conv_block_params):
        """

        :param conv_block_params:
        :return:
        """
        conv_1x1_sub_block = dict()
        conv_3x3_sub_block = dict()
        conv_input_sub_block = dict()

        for param_name, param_value in conv_block_params:
            if '1x1' in param_name:
                if 'moving_variance' in param_name:
                    conv_1x1_sub_block['bn_moving_avg_var'] = param_value
                elif 'moving_mean' in param_name:
                    conv_1x1_sub_block['bn_moving_avg_mean'] = param_value
                elif 'conv_1x1' in param_name:
                    conv_1x1_sub_block['conv_kernel'] = param_value
                elif 'gamma' in param_name:
                    conv_1x1_sub_block['bn_gamma'] = param_value
                elif 'beta' in param_name:
                    conv_1x1_sub_block['bn_beta'] = param_value
                else:
                    raise ValueError('Unrecognized params: {:s}'.format(param_name))
            elif '3x3' in param_name:
                if 'moving_variance' in param_name:
                    conv_3x3_sub_block['bn_moving_avg_var'] = param_value
                elif 'moving_mean' in param_name:
                    conv_3x3_sub_block['bn_moving_avg_mean'] = param_value
                elif 'conv_3x3' in param_name:
                    conv_3x3_sub_block['conv_kernel'] = param_value
                elif 'gamma' in param_name:
                    conv_3x3_sub_block['bn_gamma'] = param_value
                elif 'beta' in param_name:
                    conv_3x3_sub_block['bn_beta'] = param_value
                else:
                    raise ValueError('Unrecognized params: {:s}'.format(param_name))
            elif 'bn_input' in param_name:
                if 'moving_variance' in param_name:
                    conv_input_sub_block['bn_moving_avg_var'] = param_value
                elif 'moving_mean' in param_name:
                    conv_input_sub_block['bn_moving_avg_mean'] = param_value
                elif 'gamma' in param_name:
                    conv_input_sub_block['bn_gamma'] = param_value
                elif 'beta' in param_name:
                    conv_input_sub_block['bn_beta'] = param_value
                else:
                    raise ValueError('Unrecognized params: {:s}'.format(param_name))
            else:
                raise ValueError('Unrecognized param name: {:s}'.format(param_name))

        conv_1x1_fused_kernel, conv_1x1_fused_bias = self._fuse_conv_bn_weights(
            conv_kernel=conv_1x1_sub_block['conv_kernel'],
            bn_moving_avg_mean=conv_1x1_sub_block['bn_moving_avg_mean'],
            bn_moving_avg_var=conv_1x1_sub_block['bn_moving_avg_var'],
            bn_gamma=conv_1x1_sub_block['bn_gamma'],
            bn_beta=conv_1x1_sub_block['bn_beta']
        )
        conv_3x3_fused_kernel, conv_3x3_fused_bias = self._fuse_conv_bn_weights(
            conv_kernel=conv_3x3_sub_block['conv_kernel'],
            bn_moving_avg_mean=conv_3x3_sub_block['bn_moving_avg_mean'],
            bn_moving_avg_var=conv_3x3_sub_block['bn_moving_avg_var'],
            bn_gamma=conv_3x3_sub_block['bn_gamma'],
            bn_beta=conv_3x3_sub_block['bn_beta']
        )

        if conv_input_sub_block:
            conv_input_sub_block['conv_kernel'] = self._make_bn_input_layer_bn_equal_conv_kernel(
                input_channels=conv_input_sub_block['bn_beta'].shape[0]
            )
            conv_input_fused_kernel, conv_input_fused_bias = self._fuse_conv_bn_weights(
                conv_kernel=conv_input_sub_block['conv_kernel'],
                bn_moving_avg_mean=conv_input_sub_block['bn_moving_avg_mean'],
                bn_moving_avg_var=conv_input_sub_block['bn_moving_avg_var'],
                bn_gamma=conv_input_sub_block['bn_gamma'],
                bn_beta=conv_input_sub_block['bn_beta']
            )
            output_kernel = conv_3x3_fused_kernel + self._pad_1x1_kernel_to_3x3_kernel(conv_1x1_fused_kernel) + \
                            conv_input_fused_kernel
            output_bias = conv_3x3_fused_bias + conv_1x1_fused_bias + conv_input_fused_bias
        else:
            output_kernel = conv_3x3_fused_kernel + self._pad_1x1_kernel_to_3x3_kernel(conv_1x1_fused_kernel)
            output_bias = conv_3x3_fused_bias + conv_1x1_fused_bias

        return output_kernel, output_bias

    def conv_block(self, input_tensor, output_channels, stride, name, padding='VALID', apply_reparam=False):
        """

        :param input_tensor:
        :param output_channels:
        :param stride:
        :param name:
        :param padding:
        :param apply_reparam:
        :return:
        """
        with tf.variable_scope(name_or_scope=name):
            if apply_reparam:
                if padding == 'VALID':
                    input_tensor = tf.keras.layers.ZeroPadding2D().call(inputs=input_tensor)
                output = self.conv2d(
                    inputdata=input_tensor,
                    out_channel=output_channels,
                    kernel_size=3,
                    padding=padding,
                    stride=stride,
                    use_bias=True,
                    name='conv'
                )
            else:
                input_channles = input_tensor.get_shape().as_list()[-1]
                conv_3x3_input_tensor = tf.keras.layers.ZeroPadding2D().call(inputs=input_tensor) if \
                    padding == 'VALID' else input_tensor
                conv_3x3 = self.conv2d(
                    inputdata=conv_3x3_input_tensor,
                    out_channel=output_channels,
                    kernel_size=3,
                    padding=padding,
                    stride=stride,
                    use_bias=False,
                    name='conv_3x3'
                )
                bn_3x3 = self.layerbn(inputdata=conv_3x3, is_training=self._is_training, name='bn_3x3', scale=True)
                conv_1x1_input_tensor = tf.keras.layers.ZeroPadding2D(padding=0).call(inputs=input_tensor) if \
                    padding == 'VALID' else input_tensor
                conv_1x1 = self.conv2d(
                    inputdata=conv_1x1_input_tensor,
                    out_channel=output_channels,
                    kernel_size=1,
                    padding=padding,
                    stride=stride,
                    use_bias=False,
                    name='conv_1x1'
                )
                bn_1x1 = self.layerbn(inputdata=conv_1x1, is_training=self._is_training, name='bn_1x1', scale=True)

                if input_channles == output_channels and stride == 1:
                    output = tf.add_n(
                        inputs=[
                            bn_3x3,
                            bn_1x1,
                            self.layerbn(
                                inputdata=input_tensor,
                                is_training=self._is_training,
                                name='bn_input',
                                scale=True
                            ),
                        ],
                        name='add_n_fused'
                    )
                else:
                    output = tf.add_n(
                        inputs=[bn_3x3, bn_1x1],
                        name='add_n_fused'
                    )
            output = self.relu(inputdata=output, name='conv_block_output')
        return output

    def save_trained_model(self, weights_path, name, dump_to_json=False):
        """

        :param weights_path:
        :param name:
        :param dump_to_json:
        :return:
        """
        input_tensor = tf.placeholder(
            dtype=tf.float32,
            shape=[None, INPUT_TENSOR_SIZE[0], INPUT_TENSOR_SIZE[1], INPUT_CHANNELS],
            name='input_tensor'
        )

        _ = self.conv_block(
            input_tensor=input_tensor,
            output_channels=OUTPUT_CHANNELS,
            name=name,
            apply_reparam=False,
            stride=STRIDE,
            padding=PADDING
        )

        with tf.variable_scope(name_or_scope='moving_avg'):
            variable_averages = tf.train.ExponentialMovingAverage(self._cfg.SOLVER.MOVING_AVE_DECAY)
            variables_to_restore = variable_averages.variables_to_restore()

        trained_params = dict()
        saver = tf.train.Saver(variables_to_restore)
        with tf.Session() as sess:
            saver.restore(sess=sess, save_path=weights_path)

            for vv_name, vv in variables_to_restore.items():
                trained_params[vv_name] = sess.run(vv).tolist()

        if dump_to_json:
            with open('./repvgg_trainned_params.json', 'w') as file:
                json.dump(obj=trained_params, fp=file)

        return trained_params

    def convert_reparams(self, repvgg_trainned_params: dict, repvgg_converted_param_names):
        """

        :param repvgg_trainned_params:
        :param repvgg_converted_param_names:
        :return:
        """
        converted_params = dict()
        for param_name in tqdm.tqdm(repvgg_converted_param_names):
            corresponding_params = []
            for trainned_param_name, trainned_param_value in repvgg_trainned_params.items():
                corresponding_params.append((trainned_param_name, trainned_param_value))

            output_kernel, output_bias = self._fuse_conv_block_params(corresponding_params)

            if 'bias' in param_name or 'b' in param_name.split('/')[-1]:
                converted_params[param_name] = output_bias
            elif 'kernel' in param_name or 'W' in param_name.split('/')[-1]:
                converted_params[param_name] = output_kernel
            else:
                raise ValueError('Unrecognized param: {:s}'.format(param_name))

        return converted_params

    def compute_loss(self, input_tensor, label, name, reuse=False):
        """

        :param input_tensor:
        :param label:
        :param name:
        :param reuse:
        :return:
        """
        pass

    def inference(self, input_tensor, name, reuse=False):
        """

        :param input_tensor:
        :param name:
        :param reuse:
        :return:
        """
        pass

def get_model(phase, cfg):
    """

    :param phase:
    :param cfg:
    :return:
    """
    return RepVggTest(phase=phase, cfg=cfg)

def _init_args():
    """

    :return:
    """
    parser = argparse.ArgumentParser()
    parser.add_argument('--train_ckpt', type=str, help='The trainned repvgg model ckpt weights path')
    parser.add_argument('--deploy_ckpt', type=str)

    return parser.parse_args()

def save_train_params():
    """

    :return:
    """
    tf.reset_default_graph()
    cfg = config_utils.get_config(config_file_path='./config/ilsvrc_2012_repvgg.yaml')
    model = get_model(phase=PHASE, cfg=cfg)
    input_tensor = tf.placeholder(
        dtype=tf.float32,
        shape=[None, INPUT_TENSOR_SIZE[0], INPUT_TENSOR_SIZE[1], INPUT_CHANNELS],
        name='input_tensor'
    )

    _ = model.conv_block(
        input_tensor=input_tensor,
        output_channels=OUTPUT_CHANNELS,
        name="RepVgg",
        apply_reparam=False,
        stride=STRIDE,
        padding=PADDING
    )

    with tf.variable_scope(name_or_scope='moving_avg'):
        variable_averages = tf.train.ExponentialMovingAverage(cfg.SOLVER.MOVING_AVE_DECAY)
        variables_to_restore = variable_averages.variables_to_restore()

    saver = tf.train.Saver(variables_to_restore)
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        saver.save(sess=sess, save_path='./log/train/repvgg_train.ckpt')

def convert_repvgg_params():
    """

    :return:
    """
    tf.reset_default_graph()
    args = _init_args()

    # init compute graph
    cfg = config_utils.get_config(config_file_path='./config/ilsvrc_2012_repvgg.yaml')
    model = get_model(phase=PHASE, cfg=cfg)

    # save trainned repvgg params into json
    repvgg_trainned_params = dict()
    trained_params = model.save_trained_model(weights_path=args.train_ckpt, name='RepVgg')
    for param_name, param_value in trained_params.items():
        repvgg_trainned_params[param_name] = np.array(param_value, dtype=np.float32)

    # build repvgg inference compute graph
    tf.reset_default_graph()
    test_input_tensor = tf.placeholder(dtype=tf.float32, shape=[None, 224, 224, INPUT_CHANNELS], name='test_input')
    _ = model.conv_block(
        input_tensor=test_input_tensor,
        output_channels=OUTPUT_CHANNELS,
        name='RepVgg',
        apply_reparam=True,
        stride=STRIDE,
        padding=PADDING
    )
    repvgg_converted_names = []
    for vv in tf.trainable_variables():
        repvgg_converted_names.append(vv.name)

    converted_params = model.convert_reparams(
        repvgg_trainned_params=repvgg_trainned_params,
        repvgg_converted_param_names=repvgg_converted_names
    )

    saver = tf.train.Saver(tf.global_variables())

    with tf.Session() as sess:

        sess.run(tf.global_variables_initializer())
        # train_vars = enumerate(tf.trainable_variables())

        for index, vv in enumerate(tf.trainable_variables()):
            if index == 0:
                print(vv.name)
                print(sess.run(vv))
                print('*' * 20)

        for index, vv in enumerate(tf.trainable_variables()):
            kernel = converted_params[vv.name]
            sess.run(tf.assign(vv, kernel))
            if index == 0:
                print(vv.name)
                print(sess.run(vv))
                print('*' * 20)

        saver.save(sess, save_path=args.deploy_ckpt)

    print('Complete')

def check_converted_model():
    """

    :return:
    """
    tf.reset_default_graph()
    args = _init_args()

    cfg = config_utils.get_config(config_file_path='./config/ilsvrc_2012_repvgg.yaml')
    model = get_model(phase=PHASE, cfg=cfg)
    test_input_ndaray = np.random.random(
        (1, INPUT_TENSOR_SIZE[0], INPUT_TENSOR_SIZE[1], INPUT_CHANNELS)
    ).astype(np.float32)

    # init trained compute graph
    test_input_tensor = tf.placeholder(
        dtype=tf.float32,
        shape=[1, INPUT_TENSOR_SIZE[0], INPUT_TENSOR_SIZE[1], INPUT_CHANNELS],
        name='test_input'
    )
    logits = model.conv_block(
        input_tensor=test_input_tensor,
        output_channels=OUTPUT_CHANNELS,
        name="RepVgg",
        apply_reparam=False,
        stride=STRIDE,
        padding=PADDING
    )

    with tf.variable_scope(name_or_scope='moving_avg'):
        variable_averages = tf.train.ExponentialMovingAverage(cfg.SOLVER.MOVING_AVE_DECAY)
        variables_to_restore = variable_averages.variables_to_restore()

    saver = tf.train.Saver(variables_to_restore)
    with tf.Session() as sess:
        saver.restore(sess=sess, save_path=args.train_ckpt)

        train_logits = sess.run(logits, feed_dict={test_input_tensor: test_input_ndaray})
        print(train_logits)
        print('*' * 100)

    # reset compute graph
    tf.reset_default_graph()
    test_input_tensor = tf.placeholder(
        dtype=tf.float32,
        shape=[1, INPUT_TENSOR_SIZE[0], INPUT_TENSOR_SIZE[1], INPUT_CHANNELS],
        name='test_input'
    )
    logits = model.conv_block(
        input_tensor=test_input_tensor,
        output_channels=OUTPUT_CHANNELS,
        name='RepVgg',
        apply_reparam=True,
        stride=STRIDE,
        padding=PADDING
    )
    saver = tf.train.Saver(tf.global_variables())

    with tf.Session() as sess:
        saver.restore(sess=sess, save_path=args.deploy_ckpt)

        deploy_logits = sess.run(logits, feed_dict={test_input_tensor: test_input_ndaray})
        print(deploy_logits)
        print('*' * 100)

    print('========================== The diff is')
    print(((train_logits - deploy_logits) ** 2).sum())

if __name__ == '__main__':
    """

    """
    save_train_params()

    convert_repvgg_params()

    check_converted_model()


#Following tutorial from Medium: https://medium.com/deep-learning-turkey/deep-learning-lab-episode-2-cifar-10-631aea84f11e
from __future__ import print_function
import keras
from keras.datasets import cifar10
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.optimizers import SGD
from keras.utils import print_summary, to_categorical
import sys
import os
###Pre-Processing###
batch_size = 64
num_classes = 10
epochs = 100
model_name = 'keras_cifar10_model'
save_dir = os.path.join(os.getcwd(), model_name)
#Load the dataset
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
#Convert labels to categorical matrix structure
y_train = to_categorical(y_train, num_classes)
y_test = to_categorical(y_test, num_classes)
#Normalize the images in the dataset (more preprocessing)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255.0
x_test /= 255.0
###Build the model###
model = Sequential ()
model.add (Conv2D(32, (3, 3), padding='same', input_shape=x_train.shape[1:]))
model.add (Activation('relu'))
model.add (MaxPooling2D(pool_size=(2, 2)))
model.add (Dropout(0.3))

model.add (Conv2D(64, (3, 3), padding='same', input_shape=x_train.shape[1:]))
model.add (Activation('relu'))
model.add (MaxPooling2D(pool_size=(2, 2)))
model.add (Dropout(0.3))

model.add (Conv2D(128, (3, 3), padding='same', input_shape=x_train.shape[1:]))
model.add (Activation('relu'))
model.add (MaxPooling2D(pool_size=(2, 2)))
model.add (Dropout(0.3))

model.add (Flatten())
model.add (Dense(80))
model.add (Activation('relu'))
model.add (Dropout(0.3))
model.add(Dense(num_classes))
model.add (Activation('softmax'))
#Optimize weights during the backpropagation using Adam
opt = SGD (lr=0.01, momentum=0.9, decay=0.0, nesterov=False)
#Compile the model
model.compile (loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
model.fit (x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.2, shuffle=True)
#Results
scores = model.evaluate (x_test, y_test, verbose=1)
print ('Test Loss: ', scores[0])
print ('Test Accuracy: ', scores[1])


import torch
from torch import nn

from maskrcnn_benchmark.structures.boxlist_ops import boxlist_iou
from ..attribute_head.roi_attribute_feature_extractors import make_roi_attribute_feature_extractor
from ..box_head.roi_box_feature_extractors import make_roi_box_feature_extractor
from .roi_relation_feature_extractors import make_roi_relation_feature_extractor
from .roi_relation_predictors import make_roi_relation_predictor
from .inference import make_roi_relation_post_processor
from .loss import make_roi_relation_loss_evaluator
from .sampling import make_roi_relation_samp_processor

class ROIRelationHead(torch.nn.Module):
    """
    Generic Relation Head class.
    """

    def __init__(self, cfg, in_channels):
        super(ROIRelationHead, self).__init__()
        self.cfg = cfg.clone()
        # same structure with box head, but different parameters
        # these param will be trained in a slow learning rate, while the parameters of box head will be fixed
        # Note: there is another such extractor in uniton_feature_extractor
        self.union_feature_extractor = make_roi_relation_feature_extractor(cfg, in_channels)
        if cfg.MODEL.ATTRIBUTE_ON:
            self.box_feature_extractor = make_roi_box_feature_extractor(cfg, in_channels, half_out=True)
            self.att_feature_extractor = make_roi_attribute_feature_extractor(cfg, in_channels, half_out=True)
            feat_dim = self.box_feature_extractor.out_channels * 2
        else:
            self.box_feature_extractor = make_roi_box_feature_extractor(cfg, in_channels)
            feat_dim = self.box_feature_extractor.out_channels
        self.predictor = make_roi_relation_predictor(cfg, feat_dim)
        self.post_processor = make_roi_relation_post_processor(cfg)
        self.loss_evaluator = make_roi_relation_loss_evaluator(cfg)
        self.samp_processor = make_roi_relation_samp_processor(cfg)

        ### QA module
        self.q_fc1 = nn.Linear(4096, 768)
        self.softmax = nn.Softmax(dim=1)
        #self.q_fc2 = nn.Linear(2048, 1024)
        #self.q_fc3 = nn.Linear(1024, 768)
        self.his_encoder = nn.Linear(4096+768,4096)
        self.his_fc = nn.Linear(768+768,768)
        self.qa_post = nn.Linear(4096+768, 4096)

        # parameters
        self.use_union_box = self.cfg.MODEL.ROI_RELATION_HEAD.PREDICT_USE_VISION

    def forward(self, features, proposals, questions, answers, targets=None, logger=None):
        """
        Arguments:
            features (list[Tensor]): feature-maps from possibly several levels
            proposals (list[BoxList]): proposal boxes. Note: it has been post-processed (regression, nms) in sgdet mode
            targets (list[BoxList], optional): the ground-truth targets.

        Returns:
            x (Tensor): the result of the feature extractor
            proposals (list[BoxList]): during training, the subsampled proposals
                are returned. During testing, the predicted boxlists are returned
            losses (dict[Tensor]): During training, returns the losses for the
                head. During testing, returns an empty dict.
        """
        if self.training:
            # relation subsamples and assign ground truth label during training
            with torch.no_grad():
                if self.cfg.MODEL.ROI_RELATION_HEAD.USE_GT_BOX:
                    proposals, rel_labels, rel_pair_idxs, rel_binarys = self.samp_processor.gtbox_relsample(proposals, targets)
                else:
                    proposals, rel_labels, rel_pair_idxs, rel_binarys = self.samp_processor.detect_relsample(proposals, targets)
        else:
            rel_labels, rel_binarys = None, None
            rel_pair_idxs = self.samp_processor.prepare_test_pairs(features[0].device, proposals)

        # use box_head to extract features that will be fed to the later predictor processing
        roi_features = self.box_feature_extractor(features, proposals)
        #print("check original roi_features:", roi_features.size())

        #if self.cfg.MODEL.ATTRIBUTE_ON:
        #    att_features = self.att_feature_extractor(features, proposals)
        #    roi_features = torch.cat((roi_features, att_features), dim=-1)

        #if self.use_union_box:
        #    union_features = self.union_feature_extractor(features, proposals, rel_pair_idxs)
        #else:
        #    union_features = None

        ####### add the qa modules to update features here #####
        if questions is not None and answers is not None:
            #print("starting dialog:", questions.size(), answers.size())

            ## get the v_feat_in for question decoder
            s = 0
            for i in range(len(proposals)):
                num_tmp = getattr(proposals[i],"bbox").size()[0]
                e = s + num_tmp
                v_feat_tmp = roi_features[s:e,:]
                v_feat_tmp = torch.mean(v_feat_tmp,0,True)
                if i == 0:
                    v_feat_in = v_feat_tmp.unsqueeze(0)
                else:
                    v_feat_in = torch.cat((v_feat_in, v_feat_tmp.unsqueeze(0)),0)
                s = e

            for round_id in range(10):

                if round_id != 0:
                    #print("check 1:", v_feat_in.size(), his_feat.size())
                    v_feat_in = torch.cat((v_feat_in, his_feat),2)
                    #print("check 2:", v_feat_in.size())
                    v_feat_in = self.his_encoder(v_feat_in)
                    #print("check 3:", v_feat_in.size())

                ## get the question
                q_feat = self.q_fc1(v_feat_in)
                #print("check q_feat:", q_feat.size())
                q_scores = torch.bmm(q_feat,questions.transpose(1,2))
                q_idx = torch.argmax(q_scores,dim=2).squeeze(1)
                #print("check q_scores and idx:", q_scores.size(), q_idx.size(), q_idx)
                #print("check scores:", q_scores)
                for i in range(q_idx.size()[0]):
                    if i == 0:
                        qa_feat = questions[i,q_idx[i],:]+answers[i,q_idx[i],:]
                        qa_feat = qa_feat.unsqueeze(0)
                        #print("qa feat size:",qa_feat.size())
                    else:
                        qa_feat_tmp = questions[i,q_idx[i],:]+answers[i,q_idx[i],:]
                        qa_feat = torch.cat((qa_feat,qa_feat_tmp.unsqueeze(0)),0)
                #print("check qa feat:", qa_feat.size())

                if round_id == 0:
                    his_feat = qa_feat.unsqueeze(1)
                    #print("check his feat:", round_id, his_feat.size())
                else:
                    his_feat = torch.cat((his_feat, qa_feat.unsqueeze(1)),2)
                    his_feat = self.his_fc(his_feat)
                    #print("check his feat:", round_id, his_feat.size())
            #print("check final his_feat:", round_id, his_feat.size())

            ### update the original roi_features
            for i in range(len(proposals)):
                num_tmp = getattr(proposals[i],"bbox").size()[0]
                if i == 0:
                    dia_feat = his_feat[i,:,:].repeat(num_tmp,1)
                    #print("check dia_feat:", dia_feat.size())
                else:
                    dia_feat_tmp = his_feat[i,:,:].repeat(num_tmp,1)
                    dia_feat = torch.cat((dia_feat, dia_feat_tmp),0)
            #print("check final dia_feat:", roi_features.size(), dia_feat.size())
            roi_features = torch.cat((roi_features, dia_feat),1)
            roi_features = self.qa_post(roi_features)
            #print("check updated roi features:", roi_features.size())
            #exit()
        else:
            print("no dialog info")
            exit()

        if self.cfg.MODEL.ATTRIBUTE_ON:
            att_features = self.att_feature_extractor(features, proposals)
            roi_features = torch.cat((roi_features, att_features), dim=-1)

        if self.use_union_box:
            union_features = self.union_feature_extractor(features, proposals, his_feat, rel_pair_idxs)
            #print("check union features:", union_features.size())
            #exit()
        else:
            union_features = None

        # final classifier that converts the features into predictions
        # should corresponding to all the functions and layers after the self.context class
        refine_logits, relation_logits, add_losses = self.predictor(proposals, rel_pair_idxs, rel_labels, rel_binarys, roi_features, union_features, logger)

        # for test
        if not self.training:
            result = self.post_processor((relation_logits, refine_logits), rel_pair_idxs, proposals)
            return roi_features, result, {}

        loss_relation, loss_refine = self.loss_evaluator(proposals, rel_labels, relation_logits, refine_logits)

        if self.cfg.MODEL.ATTRIBUTE_ON and isinstance(loss_refine, (list, tuple)):
            output_losses = dict(loss_rel=loss_relation, loss_refine_obj=loss_refine[0], loss_refine_att=loss_refine[1])
        else:
            output_losses = dict(loss_rel=loss_relation, loss_refine_obj=loss_refine)

        output_losses.update(add_losses)

        return roi_features, proposals, output_losses

def build_roi_relation_head(cfg, in_channels):
    """
    Constructs a new relation head.
    By default, uses ROIRelationHead, but if it turns out not to be enough, just register a new class
    and make it a parameter in the config
    """
    return ROIRelationHead(cfg, in_channels)


def wrap_kawahara_processRN(filen, filemask, fitsout):
    # OEGP method
    import astropy.io.fits as pyf
    import numpy as np
    from pyird import readnoise as rn
    from pyird import large_scaled

    hdulist = pyf.open(filemask)
    hdu = hdulist[0]
    header = hdu.header
    mask = np.array(hdu.data)
    mask[mask == 0] = 0
    mask[mask > 0] = 1
    mask = np.array(mask, dtype=np.bool)
    # input img
    hdulist = pyf.open(filen)
    hdu = hdulist[0]
    header = hdu.header
    img = np.array(hdu.data)
    imgorig = np.copy(img)
    # print("# of Masked pix is "+str(len(img[mask])))
    img[mask] = None

    # Large Scale Distribution
    check = large_scaled.check_mask_filling(mask, Ncor=64)
    if check:
        LSD = large_scaled.get_LSD(img, gpscale=1024, Ncor=64, sigma=0.001)
    else:
        sys.exit('exit')
    ###############################
    img = img-LSD
    recimg = rn.RNestimate_OEGP(img)
    ###############################

    cimg = imgorig-recimg-LSD

    hdu = pyf.PrimaryHDU(cimg, header)
    hdulist = pyf.HDUList([hdu])
    hdulist.writeto(fitsout, overwrite=True)


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import geopandas as geopd
import seaborn as sns

from scipy.spatial import ConvexHull

import pyreadr

def hr_mean(xy, n_drop):
    '''
    Function to calculate the homerange using the arithmetic mean of all points.

    Input:
    xy: Series with the positional data
    n_drop: amount of positions that should be removed

    To remove the positions, the mean of all positions is calculated.
    The positions with the biggest distance to this mean are deleted.

    Returns:
    Array with the reduced amount of positions.
    '''
    #sort points by distance to mean
    ind_list = np.linalg.norm(xy - np.mean(xy, axis = 0, keepdims=True), axis = 1).argsort()
    xy_sorted = xy[ind_list, :]

    #drop last n-drop values, calculate hull
    xy_hr = xy_sorted[:-n_drop,...]
    hull = ConvexHull(xy_hr)

    return xy_hr

def hr_min_volume(xy, n_drop):
    '''
    Function to calculate the homerange using the minimum volume.

    Input:
    xy: Series with the positional data
    n_drop: amount of positions that should be removed.

    The total amount of positions is reduced in a way such
    that the volume of the ConvexHull is as small as possible.

    Returns:
    Array with the reduced amount of positions.
    '''
    #loop for every row that will be dropped:
    for k in range(n_drop):
        #create lists
        vol = []
        index = []
        #get points that make up the ConvexHull and store them in points
        #these are the points that could be dropped to reduce the volume
        hull = ConvexHull(xy)
        points = hull.vertices
        #loop over points: drop each one and store both the resulting volume and the index of the point in a list
        for i in points:
            xy_drop = np.delete(xy, i, 0)
            hull = ConvexHull(xy_drop)
            vol.append(hull.volume)
            index.append(i)
        #find the smallest volume and the corresponding index
        smallest = min(vol)
        index_smallest = vol.index(smallest)
        del_index = index[index_smallest]
        #drop the row with this index from the array
        xy = np.delete(xy,del_index,0)
    return xy

def hr_proportion(data, proportion = 0.95):
    '''
    Main function to calculate the homerange.

    Input:
    data: Dataframe where the x- and y- values of the position are stored
    in the first two columns
    proportion: proportion of positions that should be used for the homerange.

    The function calls the two functions homerange_plot_mean to calculate
    the proportion of points that are then used in the homerange calculation.

    Returns:
    xy: Array with two columns: 0: x values, 1: y values. Includes all points
    xy_proportion: Array with two columns: 0: x values, 1: y values. Includes
    only the specified proportion of points
    '''
    #create array out of input data
    xy = np.array(data.iloc[:,0:2].to_numpy())
    #estimate how many rows need to be dropped
    n_drop = int(np.ceil(xy.shape[0]*(1-proportion)))

    #distribute n_drop between the two functions
    if (proportion < 0.97):
        n_drop_min_volume = int(np.ceil(xy.shape[0]*0.03))
    else:
        n_drop_min_volume = n_drop-1
    n_drop_mean = n_drop - n_drop_min_volume

    # use homerange_plot_min_volume and homerange_plot_mean to build
    # the data frame including only the points used to build the
    # home range
    xy_new = hr_min_volume(xy, n_drop_min_volume)
    xy_proportion = hr_mean(xy_new, n_drop_mean)

    return xy, xy_proportion

def hr_plot(data, proportion = 0.95):
    '''
    Function to plot the home ranges.

    Input:
    data: should include an x and y column at first and second place
    proportion: the proportion of points that should be used to build
    the home ranges

    Uses the homerange_proportion() function to create new array only
    including the proportion of points of the input data frame that was
    specified. It then calculates the hull and plots it.

    Returns:
    hull object
    '''
    # get the data into the needed format for the plot
    # xy: Array with all data points with x as column 0 and y as column 1
    # xy_new: array only including the set proportion of x and y coordinates
    xy, xy_new = hr_proportion(data, proportion)

    # creating the hull from the set proportion of data points
    hull = ConvexHull(xy_new)

    # creating the plot
    f, ax = plt.subplots()
    # first, all data points are plotted
    ax.plot(xy[:,0], xy[:,1], 'o')
    # next, the hull is plotted
    for simplex in hull.simplices:
        ax.plot(xy_new[simplex, 0], xy_new[simplex, 1], 'k-')
    # and last the axis are set to invisible to preserve the secrecy of
    # the gps coordinates
    ax.axes.get_xaxis().set_visible(False)
    ax.axes.get_yaxis().set_visible(False)
    plt.show()

def hr_geometry_df(data, proportion = 0.95):
    '''
    Function to build a data frame including the geometry out of the
    output of homerange_proportion().

    Input:
    data: dataframe where the x- and y- values of the position are stored
    in the first two columns
    proportion: proportion of positions that should be used for the homerange.

    The function calls the homerange_proportion function and uses the return
    to build the convex hull of the data points.

    Returns:
    xy_df: data frame including the point geometry and the id of the input data
    '''

    # get the data into the needed format for the plot
    # xy: Array with all data points with x as column 0 and y as column 1
    # we do not need this here, but hr_proportion gives out 2 variables,
    # so we have to define two
    # xy_new: array only including the set proportion of x and y coordinates
    xy, xy_new = hr_proportion(data, proportion)
    hull = ConvexHull(xy_new)

    # building the geo data frame later used for analysis
    xy_df = geopd.GeoDataFrame(xy_new, geometry=geopd.points_from_xy(xy_new[:,0], xy_new[:,1]), columns = ["x", "y"])
    # adding the id again
    xy_df["id"] = data["id"][data["id"].first_valid_index()]
    # adding the area of the hull
    xy_df["area"] = hull.volume
    # dropping the x and y coordinate columns, as we already have them in the
    # original data frame, with which we will merge this data frame later
    xy_df = xy_df.drop(["x","y"], axis = 1)

    return xy_df

def hr_area(data, proportion = 0.95):
    '''
    Main function to calculate the polygons of the home ranges.

    Input:
    data: dataframe where the x- and y- values of the position are stored
    in the first two columns
    proportion: proportion of positions that should be used for the homerange.
    proportion of positions that should be used for the homerange.

    The function calls the homerange_proportion function.

    Returns:
    xy_geometry: convex hull of the input geometry
    '''
    # xy_new: data frame only including the set proportion of x and y coordinates
    # includes id of the foxes, area of the home range and geometry of the points

    xy_new = hr_geometry_df(data, proportion)

    xy_geometry = xy_new.unary_union.convex_hull

    return xy_geometry


import matplotlib.pyplot as plt
import numpy as np

LOG_FOLDER = 'logs'

def visualize(folder, base):
    logs = []
    for i in range(1, 6):
        with open('{}/{}/{}'.format(LOG_FOLDER, folder, i), 'r') as f:
            logs.append(list(map(float, f.readlines())))
    logs = np.array(logs) - base
    plt.figure(figsize=(12, 9))
    plt.plot(np.arange(logs.shape[1]), np.mean(logs, axis=0))
    plt.savefig('{}/{}_fig.png'.format(LOG_FOLDER, folder), bbox_inches='tight')
    plt.close()

visualize('p3', 0)
visualize('p4', 1000)
visualize('p6', 0)
visualize('p7', 1000)


# pip install xlrd
import xlrd
# pip install requests
import requests
import xlsxwriter
# pip install beautifulsoup4
# pip install lxml
from bs4 import BeautifulSoup

def download_xls():
    """
    Downlaods a .xls file containing data for State Population.
    :param: None
    :return: None
    """
    # URL
    url = "http://mospi.nic.in/statistical-year-book-india/2015/171"
    html_page = requests.get(url).text
    soup = BeautifulSoup(html_page, 'lxml')
    # Get table content
    table_links = []
    page_table = soup.find_all(name="table", class_="views-table")
    for content in page_table:
        table_links = content.find_all(name="a")
    # Try: Check if required_doc exists in table
    try:
        required_doc = "Mid Year Population"
        state_population_href = [link['href'] for link in table_links if required_doc in link.text][0]
        response = requests.get(state_population_href)
        # Open file with filename and write bytes
        filename = "state_population_data.xls"
        with open(filename, 'wb') as xls:
            xls.write(response.content)
            xls.close()
        print("File Downloaded!")
    except Exception as e:
        print("No Data Found!", e)

def modify_xls(quiet=True):
    """
    Modify .xls file to required format.
    :param quiet: (default: True) - Set verbosity of the function.
    :return years: (defualt: []) - A list of years extracted from the .xls file
    :return state_population_dict: (default: {}) - A dictionary of State Names (keys) and their population data (values)
    """
    book = xlrd.open_workbook("state_population_data.xls", on_demand=True)
    sh = book.sheet_by_index(0)
    state_population_dict = {}
    years = []
    for rows in range(sh.nrows):
        cell_values = [text.value for text in sh.row(rows)]
        useful_cell_values = []
        for value in cell_values:
            if value == '':
                continue
            useful_cell_values.append(str(value).strip())
        if len(useful_cell_values) == 0:
            continue
        if "State/" in useful_cell_values[0]:
            years = sorted([int(float(value)) for value in list(set(useful_cell_values[1:]))])
        if len(useful_cell_values) == 9:
            state_population_dict[useful_cell_values[0]] = [int(float(value)) for value in useful_cell_values[1:]]
    book.release_resources()
    del book
    if not quiet:
        print(f"Years: {years}")
        print(f"State Population Data: {state_population_dict}")
    return years, state_population_dict

def write_xlsx(year_list, population_dict):
    """
    Create a new .xlsx file to write required data in required format
    :param year_list: A list of years
    :param population_dict: A dictionary of "State_Name":[Yearly_Population_data_list] pair of key:value
    :return:
    """
    filename = "Yearly_State_Population.xlsx"
    book = xlsxwriter.Workbook(filename)
    sh = book.add_worksheet()
    row = 0
    col = 0
    for index, year in enumerate(year_list, start=1):
        sh.write(row, col + index, year)
    col = 0
    keys = population_dict.keys()
    for key in keys:
        row += 1
        sh.write(row, col, key)
        for index, item in enumerate(population_dict[key], start=1):
            sh.write(row, col + index, item)
    book.close()
    print(f"Sucessfully written file: {filename}")

if __name__ == "__main__":
    download_xls()
    year_list_data, population_data = modify_xls(quiet=True)
    write_xlsx(year_list=year_list_data, population_dict=population_data)


from django.contrib import admin
from .models import *
from django.contrib.auth.models import Group
from django.utils.html import format_html

# Register your models here.

admin.site.site_title = "M&G products "
admin.site.site_header ="M&G product"

class Product_Admin(admin.ModelAdmin):
    def photo_tag(self,obj):
        return format_html(f'<img src = "/media/{obj.image}" style= "height:200px;width:80px">')

class Order_Admin(admin.ModelAdmin):
    fields = ("complete",)

admin.site.unregister(Group),
admin.site.register(Product,Product_Admin),
admin.site.register(Man),
admin.site.register(Order,Order_Admin),


from collections import deque
def solution(n, computers):
    answer = 0
    visited = [0]*n
    graphs = [[] for _ in range(n) ]

    # Init graphs
    for i , computer in enumerate(computers) :
        for j , value in enumerate(computer) :
            if i==j :
                continue
            if value == 1:
                graphs[i].append(j)
    #Node
    for i in range(n):
        # 방문 한적이 없는 경우 새로운 Network 연결
        if not visited[i] :
            answer+=1
            q = deque()
            visited[i]=1
            for data in graphs[i] :
                q.append(data)

            # Network 연결
            while q :
                data = q.popleft()
                if not visited[data] :
                    visited[data]=1
                    for value in graphs[data] :
                        q.append(value)
    return answer

프로그래머스 - 네트워크.py

프로그래머스 - 네트워크.py


from collections import OrderedDict
from os import path
import os

# Flag to control if errors are raise or prints and returns
RAISE_ERRORS = False

def add_vectors(vector_1, vector_2):
    """
    Returns a list object representing the result of adding two vectors together.

    Arguments:
    vector_1 -- list representing first vector
    vector_2 -- list representing second vector

    Error checking:
    Both arguments must be of type list.
    Both vectors must of the same length.
    Only vector elements of type int can be added together.
    """
    if type(vector_1) is not list:
        if RAISE_ERRORS:
            raise TypeError('vector_1 must be of type list, got {}'
                            .format(type(vector_1)))
        else:
            print('Error: first argument is not a list')
            return

    if type(vector_2) is not list:
        if RAISE_ERRORS:
            raise TypeError('vector_2 must be of type list, got {}'
                            .format(type(vector_2)))
        else:
            print('Error: second argument is not a list')
            return

    if len(vector_1) != len(vector_2):
        if RAISE_ERRORS:
            raise ValueError('the lengths of vector_1 and vector_2 must be equal')
        else:
            print('Error: lengths of the two vectors are different')

    added_vec = []
    for i in range(len(vector_1)):
        if type(vector_1[i]) is not int:
            if RAISE_ERRORS:
                raise TypeError('vector_1[{}] was expected to be of type int but was {}'
                                .format(i, type(vector_1[i])))
            else:
                print('attempted to add incompatible {} to {}'.format(vector_1[i], vector_2[i]))
                return

        if type(vector_2[i]) is not int:
            if RAISE_ERRORS:
                raise TypeError('vector_2[{}] was expected to be of type int but was {}'
                                .format(i, type(vector_2[i])))
            else:
                print('attempted to add incompatible {} to {}'.format(vector_1[i], vector_2[i]))
                return
        added_vec.append(vector_1[i] + vector_2[i])

    return added_vec

def print_frequency(some_text):
    """
    Prints a table of letter frequencies within a string.

    Non-letter characters are ignored.
    Table is sorted alphabetically.
    Letter case is ignored.
    Two blank spaces will separate the letter from its count.

    Returns None in all cases.

    Argument:
    some_text -- string containing the text to be analysed.

    Error checking:
    The argument must be a string object.
    """
    if type(some_text) is not str:
        if RAISE_ERRORS:
            raise TypeError('some_text must be of type str, got {}'
                            .format(type(some_text)))
        else:
            print('Error: only accepts strings')
            return

    char_count = OrderedDict()
    for char in list(some_text.lower()):
        if not char.isalpha():
            continue

        if char in char_count:
            char_count[char] += 1
        else:
            char_count[char] = 1

    to_return = None
    if len(char_count) != 0:
        to_return = ""
        for char in char_count:
            to_return += "{}  {}\n".format(char, char_count[char])

    return to_return

def verbing(some_text):
    """
    Returns a string where the each word has ing added to it if it is 3
    or more characters or length and ly to shorter words.

    Argument:
    some_text -- string containing the text to be analysed.

    Error checking:
    The argument must be a string object.
    """
    if type(some_text) is not str:
        if RAISE_ERRORS:
            raise TypeError('some_text must be of type str, got {}'
                            .format(type(some_text)))
        else:
            print('Error: only accepts strings')
            return

    words = some_text.split()
    for i in range(len(words)):
        if len(words[i]) >= 3:
            words[i] += 'ing'
        else:
            words[i] += 'ly'

    return " ".join(words)

def verbing_file(file_name):
    """
    Returns the contents of a given file after applying the verbing function to each
    line in the file.

    Argument:
    file_name -- name of the file (assumed to exist in same directory from where the
                python script is executed.

    Error checking:
    The argument must be a string object.
    File must exist and be readable (note no need to distinguish these cases).
    """
    if type(file_name) is not str:
        if RAISE_ERRORS:
            raise TypeError('file_name must be of type str, got {}'
                            .format(type(file_name)))
        else:
            print('Error: only accepts strings')
            return

    # these raises are not hidden by RAISE_ERRORS because they are undefined from the assignemnt
    if not path.exists(file_name):
        raise ValueError('File {} does not exist')

    if not path.isfile(file_name):
        raise ValueError('{} is not a file')

    if not path.isfile(file_name):
        raise ValueError('{} is not a file')

    if not os.access(file_name, os.R_OK):
        raise ValueError('Cannot read from {}')

    lines = ""
    with open(file_name, 'r') as file:
        for line in file.readlines():
            lines += verbing(line) + '\n'

    return lines


import numpy as np
import matplotlib.pyplot as plt

Size = [768,1024]
MFT = np.zeros(Size)

Xpixels = np.arange(Size[1])
Ypixels = np.arange(Size[0])

Frecuency_B = np.log((Size[1]/2))/Size[1]
Frecuency_A = 1/(Frecuency_B*Size[1])
Frecuency = Frecuency_A*np.exp(Frecuency_B*Xpixels)
VaryngSin = 0.5*np.sin(2*np.pi*Frecuency)

Atenuacion_C = 16/Size[0]
Atenuacion =  np.exp(-1*Ypixels*Atenuacion_C)
Atenuacion = Atenuacion

MFT = np.matmul(Atenuacion[::-1].reshape([Size[0],1]), VaryngSin.reshape([1,Size[1]]))+0.5
MFT = MFT.clip(min=0.0,max=1.0)
##MFT = np.round(MFT * 200.0)

fig = plt.figure("Señales")
ax = fig.add_subplot(1, 2, 1)
plt.plot(Xpixels, VaryngSin)

ax = fig.add_subplot(1, 2, 2)
plt.plot(Ypixels, Atenuacion)

plt.imsave('test.jpg', MFT, cmap = plt.cm.gray)

fig = plt.figure("Imagen")
plt.imshow(MFT)
plt.axis("off")

plt.show()


import torch
import argparse
import json
import clip
import pickle
from tqdm import tqdm
from PIL import Image
from pathlib import Path
from transformers.models.gpt2.tokenization_gpt2 import GPT2Tokenizer

class COCOCaption:
    def __init__(self, annot_path: str, dataset_path: str, save_path: str) -> None:
        self.dataset_path = Path(dataset_path)
        self.save_path = Path(save_path)
        self.save_path.mkdir(parents=True, exist_ok=True)
        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

        self.model, self.preprocess = clip.load("ViT-B/32", device=self.device)
        self.tokenizer = GPT2Tokenizer.from_pretrained("gpt2")

        with open(annot_path) as f:
            dataset = json.load(f)
            self.annotations = dataset['images']
            self.dataset_name = dataset['dataset']

    def prepare(self, split):
        data = []
        annotations = list(filter(lambda x: x['split'] == split, self.annotations))

        print(f"Preparing {self.dataset_name} {split} split...")
        for annot in tqdm(annotations):
            caption = annot['sentences'][0]['raw']

        #     if self.dataset_name == "coco":
        #         image_path = self.dataset_path / annot['filepath'] / annot['filename']
        #         image_id = annot['cocoid']
        #     elif self.dataset_name == "flickr8k":
        #         image_path = self.dataset_path / "Images" / annot['filename']
        #         image_id = annot['imgid']
        #     elif self.dataset_name == "flickr30k":
        #         image_path = self.dataset_path / "flickr30k_images" / annot['filename']
        #         image_id = annot['imgid']

        #     image = self.preprocess(Image.open(image_path)).unsqueeze(0).to(self.device)

        #     with torch.inference_mode():
        #         image_features = self.model.encode_image(image).cpu()

        #     token = torch.tensor(self.tokenizer.encode(caption), dtype=torch.int64)

        #     data.append({
        #         "image_id": image_id,
        #         "embedding": image_features,
        #         "caption": caption,
        #         "token": token
        #     })

        # with open(self.save_path / f"{self.dataset_name}_{split}.pkl", 'wb') as f:
        #     pickle.dump(data, f)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--annot-path', type=str, default='data/dataset_flickr30k.json')
    parser.add_argument('--dataset-path', type=str, default='C:\\Users\\sithu\\Documents\\Datasets\\Flickr30k')
    parser.add_argument('--save-path', type=str, default='data/Flickr30k')
    args = parser.parse_args()

    dataset = COCOCaption(args.annot_path, args.dataset_path, args.save_path)

    for split in ['test', 'val', 'train']:
        dataset.prepare(split)


import h5py as h5
import pandas as pd
import glob
import matplotlib.pyplot as plt
import numpy as np
from pyproj import proj

data_file = sorted(glob.glob('/Users/isamarcortes/Dropbox/Isamar/Satellite_Imagery_Analysis/2015_2020Precip_EvapData/*.HDF5')) ## imports h5 files
list = [] #empty list to put h5 files in

for i in data_file:
    list.append(i)
    read = h5.File(i,'r')
    new = read['Grid']
    surfacePrecipitation = new['surfacePrecipitation'][...,]
    #rainwater = new['rainWater'][...,]
    #npixtotal = new['npixTotal']

    surfacePrecipitation1 = np.where(surfacePrecipitation>0, surfacePrecipitation, np.nan)
    test = np.transpose(surfacePrecipitation1)
    #print(read.keys())
    ###Indexing based on converted coordinates per region
    Venezuela = test[406,445]
    PR1 = test[430, 446]
    PR3 = test[430, 448]
    PR6 = test[430, 446]
    MX = test[445, 526]
    BarnesK = test[458, 502]
    ButtonwoodK=test[460, 502]
    CrabK = test[458, 502]
    CraneK= test[460, 501]
    FL1= test[460, 502]
    FL3 = test[460, 501]
    FL5 = test[460, 502]
    ShellK = test[458, 502]
    SidK = test[460, 502]
    UpperArsK = test[458, 502]
    Cuba2 = test[448, 506]
    Cuba3 = test[449,509]
    Belize1 = test[429, 529]
    Bahamas1= test[466, 493]
    index = np.linspace(-90,90,720)
    columns = np.linspace(-180,180,1440)

    test1 = pd.DataFrame(data = test,index=index, columns=columns)
    #test1.to_csv('/Users/isamarcortes/Desktop/HDF_Files/test.csv')

    #print(test1.head())
    #print(test1)
    #print(Bahamas1)

'''

#z_array=np.empty((1440,720))
#z_array[index, columns] = test
print(type(test))

plt.contourf(test)
plt.colorbar()
plt.xlabel('Latitude')
plt.ylabel('Longitude')
plt.title('Original')
plt.show()

plt.contourf(columns,index,test)
plt.colorbar()
plt.xlabel('Latitude')
plt.ylabel('Longitude')
plt.title('Lat Lon')
plt.show()

'''