import time
# from kflod import kfold
import numpy as np
import random
from torch.optim.adam import Adam
from torch import optim
from resnet_attn_new import *
from torchvision.models.resnet import resnet50, resnet18
from torchvision.models.densenet import densenet121
from preprocessing import get_dataset3d
import sys

def expAllAtn(data_path):
    reset_rand()
    def model_opt():
        model = AllAtn()
        optm = Adam(model.parameters())
        return model, optm

    all_targets,  all_pred=kfold(data_path,
          256,
          50,
          model_optimizer=model_opt,
          loss=nn.BCELoss(),
          name='BasicResnet',
          device='cuda:0',
          deterministic=True
          )
    return all_targets,  all_pred

def expBasicResnet(data_path):
    reset_rand()

    def model_opt():
        model = BasicResnet()
        optm = Adam(model.parameters())
        return model, optm

    all_targets,  all_pred=kfold(data_path,
          256,
          50,
          model_optimizer=model_opt,
          loss=nn.BCELoss(),
          name='BasicResnet',
          device='cuda:0',
          deterministic=True
          )
    return all_targets,  all_pred

from torchsummary import summary

def expLocalGlobal(data_path):
    reset_rand()

    def model_opt():
        model = LocalGlobalNetwork()
        # summary(model, (1, 32, 32))
        optm=optim.RMSprop(model.parameters())
        # optm = Adam(model.parameters())
        return model, optm

    all_targets,  all_pred=kfold(data_path,
          256,
          50,
          model_optimizer=model_opt,
          loss=nn.BCELoss(),
          name='LocalGlobalNetwork',
          device='cuda:0',
          deterministic=True
          )
    return all_targets,  all_pred

def expLocalGlobal_old(data_path):
    reset_rand()

    def model_opt():
        model = LocalGlobalNetwork_old()
        # summary(model, (1, 32, 32))
        optm = Adam(model.parameters())
        return model, optm

    all_targets,  all_pred=kfold(data_path,
          256,
          50,
          model_optimizer=model_opt,
          loss=nn.BCELoss(),
          name='LocalGlobalNetwork_old',
          device='cuda:0',
          deterministic=True
          )
    return all_targets,  all_pred

def expResnetTrans(data_path):
    reset_rand()

    def model_opt():
        model = resnet50(pretrained=True)
        model.avgpool = nn.AdaptiveAvgPool2d((1, 1))
        model.fc = nn.Sequential(
            nn.Linear(model.fc.in_features, 1),
            nn.Sigmoid()
        )

        optm = Adam(model.fc.parameters())
        return model, optm

    all_targets,  all_pred=kfold(data_path,
          256,
          50,
          model_optimizer=model_opt,
          loss=nn.BCELoss(),
          name='ResnetTrans',
          device='cuda:0',
          deterministic=True,
          dataset_func=get_dataset3d
          )
    return all_targets,  all_pred

def expDensenetTrans(data_path):
    reset_rand()

    def model_opt():
        model = densenet121(pretrained=True)
        model.classifier = nn.Sequential(
            nn.Linear(model.classifier.in_features, 1),
            nn.Sigmoid()
        )

        optm = Adam(model.classifier.parameters())
        return model, optm

    all_targets,  all_pred=kfold(data_path,
          256,
          50,
          model_optimizer=model_opt,
          loss=nn.BCELoss(),
          name='DensenetTrans',
          device='cuda:0',
          deterministic=True,
          #dataset_func=get_dataset3d
          )
    return all_targets,  all_pred

def expResnet18Trans(data_path):
    reset_rand()

    def model_opt():
        model = resnet18(pretrained=True)

        model.avgpool = nn.AdaptiveAvgPool2d((1, 1))
        model.fc = nn.Sequential(
            nn.Linear(model.fc.in_features, 1),
            nn.Sigmoid()
        )

        optm = Adam(model.fc.parameters())
        return model, optm

    all_targets,  all_pred=kfold(data_path,
          256,
          50,
          model_optimizer=model_opt,
          loss=nn.BCELoss(),
          name='Resnet18Trans',
          device='cuda:0',
          deterministic=True,
          dataset_func=get_dataset3d
          )
    return all_targets,  all_pred

def print_error():
    print('python <model_name> <data_path>')
    print('here is a list of experiments names:')
    for name in experiments.keys():
        print(name)

from trainer import Trainer
from preprocessing import get_dataset
from os import path
from torch.utils.data import DataLoader
import torch as T
import numpy as np
import random
from sklearn import metrics
from os import path
import matplotlib.pyplot as plt

def get_metrics(target, pred):
    prec, recall, f1, _ = metrics.precision_recall_fscore_support(target, pred>0.5, average='binary')
    accu=metrics.accuracy_score(target, pred>0.5)
    fpr, tpr, thresholds = metrics.roc_curve(target, pred)
    auc = metrics.auc(fpr, tpr)
    return prec, recall, auc, f1, accu

def calc_accuracy(x, y):
    x_th = (x > 0.5).long()
    matches = x_th == y.long()
    return matches

def reset_rand():
    seed = 1000
    T.manual_seed(seed)
    T.cuda.manual_seed(seed)
    np.random.seed(seed)
    random.seed(seed)

def kfold(src_path,
          batch_size,
          n_epochs,
          model_optimizer,
          loss,
          name,
          device,
          deterministic=False,
          parallel=False,
          dataset_func=get_dataset):

    print('------Experiment:',name,'------')
    all_pred = T.zeros(849)
    all_targets = T.zeros(849)
    i =0
    f = open(path.join('results', '{name}.txt'), 'w')
    f.write('{batch_size} {n_epochs} {model_optimizer}\n')
    for fold in range(10):

        reset_rand()

        print('------------ fold' + str(fold+1) + ' ------------')
        f.write('------------ fold {fold+1} ------------\n')
        trset, testset = dataset_func(path.join(src_path,str(fold)))
        print('Training Size: ' + str(len(trset)) + ' Validation Size: '+ str(len(testset)))
        trset = DataLoader(trset, batch_size, shuffle=True)
        testset = DataLoader(testset, batch_size, shuffle=False)
        model,optimizer = model_optimizer()
        tr = Trainer(
            trset,
            testset,
            batch_size,
            n_epochs,
            model,
            optimizer,
            loss,
            '{name}_{fold}',
            device,
            deterministic,
            parallel
        )

        tr.run()

        pred, target = tr.predict()
        all_pred[i:i+pred.shape[0]] = pred
        all_targets[i:i+target.shape[0]] = target
        i += target.shape[0]

        prec, recall, auc, f1, accu = get_metrics(target, pred)
        print('AUC: ' + str(auc) + ', '  + 'Accuracy: ' + str(accu) + ', ' +  'precession: ' + str(prec) + ', ' + 'Recall: ' +  str(recall) + ', ' + 'F1score: ' +str(f1))
        f.write('AUC: ' + str(auc) + ', '+ 'Accuracy: ' + str(accu) + ', '  +  'precession: ' + str(prec) + ', ' + 'Recall: ' +  str(recall) + ', ' + 'F1score: ' +str(f1) +'\n')

    matches = calc_accuracy(all_pred, all_targets)
    acc = matches.float().mean()
    all_pred = all_pred.numpy()
    all_targets = all_targets.numpy()

    prec, recall, auc, f1, accu = get_metrics(all_targets, all_pred)
    print('Accuracy: ' + str(accu) + ', ' + 'AUC: ' + str(auc) + ', '  +  'precession: ' + str(prec) + ',' + 'Recall: ' +  str(recall)+ ', ' + 'f1score: ' + str(f1)+ '\n')
    f.write('accuray: {acc}, accuray1: {accu}, AUC: {auc}, precession: {prec}, Recall: {recall}, f1score: {f1}')
    result = {'all_pred': all_pred, 'all_targets': all_targets}
    T.save(result, path.join('results','{name}_result'))
    f.close()
    return all_targets,  all_pred

if __name__ == '__main__':
    experiments = {
        'Resnet18Trans' : expResnet18Trans,
        'ResnetTrans' : expResnetTrans,
        'LocalGlobal_cbam' : expLocalGlobal,
        'LocalGlobal':expLocalGlobal_old,
        'BasicResnet' : expBasicResnet,
        'AllAtn' : expAllAtn,
        'DensenetTrans' : expDensenetTrans
    }

    #################################
    exp_name = 'Resnet18Trans'
    data_path = '/home/nisa.hameed/Downloads/Lung_class/lidc_img'

    st = time.time()
    all_targets,  all_pred=experiments[exp_name](data_path)
    et = time.time()
    elapsed_time = et - st
    print(' Total execution time for ',exp_name,'is', elapsed_time, 'seconds')

    #plot visualization
    fpr, tpr, _ = metrics.roc_curve(all_targets,  all_pred)
    auc = metrics.roc_auc_score(all_targets, all_pred)
    p1=plt.plot(fpr,tpr,label="AUC="+str(auc))
    # plt.ylabel('True Positive Rate')
    # plt.xlabel('False Positive Rate')
    # # plt.legend(loc=4)
    # plt.show()

# ########################################
    exp_name = 'ResnetTrans'
    data_path = '/home/nisa.hameed/Downloads/Lung_class/lidc_img'

    st = time.time()
    all_targets,  all_pred=experiments[exp_name](data_path)
    et = time.time()
    elapsed_time = et - st
    print(' Total execution time for ',exp_name,'is', elapsed_time, 'seconds')

    fpr, tpr, _ = metrics.roc_curve(all_targets,  all_pred)
    auc = metrics.roc_auc_score(all_targets, all_pred)
    p2=plt.plot(fpr,tpr,label="AUC="+str(auc))
    # plt.ylabel('True Positive Rate')
    # plt.xlabel('False Positive Rate')
#     # plt.legend(loc=4)
#     # plt.show()

#     ###########################################
    exp_name = 'AllAtn'
    data_path = '/home/nisa.hameed/Downloads/Lung_class/lidc_img'

    st = time.time()
    all_targets,  all_pred=experiments[exp_name](data_path)
    et = time.time()
    elapsed_time = et - st
    print(' Total execution time for ',exp_name,'is', elapsed_time, 'seconds')

    #added for plot
    fpr, tpr, _ = metrics.roc_curve(all_targets,  all_pred)
    auc = metrics.roc_auc_score(all_targets, all_pred)
    p3=plt.plot(fpr,tpr,label="AUC="+str(auc))
#     # plt.ylabel('True Positive Rate')
#     # plt.xlabel('False Positive Rate')
#     # plt.legend(loc=4)
#     # # plt.show()

    ###########################################
    exp_name = 'LocalGlobal'
    data_path = '/home/nisa.hameed/Downloads/Lung_class/lidc_img'

    st = time.time()
    all_targets,  all_pred=experiments[exp_name](data_path)
    et = time.time()
    elapsed_time = et - st
    print(' Total execution time for ',exp_name,'is', elapsed_time, 'seconds')

    # #added for plot
    fpr, tpr, _ = metrics.roc_curve(all_targets,  all_pred)
    auc = metrics.roc_auc_score(all_targets, all_pred)
    p4=plt.plot(fpr,tpr,label="AUC="+str(auc))

  ###################################
    exp_name = 'LocalGlobal_cbam'
    data_path = '/home/nisa.hameed/Downloads/Lung_class/lidc_img'

    st = time.time()
    all_targets,  all_pred=experiments[exp_name](data_path)
    et = time.time()
    elapsed_time = et - st
    print(' Total execution time for ',exp_name,'is', elapsed_time, 'seconds')

    fpr, tpr, _ = metrics.roc_curve(all_targets,  all_pred)
    auc = metrics.roc_auc_score(all_targets, all_pred)
    p5=plt.plot(fpr,tpr,label="AUC="+str(auc))
    #visualization
    plt.ylabel('True Positive Rate')
    plt.xlabel('False Positive Rate')
    # plt.legend((p1,p2,p3,p4,p5),('RESNET18','RESNET50','DENSENET121','LOCAL_GLOBAL','ATTENTION_LOCAL_GLOBAL'),loc='lower left')
    plt.legend((p4,p5),('LOCAL_GLOBAL','ATTENTION_LOCAL_GLOBAL'),loc=4)
    plt.show()


import pygame, random
from pygame.locals import *

def on_grid_random():
    x = random.randint(0, 590)
    y = random.randint(0, 590)
    return x // 10 * 10, y // 10 * 10

def collision(c1, c2):
    return (c1[0] == c2[0]) and (c1[1] == c2[1])

UP = 0
RIGHT = 1
DOWN = 2
LEFT = 3

pygame.init()
screen = pygame.display.set_mode((800, 800))
pygame.display.set_caption('Cobrinha')

cobrinha = [(300, 300), (310, 300), (320, 300)]
cobrinha_skin = pygame.Surface((10, 10))
cobrinha_skin.fill((0, 255, 255))

apple_pos = on_grid_random()
apple = pygame.Surface((10, 10))
apple.fill((255, 255, 255))

my_direction = LEFT

clock = pygame.time.Clock()

while True:
    clock.tick(15)
    for event in pygame.event.get():
        if event.type == QUIT:
            pygame.quit()

        if event.type == KEYDOWN:
            if event.key == K_UP:
                my_direction = UP
        if event.type == KEYDOWN:
            if event.key == K_DOWN:
                my_direction = DOWN
        if event.type == KEYDOWN:
            if event.key == K_LEFT:
                my_direction = LEFT
        if event.type == KEYDOWN:
            if event.key == K_RIGHT:
                my_direction = RIGHT

    if collision(cobrinha[0], apple_pos):
        apple_pos = on_grid_random()
        cobrinha.append((0, 0))

    for i in range(len(cobrinha) - 1, 0, -1):
        cobrinha[i] = (cobrinha[i - 1][0], cobrinha[i - 1][1])

    if my_direction == UP:
        cobrinha[0] = (cobrinha[0][0], cobrinha[0][1] - 10)
    if my_direction == DOWN:
        cobrinha[0] = (cobrinha[0][0], cobrinha[0][1] + 10)
    if my_direction == RIGHT:
        cobrinha[0] = (cobrinha[0][0] + 10, cobrinha[0][1])
    if my_direction == LEFT:
        cobrinha[0] = (cobrinha[0][0] - 10, cobrinha[0][1])

    screen.fill((0, 0, 0))
    screen.blit(apple, apple_pos)
    for pos in cobrinha:
        screen.blit(cobrinha_skin, pos)

    pygame.display.update()


# This is the file that implements a flask server to do inferences. It's the file that you will modify to
# implement the scoring for your own algorithm.

from __future__ import print_function

import os
import json
import pickle
import sys
import signal
import traceback

import flask

import math
import numpy as np
import mxnet as mx
from mxnet import nd
from mxnet.gluon.data.vision import transforms
import gluoncv as gcv
from gluoncv.data.transforms import image as timage

prefix = '/opt/ml/'
model_path = os.path.join(prefix, 'model')
artifact_file = os.path.join(model_path, 'artifact.json')
model_name = None
with open(artifact_file) as json_file:
    artifact = json.load(json_file)
    model_name = artifact['model_name']
if not model_name:
    raise RuntimeError('Unable to determine saved model name')

# A singleton for holding the model. This simply loads the model and holds it.
# It has a predict function that does a prediction based on the model and the input data.

class ScoringService(object):
    model = None                # Where we keep the model when it's loaded

    @classmethod
    def get_model(cls):
        """Get the model object for this instance, loading it if it's not already loaded."""
        if cls.model == None:
            model = gcv.model_zoo.get_model(model_name, pretrained=False, pretrained_base=False)
            model.load_parameters(os.path.join(model_path, 'model.params'), allow_missing=True)
            model.hybridize()
            cls.model = model
        return cls.model

    @classmethod
    def predict(cls, input):
        """For the input, do the predictions and return them.
        """
        clf = cls.get_model()
        output = clf(input)[0]
        predict = mx.nd.squeeze(mx.nd.argmax(output, 1)).asnumpy().astype('int32')
        return predict

# The flask app for serving predictions
app = flask.Flask(__name__)

@app.route('/ping', methods=['GET'])
def ping():
    """Determine if the container is working and healthy. In this sample container, we declare
    it healthy if we can load the model successfully."""
    health = ScoringService.get_model() is not None  # You can insert a health check here

    status = 200 if health else 404
    return flask.Response(response='\n', status=status, mimetype='application/json')

@app.route('/invocations', methods=['POST'])
def transformation():
    """Do an inference on a single batch of data. In this sample server, we take image data, decode
    image to raw frame, and apply prediction.
    """
    def preprocess(raw_image_buf, size=480,
               mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)):
        orig_image = mx.img.imdecode(raw_image_buf)
        img = timage.imresize(orig_image, size, size)
        img = mx.nd.image.to_tensor(img)
        img = mx.nd.image.normalize(img, mean=mean, std=std)
        return img.expand_dims(0), orig_image

    if flask.request.content_type in ['image/jpeg', 'image/png']:
        raw = flask.request.data
        data, orig_image = preprocess(raw)
    else:
        return flask.Response(response='This predictor only supports image data, {}'.format(flask.request.content_type), status=415, mimetype='text/plain')

    print('Invoked with {} records'.format(data.shape[0]))

    # Do the prediction
    pred = ScoringService.predict(data)

    # Convert from output to readable text
    if model_name == 'fastscnn_citys':
        classes = ('road', 'sidewalk', 'building', 'wall', 'fence', 'pole', 'traffic light', 'traffic sign', 'vegetation', 'terrain', 'sky', 'person', 'rider', 'car', 'truck', 'bus', 'train', 'motorcycle', 'bicyle')
    else:
        classes = ScoringService.get_model().classes
    result = json.dumps({'prediction': pred.tolist(), 'classes': classes})
    return flask.Response(response=result, status=200, mimetype='application/json')


#! usr/bin/python3

"""
This Program encrypts and decrypts messages
"""

import pyperclip

def Cipher():
    try:

        # mode ="encrypt" # or decrypt
        mode = input("Do you want to \" encrypt \" or \" decrypt\" ? >>")

        message=input("Insert the Message you want to {} >>".format(mode))

        #encrypting and decrypting key
        key = int(input("Please insert your {} key >>".format(mode)))

        LETTERS= "ABCDEFGHIJKLMNOPQRSTUVWXYZ"

        translated=''

        message= message.upper()

        for symbol in message:
            if symbol in LETTERS:
                num=LETTERS.find(symbol)
                if mode=="encrypt":
                    num=num+key
                elif mode=="decrypt":
                    num=num-key
                if  num >= len(LETTERS):
                    num= num- len(LETTERS)
                elif num < 0:
                    num= num + len(LETTERS)
                translated= translated + LETTERS[num]
            else:
                translated=translated + symbol

        print("")
        print("*****************************************")
        print("The {}ed message is: ".format(mode))
        print (translated)

        pyperclip.copy(translated)
    except:
        print("Check what you inputted")
        Cipher()

Cipher()


from django.urls import include, path
from rest_framework import routers

from core.views import ProductViewSet, UserViewSet, ContactViewSet, SupplierStatisticView,\
    SupplierListCreateAPIView, SupplierUpdateDeleteAPIView

app_name = "core"

router = routers.SimpleRouter()

router.register(r"product", ProductViewSet, basename='product')
router.register(r"user", UserViewSet, basename='user')
router.register(r"contact", ContactViewSet, basename='contact')

urlpatterns = [
    path("", include(router.urls)),
    path('debt', SupplierStatisticView.as_view(), name='debt'),
    path('supplier/', SupplierListCreateAPIView.as_view(), name='supplier_list'),
    path('supplier_update/<int:pk>/', SupplierUpdateDeleteAPIView.as_view(), name='supplier_update_delete'),
]


import requests
from urllib.parse import urlencode
import os
from hashlib import md5

headers = {
    'cookie': '__tasessionId=ampozol5g1563806844189; tt_webid=671649' +
    '9234256455175; WEATHER_CITY=%E5%8C%97%E4%BA%AC; UM_distinctid=16c' +
    '1a26e5e12df-040ada0c1d8081-c343162-100200-16c1a26e5e428c; CNZZDAT' +
    'A1259612802=945869726-1563806204-https%253A%252F%252Fwww.google.c' +
    'om%252F%7C1563806204; tt_webid=6716499234256455175; csrftoken=46' +
    'a41d4141000920aea9354904736a2d; s_v_web_id=2b8f7242614ce9f3dd7cbf' +
    '26d932530d',
    'host': 'www.toutiao.com',
    'referer': 'https://www.toutiao.com/search/?keyword=%E8%A1%97%E6%8B%8D',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit' +
    '/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36',
    'x-requested-with': 'XMLHttpRequest'
}

def get_page(offset):
    params = {
        'aid': '24',
        'app_name': 'web_search',
        'offset': offset,
        'format': 'json',
        'keyword': '街拍',
        'autoload': 'true',
        'count': '20',
        'en_qc': '1',
        'cur_tab': '1',
        'from': 'search_tab',
        'pd': 'synthesis',
        'timestamp': '1563782026489'
    }
    url = 'https://www.toutiao.com/api/search/content/?' + urlencode(params)
    print(url)
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            print('suc_1')
            # print(response.json())
            return response.json()
    except requests.ConnectionError:
        return None

def get_images(json):
    if json.get('data'):
        for item in json.get('data'):
            title = item.get('title')
            images = item.get('image_list')
            if images:
                for image in images:
                    print('suc_2')
                    yield {
                        'image': image.get('url'),
                        'title': title
                    }
    else:
        print('data is none')

def save_image(item):
    if not os.path.exists(item.get('title')):
        os.mkdir(item.get('title'))
    try:
        print('suc_3')
        response = requests.get(item.get('image'))
        if response.status_code == 200:
            file_path = '{0}/{1}.{2}'.format(
                item.get('title'),
                md5(response.content).hexdigest(), 'jpg')
            if not os.path.exists(file_path):
                with open(file_path, 'wb') as f:
                    f.write(response.content)
    except requests.ConnectionError:
        print('Failed to save Image')

def main(offset):
    json = get_page(offset)
    for item in get_images(json):
        save_image(item)

if __name__ == '__main__':
    groups = [x * 20 for x in range(1, 5)]
    os.chdir('../weibo_ajax/images')    # 提前切换到图片存放目录
    for val in groups:
        main(val)


import operator
import torch
from base64 import b64decode
from json import loads
import numpy as np
from matplotlib.pyplot import figure
import matplotlib as plt
from sklearn.naive_bayes import MultinomialNB, BernoulliNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from torch.utils.data import DataLoader
from torch.utils.data import Dataset
import torch.nn as nn

def read_data(json_file):
    json_object = loads(json_file)
    json_data = b64decode(json_object["data"])
    digit_vector = np.fromstring(json_data, dtype=np.ubyte)
    digit_vector = digit_vector.astype(np.float64)
    return json_object["label"], digit_vector

with open("digits.base64.json", "r") as f:
    digits = [read_data(each) for each in f.readlines()]

training_size = int(len(digits) * 0.25)
test = np.array(digits[:600])
validation = np.array(digits[600:training_size])
training = np.array(digits[training_size:])

X = list(training[:, 1])
print(type(X))
print(type(X[0]))

Y = np.array(training[:, 0]).astype("int")

X_val = list(validation[:, 1])
Y_val = np.array(validation[:, 0]).astype("int")

Test_X = list(test[:, 1])
Test_Y = np.array(test[:, 0]).astype("int")

# KNN
#
#
#
# clf = KNeighborsClassifier(n_neighbors=10)
# # clf.fit(X,Y)
# print("Test set accuracy: {:.2f}".format(clf.score(X_val, Y_val)))
#
# pre = clf.predict(Test_X)
#
#
# print(clf.score(Test_X, Test_Y))
#
#
#
# #Decision Tree
#
# tree = DecisionTreeClassifier(random_state=0) # overfit
# tree.fit(X,Y)
# print("Train set accuracy: {:.2f}".format(tree.score(X, Y)))
# print("Test set accuracy: {:.2f}".format(tree.score(X_val, Y_val)))
#
# tree = DecisionTreeClassifier(max_depth=5,random_state=0)
# tree.fit(X,Y)
# print("Train set accuracy: {:.2f}".format(tree.score(X, Y)))
# print("Test set accuracy: {:.2f}".format(tree.score(X_val, Y_val)))

# Naive Bayes

# nbM = MultinomialNB()
# nbM.fit(X,Y)
# print(nbM.score(X_val,Y_val))
#
# nbB = BernoulliNB()
# nbB.fit(X,Y)
# print("Bernoulli form: {:.2f}".format(nbB.score(X_val,Y_val)))

# KNN program
# def classifyKNN(testX, dataset, label, k):
#     datasetSize = dataset.shape[0]
#     diffMat = np.tile(testX, (datasetSize, 1)) - dataset
#     sqDiffMat = diffMat**2
#     sqDistance = sqDiffMat.sum(axis=1)
#     distance = sqDistance ** 0.5
#     sortDistance = distance.argsort()
#
#     classCount = {}
#     for i in range(k):
#         voteLabel = label[sortDistance[i]]
#         classCount[voteLabel] = classCount.get(voteLabel, 0) + 1
#
#     sortedClassCount = sorted(classCount.items(), key=operator.itemgetter(1), reverse=True)
#
#     return sortedClassCount[0][0]
#
#
# predY = []
#
# a = np.array(X)
#
# for testX in Test_X:
#     predY.append(classifyKNN(testX, a, Y, 6))
#
# score = 0
# for i in range(len(predY)):
#     if predY[i] == Test_Y[i]:
#         score += 1
#
# print(score)
# print("KNN score is {:.2f}".format(score / len(Test_Y)))
#

# SVM
# clf = SVC(C = 5)
# clf.fit(X,Y)
#
# print("SVM sklearn RBF kernal: {:.2f}".format(clf.score(X_val,Y_val)))

# clf = SVC(C=5, kernel= 'sigmoid')
# clf.fit(X,Y)
# print("SVM sklearn Sigmoid kernal: {:.2f}".format(clf.score(X_val,Y_val)))

# clf = SVC(C=5, kernel= 'linear')
# clf.fit(X,Y)
# print("SVM sklearn Linear kernal: {:.2f}".format(clf.score(X_val,Y_val)))

# Neural necwork

def get_device():
    return 'cuda' if torch.cuda.is_available() else 'cpu'

class NUMDataset(Dataset):
    def __init__(self, mode="train"):
        self.mode = mode
        if mode == 'train':
            self.X_data = torch.tensor(np.array(X))
            print(self.X_data.shape)
            self.Y_data = torch.tensor(np.array(Y))
            print(self.Y_data.shape)
        elif mode == 'val':
            self.X_data = torch.tensor(np.array(X_val))
            self.Y_data = torch.tensor(np.array(Y_val))
        else:
            self.X_data = torch.tensor(np.array(Test_X))
            self.Y_data = torch.tensor(np.array(Test_Y))

    def __getitem__(self, index):
        if self.mode in ['train', 'val']:
            return self.X_data[index], self.Y_data[index]
        else:
            return self.X_data[index]

    def __len__(self):
        return len(self.X_data)

def prep_dataLoader(mode, batch_size):
    dataset = NUMDataset(mode)
    dataloader = DataLoader(
        dataset, batch_size,
        shuffle=True
    )
    return dataloader

BATCH_SIZE = 1
train_len = NUMDataset('train').__len__()
train_loader = prep_dataLoader(mode='train', batch_size=BATCH_SIZE)
val_len = NUMDataset('val').__len__()
val_loader = prep_dataLoader(mode='val', batch_size=BATCH_SIZE)

class Classifier(nn.Module):
    def __init__(self):
        super(Classifier, self).__init__()

        self.layer1 = nn.Linear(784, 2048)
        self.layer2 = nn.Linear(2048, 512)
        self.layer3 = nn.Linear(512, 256)
        self.out = nn.Linear(256, 10)

        self.act_fn = nn.Sigmoid()

    def forward(self, x):
        x = self.layer1(x)
        x = self.act_fn(x)

        x = self.layer2(x)
        x = self.act_fn(x)

        x = self.layer3(x)
        x = self.act_fn(x)

        x = self.out(x)

        return x

def same_seeds(seed):
    torch.manual_seed(seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed(seed)
        torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True

same_seeds(1)
device = get_device()
print(f'DEVICE: {device}')
num_epoch = 20
learning_rate = 0.001

model_path = './model.ckpt'

model = Classifier().to(device)

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

best_acc = 0.0

for epoch in range(num_epoch):
    train_acc = 0.0
    train_loss = 0.0
    val_acc = 0.0
    val_loss = 0.0

    model.train()
    for i, data in enumerate(train_loader):
        inputs, labels = data
        inputs, labels = inputs.to(device), labels.to(device)
        optimizer.zero_grad()

        inputs = inputs.to(torch.float32)
        output = model(inputs)

        labels = labels.to(torch.Tensor().long())
        batch_loss = criterion(output, labels)
        _, train_pred = torch.max(output, 1)
        batch_loss.backward()
        optimizer.step()

        train_acc += (train_pred.cpu() == labels.cpu()).sum().item()

        train_loss = batch_loss.item()

    if len(X_val) > 0:
        model.eval()
        with torch.no_grad():
            for i, data in enumerate(val_loader):
                inputs, labels = data
                inputs, labels = inputs.to(device), labels.to(device)
                optimizer.zero_grad()
                inputs = inputs.to(torch.float32)
                output = model(inputs)
                labels = labels.to(torch.Tensor().long())
                batch_loss = criterion(output, labels)
                _, val_pred = torch.max(output, 1)

                val_acc += (val_pred.cpu() == labels.cpu()).sum().item()
                val_loss = batch_loss.item()
            print('[{:03d}/{:03d}] Train Acc: {:3.6f} Loss: {:3.6f} | Val Acc: {:3.6f} loss: {:3.6f}'.format(
                epoch + 1, num_epoch, train_acc / train_len, train_loss / train_len, val_acc / val_len,
                val_loss / val_len
            ))

        if val_acc > best_acc:
            best_acc = val_acc
            torch.save(model.state_dict(), model_path)
            print('saving model with acc {:.3f}'.format(best_acc / len(X_val)))

    else:
        print('[{:03d}/{:03d}] Train Acc: {:3.6f} Loss: {:3.6f}'.format(
            epoch + 1, num_epoch, train_acc / len(X), train_loss / len(train_loader)
        ))

if len(X_val) == 0:
    torch.save(model.state_dict(), model_path)
    print('saving model at last epoch')


from typing import Optional, Tuple

from cortex_xdr_client.api.authentication import Authentication
from cortex_xdr_client.api.base_api import BaseAPI
from cortex_xdr_client.api.models.action_status import GetActionStatus
from cortex_xdr_client.api.models.filters import new_request_data

class ActionsAPI(BaseAPI):
    def __init__(self, auth: Authentication, fqdn: str, timeout: Tuple[int, int]) -> None:
        super(ActionsAPI, self).__init__(auth, fqdn, "actions", timeout)

    # https://docs.paloaltonetworks.com/cortex/cortex-xdr/cortex-xdr-api/cortex-xdr-apis/response-actions/get-action-status.html
    def get_action_status(self,
                          group_action_id: int
                          ) -> Optional[GetActionStatus]:
        """
        Retrieve the status of the requested actions according to the action ID.

        :param group_action_id: String the represents the Action ID of the selected request.
        :return: Returns a GetActionStatus object if successful.
        """
        request_data = new_request_data(other={'group_action_id': group_action_id})

        response = self._call(call_name="get_action_status",
                              json_value=request_data)
        return GetActionStatus.parse_obj(response.json())

    # https://docs-cortex.paloaltonetworks.com/r/Cortex-XDR/Cortex-XDR-API-Reference/File-Retrieval-Details
    def get_file_retrieval_details(self,
                                   group_action_id: int
                                   ) -> Optional[GetActionStatus]:
        """
        Retrieve the status of the requested file retrieval action.

        :param group_action_id: String the represents the Action ID of the selected request.
        :return: Returns a GetActionStatus object if successful.
        """
        request_data = new_request_data(other={'group_action_id': group_action_id})

        response = self._call(call_name="file_retrieval_details",
                              json_value=request_data)
        return GetActionStatus.parse_obj(response.json())


"""
Skinning tools for maya.

Installation
============
Copy the **rjSkinningTools** folder to your Maya scripts directory
::
    C:/Users/<USER>/Documents/maya/scripts

Usage
=====
Add skinning shelf
::
    import rjSkinningTools
    rjSkinningTools.install()

Tools
=====
**paintSmoothWeightsCtx**
    * Paint smooth weights tool in Maya using the weights of neighbouring vertices.

**paintRemoveInfluenceCtx**
    * Remove a specific influence on vertices with the paint tool.

**tweakVertexWeights**
    * Tweak influence weights on a vertex level.

**softSelectionToWeights**
    * Convert soft selection to skin weights.

**delinearWeights**
    * De-linearize skin weights.

Code
====
"""

from maya import cmds, mel
from . import ui
import paintSmoothWeightsCtx
import paintRemoveInfluenceCtx
import ReorderAttr
__author__    = "Robert Joosten"
__version__   = "0.0.1"
__email__     = "rwm.joosten@gmail.com"

# ----------------------------------------------------------------------------

SHELF_NAME = "Skinning"
TOOLS = [
    {
        "label":"paintSkinWeights",
        "command":"ArtPaintSkinWeightsTool;",
        "annotation":"Paint Skin Weights Tool: Paint weights on smooth bound skins",
        "image1":ui.findIcon("paintSkinWeightsCtx.png"),
        "sourceType":"mel"
    },
    {
        "label":"paintSmoothWeights",
        "command":"from rjSkinningTools import paintSmoothWeightsCtx; paintSmoothWeightsCtx.paint()",
        "annotation":"Paint Smooth Weights Tool: Smooth weights on smooth bound skins",
        "image1":ui.findIcon("paintSmoothWeightsCtx.png"),
        "sourceType":"python"
    },
    {
        "label":"paintRemoveInfluence",
        "command":"import rjSkinningTools.paintRemoveInfluenceCtx.ui; rjSkinningTools.paintRemoveInfluenceCtx.ui.show()",
        "annotation":"Paint Remove Influence Tool: Remove influence on smooth bound skins",
        "image1":ui.findIcon("paintRemoveInfluenceCtx.png"),
        "sourceType":"python"
    },
    {
        "label":"tweakVertexWeights",
        "command":"import rjSkinningTools.tweakVertexWeights.ui; rjSkinningTools.tweakVertexWeights.ui.show()",
        "annotation":"Tweak Influences on a vertex level",
        "image1":ui.findIcon("tweakVertexWeights.png"),
        "sourceType":"python"
    },
    {
        "label":"softSelectionToWeights",
        "command":"import rjSkinningTools.softSelectionToWeights.ui; rjSkinningTools.softSelectionToWeights.ui.show()",
        "annotation":"Convert soft selection to skin weights",
        "image1":ui.findIcon("softSelectionToWeights.png"),
        "sourceType":"python"
    },
	{
        "label":"delinearWeights",
        "command":"import rjSkinningTools.delinearWeights.ui; rjSkinningTools.delinearWeights.ui.show()",
        "annotation":"De-linearize skin weights",
        "image1":ui.findIcon("delinearWeights.png"),
        "sourceType":"python"
    }
]

# ----------------------------------------------------------------------------

def install():
    """
    Add a new shelf in Maya with all the tools that are provided in the TOOLS
    variable. If a tab already exist new buttons that weren't registered yet
    will be added to the shelf.
    """
    # get top shelf
    gShelfTopLevel = mel.eval("$tmpVar=$gShelfTopLevel")

    # get top shelf names
    shelves = cmds.tabLayout(gShelfTopLevel, query=1, ca=1)

    # create shelf if it doesn't exist yet
    if not SHELF_NAME in shelves:
        cmds.shelfLayout(SHELF_NAME, parent=gShelfTopLevel)

    # get existing members
    existing = cmds.shelfLayout(SHELF_NAME, query=True, childArray=True ) or []
    existing = [cmds.shelfButton(e, query=True, label=True) for e in existing]

    # add modules
    for tool in TOOLS:
        if tool.get("label") in existing:
            continue

        if tool.get("image1"):
            cmds.shelfButton(style="iconOnly", parent=SHELF_NAME, **tool)
        else:
            cmds.shelfButton(style="textOnly", parent=SHELF_NAME, **tool)


import numpy as np
import cv2 as cv

img = cv.imread('imagens/catiorrinho.jpg')
cv.imshow('original', img)
cv.waitKey(0)

"""**b - Abrir uma imagem e exibir na tela a imagem invertida horizontalmente**"""

horizontal_img = np.fliplr(img)
cv.imshow('flip-horizontal', horizontal_img)
cv.waitKey(0)
cv.destroyAllWindows()


from keras.models import load_model
from tensorflow.keras.datasets import cifar10
from tensorflow.keras import utils
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, MaxPooling2D
from tensorflow.keras.layers import Dense, Dropout, Flatten
import util3 as u

(x_train, y_train), (x_test, y_test) = cifar10.load_data()

x_train_norm = x_train.astype('float32')/255
x_test_norm = x_test.astype('float32')/255

y_train_onehot = utils.to_categorical(y_train, 10)
y_test_onehot = utils.to_categorical(y_test, 10)

cnn = Sequential()

cnn.add(Conv2D(filters=32,
               kernel_size=[3,3],
               activation='relu',
               padding='same',
               input_shape=[32,32,3]))

cnn.add(Dropout(0.25))
cnn.add(MaxPooling2D(pool_size=[2,2]))

cnn.add(Conv2D(filters=64,
               kernel_size=[3,3],
               padding='same',
               activation='relu'))
cnn.add(Dropout(0.25))
cnn.add(MaxPooling2D(pool_size=[2,2]))

cnn.add(Flatten())
cnn.add(Dropout(0.25))

cnn.add(Dense(1024, activation='relu'))
cnn.add(Dropout(0.25))

cnn.add(Dense(10, activation='softmax'))

cnn.compile(loss='categorical_crossentropy',
            optimizer='adam',
            metrics='acc')

old_cnn = load_model(r'C:\Users\d19fd\Documents\tf.Keras\model\cifar10model.h5')

test_loss, test_val = cnn.evaluate(x_test_norm, y_test_onehot)

print('測試資料損失值 :', test_loss)
print('測試資料準確率 :', test_val)


import networkx as nx
import numpy as np

from mo.graph.graph import Node
from mo.ops.op import Op

class Gather(Op):
    op = 'Gather'

    def __init__(self, graph: nx.MultiDiGraph, attrs: dict):
        mandatory_props = {
            'type': __class__.op,
            'op': __class__.op,
            'axis': 0,
            'infer': __class__.infer
        }
        super().__init__(graph, mandatory_props, attrs)

    def supported_attrs(self):
        return [
            'axis',
        ]

    @staticmethod
    def infer(node: Node):
        assert len(node.in_nodes()) == 2 or len(node.in_nodes()) == 3

        # There may be three inputs in TensorFlow. The third input is axis
        if len(node.in_nodes()) == 3:
            if node.in_node(2).value is None:
                log.error("Gather is supported only with constant axis value")
                return
            node.axis = node.in_node(2).value.item()
            node.graph.remove_edge(node.in_node(2).id, node.id)

        axis = node.axis
        data = node.in_node(0)
        indices = node.in_node(1)

        # both inputs are constant
        if data.value is not None and indices.value is not None:
            node.out_node(0).value = np.take(data.value, indices.value, axis)
            node.out_node(0).shape = np.array(node.out_node(0).value.shape, dtype=np.int64)
            return

        shape = np.concatenate((data.shape[:axis], indices.shape))
        if axis < len(data.shape) - 1:
            shape = np.concatenate((shape, data.shape[axis+1:]))

        node.out_node(0).shape = np.array(shape, dtype=np.int64)


from unicodedata import name
from django.test import TestCase

from .models import Destiny

class TesteCreateDestiny(TestCase):

    def setUp(self):
        Destiny.objects.create(
        name = 'Minas',
        state = 'MG',
        city = 'Ibié'
        )

    def test_novo_destino_criado(self):
        destiny = Destiny.objects.get(name='Minas')
        self.assertEquals(destiny.__str__(), 'Minas: Ibié/MG')
        self.assertEquals(destiny.city_state(), 'Ibié/MG')

        print(
            'Nome:', destiny.name,'\n'
            'Estado:', destiny.state,'\n'
            'Cidade:', destiny.city,'\n'
        )


import os
import re
import time
import Adafruit_SSD1306
from PIL import Image
from PIL import ImageDraw
from PIL import ImageFont

class StatusDisplay:
    def __init__(self, bb8):
        #LCD.__init__(self, 0x20)
        self.display = Adafruit_SSD1306.SSD1306_128_32(rst=16)
        self.display.begin()
        self.image = Image.new('1', (self.display.width, self.display.height))
        self.draw = ImageDraw.Draw(self.image)
        self.font = ImageFont.load_default()

        self.BB8 = bb8
        self.showIP = True
        self.lastUpdateTime = 0

    def Update(self):
        t = time.time()
        if t > self.lastUpdateTime + 2:
            #self.draw.rectangle((0, 0, self.display.width, self.display.height), outline=0, fill=0)
            ip = ("IP: " + self.BB8.Network.IP) if self.BB8.Network.IP else "No Network"
            ssid = ("SSID: " + self.BB8.Network.SSID) if self.BB8.Network.SSID else "No Network"
            #if self.lastIP != ip:
            self.draw.rectangle((0, 0, self.display.width, self.display.height), outline=0, fill=0)
            self.draw.text((0, 0), ip if self.showIP else ssid, font=self.font, fill=255)
            self.display.image(self.image)
            self.display.display()
            self.showIP = not self.showIP
            #cpu = "CPU: " + os.popen("top -bn1 | tail -n +8 | awk '{ SUM += $(NF-3) } END { printf \"%.0f%%\", SUM }'").read()
            #mem = "Mem: " + os.popen("free -m | awk 'NR==2{printf \"%s/%s MB %.0f%%\", $3,$2,$3*100/$2 }'").read()
            #disk = "Disk: " + os.popen("df -h | awk '$NF==\"/\"{printf \"%.2g/%d GB %s\", $3,$2,$5}'").read()
            #self.draw.text((0, -1), ip, font=self.font, fill=255)
            #self.draw.text((0, 7), cpu, font=self.font, fill=255)
            #self.draw.text((0, 15), mem, font=self.font, fill=255)
            #self.draw.text((0, 23), disk, font=self.font, fill=255)
            #self.display.image(self.image)
            #self.display.display()
            self.lastUpdateTime = t

    def Exit(self):
        self.display.clear()
        self.display.display()

    def format(self, number):
        if number >= 100:
            return str(int(round(number)))
        elif number >= 10:
            return "%.1f" % number
        else:
            return "%.2f" % number


import torchvision.transforms as T
try:
    from torchvision.transforms import GaussianBlur
except ImportError:
    from .gaussian_blur import GaussianBlur
    T.GaussianBlur = GaussianBlur

imagenet_mean_std = [[0.485, 0.456, 0.406],[0.229, 0.224, 0.225]]

class SimCLRTransform():
    def __init__(self, image_size, mean_std=imagenet_mean_std, s=1.0):
        image_size = 224 if image_size is None else image_size
        self.transform = T.Compose([
            T.RandomResizedCrop(image_size, scale=(0.2, 1.0)),
            T.RandomHorizontalFlip(),
            T.RandomApply([T.ColorJitter(0.8*s,0.8*s,0.8*s,0.2*s)], p=0.8),
            T.RandomGrayscale(p=0.2),
            T.RandomApply([T.GaussianBlur(kernel_size=image_size//20*2+1, sigma=(0.1, 2.0))], p=0.5),
            # We blur the image 50% of the time using a Gaussian kernel. We randomly sample σ ∈ [0.1, 2.0], and the kernel size is set to be 10% of the image height/width.
            T.ToTensor(),
            T.Normalize(*mean_std)
        ])
    def __call__(self, x):
        x1 = self.transform(x)
        x2 = self.transform(x)
        return x1, x2


import yfinance as yf
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

# Collect options data for AAPL
ticker_symbol = "TSLA"
ticker = yf.Ticker(ticker_symbol)

# Get the list of available expiration dates
available_dates = ticker.options

# Choose the first expiration date for demonstration (you can choose any other)
chosen_expiry = available_dates[0]

options_data = ticker.option_chain(date=chosen_expiry)
calls = options_data.calls
puts = options_data.puts

# Merge calls and puts data on 'strike' column
merged = pd.merge(calls, puts, on='strike', suffixes=('_call', '_put'))

# Store the value (last trade price) and strike price
df = merged[['strike', 'lastPrice_call', 'volume_call']]

# Ensure the dataframe is sorted by strike price
df = df.sort_values(by='strike')

# Compute the unsmoothed second derivative
second_derivative = np.gradient(np.gradient(df['lastPrice_call'].to_numpy(), df['strike'].to_numpy()), df['strike'].to_numpy())

# Normalize the volume weights to sum to one
normalized_weights = df['volume_call'] / df['volume_call'].sum()

# Weight the second derivative by the normalized volume weights
normalized_weighted_derivative = second_derivative * normalized_weights.to_numpy()

# Plot
fig, ax1 = plt.subplots(figsize=(12, 6))

# Plot the unsmoothed second derivative
ax1.plot(df['strike'], second_derivative, label='Unsmoothed 2nd Derivative', color='blue')
ax1.plot(df['strike'], normalized_weighted_derivative, label='Normalized Weighted 2nd Derivative', color='orange', linestyle='--')
ax1.set_xlabel('Strike Price')
ax1.set_ylabel('Derivative Value')
ax1.legend(loc='upper left')

# Create a second y-axis for volume
ax2 = ax1.twinx()
ax2.plot(merged['strike'], merged['volume_call'], label='Volume (Calls)', color='green', alpha=0.5)
ax2.set_ylabel('Volume')
ax2.legend(loc='upper right')

plt.title(f'Second Derivative (Unsmoothed & Normalized Weighted) and Volume for {ticker_symbol} Calls (Expiry: {chosen_expiry})')
plt.grid(True)
plt.show()


import pytest

from advent.calendar.day_19 import part_one, part_two

def _input():
    return """Blueprint 1: Each ore robot costs 4 ore. Each clay robot costs 2 ore. Each obsidian robot costs 3 ore and 14 clay. Each geode robot costs 2 ore and 7 obsidian.
Blueprint 2: Each ore robot costs 2 ore. Each clay robot costs 3 ore. Each obsidian robot costs 3 ore and 8 clay. Each geode robot costs 3 ore and 12 obsidian.""".splitlines()

# these take a while to run, marking skip to not tax my githooks on every push
@pytest.mark.skip
def test_part_one():
    expected = 33
    actual = part_one(_input())

    assert expected == actual

@pytest.mark.skip
def test_part_two():
    expected = 56 * 62
    actual = part_two(_input())

    assert expected == actual


from dsl.create_lead import CreateLead
from ioffice.clients.client_dashboard import ClientDashboardPage
from ioffice.leads.change_status_dialog import ChangeStatusDialog
from ioffice.leads.lead_details_page import LeadDetailsPage
from utils import get_temp_data, get_common_data

class ConvertLead(CreateLead):

    def using_change_status_dialog(self):
        lead_details_page = LeadDetailsPage(self.config)
        lead_details_page.lead_actions_menu() \
            .hover_over_lead_actions() \
            .change_status()
        return ConvertLead._ChangeStatus(lead_details_page, self)

    def assert_lead_converted_to_client_exists(self):
        lead = get_temp_data(self.config, "lead")
        clientdashboard = ClientDashboardPage(self.config)
        assert clientdashboard.is_title_matches(), "Title does not match the client dashboard"
        assert clientdashboard.get_client_bar_info() == lead["person"]["firstName"] + ' ' + lead["person"]["lastName"]

    class _ChangeStatus:
        def __init__(self, parent_page, journey):
            self.config = journey.config
            self.journey = journey
            self.dialog = ChangeStatusDialog(parent_page)

        def convert_lead_to_client(self):
            data = get_common_data(self.config)["test_data"]["lead_data"]
            self.dialog.select_status(data["STATUS"])
            self.dialog.wait_until_please_wait_spinner_present()
            self.dialog.click_save_button()
            self.dialog.close_io_dialog()
            return self.journey


import argparse
import json
import os
import sys
import pandas as pd
import copy

from helpers import multitab_excel_to_single_txt
from helpers import get_protocol_map
from helpers import fetch_fastq_path
from helpers import utils
from helpers import check_experimental_design

pd.options.mode.chained_assignment = None

def rename_technology_type(technology_type, technology_dict):

    json_file = technology_dict[technology_type]

    return json_file

def get_secondary_accessions(xlsx_dict, args):
    secondary_accessions = []
    secondary_accessions.append(args.project_uuid)
    keys = ["project.geo_series_accessions","project.insdc_project_accessions","project.insdc_study_accessions","project.biostudies_accessions"]
    for key in keys:
        items = xlsx_dict["project"][key].fillna('')
        items = [item for item in items if item != '']
        if items:
            if "||" in items:
                items = items.split("||")
            else:
                items = items
            secondary_accessions.extend(items)
    return secondary_accessions

def get_person_roles(xlsx_dict):

    person_roles = utils.reformat_value(xlsx_dict, "project_contributors", "project.contributors.project_role.text", "str")
    person_roles_submitter = utils.reformat_value(xlsx_dict, "project_contributors",
                                                  "project.contributors.corresponding_contributor", "str")

    for (i, elem) in enumerate(person_roles_submitter):
        person_roles[i] = person_roles[i].lower()
        if elem == "yes":
            person_roles[i] = "submitter"
        elif elem == "no":
            if "curator" in person_roles[i]:
                person_roles[i] = "data curator"
            else:
                person_roles[i] = ""

    return person_roles

def get_author_list(xlsx_dict):

    authors = utils.reformat_value(xlsx_dict, "project_publications", "project.publications.authors", "str")[0]
    author_list = authors.replace("||",", ")

    return author_list

def generate_idf_file(work_dir, args, dataset_protocol_map, xlsx_dict, accession, idf_file_name,
                      sdrf_file_name):

    tab = '\t'
    person_roles = get_person_roles(xlsx_dict)
    secondary_accessions = get_secondary_accessions(xlsx_dict, args)
    protocol_fields = get_protocol_map.get_idf_file_protocol_fields(dataset_protocol_map)
    author_list = get_author_list(xlsx_dict)

    related_scea_accessions = None
    related_scea_accessions = args.related_scea_accession

    if related_scea_accessions:

        idf_file_contents = f"""\
MAGE-TAB Version\t1.1
Investigation Title\t{utils.reformat_value(xlsx_dict, "project", "project.project_core.project_title", "str")[0].strip('.')}
Comment[Submitted Name]\t{utils.reformat_value(xlsx_dict, "project", "project.project_core.project_short_name", "str")[0]}
Experiment Description\t{utils.reformat_value(xlsx_dict, "project", "project.project_core.project_description", "str")[0]}
Public Release Date\t{args.public_release_date}
Person First Name\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.name", lambda x: x.split(',')[0])}
Person Last Name\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.name", lambda x: x.split(',')[2])}
Person Mid Initials\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.name", lambda x: utils.get_first_letter(x.split(',')[1]))}
Person Email\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.email")}
Person Affiliation\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.institution")}
Person Address\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.address")}
Person Roles\t{tab.join(person_roles)}
Protocol Type\t{tab.join([field[0] for field in protocol_fields])}
Protocol Name\t{tab.join([field[1] for field in protocol_fields])}
Protocol Description\t{tab.join([field[2] for field in protocol_fields])}
Protocol Hardware\t{tab.join([field[3] for field in protocol_fields])}
Term Source Name\tEFO\tArrayExpress
Term Source File\thttp://www.ebi.ac.uk/efo/efo.owl\thttp://www.ebi.ac.uk/arrayexpress/
Comment[AEExperimentType]\tRNA-seq of coding RNA from single cells
Experimental Factor Name\t{tab.join(args.experimental_factors)}
Experimental Factor Type\t{tab.join(args.experimental_factors)}
Comment[EAAdditionalAttributes]\t{''}
Comment[EACurator]\t{tab.join(args.curators)}
Comment[EAExpectedClusters]\t
Comment[ExpressionAtlasAccession]\t{accession}
Comment[RelatedExperiment]\t{tab.join(related_scea_accessions)}
Comment[HCALastUpdateDate]\t{args.hca_update_date}
Comment[SecondaryAccession]\t{tab.join(secondary_accessions)}
Comment[EAExperimentType]\t{args.experiment_type}
SDRF File\t{sdrf_file_name}
Publication Title\t{utils.reformat_value(xlsx_dict, "project_publications", "project.publications.title", "str")[0].strip('.')}
Publication Author List\t{author_list}
PubMed ID\t{utils.reformat_value(xlsx_dict, "project_publications", "project.publications.pmid", "str")[0]}
Publication DOI\t{utils.reformat_value(xlsx_dict, "project_publications", "project.publications.doi", "str")[0]}
"""
    else:

        idf_file_contents = f"""\
MAGE-TAB Version\t1.1
Investigation Title\t{utils.reformat_value(xlsx_dict, "project", "project.project_core.project_title", "str")[0].strip('.')}
Comment[Submitted Name]\t{utils.reformat_value(xlsx_dict, "project", "project.project_core.project_short_name", "str")[0]}
Experiment Description\t{utils.reformat_value(xlsx_dict, "project", "project.project_core.project_description", "str")[0]}
Public Release Date\t{args.public_release_date}
Person First Name\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.name", lambda x: x.split(',')[0])}
Person Last Name\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.name", lambda x: x.split(',')[2])}
Person Mid Initials\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.name", lambda x: utils.get_first_letter(x.split(',')[1]))}
Person Email\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.email")}
Person Affiliation\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.institution")}
Person Address\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.address")}
Person Roles\t{tab.join(person_roles)}
Protocol Type\t{tab.join([field[0] for field in protocol_fields])}
Protocol Name\t{tab.join([field[1] for field in protocol_fields])}
Protocol Description\t{tab.join([field[2] for field in protocol_fields])}
Protocol Hardware\t{tab.join([field[3] for field in protocol_fields])}
Term Source Name\tEFO\tArrayExpress
Term Source File\thttp://www.ebi.ac.uk/efo/efo.owl\thttp://www.ebi.ac.uk/arrayexpress/
Comment[AEExperimentType]\tRNA-seq of coding RNA from single cells
Experimental Factor Name\t{tab.join(args.experimental_factors)}
Experimental Factor Type\t{tab.join(args.experimental_factors)}
Comment[EAAdditionalAttributes]
Comment[EACurator]\t{tab.join(args.curators)}
Comment[EAExpectedClusters]\t
Comment[ExpressionAtlasAccession]\t{accession}
Comment[HCALastUpdateDate]\t{args.hca_update_date}
Comment[SecondaryAccession]\t{tab.join(secondary_accessions)}
Comment[EAExperimentType]\t{args.experiment_type}
SDRF File\t{sdrf_file_name}
Publication Title\t{utils.reformat_value(xlsx_dict, "project_publications", "project.publications.title", "str")[0].strip('.')}
Publication Author List\t{author_list}
PubMed ID\t{utils.reformat_value(xlsx_dict, "project_publications", "project.publications.pmid", "str")[0]}
Publication DOI\t{utils.reformat_value(xlsx_dict, "project_publications", "project.publications.doi", "str")[0]}
"""

    print(f"saving {work_dir}/{idf_file_name}")
    with open(f"{work_dir}/{idf_file_name}", "w") as idf_file:
        idf_file.write(idf_file_contents)

def reformat_age(age_list):

    updated_age_list = []

    for age in age_list:
        age = str(age)
        if ' - ' in age:
            age = age.replace('-', 'to')
        elif '-' in age and ' ' not in age:
            age = age.replace('-', ' to ')
        else:
            age = age

        updated_age_list.append(age)

    return updated_age_list

def get_sample_name_key(args, df):

    if args.name == 'cs_name':
        sample_name_key = 'cell_suspension.biomaterial_core.biomaterial_name'
    elif args.name == 'cs_id':
        sample_name_key = 'cell_suspension.biomaterial_core.biomaterial_id'
    elif args.name == 'sp_name':
        sample_name_key = 'specimen_from_organism.biomaterial_core.biomaterial_name'
    elif args.name == 'sp_id':
        sample_name_key = 'specimen_from_organism.biomaterial_core.biomaterial_id'

    return sample_name_key

def get_values_from_df(df, column):
    return df[column] if column in df.columns else ['']*df.shape[0]

def add_sequence_paths(sdrf, args):

    run_accessions = list(sdrf['Comment[ENA_RUN]'])

    try:
        sra_paths = fetch_fastq_path.get_sra_path_from_ena(args.study, run_accessions)
    except:
        sra_paths = fetch_fastq_path.get_sra_path_from_sra(args.study, run_accessions)
        if not sra_paths:
            try:
                sra_paths = fetch_fastq_path.get_sra_path_from_sra(run_accessions)
            except:
                sra_paths = fetch_fastq_path.get_sra_path_from_ena(args.study, run_accessions)

    if sra_paths:
        read1_names = []
        read2_names = []
        sra_names = []
        for key in sra_paths.keys():
            read1_names.append(key + "_1.fastq.gz")
            read2_names.append(key + "_2.fastq.gz")
            sra_names.append(sra_paths[key]['files'][0])
        sdrf['Comment[read1 file]'] = read1_names
        sdrf['Comment[read2 file]'] = read2_names
        sdrf['Comment[SRA_URI]'] = sra_names
    else:
        print("Could not find paths to SRA objects.")
        sdrf['Comment[SRA_URI]'] = 'PATH NOT FOUND'
        sdrf['Comment[read1 file]'] = 'PATH NOT FOUND'
        sdrf['Comment[read2 file]'] = 'PATH NOT FOUND'

    return sdrf

def get_new_protocol_column_names(sdrf, counter):

    new_column_names = []
    for col in sdrf.columns:
        column_name = "Protocol REF" + "_" + str(counter)
        counter += 1
        new_column_names.append(column_name)

    return new_column_names, counter

def order_protocols(protocols_sdrf_before_sequencing):
    collection_protocol_cols = []
    dissociation_protocol_cols = []
    enrichment_protocol_cols = []
    differentiation_protocol_cols = []
    library_preparation_protocol_cols = []
    for col in list(protocols_sdrf_before_sequencing.columns):
        if col.split(".protocol_core.protocol_id")[0] == "collection_protocol":
            collection_protocol_cols.append(col)
        if col.split(".protocol_core.protocol_id")[0] == "dissociation_protocol":
            dissociation_protocol_cols.append(col)
        if col.split(".protocol_core.protocol_id")[0] == "enrichment_protocol":
            enrichment_protocol_cols.append(col)
        if col.split(".protocol_core.protocol_id")[0] == "differentiation_protocol":
            differentiation_protocol_cols.append(col)
        if col.split(".protocol_core.protocol_id")[0] == "library_preparation_protocol":
            library_preparation_protocol_cols.append(col)
    new_columns = [sorted(collection_protocol_cols),sorted(dissociation_protocol_cols),sorted(enrichment_protocol_cols),sorted(differentiation_protocol_cols),sorted(library_preparation_protocol_cols)]
    new_columns = [item for sublist in new_columns for item in sublist]
    protocols_sdrf_before_sequencing = protocols_sdrf_before_sequencing[new_columns]
    return protocols_sdrf_before_sequencing

def add_protocol_columns(df, dataset_protocol_map):

    def convert_term(term, name):
        return get_protocol_map.map_proto_to_id(term, dataset_protocol_map)

    def convert_row(row):
        return row.apply(lambda x: convert_term(x, row.name))

    protocols_list_before_sequencing = ['collection_protocol', 'dissociation_protocol', 'enrichment_protocol', 'differentiation_protocol', 'library_preparation_protocol']

    protocols_sdrf_before_sequencing = df[[col for (proto_type, cols) in get_protocol_map.map_of_hca_protocol_type_id_keys.items() if proto_type in
                 protocols_list_before_sequencing for col in cols]]

    protocols_sdrf_before_sequencing.fillna("", inplace=True)
    pd.set_option('display.max_columns', None)

    protocols_sdrf_before_sequencing = protocols_sdrf_before_sequencing.apply(convert_row)
    pd.set_option('display.max_columns', None)

    protocols_sdrf_before_sequencing = order_protocols(protocols_sdrf_before_sequencing)

    counter = 1
    new_column_names, counter = get_new_protocol_column_names(protocols_sdrf_before_sequencing, counter)
    protocols_sdrf_before_sequencing.columns = new_column_names

    protocols_sdrf_before_sequencing.fillna(value='', inplace=True)

    protocols_list_from_sequencing = ['sequencing_protocol']
    protocols_sdrf_from_sequencing = df[[col for (proto_type, cols) in get_protocol_map.map_of_hca_protocol_type_id_keys.items() if proto_type in protocols_list_from_sequencing for col in cols]]
    protocols_sdrf_from_sequencing = protocols_sdrf_from_sequencing.apply(convert_row)
    new_column_names, counter = get_new_protocol_column_names(protocols_sdrf_from_sequencing, counter)
    protocols_sdrf_from_sequencing.columns = new_column_names

    return protocols_sdrf_before_sequencing, protocols_sdrf_from_sequencing

def add_scea_specimen_columns(args, df, experimental_design):

    if experimental_design == "standard":

        '''Open dictionary of mapped hca2scea key:pairs for specimen metadata.'''
        with open(f"json_files/sdrf_map.json") as sdrf_map_file:
            sdrf_map = json.load(sdrf_map_file)

    else:

        if experimental_design == "cell_line_only":
            '''Open dictionary of mapped hca2scea key:pairs for specimen metadata.'''
            with open(f"json_files/sdrf_map_cell_line.json") as sdrf_map_file:
                sdrf_map = json.load(sdrf_map_file)

        elif experimental_design == "organoid":
            '''Open dictionary of mapped hca2scea key:pairs for specimen metadata.'''
            with open(f"json_files/sdrf_map_cell_line_organoid.json") as sdrf_map_file:
                sdrf_map = json.load(sdrf_map_file)

        else:
            '''Open dictionary of mapped hca2scea key:pairs for specimen metadata.'''
            with open(f"json_files/sdrf_map_organoid.json") as sdrf_map_file:
                sdrf_map = json.load(sdrf_map_file)

    '''Get user-specified HCA sample names key.'''
    sample_name_key = get_sample_name_key(args, df)

    '''Update dictionary of mapped hca2scea key:pairs with sample name key.'''
    sdrf_map.update({
        'Source Name': sample_name_key,
        'Assay Name': sample_name_key,
        'Scan Name': sample_name_key,
        'Extract Name': sample_name_key,
        'Comment[BioSD_SAMPLE]': 'cell_suspension.biomaterial_core.biosamples_accession',
        'Comment[ENA_EXPERIMENT]': 'cell_suspension.insdc_experiment.insdc_experiment_accession',
        'Comment[ENA_RUN]': 'sequence_file.insdc_run_accessions',
        'Comment[technical replicate group]': 'cell_suspension.biomaterial_core.biosamples_accession'
        })

    '''Extract the HCA metadata values using the HCA keys in sdrf_map.'''
    sdrf = pd.DataFrame({k: get_values_from_df(df, v) for k, v in sdrf_map.items()})
    sdrf = sdrf.fillna('')

    '''Reformat the age values so they are excel-friendly.'''
    sdrf['Characteristics[age]'] = reformat_age(list(sdrf['Characteristics[age]']))

    '''Reformat living status to align with SCEA controlled vocabulary.'''
    sdrf['Characteristics[organism status]'] = sdrf['Characteristics[organism status]'].apply(lambda x: 'alive' if x.lower() in ['yes', 'y'] else 'dead')

    '''Add Material Type using SCEA controlled vocabulary.'''
    sdrf['Material Type_1'] = 'cell'

    return sdrf

def generate_sdrf_file(work_dir, args, df, xlsx_dict, dataset_protocol_map, sdrf_file_name, experimental_design, technology_dict):

    '''Generate a dataframe with SCEA specimen metadata.'''
    sdrf_1 = add_scea_specimen_columns(args, df, experimental_design)

    '''Get technology-specific SCEA metadata and add to sdrf_1 dataframe.'''
    technology_type = list(xlsx_dict["library_preparation_protocol"]["library_preparation_protocol.library_construction_method.ontology_label"])[0]
    technology_type = rename_technology_type(technology_type,technology_dict)
    try:
        with open(f"json_files/{technology_type}.json") as technology_json_file:
            technology_type_dict = json.load(technology_json_file)
    except:
        print("Technology type {} is not yet supported. Please ask Ami to add it to the technology type map.".format(technology_type))
        sys.exit()
    for key in technology_type_dict.keys():
        sdrf_1[key] = technology_type_dict[key]

    '''Edit single cell isolation method if the user-specified that FACS was used.'''
    if args.facs is True:
        sdrf_1['Comment[single_cell_isolation]'] = 'FACS'*sdrf_1.shape[0]
    # To do: add 'Comment[single_cell_isolation]' == Fluidigm C1 for Fluidigm C1 experiments.

    '''To Do. Get immunophenotype and treatment/stimulus information, if there is any.'''
    #"Characteristics[immunophenotype]":"enrichment_protocol.markers"
    sdrf_1['Characteristics[immunophenotype]'] = '' * sdrf_1.shape[0]
    #"Characteristics[stimulus]":"cell_suspension.growth_conditions.drug_treatment","differentiation_protocol.small_molecules",
    sdrf_1['Characteristics[stimulus]'] = '' * sdrf_1.shape[0]

    '''Get the SRA object file names and file paths from SRA or ENA (try both).'''
    sdrf_2 = add_sequence_paths(sdrf_1, args)

    '''Check all required column names are present and reorder columns by SCEA defined order.'''

    with open(f"json_files/expected_columns.json", "r") as expected_columns_file:
        expected_columns_dict = json.load(expected_columns_file)

    if experimental_design == 'standard':
        expected_columns_ordered = expected_columns_dict['standard']
    elif experimental_design == 'organoid_only':
        expected_columns_ordered = expected_columns_dict['organoid_only']
    else:
        expected_columns_ordered = expected_columns_dict['cell_line']

    column_check = [col for col in expected_columns_ordered if col not in sdrf_2.columns]
    if column_check:
        print("Error: one or more expected columns is missing from sdrf.")
        print(column_check)
        sys.exit()
    else:
        sdrf_3 = sdrf_2[expected_columns_ordered]

    '''Add protocol columns with protocol metadata in the pre-defined SCEA column order.'''
    protocols_sdrf_before_sequencing, protocols_sdrf_from_sequencing = add_protocol_columns(df, dataset_protocol_map)
    # Remove empty Protocol REF columns.
    nan_value = float("NaN")
    protocols_sdrf_before_sequencing.replace("", nan_value, inplace=True)
    protocols_sdrf_before_sequencing.dropna(how='all', axis=1, inplace=True)
    protocols_sdrf_before_sequencing.replace(nan_value, "", inplace=True)

    idx = 18
    for col in protocols_sdrf_before_sequencing.columns:
        sdrf_3.insert(idx, col, list(protocols_sdrf_before_sequencing[col]))
        idx += 1

    idx = list(sdrf_3.columns).index('Comment[LIBRARY_SELECTION]') + 1
    for col in protocols_sdrf_from_sequencing.columns:
        sdrf_3.insert(idx, col, list(protocols_sdrf_from_sequencing[col]))
        idx += 1

    '''Fix Material Type and Protocol REF column names to SCEA standard columns.'''
    for col_name in sdrf_3.columns:
        if 'Protocol REF' in str(col_name):
            sdrf_3 = sdrf_3.rename(columns={col_name: "Protocol REF"})
        elif 'Material Type' in str(col_name):
            sdrf_3 = sdrf_3.rename(columns={col_name: "Material Type"})

    '''Remove empty columns if columns are optional.'''
    with open(f"json_files/optional_columns.json", "r") as optional_columns_file:
        optional_columns_dict = json.load(optional_columns_file)

    if experimental_design == 'standard':
        optional_columns = optional_columns_dict['standard']
    else:
        optional_columns = optional_columns_dict['cell_line']

    for column_name in optional_columns:
        if set(list(sdrf_3[column_name])) == {''}:
            sdrf_3 = sdrf_3.drop([column_name], axis=1)

    '''Write the new sdrf file to a file.'''
    if not sdrf_3.empty:
        print(f"saving {work_dir}/{sdrf_file_name}")
        sdrf_3.to_csv(f"{work_dir}/{sdrf_file_name}", sep="\t", index=False)

def create_magetab(work_dir, xlsx_dict, dataset_protocol_map, df, args, experimental_design, accession_number, technology_dict):

    accession = f"E-HCAD-{accession_number}"

    idf_file_name = f"{accession}.idf.txt"
    sdrf_file_name = f"{accession}.sdrf.txt"

    generate_idf_file(work_dir, args, dataset_protocol_map, xlsx_dict, accession, idf_file_name,
                      sdrf_file_name)
    generate_sdrf_file(work_dir, args, df, xlsx_dict, dataset_protocol_map, sdrf_file_name, experimental_design, technology_dict)

def main():
    parser = argparse.ArgumentParser(description="run hca -> scea tool")
    parser.add_argument(
        "-s",
        "--spreadsheet",
        type=str,
        required=True,
        help="Please provide a path to the HCA project spreadsheet."
    )
    parser.add_argument(
        "-id",
        "--project_uuid",
        type=str,
        required=True,
        help="Please provide an HCA ingest project submission id."
    )
    parser.add_argument(
        "-study",
        type=str,
        required=True,
        help="Please provide the SRA or ENA study accession."
    )
    parser.add_argument(
        "-name",
        type=str,
        required=False,
        default = 'cs_id',
        choices = ['cs_name','cs_id','sp_name','sp_id','other'],
        help="Please indicate which field to use as the sample name. cs=cell suspension, sp = specimen."
    )
    parser.add_argument(
        "-ac",
        "--accession_number",
        type=int,
        required=True,
        help="Provide an E-HCAD accession number. Please find the next suitable accession number by checking the google tracker sheet."
    )
    parser.add_argument(
        "-c",
        "--curators",
        nargs='+',
        required=True,
        help="space separated names of curators"
    )
    parser.add_argument(
        "-et",
        "--experiment_type",
        type=str,
        required=True,
        choices=['baseline','differential'],
        help="Please indicate whether this is a baseline or differential experimental design"
    )
    parser.add_argument(
        "--facs",
        action="store_true",
        default=None,
        help="Please specify this argument if FACS was used to isolate single cells"
    )
    parser.add_argument(
        "-f",
        "--experimental_factors",
        nargs='+',
        required=True,
        help="space separated list of experimental factors"
    )
    parser.add_argument(
        "-pd",
        "--public_release_date",
        type=str,
        required=True,
        help="Please enter the public release date in this format: YYYY-MM-DD"
    )
    parser.add_argument(
        "-hd",
        "--hca_update_date",
        type=str,
        required=True,
        help="Please enter the last time the HCA prohect submission was updated in this format: YYYY-MM-DD"
    )
    parser.add_argument(
        "-r",
        "--related_scea_accession",
        nargs='+',
        required=False,
        help="space separated list of related scea accession(s)"
    )
    parser.add_argument(
        "-o",
        "--output_dir",
        required=False,
        help="Provide full path to preferred output dir"
    )

    args = parser.parse_args()
    if not args.output_dir:
        work_dir = f"script_spreadsheets/{os.path.splitext(os.path.basename(args.spreadsheet))[0]}"
    else:
        work_dir = args.output_dir

    xlsx_dict = multitab_excel_to_single_txt.multitab_excel_to_dict(work_dir, args.spreadsheet)

    xlsx_dict = multitab_excel_to_single_txt.remove_unused_protocols(xlsx_dict)

    xlsx_dict = multitab_excel_to_single_txt.rename_protocol_columns(xlsx_dict)

    technology_dict = {
        "Fluidigm C1-based library preparation": "smart-like",
        "10X 3' v1": "10Xv1_3",
        "10X 5' v1": "10Xv1_5",
        "10X 3' v2": "10Xv2_3",
        "10X 5' v2": "10Xv2_5",
        "10X 3' v3": "10Xv3_3",
        "10X 3' v1 sequencing": "10Xv1_3",
        "10X 5' v1 sequencing": "10Xv1_5",
        "10X 3' v2 sequencing": "10Xv2_3",
        "10X 5' v2 sequencing": "10Xv2_5",
        "10X 3' v3 sequencing": "10Xv3_3",
        "10x 3' v1": "10Xv1_3",
        "10x 5' v1": "10Xv1_5",
        "10x 3' v2": "10Xv2_3",
        "10x 5' v2": "10Xv2_5",
        "10x 3' v3": "10Xv3_3",
        "10x 3' v1 sequencing": "10Xv1_3",
        "10x 5' v1 sequencing": "10Xv1_5",
        "10x 3' v2 sequencing": "10Xv2_3",
        "10x 5' v2 sequencing": "10Xv2_5",
        "10x 3' v3 sequencing": "10Xv3_3",
        "Drop-seq": "drop-seq",
        "inDrop": "drop-seq",
        "Smart-like": "smart-like",
        "Smart-seq2": "smart-seq",
        "Smart-seq": "smart-seq"
    }

    check_experimental_design.check_biomaterial_linkings(xlsx_dict)
    check_experimental_design.check_protocol_linkings(xlsx_dict)
    experimental_design = check_experimental_design.get_experimental_design(xlsx_dict)

    if experimental_design == "cell_line_only" or experimental_design == "organoid":
        check_experimental_design.check_cell_lines_linked(xlsx_dict)
    if experimental_design == "organoid_only" or experimental_design == "organoid":
        check_experimental_design.check_organoids_linked(xlsx_dict)

    check_experimental_design.check_donor_exists(xlsx_dict)
    check_experimental_design.check_specimen_exists(xlsx_dict)
    check_experimental_design.check_input_to_cell_suspension(xlsx_dict)
    check_experimental_design.check_technology_eligibility(xlsx_dict, technology_dict)
    check_experimental_design.check_species_eligibility(xlsx_dict)
    check_experimental_design.check_for_pooled_samples(xlsx_dict)

    merged_df = multitab_excel_to_single_txt.merge_dataframes(xlsx_dict,experimental_design)
    merged_df_unique_runs = merged_df.drop_duplicates(subset=['sequence_file.insdc_run_accessions'])
    clean_merged_df = multitab_excel_to_single_txt.clean_df(merged_df_unique_runs)

    df = multitab_excel_to_single_txt.create_new_protocol_columns(clean_merged_df, xlsx_dict)

    dataset_protocol_map = get_protocol_map.prepare_protocol_map(xlsx_dict, df, args)

    create_magetab(work_dir, xlsx_dict, dataset_protocol_map, df, args, experimental_design, args.accession_number, technology_dict)

if __name__ == '__main__':
    main()


#!/bin/python3

import os
from collections import defaultdict

# Complete the sherlockAndAnagrams function below.
def sherlockAndAnagrams(s):
    subs = defaultdict(int)
    count = 0
    l = len(s)
    for i in range(l - 1, 0, -1):
        for j in range(0, l, 1):
            if i + j > l:
                break
            t = s[j:i+j]
            t = str(sorted(t))
            if t not in subs:
                subs[t] = 1
            else:
                subs[t] += 1

    for k, value in subs.items():
        if value > 1:
            count += int((value - 1) * value / 2)

    return count

if __name__ == '__main__':
    fptr = open(os.environ['OUTPUT_PATH'], 'w')

    q = int(input())

    for q_itr in range(q):
        s = input()

        result = sherlockAndAnagrams(s)

        fptr.write(str(result) + '\n')

    fptr.close()

Tutorials/Interview Preparation Kit/Dictionaries and Hashmaps/Sherlock and Anagrams/code.py3


from django.shortcuts import render, redirect
from django.http import HttpResponse
from django.contrib.auth.decorators import login_required
from django.core.paginator import Paginator, PageNotAnInteger, EmptyPage
from .models import Project
from .forms import ProjectForm
from .utils import search_projects

def projects(requests):
    page = requests.GET.get("page")
    projects, search_query = search_projects(requests)

    results_per_page = 3
    paginator = Paginator(projects, results_per_page)
    try:
        projects = paginator.page(page)
    except PageNotAnInteger:
        page = 1
        projects = paginator.page(page)
    except EmptyPage:
        page = paginator.num_pages
        projects = paginator.page(page)

    context = {"projects": projects, "search_query": search_query, "paginator": paginator}
    return render(requests, "projects/projects.html", context)

def project(requests, pk):
    project = Project.objects.get(id=pk)
    context = {"project": project}
    return render(requests, "projects/single_project.html", context)

@login_required(login_url="login")
def create_project(request):
    profile = request.user.profile
    form = ProjectForm()

    if request.method == "POST":
        form = ProjectForm(request.POST, request.FILES)
        if form.is_valid():
            project = form.save(commit = False)
            project.owner = profile
            project.save()
            return redirect("account")
    context = {"form": form}
    return render(request, "projects/project_form.html", context)

@login_required(login_url="login")
def update_project(request, pk):
    profile = request.user.profile
    project = profile.project_set.get(id=pk)
    form = ProjectForm(instance=project)

    if request.method == "POST":
        form = ProjectForm(request.POST, request.FILES, instance=project)
        if form.is_valid():
            form.save()
            return redirect("account")
    context = {"form": form}
    return render(request, "projects/project_form.html", context)

@login_required(login_url="login")
def delete_project(request, pk):
    profile = request.user.profile
    project = profile.project_set.get(id=pk)
    if request.method == "POST":
        project.delete()
        return redirect("account")
    context = {"object": project}
    return render(request, 'delete_template.html', context)


import os
import datetime
import numpy as np
import pandas as pd

from matplotlib import pyplot as plt
from matplotlib.ticker import MultipleLocator, FixedLocator
from fig1_plot import plot_data
# ---------------------------
def run(win2_sparse, win2_dense, win4_sparse, win4_dense, win6_sparse, win6_dense, figname):

    plt.rcParams.update({'font.size': 10})
    fig, axes = plt.subplots(1, 3, figsize=[8, 2.5], dpi=300)

    prop_cycle = plt.rcParams['axes.prop_cycle']
    colors = prop_cycle.by_key()['color']
    # ---------------------------
    fnames = [[win2_sparse, win2_dense],
              [win4_sparse, win4_dense],
              [win6_sparse, win6_dense]]

    for i, window_size in enumerate([2, 4, 6]):
        for j, sampling in enumerate(["sparse", "dense"]):
            plot_data(axes[i], None, colors[j], fnames[i][j], linewidth=1, alpha=0.15)
            plot_data(axes[i], None, "k", fnames[i][j], linewidth=0.5, alpha=1, n_trial=1)     ## representative trajectory
            axes[i].plot([np.nan, np.nan], c=colors[j], label=sampling, linewidth=1, alpha=0.7)

        axes[i].legend()#title=f"samples per {window_size}-day window")
        axes[i].xaxis.set_major_locator(FixedLocator(np.arange(20, 168.1, 24)))  # FixedLocator
        axes[i].xaxis.set_minor_locator(FixedLocator(np.arange(20, 168.1, 8)))  # FixedLocator
        axes[i].set_xlim([20, 168])
        axes[i].grid(which = "both")

    # axis label

    axes[0].set_ylabel("Number of\nsegregating sites")
    axes[0].set_xlabel('Days post index case')
    axes[1].set_xlabel('Days post index case')
    axes[2].set_xlabel('Days post index case')

    axes[0].yaxis.set_major_locator(FixedLocator(np.arange(0, 300, 25)))  # FixedLocator
    axes[1].yaxis.set_major_locator(FixedLocator(np.arange(0, 300, 25)))  # FixedLocator
    axes[2].yaxis.set_major_locator(FixedLocator(np.arange(0, 300, 25)))  # FixedLocator

    ## add subplot label
    labels = 'ABC'.lower()
    for i, label in enumerate(labels):
        if i == 0:
            axes[i].text(-0.22, 1.1, label, transform=axes[i].transAxes, fontsize=16, fontweight='bold', va='top',
                         ha='left')
            axes[i].set_ylim([0, 300])
        else:
            axes[i].text(-0.18, 1.1, label, transform=axes[i].transAxes, fontsize=16, fontweight='bold', va='top',
                         ha='left')
            axes[i].set_ylim([0, 300])

    ## save figure
    fig.tight_layout(pad=0.5)
    fig.savefig(f'manuscript figures/pdf/{figname}.pdf')
    fig.savefig(f'manuscript figures/png/{figname}.png')

    fig.clf()
    plt.cla()

    return

if __name__ == "__main__":
    import os
    import argparse

    if os.getcwd().find("manuscript_figure_script") == -1:
        parser = argparse.ArgumentParser()
        parser.add_argument('--figname', type=str, default="figureS1")
        parser.add_argument('--win2_sparse', type=str)
        parser.add_argument('--win2_dense' , type=str)
        parser.add_argument('--win4_sparse', type=str)
        parser.add_argument('--win4_dense' , type=str)
        parser.add_argument('--win6_sparse', type=str)
        parser.add_argument('--win6_dense' , type=str)

        args = parser.parse_args()

    else:
        os.chdir(os.getcwd().replace("manuscript_figure_script", ""))
        parser = argparse.ArgumentParser()
        args = parser.parse_args()
        args.figname = "figureS1"

        args.win2_sparse = "simulated_data/simpleSEIR_R0=16e-1_mu2e-1/!!!!!!!_trial##_segsites.tsv"
        args.win2_dense  = "simulated_data/simpleSEIR_R0=16e-1_mu2e-1/!!!!!!!_trial##_segsites.tsv"
        args.win4_sparse = "simulated_data/simpleSEIR_R0=16e-1_mu2e-1/!!!!!!!_trial##_segsites.tsv"
        args.win4_dense  = "simulated_data/simpleSEIR_R0=16e-1_mu2e-1/!!!!!!!_trial##_segsites.tsv"
        args.win6_sparse = "simulated_data/simpleSEIR_R0=16e-1_mu2e-1/!!!!!!!_trial##_segsites.tsv"
        args.win6_dense  = "simulated_data/simpleSEIR_R0=16e-1_mu2e-1/!!!!!!!_trial##_segsites.tsv"

    print(args)
    run(args.win2_sparse,
        args.win2_dense,
        args.win4_sparse,
        args.win4_dense,
        args.win6_sparse,
        args.win6_dense,
        args.figname)


import time
from selenium import webdriver
from bs4 import BeautifulSoup, NavigableString, Tag
import requests
import pandas as pd
import logging
import csv
import argparse

logging.basicConfig(level=logging.CRITICAL)

parser = argparse.ArgumentParser(description='fill arguement')

parser.add_argument('--novel_name', type=str, required=False,
                    help='input the novel name')

parser.add_argument('--page', type=int, required=False, default=1000,
                    help='input the novel name')

parser.add_argument('--experiment', type=str, required=True,
                    help = 'input the execution experiment name')

parser.add_argument('--url', type=str, required=False, default="https://www.ptwxz.com/",
                    help= "input novel table of content's url")

args = parser.parse_args()

logging.debug("novel name", args.novel_name)
logging.debug("novel page", args.page)

count_page = args.page

browser = webdriver.Chrome()
url=args.url
browser.get(url)

soup = BeautifulSoup(browser.page_source, 'html.parser')

data = pd.DataFrame()

#count the page
count = 0
for i in soup.find_all('li'):
    for j in i.find_all('a', href=True):

        if(count > count_page):
            break

        newurl = url+'/'+j.get('href')
        response = requests.get(newurl, verify=False, headers={
            "User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36"
        })
        print("new url", newurl)
        logging.debug("status code ", response.status_code)
        logging.debug("web encode ",response.encoding)
        logging.debug("apparent encode", response.apparent_encoding)
        response.encoding = response.apparent_encoding

        soup = BeautifulSoup(response.text, 'html.parser')

        title = soup.find("title")

        text = ""

        for i in soup.find_all('br'):
            next_s = i.nextSibling
            if(next_s and type(next_s) == "<class 'bs4.element.NavigableString'>"):
                continue
            if(isinstance(next_s, Tag)):
                continue
            #print(next_s)
            text += str(next_s)
        if(title.string != "404 Not Found"):
            series = pd.DataFrame({
                "title": title.string,
                "desc" : text
            }, index=[str(count)])
            data = data.append(series)
            count += 1
        else:
            print("error 404")

data_csv = data.to_csv(args.experiment+'.csv')
print(data_csv)


import typer
from motion_marmot.advanced_motion_filter import AdvancedMotionFilter

app = typer.Typer()

@app.command()
def run():
    amf = AdvancedMotionFilter("model/scene_knn_model", 0, 0)
    print(amf)

def main():
    """Main program"""
    app()

if __name__ == "__main__":
    main()


import argparse
import sys
from typing import Dict

from kenning.cli.autocompletion import configure_autocomplete
from kenning.cli.config import (
    AVAILABLE_COMMANDS,
    MAP_COMMAND_TO_SCENARIO,
    SUB_DEST_FORM,
    setup_base_parser,
)
from kenning.cli.parser import (
    Parser,
    ParserHelpException,
    print_help_from_parsers,
)
from kenning.utils.excepthook import (
    MissingKenningDependencies,
    find_missing_optional_dependency,
)
from kenning.utils.logger import KLogger

def main():
    """
    The entrypoint of Kenning CLI.

    Creates and manages parsers, runs subcommands, and handle errors.
    """
    verbosity = "WARNING"
    if (
        "--verbosity" in sys.argv
        and len(sys.argv) > sys.argv.index("--verbosity") + 1
    ):
        verbosity = sys.argv[sys.argv.index("--verbosity") + 1]
    KLogger.configure()
    KLogger.set_verbosity(level=verbosity)
    configure_autocomplete()
    parser, parsers = setup_base_parser()

    # Get only subcommands and help
    i = 1
    while i < len(sys.argv) and sys.argv[i] in AVAILABLE_COMMANDS:
        i += 1

    # Parse subcommands and help
    args, rem = parser.parse_known_args(sys.argv[1:i])
    args, rem = parser.parse_known_args(args=rem, namespace=args)
    rem += sys.argv[i:]

    # Retrieve parsed subcommands
    i = 0
    subcommands = []
    while getattr(args, SUB_DEST_FORM.format(i), None) is not None:
        subcommands.append(getattr(args, SUB_DEST_FORM.format(i)))
        i += 1

    if not subcommands:
        parser.print_help()
        if rem:
            parser.exit(
                2,
                f"{parser.prog}: error: '{rem[0]}' doesn't match any subcommand",  # noqa: E501
            )
        return

    # Creating parent-parsers from scenarios and completing description
    groups = {}
    parse_all = True
    used_configs = set()
    parents, description = [], []
    seq_description = len(subcommands) > 1
    if seq_description:
        description.append("Running in sequence:")
    for subcommand in subcommands:
        scenario = MAP_COMMAND_TO_SCENARIO[subcommand]
        if isinstance(scenario.description, Dict):
            desc = scenario.description[subcommand]
        else:
            desc = scenario.description[1:]
        description.append(("- " if seq_description else "") + desc)
        if scenario.configure_parser not in used_configs:
            parents.append(
                scenario.configure_parser(types=subcommands, groups=groups)[0]
            )
            used_configs.add(scenario.configure_parser)
            parse_all = parse_all and scenario.parse_all
    description = "\n".join(description)

    # The main parser without subcommands
    parser = Parser(
        "kenning " + " ".join(subcommands),
        conflict_handler="resolve",
        parents=parents,
        description=description,
        add_help=False,
    )

    def _print_help():
        """
        Prints help message for non sequenced subcommands.
        """
        print_help_from_parsers(
            "kenning " + " ".join(subcommands),
            [parsers[tuple(subcommands)], parser],
            description,
        )

    # Print help, including possible subcommands
    if args.help:
        _print_help()
        return

    errors = []
    # Parse arguments
    if parse_all:
        try:
            args = parser.parse_args(
                args=sys.argv[1 + len(subcommands) :], namespace=args
            )
        except ParserHelpException:
            args.help = True
        if args.help:
            _print_help()
            return
        rem = []
    else:
        # Parse only known args if scenario will add more arguments
        try:
            args, rem = parser.parse_known_args(
                sys.argv[1 + len(subcommands) :], namespace=args
            )
        except ParserHelpException as ex:
            errors.append(ex.error)

    # Run subcommands
    used_functions = set()
    for subcommand in subcommands:
        run = MAP_COMMAND_TO_SCENARIO[subcommand].run
        if run in used_functions:
            continue
        try:
            try:
                result = run(args, not_parsed=rem)
                if result:
                    parser.error(
                        f"`{subcommand}` subcommand did not end successfully",
                        print_usage=False,
                    )
            except ModuleNotFoundError as er:
                extras = find_missing_optional_dependency(er.name)
                if not extras:
                    raise
                er = MissingKenningDependencies(
                    name=er.name, path=er.path, optional_dependencies=extras
                )
                raise

            except argparse.ArgumentError as er:
                parser.error(er.message)
                return 2
        except ParserHelpException as ex:
            # Prepare combined errors
            if ex.error is not None:
                errors.append(ex.error)
            if len(errors) > 1:
                ex.error = "\n"
                for error in errors:
                    ex.error += f"- {error}\n"
            elif errors:
                ex.error = errors[0]
            # Print help with errors
            ex.print(
                parser,
                [parsers[tuple(subcommands)], parser],
                description,
            )
            return

        used_functions.add(run)


from selenium import webdriver
from selenium.webdriver.common.by import By
from time import sleep

driver = webdriver.Firefox(executable_path="C://Users//seker//Documents//driver//geckodriver.exe")
driver.get("https://www.rahulshettyacademy.com/AutomationPractice/")
driver.maximize_window()
driver.implicitly_wait(10)

row = len(driver.find_elements(By.XPATH,"//table/tbody/tr"))
col = len(driver.find_elements(By.XPATH,"//table/tbody/tr/th"))

print('Number of the rows:', row)
print('Number of the col :', col)

for r in range(2,row+1):
    for c in range(1,col+1):
        value = driver.find_element(By.XPATH,"//body[1]/div[3]/div[1]/fieldset[1]/table[1]/tbody[1]/tr["+str(r)+"]/td["+str(c)+"]").text
        print(value,end='   ')
    print()

driver.quit()


from flask_ngrok import run_with_ngrok
from flask import Flask, render_template, request
import numpy as np
import sys
import os
sys.path.append(os.path.abspath("./model"))
from load import *
import tensorflow.keras.backend as K
from tensorflow.keras.preprocessing import image
from werkzeug import secure_filename
from werkzeug.routing import Rule

K.clear_session()

global model, food_classes

app = Flask(__name__)

path_trained_model = "model/best_model_58class_all.hdf5"
model = init(path_trained_model)
food_classes = init_food_classes()

# Gère le routing
app.url_map.add(Rule('/', endpoint="predict"))
app.url_map.add(Rule('/predict/', endpoint="predict"))

# IMPORTANT : Décommenter la ligne ci-dessous pour exécuter NGROK et sonner accès à l'application en ligne
# run_with_ngrok(app)

def do_prediction(food_img_path):
    K.clear_session()
    img = image.load_img(food_img_path, target_size=(299, 299))
    img = image.img_to_array(img)
    img = np.expand_dims(img, axis=0)
    img /= 255.

    pred = model.predict(img)

    indexes = np.argsort(pred[0])[::-1][:3]

    food_probas = np.array(pred[0])[indexes]
    food_probas = [round(proba*100, 2) if idx == 0 else round(proba*100, 5) for idx, proba in enumerate(food_probas)]

    food_classes.sort()

    pred_values = [food_classes[index].capitalize().replace("_", " ") for index in indexes]

    print(pred_values)

    return render_template("draw.html", food_image = os.path.join("..", food_img_path), food_pred = pred_values, food_probas = food_probas)

# @app.route("/")
# @app.route("/predict/")
# @app.route('/uploader/', methods=["GET", "POST"])
@app.endpoint('predict')
def predict():
    path_images_validation = os.path.join("static", "images_validation")
    path_list_food_test = os.path.join("/", "home" , "sully" , "code", "practice", "portfolio_data", "food-recognizer", "datasets", "google_images_dataset", "images", "test")

    if request.method == 'POST':
        f = request.files['file']
        f.save(os.path.join(path_images_validation, secure_filename(f.filename)))
        print('file uploaded successfully')

        food_img = secure_filename(f.filename)
        food_img_path = os.path.join(path_images_validation, food_img)
    else:
        food_validation_list = os.listdir(path_images_validation)
        food_img = food_validation_list[np.random.randint(len(food_validation_list))]
        food_img_path = os.path.join(path_images_validation, food_img)

    return do_prediction(food_img_path)

if __name__ == "__main__":
    app.run()


# Written by Hayatu Abdullahi

from PIL import ImageTk, Image
import cv2
from os.path import splitext
import numpy as np
import matplotlib.pyplot as plt

# Obtain camera intrinsics for the images
def camera_intrinsic(camera):
    global fx, fy, cx, cy, depth
    depth = 0
    fx = float(((str(camera[0]).lstrip('[').rstrip(']'))))
    fy = float(((str(camera[1]).lstrip('[').rstrip(']'))))
    cx = float(((str(camera[2]).lstrip('[').rstrip(']'))))
    cy = float(((str(camera[3]).lstrip('[').rstrip(']'))))
    print('Camera Intrinsics:', fx,fy,cx,cy)

    return fx, fy, cx, cy

# Convert RGB images to match depth size if necessary
def convert(imgselect, depth_map, depth_map2):
    global img4
    global img7
    global rgb
    global rgb2

    rgb = Image.open(imgselect)
    rgb2 = np.asarray(rgb)
    dp_size = depth_map.shape
    if rgb2.size != depth_map.size:
        img =cv2.resize(rgb2,(dp_size[1],dp_size[0]))
        img4 = Image.fromarray(img)

        if img4.mode != 'RGB':
            img4 = img4.convert('RGB')

    pic2 = Image.open('images/8.jpeg')
    #pic2 = Image.open('images\8.jpeg')
    rgb3 = np.asarray(pic2)
    dp_size2 = depth_map2.shape
    if rgb3.size != depth_map2.size:
        img6 =cv2.resize(rgb3,(dp_size[1],dp_size[0]))
        img7 = Image.fromarray(img6)

        if img7.mode != 'RGB':
            img7 = img7.convert('RGB')

    print('Depth Size', depth_map.shape)
    print('Image Size',img.shape)
    #depth_map2 = depth_map.reshape(2016, 1512, 3)
    #print (depth_map2.shape)

    return img4, rgb, img7


from django.shortcuts import render, redirect
from .models import Product
from .form import ProductForm
from django.http import Http404
from merchan.models import Merchandise

def index(request):
    return render(request, 'product/index.html')

def create_product(request):
    data = {}
    if request.method == "GET":
        group_form = ProductForm()
        data['group_form'] = group_form
        return render(request, 'product/create.html', context=data)
    elif request.method == "POST":
        group_form = ProductForm(request.POST, request.FILES)
        group_form.save()
        return redirect('/product/list')

def delete_product(request, slug):
    data = {}
    if not request.user.is_authenticated or not Product.objects.filter(slug=slug).exists():
        raise Http404
    product = Product.objects.get(slug=slug)
    if request.method == "GET":
        data['product'] = product
        return render(request, 'product/delete.html', context=data)
    elif request.method == "POST":
        product.delete()
        return redirect('/product/list')

def product_detail(request, slug):
    data = {}
    if not Product.objects.filter(slug=slug).exists():
        raise Http404
    product = Product.objects.get(slug=slug)

    # Секция со студентами
    data['product'] = product
    merchan = Merchandise.objects.filter(product__id=product.id)
    data['merchan'] = merchan
    return render(request, 'product/details.html', context=data)

def product_list(request):
    data = {}
    all_product = Product.objects.all()
    data['product'] = all_product
    return render(request, 'product/list.html', context=data)

def update_product(request, slug):
    data = {}
    if not request.user.is_authenticated or not Product.objects.filter(slug=slug).exists():
        raise Http404
    product = Product.objects.get(slug=slug)
    data['product'] = product
    if request.method == "GET":
        product_form = ProductForm(instance=product)
        data['product_form'] = product_form
        return render(request, 'product/update.html', context=data)
    elif request.method == "POST":
        product_form = ProductForm(request.POST)
        if product_form.is_valid():
            product.image = product_form.cleaned_data['image']
            product.name = product_form.cleaned_data['name']
            product.about = product_form.cleaned_data['about']
            product.max_presets = product_form.cleaned_data['max_presets']
            product.is_discount_preset = product_form.cleaned_data['is_discount_preset']
            product.save()
        else:
            print('NOT VALID FORM!')

        return redirect('/product/list')