### Import required python modules
from flask import abort
import platform
import os
import itertools
from os import makedirs, mkdir, walk
from openpyxl import load_workbook
from os.path import (
    isdir,
    join,
    splitext,
    basename,
    exists,
    expanduser,
    getsize,
)
import pandas as pd
import time
import shutil
import subprocess
import re
from timeit import default_timer as timer
from datetime import timedelta
import gevent
import pathlib
from datetime import datetime, timezone
import requests
from permissions import pennsieve_get_current_user_permissions
from utils import get_dataset_id, create_request_headers
from namespaces import NamespaceEnum, get_namespace_logger
from openpyxl.styles import PatternFill
from openpyxl import load_workbook
from utils import load_metadata_to_dataframe

namespace_logger = get_namespace_logger(NamespaceEnum.ORGANIZE_DATASETS)
from authentication import get_access_token

### Global variables
create_soda_json_progress = 0
create_soda_json_total_items = 0
create_soda_json_completed = 0
curateprogress = " "
curatestatus = " "
curateprintstatus = " "
total_dataset_size = 1
curated_dataset_size = 0
start_time = 0

userpath = expanduser("~")
configpath = join(userpath, ".pennsieve", "config.ini")
submitdataprogress = " "
submitdatastatus = " "
submitprintstatus = " "
total_file_size = 1
uploaded_file_size = 0
start_time_bf_upload = 0
start_submit = 0
metadatapath = join(userpath, "SODA", "SODA_metadata")

myds = ""
initial_bfdataset_size = 0
upload_directly_to_ps = 0
initial_bfdataset_size_submit = 0

forbidden_characters = '<>:"/\|?*'
forbidden_characters_bf = '\/:*?"<>'

PENNSIEVE_URL = "https://api.pennsieve.io"

from namespaces import NamespaceEnum, get_namespace_logger
namespace_logger = get_namespace_logger(NamespaceEnum.MANAGE_DATASETS)

METADATA_FILES_SPARC = [
        "submission.xlsx",
        "submission.csv",
        "submission.json",
        "dataset_description.xlsx",
        "dataset_description.csv",
        "dataset_description.json",
        "subjects.xlsx",
        "subjects.csv",
        "subjects.json",
        "samples.xlsx",
        "samples.csv",
        "samples.json",
        "README.txt",
        "CHANGES.txt",
        "code_description.xlsx",
        "inputs_metadata.xlsx",
        "outputs_metadata.xlsx",
    ]

### Internal functions
def TZLOCAL():
    """
        Get current local timezone
    """
    return datetime.now(timezone.utc).astimezone().tzinfo

## these subsequent CheckLeafValue and traverseForLeafNodes functions check for the validity of file paths,
## and folder and file size
def checkLeafValue(leafName, leafNodeValue):
    error, c = "", 0
    total_dataset_size = 1
    curatestatus = ""

    if isinstance(leafNodeValue, list):
        filePath = leafNodeValue[0]

        if exists(filePath):
            filePathSize = getsize(filePath)
            if filePathSize == 0:
                c += 1
                error = error + filePath + " is 0 KB
"
            else:
                total_dataset_size += filePathSize
        else:
            c += 1
            error = error + filePath + " doesn not exist!
"

    elif isinstance(leafNodeValue, dict):
        c += 1
        error = error + leafName + " is empty
"

    if c > 0:
        error += "
Please remove invalid files/folders from your dataset and try again"
        curatestatus = "Done"
        abort(400, error)

    return [True, total_dataset_size - 1]

def traverseForLeafNodes(datasetStructure):
    total_dataset_size = 1

    for key in datasetStructure:
        if isinstance(datasetStructure[key], list):

            returnedOutput = checkLeafValue(key, datasetStructure[key])

            if returnedOutput[0]:
                total_dataset_size += returnedOutput[1]

        else:

            if len(datasetStructure[key]) == 0:
                returnedOutput = checkLeafValue(key, datasetStructure[key])

            else:
                # going one step down in the object tree
                traverseForLeafNodes(datasetStructure[key])

    return total_dataset_size

######## CREATE FILES FOR CURATE_DATASET FUNCTION
def createFiles(jsonpath, fileKey, distdir, listallfiles):
    # fileKey is the key in json structure that is a file (meaning that its value is an array)
    # note: this fileKey can be the new name of the file (if renamed by users in SODA)
    # (or can be the original basename)

    srcfile = jsonpath[fileKey][0]
    distfile = distdir
    listallfiles.append([srcfile, distfile])

def ignore_empty_high_level_folders(jsonObject):
    items_to_delete = [
        folder for folder in jsonObject.keys() if len(jsonObject[folder].keys()) == 0
    ]

    for item in items_to_delete:
        del jsonObject[item]

    return jsonObject

def generate_dataset_locally(destinationdataset, pathdataset, newdatasetname, jsonpath):
    """
    Associated with 'Generate' button in the 'Generate dataset' section of SODA interface
    Checks validity of files / paths / folders and then generates the files and folders
    as requested along with progress status

    Args:
        destinationdataset: type of destination dataset ('modify existing', 'create new', or 'upload to Pennsieve')
        pathdataset: destination path of new dataset if created locally or name of Pennsieve account (string)
        newdatasetname: name of the local dataset or name of the dataset on Pennsieve (string)
        manifeststatus: boolean to check if user request manifest files
        jsonpath: path of the files to be included in the dataset (dictionary)
    """
    global curatestatus  # set to 'Done' when completed or error to stop progress tracking from front-end
    global curateprogress  # GUI messages shown to user to provide update on progress
    global curateprintstatus  # If = "Curating" Progress messages are shown to user
    global total_dataset_size  # total size of the dataset to be generated
    global curated_dataset_size  # total size of the dataset generated (locally or on Pennsieve) at a given time
    global start_time
    global myds
    global upload_directly_to_ps
    global start_submit
    global initial_bfdataset_size

    curateprogress = " "
    curatestatus = ""
    curateprintstatus = " "
    error, c = "", 0
    curated_dataset_size = 0
    start_time = 0
    upload_directly_to_ps = 0
    start_submit = 0
    initial_bfdataset_size = 0

    jsonstructure_non_empty = ignore_empty_high_level_folders(jsonpath)

    if destinationdataset == "create new":
        if not isdir(pathdataset):
            curatestatus = "Done"
            abort(400, "Please select a valid folder for new dataset")
        if not newdatasetname:
            curatestatus = "Done"
            abort(400, "Please enter a valid name for new dataset folder")
        if check_forbidden_characters(newdatasetname):
            curatestatus = "Done"
            abort(400,  "A folder name cannot contain any of the following characters " + forbidden_characters)

    total_dataset_size = 1

    # check if path in jsonpath are valid and calculate total dataset size
    total_dataset_size = traverseForLeafNodes(jsonstructure_non_empty)
    total_dataset_size = total_dataset_size - 1

    # CREATE NEW
    if destinationdataset == "create new":
        try:
            listallfiles = []
            pathnewdatasetfolder = join(pathdataset, newdatasetname)
            pathnewdatasetfolder = return_new_path(pathnewdatasetfolder)
            open_file(pathnewdatasetfolder)

            curateprogress = "Started"
            curateprintstatus = "Curating"
            start_time = time.time()
            start_submit = 1

            pathdataset = pathnewdatasetfolder
            mkdir(pathdataset)
            create_dataset(pathdataset, jsonstructure_non_empty, listallfiles)

            curateprogress = "New dataset created"
            curateprogress = "Success: COMPLETED!"
            curatestatus = "Done"

        except Exception as e:
            curatestatus = "Done"
            raise e

def create_folder_level_manifest(jsonpath, jsondescription):
    """
    Function to create manifest files for each SPARC folder.
    Files are created in a temporary folder

    Args:
        datasetpath: path of the dataset (string)
        jsonpath: all paths in json format with key being SPARC folder names (dictionary)
        jsondescription: description associated with each path (dictionary)
    Action:
        Creates manifest files in xslx format for each SPARC folder
    """
    global total_dataset_size
    local_timezone = TZLOCAL()

    try:
        datasetpath = metadatapath
        shutil.rmtree(datasetpath) if isdir(datasetpath) else 0
        makedirs(datasetpath)
        folders = list(jsonpath.keys())
        # In each SPARC folder, generate a manifest file
        for folder in folders:
            if jsonpath[folder] != []:
                # Initialize dataframe where manifest info will be stored
                df = pd.DataFrame(
                    columns=[
                        "filename",
                        "timestamp",
                        "description",
                        "file type",
                        "Additional Metadata",
                    ]
                )
                # Get list of files/folders in the the folder
                # Remove manifest file from the list if already exists
                folderpath = join(datasetpath, folder)
                allfiles = jsonpath[folder]
                alldescription = jsondescription[folder + "_description"]

                countpath = -1
                for pathname in allfiles:
                    countpath += 1
                    if (
                        basename(pathname) == "manifest.csv"
                        or basename(pathname) == "manifest.xlsx"
                    ):
                        allfiles.pop(countpath)
                        alldescription.pop(countpath)

                # Populate manifest dataframe
                filename, timestamp, filetype, filedescription = [], [], [], []
                countpath = -1
                for paths in allfiles:
                    if isdir(paths):
                        key = basename(paths)
                        alldescription.pop(0)
                        for subdir, dirs, files in os.walk(paths):
                            for file in files:
                                gevent.sleep(0)
                                filepath = pathlib.Path(paths) / subdir / file
                                mtime = filepath.stat().st_mtime
                                lastmodtime = datetime.fromtimestamp(mtime).astimezone(
                                    local_timezone
                                )
                                timestamp.append(
                                    lastmodtime.isoformat()
                                    .replace(".", ",")
                                    .replace("+00:00", "Z")
                                )
                                full_filename = filepath.name

                                if folder == "main":  # if file in main folder
                                    filename.append(
                                        full_filename
                                    ) if folder == "" else filename.append(
                                        join(folder, full_filename)
                                    )
                                else:
                                    subdirname = os.path.relpath(
                                        subdir, paths
                                    )  # gives relative path of the directory of the file w.r.t paths
                                    if subdirname == ".":
                                        filename.append(join(key, full_filename))
                                    else:
                                        filename.append(
                                            join(key, subdirname, full_filename)
                                        )

                                fileextension = splitext(full_filename)[1]
                                if (
                                    not fileextension
                                ):  # if empty (happens e.g. with Readme files)
                                    fileextension = "None"
                                filetype.append(fileextension)
                                filedescription.append("")
                    else:
                        gevent.sleep(0)
                        countpath += 1
                        filepath = pathlib.Path(paths)
                        file = filepath.name
                        filename.append(file)
                        mtime = filepath.stat().st_mtime
                        lastmodtime = datetime.fromtimestamp(mtime).astimezone(
                            local_timezone
                        )
                        timestamp.append(
                            lastmodtime.isoformat()
                            .replace(".", ",")
                            .replace("+00:00", "Z")
                        )
                        filedescription.append(alldescription[countpath])
                        if isdir(paths):
                            filetype.append("folder")
                        else:
                            fileextension = splitext(file)[1]
                            if (
                                not fileextension
                            ):  # if empty (happens e.g. with Readme files)
                                fileextension = "None"
                            filetype.append(fileextension)

                df["filename"] = filename
                df["timestamp"] = timestamp
                df["file type"] = filetype
                df["description"] = filedescription

                makedirs(folderpath)
                # Save manifest as Excel sheet
                manifestfile = join(folderpath, "manifest.xlsx")
                df.to_excel(manifestfile, index=None, header=True)
                wb = load_workbook(manifestfile)
                ws = wb.active
                blueFill = PatternFill(
                    start_color="9DC3E6", fill_type="solid"
                )
                greenFill = PatternFill(
                    start_color="A8D08D", fill_type="solid"
                )
                yellowFill = PatternFill(
                    start_color="FFD965", fill_type="solid"
                )
                ws['A1'].fill = blueFill
                ws['B1'].fill = greenFill
                ws['C1'].fill = greenFill
                ws['D1'].fill = greenFill
                ws['E1'].fill = yellowFill
                wb.save(manifestfile)
                total_dataset_size += path_size(manifestfile)
                jsonpath[folder].append(manifestfile)

        return jsonpath

    except Exception as e:
        raise e

def check_forbidden_characters(my_string):
    """
    Check for forbidden characters in file/folder name

    Args:
        my_string: string with characters (string)
    Returns:
        False: no forbidden character
        True: presence of forbidden character(s)
    """
    regex = re.compile("[" + forbidden_characters + "]")
    if regex.search(my_string) == None and "\\" not in r"%r" % my_string:
        return False
    else:
        return True

def folder_size(path):
    """
    Provides the size of the folder indicated by path

    Args:
        path: path of the folder (string)
    Returns:
        total_size: total size of the folder in bytes (integer)
    """
    total_size = 0

    for path, _, files in walk(path):
        for f in files:
            fp = join(path, f)
            total_size += getsize(fp)
    return total_size

def open_file(file_path):
    """
    Opening folder on all platforms
    https://stackoverflow.com/questions/6631299/python-opening-a-folder-in-explorer-nautilus-mac-thingie

    Args:
        file_path: path of the folder (string)
    Action:
        Opens file explorer window to the given path
    """
    try:
        if platform.system() == "Windows":
            subprocess.Popen(r"explorer /select," + str(file_path))
        elif platform.system() == "Darwin":
            subprocess.Popen(["open", file_path])
        else:
            subprocess.Popen(["xdg-open", file_path])
    except Exception as e:
        raise e

def path_size(path):
    """
    Returns size of the path, after checking if it's a folder or a file
    Args:
        path: path of the file/folder (string)
    Returns:
        total_size: total size of the file/folder in bytes (integer)
    """
    if isdir(path):
        return folder_size(path)
    else:
        return getsize(path)

def mycopyfile_with_metadata(src, dst, *, follow_symlinks=True):
    """
    Copy file src to dst with metadata (timestamp, permission, etc.) conserved

    Args:
        src: source file (string)
        dst: destination file (string)
    Returns:
        dst
    """
    if not follow_symlinks and os.path.islink(src):
        os.symlink(os.readlink(src), dst)
    else:
        with open(src, "rb") as fsrc:
            with open(dst, "wb") as fdst:
                mycopyfileobj(fsrc, fdst)
    shutil.copystat(src, dst)
    return dst

def mycopyfileobj(fsrc, fdst, length=16 * 1024 * 16):
    """
    Helper function to copy file

    Args:
        fsrc: source file opened in python (file-like object)
        fdst: destination file accessed in python (file-like object)
        length: copied buffer size in bytes (integer)
    """
    global curateprogress
    global total_dataset_size
    global curated_dataset_size
    while True:
        buf = fsrc.read(length)
        if not buf:
            break
        gevent.sleep(0)
        fdst.write(buf)
        curated_dataset_size += len(buf)

def return_new_path(topath):
    """
    This function checks if a folder already exists and in such cases,
    appends (2) or (3) etc. to the folder name

    Args:
        topath: path where the folder is supposed to be created (string)
    Returns:
        topath: new folder name based on the availability in destination folder (string)
    """
    if exists(topath):
        i = 2
        while True:
            if not exists(topath + " (" + str(i) + ")"):
                return topath + " (" + str(i) + ")"
            i += 1
    else:
        return topath

def create_dataset(recursivePath, jsonStructure, listallfiles):
    """
    Associated with 'Create new dataset locally' option of SODA interface
    for creating requested folders and files to the destination path specified

    Args:
        recursivePath: destination path for creating new folders and files (recursively) (string)
        jsonStructure: all paths (dictionary)
    Action:
        Creates the folders and files specified
    """

    global curateprogress

    for key in jsonStructure:

        if isinstance(jsonStructure[key], dict):

            if not exists(join(recursivePath, key)):
                mkdir(join(recursivePath, key))

            outputpath = join(recursivePath, key)

            create_dataset(outputpath, jsonStructure[key], listallfiles)

        elif isinstance(jsonStructure[key], list):

            outputpath = join(recursivePath, key)

            createFiles(jsonStructure, key, outputpath, listallfiles)

            for fileinfo in listallfiles:
                srcfile = fileinfo[0]
                distfile = fileinfo[1]
                curateprogress = "Copying " + str(srcfile)

                mycopyfile_with_metadata(srcfile, distfile)

def create_soda_json_object_backend(
    soda_json_structure, root_folder_path, irregularFolders, replaced
):
    """
    This function is meant for importing local datasets into SODA.
    It creates a json object with the structure of the dataset.
    """
    global create_soda_json_progress  # amount of items counted during recursion
    global create_soda_json_total_items  # counts the total items in folder
    global create_soda_json_completed  # completed progress is either 0 or 1
    global METADATA_FILES_SPARC

    high_level_sparc_folders = [
        "code",
        "derivative",
        "docs",
        "primary",
        "protocol",
        "source",
    ]

    dataset_folder = soda_json_structure["dataset-structure"] = {"folders": {}}

    def recursive_structure_create(dataset_structure, folder_path, high_lvl_folder_name):
        global create_soda_json_progress
        # going within high level folders
        # add manifest details if manifest exists
        manifest_object = {
            "filename": "",
            "timestamp": "",
            "description": "",
            "additional-metadata": "",
        }

        lastSlash = folder_path.rfind("/") + 1
        folder_name = folder_path[lastSlash:]

        if folder_name in replaced.keys():
            folder_name = replaced[folder_name]

        # Check if folder is in irregular folders
        if folder_path in irregularFolders:
            index_check = irregularFolders.index(folder_path)
            modified_name = replaced[os.path.basename(folder_path)]
            folder_path = irregularFolders[index_check]
        entries = os.listdir(folder_path)
        for entry in entries:
            gevent.sleep(0)
            item_path = os.path.normpath(os.path.join(folder_path, entry))

            if os.path.isfile(item_path) is True:
                # check manifest to add metadata
                if entry[0:1] != ".":
                    create_soda_json_progress += 1
                if entry[0:1] != "." and entry[0:8] != "manifest":
                    # no hidden files or manifest files included
                    if high_lvl_folder_name in soda_json_structure["starting-point"]:
                        if (
                            soda_json_structure["starting-point"][high_lvl_folder_name]["path"]
                            != ""
                        ):
                            # checks if there is a path to a manifest
                            manifest_path = soda_json_structure["starting-point"][
                                high_lvl_folder_name
                            ]["path"]
                            ext_index = manifest_path.rfind(".")
                            extension = manifest_path[ext_index:]
                            if extension == ".xlsx":
                                for key in soda_json_structure["starting-point"][
                                    high_lvl_folder_name
                                ]["manifest"]:
                                    extra_columns = False
                                    if(len(key) > 5):
                                        extra_columns = True
                                        extra_columns_dict = dict(itertools.islice(key.items(), 5, len(key)))
                                    # description metadata
                                    if key["filename"] in item_path:
                                        if key["description"] != "":
                                            manifest_object["description"] = key[
                                                "description"
                                            ]
                                        else:
                                            manifest_object["description"] = ""
                                    # additional metadata
                                        if key["Additional Metadata"] != "":
                                            manifest_object["additional-metadata"] = key[
                                                "Additional Metadata"
                                            ]
                                        else:
                                            manifest_object["additional-metadata"] = ""

                                        if extra_columns:
                                            manifest_object["extra_columns"] = extra_columns_dict

                            elif extension == ".csv":
                                for key in soda_json_structure["starting-point"][
                                    folder_name
                                ]["manifest"]:
                                    extra_columns = False
                                    if len(key) > 5:
                                        extra_columns = True
                                        extra_columns_dict = dict(itertools.islice(key.items(), 5, len(key)))

                                    if (
                                        soda_json_structure["starting-point"][
                                            folder_name
                                        ]["manifest"][key]["filename"]
                                        == entry
                                    ):
                                        if (
                                            soda_json_structure["starting-point"][
                                                folder_name
                                            ][key]["description"]
                                            != None
                                        ):
                                            manifest_object[
                                                "description"
                                            ] = soda_json_structure["starting-point"][
                                                folder_name
                                            ][
                                                key
                                            ][
                                                "description"
                                            ]
                                        else:
                                            manifest_object["description"] = ""
                                    if (
                                        soda_json_structure["starting-point"][
                                            folder_name
                                        ]["manifest"][key]["Additional Metadata"]
                                        != None
                                    ):
                                        manifest_object[
                                            "additional-metadata"
                                        ] = soda_json_structure["starting-point"][
                                            folder_name
                                        ][
                                            "manifest"
                                        ][
                                            key
                                        ][
                                            "Additional Metadata"
                                        ]
                                    else:
                                        manifest_object["additional-metadata"] = ""

                                    if extra_columns:
                                        manifest_object["extra_columns"] = extra_columns_dict
                    # create json
                    if "extra_columns" in manifest_object:
                        dataset_structure["files"][entry] = {
                            "path": item_path,
                            "type": "local",
                            "action": ["existing"],
                            "description": manifest_object["description"],
                            "additional-metadata": manifest_object["additional-metadata"],
                            "extra_columns": manifest_object["extra_columns"]
                        }
                    else:
                        dataset_structure["files"][entry] = {
                            "path": item_path,
                            "type": "local",
                            "action": ["existing"],
                            "description": manifest_object["description"],
                            "additional-metadata": manifest_object["additional-metadata"],
                        }
            elif os.path.isdir(item_path) is True:
                create_soda_json_progress += 1
                if item_path in irregularFolders:
                    index_check = irregularFolders.index(item_path)
                    modified_name = replaced[os.path.basename(item_path)]

                    dataset_structure["folders"][modified_name] = {
                        "folders": {},
                        "files": {},
                        "path": item_path,
                        "type": "local",
                        "action": ["existing"],
                        "original-name": entry,
                    }
                    for folder in dataset_structure["folders"][modified_name][
                        "folders"
                    ]:
                        updated_path = dataset_structure["folders"][modified_name][
                            folder
                        ]["path"]
                        recursive_structure_create(
                            dataset_structure["folders"][modified_name][folder],
                            updated_path,
                            high_lvl_folder_name
                        )
                else:
                    dataset_structure["folders"][entry] = {
                        "folders": {},
                        "files": {},
                        "path": item_path,
                        "type": "local",
                        "action": ["existing"],
                    }

        for folder in dataset_structure["folders"]:
            updated_path = dataset_structure["folders"][folder]["path"]
            recursive_structure_create(
                dataset_structure["folders"][folder], updated_path, high_lvl_folder_name
            )

    # BEGIN

    # count the amount of items in folder
    create_soda_json_total_items = 0
    for root, dirs, filenames in os.walk(root_folder_path):
        # walk through all folders and it's subfolders
        for Dir in dirs:
            # does not take hidden folders or manifest folders
            if Dir[0:1] != "." and Dir[0:8] != "manifest":
                create_soda_json_total_items += 1
        for fileName in filenames:
            if root == root_folder_path and fileName in METADATA_FILES_SPARC:
                # goes through all files and does not count hidden files
                create_soda_json_total_items += 1
            else:
                if fileName[0:1] != ".":
                    create_soda_json_total_items += 1

    # reading high level folders
    create_soda_json_completed = 0
    create_soda_json_progress = 0
    entries = os.listdir(root_folder_path)

    for entry in entries:
        # begin going through high level folders
        gevent.sleep(0)
        item_path = root_folder_path + "/" + entry
        # high level folder paths
        if os.path.isfile(item_path) is True:
            if entry[0:1] != "." and entry in METADATA_FILES_SPARC:
                # is not a hidden folder
                create_soda_json_progress += 1
                soda_json_structure["metadata-files"][entry] = {
                    "path": item_path,
                    "type": "local",
                    "action": ["existing"],
                }
            # do file work here
        elif os.path.isdir(item_path) is True:
            create_soda_json_progress += 1
            # add item to soda
            if item_path in irregularFolders:
                index_check = irregularFolders.index(item_path)
                modified_name = replaced[index_check]
                folder_name = modified_name
                dataset_folder["folders"][folder_name] = {
                    "folders": {},
                    "files": {},
                    "path": item_path,
                    "type": "local",
                    "action": ["existing"],
                    "original-basename": item_path[(item_path.rfind("/") + 1) :],
                }
            else:
                if entry in high_level_sparc_folders:
                    dataset_folder["folders"][entry] = {
                        "folders": {},
                        "files": {},
                        "path": item_path,
                        "type": "local",
                        "action": ["existing"],
                    }
            soda_json_structure["starting-point"][entry] = {"path": ""}

    for folder in dataset_folder["folders"]:
        # go through high level folders again
        high_lvl_path = root_folder_path + "/" + folder
        temp_csv = high_lvl_path + "/manifest.csv"
        temp_xlsx = high_lvl_path + "/manifest.xlsx"
        if os.path.exists(temp_csv) == True:
            temp_file_path_csv = root_folder_path + "/" + folder + "/" + "manifest.csv"
            csv_data = pd.read_csv(temp_file_path_csv)
            csv_data.fillna("", inplace=True)
            json_format = csv_data.to_dict(orient="records")
            soda_json_structure["starting-point"][folder]["path"] = temp_file_path_csv
            soda_json_structure["starting-point"][folder]["manifest"] = json_format
        if os.path.exists(temp_xlsx) == True:
            temp_file_path_xlsx = (
                root_folder_path + "/" + folder + "/" + "manifest.xlsx"
            )
            excel_data = pd.read_excel(temp_file_path_xlsx, sheet_name="Sheet1")
            excel_data.fillna("", inplace=True)
            json_format = excel_data.to_dict(orient="records")
            soda_json_structure["starting-point"][folder]["path"] = temp_file_path_xlsx
            soda_json_structure["starting-point"][folder]["manifest"] = json_format
        recursive_structure_create(dataset_folder["folders"][folder], high_lvl_path, folder)

    create_soda_json_completed = 1
    return soda_json_structure

def monitor_local_json_progress():
    """
    Function for monitoring progress of json_object_creation
    Used for progress bar
    """
    global create_soda_json_completed
    global create_soda_json_total_items
    global create_soda_json_progress
    progress_percentage = (
        create_soda_json_progress / create_soda_json_total_items
    ) * 100

    return {
        "create_soda_json_progress": create_soda_json_progress,
        "create_soda_json_total_items": create_soda_json_total_items,
        "progress_percentage": progress_percentage,
        "create_soda_json_completed": create_soda_json_completed
    }

def import_pennsieve_dataset(soda_json_structure, requested_sparc_only=True):
    global namespace_logger
    high_level_sparc_folders = [
        "code",
        "derivative",
        "docs",
        "primary",
        "protocol",
        "source",
    ]
    manifest_sparc = ["manifest.xlsx", "manifest.csv"]
    high_level_metadata_sparc = [
        "submission.xlsx",
        "submission.csv",
        "submission.json",
        "dataset_description.xlsx",
        "dataset_description.csv",
        "dataset_description.json",
        "subjects.xlsx",
        "subjects.csv",
        "subjects.json",
        "samples.xlsx",
        "samples.csv",
        "samples.json",
        "README.txt",
        "CHANGES.txt",
        "code_description.xlsx",
        "inputs_metadata.xlsx",
        "outputs_metadata.xlsx",
    ]
    double_extensions = [
        ".ome.tiff",
        ".ome.tif",
        ".ome.tf2,",
        ".ome.tf8",
        ".ome.btf",
        ".ome.xml",
        ".brukertiff.gz",
        ".mefd.gz",
        ".moberg.gz",
        ".nii.gz",
        ".mgh.gz",
        ".tar.gz",
        ".bcl.gz",
    ]

    global create_soda_json_completed
    global create_soda_json_total_items
    global create_soda_json_progress
    create_soda_json_progress = 0
    create_soda_json_total_items = 0
    create_soda_json_completed = 0

    # ["extensions"] doesn't seem to be returned by the Pennsieve API anymore
    def verify_file_name(file_name, extension):
        global namespace_logger
        if extension == "":
            return file_name

        double_ext = False
        for ext in double_extensions:
            if file_name.find(ext) != -1:
                double_ext = True
                break

        extension_from_name = ""

        if double_ext == False:
            extension_from_name = os.path.splitext(file_name)[1]
        else:
            extension_from_name = (
                os.path.splitext(os.path.splitext(file_name)[0])[1]
                + os.path.splitext(file_name)[1]
            )

        if extension_from_name == ("." + extension):
            return file_name
        else:
            return file_name + ("." + extension)

    def createFolderStructure(subfolder_json, pennsieve_client_or_token, manifest):
        """
            Function for creating the Pennsieve folder structure for a given dataset as an object stored locally.
            Arguments:
                subfolder_json: The json object containing the folder structure of the dataset
                pennsieve_client: The Pennsieve client object
                manifest: The manifest object for the dataset
        """
        # root level folder will pass subfolders into this function and will recursively check if there are subfolders while creating the json structure
        global namespace_logger
        global create_soda_json_progress

        collection_id = subfolder_json["path"]

        headers = create_request_headers(pennsieve_client_or_token)

        r = requests.get(f"{PENNSIEVE_URL}/packages/{collection_id}", headers=headers)
        r.raise_for_status()
        subfolder = r.json()["children"]

        for items in subfolder:
            folder_item_name = items["content"]["name"]
            create_soda_json_progress += 1
            item_id = items["content"]["id"]
            # is a file name check if there are additional manifest information to attach to files
            if item_id[2:9] == "package":
                if (
                    folder_item_name[0:8] != "manifest"
                ):  # manifest files are not being included in json structure

                    # verify file name first (used for legacy Pennsieve datasets)
                    if("extension" not in subfolder):
                        folder_item_name = verify_file_name(folder_item_name, "")
                    else:
                        folder_item_name = verify_file_name(folder_item_name, subfolder["extension"])

                    # verify timestamps
                    timestamp = items["content"]["createdAt"].replace('.', ',')
                    paths_list = [*subfolder_json["bfpath"]]
                    subfolder_json["files"][folder_item_name] = {
                        "action": ["existing"],
                        "path": item_id,
                        "bfpath": paths_list,
                        "timestamp": timestamp,
                        "type": "bf",
                        "additional-metadata": "",
                        "description": "",
                    }

                    # creates path for folder_item_name (stored in temp_name)
                    if len(subfolder_json["files"][folder_item_name]["bfpath"]) > 1:
                        temp_name = '/'.join(subfolder_json["files"][folder_item_name]["bfpath"][1:]) + "/" + folder_item_name
                    else:
                        temp_name = folder_item_name

                    if len(manifest.keys()) > 0:
                        # Dictionary that has the required manifest headers in lowercase and without spaces as keys
                        # and the correct manifest headers as values
                        defaultManifestHeadersNameMapped = {
                            "filename": "filename",
                            "timestamp": "timestamp",
                            "description": "description",
                            "filetype": "file type",
                            "additionalmetadata": "additional-metadata",
                        }

                        # Dictionary that will be used to store the correct manifest headers as keys
                        # and the values from the manifest as values
                        updated_manifest = {}

                        # Go through the imported manifest keys and change the keys to the correct name
                        # For example if the key is "File Name" change it to "filename"
                        for manifestKey in manifest.keys():
                            # Make the key lowercase
                            sterilizedKeyName = manifestKey.lower().replace(" ", "")
                            if sterilizedKeyName in defaultManifestHeadersNameMapped.keys():
                                # change the key to the correct name
                                # For example if the key name is "filetype" change it to "file type"
                                newManifestKeyName = defaultManifestHeadersNameMapped[sterilizedKeyName]
                                # Add the new key/value to the updated manifest
                                updated_manifest[newManifestKeyName] = manifest[manifestKey]
                            else:
                                # Keep the key/value the same and add it to the updated manifest
                                updated_manifest[manifestKey] = manifest[manifestKey]

                        if "filename" in updated_manifest.keys():
                            for manifestKey in updated_manifest.keys():
                                location_index = ""
                                # get the index of the file name in the manifest
                                if (temp_name in updated_manifest["filename"].values()):
                                    location_index = list(updated_manifest["filename"].values()).index(
                                        temp_name
                                    )
                                # This is for the case where the file name in the manifest has a slash at the beginning
                                # which is the case for files in the root folder
                                elif ("/" + temp_name in updated_manifest["filename"].values()):
                                    location_index = list(updated_manifest["filename"].values()).index(
                                        "/" + temp_name
                                    )
                                else:
                                    # break out of the for loop if the file name is not in the manifest
                                    break

                                # check if the key is in the required manifest headers, if it is, update the folder_item_name value
                                # corresponding to the key
                                if manifestKey in defaultManifestHeadersNameMapped.values():
                                    if updated_manifest[manifestKey][location_index] != "":
                                        if folder_item_name[0:1] == "/":
                                            subfolder_json["files"][folder_item_name[:1]][manifestKey] = updated_manifest[manifestKey][location_index]
                                        else:
                                            subfolder_json["files"][folder_item_name][manifestKey] = updated_manifest[manifestKey][location_index]
                                # if the key is not in the required manifest headers, add it to the extra columns folder_item_name value
                                else :
                                    # if the extra columns key does not exist, create it
                                    if "extra_columns" not in subfolder_json["files"][folder_item_name]:
                                        subfolder_json["files"][folder_item_name]["extra_columns"] = {}

                                    if updated_manifest[manifestKey][location_index] != "":
                                        subfolder_json["files"][folder_item_name]["extra_columns"][manifestKey] = updated_manifest[manifestKey][location_index]
                                    else:
                                        subfolder_json["files"][folder_item_name]["extra_columns"][manifestKey] = ""
                        else:
                            # filename not in updated manifest so recreate standard headers if they don't exist
                            # loop through the updated manifest keys and if header matches standard header add content else recreate
                            if len(updated_manifest.keys()) > 0:
                                location_index = ""
                                for manifestKey in updated_manifest.keys():
                                    if temp_name in updated_manifest[manifestKey].values():
                                        # file_names found
                                        location_index = list(updated_manifest[manifestKey].values()).index(
                                        temp_name
                                        )
                                    if ("/" + temp_name in updated_manifest[manifestKey].values()):
                                        location_index = list(updated_manifest[manifestKey].values()).index(
                                        "/" + temp_name
                                        )
                                    if location_index != "":
                                        if manifestKey in defaultManifestHeadersNameMapped.values():
                                            if folder_item_name[0:1] == "/":
                                                subfolder_json["files"][folder_item_name[1:]][manifestKey] = updated_manifest[manifestKey][location_index]
                                            else:
                                                subfolder_json["files"][folder_item_name][manifestKey] = updated_manifest[manifestKey][location_index]
                                        else:
                                            if "extra_columns" not in subfolder_json["files"][folder_item_name]:
                                                subfolder_json["files"][folder_item_name]["extra_columns"] = {}
                                            subfolder_json["files"][folder_item_name]["extra_columns"][manifestKey] = updated_manifest[manifestKey][location_index]

            else:  # another subfolder found
                paths_list = [*subfolder_json["bfpath"], folder_item_name]
                subfolder_json["folders"][folder_item_name] = {
                    "action": ["existing"],
                    "path": item_id,
                    "bfpath": paths_list,
                    "files": {},
                    "folders": {},
                    "type": "bf",
                }

        if len(subfolder_json["folders"].keys()) != 0:  # there are subfolders
            for folder in subfolder_json["folders"].keys():
                subfolder = subfolder_json["folders"][folder]
                createFolderStructure(subfolder, pennsieve_client_or_token, manifest)

    # START
    start = timer()
    token = get_access_token()

    # check that the Pennsieve dataset is valid
    try:
        bf_dataset_name = soda_json_structure["bf-dataset-selected"]["dataset-name"]
    except Exception as e:
        raise e

    selected_dataset_id = get_dataset_id(token, bf_dataset_name)

    # check that the user has permission to edit this dataset
    try:
        role = pennsieve_get_current_user_permissions(selected_dataset_id, token)["role"]
        if role not in ["owner", "manager", "editor"]:
            curatestatus = "Done"
            raise Exception("You don't have permissions for uploading to this Pennsieve dataset")
    except Exception as e:
        abort(401, "You do not have permissions to edit upload this Pennsieve dataset.")

    # surface layer of dataset is pulled. then go through through the children to get information on subfolders
    manifest_dict = {}
    manifest_error_message = []
    soda_json_structure["dataset-structure"] = {
        "files": {},
        "folders": {},
    }

    # headers for making requests to Pennsieve's api
    headers = create_request_headers(token)

    # root of dataset is pulled here (high level folders/files are gathered here)
    # root_folder is the files and folders within root
    r = requests.get(f"{PENNSIEVE_URL}/datasets/{selected_dataset_id}", headers=headers)
    r.raise_for_status()
    root_folder = r.json()["children"]

    # Get the amount of files/folders in the dataset
    r = requests.get(f"{PENNSIEVE_URL}/datasets/{selected_dataset_id}/packageTypeCounts", headers=headers)
    r.raise_for_status()
    packages_list = r.json()

    # root's children files
    for count in packages_list.values():
        create_soda_json_total_items += int(count)

    # Go through the content in the root folder
    for items in root_folder:
        item_id = items["content"]["id"]
        item_name = items["content"]["name"]
        # If package type is Collection, then it is a folder
        if items["content"]["packageType"] == "Collection" and item_name in high_level_sparc_folders:
            create_soda_json_progress += 1
            soda_json_structure["dataset-structure"]["folders"][item_name] = {
                "type": "bf",
                "path": item_id,
                "action": ["existing"],
                "files": {},
                "folders": {},
                "bfpath": [item_name],
            }

            manifest_dict[item_name] = {}
            # Check the content of the folder to see if a manifest file exists
            r = requests.get(f"{PENNSIEVE_URL}/packages/{item_id}", headers=headers)
            r.raise_for_status()
            folder_content = r.json()["children"]

            if len(folder_content) > 0:
                for package in folder_content:
                    package_name = package["content"]["name"]
                    if package_name in manifest_sparc:
                        # item is manifest
                        df = ""
                        package_id = package["content"]["id"]
                        try:
                            if package_name.lower() == "manifest.xlsx":
                                df = load_metadata_to_dataframe(package_id, "excel", token)
                                df = df.fillna("")
                            else:
                                df = load_metadata_to_dataframe(package_id, "csv", token)
                                df = df.fillna("")
                            manifest_dict[item_name].update(df.to_dict())
                        except Exception as e:
                            manifest_error_message.append(
                                items["content"]["name"]
                            )

                high_lvl_folder_dict = soda_json_structure["dataset-structure"]["folders"][
                    item_name
                ]

                if item_name in manifest_dict:
                    createFolderStructure(
                        high_lvl_folder_dict, token, manifest_dict[item_name]
                    )  # passing item's json and the collection ID

        else:
            # Item is a metadata file
            if item_name in high_level_metadata_sparc:
                create_soda_json_progress += 1
                if "metadata-files" in soda_json_structure.keys():
                    soda_json_structure["metadata-files"][item_name] = {
                        "type": "bf",
                        "action": ["existing"],
                        "path": item_id,
                    }

    success_message = (
        "Data files under a valid high-level SPARC folders have been imported"
    )
    create_soda_json_completed = 1

    end = timer()
    namespace_logger.info(f"Time to import {soda_json_structure['bf-dataset-selected']['dataset-name']} dataset: {timedelta(seconds=end - start)}")
    return {
        "soda_object": soda_json_structure,
        "success_message": success_message,
        "manifest_error_message": manifest_error_message,
        "import_progress": create_soda_json_progress,
        "import_total_items": create_soda_json_total_items,
    }

def monitor_pennsieve_json_progress():
    """
    Function for monitoring progress of soda_json object
    Used for progress bar
    """
    global create_soda_json_completed
    global create_soda_json_total_items
    global create_soda_json_progress

    if create_soda_json_progress != 0:
        progress_percentage = (
            create_soda_json_progress / create_soda_json_total_items
        ) * 100
    else:
        progress_percentage = 0
    return {
        "import_progress": create_soda_json_progress,
        "import_total_items": create_soda_json_total_items,
        "import_progress_percentage": progress_percentage,
        "import_completed_items": create_soda_json_completed,
    }


import numpy as np
import scipy.optimize
import uncertainties as unc
import uncertainties.unumpy as unp
from scipy import optimize
import scipy.constants as const
kboltzmann = const.k
import matplotlib.pyplot as plt
from uncertainties import ufloat
from uncertainties import correlated_values, correlation_matrix
from uncertainties.unumpy import (nominal_values as noms, std_devs as stds)
from matrix2latex import matrix2latex

k_B=1.3806485e-23
e=1.602e-19

#################################
### Our statistical functions ###
#################################

def mittel(x, deltadof=1):
    return ufloat(np.mean(x),np.std(x,ddof=deltadof)/np.sqrt(len(x)))

########################################
### Our helper functions (fits, etc) ###
########################################

def linfit(x,a,b):
    return a*x+b

def hyp(x,a,b):
    return a/x+b

def lin(x,a,offset=0):
    return a*x + offset

def exp(x,a,b,c):
    return a*np.exp(x*b)+c

def lnint(T,current,heatingRate):
	array = np.array([])
	for t in T:
		array = np.append(array,[np.trapz(current[T>=t],T[T>=t])/(current[T == t]*heatingRate)])
	return array

########################
### Reading the data ###
########################

temp1_2, current1_2 = np.genfromtxt('data/1_2.txt', unpack=True)
temp2, current2 = np.genfromtxt('data/2.txt', unpack=True)
# Temp in Celsius, Current in pA (10^-12)
temps = [temp1_2, temp2] # Eine Liste der beiden Temperaturmessreihen (1_2 und 2)
currents = [current1_2, current2]

############################
### Untergrund bestimmen ###   Vielleicht sind hier die Werte für i einfach so klein dass er da nichts findet... ist mir erst zu spät eingefallen....
############################

expfitx1=np.append(temp1_2[5:19],temp1_2[55:72])
expfity1=np.append(current1_2[5:19], current1_2[55:72])

params1, covariance_matrix = optimize.curve_fit(exp, expfitx1, expfity1)
a,b,c= correlated_values(params1, covariance_matrix)
print('Fit für 1.2K/min:')
print('a=', a)
print('b=', b)
print('c=', c)

expfitx2=np.append(temp2[11:17],temp2[37:51])
expfity2=np.append(current2[11:17], current2[37:51])

params2, covariance_matrix = optimize.curve_fit(exp, expfitx2, expfity2)
a, b, c= correlated_values(params2, covariance_matrix)
print('Fit für 2K/min:')
print('a=', a)
print('b=', b)
print('c=', c)

expfitx=[expfitx1, expfitx2]
expfity=[expfity1, expfity2]
params=[params1, params2]
############################
### Plot the actual data ###
############################

labels = ['Niedigere Heizrate', 'Höhere Heizrate']
names = ['build/depolarisationskurve_1_2.pdf', 'build/depolarisationskurve_2.pdf']
x=np.linspace(-100,100,1000)
xlims = [-70,65]
ylims = [-12.5,17.5]

for i in (0,1):
    plt.plot(temps[i], currents[i], 'bx', alpha=0.75, mew=0.5, label = labels[i])
    plt.plot(expfitx[i], expfity[i], 'rx', label = 'Werte für den Fit')
    plt.plot(x, exp(x, *params[i]),'g-', label='Ausgleichsrechnung', linewidth=0.5)
    plt.plot(temps[i], currents[i]-exp(temps[i],*params[i]), 'gx', mew=0.5,  label='Messwerte mit Untergrundkorrektur')
    plt.ylim(ylims)
    plt.xlim(xlims)
    plt.grid()
    plt.xlabel(r'$T/$°C')
    plt.ylabel(r'$I/$pA')
    plt.legend(loc='best')
    plt.savefig(names[i])
    plt.clf()


from sqlalchemy.sql.expression import bindparam
from datetime import datetime as dt
import sqlalchemy
import sys

def create_db(engine):
    metadata = sqlalchemy.MetaData(engine)
    # Create tables with the appropriate Columns
    calibration = sqlalchemy.Table('calibration',
                                   metadata,
                                   sqlalchemy.Column('calibration_pkey',
                                                     sqlalchemy.Integer,
                                                     primary_key=True,
                                                     nullable=False),
                                   sqlalchemy.Column('xml_file', sqlalchemy.String),
                                   sqlalchemy.Column('modified_filename', sqlalchemy.String),
                                   sqlalchemy.Column('timestamp', sqlalchemy.DateTime),
                                   sqlalchemy.Column('compound', sqlalchemy.String),
                                   sqlalchemy.Column('curve_type', sqlalchemy.String),
                                   sqlalchemy.Column('origin', sqlalchemy.String, nullable=True),
                                   sqlalchemy.Column('weighting', sqlalchemy.String, nullable=True),
                                   sqlalchemy.Column('slope', sqlalchemy.String, nullable=True),
                                   sqlalchemy.Column('rsquared', sqlalchemy.Float, nullable=True),
                                   sqlalchemy.Column('cc', sqlalchemy.Float, nullable=True),
                                   sqlalchemy.Column('int_std_id', sqlalchemy.Integer),
                                   sqlalchemy.Column('int_std', sqlalchemy.String))

    results = sqlalchemy.Table('results', metadata,
                               sqlalchemy.Column('results_pkey',
                                                 sqlalchemy.Integer,
                                                 primary_key=True,
                                                 nullable=False),
                               sqlalchemy.Column('xml_file', sqlalchemy.String),
                               sqlalchemy.Column('modified_filename', sqlalchemy.String),
                               sqlalchemy.Column('lc_batch', sqlalchemy.String),
                               sqlalchemy.Column('injection_time', sqlalchemy.DateTime),
                               sqlalchemy.Column('vial', sqlalchemy.String),
                               sqlalchemy.Column('sample', sqlalchemy.String),
                               sqlalchemy.Column('sample_id', sqlalchemy.Integer),
                               sqlalchemy.Column('sample_type', sqlalchemy.String),
                               sqlalchemy.Column('compound', sqlalchemy.String),
                               sqlalchemy.Column('compound_id', sqlalchemy.Integer),
                               sqlalchemy.Column('peak_area', sqlalchemy.Float, nullable=True),
                               sqlalchemy.Column('concentration', sqlalchemy.Float, nullable=True),
                               sqlalchemy.Column('RT', sqlalchemy.Float, nullable=True),
                               sqlalchemy.Column('SN', sqlalchemy.Float, nullable=True),
                               sqlalchemy.Column('confirming_ion_area', sqlalchemy.Float, nullable=True),
                               sqlalchemy.Column('int_std_id', sqlalchemy.Integer),
                               sqlalchemy.Column('int_std', sqlalchemy.String),
                               sqlalchemy.Column('is_peak_area', sqlalchemy.Float, nullable=True),
                               sqlalchemy.Column('is_RT', sqlalchemy.Float, nullable=True),
                               sqlalchemy.Column('is_SN', sqlalchemy.Float, nullable=True),
                               sqlalchemy.Column('is_confirming_ion_area', sqlalchemy.Float, nullable=True))

    batch = sqlalchemy.Table('batch', metadata,
                             sqlalchemy.Column('batch_pkey',
                                               sqlalchemy.Integer,
                                               primary_key=True,
                                               nullable=False),
                             sqlalchemy.Column('xml_file', sqlalchemy.String),
                             sqlalchemy.Column('modified_filename', sqlalchemy.String),
                             sqlalchemy.Column('timestamp', sqlalchemy.DateTime),
                             sqlalchemy.Column('instrument', sqlalchemy.String),
                             sqlalchemy.Column('numb_samples', sqlalchemy.Integer))
    # Implement the creation
    metadata.create_all(engine)
    return [calibration, batch, results]

def execute_calibration(db_table, value_dictionary):
    cstmt = db_table.update().where(db_table.c.calibration_pkey == bindparam("_id")).values(value_dictionary)
    return cstmt

def execute_result(db_table, value_dictionary):
    rstmt = db_table.update().where(db_table.c.results_pkey == bindparam("_id")).values(value_dictionary)
    return rstmt

def get_table_schema(table):
    if str(table.name) == 'batch':
        values = {
            'xml_file': bindparam('xml_file'),
            'modified_filename': bindparam('modified_filename'),
            'timestamp': bindparam('timestamp'),
            'instrument': bindparam('instrument'),
            'numb_samples': bindparam('numb_samples')
        }
        stmt = table.update().where(table.c.batch_pkey == bindparam("_id")).values(values)
        return stmt
    if str(table.name) == 'calibration':
        values = {
            'xml_file': bindparam('xml_file'),
            'modified_filename': bindparam('modified_filename'),
            'timestamp': bindparam('timestamp'),
            'compound': bindparam('compound'),
            'curve_type': bindparam('curve_type'),
            'origin': bindparam('origin'),
            'weighting': bindparam('weighting'),
            'slope': bindparam('slope'),
            'rsquared': bindparam('rsquared'),
            'cc': bindparam('cc'),
            'int_std_id': bindparam('int_std_id'),
            'int_std': bindparam('int_std')
        }
        stmt = execute_calibration(table, values)
        return stmt
    if str(table.name) == 'results':
        values = {
            'xml_file': bindparam('xml_file'),
            'modified_filename': bindparam('modified_filename'),
            'lc_batch': bindparam('lc_batch'),
            'injection_time': bindparam('injection_time'),
            'vial': bindparam('vial'),
            'sample': bindparam('sample'),
            'sample_id': bindparam('sample_id'),
            'sample_type': bindparam('sample_type'),
            'compound': bindparam('compound'),
            'compound_id': bindparam('compound_id'),
            'peak_area': bindparam('peak_area'),
            'concentration': bindparam('concentration'),
            'RT': bindparam('RT'),
            'SN': bindparam('SN'),
            'confirming_ion_area': bindparam('confirming_ion_area'),
            'int_std_id': bindparam('int_std_id'),
            'int_std': bindparam('int_std'),
            'is_peak_area': bindparam('is_peak_area'),
            'is_RT': bindparam('is_RT'),
            'is_SN': bindparam('is_SN'),
            'is_confirming_ion_area': bindparam('is_confirming_ion_area'),
        }
        stmt = execute_result(table, values)
        return stmt

def execute_table_changes(engine_, db_table, replacement_data):
    statement = get_table_schema(db_table)
    for item in replacement_data:
        engine_.execute(statement, [item])

def execute_batch(engine_, db_table, replacement_data):
    bstmt = get_table_schema(db_table)
    engine_.execute(bstmt, [replacement_data])

def grab_entries_from_db(engine, file, table):
    s = sqlalchemy.sql.select([table]).where(table.c.xml_file == file)
    result = engine.execute(s)
    res = result.fetchall()
    return res

def check_matching_file_entry_length(engine, tables, xml_file, data_to_replace):
    calibrator_length, sample_number = len(data_to_replace[0]), data_to_replace[1][4]
    cal_table, batch_table, results_table = tables
    cal_table_data = grab_entries_from_db(engine, xml_file, cal_table)
    batch_table_data = grab_entries_from_db(engine, xml_file, batch_table)
    batch_length = batch_table_data[0][5]
    results_table_data = grab_entries_from_db(engine, xml_file, results_table)
    if (len(cal_table_data) == calibrator_length) and (batch_length == sample_number):
        return [cal_table_data, batch_table_data, results_table_data]
    else:
        return [False, False, False]

def check_if_in_db(engine, xml_file, calibration):
    s = sqlalchemy.sql.select([calibration.c.xml_file]).where(calibration.c.xml_file == xml_file)
    result = engine.execute(s)
    res = result.fetchone()
    if res:
        return True
    else:
        return False

def add_to_calibration_table(df, engine):
    result, = engine.execute('SELECT max(calibration_pkey) FROM calibration')
    result = result[0]
    if result is None:
        result = -1
    result = result + 1
    df.index += result
    df.to_sql('calibration', engine, if_exists="append", index_label='calibration_pkey')

def add_to_result_table(df, engine):
    result, = engine.execute('SELECT max(results_pkey) FROM results')
    result = result[0]
    if result is None:
        result = -1
    result = result + 1
    df.index += result
    df.to_sql('results', engine, if_exists='append', index_label='results_pkey')

def add_to_batch_table(batch_data, engine, batch):
    max_id, = engine.execute('SELECT max(batch_pkey) FROM batch')
    max_id = max_id[0]
    if max_id is None:
        max_id = -1
    new_id = max_id + 1
    ins = batch.insert().values(
                                batch_pkey=new_id,
                                xml_file=batch_data[0],
                                modified_filename=batch_data[1],
                                timestamp=dt.strptime(batch_data[2], '%Y-%m-%d %H:%M:%S'),
                                instrument=batch_data[3],
                                numb_samples=batch_data[4])
    engine.execute(ins)

def make_engine(db_type):
    if db_type == 'postgres':
        # change to fit actual path
        engine = sqlalchemy.create_engine('postgresql://username:password@localhost/opiates')
    elif db_type == 'sqlite':
        # change to fit actual path
        engine = sqlalchemy.create_engine('sqlite:////Path/to/file/opiates.db')
    else:
        print('No recognizied database type selected. Will terminate process.')
        sys.exit()
    table_list = create_db(engine)
    return engine, table_list


import itertools
import warnings
import statsmodels.api as sm
import matplotlib.pyplot as plt
import pandas as pd
from matplotlib import pyplot
from statsmodels.tsa.arima_model import ARIMA
from statsmodels.tsa.seasonal import seasonal_decompose
from statsmodels.tsa.stattools import adfuller
import seaborn as sns
from statsmodels.tsa.arima_model import ARIMAResults

# from statsmodels.tsa.arima_model import SARIMAResults

warnings.filterwarnings('ignore')

def adfuller_test(sales):
    """Method will test weather data is stationary or not"""
    rolmean = pd.Series(sales).rolling(window=12).mean()
    rolstd = pd.Series(sales).rolling(window=12).std()

    # plt.plot(sales, color='blue', label='Original')
    # plt.plot(rolmean, color='red', label='Rolling Mean')
    # plt.plot(rolstd, color='black', label='Rolling Std')
    # plt.legend(loc='best')
    # plt.title('Rolling Mean & Standard Deviation')
    # plt.xlabel("Date")
    # plt.ylabel('No Of Booking Requests')
    # plt.show()

    result = adfuller(sales)
    labels = ['ADF Test Statistic', 'p-value', '#Lags Used', 'Number of Observations Used']
    for value, label in zip(result, labels):
        print(label + ' : ' + str(value))
    if result[1] <= 0.05:
        print("strong evidence against the null hypothesis(Ho), reject the null hypothesis. "
              "Data has no unit root and is stationary")
    else:
        print("weak evidence against null hypothesis, time series has a unit root, indicating it is non-stationary ")

def train_arima_sarima(df):
    df.dropna(inplace=True)
    df["no_of_requests"] = 1
    print(df["no_of_requests"].describe())
    df.rename(columns={'index': 'filedate'}, inplace=True)
    df['filedate'] = pd.to_datetime(df['filedate'])
    df["no_of_requests"] = df["no_of_requests"].apply(pd.to_numeric, errors='coerce')
    df = df.resample('d', on='filedate').sum().dropna(how='all')
    print(df["no_of_requests"].describe())
    # sns.set_style('darkgrid')
    # df["no_of_requests"].plot(kind='line', legend='reverse', title='Visualizing average number of request plot')
    # plt.xlabel("Month'S")
    # plt.ylabel('No Of Booking Requests')
    # plt.legend(loc='upper right', shadow=True)
    # plt.show()

    # result = seasonal_decompose(df["no_of_requests"])
    # result.plot()
    # pyplot.show()

    adfuller_test(df["no_of_requests"])

    df['Booking First Difference'] = df['no_of_requests'] - df['no_of_requests'].shift(1)

    # print(df["Booking First Difference"].describe())
    adfuller_test(df["Booking First Difference"].dropna())

    # fig = plt.figure(figsize=(12, 8))
    # ax1 = fig.add_subplot(211)
    # fig = sm.graphics.tsa.plot_acf(df["Booking First Difference"].iloc[6:], lags=13, ax=ax1)
    # ax2 = fig.add_subplot(212)
    # fig = sm.graphics.tsa.plot_pacf(df["Booking First Difference"].iloc[6:], lags=13, ax=ax2)
    # plt.show()

    train = df["no_of_requests"].dropna()[:100]
    test = df["no_of_requests"].dropna()[100:]
    print(train.head())

    param_dict = {}
    p = d = q = range(0, 5)
    pdq = list(itertools.product(p, d, q))

    for param in pdq:
        try:
            model = ARIMA(train, order=param)
            results = model.fit(disp=0)
            print(param, results.aic)
            param_dict[abs(results.aic)] = param
        except:
            continue

    arima_param = param_dict.get(min(list(param_dict.keys())))
    print("Training on order", arima_param)
    model = ARIMA(train, order=arima_param)
    results = model.fit(disp=0)
    prediction = results.forecast(steps=36)[0]
    plt.plot(test.values)
    plt.plot(prediction, color='red')
    plt.xlabel("Day'S")
    plt.ylabel('No Of Booking Requests')
    plt.show()
    print(prediction)

    results.save('model_arima.pkl')
    # load model
    loaded = ARIMAResults.load('model_arima.pkl')
    future = loaded.forecast(steps=20)[0]
    print(future)

    # seasonal_pdq = [(x[0], x[1], x[2], 12) for x in list(itertools.product(p, d, q))]
    # print('Seasonal ARIMA result...')
    # for param in pdq:
    #     for param_seasonal in seasonal_pdq:
    #         try:
    #             mod = sm.tsa.statespace.SARIMAX(train,
    #                                             order=param,
    #                                             seasonal_order=param_seasonal,
    #                                             enforce_stationarity=True,
    #                                             enforce_invertibility=False)
    #             results = mod.fit(disp=0)
    #             print('ARIMA{}x{}12 - AIC:{}'.format(param, param_seasonal, results.aic))
    #         except:
    #             continue

    mod = sm.tsa.statespace.SARIMAX(train,
                                    order=arima_param,
                                    seasonal_order=(0, 1, 0, 52),
                                    enforce_stationarity=True,
                                    enforce_invertibility=False)
    results = mod.fit(disp=0)
    prediction = results.forecast(steps=36)
    print("Prediction", prediction)
    results.save('model_sarima.pkl')
    # load model
    loaded = ARIMAResults.load('model_sarima.pkl')
    # plt.plot(test.values)
    # plt.plot(prediction.values, color='red')
    # plt.xlabel("Day'S")
    # plt.ylabel('No Of Booking Requests')
    # plt.show()

# if __name__ == '__main__':
#
#     df = pd.read_csv('../Models/feature_file.csv')
#     predict_sas(df)


from bs4 import BeautifulSoup
import re
import csv

RPP = 30
NEED = 3100
N_PG = int(NEED/RPP) + 1

def read_soup(url, decode_type=None):
    import requests
    r = requests.get(url)
    encoding = r.encoding if 'charset' in r.headers.get('content-type', '').lower() else None
    soup = BeautifulSoup(r.content, 'html.parser', from_encoding=encoding)
    return soup

class Restaurant:
    def __init__(self, idx, name, single_page_url):
        self.id = idx
        self.name = name
        self.url = single_page_url
        self.rate = None
        self.reviews = None
        self.money = None
        self.type = list()
        self.address_street = None
        self.address_locality = None
        self.phone = None
        self.sun_hours = None
        self.mon_hours = None
        self.tues_hours = None
        self.wed_hours = None
        self.thur_hours = None
        self.fri_hours = None
        self.sat_hours = None
        self.crawl_from_url()

    def set_hours(self, day_text, hours):
        if day_text == "Sunday":
            self.sun_hours = hours
        elif day_text == "Monday":
            self.mon_hours = hours
        elif day_text == "Tuesday":
            self.tues_hours = hours
        elif day_text == "Wednesday":
            self.wed_hours = hours
        elif day_text == "Thursday":
            self.thur_hours = hours
        elif day_text == "Friday":
            self.fri_hours = hours
        elif day_text == "Saturday":
            self.sat_hours = hours

    def crawl_from_url(self):
        try:
            soup = read_soup(self.url)
            header_rating = soup.find(class_="header_rating")
            self.rate = header_rating.find("span", {"property": "ratingValue"})["content"]
            self.reviews = header_rating.find("span", {"property": "count"}).text.replace(",", "")
            header_money = soup.find(class_="header_tags rating_and_popularity")
            self.money = header_money.text if header_money else None
            header_types = soup.find(class_="header_links rating_and_popularity")
            self.type = [t.text for t in header_types.find_all("a")] if header_types else list()
            header_street = soup.find(class_="street-address")
            self.address_street = header_street.text if header_street else None
            header_locality = soup.find(class_="locality")
            self.address_locality = header_locality.text if header_locality else None
            header_phone = soup.find(class_="ui_icon phone").next_sibling
            self.phone = header_phone.text if header_phone else None
            hours = soup.find(class_="hours content")
            if hours:
                week = hours.find(class_="detail")
                while week:
                    self.set_hours(week.find(class_="day").text, week.find(class_="hoursRange").text)
                    week = week.next_sibling.next_sibling
        except Exception as e:
            print(e)
            print("{} {} \n{}".format(self.id, self.name, self.url))

    def writerow_to_csv(self, csvwriter):
        info = [self.id, self.name, self.rate, self.reviews, self.money]
        for i in range(3):
            try:
                info.append(self.type[i])
            except IndexError:
                info.append(None)
        info.extend([self.address_street, self.address_locality, self.phone])
        info.extend([self.sun_hours, self.mon_hours, self.tues_hours, self.wed_hours, self.thur_hours, self.fri_hours, self.sat_hours])
        csvwriter.writerow(info)

# crawl website information
idx = 1
restaurants = list()
print("Start crawling")
for page in range(N_PG):
    API_url = 'https://www.tripadvisor.com/RestaurantSearch?Action=PAGE&geo=32655&ajax=1&itags=10591&sortOrder=relevance&o=a{}&availSearchEnabled=false'\
            .format(page*30)
    soup = read_soup(API_url)
    reslist_regex = re.compile(".*listing.*")
    reslist = soup.find_all("div", class_=reslist_regex)
    for res in reslist:
        title = res.find(class_="property_title")
        name = title.text.strip()
        url = "https://www.tripadvisor.com" + title['href']
        r_object = Restaurant(idx, name, url)
        restaurants.append(r_object)
        idx += 1

# convert to csv
print("Start convert to csv")

with open('tripadvisor_la.csv', 'w', newline='') as csvfile:
    csvwriter = csv.writer(csvfile, delimiter=',',
                           quotechar='"', quoting=csv.QUOTE_MINIMAL)
    csvwriter.writerow(["id", "name", "rate", "n_reviews", "money", "type_1", "type_2", "type_3", "address_street", "address_locality", "phone", \
                        "sun_hours", "mon_hours", "tue_hours", "wed_hours", "thur_hours", "fri_hours", "sat_hours"])
    for robj in restaurants:
        robj.writerow_to_csv(csvwriter)


"""Strings encoder and decoder."""
from typing import List

class Codec:
    """Strings encoder and decoder."""

    SEPARATOR = ":"

    def encode(self, strs: List[str]) -> str:
        """Encodes a list of strings to a single string."""
        return "".join(f"%d{Codec.SEPARATOR}%s" % (len(s), s) for s in strs)

    def decode(self, s: str) -> List[str]:
        """Decodes a single string to a list of strings."""
        idx = 0
        strs: List[str] = []

        while idx < len(s):
            sep_idx = s.find(Codec.SEPARATOR, idx)
            s_len = int(s[idx:sep_idx])
            idx = sep_idx + 1 + s_len
            strs.append(s[sep_idx + 1 : idx])

        return strs

if __name__ == "__main__":
    codec = Codec()
    strs = ["Hello", "World", "This is cse", "from sust."]
    e_str = codec.encode(strs)
    print(f"Encoded: {e_str}, type={type(e_str)}")
    d_str = codec.decode(e_str)
    print(f"Decoded: {d_str}, type={type(d_str)}")
    assert d_str == strs


#Usage
# python train_test_split.py
import os
import config
import shutil
import random

class DataSpliter ():

  def buildDataset(self, imagesPath, dtype):
    try:
      path = os.path.join(config.BASE_PATH, dtype)

      if os.path.isdir(path):
        shutil.rmtree(path)

      if not os.path.isdir(path):
        os.mkdir(path)

      for imgPath in imagesPath:
        imagePath = imgPath
        # annotation file full path
        annotationName = imgPath.split(os.path.sep)[-1].replace('.jpg', '.xml')
        annotationPath = os.path.join(config.ANNOTATIONS_PATH, annotationName)
        # coopy images and annotations to corresponding file
        shutil.copy2(imagePath, path)
        shutil.copy2(annotationPath, path)
    except Exception as e:
      raise e

  def split(self):
    try:
      # garb the full path of all images
      imagesPath = [os.path.join(config.IMAGES_PATH, imageName) for imageName in os.listdir(config.IMAGES_PATH)]
      random.shuffle(imagesPath)
      # split the images list into test and train
      limit = int(len(imagesPath) * config.SPLIT)

      trainset = imagesPath[:limit]
      testset = imagesPath[limit:]
      #build test and train dataset
      self.buildDataset(trainset, 'train')
      self.buildDataset(testset, 'test')
    except Exception as e:
      raise e

if __name__ == '__main__':
  spliter = DataSpliter()
  spliter.split()


import requests
from datetime import timedelta
import datetime
import json
from time import sleep
import config
url = "https://app.najva.com/api/v1/notifications/"
#print(url)
currentDT = datetime.datetime.now()
now_plus_10s=currentDT+timedelta(seconds=+10)
now = now_plus_10s.strftime("%Y-%m-%dT%H:%M:%SZ")
#print(currentDT)
#print(now_plus_10s)
#time = datetime.datetime.now() + timedelta(minutes=10) # this time is an example. replace it with your own time.
#print(time)
#print("+++++++++++++")
#print(now_plus_5m.strftime("%Y/%m/%d - %H:%M:%S"))
symbol='BNBUSDT'
price =270
data={
  "api_key":config.apikey,
    "title":symbol,
    "body":f"{symbol} Triggered on {price}",
    "onclick_action":"open-link",
    "url":f"https://www.binance.com/en/trade/{symbol}?layout=pro",
    "content":f"{symbol} Price : {price}",
    # "content":"image should be ۳۶۰×۷۲۰ or 720x300",
    "icon":"https://cdn.jsdelivr.net/gh/atomiclabs/cryptocurrency-icons@9ab8d6934b83a4aa8ae5e8711609a70ca0ab1b2b/128/icon/bnb.png",
    # "image":"https://cdn.jsdelivr.net/gh/atomiclabs/cryptocurrency-icons@9ab8d6934b83a4aa8ae5e8711609a70ca0ab1b2b/128/icon/bnb.png",
    "sent_time": now ,
    "one_signal_enabled":"false"
}
#print(data)
payload = json.dumps(data)
headers = {
    'authorization': f"Token {config.token}",
    'content-type': "application/json",
    'cache-control': "no-cache",
    }

response = requests.request("POST", url, data=payload, headers=headers)
print(response.json())
#print(response.text)


import face_recognition
import cv2
from gtts import gTTS
from playsound import playsound

def sintesi_vocale2(testo):
    tts = gTTS(text=testo, lang='it')
    tts.save("riconoscimento_facciale/results.mp3")
    playsound("riconoscimento_facciale/results.mp3")

def face_match(image1, image2, image3):
    # lavoro sulla prima immagine
    first_image = face_recognition.load_image_file(image1)
    encode_image = face_recognition.face_encodings(first_image)[0]

    # lavoro sulla seconda immagine
    second_image = face_recognition.load_image_file(image2)
    encode_image2 = face_recognition.face_encodings(second_image)[0]

    third_image = face_recognition.load_image_file(image3)
    encode_image3 = face_recognition.face_encodings(third_image)[0]

    multiple_image_encode = [
        encode_image,
        encode_image2,
    ]

    # confronto le immagini
    match= face_recognition.compare_faces(multiple_image_encode, encode_image3)

    #TERMINATO IL MATCH TRA LE IMMAGINI
    # LAVORO SU ENTRAMBE LE IMMAGINI PER AVERE L'INQUADRATURA SUL VOLTO

    #Trovo la posizione dei volti nelle immagini
    posizione_faccia = face_recognition.face_locations(first_image)
    posizione_faccia2 = face_recognition.face_locations(second_image)
    posizione_faccia3 = face_recognition.face_locations(third_image)

    #lavoro con cv2 sulle immagini
    image=[

    cv2.imread(image1),
    cv2.imread(image2),
    cv2.imread(image3)

    ]

    #   inquadratura
    for (top, right, bottom, left) in posizione_faccia:
        cv2.rectangle(image[0], (left, top), (right, bottom), (0,0,255))

    #   inquadratura seconda immagine
    for (top, right, bottom, left) in posizione_faccia2:
        cv2.rectangle(image[1], (left, top), (right, bottom), (229, 43, 80))

        #   inquadratura terza immagine
    for (top, right, bottom, left) in posizione_faccia3:
        cv2.rectangle(image[2], (left, top), (right, bottom), (85, 107, 47))

    # Visualizzo le immagini a schermo
    for x in range(len(image)):
        cv2.imshow("immagine"+str(x), image[x])

    #restituisco vocalemnte il risultato del match
    sintesi_vocale2(testo = "riconoscimento facciale eseguito correttamente, esito positivo!!" if match[0] else "riconoscimento facciale eseguito correttamente, esito negativo!!")

    #ciclo per terminare con un comando lo script
    while True:
        if cv2.waitKey(1) & 0xFF == ord("q"):
            break

    cv2.destroyAllWindows()

face_match("riconoscimento_facciale\obama.jpg", "riconoscimento_facciale\prova.jpg", "riconoscimento_facciale\obama1.jpg")

input()


from onto_tool import onto_tool
import csv
from rdflib import Graph, URIRef, Literal
from rdflib.namespace import RDFS, SKOS

def lists_equal(list_one, list_two):
    return len(list_one) == len(list_two) and sorted(list_one) == sorted(list_two)

def test_sparql_queries():
    onto_tool.main([
        'bundle', '-v', 'output', 'tests-output/bundle', 'tests/bundle/sparql.yaml'
    ])
    with open('tests-output/bundle/sparql.csv') as csvfile:
        actual = list(row for row in csv.DictReader(csvfile))
    expected = [
        {'s': 'https://data.clientX.com/d/topOntology',
         'p': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type',
         'o': 'http://www.w3.org/2002/07/owl#Ontology'},
        {'s': 'https://data.clientX.com/d/topOntology',
         'p': 'http://www.w3.org/2000/01/rdf-schema#definedBy',
         'o': 'urn:test-sparql-queries'}
    ]
    assert actual == expected

def test_sparql_updates():
    onto_tool.main([
        '-k', 'bundle', '-v', 'output', 'tests-output/bundle/endpoint_sparql', 'tests/bundle/sparql_update.yaml'
    ])

    with open('tests-output/bundle/endpoint_sparql/sparql_update_select.csv') as csvfile:
        actual = list(row for row in csv.DictReader(csvfile))
    expected = [
        {'person': 'http://example.com/John',
         'name': 'John'},
        {'person': 'http://example.com/Jane',
         'name': 'Jane'},
    ]
    assert actual == expected

    constructed_graph = Graph()
    constructed_graph.parse('tests-output/bundle/endpoint_sparql/sparql_update_construct.xml', format='xml')
    labels = list(constructed_graph.subject_objects(SKOS.prefLabel))
    assert lists_equal([(URIRef('http://example.com/John'), Literal('John Johnson')),
                        (URIRef('http://example.com/Jane'), Literal('Jane Johnson'))],
                       labels)

def test_each_file():
    onto_tool.main([
        'bundle', '-v', 'output', 'tests-output/bundle', 'tests/bundle/sparql-each.yaml'
    ])

    # Verify CONSTRUCT
    constructed_graph = Graph()
    constructed_graph.parse('tests-output/bundle/each/construct/upper_ontology.ttl', format='turtle')
    labels = list(constructed_graph.subject_objects(SKOS.prefLabel))
    assert len(labels) == 5
    constructed_graph = Graph()
    constructed_graph.parse('tests-output/bundle/each/construct/domain_ontology.ttl', format='turtle')
    labels = list(constructed_graph.subject_objects(SKOS.prefLabel))
    assert len(labels) == 6

    # Verify SELECT
    with open('tests-output/bundle/each/select/upper_ontology.csv') as csvfile:
        actual = list(row['label'] for row in csv.DictReader(csvfile))
    expected = ["Person", "Upper Ontology", "has phone number", "is friend of", "is private"]
    assert actual == expected
    with open('tests-output/bundle/each/select/domain_ontology.csv') as csvfile:
        actual = list(row['label'] for row in csv.DictReader(csvfile))
    expected = ["Domain Ontology", "School", "Student", "Teacher", "teaches", "works for"]
    assert actual == expected

    # Verify UPDATE
    assert actual == expected
    constructed_graph = Graph()
    constructed_graph.parse('tests-output/bundle/each/update/upper_ontology.ttl', format='turtle')
    labels = list(constructed_graph.subject_objects(SKOS.prefLabel))
    assert len(labels) == 5
    assert not list(constructed_graph.subject_objects(RDFS.label))
    constructed_graph = Graph()
    constructed_graph.parse('tests-output/bundle/each/update/domain_ontology.ttl', format='turtle')
    labels = list(constructed_graph.subject_objects(SKOS.prefLabel))
    assert len(labels) == 6
    assert not list(constructed_graph.subject_objects(RDFS.label))


import sgtk
import datetime
from sgtk.platform.qt import QtGui, QtCore
from tank_vendor import six
from .label_base_widget import LabelBaseWidget
from .shotgun_field_meta import ShotgunFieldMeta

shotgun_globals = sgtk.platform.import_framework(
    "tk-framework-shotgunutils", "shotgun_globals"
)

@six.add_metaclass(ShotgunFieldMeta)
class DateAndTimeWidget(LabelBaseWidget):
    """
    Display a ``date_time`` field value as returned by the Shotgun API.
    """

    _DISPLAY_TYPE = "date_time"

    def _display_value(self, value):
        """
        Set the value displayed by the widget.

        :param value: The value returned by the Shotgun API to be displayed
        """
        self.setText(self._string_value(value))
        self.setToolTip(self._tooltip_value(value, postfix=" %I:%M%p"))

    def _ensure_datetime(self, value):
        """
        Ensures the supplied value is a python datetime object.
        """

        # shotgun_model converts datetimes to floats representing unix time so
        # handle that as a valid value as well
        if not isinstance(value, datetime.datetime):
            value = datetime.datetime.fromtimestamp(value)
        return value

    def _string_value(self, value):
        """
        Convert the Shotgun value for this field into a string

        :param value: The value to convert into a string
        :type value: :class:`datetime.datetime` or a float representing unix time
        """
        dt = self._ensure_datetime(value)

        # return a human readable time format. The postfix formatter used here
        # is '%I:%M%p' which is:
        #
        #   %I = Hour (12-hour clock) as a decimal number [01,12]
        #   %M = Minute as a decimal number [00,59]
        #   %p = Locale's equivalent of either AM or PM
        #
        # resulting in a value like: 01:37AM
        return shotgun_globals.create_human_readable_timestamp(dt, postfix=" %I:%M%p")

    def _tooltip_value(self, value, postfix=""):
        """
        Convert the Shotgun value for this field into a tooltip string

        :param value: The value to convert into a string
        :type value: :class:`datetime.datetime` or a float representing unix time
        """
        dt = self._ensure_datetime(value)
        return dt.strftime("%x" + postfix)

@six.add_metaclass(ShotgunFieldMeta)
class DateAndTimeEditorWidget(QtGui.QDateTimeEdit):
    """
    Allows editing of a ``date_time`` field value as returned by the Shotgun API.

    Pressing ``Enter`` or ``Return`` when the widget has focus will cause the
    value to be applied and the ``value_changed`` signal to be emitted.
    """

    _EDITOR_TYPE = "date_time"

    # minimum width just to prevent the widget from being too squished
    _MINIMUM_WIDTH = 100

    def get_value(self):
        """
        :return: The internal value being displayed by the widget.
        """
        value = self.dateTime()

        try:
            # pyside
            return value.toPython()
        except AttributeError:
            # pyqt
            return value.toPyDateTime()

    def keyPressEvent(self, event):
        """
        Provides shortcuts for applying modified values.

        :param event: The key press event object
        :type event: :class:`~PySide.QtGui.QKeyEvent`
        """

        if event.key() in [QtCore.Qt.Key_Enter, QtCore.Qt.Key_Return]:
            self.value_changed.emit()
        else:
            super(DateAndTimeEditorWidget, self).keyPressEvent(event)

    def setup_widget(self):
        """
        Prepare the widget for display.

        Called by the metaclass during initialization.
        """
        self.setCalendarPopup(True)
        self.setMinimumWidth(self._MINIMUM_WIDTH)

    def _display_default(self):
        """
        Display the default value of the widget.
        """
        self.clear()

    def _display_value(self, value):
        """
        Set the value displayed by the widget.

        :param value: The value returned by the Shotgun API to be displayed
        """
        # shotgun_model converts datetimes to floats representing unix time so
        # handle that as a valid value as well
        if not isinstance(value, datetime.datetime):
            value = datetime.datetime.fromtimestamp(value)
        self.setDateTime(value)


import pandas as pd
import transformers
from transformers import BertModel, BertTokenizer
import torch
from torch import nn, optim
from torch.utils.data import Dataset, DataLoader
from torch.optim.lr_scheduler import LambdaLR

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
def preprocess_data(df):
    print("Preprocessing data...")
    encoded_reviews = tokenizer(
        df['text'].tolist(),
        add_special_tokens=True,
        max_length=128,
        padding='max_length',
        truncation=True,
        return_attention_mask=True,
        return_tensors='pt'
    )
    input_ids = encoded_reviews['input_ids']
    print(input_ids.shape)
    attention_masks = encoded_reviews['attention_mask']
    print(attention_masks.shape)
    labels = torch.tensor(df['label'].values)
    print(labels.shape)
    return input_ids, attention_masks, labels

class AmazonReviewDataset(Dataset):
    def __init__(self, input_ids, attention_masks, labels):
        self.input_ids = input_ids
        self.attention_mask = attention_masks
        self.labels = labels

    def __len__(self):
        return len(self.labels)

    def __getitem__(self, idx):
        return {
            'input_ids': self.input_ids[idx],
            'attention_mask': self.attention_mask[idx],
            'labels': self.labels[idx]
        }


from django.urls import path
from api_doc import views

urlpatterns = [
    path('getAllSportSection/', views.getAllSportSection),
    path('getSportSectionById/<int:pk>', views.getSportSectionById),
    path('getSportSectionType/', views.getSportSectionType),
    path('getSportSectionByTypeId/<int:pk>', views.getSportSectionByTypeId),
    # path('deleteSportSection/<int:pk>', views.deleteSportSection),
    # path('updateSportSection/', views.SportSection.updateSportSection),
]


import numpy as np
import pandas as pd
from matplotlib import pyplot as plt

def data_init():
    spreadsheet_file = pd.ExcelFile('data.xlsx')
    worksheets = spreadsheet_file.sheet_names
    island_coordinates = []
    island_numbers = []
    locationDF = pd.read_excel('data.xlsx', sheet_name="Locations", header=None, skiprows=1)
    for index, row in locationDF.iterrows():
        island_coordinates.append([row[1], row[2]])
        island_numbers.append(row[0])
    return [island_numbers, island_coordinates]

def storm_init():
    storms = []
    stormDF = pd.read_excel('data.xlsx', sheet_name="Storms", header=None, skiprows=2)
    for index, row in stormDF.iterrows():
        storms.append([row[0], row[1], row[2]])
    return storms

def main():
    island_numbers, island_coordinates = data_init()
    storms = storm_init()
    #optimum_route = [1, 23, 6, 4, 22, 13, 2, 3, 5, 28, 20, 11, 18, 16, 24, 30, 21, 15, 27, 12, 19, 8, 29, 9, 10, 25, 26, 17, 14, 7, 1]
    #optimum_route = [1, 23, 13, 28, 6, 4, 22, 2, 3, 5, 20, 11, 18, 16, 24, 30, 21, 15, 27, 12, 19, 8, 29, 9, 10, 25, 26, 17, 14, 7, 1]
    optimum_route = [1, 23, 13, 2, 22, 5, 3, 4, 6, 28, 18, 24, 16, 11, 20, 10, 9, 29, 8, 19, 12, 27, 15, 30, 21, 26, 25, 17, 14, 7, 1]
    data = np.array(island_coordinates)
    x, y = data.T
    plt.scatter(x, y)

    for i in range(len(island_coordinates)):
        x = island_coordinates[i][0]
        y = island_coordinates[i][1]
        plt.plot(x, y, 'bo')
        plt.text(x * (1 + 0.01), y * (1 + 0.01), i + 1, fontsize=12)

    for i in range(len(island_coordinates) - 1):
        # i = 0 .... n-1
        plt.plot([island_coordinates[optimum_route[i] - 1][0], island_coordinates[optimum_route[i+1] - 1][0]],
                 [island_coordinates[optimum_route[i] - 1][1], island_coordinates[optimum_route[i+1] - 1][1]], 'green', linestyle=':', marker='')

    for i in range(len(storms)):
        x = storms[i][0]
        y = storms[i][1]
        r = storms[i][2]
        circle1 = plt.Circle((x, y), r, color='r', alpha=0.3, fill=False, label="storm:" + str(i + 1))
        label = plt.annotate(i + 1, xy=(x, y), fontsize=12,
                            verticalalignment='center', horizontalalignment='center')
        plt.gcf().gca().add_artist(circle1)

    plt.gca().set_aspect('equal', adjustable='box')
    plt.draw()

    plt.show()
    plt.savefig('plot.png', dpi=100)

if __name__ == '__main__':
    main()


import cufflinks as cf
cf.go_offline()
cf.set_config_file(offline=False, world_readable=True)
import torch
import numpy as np
from Utils import *

def train_step(x ,y, model, loss_fn, optimizer):
# Sets model to train mode
    model.train()

    # Makes predictions
    ypred = model(x)

    # Computes loss
    loss = loss_fn(y, ypred)

    # Computes gradients
    loss.backward()

    # Updates parameters and zeroes gradients
    optimizer.step()
    optimizer.zero_grad()

    # Returns the loss
    return loss.item()

def fit_train(n_epochs, train_dl, valid_dl, model, loss_function, optimizer, batch_size, input_dim):
    train_losses = []
    val_losses = []
    device = "cuda" if torch.cuda.is_available() else "cpu" # check if cuda available.
    print("Training is starting...")
    for epoch in range(1, n_epochs + 1):

        batch_losses = []
        for x_batch, y_batch in train_dl:
            x_batch = x_batch.view([batch_size, -1, input_dim]).to(device)
            y_batch = y_batch.to(device)
            # use train step to train the model
            loss = train_step(x_batch, y_batch, model=model, loss_fn=loss_function, optimizer=optimizer)
            batch_losses.append(loss)
        # append losses to global variables
        training_loss = np.mean(batch_losses)
        train_losses.append(training_loss)

        # check if validation set is available
        if valid_dl is not None:

            with torch.no_grad():
                batch_val_losses = []
                for x_val, y_val in valid_dl:
                    # model in evaluation mode
                    model.eval()
                    x_val = x_val.view([batch_size, -1, input_dim]).to(device)
                    y_val = y_val.to(device)
                    # make prediction
                    yhat = model(x_val)
                    val_loss = loss_function(y_val, yhat).item()
                    batch_val_losses.append(val_loss)

                # vallid global losses
                validation_loss = np.mean(batch_val_losses)
                val_losses.append(validation_loss)

            if (epoch <= 10) | (epoch % 50 == 0):
                # if validation available print all
                print(
                    f"[{epoch}/{n_epochs}] Training loss: {training_loss:.4f}\t Validation loss: {validation_loss:.4f}")
        else:
            # print only train loss if no validation available
            print(f"[{epoch}/{n_epochs}] Training loss: {training_loss:.4f}")
    return [train_losses, val_losses]

def test_fit(model, test_dl_one, input_dim):
    batch_size = 1
    device = "cuda" if torch.cuda.is_available() else "cpu"  # check if cuda available.
    with torch.no_grad():
        predictions = []
        values = []
        for x_test, y_test in test_dl_one:
            # reshape
            x_test = x_test.view([batch_size, -1, input_dim]).to(device)
            y_test = y_test.to(device)

            model.eval()
            # make prediction in evaluation mode the model
            yhat = model(x_test)
            predictions.append(yhat.to(device).detach().numpy())
            values.append(y_test.to(device).detach().numpy())
    return predictions, values