from rest_framework import serializers

from .models import ROLES, User

class UserSignUpSerializer(serializers.ModelSerializer):
    class Meta:
        model = User
        fields = ('email', 'username')

class UserAuthSerializer(serializers.ModelSerializer):
    confirmation_code = serializers.CharField(required=True)

    class Meta:
        model = User
        fields = ('username', 'confirmation_code')

class UserCreateSerializer(serializers.ModelSerializer):
    role = serializers.ChoiceField(ROLES, required=False)

    class Meta:
        model = User
        fields = (
            'username', 'email', 'first_name', 'last_name', 'bio', 'role',
        )

class CurrentUserSerializer(serializers.ModelSerializer):
    class Meta:
        model = User
        fields = (
            'username', 'email', 'first_name', 'last_name', 'bio', 'role',
        )
        read_only_fields = ('username', 'email', 'role',)


from check_sequences import *
from itertools import permutations

# Intuitive, but too long
"""
for i in range(1000,10000): # n1 = i
    if i % 100 > 9 and is_octagonal(i):
        print(i)
        for j in range(10,100):
            n2 = (i % 100) * 100 + j
            for k in range(10,100):
                n3 = (n2 % 100) * 100 + k
                for l in range(10,100):
                    n4 = (n3 % 100) * 100 + l
                    for m in range(10,100):
                        n5 = (n4 % 100) * 100 + m
                        n6 = (n5 % 100) * 100 + (i // 100)
                        check = [i,n2,n3,n4,n5,n6]
                        if len(set(check)) == 6:
                            # print(check) # tracker
                            properties = [is_triangle,is_square,is_pentagonal,is_hexagonal,is_heptagonal]
                            satisfied = []
                            for el in check[1:]:
                                for f in properties:
                                    if f(el) and f not in satisfied:
                                        satisfied.append(f)
                                        break
                            if len(satisfied) == 5:
                                print(check)
"""
def is_cyclic(lst):
    for i in range(len(lst)):
        if lst[i] // 100 != lst[i-1] % 100:
            return False
    return True

def main():
    l1 = list(filter(is_triangle,range(1000,10000)))
    l2 = list(filter(is_square,range(1000,10000)))
    l3 = list(filter(is_pentagonal,range(1000,10000)))
    l4 = list(filter(is_hexagonal,range(1000,10000)))
    l5 = list(filter(is_heptagonal,range(1000,10000)))
    l6 = list(filter(is_octagonal,range(1000,10000)))

    check = [l1,l2,l3,l4,l5,l6]
    for a,b,c,d,e,f in permutations(check):
        for e1 in a:
            for e2 in list(filter(lambda x:x // 100 == e1 % 100,b)):
                for e3 in list(filter(lambda x:x // 100 == e2 % 100,c)):
                    for e4 in list(filter(lambda x:x // 100 == e3 % 100,d)):
                        for e5 in list(filter(lambda x:x // 100 == e4 % 100,e)):
                            for e6 in list(filter(lambda x:x // 100 == e5 % 100,f)):
                                if is_cyclic([e1,e2,e3,e4,e5,e6]):
                                    print(e1, e2, e3, e4, e5, e6)
                                    print("Answer:",sum([e1,e2,e3,e4,e5,e6]))
                                    return

main()


"""Houses the Service-Now Property"""

from typing import TypeVar, Type, Dict, Any, Union

from pyservicenow.types.models._abstract_servicenow_property import AbstractServiceNowProperty

from json import dumps

S = TypeVar("S", bound="ServiceNowProperty")

class ServiceNowProperty(AbstractServiceNowProperty):
    """Service-Now Property type"""

    def __init__(self) -> None:
        super().__init__()

        self._display_value = ""
        self._value = ""
        self._link = ""

    @property
    def display_value(self) -> str:
        """Gets/Sets the display value

        Returns:
            str: The display value
        """

        return self._display_value

    @display_value.setter
    def display_value(self, value: str) -> None:
        self._display_value = value

    @property
    def value_link(self) -> str:
        """Gets/Sets the link

        Returns:
            str: The link
        """

        return self._link

    @value_link.setter
    def value_link(self, value: str) -> None:
        self._link = value

    @property
    def actual_value(self) -> str:
        """Gets/Sets the value

        Returns:
            str: The value
        """

        return self._value

    @actual_value.setter
    def actual_value(self, value: str) -> None:
        self._value = value

    @property
    def as_dict(self) -> Dict[str, Any]:

        return {
            "display_value": self.display_value,
            "value": self.actual_value,
            "link": self.value_link,
        }

    def __json__(self) -> str:

        return dumps(self.as_dict)

    @classmethod
    def __fromjson__(cls: Type[S], value: Union[str, Dict]) -> S:

        _value = cls()

        if not isinstance(value, str) and not isinstance(value, dict):
            return _value

        if value is None:
            pass
        elif isinstance(value, str):
            _value.actual_value = value
        else:
            _value.display_value = value.get("display_value", "")
            _value.actual_value = value.get("value", "")
            _value.value_link = value.get("link", "")

        return _value


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import datetime
import logging
import pathlib
import subprocess
import time
from multiprocessing import Process

import carlhauser_server.Configuration.database_conf as database_conf
import carlhauser_server.Configuration.distance_engine_conf as distance_engine_conf
import carlhauser_server.Configuration.feature_extractor_conf as feature_extractor_conf
import carlhauser_server.Configuration.webservice_conf as webservice_conf
import common.ImportExport.json_import_export as json_import_export
from carlhauser_server.Helpers.arg_parser import ConfArgs
from common.environment_variable import get_homedir
from common.environment_variable import load_server_logging_conf_file
from psrecord.main import monitor

load_server_logging_conf_file()

# Small class to handle processus
class WorkerProcessus:
    def __init__(self, worker_path: pathlib.Path):
        self.worker_path: pathlib.Path = worker_path
        self.process: subprocess.Popen = None  # processus: subprocess.Popen
        self.start_time = None

        self.logger = logging.getLogger(__name__)

    def launch(self, db_conf: database_conf.Default_database_conf = None,
               dist_conf: distance_engine_conf.Default_distance_engine_conf = None,
               fe_conf: feature_extractor_conf.Default_feature_extractor_conf = None,
               ws_conf: webservice_conf.Default_webservice_conf = None,
               mode=None):
        """
        Construct an argument list and launch the process.
        :param db_conf: configuration file
        :param dist_conf: configuration file
        :param fe_conf: configuration file
        :param ws_conf: configuration file
        :param mode:  configuration element
        :return: Nothing
        """
        # Construct an argument list to be 'popen' as a new process
        arg_list = [str(self.worker_path)]

        # Save current configuration in files
        # Using self.worker_path.parent
        if db_conf is not None:
            tmp_db_conf_path = get_homedir() / "tmp_db_conf.json"
            json_import_export.save_json(db_conf, file_path=tmp_db_conf_path)
            arg_list.append(ConfArgs.DB_CONF_ARG)
            arg_list.append(str(tmp_db_conf_path.resolve()))

        if dist_conf is not None:
            tmp_dist_conf_path = get_homedir() / "tmp_dist_conf.json"
            json_import_export.save_json(dist_conf, file_path=tmp_dist_conf_path)
            arg_list.append(ConfArgs.DIST_CONF_ARG)
            arg_list.append(str(tmp_dist_conf_path.resolve()))

        if fe_conf is not None:
            tmp_fe_conf_path = get_homedir() / "tmp_fe_conf.json"
            json_import_export.save_json(fe_conf, file_path=tmp_fe_conf_path)
            arg_list.append(ConfArgs.FE_CONF_ARG)
            arg_list.append(str(tmp_fe_conf_path.resolve()))

        if ws_conf is not None:
            tmp_ws_conf_path = get_homedir() / "tmp_ws_conf.json"
            json_import_export.save_json(ws_conf, file_path=tmp_ws_conf_path)
            arg_list.append(ConfArgs.WS_CONF_ARG)
            arg_list.append(str(tmp_ws_conf_path.resolve()))

        if mode is not None:
            arg_list.append(ConfArgs.MODE_ARG)
            arg_list.append(mode)

        # Save starting time
        self.start_time = datetime.datetime.now()

        # Launch worker
        self.logger.debug(f"launching process as : {arg_list}")
        self.process = subprocess.Popen(arg_list)

    def shutdown(self, grace_time=10) -> bool:
        """
        Tru to stop within <gracetime> seconds the current processus/worker
        :param grace_time: maximum waiting time
        :return: True if stopped, False otherwise
        """
        self.logger.info(f"Trying to stop (terminate) {self.process}")
        try:
            self.process.terminate()
            time.sleep(0.2)
        finally:
            try:
                self.process.wait(timeout=grace_time)
            except subprocess.TimeoutExpired:
                self.logger.info(f"Processus {self.process} did not terminate in time. Trying to kill it.")
            finally:
                try:
                    self.process.kill()
                    self.logger.info(f"Processus exited with {self.process.returncode}")
                    return True
                except subprocess.TimeoutExpired:
                    self.logger.info(f"Processus {self.process} is still alive .. Don't know how to stop it.")
                    return False

    def is_running(self) -> bool:
        """
        Check if current processus is running
        :return: True if process running, False otherwise
        """

        if self.process.poll() is None:
            return True
        return False

    def check_status(self, id_to_display):
        """
        Check and display status on logging output
        :param id_to_display: # of the worker, if you want to print one
        :return: True if process running, False otherwise
        """

        curr_str = f"==> Worker {id_to_display} : Launched on {self}"

        is_running = self.is_running()
        if is_running:
            curr_str += " status : running ..."
        else:
            curr_str += " status : terminated or ended ..."

        self.logger.info(curr_str)
        return is_running

    def wait_until_stopped(self, timeout: int = 60) -> bool:
        """
        Wait until the worker is stopped (= terminated)
        Put timeout -1 if you don't want to function to timeout
        :param timeout: maximal waiting time for the process to stop
        :return: True if stopped, False otherwise
        """

        start = time.time()

        while self.is_running():
            time.sleep(5)
            if timeout != -1 and abs(time.time() - start) > timeout:
                self.logger.warning("Waiting for worker to stop has timeout-ed.")
                return False

        self.logger.debug(f"Worker stopped after {round(abs(time.time() - start), 3)}s.")
        return True

    def monitor_worker(self, interval: int = 1):

        # Create path where to save pictures and logs
        curr_name = str(self.worker_path.name) + "_at_" + str(self.start_time)
        logpath = pathlib.Path(get_homedir() / (curr_name + ".log"))
        graphpath = pathlib.Path(get_homedir() / (curr_name + ".pdf"))
        # monitor(self.process.pid, logfile=logpath, plot=graphpath, include_children=True)
        # pid, logfile = None, plot = None, duration = None, interval = None,
        # Launch the monitoring process
        Process(target=monitor, args=(self.process.pid, str(logpath), str(graphpath), interval, True))

    # ==================== To string ====================

    # Overwrite to print the content of the cluster instead of the cluster memory address
    def __repr__(self):
        return self.get_str()

    def __str__(self):
        return self.get_str()

    def get_str(self):
        return ''.join(map(str, [' processus=', self.process,
                                 ' worker_path=', str(self.worker_path),
                                 ' start_time=', self.start_time]))


import time
import json
from display import Display
import const

class Timer():
    def __init__(self):
        self.min = 0
        self.text = '00:00'
        self.seconds = 0
        self.display = Display()
        self.cancel = False

    def setTimer(self, min):
        self.min = min
        self.countdown_min(self.min)

    def checkCancel(self):
        data = self.checkdata()
        self.cancel = data

    def checkdata(self): #check json file for any changes
        with open(const.data_location) as f:
            data = json.load(f)
        return data['timer']
    def setCancel(self):
        with open(const.data_location, 'r', encoding='utf-8') as outjson:
            json_data = json.load(outjson)
            json_data['timer'] = False

        with open(const.data_location, 'w') as output:
            json.dump(json_data, output, indent=4)

    def countdown_min(self, t):
        seconds = t * 60
        self.seconds = seconds
        while seconds:
            self.checkCancel()
            if self.cancel == False:
                mins,sec = divmod(seconds,60)
                timer = '{:02d}:{:02d}'.format(mins, sec)
                self.text = timer
                self.display.update_screen(timer)
                time.sleep(0.94)
                seconds-=1
            else:
                break
        self.cancel = False
        self.setCancel()


"""Create art_no column in databses."""

import pathlib

import pandas as pd

from ._message import message

def create_art_no_column(root_dir):
    """Create art_no column in databases."""
    #
    message("Assign REC-No identifier to each record")

    processed_dir = pathlib.Path(root_dir) / "databases"
    files = list(processed_dir.glob("_*.zip"))
    for file in files:
        data = pd.read_csv(file, encoding="utf-8", compression="zip")
        data["art_no"] = range(1, len(data) + 1)
        data.to_csv(file, sep=",", encoding="utf-8", index=False, compression="zip")


from typing import List, Tuple
import numpy as np
from exception import *
from agent import BaseAgent

class TicTacToe:
    def reset(self, opponent: BaseAgent, turn=True) -> np.ndarray:
        self._opponent = opponent
        self._board = np.zeros((3, 3), dtype=int)
        if not turn:
            agent_action = self._opponent.action(self._board)
            self._board[agent_action[0]][agent_action[1]] = -1
        return self._board.copy()

    def step(self, action: Tuple[int, int]) -> Tuple[np.ndarray, int, bool, dict]:
        '''
        Walks through the game with given action, returns the next state, reward, done, and info.
        @param Tuple[int, int] action: action to take
        @return: next state, reward, done, and info (for debugging purposes)
        @rtype: Tuple[np.ndarray, int, bool, dict]
        '''
        # Consider `reward = -1` as not only being lose but also wrong position.
        # First, place on the board for my turn
        # Assertive (the case where the action is out of range)
        if not (0 <= action[0] <= 2 and 0 <= action[1] <= 2):
            print('Action is out of action space.')
            return self._board.copy(), -1, True, {}

        if not self._possible(action):
            return self._board.copy(), -1, True, {}
        self._board[action[0]][action[1]] = 1
        if self._win():
            return self._board.copy(), 1, True, {}
        if self._all_occupied():
            return self._board.copy(), 0, True, {}

        # It's the opponent's turn
        opponent_action = self._opponent.action(-self._board)
        if not self._possible(opponent_action):
            return self._board.copy(), -1, True, {}
        self._board[opponent_action[0]][opponent_action[1]] = -1
        if self._win():
            return self._board.copy(), -1, True, {}
        if self._all_occupied():
            return self._board.copy(), 0, True, {}

        return self._board.copy(), 0, False, {}

    def _possible(self, action: Tuple[int, int]) -> bool:
        return self._board[action[0]][action[1]] == 0

    def _check(self, v: List[int]) -> int:
        _sum = sum(v)
        if _sum == 3:
            return 1
        elif _sum == -3:
            return -1
        else:
            return 0

    def _win(self) -> int: # check the winner of the game. If it's not done, return -1.
        vecs: list = [self._board.tolist()[0], self._board.T.tolist()[0], self._board.diagonal(), np.flip(self._board, 1).diagonal()]

        for vec in vecs:
            checked = self._check(vec)
            if checked != 0:
                return checked
        return 0

    def _all_occupied(self) -> bool:
        return np.square(self._board).sum() == 9

    def render(self) -> None:
        print("+---+---+---+")
        for row in range(3):
            print("|", end = '')
            for col in range(3):
                shape = ' '
                if self._board[row][col] == 1:
                    shape = 'O'
                if self._board[row][col] == -1:
                    shape = 'X'
                print(" {} |".format(shape), end = '')
            print("\n+---+---+---+")


import requests
import simpleaudio
import tempfile
import json

host = "127.0.0.1"  # "localhost"でも可能だが、処理が遅くなる
port = 50021

class SyntheticVoice:
    def __init__(self):
        self.host = host
        self.port = port

    def speaking(self, text):
        params = (
            ("text", text),
            ("speaker", 1)  # 音声の種類をInt型で指定
        )
        # 以下の検索結果でどの数値がどういった声なのか記載されている
        # https://github.com/VOICEVOX/voicevox_resource/search?q=styleId

        response1 = requests.post(
            f"http://{self.host}:{self.port}/audio_query",
            params=params
        )

        response2 = requests.post(
            f"http://{self.host}:{self.port}/synthesis",
            headers={"Content-Type": "application/json"},
            params=params,
            data=json.dumps(response1.json())
        )

        with tempfile.TemporaryDirectory() as tmp:
            with open(f"{tmp}/audi.wav", "wb") as f:
                f.write(response2.content)
                wav_obj = simpleaudio.WaveObject.from_wave_file(
                    f"{tmp}/audi.wav")
                play_obj = wav_obj.play()
                play_obj.wait_done()


import sys

from binaryninja.update import UpdateChannel, are_auto_updates_enabled, set_auto_updates_enabled, is_update_installation_pending, install_pending_update
from binaryninja import core_version
import datetime

channel = None
versions = []

def load_channel(newchannel):
  global channel
  global versions
  if (channel is not None and newchannel == channel.name):
    print("Already on the selected channel: {}".format(newchannel))
  else:
    try:
      print("Loading channel %s" % newchannel)
      channel = UpdateChannel[newchannel]
      print("Loading versions...")
      versions = channel.versions
    except Exception:
      print("%s is not a valid channel name. Defaulting to " % chandefault)
      channel = UpdateChannel[chandefault]

def select(version):
  date = datetime.datetime.fromtimestamp(version.time).strftime('%c')
  print("Version:\t%s" % version.version)
  print("Updated:\t%s" % date)
  print("Notes:\n\n-----\n%s" % version.notes)
  print(version.update())
  if is_update_installation_pending():
    print("Installing...")
    #note that the GUI will be launched after update but should still do the upgrade headless
    install_pending_update()
  # forward updating won't work without reloading
  sys.exit()

def main():
  global channel
  done = False
  # NOTE(artem): We need the dev version, for now
  load_channel("dev")
  set_auto_updates_enabled(True)
  select(channel.latest_version)

if __name__ == "__main__":
  try:
    main()
  except Exception as e:
    print("Binary Ninja update may have failed. Caught an exception trying to update: {}".format(str(e)))


import boto3
import json
import decimal
from decimal import Decimal
from boto3.dynamodb.conditions import Key, Attr

class DecimalEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, decimal.Decimal):
            return float(obj)
        return json.JSONEncoder.default(self, obj)

def get_kneipen(lat, long, distance):
    dynamodb = boto3.resource("dynamodb", region_name='eu-central-1')

    table = dynamodb.Table('geo-data')
    fe = Key('lat').between(Decimal(lat) - Decimal(distance), Decimal(lat) + Decimal(distance))
    fe2 = Key('long').between(Decimal(long) - Decimal(distance), Decimal(long) + Decimal(distance))

    response = table.scan(
            FilterExpression=fe & fe2,
            )

    return response["Items"]

def handler(event, context):
    try:
        return response(get_kneipen(event["queryStringParameters"]["lat"],event["queryStringParameters"]["long"],event["queryStringParameters"]["distance"]) , 200)
    except Exception as e:
        return response('Error', 400)

def response(message, status_code):
    return {
        "statusCode": status_code,
        "headers": {
            "X-Bier": "gerne doch",
            "Access-Control-Allow-Origin": "*"
        },
        "body": json.dumps(message, cls=DecimalEncoder),
        "isBase64Encoded": False
    }


# This script plots the MBAR vs true data

import matplotlib
import numpy
import sys
import os
if os.environ.get('DISPLAY','') == '': # then no display found. Use non-interactive Agg backend'
    matplotlib.use('Agg')
import matplotlib.pyplot as plt

MW = float(sys.argv[1])
true_data_file = sys.argv[2]               # The file that is used to minimize the deviations against, i.e. the target. It could be experimental data or a simulation results of a force field from literature.
mbar_data_file = sys.argv[3]               # The file that contains MBAR results at ITIC state points
true_data_label = sys.argv[4]               # Label for true data
output_figure_filename = sys.argv[5]        # Output file name
try:
    dsim_data_file = sys.argv[6]            # If provided, this argument is a file containing direct simulation results
except:
    isDSIM = False
else:
    isDSIM = True

R_const = 8.31446261815324

# Conversion factors
atm_to_mpa = 0.101325
kcalmol_to_kjmol = 4.184

bar_to_mpa = 0.1
kelvin_to_kjmol = 0.0083

# Import data
true_data = numpy.loadtxt(true_data_file, skiprows=1, usecols=(0,1,2,5,11))
true_temp_k = true_data[:,0]
true_rho_gcc = true_data[:,1]
true_p_mpa = true_data[:,2] * atm_to_mpa
true_u_kjmol = true_data[:,3] * kcalmol_to_kjmol
Nmolec = true_data[:,4]

mbar_data = numpy.loadtxt(mbar_data_file, skiprows=1, usecols=(0,1,3,5,2))
mbar_temp_k = mbar_data[:,0]
mbar_rho_gcc = mbar_data[:,1]
mbar_u_kjmol = mbar_data[:,2] * kelvin_to_kjmol
mbar_p_mpa = mbar_data[:,3] * bar_to_mpa
Neff = mbar_data[:,4]

if isDSIM:
    dsim_data = numpy.loadtxt(dsim_data_file, skiprows=1, usecols=(0,1,2,5,11))
    dsim_temp_k = dsim_data[:,0]
    dsim_rho_gcc = dsim_data[:,1]
    dsim_p_mpa = dsim_data[:,2] * atm_to_mpa
    dsim_u_kjmol = dsim_data[:,3] * kcalmol_to_kjmol

# Calcualte properties
true_u_hat = true_data[:,3] * kcalmol_to_kjmol / Nmolec / R_const / true_temp_k * 1e3
true_z = true_p_mpa * MW / ( true_rho_gcc  * R_const * true_temp_k )
true_zminus1overRho = ( true_z - 1 ) / true_rho_gcc

mbar_u_hat = mbar_data[:,2] * kelvin_to_kjmol / Nmolec / R_const / mbar_temp_k * 1e3
mbar_z = mbar_p_mpa * MW / ( mbar_rho_gcc  * R_const * mbar_temp_k )
mbar_zminus1overRho = ( mbar_z - 1 ) / mbar_rho_gcc

if isDSIM:
    dsim_u_hat = dsim_data[:,3] * kcalmol_to_kjmol / Nmolec / R_const / dsim_temp_k * 1e3
    dsim_z = dsim_p_mpa * MW / ( dsim_rho_gcc  * R_const * dsim_temp_k )
    dsim_zminus1overRho = ( dsim_z - 1 ) / dsim_rho_gcc

# Initiate plot
plt.figure(num=None, figsize=(24, 7), dpi=100, facecolor='w', edgecolor='w')
plt.subplots_adjust(left=0.04, bottom=None, right=0.99, top=None, wspace=0.3, hspace=None)
font = {'weight' : 'normal', 'size' : 14}
matplotlib.rc('font', **font)

# (Z-1)/rho vs. rho plot
plt.subplot(1, 3, 1 )
plt.xlabel("$\\rho$ [$\mathrm{g/cm^3}$]")
plt.ylabel("$(Z-1)/\\rho}$ [$\mathrm{cm^3/g}$]")
plt.scatter(true_rho_gcc, true_zminus1overRho, marker="o", facecolors='none', edgecolors='k', label=true_data_label)
plt.scatter(mbar_rho_gcc, mbar_zminus1overRho, marker="o", facecolors='none', edgecolors='r', label='MBAR')
if isDSIM:
    plt.scatter(dsim_rho_gcc, dsim_zminus1overRho, marker="o", facecolors='none', edgecolors='g', label='Direct dsimulation')
for i in range(0,len(mbar_rho_gcc)):
    plt.text(mbar_rho_gcc[i]+0.01, mbar_zminus1overRho[i], str(round(Neff[i],1)), color='red', fontsize=9, alpha=0.9, bbox=dict(color='red', alpha=0.01))
plt.legend()

# Z vs. 1000/T plot
plt.subplot(1, 3, 2 )
plt.xlabel("$1000/T$ [K$^{-1}$]")
plt.ylabel("$Z$")
plt.scatter(1000.0 / true_temp_k[:10], true_z[:10], marker="o", facecolors='none', edgecolors='k')
plt.scatter(1000.0 / true_temp_k[21:], true_z[21:], marker="o", facecolors='none', edgecolors='k')
plt.scatter(1000.0 / mbar_temp_k[:10], mbar_z[:10], marker="o", facecolors='none', edgecolors='r')
plt.scatter(1000.0 / mbar_temp_k[21:], mbar_z[21:], marker="o", facecolors='none', edgecolors='r')
if isDSIM:
    plt.scatter(1000.0 / dsim_temp_k[:10], dsim_z[:10], marker="o", facecolors='none', edgecolors='k')
    plt.scatter(1000.0 / dsim_temp_k[21:], dsim_z[21:], marker="o", facecolors='none', edgecolors='k')

# TU^ vs 1000/T plot
plt.subplot(1, 3, 3 )
#plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))
plt.xlabel("$1000/T$ [K$^{-1}$]")
plt.ylabel("$T\hat{U}$ [K]")
plt.scatter(1000.0 / true_temp_k[:10], true_u_hat[:10] * true_temp_k[:10], marker="o", facecolors='none', edgecolors='k')
plt.scatter(1000.0 / true_temp_k[21:], true_u_hat[21:] * true_temp_k[21:], marker="o", facecolors='none', edgecolors='k')
plt.scatter(1000.0 / mbar_temp_k[:10], mbar_u_hat[:10] * mbar_temp_k[:10], marker="o", facecolors='none', edgecolors='r')
plt.scatter(1000.0 / mbar_temp_k[21:], mbar_u_hat[21:] * mbar_temp_k[21:], marker="o", facecolors='none', edgecolors='r')
if isDSIM:
    plt.scatter(1000.0 / dsim_temp_k[:10], dsim_u_hat[:10] * dsim_temp_k[:10], marker="o", facecolors='none', edgecolors='k')
    plt.scatter(1000.0 / dsim_temp_k[21:], dsim_u_hat[21:] * dsim_temp_k[21:], marker="o", facecolors='none', edgecolors='k')

plt.savefig(output_figure_filename)
plt.close()


import numpy as np
import math
import csv
import matplotlib.pyplot as plt
from scipy.io import loadmat

print("Initializations")
train_set = loadmat(file_name="train.mat", mat_dtype=True)
test_set = loadmat(file_name="test.mat", mat_dtype=True)

train_data = np.transpose(train_set["train_images"])
train_labels = train_set["train_labels"].reshape(1,-1)[0]
test_data = test_set["test_images"]

random_state = np.random.get_state()
np.random.shuffle(train_data)
np.random.set_state(random_state)
np.random.shuffle(train_labels)

num_train = train_data.shape[0]
num_test = test_data.shape[0]

for i in range(0, num_train):
    train_data[i] = train_data[i] / np.linalg.norm(train_data[i])

for i in range(0, num_test):
    test_data[i] = test_data[i] / np.linalg.norm(test_data[i])
test_data = np.transpose(test_data)

pred_labels = np.zeros(num_test)
set_size = [num_train]
for si in range(0, len(set_size)):
    sz = set_size[si]
    print("Slicing, Sorting, Reshaping")
    train_slice = train_data[0:sz]
    label_slice = train_labels[0:sz]

    index = label_slice.argsort()
    t_sort = train_slice[index[::1]]
    l_sort = label_slice[index[::1]]
    t_flat = t_sort.reshape(len(t_sort), -1)
    t_flat = np.transpose(t_flat)
    for i in range(0, sz):
        t_flat[:, i] = t_flat[:, i] / np.linalg.norm(t_flat[:, i])

    priors = np.zeros(10)
    for i in l_sort:
        priors[i] = priors[i] + 1

    digit_sect = np.zeros(11)
    for i in range(0, 10):
        digit_sect[i+1] = digit_sect[i]+priors[i]

    priors = priors / len(l_sort)

    print("Covariance Matrix")
    covmat = np.zeros((10,784,784))
    mu = np.zeros((10, 784))
    for i in range(0,10):
        ai = digit_sect[i]
        ae = digit_sect[i+1]
        covmat[i,:,:] = np.cov(t_flat[:, ai:ae])
        mu[i, :] = np.mean(t_flat[:, ai:ae], axis=1)

    a = 0.001
    covmat_inv = np.zeros((10,784,784))
    seconds = np.zeros(10)
    thirds = np.zeros(10)
    for c in range(0,10):
        covmat_id = covmat[c]+a*np.identity(784)
        covmat_inv[c] = np.linalg.inv(covmat_id)
        sign, val = np.linalg.slogdet(covmat_id)
        seconds[c] = sign * val
        thirds[c] = np.log(priors[c])

    print("Predictions")
    for i in range(0, len(pred_labels)):
        if i % 1000 == 0: print("i: " + str(i))
        x = test_data[:, i]
        max_c = 0
        max_disc = -99999999
        for c in range(0, 10):
            x_mu = x - mu[c]
            qc = np.dot(x_mu, np.dot(covmat_inv[c], x_mu))
            curr = -0.5*qc - 0.5*seconds[c] + thirds[c]
            if curr > max_disc:
                max_c = c
                max_disc = curr
        pred_labels[i] = max_c

with open('test_labels.csv', 'wb') as f:
    writer = csv.writer(f)
    writer.writerow(['Id'] + ['Category'])
    for i in range(0, len(pred_labels)):
        writer.writerow([i+1] + [int(pred_labels[i])])

hw3 - Gaussian Classifiers (LDA and QDA)/code/prob5div.py


from typing import TYPE_CHECKING

from .client_assertion import ClientAssertionCredential
from ..._credentials.token_exchange import TokenFileMixin

if TYPE_CHECKING:
    # pylint:disable=unused-import,ungrouped-imports
    from typing import Any

class TokenExchangeCredential(ClientAssertionCredential, TokenFileMixin):
    def __init__(self, tenant_id: str, client_id: str, token_file_path: str, **kwargs: "Any") -> None:
        super().__init__(
            tenant_id=tenant_id,
            client_id=client_id,
            get_assertion=self.get_service_account_token,
            token_file_path=token_file_path,
            **kwargs
        )


import cvxpy as cvx
import numpy as np

def copy_prob(prob):
    return cvx.Problem(prob.objective, prob.constraints)

def get_scs_data(prob):
    data = prob.get_problem_data('SCS')

    # convert sparse indices and pointers to int64 so that CySCS doesn't
    # need to convert them itself (which raises a warning).
    A = data['A']
    A.indptr = A.indptr.astype(np.int64)
    A.indices = A.indices.astype(np.int64)

    return data

def form_prox(prob, x_vars):
    """ Given a CVXPY problem, form its prox.

    Given problem: min f(x),

    form the prox problem:

    min f(x) + tau*||x-x0||^2

    Parameters
    ----------
    prob: CVXPY problem
    x_vars: dict
        Dict of k:v pairs, where k is a string, and v is a CVXPY Variable that appears in `prob`

    Returns
    -------
    pxprob: CVXPY prox problem
    x0_vars: dict
        Dict of k:v pairs, where v is CVXPY Parameter object, corresponding to prox input x0
        x0_vars also contains special key '__tau', which corresponds to the regularization parameter
    """

    tau = cvx.Parameter(sign="positive")
    x0_vars = {'__tau':tau}

    obj = 0
    for k in x_vars:
        x = x_vars[k]
        x0 = cvx.Parameter(*x.size)
        x0_vars[k] = x0

        obj = obj + tau*cvx.sum_squares(x - x0)

    pxprob = cvx.Problem(prob.objective + cvx.Minimize(obj), prob.constraints)

    return pxprob, x0_vars

def rand_param_vals(x0_vars):
    """ Set the CVXPY Parameter values for the x0 prox input parameters to random numpy
    arrays of the appropriate size.

    We use this to detect where the parameters get mapped to in the SCS stuffing.

    Modifies the x0_vars in place!
    """
    for k in x0_vars:
        x = x0_vars[k]
        s = x.size

        if k == '__tau':
            x.value = np.random.rand()+1
        else:
            x.value = np.random.randn(*s)

def param_map(pxprob, x0_vars):
    """
    get the location for taus and x0 parameter input
    in terms of the stuffed SCS problem

    Parameters
    ----------
    prox_prob: CVXPY prox problem
    x0_vars: dict
        dictionary of x0 keys and __tau that point to cvxpy Parameter objects

    Returns
    -------
    dict
        mapping of variable names (including '__tau') to indices of the b, c vectors in
        stuffed SCS problem

    Notes
    -----
    xs get multiplied by -2.
    expect '__tau' to be an array of random locations to c
    expect the other elements to be slice objects of continuous chunks
    """
    # set the x0 parameters to random values, and then find those random values in the vectors
    rand_param_vals(x0_vars)

    data = get_scs_data(pxprob)

    # tau may be mapped to multiple locations in the c vector
    taus, = np.where(data['c']==x0_vars['__tau'].value)

    indmap = {'__tau':taus}

    b = data['b']
    for k in x0_vars:
        if k != '__tau':
            # because x.value sometimes returns a float, and sometimes returns a 2d np.Matrix...
            # this cleans it all up to be a 1d array in either case
            x = np.atleast_1d(np.squeeze(np.array(x0_vars[k].value)))
            ind, = np.where(b==-2*x[0])
            ind = ind[0]

            indmap[k] = slice(ind,ind+len(x))

    return data, indmap

def restuff(data, indmap, x0_vals):
    """ Modify the b,c data in `data` to reflect the x0 prox values
    in x0_vals (which should be appropriately sized numpy arrays or scalars).

    indmap is the mapping from variable names to b,c indices

    Notes
    -----
    tau maps to tau in c, but x maps to -2*x in b.
    """

    c = data['c']
    c[indmap['__tau']] = x0_vals['__tau']

    b = data['b']
    for k in x0_vals:
        if k != '__tau':
            b[indmap[k]] = -2*x0_vals[k]

def dummy_scs_output(data):
    """ `data` is a dict of SCS input data
    """
    x = np.random.randn(*data['c'].shape)
    y = np.random.randn(*data['b'].shape)

    out = {'info': {'dobj': 0,
          'iter': 0,
          'pobj': 0,
          'relGap': 0,
          'resDual': 0,
          'resInfeas': 0,
          'resPri': 0,
          'resUnbdd': 0,
          'setupTime': 0,
          'solveTime': 0,
          'status': 'Solved',
          'statusVal': 1},
         'x': x,
         'y': y,
         's': y}

    return out

def get_solmap(prob, x_vars, data=None):
    """
    Parameters
    ----------
    x_vars: dict
        k:v pairs, where v is a CVXPY Variable

    Returns
    -------
    dict
        elements are Python `slice` objects.
    """
    if data is None:
        data = get_scs_data(prob)

    out = dummy_scs_output(data)

    prob.unpack_results('SCS', out)

    solmap = {}
    for k in x_vars:
        x = np.atleast_1d(np.squeeze(np.array(x_vars[k].value)))
        ind, = np.where(x[0]==out['x'])
        ind = ind[0]
        solmap[k] = slice(ind,ind+len(x))

    return solmap

def extract_sol(scs_x, solmap):
    """ Extract a solution from the SCS output variable `x`.
    solmap is a dict mapping variable names to indices (slices) of x.
    """
    x_vals = {}
    for k in solmap:
        x_vals[k] = scs_x[solmap[k]]
        if len(x_vals[k]) == 1:
            x_vals[k] = x_vals[k][0]

    return x_vals


import discord
import json
import os
import requests
from discord.ext import tasks
from datetime import datetime
from stayonline import keep_alive

data = open('bdays.json')
check = (json.load(data))

intents = discord.Intents.default()
intents.message_content = True
client = discord.Client(intents=intents)

@client.event
async def on_ready():
    print('Logged in as {0.user}', format(client))
    send_message.start()

#date and loop part start here:
@tasks.loop(hours=3)
async def send_message():
    now = datetime.now()
    tdate = now.strftime("%d-%m")

    for i in range(0, 30):
        if check[i]['Bday'] == tdate:
            channel = client.get_channel(1014507327497437267)
            await channel.send(
                f"@everyone Its {check[i]['Name']} ,{check[i]['Roll']}'s Birthday"
            )

#chat part start here:
@client.event
async def on_message(message):
    if message.author == client.user:
        return
    if message.author != client.user:
        if str(message.content) == "hello":
            print(str(message))
            await message.channel.send("works")
            return
        if str(message.content) == "/json":
          await message.channel.send(check)
          return
        if str(message.content) == "/joke" or "tell me a joke":
          req = requests.get("https://v2.jokeapi.dev/joke/Programming")
          joke = req.json()
          if joke['type'] == 'twopart':
            await message.channel.send(joke['setup'])
            await message.channel.send(joke['delivery'])
          else:
            await message.channel.send(joke['joke'])
keep_alive()
token = os.environ.get('code')
client.run(token)


from flask import json

class Data:

    def __init__(self, modules):
        self.modules = modules

    def check_year(self, init, end):
        if init is None or end is None:
            return False
        return True

    def generate_year(self, init, end):
        init_year = int(init)
        end_year = int(end)

        list_year = []
        while (init_year <= end_year):
            list_year.append(init_year)
            init_year = init_year + 1

        return list_year

    def generate_period(self, init, end):
        init_year = int(init)
        end_year = int(end)

        list_period = []
        while (init_year < end_year):
            next_year = init_year + 1
            list_period.append([init_year, next_year])
            init_year = init_year + 1

        return list_period

    def iszero(self, a):
        return abs(a)<1e-9

    def calculate_gcf(self, a, b):
        if self.iszero(b):
            return a

        return self.calculate_gcf(b, a % b)

    def retrieve(self, params):
        init_year = params['request'].args.get('init')
        end_year = params['request'].args.get('end')
        query_type = params['type']

        if not self.check_year(init_year, end_year):
            return 'Please enter `init` and `end` as parameters.'

        list_year = self.generate_year(init_year, end_year)
        list_income_data = self.modules['income'].get_data(query_type, init_year, end_year)
        list_wealth_data = self.modules['wealth'].get_data(query_type, init_year, end_year)

        list_income = []
        list_wealth = []

        for item in list_income_data:
            list_income.append(item[1])

        for item in list_wealth_data:
            list_wealth.append(item[1])

        res_json = {
            'year': list_year,
            'income': list_income,
            'wealth': list_wealth
        }

        return json.dumps(res_json)

    def inequality(self, params):
        init_year = params['request'].args.get('init')
        end_year = params['request'].args.get('end')
        query_type = params['type']

        if not self.check_year(init_year, end_year):
            return 'Please enter `init` and `end` as parameters.'

        list_year = self.generate_year(init_year, end_year)

        list_data = []
        list_data = self.modules[query_type].get_data('all', init_year, end_year)

        factor = []
        for item in list_data:
            factor.append(self.calculate_gcf(float(item[2]), float(item[1])))

        res_json = {
            'year': list_year,
            'factor': factor
        }

        return json.dumps(res_json)

    def saving_capacity(self, params):
        request_data = params['request'].json
        init_year = request_data[u'init']
        end_year = request_data[u'end']
        group = request_data[u'group']

        if not self.check_year(init_year, end_year):
            return 'Please enter `init` and `end` as parameters.'

        list_data = self.modules['general'].get_data(init_year, end_year)
        list_data_temp = {}

        for item in list_data:
            list_data_temp[item[1]] = item

        list_period = self.generate_period(init_year, end_year)
        list_saving_capacity = []

        for period in list_period:
            x = period[0]
            y = period[1]
            total = 0

            if group == 10 or group == 50:
                total = (list_data_temp[y][3] - list_data_temp[x][3]) / list_data_temp[x][2]
            else:
                total = (list_data_temp[y][5] - list_data_temp[x][5]) / list_data_temp[x][4]

            list_saving_capacity.append(float(total))

        res_json = {
            'group': group,
            'period': list_period,
            'saving_capacity': list_saving_capacity
        }

        return json.dumps(res_json)

    def prediction(self, params):
        request_data = params['request'].json
        n_year = request_data[u'years']
        group = request_data[u'group']
        query_type = params['type']

        list_data = self.modules[query_type].get_data(group)

        values = []
        for item in list_data:
            values.append((item[0], float(item[1])))

        coef = params['regression'].calculate(values)

        i = 1
        prediction = []
        while (i <= n_year):
            year = 2014 + i
            p = params['regression'].predict(year, coef[u'm'], coef[u'b'])
            prediction.append(p)
            i = i + 1

        res_json = {
            'group': group,
            'prediction': prediction
        }

        return json.dumps(res_json)


# -*- coding: utf-8 -*-

### Import required python modules
import logging

from gevent import monkey; monkey.patch_all()
import platform
import os
from os import listdir, stat, makedirs, mkdir, walk, remove, pardir
from os.path import isdir, isfile, join, splitext, getmtime, basename, normpath, exists, expanduser, split, dirname, getsize, abspath
import pandas as pd
import time
from time import strftime, localtime
import shutil
from shutil import copy2
from configparser import ConfigParser
import numpy as np
from collections import defaultdict
import subprocess
from websocket import create_connection
import socket
import errno
import re
import gevent
from blackfynn import Blackfynn
from blackfynn.log import get_logger
from blackfynn.api.agent import agent_cmd, validate_agent_installation
from blackfynn.api.agent import AgentError, check_port, socket_address
from blackfynn import Settings
from urllib.request import urlopen
import json
import collections
from threading import Thread
import pathlib

from openpyxl import load_workbook
from openpyxl import Workbook
from openpyxl.styles import PatternFill, Font
from docx import Document

from string import ascii_uppercase
import itertools

from datetime import datetime, timezone

### Global variables
curateprogress = ' '
curatestatus = ' '
curateprintstatus = ' '
total_dataset_size = 1
curated_dataset_size = 0
start_time = 0

userpath = expanduser("~")
configpath = join(userpath, '.blackfynn', 'config.ini')
submitdataprogress = ' '
submitdatastatus = ' '
submitprintstatus = ' '
total_file_size = 1
uploaded_file_size = 0
start_time_bf_upload = 0
start_submit = 0
metadatapath = join(userpath, 'SODA', 'SODA_metadata')

bf = ""
myds = ""
initial_bfdataset_size = 0
upload_directly_to_bf = 0
initial_bfdataset_size_submit = 0

forbidden_characters = '<>:"/\|?*'
forbidden_characters_bf = '\/:*?"<>'

DEV_TEMPLATE_PATH = join(dirname(__file__), "..", "file_templates")

# once pysoda has been packaged with pyinstaller
# it becomes nested into the pysodadist/api directory
PROD_TEMPLATE_PATH = join(dirname(__file__), "..", "..", "file_templates")
TEMPLATE_PATH = DEV_TEMPLATE_PATH if exists(DEV_TEMPLATE_PATH) else PROD_TEMPLATE_PATH

logging.basicConfig(level=logging.DEBUG, filename=os.path.join(os.path.expanduser("~"), f"{__name__}.log"))
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
handler = logging.FileHandler(os.path.join(os.path.expanduser("~"), f"{__name__}.log"))
handler.setLevel(logging.DEBUG)
logger.addHandler(handler)

### Manage datasets (Blackfynn interface)
def bf_add_account(keyname, key, secret):
    """
    Associated with 'Add account' button in 'Login to your Blackfynn account' section of SODA

    Args:
        keyname: Name of the account to be associated with the given credentials (string)
        key: API key (string)
        secret: API Secret (string)
    Action:
        Adds account to the Blackfynn configuration file (local machine)
    """
    try:
        error, c = '', 0
        keyname = keyname.strip()
        if (not keyname) or (not key) or (not secret):
            raise Exception('Error: Please enter valid keyname, key, and/or secret')

        if (keyname.isspace()) or (key.isspace()) or (secret.isspace()):
            raise Exception('Error: Please enter valid keyname, key, and/or secret')

        bfpath = join(userpath, '.blackfynn')
        # Load existing or create new config file
        config = ConfigParser()
        if exists(configpath):
            config.read(configpath)
            if config.has_section(keyname):
                raise Exception('Error: Key name already exists')
        else:
            if not exists(bfpath):
                mkdir(bfpath)
            if not exists(join(bfpath, 'cache')):
                mkdir(join(bfpath, 'cache'))

        # Add agent section
        agentkey = 'agent'
        if not config.has_section(agentkey):
            config.add_section(agentkey)
            config.set(agentkey, 'proxy_local_port', '8080')
            # config.set(agentkey, 'cache_base_path', join(bfpath, 'cache'))
            config.set(agentkey, 'uploader', 'true')
            config.set(agentkey, 'cache_hard_cache_size', '10000000000')
            config.set(agentkey, 'status_port', '11235')
            config.set(agentkey, 'metrics', 'true')
            config.set(agentkey, 'cache_page_size', '100000')
            config.set(agentkey, 'proxy', 'true')
            config.set(agentkey, 'cache_soft_cache_size', '5000000000')
            config.set(agentkey, 'timeseries_local_port', '9090')
            config.set(agentkey, 'timeseries', 'true')

        # Add new account
        config.add_section(keyname)
        config.set(keyname, 'api_token', key)
        config.set(keyname, 'api_secret', secret)

        with open(configpath, 'w') as configfile:
            config.write(configfile)

    except Exception as e:
        raise e

    # Check key and secret are valid, if not delete account from config
    try:
        bf = Blackfynn(keyname)

        if not config.has_section("global"):
            config.add_section("global")

        default_acc = config["global"]
        default_acc["default_profile"] = keyname

        with open(configpath, 'w') as configfile:
            config.write(configfile)
        return f'Successfully added account {str(bf)}'

    except:
        bf_delete_account(keyname)
        raise Exception('Authentication Error: please check that key name, key, and secret are entered properly')

def check_forbidden_characters(my_string):
    """
    Check for forbidden characters in file/folder name

    Args:
        my_string: string with characters (string)
    Returns:
        False: no forbidden character
        True: presence of forbidden character(s)
    """
    regex = re.compile('[' + forbidden_characters + ']')
    return regex.search(my_string) is not None or "\\" in r"%r" % my_string

def check_forbidden_characters_bf(my_string):
    """
    Check for forbidden characters in blackfynn file/folder name

    Args:
        my_string: string with characters (string)
    Returns:
        False: no forbidden character
        True: presence of forbidden character(s)
    """
    regex = re.compile('[' + forbidden_characters_bf + ']')
    return regex.search(my_string) is not None or "\\" in r"%r" % my_string

def bf_delete_account(keyname):
    """
    Args:
        keyname: name of local Blackfynn account key (string)
    Action:
        Deletes account information from the Blackfynn config file
    """
    config = ConfigParser()
    config.read(configpath)
    config.remove_section(keyname)
    with open(configpath, 'w') as configfile:
        config.write(configfile)

def bf_account_list():
    """
    Action:
        Returns list of accounts stored in the system
    """
    try:
        accountlist = ['Select']
        if exists(configpath):
            config = ConfigParser()
            config.read(configpath)
            accountname = config.sections()
            accountnamenoglobal = list(accountname)
            # if accountnamenoglobal:
            for n in accountnamenoglobal:
                try:
                    bfn = Blackfynn(n)
                    accountlist.append(n)
                except Exception as e:
                    pass
            with open(configpath, 'w') as configfile:
                config.write(configfile)
        return accountlist
            # My accountlist

    except Exception as e:
        raise e

def bf_default_account_load():
    """
    Action:
        Returns the first valid account as the default account
    """
    try:
        accountlist = []
        if exists(configpath):
            config = ConfigParser()
            config.read(configpath)
            keys = config.sections()
            accountlist = []
            if "global" in keys:
                default_acc = config["global"]
                if "default_profile" in default_acc:
                    n = default_acc["default_profile"]
                    try:
                        bfn = Blackfynn(n)
                        accountlist.append(n)
                    except Exception as e:
                        return accountlist
            # accountnamenoglobal = [n for n in accountname]
            # if accountnamenoglobal:
            #     for n in accountnamenoglobal:
            #         try:
            #             bfn = Blackfynn(n)
            #             accountlist.append(n)
            #             break
            #         except:
            #             pass
            #     with open(configpath, 'w') as configfile:
            #         config.write(configfile)
        return accountlist
    except Exception as e:
        raise e

def bf_dataset_account(accountname):
    """
    This function filters dataset dropdowns across SODA by the permissions granted to users.

    Input: BFaccountname
    Output: a filtered dataset list with objects as elements: {"name": dataset's name, "id": dataset's id, "role": permission}

    """
    # # current_user = bf._api._get('/user')
    # # current_user_id = current_user['id']
    # # for ds in bf.datasets():
    # #     dataset_list.append(ds.name)
    # dataset_list.sort(key=lambda v: v.upper()) # Returning the list of datasets in alphabetical order
    # dataset_list.insert(0, ['Select dataset'])
    # return dataset_list

    bf = Blackfynn(accountname)
    bfaccountname = bf.profile.id
    datasets_list = bf.datasets()

    all_bf_datasets = []

    def filter_dataset(datasets_list, store=None):
        if store is None:
            store = []
        for dataset in datasets_list:
            selected_dataset_id = dataset.id
            user_role = bf._api._get(f'/datasets/{str(selected_dataset_id)}/role')['role']
            store.append({"id": selected_dataset_id, "name": dataset.name, "role": user_role})
        return store

    #filter_dataset(datasets_list)
    store = []
    threads = []
    nthreads = 8
    # create the threads
    for i in range(nthreads):
        sub_datasets_list = datasets_list[i::nthreads]
        t = Thread(target=filter_dataset, args=(sub_datasets_list, store))
        threads.append(t)

    # start the threads
    [ t.start() for t in threads ]
    # wait for the threads to finish
    [ t.join() for t in threads ]

    sorted_bf_datasets = sorted(store, key=lambda k: k['name'].upper())

    return sorted_bf_datasets

def get_username(accountname):
    """
    Input: User's accountname and the name of the selected dataset

    Output: User's name
    """

    bf = Blackfynn(accountname)
    return bf.profile.first_name + " " + bf.profile.last_name

def bf_account_details(accountname):
    """
    Args:
        accountname: name of local Blackfynn account key (string)
    Return:
        acc_details: account user email and organization (string)
    Action:
        Returns: return details of user associated with the account
    """
    try:
        bf = Blackfynn(accountname)
        acc_details = "User email: " + bf.profile.email + "
"
        acc_details = acc_details + "Organization: " + bf.context.name

        if exists(configpath):
            config = ConfigParser()
            config.read(configpath)

        if not config.has_section("global"):
            config.add_section("global")
            config.set("global", "default_profile", accountname)
        else:
            config["global"]["default_profile"] = accountname

        with open(configpath, 'w') as configfile:
            config.write(configfile)

        ## return account details and datasets where such an account has some permission
        return acc_details

    except Exception as e:
        raise e

def bf_new_dataset_folder(datasetname, accountname):
    """
    Associated with 'Create' button in 'Create new dataset folder'

    Args:
        datasetname: name of the dataset to be created (string)
        accountname: account in which the dataset needs to be created (string)
    Action:
        Creates dataset for the account specified
    """
    try:
        error, c = '', 0
        datasetname = datasetname.strip()

        if check_forbidden_characters_bf(datasetname):
            error = (
                f'{error}Error: A Blackfynn dataset name cannot contain any of the following characters: '
                + forbidden_characters_bf
                + "
"
            )

            c += 1

        if (not datasetname):
            error = f'{error}Error: Please enter valid dataset name
'
            c += 1

        if (datasetname.isspace()):
            error = error + 'Error: Please enter valid dataset name' + "
"
            c += 1

        try:
            bf = Blackfynn(accountname)
        except Exception as e:
            error = error + 'Error: Please select a valid Blackfynn account' + "
"
            c += 1

        if c>0:
            raise Exception(error)

        dataset_list = [ds.name for ds in bf.datasets()]
        if datasetname in dataset_list:
            raise Exception('Error: Dataset name already exists')
        else:
            bf.create_dataset(datasetname)

    except Exception as e:
        raise e

def bf_rename_dataset(accountname, current_dataset_name, renamed_dataset_name):
    """
    Args:
        accountname: account in which the dataset needs to be created (string)
        current_dataset_name: current name of the dataset
        renamed_dataset_name: new name of the dataset

    Action:
        Creates dataset for the account specified
    """
    error, c = '', 0
    datasetname = renamed_dataset_name.strip()

    if check_forbidden_characters_bf(datasetname):
        error = (
            f'{error}Error: A Blackfynn dataset name cannot contain any of the following characters: '
            + forbidden_characters_bf
            + "
"
        )

        c += 1

    if (not datasetname):
        error = f'{error}Error: Please enter valid new dataset name
'
        c += 1

    if (datasetname.isspace()):
        error = error + 'Error: Please enter valid new dataset name' + "
"
        c += 1

    try:
        bf = Blackfynn(accountname)
    except Exception as e:
        error = error + 'Error: Please select a valid Blackfynn account' + "
"
        c += 1

    if c>0:
        raise Exception(error)

    try:
        myds = bf.get_dataset(current_dataset_name)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn dataset'
        raise Exception(error)

    try:
        role = bf_get_current_user_permission(bf, myds)
        if role not in ['owner', 'manager']:
            error = "Error: You don't have permissions to change the name of this Blackfynn dataset"
            raise Exception(error)
    except Exception as e:
        raise e

    dataset_list = [ds.name for ds in bf.datasets()]
    if datasetname in dataset_list:
        raise Exception('Error: Dataset name already exists')
    myds = bf.get_dataset(current_dataset_name)
    selected_dataset_id = myds.id
    jsonfile = {'name': datasetname}
    bf._api.datasets._put(f'/{str(selected_dataset_id)}', json=jsonfile)

def clear_queue():

    command = [agent_cmd(), "upload-status", "--cancel-all"]

    return subprocess.run(command, check=True)

def agent_running():
    listen_port = 11235

    try:
        logger.debug("Checking port %s", listen_port)
        create_connection(socket_address(listen_port)).close()

    except socket.error as e:

        if e.errno != errno.ECONNREFUSED:
            raise
        logger.debug("No agent found, port %s OK", listen_port)
        return True
    else:
        raise AgentError("The Blackfynn agent is already running. Please go to your Task Manager/Activity Monitor to stop any running blackfynn_agent processes and try again")

def bf_submit_dataset(accountname, bfdataset, pathdataset):
    """
    Associated with 'Submit dataset' button in 'Submit new dataset' section
    Uploads the specified folder to the specified dataset on Blackfynn account

    Input:
        accountname: account in which the dataset needs to be created (string)
        bfdataset: name of the dataset on Blackfynn (string)
        pathdataset: path of dataset on local machine (string)
    Action:
        Uploads dataset on Blackfynn account
    """
    global submitdataprogress
    global submitdatastatus
    global total_file_size
    global uploaded_file_size
    global submitprintstatus
    global start_time_bf_upload
    global bf
    global myds
    global start_submit
    global initial_bfdataset_size_submit

    submitdataprogress = ' '
    submitdatastatus = ' '
    uploaded_file_size = 0
    submitprintstatus = ' '
    start_time_bf_upload = 0
    initial_bfdataset_size_submit = 0
    start_submit = 0

    try:
        bf = Blackfynn(accountname)
    except Exception as e:
        submitdatastatus = 'Done'
        error = 'Error: Please select a valid Blackfynn account'
        raise Exception(error)

    error, c = '', 0
    try:
        myds = bf.get_dataset(bfdataset)
    except Exception as e:
        submitdatastatus = 'Done'
        error = error + 'Error: Please select a valid Blackfynn dataset' + '
'
        c += 1

    if not isdir(pathdataset):
        submitdatastatus = 'Done'
        error = error + 'Error: Please select a valid local dataset folder' + '
'
        c += 1

    if c>0:
        raise Exception(error)

    error, c = '', 0
    total_file_size = 1
    try:
        for path, dirs, files in walk(pathdataset):
            for f in files:
                fp = join(path, f)
                mypathsize = getsize(fp)
                if mypathsize == 0:
                    c += 1
                    error = error + fp + ' is 0 KB
'
                else:
                    total_file_size += mypathsize
            for d in dirs:
                dp = join(path,d)
                myfoldersize = folder_size(dp)
                if myfoldersize == 0:
                    c += 1
                    error = error + dp + ' is empty
'
    except Exception as e:
        raise e

    if c>0:
        submitdatastatus = 'Done'
        error = error + '
Please remove invalid files/folders from your dataset before uploading'
        raise Exception(error)

    total_file_size = total_file_size - 1

    role = bf_get_current_user_permission(bf, myds)
    if role not in ['owner', 'manager', 'editor']:
        submitdatastatus = 'Done'
        error = "Error: You don't have permissions for uploading to this Blackfynn dataset"
        raise Exception(error)

    ## check if agent is installed
    try:
        validate_agent_installation(Settings())
    except AgentError:
        raise AgentInstallationError('The Blackfynn agent is not installed on your computer. Visit the Blackfynn Agent website at "https://developer.blackfynn.io/agent" for installation instructions.')

    clear_queue()
    try:
        ## check if agent is running in the background
        agent_running()
        def calluploadfolder():

            try:

                global submitdataprogress
                global submitdatastatus

                myds = bf.get_dataset(bfdataset)

                for filename in listdir(pathdataset):
                    filepath = join(pathdataset, filename)
                    if isdir(filepath):
                        submitdataprogress = "Uploading folder '%s' to dataset '%s \n' " %(filepath, bfdataset)
                        myds.upload(filepath, recursive=True, use_agent=True)
                    else:
                        submitdataprogress = "Uploading file '%s' to dataset '%s \n' " %(filepath, bfdataset)
                        myds.upload(filepath, use_agent=True)
                submitdataprogress = 'Success: COMPLETED!'
                submitdatastatus = 'Done'

            except Exception as e:
                raise e

        submitprintstatus = 'Uploading'
        start_time_bf_upload = time.time()
        initial_bfdataset_size_submit = bf_dataset_size()
        start_submit = 1
        gev = []
        gev.append(gevent.spawn(calluploadfolder))
        gevent.sleep(0)
        gevent.joinall(gev)
        submitdatastatus = 'Done'

        try:
            return gev[0].get()
        except Exception as e:
            raise e

    except Exception as e:
        submitdatastatus = 'Done'
        raise e

def submit_dataset_progress():
    """
    Keeps track of the dataset submission progress
    """
    global submitdataprogress
    global submitdatastatus
    global submitprintstatus
    global total_file_size
    global uploaded_file_size
    global start_time_bf_upload
    global start_submit
    global initial_bfdataset_size_submit

    if start_submit == 1:
        uploaded_file_size = bf_dataset_size() - initial_bfdataset_size_submit
        elapsed_time = time.time() - start_time_bf_upload
        elapsed_time_formatted = time_format(elapsed_time)
        elapsed_time_formatted_display = '
' + 'Elapsed time: ' + elapsed_time_formatted + '
'
    else:
        uploaded_file_size = 0
        elapsed_time_formatted = 0
        elapsed_time_formatted_display = '
' + 'Initiating...' + '
'
    #gevent.sleep(0)
    return (submitdataprogress + elapsed_time_formatted_display, submitdatastatus, submitprintstatus, total_file_size, uploaded_file_size, elapsed_time_formatted)

def bf_get_users(selected_bfaccount):
    """
    Function to get list of users belonging to the organization of
    the given Blackfynn account

    Args:
      selected_bfaccount: name of selected Blackfynn acccount (string)
    Retun:
        list_users : list of users (first name -- last name) associated with the organization of the
        selected Blackfynn account (list of string)
    """
    try:
        # def get_users_list():
        bf = Blackfynn(selected_bfaccount)
        organization_name = bf.context.name
        organization_id = bf.context.id
        list_users = bf._api._get(f'/organizations/{str(organization_id)}/members')
        list_users_first_last = []
        for i in range(len(list_users)):
                first_last = list_users[i]['firstName'] + ' ' + list_users[i]['lastName']
                list_users_first_last.append(first_last)
        list_users_first_last.sort() # Returning the list of users in alphabetical order
        return list_users_first_last
            # list_users_first_last = gevent.spawn(get_users_list())
            # gevent.sleep(0)
            # return list_users_first_last
    except Exception as e:
        raise e

def bf_get_teams(selected_bfaccount):
    """
    Args:
      selected_bfaccount: name of selected Blackfynn acccount (string)
    Return:
        list_teams : list of teams (name) associated with the organization of the
        selected Blackfynn account (list of string)
    Action:
        Provides list of teams belonging to the organization of
        the given Blackfynn account
    """
    try:
        bf = Blackfynn(selected_bfaccount)
        organization_name = bf.context.name
        organization_id = bf.context.id
        list_teams = bf._api._get(f'/organizations/{str(organization_id)}/teams')
        list_teams_name = [
            list_teams[i]['team']['name'] for i in range(len(list_teams))
        ]

        list_teams_name.sort() # Returning the list of teams in alphabetical order
        return list_teams_name
    except Exception as e:
        raise e

def bf_get_permission(selected_bfaccount, selected_bfdataset):

    """
    Function to get permission for a selected dataset

    Args:
        selected_bfaccount: name of selected Blackfynn acccount (string)
        selected_bfdataset: name of selected Blackfynn dataset (string)
    Output:
        list_permission: list of permission (first name -- last name -- role) associated with the
        selected dataset (list of string)
    """

    try:
        bf = Blackfynn(selected_bfaccount)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn account'
        raise Exception(error)

    try:
        myds = bf.get_dataset(selected_bfdataset)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn dataset' + '
'
        raise Exception(error)

    try:
        # user permissions
        selected_dataset_id = myds.id
        list_dataset_permission = bf._api._get(
            f'/datasets/{str(selected_dataset_id)}/collaborators/users'
        )

        list_dataset_permission_first_last_role = []
        for i in range(len(list_dataset_permission)):
            first_name = list_dataset_permission[i]['firstName']
            last_name = list_dataset_permission[i]['lastName']
            role = list_dataset_permission[i]['role']
            list_dataset_permission_first_last_role.append('User: ' +  first_name + ' ' + last_name + ' , role: ' + role)

        # team permissions
        list_dataset_permission_teams = bf._api._get(
            f'/datasets/{str(selected_dataset_id)}/collaborators/teams'
        )

        for i in range(len(list_dataset_permission_teams)):
            team_keys = list(list_dataset_permission_teams[i].keys())
            if 'role' in team_keys:
                team_name = list_dataset_permission_teams[i]['name']
                team_role = list_dataset_permission_teams[i]['role']
                list_dataset_permission_first_last_role.append('Team: ' + team_name + ', role: ' + team_role)

        # Organization permissions
        list_dataset_permission_organizations = bf._api._get(
            f'/datasets/{str(selected_dataset_id)}/collaborators/organizations'
        )

        if type(list_dataset_permission_organizations) is dict:
                organization_keys = list(list_dataset_permission_organizations.keys())
                if 'role' in organization_keys:
                    organization_name = list_dataset_permission_organizations['name']
                    organization_role = list_dataset_permission_organizations['role']
                    list_dataset_permission_first_last_role.append('Organization: ' + organization_name + ', role: ' + organization_role)
        else:
            for i in range(len(list_dataset_permission_organizations)):
                organization_keys = list(list_dataset_permission_organizations[i].keys())
                if 'role' in organization_keys:
                    organization_name = list_dataset_permission_organizations[i]['name']
                    organization_role = list_dataset_permission_organizations[i]['role']
                    list_dataset_permission_first_last_role.append('Organization: ' + organization_name + ', role: ' + organization_role)

        return list_dataset_permission_first_last_role

    except Exception as e:
        raise e

def bf_get_current_user_permission(bf, myds):

    """
    Function to get the permission of currently logged in user for a selected dataset

    Args:
        bf: logged Blackfynn acccount (dict)
        myds: selected Blackfynn dataset (dict)
    Output:
        permission of current user (string)
    """

    try:
        selected_dataset_id = myds.id
        return bf._api._get(f'/datasets/{str(selected_dataset_id)}/role')['role']

    except Exception as e:
        raise e

def bf_add_permission(selected_bfaccount, selected_bfdataset, selected_user, selected_role):

    """
    Function to add/remove permission for a suser to a selected dataset

    Args:
        selected_bfaccount: name of selected Blackfynn acccount (string)
        selected_bfdataset: name of selected Blackfynn dataset (string)
        selected_user: name (first name -- last name) of selected Blackfynn user (string)
        selected_role: desired role ('manager', 'viewer', 'editor', 'remove current permissions') (string)
    Return:
        success or error message (string)
    """

    error = ''
    try:
        bf = Blackfynn(selected_bfaccount)
    except Exception as e:
        error += 'Error: Please select a valid Blackfynn account'
        raise Exception(error)

    c = 0
    try:
        myds = bf.get_dataset(selected_bfdataset)
    except Exception as e:
        error = f'{error}Error: Please select a valid Blackfynn dataset
'
        c += 1

    try:
        organization_name = bf.context.name
        organization_id = bf.context.id
        list_users = bf._api._get(f'/organizations/{str(organization_id)}/members')
        dict_users = {}
        list_users_firstlast = []
        for i in range(len(list_users)):
            list_users_firstlast.append(list_users[i]['firstName'] + ' ' + list_users[i]['lastName'] )
            dict_users[list_users_firstlast[i]] = list_users[i]['id']
        if selected_user not in list_users_firstlast:
            error = error + 'Error: Please select a valid user' + '
'
            c += 1
    except Exception as e:
        raise e

    if selected_role not in ['manager', 'viewer', 'editor', 'owner', 'remove current permissions']:
        error = error + 'Error: Please select a valid role' + '
'
        c += 1

    if c > 0:
        raise Exception(error)
    try:
        selected_dataset_id = myds.id
        selected_user_id = dict_users[selected_user]

        # check that currently logged in user is a manager or a owner of the selected dataset (only manager and owner can change dataset permission)
        current_user = bf._api._get('/user')
        first_name_current_user = current_user['firstName']
        last_name_current_user = current_user['lastName']
        list_dataset_permission = bf._api._get(
            f'/datasets/{str(selected_dataset_id)}/collaborators/users'
        )

        c = 0
        for i in range(len(list_dataset_permission)):
            first_name = list_dataset_permission[i]['firstName']
            last_name = list_dataset_permission[i]['lastName']
            role = list_dataset_permission[i]['role']
            user_id = list_dataset_permission[i]['id']
            if role not in ['owner', 'manager']:
                if (first_name == first_name_current_user and last_name == last_name_current_user):
                    raise Exception('Error: you must be dataset owner or manager to change its permissions')
            elif selected_role == 'owner' and role != 'owner':
                if (first_name == first_name_current_user and last_name == last_name_current_user):
                    raise Exception('Error: you must be dataset owner to change the ownership')
            elif (first_name == first_name_current_user and last_name == last_name_current_user):
                c += 1
            #check if selected user is owner, dataset permission cannot be changed for owner
            if user_id == selected_user_id and role == 'owner':
                raise Exception("Error: owner's permission cannot be changed")

        if (c == 0):
            raise Exception('Error: you must be dataset owner or manager to change its permissions')

        if (selected_role == 'remove current permissions'):

            bf._api.datasets._del(
                f'/{str(selected_dataset_id)}'
                + '/collaborators/users'.format(
                    dataset_id=selected_dataset_id
                ),
                json={'id': selected_user_id},
            )

            return "Permission removed for " + selected_user
        elif selected_role == 'owner':
                #check if currently logged in user is owner of selected dataset (only owner can change owner)

                # change owner
            bf._api.datasets._put(
                f'/{str(selected_dataset_id)}'
                + '/collaborators/owner'.format(
                    dataset_id=selected_dataset_id
                ),
                json={'id': selected_user_id},
            )

            return "Permission " + "'" + selected_role + "' " +  " added for " + selected_user
        else:
            bf._api.datasets._put(
                f'/{str(selected_dataset_id)}'
                + '/collaborators/users'.format(
                    dataset_id=selected_dataset_id
                ),
                json={'id': selected_user_id, 'role': selected_role},
            )

            return "Permission " + "'" + selected_role + "' " +  " added for " + selected_user
    except Exception as e:
            raise e

def bf_add_permission_team(selected_bfaccount, selected_bfdataset, selected_team, selected_role):

    """
    Function to add/remove permission fo a team to a selected dataset

    Args:
        selected_bfaccount: name of selected Blackfynn acccount (string)
        selected_bfdataset: name of selected Blackfynn dataset (string)
        selected_team: name of selected Blackfynn team (string)
        selected_role: desired role ('manager', 'viewer', 'editor', 'remove current permissions') (string)
    Return:
        success or error message (string)
    """

    error = ''

    try:
        bf = Blackfynn(selected_bfaccount)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn account'
        raise Exception(error)

    try:
        if (
            selected_team == 'SPARC Data Curation Team'
        ) and bf.context.name != 'SPARC Consortium':
            raise Exception('Error: Please login under the Blackfynn SPARC Consortium organization to share with the Curation Team')
        if (
            selected_team == 'SPARC Embargoed Data Sharing Group'
        ) and bf.context.name != 'SPARC Consortium':
            raise Exception('Error: Please login under the Blackfynn SPARC Consortium organization to share with the SPARC consortium group')
    except Exception as e:
        raise e

    c = 0

    try:
        myds = bf.get_dataset(selected_bfdataset)
    except Exception as e:
        error = f'{error}Error: Please select a valid Blackfynn dataset
'
        c += 1

    try:
        organization_name = bf.context.name
        organization_id = bf.context.id
        list_teams = bf._api._get(f'/organizations/{str(organization_id)}/teams')
        dict_teams = {}
        list_teams_name = []
        for i in range(len(list_teams)):
                list_teams_name.append(list_teams[i]['team']['name'])
                dict_teams[list_teams_name[i]] = list_teams[i]['team']['id']
        if selected_team not in list_teams_name:
           error = error + 'Error: Please select a valid team' + '
'
           c += 1
    except Exception as e:
        raise e

    if selected_role not in ['manager', 'viewer', 'editor', 'remove current permissions']:
        error = error + 'Error: Please select a valid role' + '
'
        c += 1

    if c > 0:
        raise Exception(error)

    try:
        selected_dataset_id = myds.id
        selected_team_id = dict_teams[selected_team]

        # check that currently logged in user is a manager or a owner of the selected dataset (only manager and owner can change dataset permission)
        current_user = bf._api._get('/user')
        first_name_current_user = current_user['firstName']
        last_name_current_user = current_user['lastName']
        list_dataset_permission = bf._api._get(
            f'/datasets/{str(selected_dataset_id)}/collaborators/users'
        )

        c = 0
        for i in range(len(list_dataset_permission)):
            first_name = list_dataset_permission[i]['firstName']
            last_name = list_dataset_permission[i]['lastName']
            role = list_dataset_permission[i]['role']
            if (first_name == first_name_current_user and last_name == last_name_current_user):
                if role not in ['owner', 'manager']:
                    user_id = list_dataset_permission[i]['id']
                    raise Exception('Error: you must be dataset owner or manager to change its permissions')
                else:
                    c += 1
        if (c == 0):
            raise Exception('Error: you must be dataset owner or manager to change its permissions')

        if (selected_role == 'remove current permissions'):

            bf._api.datasets._del(
                f'/{str(selected_dataset_id)}'
                + '/collaborators/teams'.format(
                    dataset_id=selected_dataset_id
                ),
                json={'id': selected_team_id},
            )

            return "Permission removed for " + selected_team
        else:
            bf._api.datasets._put(
                f'/{str(selected_dataset_id)}'
                + '/collaborators/teams'.format(
                    dataset_id=selected_dataset_id
                ),
                json={'id': selected_team_id, 'role': selected_role},
            )

            return "Permission " + "'" + selected_role + "' " +  " added for " + selected_team
    except Exception as e:
            raise e

"""
    Function to get current subtitle associated with a selected dataset

    Args:
        selected_bfaccount: name of selected Blackfynn acccount (string)
        selected_bfdataset: name of selected Blackfynn dataset (string)
    Return:
        License name, if any, or "No license" message
    """
def bf_get_subtitle(selected_bfaccount, selected_bfdataset):

    try:
        bf = Blackfynn(selected_bfaccount)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn account'
        raise Exception(error)

    try:
        myds = bf.get_dataset(selected_bfdataset)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn dataset'
        raise Exception(error)

    try:
        selected_dataset_id = myds.id
        dataset_info = bf._api._get(f'/datasets/{str(selected_dataset_id)}')
        return dataset_info['content']['description']
    except Exception as e:
        raise Exception(e)

"""
    Args:
        selected_bfaccount: name of selected Blackfynn acccount (string)
        selected_bfdataset: name of selected Blackfynn dataset (string)
        input_subtitle: subtitle limited to 256 characters (string)
    Action:
        Add/change subtitle for a selected dataset
    Return:
        Success messsge or error
    """
def bf_add_subtitle(selected_bfaccount, selected_bfdataset, input_subtitle):

    try:
        bf = Blackfynn(selected_bfaccount)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn account'
        raise Exception(error)

    try:
        myds = bf.get_dataset(selected_bfdataset)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn dataset'
        raise Exception(error)

    try:
        role = bf_get_current_user_permission(bf, myds)
        if role not in ['owner', 'manager']:
            error = "Error: You don't have permissions for editing metadata on this Blackfynn dataset"
            raise Exception(error)
    except Exception as e:
        raise e

    try:
        selected_dataset_id = myds.id
        jsonfile = {'description': input_subtitle}
        bf._api.datasets._put(f'/{str(selected_dataset_id)}', json=jsonfile)
        return 'Subtitle added!'
    except Exception as e:
        raise Exception(e)

"""
    Function to get current description associated with a selected dataset

    Args:
        selected_bfaccount: name of selected Blackfynn acccount (string)
        selected_bfdataset: name of selected Blackfynn dataset (string)
    Return:
        Description (string with markdown code)
    """
def bf_get_description(selected_bfaccount, selected_bfdataset):

    try:
        bf = Blackfynn(selected_bfaccount)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn account'
        raise Exception(error)

    try:
        myds = bf.get_dataset(selected_bfdataset)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn dataset'
        raise Exception(error)

    try:
        selected_dataset_id = myds.id
        dataset_readme_info = bf._api._get(
            f'/datasets/{str(selected_dataset_id)}/readme'
        )

        return dataset_readme_info['readme']
    except Exception as e:
        raise Exception(e)

"""
    Args:
        selected_bfaccount: name of selected Blackfynn acccount (string)
        selected_bfdataset: name of selected Blackfynn dataset (string)
        markdown_input: description with markdown formatting (string)
    Action:
        Add/change desciption for a selected dataset
    Return:
        Success messsge or error
    """
def bf_add_description(selected_bfaccount, selected_bfdataset, markdown_input):

    try:
        bf = Blackfynn(selected_bfaccount)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn account'
        raise Exception(error)

    try:
        myds = bf.get_dataset(selected_bfdataset)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn dataset'
        raise Exception(error)

    try:
        role = bf_get_current_user_permission(bf, myds)
        if role not in ['owner', 'manager']:
            error = "Error: You don't have permissions for editing metadata on this Blackfynn dataset"
            raise Exception(error)
    except Exception as e:
        raise e

    try:
        selected_dataset_id = myds.id
        jsonfile = {'readme': markdown_input}
        bf._api.datasets._put(f'/{str(selected_dataset_id)}/readme', json=jsonfile)
        return 'Description added!'
    except Exception as e:
        raise Exception(e)

"""
    Function to get url of current banner image associated with a selected dataset

    Args:
        selected_bfaccount: name of selected Blackfynn acccount (string)
        selected_bfdataset: name of selected Blackfynn dataset (string)
    Return:
        url of banner image (string)
    """
def bf_get_banner_image(selected_bfaccount, selected_bfdataset):

    try:
        bf = Blackfynn(selected_bfaccount)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn account'
        raise Exception(error)

    try:
        myds = bf.get_dataset(selected_bfdataset)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn dataset'
        raise Exception(error)

    try:
        selected_dataset_id = myds.id
        dataset_banner_info = bf._api._get(
            f'/datasets/{str(selected_dataset_id)}/banner'
        )

        list_keys = dataset_banner_info.keys()
        return (
            dataset_banner_info['banner']
            if 'banner' in list_keys
            else 'No banner image'
        )

    except Exception as e:
        raise Exception(e)

"""
    Function to add banner to a selected dataset

    Args:
        selected_bfaccount: name of selected Blackfynn acccount (string)
        selected_bfdataset: name of selected Blackfynn dataset (string)
        selected_banner_image: name of selected Blackfynn dataset (data-uri)
    Return:
        Success or error message
    """
def bf_add_banner_image(selected_bfaccount, selected_bfdataset, banner_image_path):

    try:
        bf = Blackfynn(selected_bfaccount)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn account'
        raise Exception(error)

    try:
        myds = bf.get_dataset(selected_bfdataset)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn dataset'
        raise Exception(error)

    try:
        role = bf_get_current_user_permission(bf, myds)
        if role not in ['owner', 'manager']:
            error = "Error: You don't have permissions for editing metadata on this Blackfynn dataset"
            raise Exception(error)
    except Exception as e:
        raise Exception(error)

    try:
        selected_dataset_id = myds.id
        def upload_image():
            with open(banner_image_path, "rb") as f:
                bf._api._put(
                    f'/datasets/{str(selected_dataset_id)}/banner', files={"banner": f}
                )
        #delete banner image folder if it is located in SODA
        gevent.spawn(upload_image())
        image_folder = dirname(banner_image_path)
        if isdir(image_folder) and ('SODA' in image_folder):
            shutil.rmtree(image_folder, ignore_errors=True)
        return('Uploaded!')
    except Exception as e:
        raise Exception(e)

"""
    Function to get current license associated with a selected dataset

    Args:
        selected_bfaccount: name of selected Blackfynn acccount (string)
        selected_bfdataset: name of selected Blackfynn dataset (string)
    Return:
        License name, if any, or "No license" message
    """
def bf_get_license(selected_bfaccount, selected_bfdataset):

    try:
        bf = Blackfynn(selected_bfaccount)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn account'
        raise Exception(error)

    try:
        myds = bf.get_dataset(selected_bfdataset)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn dataset'
        raise Exception(error)

    try:
        selected_dataset_id = myds.id
        dataset_info = bf._api._get(f'/datasets/{str(selected_dataset_id)}')
        list_keys = dataset_info['content'].keys()
        return (
            dataset_info['content']['license']
            if 'license' in list_keys
            else 'No license is currently assigned to this dataset'
        )

    except Exception as e:
        raise Exception(e)

"""
    Args:
        selected_bfaccount: name of selected Blackfynn acccount (string)
        selected_bfdataset: name of selected Blackfynn dataset (string)
        selected_license: name of selected license (string)
    Action:
        Add/change license for a selected dataset
    Return:
        Success messsge or error
    """
def bf_add_license(selected_bfaccount, selected_bfdataset, selected_license):

    try:
        bf = Blackfynn(selected_bfaccount)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn account'
        raise Exception(error)

    try:
        myds = bf.get_dataset(selected_bfdataset)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn dataset'
        raise Exception(error)

    try:
        role = bf_get_current_user_permission(bf, myds)
        if role not in ['owner', 'manager']:
            error = "Error: You don't have permissions for editing metadata on this Blackfynn dataset"
            raise Exception(error)
    except Exception as e:
        raise e

    try:
        allowed_licenses_list = [
            'Community Data License Agreement – Permissive',
            'Community Data License Agreement – Sharing',
            'Creative Commons Zero 1.0 Universal',
            'Creative Commons Attribution',
            'Creative Commons Attribution - ShareAlike',
            'Open Data Commons Open Database',
            'Open Data Commons Attribution',
            'Open Data Commons Public Domain Dedication and License',
            'Apache 2.0',
            'GNU General Public License v3.0',
            'GNU Lesser General Public License',
            'MIT',
            'Mozilla Public License 2.0'
            ]
        if selected_license not in allowed_licenses_list:
            error = 'Error: Please select a valid license'
            raise Exception(error)
        selected_dataset_id = myds.id
        jsonfile = {'license': selected_license}
        bf._api.datasets._put(f'/{str(selected_dataset_id)}', json=jsonfile)
        return 'License added!'
    except Exception as e:
        raise Exception(e)

"""
    Function to get current status for a selected dataset

    Args:
        selected_bfaccount: name of selected Blackfynn acccount (string)
        selected_bfdataset: name of selected Blackfynn dataset (string)
    Return:
        List of available status options for the account (list of string).
        Current dataset status (string)
    """
def bf_get_dataset_status(selected_bfaccount, selected_bfdataset):

    try:
        bf = Blackfynn(selected_bfaccount)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn account'
        raise Exception(error)

    try:
        myds = bf.get_dataset(selected_bfdataset)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn dataset'
        raise Exception(error)

    try:
        #get list of available status options
        organization_id = bf.context.id
        list_status = bf._api._get(
            f'/organizations/{str(organization_id)}/dataset-status'
        )

        #get current status of select dataset
        selected_dataset_id = myds.id
        dataset_current_status = bf._api._get(
            f'/datasets/{str(selected_dataset_id)}'
        )['content']['status']

        return [list_status, dataset_current_status]
    except Exception as e:
        raise e

"""
    Function to get current status for a selected dataset

    Args:
        selected_bfaccount: name of selected Blackfynn acccount (string)
        selected_bfdataset: name of selected Blackfynn dataset (string)
        selected_status: display name of selected status (string)
    Return:
        success message
    """
def bf_change_dataset_status(selected_bfaccount, selected_bfdataset, selected_status):

    try:
        bf = Blackfynn(selected_bfaccount)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn account'
        raise Exception(error)

    try:
        myds = bf.get_dataset(selected_bfdataset)
    except Exception as e:
        error = 'Error: Please select a valid Blackfynn dataset'
        raise Exception(error)

    try:
        role = bf_get_current_user_permission(bf, myds)
        if role not in ['owner', 'manager']:
            error = "Error: You don't have permissions for changing the status of this Blackfynn dataset"
            raise Exception(error)
    except Exception as e:
        raise e

    try:
        #find name corresponding to display name or show error message
        organization_id = bf.context.id
        list_status = bf._api._get(
            f'/organizations/{str(organization_id)}/dataset-status'
        )

        c = 0
        for option in list_status:
            if option['displayName'] == selected_status:
                new_status = option['name']
                c += 1
                break
        if c==0:
            error = "Error: Selected status is not available for this blackfynn account"
            raise Exception(error)

        #gchange dataset status
        selected_dataset_id = myds.id
        jsonfile = {'status': new_status}
        bf._api.datasets._put(f'/{str(selected_dataset_id)}', json=jsonfile)
        return "Success: Changed dataset status to '"+ selected_status +"'"
    except Exception as e:
        raise e


#함수형모델 앙상블 + RMSE, R2넣기
# 데이터가 x 는 2개, y는 1개인 모델 output은 1개
#1. 데이터

import numpy as np
#모델1.
x1 = np.array([range(1,101), range(711,811), range(100)])
x2 = np.array([range(4,104), range(761,861), range(100)])

#모델2.
y1 = np.array([range(101,201), range(311, 411), range(100)])

x1 = np.transpose(x1)
x2 = np.transpose(x2)

y1 = np.transpose(y1)

#train, test분리 (슬라이싱 대신에 train_test_split)
from sklearn.model_selection import train_test_split
x1_train, x1_test, y1_train, y1_test = train_test_split(x1, y1, shuffle=True, train_size =0.7)
x2_train, x2_test = train_test_split(x2,shuffle=True, train_size =0.7)

print("x1_test", x1_test.shape)
print("x2_test", x2_test.shape)
print("y1_test", y1_test.shape)

#함수형 모델 만들기
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import Dense, Input

#모델1.
input1 = Input(shape =(3,))
dense1 = Dense(10, activation='relu', name='king1')(input1) #모델 이름이 중복될 때 , 혹은 dense이름 변경
dense2 = Dense(7, activation='relu', name='king2')(dense1)
dense3 = Dense(5, activation='relu', name='king3')(dense2)
output1 = Dense(3, activation ='linear', name='king4')(dense3)

#모델2.
input2 = Input(shape =(3,))
dense1_1 = Dense(15, activation='relu')(input2)
dense2_1 = Dense(11, activation='relu')(dense1_1)
output2 = Dense(3, activation ='linear')(dense2_1)

#모델끼리는 별 영향을 끼치지 않는다
##########모델 병합, concantenate 여러가지 방법으로import
from tensorflow.keras.layers import Concatenate, concatenate
# from keras.layers.merge import Concatenate, concatenate
# from keras.layers import Concatentate, concatenate

#########모델 2개 병합하기
#소문자 concatenate
#merge1 = concatenate([output1, output2])#리스트로 묶기

#방법 1. Concatenate대문자 사용법
#merge1 = Concatenate()([output1, output2]) #대문자 Concatenate 면 class다
#방법 2.
merge1 = Concatenate(axis=1)([output1, output2]) #axis축 // 여기서의 output은 y축의 데이터 개수가 아니고 모델 1의output, 모델2의output
middle1 = Dense(30)(merge1) #함수형으로 모델 병합하기
middle1 = Dense(7)(middle1)
middle1 = Dense(11)(middle1)

'''
merge1 = concatenate([output1, output2])#리스트로 묶기
#앞에 변수명 같이 명시 해도 가능.
middle1 = Dense(30)(merge1) #함수형으로 모델 병합하기
middle2 = Dense(7)(middle1)
middle3 = Dense(11)(middle2) #이렇게 middle1도 가능?
'''
########### output 만들기 (분기) y갯수에 따라서
output3 = Dense(30)(middle1)#middle1에서 받아온다
output3_1 = Dense(7)(output3)
output3_2 = Dense(3)(output3_1)#outputs에 마지막 아웃풋 넣기

# output4 = Dense(15)(middle1)
# output4_1 = Dense(14)(output4)
# output4_2 = Dense(11)(output4_1)
# output4_3 = Dense(3)(output4_2)#마지막 아웃풋 넣기

#모델 정의하기
model = Model (inputs =[input1, input2], outputs =output3_2)
                        #모델 1,2넣기               #ㄴ여기아웃풋은 input할 때의 output이 아니라 output할 때 마지막 output변수명
model.summary()

#concatenate는 연산하지 않고 병합만 해준다

# 컴파일, 훈련
model.compile(loss= 'mse', optimizer='adam', metrics=['mae'])
model.fit([x1_train, x2_train], y1_train, epochs=100, batch_size=8, validation_split=0.25, verbose=1)

# 평가
result = model.evaluate([x1_test, x2_test], y1_test, batch_size=8)
# (loss, 메트릭스의 mse)가 나온다

print('result:' , result)
#result: [1.739203691482544, 0.4144916236400604, 1.3247121572494507, 0.47640863060951233, 0.80832839012146]

#예측 = 훈련시킨 값이 나온다 (accuracy 의 정확도를 위해서 예측이 필요하다) x값을 넣어서 예측 y값을 나오게
y_predict = model.predict([x1_test, x2_test]) #y_predict라는 변수에 위에서 validation_split으로 쪼갠 test데이터에서 y값을 예측하자
print('결과물: \n :' , y_predict)

#RMSE
#사이킷런 지표
from sklearn.metrics import mean_squared_error
def RMSE(y1_test, y_predict) : #y1_test validation_split값의 y예측 테스트 데이터를 가져오고, 예측 데이터를 넣을 변수명
    return np.sqrt(mean_squared_error(y1_test, y_predict))
print('RMSE:' ,RMSE(y1_test, y_predict))

#R2
from sklearn.metrics import r2_score
r2 = r2_score(y1_test, y_predict)
print('R2:', r2)