__author__ = 'Neil Butcher'

import sqlite3
from Events import MultipleEventsSavingObject
from Rota_System.Roles import GlobalRoleList, Role
from Rota_System.StandardTimes import StandardEventTimes
from Rota_System.Worker import Worker
from Duration import DurationSavingObject
from Error import DatabaseSaveLoadError
import time
from datetime import datetime

class InstitutionSavingObject(object):
    '''
    classdocs
    '''

    def __init__(self, institution, filename=None):
        if filename:
            self._filename = filename
        else:
            self._filename = institution.name + '.db'

        self._institution = institution
        self._connection = sqlite3.connect(self._filename, detect_types=sqlite3.PARSE_DECLTYPES)

    def load(self):

        self.loadRoles()
        self.loadTimes()
        self.loadPopulation()
        self.loadTemplates()
        self.loadDurations()

    def createTables(self):
        with self._connection:
            self._createRolesTables()
            self._createPopulationTables()
            self._createTimesTable()
        MultipleEventsSavingObject(None, self._filename, None).beginNew()
        DurationSavingObject(None, self._filename).beginNew()

    def populateTables(self):
        with self._connection:
            self._populateRolesTables()
            self._populatePopulationTables()
            self._populateTimesTable()
        e = MultipleEventsSavingObject(self._institution.templates, self._filename, None)
        d = DurationSavingObject(self._institution.durations, self._filename)
        e.populateTables()
        d.populateTables()

    def loadTemplates(self):
        MultipleEventsSavingObject(None, self._filename, None).loadEvents(self._institution)

    def loadDurations(self):
        if self._personCacheDict:
            DurationSavingObject(None, self._filename).loadTables(self._institution, self._personCacheDict)
        else:
            raise DatabaseSaveLoadError('The population must be loaded before appointments can be loaded')

    def loadTimes(self):
        with self._connection:
            self._loadInTimesTable()

    def loadRoles(self):
        with self._connection:
            self._loadInRolesTables()

    def loadPopulation(self):
        with self._connection:
            self._loadInRolesTables()
            self._loadInPopulationTables()

    def _createRolesTables(self):
        _c = self._connection.cursor()
        try:
            _c.execute('''DROP TABLE Roles''')
        except sqlite3.OperationalError:
            pass
        _c.execute('''CREATE TABLE Roles
             (description TEXT NOT NULL, priority INTEGER, code TEXT,PRIMARY KEY (description))''')
        try:
            _c.execute('''DROP TABLE RolesCompatibility''')
        except sqlite3.OperationalError:
            pass
        _c.execute('''CREATE TABLE RolesCompatibility
             (role1Description TEXT NOT NULL, role2Description TEXT NOT NULL)''')

    @staticmethod
    def roleTupple(role):
        return role.description, role.priority, role.code

    def _populateRolesTables(self):
        _c = self._connection.cursor()
        _c.execute('''DELETE FROM Roles''')
        roleTupples = map(self.roleTupple, GlobalRoleList.roles)
        _c.executemany('INSERT INTO Roles VALUES (?,?,?)', roleTupples)

        _c.execute('''DELETE FROM RolesCompatibility''')
        for role1 in GlobalRoleList.roles:
            for role2 in role1.compatibilities.roles:
                _c.execute('INSERT INTO RolesCompatibility VALUES (?,?)', (role1.description, role2.description))

    def _loadInRolesTables(self):
        _c = self._connection.cursor()
        GlobalRoleList.clear()
        for row in _c.execute('SELECT * FROM Roles ORDER BY priority'):
            GlobalRoleList.add_role(Role(row[0], row[2], row[1]))

        for row in _c.execute('SELECT * FROM RolesCompatibility'):
            role1 = GlobalRoleList.role_from_desc(row[0])
            role2 = GlobalRoleList.role_from_desc(row[1])
            role1.compatibilities.add(role2)

    def _createPopulationTables(self):
        _c = self._connection.cursor()
        try:
            _c.execute('''DROP TABLE Population''')
        except sqlite3.OperationalError:
            pass
        _c.execute('''CREATE TABLE Population
             (name TEXT NOT NULL, address TEXT, email TEXT, phone TEXT,PRIMARY KEY (name))''')
        try:
            _c.execute('''DROP TABLE PopulationAvailability''')
        except sqlite3.OperationalError:
            pass
        _c.execute('''CREATE TABLE PopulationAvailability
             (personName TEXT NOT NULL, unavailableDate date NOT NULL)''')
        self._connection.commit()
        try:
            _c.execute('''DROP TABLE PopulationQualifications''')
        except sqlite3.OperationalError:
            pass
        _c.execute('''CREATE TABLE PopulationQualifications
             (personName TEXT NOT NULL, qualification TEXT NOT NULL)''')

    @staticmethod
    def personTupple(person):
        return person.name, person.address, person.email, person.phone_number

    def _populatePopulationTables(self):
        _c = self._connection.cursor()
        _c.execute('''DELETE FROM Population''')
        tupples = map(self.personTupple, self._institution.people)
        _c.executemany('INSERT INTO Population VALUES (?,?,?,?)', tupples)

        _c.execute('''DELETE FROM PopulationAvailability''')
        _c.execute('''DELETE FROM PopulationQualifications''')
        for person in self._institution.people:
            for date in person.blacklisted_dates():
                _c.execute('INSERT INTO PopulationAvailability VALUES (?,?)', (person.name, date))
            for role in person.roles():
                _c.execute('INSERT INTO PopulationQualifications VALUES (?,?)', (person.name, role.description))

    def _loadInPopulationTables(self):
        _c = self._connection.cursor()
        self._institution.people = []
        personCacheDict = {'Name': 'PersonObject'}
        for row in _c.execute('SELECT * FROM Population ORDER BY name'):
            p = Worker()
            p.name = row[0]
            p.address = row[1]
            p.email = row[2]
            p.phoneNumber = row[3]
            self._institution.people.append(p)
            personCacheDict[p.name] = p

        for row in _c.execute('SELECT * FROM PopulationAvailability'):
            personCacheDict[row[0]].blacklist_date(row[1])

        for row in _c.execute('SELECT * FROM PopulationQualifications'):
            r = GlobalRoleList.role_from_desc(row[1])
            personCacheDict[row[0]].roles().add(r)

        self._personCacheDict = personCacheDict

    def _createTimesTable(self):
        _c = self._connection.cursor()
        try:
            _c.execute('''DROP TABLE StandardEventTimes''')
        except sqlite3.OperationalError:
            pass
        _c.execute('''CREATE TABLE StandardEventTimes
             (name TEXT NOT NULL, time TEXT,PRIMARY KEY (name))''')

    @staticmethod
    def timeTupple(item):
        return item[0], item[1].strftime("%H:%M")

    def _populateTimesTable(self):
        _c = self._connection.cursor()
        _c.execute('''DELETE FROM StandardEventTimes''')
        timeTupples = map(self.timeTupple, StandardEventTimes)
        _c.executemany('INSERT INTO StandardEventTimes VALUES (?,?)', timeTupples)

    def _loadInTimesTable(self):
        _c = self._connection.cursor()
        StandardEventTimes[:] = []
        for row in _c.execute('SELECT * FROM StandardEventTimes'):
            t = (datetime(*time.strptime(row[1], "%H:%M")[0:6])).time()
            StandardEventTimes.append((row[0], t))


'''
Calculation files in VASP
'''

from logging import getLogger
import os

from ...IO import read_input as rin

logger = getLogger('cryspy')

def check_input_vasp():
    # ---------- prepare rin.jobfile, POTCAR, INCAR
    calc_inputs = [rin.jobfile, 'POTCAR', 'INCAR']

    # ------ check required files
    for f in calc_inputs:
        if f == 'INCAR':
            fincars = ['INCAR_{}'.format(i) for i in range(1, rin.nstage+1)]
            for ff in fincars:
                if not os.path.isfile('./calc_in/' + ff):
                    logger.error('Could not find ./calc_in/' + ff)
                    raise SystemExit(1)
        else:
            if not os.path.isfile('./calc_in/' + f):
                logger.error('Could not find ./calc_in/' + f)
                raise SystemExit(1)


import math
import numpy as np
import matplotlib.pyplot as plt
from random import gauss
import matplotlib as mpl
from matplotlib.colors import ListedColormap
import seaborn as sns

# boundary condition
def BC(x, y):
    if x < -1:
        x = -2 - x
    else:
        if x > 1:
            x = 2 - x
    if y < -1:
        y = -2 - y
    else:
        if y > 1:
            y = 2 - y
    return x, y

# for easier setting, I put all the code into a class, developed by York and The Kite
class CMM3(object):
    def __init__(self):
        # ----------------------------------------
        # init conditions / interface
        # time set, "h" is a step time
        self.time_max = 0.4
        self.h = 0.05
        # diffusivity
        self.D = 0.01
        # domain size
        self.x_min = -1
        self.x_max = 1
        self.y_min = -1
        self.y_max = 1
        # number of axial gridlines and total grids
        self.Nx = 64
        self.Ny = 64
        self.grids = self.Nx * self.Ny
        # number of particles
        self.Np = 65536
        # init velocity
        self.vel = 0
        # -----------------------------------------
        # for temp particle position data save
        self.x_blue = []
        self.y_blue = []
        self.x_red = []
        self.y_red = []
        # figure and save the interval of the gridlines
        self.ivl_grid_y = (self.y_max - self.y_min) / self.Ny
        self.ivl_grid_x = (self.x_max - self.x_min) / self.Nx
        # data0 is set to count and save the number of red particles and blue particles
        self.data0 = np.zeros((self.Nx, self.Ny, 2))
        # data is set to figure the proportion of blue particles in each grid
        self.data = np.zeros((self.Nx, self.Ny))

    # the func below is extends the classical Euler method
    def Euler_method(self, Xp):
        # create a gauss random dx
        r = gauss(0, 1)
        # equation 6
        # 'r' is random numbers with the standard Gaussian probability
        # 'vel' is the (known) velocity components evaluated at the particle position
        Xp += self.vel * self.h + math.sqrt(2 * self.D) * math.sqrt(self.h) * r
        return Xp

    # this func is used to update the particles after a step time, the movement of each particle is built by York,
    def go_a_step(self):
        for i in range(len(self.x_blue)):
            self.x_blue[i] = self.Euler_method(self.x_blue[i])
            self.y_blue[i] = self.Euler_method(self.y_blue[i])
            # use the boundary condition above
            self.x_blue[i], self.y_blue[i] = BC(self.x_blue[i], self.y_blue[i])
        for i in range(len(self.x_red)):
            self.x_red[i] = self.Euler_method(self.x_red[i])
            self.y_red[i] = self.Euler_method(self.y_red[i])
            # use the boundary condition above
            self.x_red[i], self.y_red[i] = BC(self.x_red[i], self.y_red[i])

    # this func is used to setup the particles in one traverse, built by York
    def setup(self):
        # to reset the coordinates and quantities for the circulation in main()
        self.x_blue = []
        self.y_blue = []
        self.x_red = []
        self.y_red = []
        # use np.random to init the particles
        for i in range(self.Np):
            # temp val for a random particle
            tx = np.random.uniform(self.x_min, self.x_max)
            ty = np.random.uniform(self.y_min, self.y_max)
            ivl_xs = math.ceil((tx - self.x_min) / self.ivl_grid_x) - 1
            ivl_ys = math.ceil((ty - self.y_min) / self.ivl_grid_y) - 1
            # select the color of this particle and count each color in data0
            if math.sqrt(math.pow(tx, 2) + math.pow(ty, 2)) < 0.3:
                self.x_blue.append(tx)
                self.y_blue.append(ty)
                self.data0[ivl_xs][ivl_ys][0] += 1
            else:
                self.x_red.append(tx)
                self.y_red.append(ty)
                self.data0[ivl_xs][ivl_ys][1] += 1

    def grid(self):
        # to locate which grid is the new red particle in and add it to data0[1]
        for i in range(len(self.x_blue)):
            ivl_xs = math.ceil((self.x_blue[i] - self.x_min) / self.ivl_grid_x) - 1
            ivl_ys = math.ceil((self.y_blue[i] - self.y_min) / self.ivl_grid_y) - 1
            self.data0[ivl_xs][ivl_ys][0] += 1
        # to locate which grid is the new red particle in and add it to data0[1]
        for i in range(len(self.x_red)):
            ivl_xs = math.ceil((self.x_red[i] - self.x_min) / self.ivl_grid_x) - 1
            ivl_ys = math.ceil((self.y_red[i] - self.y_min) / self.ivl_grid_y) - 1
            self.data0[ivl_xs][ivl_ys][1] += 1
        # transfer the data0 into data by calculating the proportion of blue particles in each grid
        for i in range(self.Nx):
            for j in range(self.Ny):
                self.data[i][j] = self.data0[i][j][0] / (self.data0[i][j][0] + self.data0[i][j][1])

    # show1 and show2 are made by The Kite
    # the visualization of particle form
    def show1(self):
        # set the figure and pass in the coordinates of blue and red particles
        plt.figure(figsize=(10, 8))
        plt.scatter(self.x_blue, self.y_blue, s=1, c='b')
        plt.scatter(self.x_red, self.y_red, s=1, c='r')
        # set the layout of axis and title
        plt.title("2D problem", fontname='Arial', fontsize=30, weight='bold')
        plt.xlabel("x", fontname='Arial', fontsize=20, weight='bold')
        plt.ylabel("y", fontname='Arial', fontsize=20, weight='bold')
        plt.xlim([-1, 1])
        plt.ylim([-1, 1])
        plt.xticks([-1 + i * 0.5 for i in range(5)])
        plt.yticks([-1 + i * 0.5 for i in range(5)])
        # set the parameters of the colorbar in particle form and create it
        cmap = ListedColormap(["red", "blue"])
        norm = mpl.colors.Normalize(vmin=0, vmax=1)
        plt.colorbar(
            mpl.cm.ScalarMappable(cmap=cmap, norm=norm)
        )
        # show
        plt.show()

    # the visualization of grid form
    def show2(self, colors):
        # pass in data and create the heatmap
        sns_plot = sns.heatmap(self.data, vmin=0, vmax=1, cmap=colors)
        # set the layout of axis and title
        plt.title("2D problem", fontname='Arial', fontsize=30, weight='bold')
        sns_plot.set_xlabel("x", fontname='Arial', fontsize=20, weight='bold')
        sns_plot.set_ylabel("y", fontname='Arial', fontsize=20, weight='bold')
        plt.axis('off')
        # show
        plt.show()

    '''
    # This code is built by York, and used to test
    def show(self):
        plt.scatter(self.x_red, self.y_red, s=0.5, color='red')
        plt.scatter(self.x_blue, self.y_blue, s=0.5, color='blue')
        plt.xlabel("x")
        plt.ylabel("y")
        plt.show()
    '''

    # main code for this class
    def main(self):
        # the defination of colorbar of gird form
        colors1 = [(r, g, b) for (r, g, b) in zip(np.linspace(1, 0.8, 7), np.linspace(0, 0, 7), np.linspace(0, 0.9, 7))]
        colors2 = [(r, g, b) for (r, g, b) in zip(np.linspace(0.7, 0, 6), np.linspace(0, 1, 6), np.linspace(1, 0, 6))]
        colors3 = [(r, g, b) for (r, g, b) in zip(np.linspace(0, 0, 7), np.linspace(0.7, 0, 7), np.linspace(0, 1, 7))]
        colors = colors1 + colors2 + colors3

        print("This is CMM3 group's project of 2D probelms")
        while True:
            choice = int(input("for particle press 0,for grid press 1,press anything to quit\n"))
            # the visualization of particle form
            if choice == 0:
                # setup the init list of particles
                self.setup()
                # show the first graph when t = 0
                self.show1()
                # cycle in Classic Euler Method step by step
                for i in range(int(self.time_max / self.h)):
                    self.data = np.zeros((self.Nx, self.Ny))
                    self.go_a_step()
                    self.show1()
                # after the iteration recreate data0 and data
                self.data0 = np.zeros((self.Nx, self.Ny, 2))
                self.data = np.zeros((self.Nx, self.Ny))
            # the visualization of grid form
            elif choice == 1:
                # setup the init list of particles, data0 and data
                self.setup()
                # show the first graph when t = 0, data0 and data
                self.grid()
                self.show2(colors)
                # cycle in Classic Euler Method step by step
                for i in range(int(self.time_max / self.h)):
                    self.data = np.zeros((self.Nx, self.Ny))
                    self.go_a_step()
                    self.grid()
                    self.show2(colors)
                # after the iteration recreate data0 and data
                self.data0 = np.zeros((self.Nx, self.Ny, 2))
                self.data = np.zeros((self.Nx, self.Ny))
            # quit the project
            else:
                return

# main code for the whole project
if __name__ == '__main__':
    # build a instance
    Project = CMM3()
    # to run main code
    Project.main()


from django.core.management.base import BaseCommand, CommandError

from main.models import Publication
from collections import defaultdict
import pandas as pd

class Command(BaseCommand):
    help = 'Adds the bio.tools to the publications model'

    def add_arguments(self, parser):
        parser.add_argument('biotoolsid2pmid', default=None)

    def handle(self, *args, **options):
        try:
            biotoolsid2pmid = options['biotoolsid2pmid']
        except:
            raise CommandError('Please provide an biotoolsid2pmid csv')

        df = pd.read_csv(biotoolsid2pmid, sep='\t', names = ["biotoolsID","pmid"])
        pmid2biotoolsid = dict(zip(list(df.pmid), list(df.biotoolsID)))

        to_update = []
        for pub in Publication.objects.all():
            if pmid2biotoolsid:
                if pub.pubmed_id in pmid2biotoolsid:
                    pub.biotools_id = pmid2biotoolsid[pub.pubmed_id]
                    to_update.append(pub)

        Publication.objects.bulk_update(to_update, ["biotools_id"])

        self.stdout.write(self.style.SUCCESS(f'Updated {len(to_update)} publications'))


from flask import Flask, render_template, g, request, redirect, url_for
from flask_login import LoginManager, current_user, login_required, login_user, \
    logout_user
import json
from oauthlib.oauth2 import WebApplicationClient
import os
import requests
from werkzeug.local import LocalProxy

from sql.repository import Repository
from sql.sqlite_connection import create_sqlite_connection
from models.employee import Employee
from models.department import Department
from models.user import User

PROJECT_PATH = os.path.abspath(os.path.join(__file__, ".."))

config_file_path = os.path.join(PROJECT_PATH, "config", "configuration.json")

# open configuration file
with open(config_file_path) as configuration_file:
    configuration_dict = json.loads(configuration_file.read())

# get path to google client id and token
google_client_id_path = os.path.join(
    PROJECT_PATH,
    configuration_dict["google_client_id"]
)

if not os.path.exists(google_client_id_path):
    print(f"You need the token file at {google_client_id_path}")
    exit(1)

google_client_secret_path = os.path.join(
    PROJECT_PATH,
    configuration_dict["google_client_secret"]
)

if not os.path.exists(google_client_secret_path):
    print(f"You need the token file at {google_client_secret_path}")
    exit(1)

# get google client id and token
with open(google_client_id_path) as google_client_id_file:
    GOOGLE_CLIENT_ID = google_client_id_file.readline()
    if GOOGLE_CLIENT_ID[-1] == "\n":
        GOOGLE_CLIENT_ID = GOOGLE_CLIENT_ID[:-1]

with open(google_client_secret_path) as google_client_secret_file:
    GOOGLE_CLIENT_SECRET = google_client_secret_file.readline()
    if GOOGLE_CLIENT_SECRET[-1] == "\n":
        GOOGLE_CLIENT_SECRET = GOOGLE_CLIENT_SECRET[:-1]

# google discovery document url
GOOGLE_DISCOVERY_URL = (
    "https://accounts.google.com/.well-known/openid-configuration"
)

DATABASE_NAME = configuration_dict["database"]

DATABASE_PATH = os.path.join(PROJECT_PATH, DATABASE_NAME)

COLUMNS = {"id", "department_id", "age", "salary"}

# create DB schema if it doesn't exist yet
with create_sqlite_connection(DATABASE_PATH) as connection:
    repository = Repository(connection)
    repository.create_schema()
    repository.commit()

# Flask set-up
application = Flask(
    __name__,
    template_folder=os.path.join(PROJECT_PATH, "templates"),
    static_folder=os.path.join(PROJECT_PATH, "static")
)

# set-up login manger
application.secret_key = GOOGLE_CLIENT_SECRET
login_manager = LoginManager()
login_manager.init_app(application)

client = WebApplicationClient(GOOGLE_CLIENT_ID)

def get_google_provider_cfg():
    return requests.get(GOOGLE_DISCOVERY_URL).json()

@login_manager.user_loader
def load_user(user_id):
    return context.repository.get_admin(user_id)

@login_manager.unauthorized_handler
def unathorized():
    return redirect(url_for("login"))

# set-up db context
class Context:
    def __init__(self):
        self.connection = create_sqlite_connection(DATABASE_PATH)
        self.repository = Repository(self.connection)

    def __enter__(self):
        self.connection.__enter()

    def __exit__(self):
        self.connection.close()

def inject_context():
    if "context" not in g:
        g.context = Context()
    return g.context

def teardown_context(exception):
    context = g.pop("context", None)
    if context is not None:
        context.__exit__()

context = LocalProxy(inject_context)

@application.route("/", endpoint="root")
@application.route("/departments", endpoint="departments")
@login_required
def departments():
    if "sort" in request.args:
        departments = context.repository.get_departments_desc()
    else:
        departments = context.repository.get_departments()
    return render_template(
        "departments.html",
        departments=departments
    )

@application.route("/login")
def login():
    if request.args.get("error") == "1":
        return render_template(
            "login.html",
            error="User email not available or not verified by Google."
        )
    if request.args.get("error") == "2":
        return render_template(
            "login.html",
            error="You are not on the list of administrators"
        )
    if not request.args:
        return render_template("login.html")
    else:
        return "Smth went wrong", 400

@application.route("/authorization")
def authorization():
    # get google authorization endpoint
    google_provider_cfg = get_google_provider_cfg()
    authorization_endpoint = google_provider_cfg["authorization_endpoint"]

    # authorization request
    request_uri = client.prepare_request_uri(
        authorization_endpoint,
        redirect_uri=request.base_url + "/callback",
        scope=["openid", "email", "profile"],
    )
    return redirect(request_uri)

@application.route("/authorization/callback")
def authorization_callback():
    code = request.args.get("code")

    google_provider_cfg = get_google_provider_cfg()
    token_endpoint = google_provider_cfg["token_endpoint"]

    # exchanging received code on token
    token_url, headers, body = client.prepare_token_request(
        token_endpoint,
        authorization_response=request.url,
        redirect_url=request.base_url,
        code=code
    )
    token_response = requests.post(
        token_url,
        headers=headers,
        data=body,
        auth=(GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET),
    )

    # use token to get user info
    client.parse_request_body_response(json.dumps(token_response.json()))

    userinfo_endpoint = google_provider_cfg["userinfo_endpoint"]
    uri, headers, body = client.add_token(userinfo_endpoint)
    userinfo_response = requests.get(uri, headers=headers, data=body)

    # parse user info if email is verified
    if userinfo_response.json().get("email_verified"):
        unique_id = userinfo_response.json()["sub"]
        users_email = userinfo_response.json()["email"]
        picture = userinfo_response.json()["picture"]
        users_name = userinfo_response.json()["given_name"]
    else:
        return redirect(url_for("login", error="1"), 302)

    user = User(
        unique_id, users_name, users_email, picture
    )

    # add user into users table if he is not an admin
    if context.repository.is_user_admin(user) is not True:
        error = context.repository.add_user(user)
        if not error:
            context.repository.commit()
        return redirect(url_for("login", error="2"), 302)

    # login user if he is an admin
    login_user(user)

    return redirect(url_for("root"))

@application.route("/logout")
@login_required
def logout():
    logout_user()
    return redirect(url_for("login"))

@application.route("/employees", endpoint="employees")
@login_required
def employees():
    sort = request.args.get("sort")
    direction = request.args.get("direction")
    if sort:
        if sort not in COLUMNS:
            return "Bad request", 400
        if direction == "desc":
            employees = context.repository.get_employees_desc(sort)

        else:
            employees = context.repository.get_employees(sort)
    else:
        employees = context.repository.get_employees()
    return render_template(
        "employees.html",
        employees=employees,
        departments=context.repository.get_departments()
    )

@application.route("/departments/<int:id>", endpoint="edit_department")
@login_required
def edit_department(id):
    sort = request.args.get("sort")
    direction = request.args.get("direction")
    if sort:
        if sort not in COLUMNS:
            return "Bad request", 400
        if direction == "desc":
            employees = context.repository\
                .get_employees_by_department_id_desc(id, sort)
        else:
            employees = context.repository\
                .get_employees_by_department_id(id, sort)
    else:
        employees = context.repository.get_employees_by_department_id(id)
    return render_template(
        "edit_department.html",
        employees=employees,
        department=context.repository.get_department_by_id(id)
    )

@application.route("/departments/search", endpoint="search-departments")
@login_required
def display_department_search_result():
    sort_field = request.args.get("sort")
    search_query = request.args.get("q")
    if sort_field:
        if sort_field != "id":
            return "Bad request", 400
        departments = context.repository.search_department_by_name_desc(
            search_query)
    else:
        departments = context.repository\
            .search_department_by_name(search_query)
    if not departments:
        return "No results", 418
    if search_query:
        return render_template(
            "departments.html",
            departments=departments,
            search_query=request.args["q"]
        )

@application.route(
    "/departments/<int:id>/search",
    endpoint="search-in-department")
@login_required
def search_in_department_view(id):
    sort_field = request.args.get("sort")
    direction = request.args.get("direction")
    search_query = request.args.get("q")
    if sort_field:
        if sort_field not in COLUMNS:
            return "Bad request", 400
        if direction == "desc":
            employees = context.repository.search_employees_in_department_desc(
                id, search_query, sort_field)
        else:
            employees = context.repository\
                .search_employees_in_department(id, search_query, sort_field)
    else:
        employees = context.repository\
            .search_employees_in_department(id, search_query)
    if not employees:
        return "No results", 418
    if search_query:
        return render_template(
            "edit_department.html",
            department=context.repository.get_department_by_id(id),
            employees=employees,
            search_query=search_query
        )

@application.route("/employees/search", endpoint="search-employees")
@login_required
def display_employees_search_result():
    sort_field = request.args.get("sort")
    direction = request.args.get("direction")
    search_query = request.args.get("q")
    if sort_field:
        if sort_field not in COLUMNS:
            return "Bad request", 400
        if direction == "desc":
            employees = context.repository\
                .search_employees_desc(search_query, sort_field)
        else:
            employees = context.repository\
                .search_employees(search_query, sort_field)
    else:
        employees = context.repository.search_employees(search_query)
    if not employees:
        return "No results", 418
    if search_query:
        return render_template(
            "employees.html",
            employees=employees,
            departments=context.repository.get_departments(),
            search_query=search_query
        )

@application.route("/employees/<int:id>", endpoint="edit_employee")
@login_required
def edit_employee(id):
    employee = context.repository.get_employee_by_id(id)
    if employee == "Incorrect data":
        return "Incorrect data", 418
    return render_template(
        "edit_employee.html",
        employee=employee,
        departments=context.repository.get_departments()
    )

@application.route("/admins", endpoint="admins")
@login_required
def admins():
    admins = context.repository.get_admins()
    users = context.repository.get_users()
    return render_template(
        "admins.html",
        admins=admins,
        users=users
    )

# api methods
@application.route("/api/departments/<int:id>", methods=["DELETE"])
@login_required
def delete_department(id):
    context.repository.delete_department(id)
    context.repository.commit()
    return "", 204

@application.route(
    "/api/employees/<int:id>/<int:department_id>",
    methods=["DELETE"])
@login_required
def delete_employee(id, department_id):
    context.repository.delete_employee(id, department_id)
    context.repository.commit()
    return "", 204

@application.route("/api/departments/new-department", methods=["POST"])
@login_required
def new_department():
    new_department_data = Department.from_tuple(
        tuple([0] + request.data.decode("utf-8").split(",")))
    error = context.repository.add_department(
        new_department_data.name,
        new_department_data.country,
        new_department_data.city,
        new_department_data.address,
        new_department_data.department_manager
    )
    if error == "DEPARTMENT NAME DUPLICATE":
        return "This department name is occupied", 400
    context.repository.commit()
    return "", 204

@application.route("/api/employees/new-employee", methods=["POST"])
@login_required
def new_employee():
    new_employee_data = Employee.from_tuple(
        tuple([0] + request.data.decode("utf-8").split(",")))
    error = context.repository.add_employee(
        new_employee_data.department_id,
        new_employee_data.first_name,
        new_employee_data.last_name,
        new_employee_data.gender,
        new_employee_data.age,
        new_employee_data.position,
        new_employee_data.salary
    )
    if error == "DUPLICATE MEMBERS":
        return "Duplicate members", 400
    context.repository.commit()
    return "", 204

@application.route("/api/department/<int:id>/rename", methods=["POST"])
@login_required
def rename_department(id):
    new_name = request.data.decode("utf-8")
    error = context.repository.rename_department(id, new_name)
    if error == "DUPLICATE NAMES":
        return "DUPLICATE NAMES", 418
    context.repository.commit()
    return "", 204

@application.route("/api/employees/<int:id>/edit", methods=["POST"])
@login_required
def edit_employee_api(id):
    error = context.repository.edit_employee(Employee.from_tuple(
        [id] + request.data.decode("utf-8").split(",")
    ))
    if error:
        return error, 418
    context.repository.commit()
    return "", 204

@application.route("/api/admins/add", methods=["POST"])
@login_required
def add_admin():
    id = request.data.decode("utf-8")
    user = context.repository.get_user(id)
    error = context.repository.add_admin(user)
    if error == "Duplicate admins":
        return error, 418
    context.repository.delete_user(id)
    context.repository.commit()
    return "", 204

if __name__ == "__main__":
    application.run(ssl_context="adhoc", port=443)


# -*- coding: utf-8 -*-

import logging

def get_logger():
    # 设置logger等级
    logger = logging.getLogger()
    logger.setLevel(logging.INFO)
    # 设置文件流和标准输出流stream
    fh = logging.FileHandler('logger2')
    sh = logging.StreamHandler()
    # 设置文件流的消息格式
    fm = logging.Formatter("%(asctime)s [%(lineno)s] %(message)s")
    fh.setFormatter(fm)
    logger.addHandler(fh)
    # 设置标准输出的消息格式
    sh.setFormatter(fm)
    logger.addHandler(sh)
    return logger

logger = get_logger()
logger.info('info message')
logger.debug('debug message')
logger.warning('warning message')
logger.error('error message')
logger.critical('critial message')

python基础/第三方模块/logging模块/test_如何配置logger可以同时写文件和控制台显示.py

test_如何配置logger可以同时写文件和控制台显示.py


# IMPORTS
import json
#Read Json
with open('account.json', 'r') as acc1:
    acc = json.load(acc1)

# Input Part and Reading commands
#Get Name
if acc["name"] == "":
    acc["name"] = input("--Please, enter your name:  ")
    with open('account.json', 'w') as acc2:
        json.dump(acc, acc2)
else:
    pass
print(f"--Welcome, {acc['name']}.")
#Get Observation Type
a = input("--[statistics] (1) or [probability] (2)?:  ")
if a=="statistics" or a=="1":
    from statistics_lib.operations import observType
    x = input(f"--Select observation type - {list(observType.keys())}:  ")
    className = observType[x]
    runnerObj = className()
    runnerObj.get_output()
elif a=="probability" or a=="2":
    from probability_lib.operations import distType
    x = input(f"--Select distribution type - {list(distType.keys())}:  ")
    className = distType[x]
    runnerObj = className()
    runnerObj.get_output()
else:
    raise ImportError("You should import correct library!")


from persistent.dict import PersistentDict
from persistent import Persistent

from zope.schema._field import Choice
from zope.schema._field import Date
from zope.schema import TextLine, Bool
from zope.schema import Int
from zope.location.location import Location
from zope.location.location import locate
from zope.html.field import HtmlFragment
from zope.interface import Interface
from zope.interface import implements
from zope.interface import implementer
from zope.component import adapts
from zope.component import adapter
from zope.container.btree import BTreeContainer
from zope.container.ordered import OrderedContainer
from zope.lifecycleevent.interfaces import IObjectRemovedEvent
from zope.schema.vocabulary import SimpleVocabulary
from zope.schema.vocabulary import SimpleTerm

import z3c.form.field

from schooltool.app.app import InitBase, StartUpBase
from schooltool.app.interfaces import ISchoolToolApplication
from schooltool.basicperson.interfaces import IEnumFieldDescription
from schooltool.basicperson.interfaces import IIntFieldDescription
from schooltool.basicperson.interfaces import IDemographicsFields
from schooltool.basicperson.interfaces import IBasicPerson
from schooltool.basicperson.interfaces import IDemographics
from schooltool.basicperson.interfaces import IFieldDescription
from schooltool.common import SchoolToolMessage as _

class IDemographicsForm(Interface):
    """Interface for fields that are supposed to get stored in person demographics."""

class PersonDemographicsDataContainer(BTreeContainer):
    """Storage for demographics information for all persons."""

class InvalidKeyError(Exception):
    """Key is not in demographics fields."""

class PersonDemographicsData(PersistentDict):
    """Storage for demographics information for a person."""

    def isValidKey(self, key):
        app = ISchoolToolApplication(None)
        demographics_fields = IDemographicsFields(app)
        return key in demographics_fields

    def __repr__(self):
        return '%s: %s' % (
            object.__repr__(self),
            super(PersonDemographicsData, self).__repr__())

    def __setitem__(self, key, v):
        if not self.isValidKey(key):
            raise InvalidKeyError(key)
        super(PersonDemographicsData, self).__setitem__(key, v)

    def __getitem__(self, key):
        if key not in self and self.isValidKey(key):
            self[key] = None
        return super(PersonDemographicsData, self).__getitem__(key)

@adapter(IBasicPerson)
@implementer(IDemographics)
def getPersonDemographics(person):
    app = ISchoolToolApplication(None)
    pdc = app['schooltool.basicperson.demographics_data']
    demographics = pdc.get(person.username, None)
    if demographics is None:
        pdc[person.username] = demographics = PersonDemographicsData()
    return demographics

@adapter(IBasicPerson, IObjectRemovedEvent)
def removePersonDemographicsSubscriber(person, event):
    app = ISchoolToolApplication(None)
    pdc = app['schooltool.basicperson.demographics_data']
    demographics = pdc.get(person.username, None)
    if demographics is not None:
        del pdc[person.username]

class DemographicsFormAdapter(object):
    implements(IDemographicsForm)
    adapts(IBasicPerson)

    def __init__(self, context):
        self.__dict__['context'] = context
        self.__dict__['demographics'] = IDemographics(self.context)

    def __setattr__(self, name, value):
        self.demographics[name] = value

    def __getattr__(self, name):
        return self.demographics.get(name, None)

class DemographicsFields(OrderedContainer):
    implements(IDemographicsFields)

    def filter_key(self, key):
        """Return the subset of fields whose limited_keys list is either
           empty, or it contains the key passed."""
        result = []
        for field in self.values():
            if not field.limit_keys or key in field.limit_keys:
                result.append(field)
        return result

    def filter_keys(self, keys):
        """Return the subset of fields whose limited_keys list is either
           empty, or it contains one of the keys passed."""
        result = []
        for field in self.values():
            limit_keys = field.limit_keys
            if not limit_keys or [key for key in keys if key in limit_keys]:
                result.append(field)
        return result

def setUpDefaultDemographics(app):
    dfs = DemographicsFields()
    app['schooltool.basicperson.demographics_fields'] = dfs
    locate(dfs, app, 'schooltool.basicperson.demographics_fields')
    dfs['ID'] = TextFieldDescription('ID', _('ID'))
    dfs['ethnicity'] = EnumFieldDescription('ethnicity', _('Ethnicity'))
    dfs['ethnicity'].items = [_('American Indian or Alaska Native'),
                              _('Asian'),
                              _('Black or African American'),
                              _('Native Hawaiian or Other Pacific Islander'),
                              _('White')]
    dfs['language'] = TextFieldDescription('language', _('Language'))
    dfs['placeofbirth'] = TextFieldDescription('placeofbirth', _('Place of birth'))
    dfs['citizenship'] = TextFieldDescription('citizenship', _('Citizenship'))

class DemographicsAppStartup(StartUpBase):
    def __call__(self):
        if 'schooltool.basicperson.demographics_fields' not in self.app:
            setUpDefaultDemographics(self.app)
        if 'schooltool.basicperson.demographics_data' not in self.app:
            self.app['schooltool.basicperson.demographics_data'] = PersonDemographicsDataContainer()

class DemographicsInit(InitBase):
    def __call__(self):
        setUpDefaultDemographics(self.app)
        self.app['schooltool.basicperson.demographics_data'] = PersonDemographicsDataContainer()

@implementer(IDemographicsFields)
@adapter(ISchoolToolApplication)
def getDemographicsFields(app):
    return app['schooltool.basicperson.demographics_fields']

class FieldDescription(Persistent, Location):
    implements(IFieldDescription)
    limit_keys = []
    description = None

    def __init__(self, name, title, required=False, limit_keys=[],
                 description=None):
        self.name, self.title, self.required, self.limit_keys = (name,
            title, required, limit_keys)
        self.description = description

    def setUpField(self, form_field):
        form_field.description = self.description
        form_field.required = self.required
        form_field.__name__ = str(self.__name__)
        form_field.interface = IDemographicsForm
        return z3c.form.field.Fields(form_field)

# XXX: IMHO all IDNA conversions should be replaced by punycode.
#      64 max length limitation simply breaks things too often.
class IDNAVocabulary(SimpleVocabulary):

    def createTerm(cls, *args):
        """Create a single term from data.

        Encode the value using idna encoding so it would look sane in
        the form if it's ascii, but still work if it uses unicode.
        """
        value = args[0]
        token = value.encode('idna')
        title = value
        return SimpleTerm(value, token, title)
    createTerm = classmethod(createTerm)

class EnumFieldDescription(FieldDescription):
    implements(IEnumFieldDescription)

    items = []

    def makeField(self):
        return self.setUpField(Choice(
                title=unicode(self.title),
                vocabulary=IDNAVocabulary.fromValues(self.items)
                ))

class DateFieldDescription(FieldDescription):

    def makeField(self):
        return self.setUpField(Date(title=unicode(self.title)))

class TextFieldDescription(FieldDescription):

    def makeField(self):
        return self.setUpField(TextLine(title=unicode(self.title)))

class BoolFieldDescription(FieldDescription):

    def makeField(self):
        return self.setUpField(Bool(title=unicode(self.title)))

class DescriptionFieldDescription(FieldDescription):

    def makeField(self):
        return self.setUpField(HtmlFragment(title=unicode(self.title)))

class IntFieldDescription(FieldDescription):

    implements(IIntFieldDescription)

    min_value = None
    max_value = None

    def makeField(self):
        return self.setUpField(Int(title=unicode(self.title),
                                   min=self.min_value,
                                   max=self.max_value))


from zope.interface import Interface
from zope.container.constraints import contains
from jpl.mcl.protocol import _
from plone.app.textfield import RichText
from zope import schema

class IAsserter(Interface):
    '''An object that describes subjects with a known predicate and a given object'''
    def characterize(obj):
        '''Characterize some subject using a known predicate for complementary ``obj``.  Returns a sequence of doubles
        containing a predicate URI (a URIRef) and an appropriate Literal or URIRef object.'''

class IOrgan(Interface):
    '''An object that describes an Organ'''

class IProtocolFolder(Interface):
    '''Study folder.'''
    contains('jpl.mcl.protocol.protocol.IProtocol', 'jpl.mcl.protocol.interfaces.IProtocolFolder')
    title = schema.TextLine(
        title=_(u'Title'),
        description=_(u'Descriptive name of this folder.'),
        required=True,
    )
    description = RichText(
        title=_(u'Description'),
        description=_(u'A short summary of this folder.'),
        required=False,
    )
    rdfDataSource = schema.TextLine(
        title=_(u'RDF Data Source'),
        description=_(u'URL to a source of Resource Description Format data that mandates the contents of this folder.'),
        required=False,
    )


import numpy as np
import librosa
import soundfile
import os
import json
import tqdm
import sys

pause_threshold=float(sys.argv[1])

base_path='/home/german/Documentos/Trust/First_Impressions_Dataset/OCEAN_new_structure/'
out_path=base_path+'data/audio_IPU_'+str(pause_threshold)+'/'

if not os.path.exists(out_path):
    os.makedirs(out_path)

with open (base_path+'experiments/audio_lists/all_audio_complete_set.txt') as filename:
    list=filename.read().splitlines()

for file in tqdm.tqdm(list):

    parent_path=out_path+'/'+file.split('/')[0]

    if not os.path.exists(parent_path):
        os.mkdir(parent_path)

    with open (base_path+'data/JSON/'+file+'.JSON') as jsonfile:
        dict=json.load(jsonfile)

    y,fs=librosa.core.load(base_path+'data/audio/'+file+'.wav', sr=None)

    IPU=pause_threshold*fs

    final=np.array([])
    n_segment=0
    n_sub_segment=1

    if 'words' in dict:

        # Creo lista con las palabras reconocidas

        words=[x for x in dict['words'] if 'start' in x.keys()]

        for i in range(len(words)-1):

            start=int(words[i]['start']*fs)
            end=int(words[i]['end']*fs)

            speech=y[start:end]

            pause_end=int(words[i+1]['start']*fs)
            pause_init=int(words[i]['end']*fs)

            no_speech=y[pause_init:pause_end]

            pause_duration=pause_end-pause_init

            if pause_duration < IPU:

                segment=np.append(speech,no_speech)
                final=np.append(final,segment)
                n_sub_segment+=1

            else:
                if n_sub_segment==1:
                    final=speech

                n_segment+=1
                soundfile.write(out_path+'%s_%d.wav'% (file,n_segment),final,fs)

                final=np.array([])
                n_sub_segment=1


"""
setup
~~~~~

Script for creating the lerpy package.
"""
import setuptools

from lerpy import __version__

with open('requirements.txt') as fh:
    reqs = fh.readlines()

with open('README.rst') as fh:
    long_desc = fh.read()

setuptools.setup(
    name='lerpy',
    version=__version__.__version__,
    description=__version__.__description__,
    long_description=long_desc,
    long_description_content_type='text/x-rst',
    url='https://github.com/pji/lerpy',
    author=__version__.__author__,
    install_requires=reqs,
    author_email='pji@mac.com',
    packages=setuptools.find_packages(),
    python_requires='>=3.9',
    zip_safe=False
)


import argparse
import sys
import pandas as pd
import numpy as np
import tensorflow as tf

from models import ConvNet_Models
from models import RecurNet_Models
from models import Feed_Forward

from train import Trainer

parser = argparse.ArgumentParser()

# Positional Arguments
parser.add_argument("DATASET_PATH", help="Path to Dataset File (.CSV)", type=str)
parser.add_argument("MODEL", help="Neural Network Model to Train", type=str)
models = ['feed_forward', 'multi_column_cnn', 'simple_rnn', 'bidirectional_rnn', 'vote_multi_column_cnn']

# Optional Arguments
parser.add_argument("--save_model", help="Where to save trained model? Default='model/model.ckpt'", type=str, default='model/model.ckpt')
parser.add_argument("--epochs", help="How many Epochs to train for? Default=10", type=int, default=10)
parser.add_argument("--batch_size", help="What will be the batch Size? Default=128", type=int, default=128)
parser.add_argument("--learning_rate", help="What will be the Learning rate? Default = 0.001", type=float, default=0.001)
parser.add_argument("--feed_forward_architecture", help="If training Feed Forward Network, what will be the Architecture? Specify Number of neurons in each hidden layer EX: [500,500,500]", default="[500,500,500]")

args = parser.parse_args()

if not args.DATASET_PATH or not (args.DATASET_PATH.endswith(".csv")):
    print("No Dataset To Train or Invalid Dataset File!")
    print("'python3 main.py -h' for help")
    sys.exit()
if not args.MODEL or not(args.MODEL in models):
    print("Invalid Model to Train")
    print("USAGE:", models)
    sys.exit()

dataset_path = args.DATASET_PATH
model_to_train = args.MODEL

path_to_save_model = args.save_model
total_epochs = args.epochs
batch_size = args.batch_size
learning_rate = args.learning_rate
architecture = args.feed_forward_architecture

def read_data(dataset_path):
    data = pd.read_csv(dataset_path)
    data = data.values
    values = list(data[:,0])

    labels = []
    for val in values:
        one_hot = np.zeros(10)
        one_hot[val] += 1
        labels.append(np.asarray(one_hot, np.float32))

    images = []
    for val in data:
        images.append(np.asarray(val[1:], np.float32))

    return images, labels

images, labels = read_data(dataset_path)

if model_to_train in ["multi_column_cnn", "vote_multi_column_cnn"]:
    img_w = 28
    img_h = 28
    no_classes = 10
    channels = 1

    x = tf.placeholder("float", [None, img_w, img_h, channels])
    y = tf.placeholder("float", [None, no_classes])

    if model_to_train == "multi_column_cnn":
        model_to_train = ConvNet_Models.Multi_Column_DeepNet()
    else:
        model_to_train = ConvNet_Models.Vote_Multi_Column_DeepNet()

    size = [-1, img_h, img_w, channels]

elif model_to_train in ['simple_rnn', 'bidirectional_rnn']:
    hidden_features = 512
    no_classes = 10
    timesteps = 28
    input_size = 28

    x = tf.placeholder("float", [None, timesteps, input_size])
    y = tf.placeholder("float", [None, no_classes])

    if model_to_train == "simple_rnn":
        model_to_train = RecurNet_Models.Simple_RNN(hidden_features=hidden_features, no_classes=no_classes,
                                                    timesteps=timesteps)
    else:
        model_to_train = RecurNet_Models.Bidirectional_RNN(hidden_features=hidden_features, no_classes=no_classes,
                                                    timesteps=timesteps)

    size = [-1, timesteps, input_size]

elif model_to_train == "feed_forward":
    architecture = architecture.replace("[", '')
    architecture = architecture.replace("]", '')
    architecture = architecture.split(',')

    input_size = 784
    no_classes = 10

    architecture = [int(x) for x in architecture]

    architecture = [784] + architecture + [10]

    size = [-1, input_size]

    x = tf.placeholder("float", [None, input_size])
    y = tf.placeholder("float", [None, no_classes])

    model_to_train = Feed_Forward.Feed_Forward(architecture)

images = np.reshape(images, size)

trainer = Trainer(x, y)

trainer.train_network(x, dataset=[images, labels], model=model_to_train.model,
            learning_rate=learning_rate, save_model_path=path_to_save_model,
            total_epochs=total_epochs, batch_size=batch_size)

#


import json
from urllib.parse import urlparse

try:
    from json.decoder import JSONDecodeError
except ImportError:
    # json decoder in Python3.4 raises ValueError on invalid json
    JSONDecodeError = ValueError
import logging
import sys
from os import getenv, environ
from socket import gaierror, timeout

from h2.exceptions import ProtocolError
from hypertemp import HTTP20Connection
from hypertemp.http20.exceptions import StreamResetError

from ssl import SSLError

log = logging.getLogger(__name__)

class MaxNumberOfRequestsError(Exception):
    """Thrown when remote peer closes the connection because
    maximum number of requests were served through a single connection.
    """

class Http2Client(object):
    """Base class for an http2 client."""
    _no_conn_err = "No connection with server, call reconnect_to_server()"

    def __init__(self, server_url):
        self._server_url = getenv('ORANGE_EMBEDDING_API_URL', server_url)
        self._server_connection = self._connect_to_server()
        self._max_concurrent_streams = self._read_max_concurrent_streams()

    def reconnect_to_server(self):
        self.disconnect_from_server()
        self._server_connection = self._connect_to_server()
        self._max_concurrent_streams = self._read_max_concurrent_streams()
        return self.is_connected_to_server()

    def disconnect_from_server(self):
        if self._server_connection:
            try:
                self._server_connection.close()
            except (ConnectionError, SSLError):
                log.error("Error when disconnecting from server", exc_info=True)

        self._server_connection = None
        self._max_concurrent_streams = None

    def _connect_to_server(self):
        host = port = proxy = None
        if "http_proxy" in environ:
            proxy = environ["http_proxy"]
        elif "https_proxy" in environ:
            proxy = environ["https_proxy"]
        if proxy is not None:
            url = urlparse(proxy)
            host = url.hostname
            port = url.port
        return HTTP20Connection(host=self._server_url, force_proto='h2',
                                proxy_host=host, proxy_port=port)

    def _read_max_concurrent_streams(self):
        if not self._server_ping_successful():
            return None

        # pylint: disable=protected-access
        max_concurrent_streams = (self._server_connection._conn._obj
                                  .remote_settings.max_concurrent_streams)

        if not max_concurrent_streams <= sys.maxsize:
            return 1

        return max_concurrent_streams

    def is_connected_to_server(self):
        if not self._server_connection:
            return False
        if not self._max_concurrent_streams:
            return False
        return self._server_ping_successful()

    def _server_ping_successful(self):
        try:
            self._server_connection.ping(bytes(8))
        except (OSError, TimeoutError, ConnectionError, gaierror, timeout):
            log.error("Remote server not reachable", exc_info=True)
            return False
        return True

    def _send_request(self, method, url, headers, body_bytes):
        if not self._server_connection or not self._max_concurrent_streams:
            raise ConnectionError(self._no_conn_err)

        try:
            return self._server_connection.request(
                method=method,
                url=url,
                body=body_bytes,
                headers=headers
            )
        except (ConnectionError, BrokenPipeError) as error:
            self.disconnect_from_server()
            log.error("Request sending failed")
            raise error

    def _get_json_response_or_none(self, stream_id):
        if not self._server_connection or not self._max_concurrent_streams:
            raise ConnectionError(self._no_conn_err)

        try:
            response_raw = self._server_connection.get_response(stream_id)
            response_txt = response_raw.read().decode()
            return json.loads(response_txt)

        except JSONDecodeError:
            log.warning("Response skipped (not valid json)", exc_info=True)
            return None

        except StreamResetError:
            log.warning(
                "Response skipped (request didn't reach the server "
                "or was malformed)",
                exc_info=True)
            return None

        except ProtocolError:
            error = MaxNumberOfRequestsError(
                "Maximum number of http2 requests through a single "
                "connection exceeded")
            log.warning(error, exc_info=True)
            raise error

        except OSError:
            self.disconnect_from_server()
            error = ConnectionError("Response receiving failed")
            log.error(error, exc_info=True)
            raise error


import enum

from sqlalchemy import Enum
from sqlalchemy_utils.types import UUIDType

from invenio_db import db
from invenio_accounts.models import User as InvenioUser

from storm_project.project.records.models import ResearchProjectMetadata
from storm_workflow.workflow.records.models import ResearchWorkflowMetadata

from storm_commons.records.model import BaseRecordModel

class ExecutionTaskStatus(enum.Enum):
    """Execution Task status."""

    # General
    CREATED = "created"
    FAILED = "failed"
    QUEUED = "queued"
    FINISHED = "finished"

    RUNNING = "running"

class ExecutionTaskModel(db.Model, BaseRecordModel):
    """Execution Task database model."""

    __tablename__ = "execution_execution_tasks"

    #
    # Execution Task
    #
    service = db.Column(db.String)

    status = db.Column(Enum(ExecutionTaskStatus), default=ExecutionTaskStatus.CREATED)

    #
    # Execution Task User owner
    #
    user_id = db.Column(db.Integer, db.ForeignKey(InvenioUser.id))
    user = db.relationship(InvenioUser)

    #
    # Associated project
    #
    project_id = db.Column(UUIDType, db.ForeignKey(ResearchProjectMetadata.id))
    project = db.relationship(ResearchProjectMetadata)

    #
    # Related pipeline
    #
    workflow_id = db.Column(UUIDType, db.ForeignKey(ResearchWorkflowMetadata.id))
    workflow = db.relationship(ResearchWorkflowMetadata)

__all__ = (
    "ExecutionTaskModel",
    "ExecutionTaskStatus",
)


import os
import sys
from threading import Thread

from telegram.ext import CommandHandler

from apps.management.models import CustomUser

class RestartBotHandlerGenerator:
    """Обработчик рестарта бота."""

    @classmethod
    def self_load(cls, dp=None) -> None:
        def stop_and_restart():
            dp.stop()
            os.execl(sys.executable, sys.executable, *sys.argv)

        def restart(update, context):
            update.message.reply_text('Bot is restarting...')
            Thread(target=stop_and_restart).start()

        try:
            dp.add_handler(
                CommandHandler(
                    'restart', restart, filters=CustomUser.get_tgfilters_managers_username()
                )
            )
        except ValueError:
            # Нет ни одного пользователя. Первый запуск бота или никому не назначены права.
            dp.add_handler(
                CommandHandler('restart', restart)
            )


# Digit Cancelling Fractions https://projecteuler.net/problem=33

from fractions import Fraction

def canDoStupidMath(numerator, denominator):
    value = numerator/denominator
    stupidValue = 0
    num = [int(i) for i in str(numerator)]
    den = [int(i) for i in str(denominator)]
    for i in num:
        if i in den:
            den.remove(i)
            num.remove(i)
            stupidValue = num[0]/den[0]
    if stupidValue == value:
        return True
    return False

answers = []
for n in range(11, 99):
    for d in range(11, 99):
        if n >= d or '0' in str(n) or '0' in str(d):
            continue
        if canDoStupidMath(n, d):
            answers.append([n,d])

numAns = 1
denAns = 1
for fractions in answers:
    numAns *= fractions[0]
    denAns *= fractions[1]
print(Fraction(numAns, denAns))


# -*- coding: utf-8 -*-
"""
Created on Thu Dec 25 19:01:54 2014

@author: Huapu (Peter) Pan
"""
import os
import datetime
import pytz

Level={
'NOTSET':0, # show more info
'DEBUG' : 10,
'INFO' : 20,
'ERROR' : 30} # show less info

class SimpleLoggerClass(object):
    def __init__(self, filename, logLevel, folderPath='default', addTime=True):
        """ determine US Eastern time zone depending on EST or EDT """
        if datetime.datetime.now(pytz.timezone('US/Eastern')).tzname() == 'EDT':
            self.USeasternTimeZone = pytz.timezone('Etc/GMT+4')
        elif datetime.datetime.now(pytz.timezone('US/Eastern')).tzname() == 'EST':
            self.USeasternTimeZone = pytz.timezone('Etc/GMT+5')
        else:
            self.USeasternTimeZone = None
        self.addTime = addTime  # True: add local time str in front of the records
        self.filename = filename # User defined fileName
        self._logLevel = Level[logLevel] #
        if folderPath == 'default':
            self.folderPath = os.path.join(os.getcwd(), 'Log')
        else:
            self.folderPath = folderPath
        if not os.path.isdir(self.folderPath):
            os.makedirs(self.folderPath)
            print (__name__+'::__init__: WARNING, create a folder of "Log" at %s' %(self.folderPath,))

    def _write_to_log(self, msg, verbose=True):
        currentTime = datetime.datetime.now(tz = self.USeasternTimeZone)
        if verbose:
            print (msg)
        self.open_file()
        if self.addTime:
            self._log_file.write(str(currentTime) + ": " + msg + '\n')
        else:
            self._log_file.write(msg + '\n')
        self.close_log()

    def notset(self, msg, verbose=True):
        if (self._logLevel <= Level['NOTSET']):
            self._write_to_log(msg, verbose=verbose)

    def debug(self, msg, verbose=True):
        if (self._logLevel <= Level['DEBUG']):
            self._write_to_log(msg, verbose=verbose)

    def info(self, msg, verbose=True):
        if (self._logLevel <= Level['INFO']):
            self._write_to_log(msg, verbose=verbose)

    def error(self, msg, verbose=True):
        if (self._logLevel <= Level['ERROR']):
            self._write_to_log(msg, verbose=verbose)

    def record(self, *arg, **kwrs):
        msg = ''
        for ct in arg:
            msg += str(ct) + ' '
        for ct in kwrs:
            msg += str(kwrs[ct]) + ' '
        self._write_to_log(msg, verbose=False)

    def close_log(self):
        self._log_file.close()

    def open_file(self):
        self._log_file = open(os.path.join(self.folderPath, self.filename), 'a')

if __name__=='__main__':
    #c=  SimpleLoggerClass('TestLog.txt', 'DEBUG', addTime=False)
    c=  SimpleLoggerClass('TestLog.txt', 'INFO')
    c.info('test test')
    c.info('test 1')
    c.info('test 2', verbose=False)
    c.info('test 3')
    c.info('test 4')
    #c.record(a=1, b=2, c=3)
    c.record(1, 2, 3,'test',1,'retest',4, a=4, b=5, c=6)


# -*- coding: utf-8 -*-
"""
Created on Thu Feb 27 12:20:12 2020

@author: Stefan
"""
import numpy as np
import matplotlib

E1_input = np.loadtxt("E1.txt", dtype='i', delimiter='\t')
V1_input = np.loadtxt("V1.txt", dtype='i', delimiter='\t')
test_image=np.reshape(E1_input[:,0],(50,50))
print(test_image.shape)
print(type(E1_input))

E1=np.ravel(E1_input,order="F")
E1=np.reshape(E1,(12000,50,50,1))
V1=np.ravel(V1_input,order="F")
V1=np.reshape(V1,(12000,50,50,1))

x_train=E1[0:11000,:,:,:]
x_test=E1[11000:,:,:,:]
y_train=V1[0:11000,:,:,:]
y_test=V1[11000:,:,:,:]

idx=np.random.randint(0,high=999,size=100)

human_performance_X=x_test[idx,:,:,:]
human_performance_Y=y_test[idx,:,:,:]

for i in range(0,100):
    image_file_name_X='X_' + str(i)+'.png'
    matplotlib.image.imsave(image_file_name_X,human_performance_X[i,:,:,0])
    image_file_name_Y='Y_' + str(i)+'.png'
    matplotlib.image.imsave(image_file_name_Y,human_performance_Y[i,:,:,0])


# %%
# We are importing everything we need for this example:
import os.path as op
import shutil
import csv
import os

import mne
import mne_bids

from mne_bids import write_raw_bids, BIDSPath, print_dir_tree
from mne_bids.stats import count_events
import json
import numpy as np
import matplotlib.pyplot as plt

def convert_to_bids(raw, ica_component=None, ica_topo_figs=None, ica_dict=None, bad_channel_dict=None, sub_id='06', ses='LittlePrince',
                    task='Reading', run=1, bids_root='derivative/preproc',
                    dataset_name='Novel Reading', dataset_type='derivative',
                    author='Xinyu Mou, Cuilin He, Liwei Tan', line_freq=50):
    '''
    :param raw: the pre-processed raw data which you want to save into BIDS format
    :param ica_component: The ICA components when you pre-process the data
    :param ica_topo_figs: Topography of ICA components
    :param ica_dict: This is a dict containing information of the ICA components, It should
           satisfy the following format: {'shape': np.ndarray, 'exclude': list}
           The first value represents the shape of the ICA components: (n_components, time_length)
           The second value represents the excluded ICA components when pre-processing the data
    :param bad_channel_dict: a dict containing information of the marked bad channel when pre-processing
           the data. It should satisfy the following format: {'bad channel': list}
    :param sub_id: The id of the subject you are processing
    :param ses: The session of the current data
    :param run: the run of the current data
    :param bids_root: The root of your dataset
    :param dataset_name: The name of the dataset, which will be saved in the dataset_description.json
    :param dataset_type: The type of the dataset, can be 'raw' or 'derivative', which will be saved in the dataset_description.json
    :param author: The author of the dataset, which will be saved in the dataset_description.json
    :param line_freq: line frequency of your raw data, normally 50 in China

    This function convert a raw eeg to BIDS format with some sidecar files. The EEG data will be
    saved following BrainVision format to satisfy the requirement of BIDS. Make sure that information in the
    data will not be lost when converting the data format.

    The parameter ica_component, ica_dict, bad_channel_dict are optional.
    '''
    raw.info["line_freq"] = line_freq  # specify power line frequency as required by BIDS

    # save files in need, including information about coordinate system, JSON of the eeg, channel information, ...
    bids_path = BIDSPath(subject=sub_id, session=ses, task=task, run=run, root=bids_root)
    basename = str(bids_path.basename)

    events, event_id = mne.events_from_annotations(raw)
    write_raw_bids(raw, bids_path, format='BrainVision', allow_preload=True, overwrite=True)

    fpath = bids_path.fpath
    eeg_file_directory = str(fpath.parent)

    ica_component_path = eeg_file_directory + '/' + basename + '_ica_components.npy'
    ica_topo_path = eeg_file_directory + '/' + basename + '_ica_components_topography'
    ica_json_path = eeg_file_directory + '/' + basename + '_ica_components.json'
    bad_channel_path = eeg_file_directory + '/' + basename + '_bad_channels.json'

    # save ICA components
    if ica_component is not None:
        np.save(ica_component_path, ica_component)

    # save ICA component JSON file
    if ica_dict is not None:
        ica_component_json = json.dumps(ica_dict, sort_keys=False, indent=4, separators=(',', ': '))
        f = open(ica_json_path, 'w')
        f.write(ica_component_json)

    # save bad channel record JSON file
    if bad_channel_dict is not None:
        bad_channel_json = json.dumps(bad_channel_dict, sort_keys=False, indent=4, separators=(',', ': '))
        f = open(bad_channel_path, 'w')
        f.write(bad_channel_json)

    if ica_topo_figs is not None:
        fig_num = 1
        if isinstance(ica_topo_figs, list):
            for topo in ica_topo_figs:
                if fig_num < 10:
                    fig_num_str = str(0) + str(fig_num)
                else:
                    fig_num_str = str(fig_num)
                topo.savefig(ica_topo_path + '_' + fig_num_str + '.png')
                fig_num += 1
        else:
            ica_topo_figs.savefig(ica_topo_path + '.png')

    print_dir_tree(bids_root)

    readme = op.join(bids_root, "README")
    with open(readme, "r", encoding="utf-8-sig") as fid:
        text = fid.read()
    print(text)

    mne_bids.make_dataset_description(path=bids_root, name=dataset_name, dataset_type=dataset_type,
                                      authors=author, overwrite=True)


# import requests
import aiohttp

from ...core.exceptions import BadRequest

INTERNAL_API_URL = 'https://sandbox-con.kakao.com/admin/internal-api/hello-mis/'

async def request_krew_info_from_ldap_id(ldap_id):
    async with aiohttp.ClientSession() as session:
        async with session.get(url=INTERNAL_API_URL, params={'name': ldap_id}) as response:
            response_json = await response.json()

    if 'error' in response_json:
        raise BadRequest()

    return response_json


import numpy as np
from xml.etree import ElementTree
from sint.base_dataset import BaseDataset

class Dataset(BaseDataset):

    def __init__(self, path_to_in_xml, valid_part=0.1, test_part=0.1):
        super(Dataset, self).__init__(path_to_in_xml, valid_part=valid_part, test_part=test_part)
        self.int_dict = {
            '0': 0,
            '11': 1,
            '30': 2,
            '40': 3,
            '50': 4,
            '70': 5,
            '110': 6
        }

    def parse_xml(self, return_intonation=False):
        tree_root = ElementTree.parse(self._path_to_in_xml).getroot()

        word_tags = []
        intonation_class = []
        content_tags = []

        word_ids = []
        for sentence in tree_root.iter('sentence'):
            # sentence_words = sentence.findall('word')
            tags_count = len(sentence)
            words_count = 0
            for idx in range(tags_count):
                if sentence[idx].tag == 'word':
                    word_ids.append(words_count)
                    words_count += 1
                    word_tags.append(sentence[idx])
                    if idx != tags_count-1:
                        if sentence[idx+1].tag == 'intonation':
                            intonation_class.append(sentence[idx+1].get('type'))
                        else:
                            intonation_class.append('0')
                    else:
                        intonation_class.append('0')

        intonation_class = list(map(lambda x: self.int_dict[x], intonation_class))
        intonation_class = np.array(intonation_class).astype(np.int)

        for content in tree_root.iter('content'):
            content_tags.append(content)

        words = []
        attrs = []
        for idx, word in enumerate(word_tags):
            _word = word.get('original')

            attr = {}
            if _word is not None:
                words.append(_word)
                attr['class'] = word.find('dictitem').get('subpart_of_speech')

                if idx+1 < len(content_tags):
                    p_end = content_tags[idx+1].get('PunktEnd')
                    if p_end is not None:
                        attr['punkt_end'] = int(p_end)
                    else:
                        attr['punkt_end'] = 0
                else:
                    attr['punkt_end'] = 0
                attr['pos'] = word_ids[idx]

                attrs.append(attr)

        if return_intonation:
            return words, attrs, intonation_class

        return words, attrs

    def create_ds(self, use_th_gr=False):
        words, attrs = self.parse_xml()

        classes = list(map(lambda x: x['class'], attrs))
        classes = np.array(classes, dtype=np.int)
        feats = self.get_features(words, attrs)

        if use_th_gr:
            feats = self.gen_three_grams(feats)

        feats, classes = Dataset.random_shuffle(feats, classes)
        num_ex = classes.shape[0]

        train_bound = int(num_ex * (1 - self._valid_part - self._test_part))
        valid_bound = int(num_ex * (1 - self._valid_part))

        self._train_set = {
            'data': feats[:train_bound, :],
            'labels': classes[:train_bound]
        }

        self._valid_set = {
            'data': feats[train_bound:valid_bound, :],
            'labels': classes[train_bound:valid_bound]
        }

        self._test_set = {
            'data': feats[valid_bound:, :],
            'labels': classes[valid_bound:]
        }

    def get_features(self, words_array, attr_array):
        feature_array = []

        for i in range(len(words_array)):
            word_features = list()

            last_sym = words_array[i][-1]

            if last_sym in self._sym_keys:
                word_features.append(self._sym_dict[last_sym])
            else:
                word_features.append(0)
            words_array[i] = words_array[i].strip(',. -:;!?')
            word_features.append(int(words_array[i][0].isupper()))
            word_features.append(len(words_array[i]))
            count_of_vowels = sum(map(Dataset.is_vowel, words_array[i]))
            word_features.append(count_of_vowels)
            word_features.append(ord(words_array[i][0]))
            word_features.append(ord(words_array[i][-1]))
            word_features.append(int(Dataset.has_symbol(words_array[i], 'ъ')))
            word_features.append(int(Dataset.has_symbol(words_array[i], 'ь')))
            word_features.append(int(Dataset.has_symbol(words_array[i], 'й')))
            word_features.append(attr_array[i]['pos'])

            if i < len(words_array) - 1:
                if attr_array[i + 1]['pos'] != 0:
                    word_features.append(0)
                else:
                    word_features.append(1)
            else:
                word_features.append(1)

            feature_array.append(np.array(word_features))

        return np.stack(feature_array)

    @staticmethod
    def gen_three_grams(feats):
        num_ex, num_feats = feats.shape
        three_grams_arr = np.zeros(shape=(num_ex, num_feats*3))

        for i in range(num_ex):
            if feats[i][-2] == 0:
                f_feat = np.zeros(num_feats)
            else:
                f_feat = feats[i-1]

            if feats[i][-1] == 1:
                l_feat = np.zeros(num_feats)
            else:
                l_feat = feats[i+1]

            three_grams_arr[i] = np.concatenate((f_feat, feats[i], l_feat))

        return three_grams_arr

    @staticmethod
    def has_symbol(in_str, sym):
        return sym in in_str

    @staticmethod
    def is_vowel(in_sym):
        vowel_set = set("аоиеёэыуюяАОИЕЁЭЫУЮЯ")

        if in_sym in vowel_set:
            return 1
        return 0

    @staticmethod
    def random_shuffle(x, y):
        num_ex = x.shape[0]

        perm = np.arange(num_ex)
        np.random.shuffle(perm)

        x = x[perm]
        y = y[perm]

        return x, y


import sqlite3
import argparse
import json
from pathlib import Path
import re

from collections import defaultdict

_PID_TO_DEVICE = None

# Code adapted from https://github.com/ezyang/nvprof2json

def parse_args():
    parser = argparse.ArgumentParser(description='Convert nsight systems sqlite output to Google Event Trace compatible JSON.')
    parser.add_argument("-f", '--filename', help="Path to the input sqlite file.", required=True)
    parser.add_argument("-o", "--output", help="Output file name, default to same as input with .json extension.")
    parser.add_argument("-t", "--activity-type", help="Type of activities shown. Default to all.", default=["kernel", "nvtx-kernel"], choices=['kernel', 'nvtx', "nvtx-kernel", "cuda-api"], nargs="+")
    parser.add_argument("--nvtx-event-prefix", help="Filter NVTX events by their names' prefix.", type=str, nargs="*")
    parser.add_argument("--nvtx-color-scheme", help="""Color scheme for NVTX events.
                                                    Accepts a dict mapping a string to one of chrome tracing colors.
                                                    Events with names containing the string will be colored.
                                                    E.g. {"send": "thread_state_iowait", "recv": "thread_state_iowait", "compute": "thread_state_running"}
                                                    For details of the color scheme, see links in https://github.com/google/perfetto/issues/208
                                                    """, type=json.loads, default={})
    args = parser.parse_args()
    if args.output is None:
        args.output = Path(args.filename).with_suffix(".json")
    return args

class ActivityType:
    KERNEL = "kernel"
    NVTX_CPU = "nvtx"
    NVTX_KERNEL = "nvtx-kernel"
    CUDA_API = "cuda-api"

def munge_time(t):
    """Take a timestamp from nsys (ns) and convert it into us (the default for chrome://tracing)."""
    # For strict correctness, divide by 1000, but this reduces accuracy.
    return t / 1000.

# For reference of the schema, see
# https://docs.nvidia.com/nsight-systems/UserGuide/index.html#exporter-sqlite-schema

def parse_cupti_kernel_events(conn: sqlite3.Connection, strings: dict):
    """
    Copied from the docs:
    CUPTI_ACTIVITY_KIND_KERNEL
    start                       INTEGER   NOT NULL,                    -- Event start timestamp (ns).
    end                         INTEGER   NOT NULL,                    -- Event end timestamp (ns).
    deviceId                    INTEGER   NOT NULL,                    -- Device ID.
    contextId                   INTEGER   NOT NULL,                    -- Context ID.
    streamId                    INTEGER   NOT NULL,                    -- Stream ID.
    correlationId               INTEGER,                               -- REFERENCES CUPTI_ACTIVITY_KIND_RUNTIME(correlationId)
    globalPid                   INTEGER,                               -- Serialized GlobalId.
    demangledName               INTEGER   NOT NULL,                    -- REFERENCES StringIds(id) -- Kernel function name w/ templates
    shortName                   INTEGER   NOT NULL,                    -- REFERENCES StringIds(id) -- Base kernel function name
    mangledName                 INTEGER,                               -- REFERENCES StringIds(id) -- Raw C++ mangled kernel function name
    launchType                  INTEGER,                               -- REFERENCES ENUM_CUDA_KRENEL_LAUNCH_TYPE(id)
    cacheConfig                 INTEGER,                               -- REFERENCES ENUM_CUDA_FUNC_CACHE_CONFIG(id)
    registersPerThread          INTEGER   NOT NULL,                    -- Number of registers required for each thread executing the kernel.
    gridX                       INTEGER   NOT NULL,                    -- X-dimension grid size.
    gridY                       INTEGER   NOT NULL,                    -- Y-dimension grid size.
    gridZ                       INTEGER   NOT NULL,                    -- Z-dimension grid size.
    blockX                      INTEGER   NOT NULL,                    -- X-dimension block size.
    blockY                      INTEGER   NOT NULL,                    -- Y-dimension block size.
    blockZ                      INTEGER   NOT NULL,                    -- Z-dimension block size.
    staticSharedMemory          INTEGER   NOT NULL,                    -- Static shared memory allocated for the kernel (B).
    dynamicSharedMemory         INTEGER   NOT NULL,                    -- Dynamic shared memory reserved for the kernel (B).
    localMemoryPerThread        INTEGER   NOT NULL,                    -- Amount of local memory reserved for each thread (B).
    localMemoryTotal            INTEGER   NOT NULL,                    -- Total amount of local memory reserved for the kernel (B).
    gridId                      INTEGER   NOT NULL,                    -- Unique grid ID of the kernel assigned at runtime.
    sharedMemoryExecuted        INTEGER,                               -- Shared memory size set by the driver.
    graphNodeId                 INTEGER,                               -- REFERENCES CUDA_GRAPH_EVENTS(graphNodeId)
    sharedMemoryLimitConfig     INTEGER                                -- REFERENCES ENUM_CUDA_SHARED_MEM_LIMIT_CONFIG(id)
    """
    per_device_kernel_rows = defaultdict(list)
    per_device_kernel_events = defaultdict(list)
    for row in conn.execute("SELECT * FROM CUPTI_ACTIVITY_KIND_KERNEL"):
        per_device_kernel_rows[row["deviceId"]].append(row)
        event = {
                "name": strings[row["shortName"]],
                "ph": "X", # Complete Event (Begin + End event)
                "cat": "cuda",
                "ts": munge_time(row["start"]),
                "dur": munge_time(row["end"] - row["start"]),
                "tid": "Stream {}".format(row["streamId"]),
                "pid": "Device {}".format(row["deviceId"]),
                "args": {
                    # TODO: More
                    },
                }
        per_device_kernel_events[row["deviceId"]].append(event)
    return per_device_kernel_rows, per_device_kernel_events

def link_pid_with_devices(conn: sqlite3.Connection):
    # map each pid to a device. assumes each pid is associated with a single device
    global _PID_TO_DEVICE
    if _PID_TO_DEVICE is None:
        pid_to_device = {}
        for row in conn.execute("SELECT DISTINCT deviceId, globalPid / 0x1000000 % 0x1000000 AS PID FROM CUPTI_ACTIVITY_KIND_KERNEL"):
            assert row["PID"] not in pid_to_device, \
                f"A single PID ({row['PID']}) is associated with multiple devices ({pid_to_device[row['PID']]} and {row['deviceId']})."
            pid_to_device[row["PID"]] = row["deviceId"]
        _PID_TO_DEVICE = pid_to_device
    return _PID_TO_DEVICE

def parse_nvtx_events(conn: sqlite3.Connection, event_prefix=None, color_scheme={}):
    """
    Copied from the docs:
    NVTX_EVENTS
    start                       INTEGER   NOT NULL,                    -- Event start timestamp (ns).
    end                         INTEGER,                               -- Event end timestamp (ns).
    eventType                   INTEGER   NOT NULL,                    -- NVTX event type enum value. See docs for specifics.
    rangeId                     INTEGER,                               -- Correlation ID returned from a nvtxRangeStart call.
    category                    INTEGER,                               -- User-controlled ID that can be used to group events.
    color                       INTEGER,                               -- Encoded ARGB color value.
    text                        TEXT,                                  -- Optional text message for non registered strings.
    globalTid                   INTEGER,                               -- Serialized GlobalId.
    endGlobalTid                INTEGER,                               -- Serialized GlobalId. See docs for specifics.
    textId                      INTEGER   REFERENCES StringIds(id),    -- StringId of the NVTX domain registered string.
    domainId                    INTEGER,                               -- User-controlled ID that can be used to group events.
    uint64Value                 INTEGER,                               -- One of possible payload value union members.
    int64Value                  INTEGER,                               -- One of possible payload value union members.
    doubleValue                 REAL,                                  -- One of possible payload value union members.
    uint32Value                 INTEGER,                               -- One of possible payload value union members.
    int32Value                  INTEGER,                               -- One of possible payload value union members.
    floatValue                  REAL,                                  -- One of possible payload value union members.
    jsonTextId                  INTEGER,                               -- One of possible payload value union members.
    jsonText                    TEXT                                   -- One of possible payload value union members.

    NVTX_EVENT_TYPES
    33 - NvtxCategory
    34 - NvtxMark
    39 - NvtxThread
    59 - NvtxPushPopRange
    60 - NvtxStartEndRange
    75 - NvtxDomainCreate
    76 - NvtxDomainDestroy
    """

    if event_prefix is None:
        match_text = ''
    else:
        match_text = " AND "
        if len(event_prefix) == 1:
            match_text += f"NVTX_EVENTS.text LIKE '{event_prefix[0]}%'"
        else:
            match_text += "("
            for idx, prefix in enumerate(event_prefix):
                match_text += f"NVTX_EVENTS.text LIKE '{prefix}%'"
                if idx == len(event_prefix) - 1:
                    match_text += ")"
                else:
                    match_text += " OR "

    per_device_nvtx_rows = defaultdict(list)
    per_device_nvtx_events = defaultdict(list)
    pid_to_device = link_pid_with_devices(conn)
    # eventType 59 is NvtxPushPopRange, which corresponds to torch.cuda.nvtx.range apis
    for row in conn.execute(f"SELECT start, end, text, globalTid / 0x1000000 % 0x1000000 AS PID, globalTid % 0x1000000 AS TID FROM NVTX_EVENTS WHERE NVTX_EVENTS.eventType == 59{match_text};"):
        text = row['text']
        pid = row['PID']
        tid = row['TID']
        per_device_nvtx_rows[pid_to_device[pid]].append(row)
        assert pid in pid_to_device, f"PID {pid} not found in the pid to device map."
        event = {
                "name": text,
                "ph": "X", # Complete Event (Begin + End event)
                "cat": "nvtx",
                "ts": munge_time(row["start"]),
                "dur": munge_time(row["end"] - row["start"]),
                "tid": "NVTX Thread {}".format(tid),
                "pid": "Device {}".format(pid_to_device[pid]),
                "args": {
                    # TODO: More
                    },
                }
        if color_scheme:
            for key, color in color_scheme.items():
                if re.search(key, text):
                    event["cname"] = color
                    break
        per_device_nvtx_events[pid_to_device[pid]].append(event)
    return per_device_nvtx_rows, per_device_nvtx_events

def parse_cuda_api_events(conn: sqlite3.Connection, strings: dict):
    """
    Copied from the docs:
    CUPTI_ACTIVITY_KIND_RUNTIME
    start                       INTEGER   NOT NULL,                    -- Event start timestamp (ns).
    end                         INTEGER   NOT NULL,                    -- Event end timestamp (ns).
    eventClass                  INTEGER   NOT NULL,                    -- CUDA event class enum value. See docs for specifics.
    globalTid                   INTEGER,                               -- Serialized GlobalId.
    correlationId               INTEGER,                               -- ID used to identify events that this function call has triggered.
    nameId                      INTEGER   NOT NULL   REFERENCES StringIds(id), -- StringId of the function name.
    returnValue                 INTEGER   NOT NULL,                    -- Return value of the function call.
    callchainId                 INTEGER   REFERENCES CUDA_CALLCHAINS(id) -- ID of the attached callchain.
    """
    pid_to_devices = link_pid_with_devices(conn)
    per_device_api_rows = defaultdict(list)
    per_device_api_events = defaultdict(list)
    # event type 0 is TRACE_PROCESS_EVENT_CUDA_RUNTIME
    for row in conn.execute(f"SELECT start, end, globalTid / 0x1000000 % 0x1000000 AS PID, globalTid % 0x1000000 AS TID, correlationId, nameId FROM CUPTI_ACTIVITY_KIND_RUNTIME;"):
        text = strings[row['nameId']]
        pid = row['PID']
        tid = row['TID']
        correlationId = row['correlationId']
        per_device_api_rows[pid_to_devices[pid]].append(row)
        event = {
                "name": text,
                "ph": "X", # Complete Event (Begin + End event)
                "cat": "cuda_api",
                "ts": munge_time(row["start"]),
                "dur": munge_time(row["end"] - row["start"]),
                "tid": "CUDA API Thread {}".format(tid),
                "pid": "Device {}".format(pid_to_devices[pid]),
                "args": {
                        "correlationId": correlationId,
                    },
                }
        per_device_api_events[pid_to_devices[pid]].append(event)
    return per_device_api_rows, per_device_api_events

def _find_overlapping_intervals(nvtx_rows, cuda_api_rows):
    mixed_rows = []
    for nvtx_row in nvtx_rows:
        start = nvtx_row["start"]
        end = nvtx_row["end"]
        mixed_rows.append((start, 1, "nvtx", nvtx_row))
        mixed_rows.append((end, -1, "nvtx", nvtx_row))
    for cuda_api_row in cuda_api_rows:
        start = cuda_api_row["start"]
        end = cuda_api_row["end"]
        mixed_rows.append((start, 1, "cuda_api", cuda_api_row))
        mixed_rows.append((end, -1, "cuda_api", cuda_api_row))
    mixed_rows.sort(key=lambda x: (x[0], x[1], x[2]))
    active_intervals = []
    result = defaultdict(list)
    for _, event_type, event_origin, orig_event in mixed_rows:
        if event_type == 1:
            # start
            if event_origin == "nvtx":
                active_intervals.append(orig_event)
            else:
                for event in active_intervals:
                    result[event].append(orig_event)
        else:
            # end
            if event_origin == "nvtx":
                active_intervals.remove(orig_event)
    return result

def link_nvtx_events_to_kernel_events(strings: dict,
                                      pid_to_device: dict[int, int],
                                      per_device_nvtx_rows: dict[int, list],
                                      per_device_cuda_api_rows: dict[int, list],
                                      per_device_cuda_kernel_rows: dict[int, list],
                                      per_device_kernel_events: dict[int, list]):
    """
    Link NVTX events to cupti kernel events. This is done by first matching
    the nvtx ranges with CUDA API calls by timestamp. Then, retrieve the
    corresponding kernel events using the correlationId from CUDA API calls.
    """
    result = {}
    for device in pid_to_device.values():
        event_map = _find_overlapping_intervals(per_device_nvtx_rows[device], per_device_cuda_api_rows[device])
        correlation_id_map = defaultdict(dict)
        for cuda_api_row in per_device_cuda_api_rows[device]:
            correlation_id_map[cuda_api_row["correlationId"]]["cuda_api"] = cuda_api_row
        for kernel_row, kernel_trace_event in zip(per_device_cuda_kernel_rows[device], per_device_kernel_events[device]):
            correlation_id_map[kernel_row["correlationId"]]["kernel"] = kernel_row
            correlation_id_map[kernel_row["correlationId"]]["kernel_trace_event"] = kernel_trace_event
        for nvtx_row, cuda_api_rows in event_map.items():
            kernel_start_time = None
            kernel_end_time = None
            for cuda_api_row in cuda_api_rows:
                if "kernel" not in correlation_id_map[cuda_api_row["correlationId"]]:
                    # other cuda api event, ignore
                    continue
                kernel_row = correlation_id_map[cuda_api_row["correlationId"]]["kernel"]
                kernel_trace_event = correlation_id_map[cuda_api_row["correlationId"]]["kernel_trace_event"]
                if "NVTXRegions" not in kernel_trace_event["args"]:
                    kernel_trace_event["args"]["NVTXRegions"] = []
                kernel_trace_event["args"]["NVTXRegions"].append(nvtx_row["text"])
                if kernel_start_time is None or kernel_start_time > kernel_row["start"]:
                    kernel_start_time = kernel_row["start"]
                if kernel_end_time is None or kernel_end_time < kernel_row["end"]:
                    kernel_end_time = kernel_row["end"]
            if kernel_start_time is not None and kernel_end_time is not None:
                result[nvtx_row] = (kernel_start_time, kernel_end_time)
    return result

def parse_all_events(conn: sqlite3.Connection, strings: dict, activities=None, event_prefix=None, color_scheme={}):
    if activities is None:
        activities = [ActivityType.KERNEL, ActivityType.NVTX_CPU, ActivityType.NVTX_KERNEL]
    if ActivityType.KERNEL in activities or ActivityType.NVTX_KERNEL in activities:
        per_device_kernel_rows, per_device_kernel_events = parse_cupti_kernel_events(conn, strings)
    if ActivityType.NVTX_CPU in activities or ActivityType.NVTX_KERNEL in activities:
        per_device_nvtx_rows, per_device_nvtx_events = parse_nvtx_events(conn, event_prefix=event_prefix, color_scheme=color_scheme)
    if ActivityType.CUDA_API in activities or ActivityType.NVTX_KERNEL in activities:
        per_device_cuda_api_rows, per_device_cuda_api_events = parse_cuda_api_events(conn, strings)
    if ActivityType.NVTX_KERNEL in activities:
        pid_to_device = link_pid_with_devices(conn)
        nvtx_kernel_event_map = link_nvtx_events_to_kernel_events(strings, pid_to_device, per_device_nvtx_rows, per_device_cuda_api_rows, per_device_kernel_rows, per_device_kernel_events)
    traceEvents = []
    if ActivityType.KERNEL in activities:
        for k, v in per_device_kernel_events.items():
            traceEvents.extend(v)
    if ActivityType.NVTX_CPU in activities:
        for k, v in per_device_nvtx_events.items():
            traceEvents.extend(v)
    if ActivityType.CUDA_API in activities:
        for k, v in per_device_cuda_api_events.items():
            traceEvents.extend(v)
    if ActivityType.NVTX_KERNEL in activities:
        for nvtx_event, (kernel_start_time, kernel_end_time) in nvtx_kernel_event_map.items():
            event = {
                "name": nvtx_event["text"],
                "ph": "X", # Complete Event (Begin + End event)
                "cat": "nvtx-kernel",
                "ts": munge_time(kernel_start_time),
                "dur": munge_time(kernel_end_time - kernel_start_time),
                "tid": "NVTX Kernel Thread {}".format(nvtx_event["tid"]),
                "pid": "Device {}".format(pid_to_device[nvtx_event["pid"]]),
                "args": {
                    # TODO: More
                    },
                }
            traceEvents.append(event)
    return traceEvents

def nsys2json():
    args = parse_args()
    conn = sqlite3.connect(args.filename)
    conn.row_factory = sqlite3.Row

    strings = {}
    for r in conn.execute("SELECT id, value FROM StringIds"):
        strings[r["id"]] = r["value"]

    traceEvents = parse_all_events(conn, strings, activities=args.activity_type, event_prefix=args.nvtx_event_prefix, color_scheme=args.nvtx_color_scheme)
    # make the timelines appear in pid and tid order
    traceEvents.sort(key=lambda x: (x["pid"], x["tid"]))

    with open(args.output, 'w') as f:
        json.dump(traceEvents, f)

if __name__ == "__main__":
    nsys2json()