# -*- coding: utf-8 -*-
"""
/***************************************************************************
 ClipRaster
                                 A QGIS plugin
 This plugin allows you to split raster files based on given polygon data.
 Generated by Plugin Builder: http://g-sherman.github.io/Qgis-Plugin-Builder/
                              -------------------
        begin                : 2020-12-13
        git sha              : $Format:%H$
        copyright            : (C) 2020 by Murat ÇALIŞKAN
        email                : caliskan.murat.20@gmail.com
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
"""
from qgis.PyQt.QtCore import QSettings, QTranslator, QCoreApplication
from qgis.PyQt.QtGui import QIcon
from qgis.PyQt.QtWidgets import QAction, QFileDialog
from qgis.utils import iface

from qgis.core import Qgis, QgsProject, QgsGeometry
import processing

# Initialize Qt resources from file resources.py
from .resources import *
# Import the code for the dialog
from .easy_raster_splitter_dialog import ClipRasterDialog
import os.path

import numpy as np
from osgeo import gdal
import os

class ClipRaster:
    """QGIS Plugin Implementation."""

    def __init__(self, iface):
        """Constructor.

        :param iface: An interface instance that will be passed to this class
            which provides the hook by which you can manipulate the QGIS
            application at run time.
        :type iface: QgsInterface
        """
        # Save reference to the QGIS interface
        self.iface = iface
        # initialize plugin directory
        self.plugin_dir = os.path.dirname(__file__)
        # initialize locale
        locale = QSettings().value('locale/userLocale')[0:2]
        locale_path = os.path.join(
            self.plugin_dir,
            'i18n',
            'ClipRaster_{}.qm'.format(locale))

        if os.path.exists(locale_path):
            self.translator = QTranslator()
            self.translator.load(locale_path)
            QCoreApplication.installTranslator(self.translator)

        # Declare instance attributes
        self.actions = []
        self.menu = self.tr(u'&Easy Raster Splitter')

        # Check if plugin was started the first time in current QGIS session
        # Must be set in initGui() to survive plugin reloads
        self.first_start = None

    # noinspection PyMethodMayBeStatic
    def tr(self, message):
        """Get the translation for a string using Qt translation API.

        We implement this ourselves since we do not inherit QObject.

        :param message: String for translation.
        :type message: str, QString

        :returns: Translated version of message.
        :rtype: QString
        """
        # noinspection PyTypeChecker,PyArgumentList,PyCallByClass
        return QCoreApplication.translate('ClipRaster', message)

    def add_action(
        self,
        icon_path,
        text,
        callback,
        enabled_flag=True,
        add_to_menu=True,
        add_to_toolbar=True,
        status_tip=None,
        whats_this=None,
        parent=None):
        """Add a toolbar icon to the toolbar.

        :param icon_path: Path to the icon for this action. Can be a resource
            path (e.g. ':/plugins/foo/bar.png') or a normal file system path.
        :type icon_path: str

        :param text: Text that should be shown in menu items for this action.
        :type text: str

        :param callback: Function to be called when the action is triggered.
        :type callback: function

        :param enabled_flag: A flag indicating if the action should be enabled
            by default. Defaults to True.
        :type enabled_flag: bool

        :param add_to_menu: Flag indicating whether the action should also
            be added to the menu. Defaults to True.
        :type add_to_menu: bool

        :param add_to_toolbar: Flag indicating whether the action should also
            be added to the toolbar. Defaults to True.
        :type add_to_toolbar: bool

        :param status_tip: Optional text to show in a popup when mouse pointer
            hovers over the action.
        :type status_tip: str

        :param parent: Parent widget for the new action. Defaults None.
        :type parent: QWidget

        :param whats_this: Optional text to show in the status bar when the
            mouse pointer hovers over the action.

        :returns: The action that was created. Note that the action is also
            added to self.actions list.
        :rtype: QAction
        """

        icon = QIcon(icon_path)
        action = QAction(icon, text, parent)
        action.triggered.connect(callback)
        action.setEnabled(enabled_flag)

        if status_tip is not None:
            action.setStatusTip(status_tip)

        if whats_this is not None:
            action.setWhatsThis(whats_this)

        if add_to_toolbar:
            # Adds plugin icon to Plugins toolbar
            self.iface.addToolBarIcon(action)

        if add_to_menu:
            self.iface.addPluginToRasterMenu(
                self.menu,
                action)

        self.actions.append(action)

        return action

    def initGui(self):
        """Create the menu entries and toolbar icons inside the QGIS GUI."""

        icon_path = ':/plugins/easy_raster_splitter/icon.png'
        self.add_action(
            icon_path,
            text=self.tr(u'Easy Raster Splitter'),
            callback=self.run,
            parent=self.iface.mainWindow())

        # will be set False in run()
        self.first_start = True

    def unload(self):
        """Removes the plugin menu item and icon from QGIS GUI."""
        for action in self.actions:
            self.iface.removePluginRasterMenu(
                self.tr(u'&Easy Raster Splitter'),
                action)
            self.iface.removeToolBarIcon(action)

    def selectOutput(self):
        self.dlg.lineEdit_outRaster.setText("")
        self.OutputDir = QFileDialog.getExistingDirectory(None, 'Open working directory', "", QFileDialog.ShowDirsOnly)
        self.dlg.lineEdit_outRaster.setText(self.OutputDir)
        self.dlg.button_box.setEnabled(True)

    def getLayers(self):
        self.dlg.comboBox_in_vector.clear()
        self.dlg.comboBox_in_raster.clear()

        self.dlg.comboBox_in_vector.addItems(self.vector_layers.keys())
        self.dlg.comboBox_in_raster.addItems(self.raster_layers.keys())

    def fillFields(self):
        self.dlg.comboBox_names.clear()
        self.selectedLayer = self.vector_layers[self.dlg.comboBox_in_vector.currentText()]
        self.fields = ["fid"] + [field.name() for field in self.selectedLayer.fields() if (field.typeName().lower()=='string') or (field.typeName().lower().startswith("int"))]
        self.dlg.comboBox_names.addItems(self.fields)

    def toggleCheckBox(self, name):
        if name == "full":
            if self.dlg.checkBox_full.isChecked():
                self.dlg.lineEdit_name.setEnabled(True)
                self.dlg.lineEdit_name.setText("")
            else:
                self.dlg.lineEdit_name.setEnabled(False)
                self.dlg.lineEdit_name.setText("")

        elif name == "feature":
            if self.dlg.checkBox_feature.isChecked():
                self.dlg.comboBox_names.setEnabled(True)
                self.fillFields()
            else:
                self.dlg.comboBox_names.setEnabled(False)
                self.dlg.comboBox_names.clear()

    def getResults(self, layer, raster, out_folder, geot, nodata, outputNameFeatureField, outputNameFull, erased=False, split_features=False, split_full=True):

        if split_full:
            if not outputNameFull:
                outputNameFull = "outRaster_full"

            self.out_file = os.path.join(out_folder, outputNameFull + ".tif")

            if not erased:
                params = {
                      'INPUT': self.dlg.comboBox_in_raster.currentText(),
                      'MASK': self.dlg.comboBox_in_vector.currentText(),
                      'NODATA': None,
                      'ALPHA_BAND': False,
                      'CROP_TO_CUTLINE': True,
                      'KEEP_RESOLUTION': False,
                      'OPTIONS': 'COMPRESS=LZW',
                      'DATA_TYPE': 0,
                      'OUTPUT': self.out_file,
                }

                self.result = processing.run('gdal:cliprasterbymasklayer', params)

                if self.dlg.checkBox_add.isChecked():
                    iface.addRasterLayer(self.out_file, outputNameFull)

            else:
                self.raster_layer_name = self.dlg.comboBox_in_raster.currentText()
                self.vector_layer_name = self.dlg.comboBox_in_vector.currentText()

                self.vectorLayer = QgsProject.instance().mapLayersByName(self.vector_layer_name)[0]

                self.paramsExtent = {
                    'INPUT':self.raster_layer_name,
                    'ROUND_TO':0,
                    'OUTPUT':'TEMPORARY_OUTPUT'
                }

                self.rasterExtent = processing.run("native:polygonfromlayerextent", self.paramsExtent)
                self.rasterExtent = self.rasterExtent["OUTPUT"]

                self.extentFeatures = {i.id():i.geometry() for i in self.rasterExtent.getFeatures()}
                self.vectorFeatures = [i.geometry() for i in self.vectorLayer.getFeatures()]

                for geomLayer in self.vectorFeatures:
                    for k, geomExtent in self.extentFeatures.items():
                        self.extentFeatures[k] = geomExtent.difference(geomLayer)

                self.rasterExtent.startEditing()
                for fid,geom in self.extentFeatures.items():
                    _ = self.rasterExtent.changeGeometry(fid,geom)
                self.rasterExtent.commitChanges()

                self.paramsClip = {
                    'INPUT': self.raster_layer_name,
                    'MASK': self.rasterExtent,
                    'ALPHA_BAND': False,
                    'CROP_TO_CUTLINE': True,
                    'KEEP_RESOLUTION': False,
                    'OPTIONS': 'COMPRESS=LZW',
                    'DATA_TYPE': 0,
                    'OUTPUT': self.out_file,
                }

                _ = processing.run('gdal:cliprasterbymasklayer', self.paramsClip)

                if self.dlg.checkBox_add.isChecked():
                    iface.addRasterLayer(self.out_file, outputNameFull)

        if split_features:
            if erased:
                self.raster_layer_name = self.dlg.comboBox_in_raster.currentText()
                self.vector_layer_name = self.dlg.comboBox_in_vector.currentText()

                self.vectorLayer = QgsProject.instance().mapLayersByName(self.vector_layer_name)[0]

                self.paramsExtent = {
                    'INPUT':self.raster_layer_name,
                    'ROUND_TO':0,
                    'OUTPUT':'TEMPORARY_OUTPUT'
                }

                self.rasterExtent = processing.run("native:polygonfromlayerextent", self.paramsExtent)
                self.rasterExtent = self.rasterExtent["OUTPUT"]
                self.extentFeatures = {i.id():i.geometry() for i in self.rasterExtent.getFeatures()}

                self.currentFilterText = self.vectorLayer.subsetString()

                if outputNameFeatureField != "fid":
                    self.features = [str(i[outputNameFeatureField]) for i in self.vectorLayer.getFeatures()]
                else:
                    self.features = [str(i.id()) for i in self.vectorLayer.getFeatures()]

                for name in set(self.features):
                    self.out_file = os.path.join(out_folder, name+".tif")

                    if outputNameFeatureField =="fid":
                        if self.vectorLayer.storageType()=='Memory storage':
                            self.q = '$id = \'{}\''.format(name)
                        else:
                            self.q = '"fid" = \'{}\''.format(name)
                    else:
                        self.q = '"{}" = \'{}\''.format(outputNameFeatureField, name)

                    self.vectorLayer.setSubsetString(self.q)

                    self.vectorFeatures = [i.geometry() for i in self.vectorLayer.getFeatures()]
                    for k, geomExtent in self.extentFeatures.items():
                        self.geomExtent = QgsGeometry.fromRect(geomExtent.boundingBox())
                        for geomLayer in self.vectorFeatures:
                            self.extentFeatures[k] = self.geomExtent.difference(geomLayer)

                    self.rasterExtent.startEditing()
                    for fid,geom in self.extentFeatures.items():
                        _ = self.rasterExtent.changeGeometry(fid,geom)
                    self.rasterExtent.commitChanges()

                    self.params = {
                        'INPUT': self.raster_layer_name,
                        'MASK': self.rasterExtent,
                        'ALPHA_BAND': False,
                        'CROP_TO_CUTLINE': True,
                        'KEEP_RESOLUTION': False,
                        'OPTIONS': 'COMPRESS=LZW',
                        'DATA_TYPE': 0,
                        'OUTPUT': self.out_file,
                    }
                    self.result = processing.run('gdal:cliprasterbymasklayer', self.params)

                    if self.dlg.checkBox_add.isChecked():
                        iface.addRasterLayer(self.out_file, str(name))

                self.vectorLayer.setSubsetString(self.currentFilterText)

            else:

                self.rasterLayer = QgsProject.instance().mapLayersByName(self.dlg.comboBox_in_raster.currentText())[0]
                self.vectorLayer = QgsProject.instance().mapLayersByName(self.dlg.comboBox_in_vector.currentText())[0]

                self.currentFilterText = self.vectorLayer.subsetString()

                if outputNameFeatureField != "fid":
                    self.features = [str(i[outputNameFeatureField]) for i in self.vectorLayer.getFeatures()]
                else:
                    self.features = [str(i.id()) for i in self.vectorLayer.getFeatures()]

                for name in set(self.features):
                    self.out_file = os.path.join(out_folder, name+".tif")
                    if outputNameFeatureField =="fid":
                        if self.vectorLayer.storageType()=='Memory storage':
                            self.q = '$id = \'{}\''.format(name)
                        else:
                            self.q = '"fid" = \'{}\''.format(name)
                    else:
                        self.q = '"{}" = \'{}\''.format(outputNameFeatureField, name)

                    self.vectorLayer.setSubsetString(self.q)

                    self.params = {
                        'INPUT': self.rasterLayer,
                        'MASK': self.vectorLayer,
                        'ALPHA_BAND': False,
                        'CROP_TO_CUTLINE': True,
                        'KEEP_RESOLUTION': False,
                        'OPTIONS': 'COMPRESS=LZW',
                        'DATA_TYPE': 0,
                        'OUTPUT': self.out_file,
                    }
                    self.result = processing.run('gdal:cliprasterbymasklayer', self.params)

                    if self.dlg.checkBox_add.isChecked():
                        iface.addRasterLayer(self.out_file, str(name))

                self.vectorLayer.setSubsetString(self.currentFilterText)

    def run(self):
        """Run method that performs all the real work"""
        # Create the dialog with elements (after translation) and keep reference
        # Only create GUI ONCE in callback, so that it will only load when the plugin is started
        if self.first_start == True:
            self.dlg = ClipRasterDialog()
            self.vector_layers = {layer.name():layer for layer in QgsProject.instance().mapLayers().values() if (layer.type()== 0 and layer.wkbType() in (3,6))}
            self.raster_layers = {layer.name():layer for layer in QgsProject.instance().mapLayers().values() if layer.type()== 1}
            self.getLayers()

            self.dlg.checkBox_full.toggled.connect(lambda x:self.toggleCheckBox("full"))
            self.dlg.checkBox_feature.toggled.connect(lambda x:self.toggleCheckBox("feature"))
            self.dlg.comboBox_in_vector.currentTextChanged.connect(self.fillFields)
            self.dlg.toolButton_selectOutFolder.clicked.connect(self.selectOutput)

        # show the dialog
        self.dlg.show()

        # Run the dialog event loop
        result = self.dlg.exec_()
        # See if OK was pressed
        if result:
            # Do something useful here - delete the line containing pass and
            # substitute with your code.

            self.rasterName = self.dlg.comboBox_in_raster.currentText()
            self.vectorName = self.dlg.comboBox_in_vector.currentText()

            self.raster_path = self.raster_layers[self.rasterName].source()
            self.vector_layer = self.vector_layers[self.vectorName]

            self.out_folder = self.dlg.lineEdit_outRaster.text()
            self.erased = False if self.dlg.radioButton_clip.isChecked() else True
            self.split_features = True if self.dlg.checkBox_feature.isChecked() else False
            self.split_full = True if self.dlg.checkBox_full.isChecked() else False
            self.outputNameFeatureField = self.dlg.comboBox_names.currentText()
            self.outputNameFull = self.dlg.lineEdit_name.text()

            self.raster = gdal.Open(self.raster_path)
            self.geot = self.raster.GetGeoTransform()
            self.bands = [self.raster.GetRasterBand(band_no) for band_no in range(1, self.raster.RasterCount+1)]
            self.dtype = self.bands[0].DataType
            self.nodata = self.bands[0].GetNoDataValue()

            if self.nodata is None:
                self.arrays = [band.ReadAsArray(0,0,0,0) for band in self.bands]
                try:
                    self.nanofbands = [np.iinfo(array.dtype).min for array in self.arrays]
                    self.nodata = min(self.nanofbands)
                except:
                    self.nanofbands = [np.finfo(array.dtype).min for array in self.arrays]
                    self.nodata = min(self.nanofbands)

            if self.out_folder:
                self.getResults(self.vector_layer, self.raster, self.out_folder, self.geot, self.nodata, self.outputNameFeatureField, self.outputNameFull, self.erased, self.split_features, self.split_full)
                self.iface.messageBar().pushMessage("Success", "Raster was clipped/erased successfully!" , level=Qgis.Success, duration=5)


import os
import tqdm
import numpy as np
from pathlib import Path
from PIL import Image

clsID_to_trID = {
    0: 255,
    1: 0,
    2: 1,
    3: 2,
    4: 3,
    5: 4,
    6: 5,
    7: 6,
    8: 7,
    9: 8,
    10: 9,
    11: 10,
    12: 11,
    13: 12,
    14: 13,
    15: 14,
    16: 15,
    17: 16,
    18: 17,
    19: 18,
    20: 19,
    255: 255,
}

clsID_to_trID_w_background = clsID_to_trID.copy()
clsID_to_trID_w_background[0] = 20

def convert_to_trainID(
    maskpath, out_mask_dir, is_train, clsID_to_trID=clsID_to_trID, suffix="",
):
    mask = np.array(Image.open(maskpath))
    mask = np.vectorize(clsID_to_trID.get)(mask).astype(np.uint8)

    seg_filename = (
        os.path.join(out_mask_dir, "train" + suffix, os.path.basename(maskpath))
        if is_train
        else os.path.join(out_mask_dir, "val" + suffix, os.path.basename(maskpath))
    )
    if len(np.unique(mask)) == 1 and (np.unique(mask)[0] == 255 or np.unique(mask)[0] == 20):
        # ignore images with only background
        return
    Image.fromarray(mask).save(seg_filename, "PNG")

def download_dataset(ds_path):
    """
    Downloads the dataset
    """
    print('Downloading dataset...')
    # Downloading images
    os.system('wget http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar')
    os.system(f'tar -xvf VOCtrainval_11-May-2012.tar -C {ds_path.parent.parent}')
    os.system(f'rm VOCtrainval_11-May-2012.tar')
    # Downloading annotations
    os.system('wget https://www.dropbox.com/s/oeu149j8qtbs1x0/SegmentationClassAug.zip')
    os.system('unzip SegmentationClassAug.zip -d ' + str(ds_path))
    os.system('rm SegmentationClassAug.zip')

def main():
    dataset_dir = Path(os.getenv("DETECTRON2_DATASETS", "datasets"))
    voc_path = dataset_dir / "VOCdevkit" / "VOC2012"

    if not voc_path.exists():
        download_dataset(voc_path)

    out_mask_dir = voc_path / "annotations_detectron2"

    for name in ["val"]:
        os.makedirs((out_mask_dir / name), exist_ok=True)
        os.makedirs((out_mask_dir / (name + '_bg')), exist_ok=True)

        val_list = [
            os.path.join(voc_path, "SegmentationClassAug", f + ".png")
            for f in np.loadtxt(os.path.join(voc_path, "ImageSets/Segmentation/val.txt"), dtype=np.str).tolist()
        ]

        for file in tqdm.tqdm(val_list):
            convert_to_trainID(file, out_mask_dir, is_train=False)
            convert_to_trainID(file, out_mask_dir, is_train=False,
                               clsID_to_trID=clsID_to_trID_w_background, suffix="_bg")

if __name__ == "__main__":
    main()


from requests import exceptions, get
import logging

def __downloadFilings(cik: str) -> list:
    """Function to download the XML text of listings pages for a given CIK
    from the EDGAR database.

    Arguments:
        cik {str} -- Target CIK.

    Returns:
        list -- List of page XML, comprising full listing metadata for CIK.
    """

    idx = 0  # Current page index
    end = False  # Flags for loop
    count = 100  # Number of results per page (limited by SEC)

    # Text indicating next page exists
    next_page_text = 'rel="next" type="application/atom+xml" />'

    pages = []

    while not end:
        # Making request
        page_text = __makeRequest(cik=cik, start_idx=idx, count=count)
        end = (page_text.find(next_page_text) == -1)  # Update end flag
        idx += count  # Increment index for next page
        pages.append(page_text)  # Save page text

    return pages

def __makeRequest(cik: str, start_idx: int, count: int, retry: bool=False) \
    -> str:
    """Function to make a request to the EDGAR system to retrieve XML with
    listings for a given CIK.

    Arguments:
        cik {str} -- Target CIK.
        start_idx {int} -- Start index (for pagination).
        count {int} -- Count of results per page.

    Keyword Arguments:
        retry {bool} -- Flag for auto-retry (default: {False}).

    Raises:
        exceptions.RetryError -- Raised if the request retry fails.

    Returns:
        str -- Page text with XML listing metadata for the target CIK.
    """

    base_url = 'https://sec.gov/cgi-bin/browse-edgar'

    # Building parameters for request wit hticker
    # Note: Order of parameters here is important; null params are
    #       necessary cuz the SEC is silly
    params = {
        'action': 'getcompany',
        'CIK': cik,
        'type': '',
        'dateb': '',
        'owner': 'exclude',
        'start': start_idx,
        'count': count,
        'output': 'atom'
    }

    # Making request
    r = get(url=base_url, params=params)

    # Retry request if it fails; raise Exception if it still doesn't work
    if (not r.ok) and (not retry):
        logging.warn('Listings request failed for CIK {0}. Trying again...'
                     .format(cik))
        return __makeRequest(cik=cik, start_idx=start_idx, count=count,
                             retry=True)
    elif not r.ok:
        logging.error('Listings request failed for CIK {0}'.format(cik))
        raise exceptions.RetryError('Listings request retry failed for CIK {0}'
                                    .format(cik))

    return r.text


# local libs

from categories import Category
from items import VisitsOutsideUKItem
from utils import regex_for_registered, regex_for_amount, get_regex_pair_search

class VisitsOutsideUK(Category):
	def __init__(self):
		"""
		Visits outside the UK
		"""

		# Init the class, then set some class specific variables
		Category.__init__(self)

		# category info
		self.category_id = 5
		self.category_type = 'visits_outside_uk'
		self.category_description = 'Vists outside UK'
		self.isCurrency = True

	def do_logic(self, raw_string):
		"""
		Method performing the logic of parsing raw data into dictionary
		"""

		amount = 0
		next_id = len(self.items) + 1
		item_id = '%04d' % next_id

		# not much we can really split on
		pretty = raw_string.split(' (Registered')[0]
		registered = regex_for_registered(raw_string)

		# hold a list of lists with search pair, to then regex out the value between them
		regex_pairs = []
		regex_pairs.append(['Name of donor: ', 'Address of donor: '])
		regex_pairs.append(['Address of donor: ', 'Amount of donation '])
		regex_pairs.append(['Address of donor: ', 'Estimate of the probable value'])
		regex_pairs.append(['Destination of visit: ', 'Dates of visit: '])
		regex_pairs.append(['Destination of visit: ', 'Date of visit: '])
		regex_pairs.append(['Dates of visit: ', 'Purpose of visit: '])
		regex_pairs.append(['Date of visit: ', 'Purpose of visit: '])
		regex_pairs.append(['Purpose of visit: ', r'\(Registered '])

		amount = regex_for_amount(raw_string)

		donor = None
		address = None
		destination = None
		dates = None
		purpose = None

		for pair in regex_pairs:
			regex_search = get_regex_pair_search(pair, raw_string)

			if regex_search:
				value = regex_search.group(1)

				if 'name of donor' in pair[0].lower():
					donor = value
				elif 'address of donor' in pair[0].lower():
					address = value
				elif 'destination' in pair[0].lower():
					destination = value
				elif 'date' in pair[0].lower():
					dates = value
				elif 'dates' in pair[0].lower():
					dates = value
				elif 'purpose' in pair[0].lower():
					pretty = value

		item = VisitsOutsideUKItem(item_id, self.category_id, raw_string, pretty, registered, amount)
		item.donor = donor
		item.address = address
		item.destination = destination
		item.dates = dates

		self.items.append(item)


import sqlite3

DATABASE_PATH = "/Users/nick/Dropbox/lab_notebook/projects_and_data/mnc/analysis_and_data/patch_clamp/data/summary_data/spiking_201912-202001/patch_data_batch.db"

def persistent_connection_to_db(path):
    """returns a connection object for the `path` to a database"""
    return sqlite3.connect(path)

def add_to_db(path, query_string):
    """uses a context manager to open a connection to database at `path` and execute
    query in `query_string`. Returns a 0 for success or a 1 for failure."""
    try:
        con = sqlite3.connect(path)
        with con:
            con.execute(query_string)
        con.close()
        return 0
    except Exception as e:
        print(f"Failed to execute. Query: {query_string}\n with error:\n{e}")
        con.close()
        return 1

def add_to_db_parameterized(path, query_string, data):
    """uses a context manager to open a connection to database at `path` and execute
    query in `query_string`. Returns a 0 for success or a 1 for failure."""
    try:
        con = sqlite3.connect(path)
        with con:
            con.execute(query_string, data)
        con.close()
        return 0
    except Exception as e:
        print(f"Failed to execute. Query: {query_string}\n with error:\n{e}")
        con.close()
        return 1

def get_paths_for_protocol(path_to_db, prot):
    """returns a list of paths from metadata which matching the protocol `prot` and
    `include` is `yes` or `maybe`"""
    query = "SELECT fpath FROM metadata WHERE protocol = ? AND (include = 'yes' OR include = 'maybe')"
    try:
        con = sqlite3.connect(path_to_db)
        con.row_factory = sqlite3.Row
        with con:
            things = con.execute(query, (prot,)).fetchall()
        con.close()
        return sorted([s["fpath"] for s in things])
    except Exception as e:
        print(f"Failed to execute. Query: {query}\n with error:\n{e}")
        con.close()
        return 1

def sql_data_as_dict(path_to_db: str, select_query: str) -> list:
    """Returns data from an query as a list of dicts."""
    try:
        con = sqlite3.connect(path_to_db)
        con.row_factory = sqlite3.Row
        things = con.execute(select_query).fetchall()
        unpacked = [{k: item[k] for k in item.keys()} for item in things]
        return unpacked
    except Exception as e:
        print(f"Failed to execute. Query: {select_query}\n with error:\n{e}")
        return []
    finally:
        con.close()

def list_of_ints_to_str(list_of_ints):
    return ",".join([str(i) for i in list_of_ints])

def list_of_floats_to_str(list_of_floats):
    return ",".join([str(i) for i in list_of_floats])

def str_list_to_list_floats(str_list):
    if not str_list:
        return []
    return [float(i) for i in str_list.split(",")]

def str_list_to_list_ints(str_list):
    if not str_list:
        return []
    return [int(i) for i in str_list.split(",")]


import jax
import jax.numpy as jnp
import numpy as np
import logging

from ase.neighborlist import primitive_neighbor_list
from functools import partial
from typing import Dict
from tqdm import tqdm

from mlff.geometric.metric import coordinates_to_distance_matrix, coordinates_to_distance_matrix_mic
from mlff.utils import PrimitiveNeighbors

def get_indices(R: np.ndarray, z: np.ndarray, r_cut: float, cell: np.ndarray = None, mic: bool = False):
    """
        For the `n_data` data frames, return index lists for centering and neighboring atoms given some cutoff radius
        `r_cut` for each structure. As atoms may leave or enter the neighborhood for a given atom within the dataset,
        one can have different lengths for the index lists, even for same structures. Thus, the index lists are padded
        wrt the length `n_pairs_max` of the longest index list observed over all frames in the coordinates `R` across
        all structures. Note, that this is suboptimal if one has a wide range number of atoms in the same dataset. The
        coordinates `R` and the atomic types `z` can also be already padded (see `mlff.src.padding.padding`) and are
        assumed to be padded with `0` if padded. Index values are padded with `-1`.

        Args:
            R (Array): Atomic coordinates, shape: (n_data,n,3)
            z (Array): Atomic types, shape: (n_data, n)
            r_cut (float): Cutoff distance
            cell (Array): Unit cell with lattice vectors along rows (ASE default), shape: (n_data,3,3)
            mic (bool): Minimal image convention for periodic boundary conditions

        Returns: Tuple of centering and neighboring indices, shape: Tuple[(n_pairs_max), (n_pairs_max)]

        """
    if mic is True:
        raise DeprecationWarning('`get_indices()` with `mic=True` is deprecated in favor of `get_pbc_neighbors()`.')

    n = R.shape[-2]
    n_data = R.shape[0]
    idx = np.indices((n, n))

    if mic:
        cell_lengths = np.linalg.norm(cell, axis=-1).reshape(-1)  # shape: (n_data*3)
        if r_cut < 0.5 * min(cell_lengths):
            distance_fn = lambda r, c: coordinates_to_distance_matrix_mic(r, c)
        else:
            raise NotImplementedError(f'Minimal image convention currently only implemented for '
                                      f'r_cut < 0.5*min(cell_lengths), but r_cut={r_cut} and 0.5*min(cell_lengths) = '
                                      f'{0.5 * min(cell_lengths)}. Consider using `get_pbc_indices` which uses ASE under '
                                      f'the hood. However, the latter takes ~15 times longer so maybe reduce r_cut.')
    else:
        distance_fn = lambda r, _: coordinates_to_distance_matrix(r)
        cell = np.zeros(len(R))

    @jax.jit
    def neigh_filter(_z, _R, _c):
        Dij = distance_fn(_R, _c).squeeze(axis=-1)  # shape: (n,n)
        msk_ij = (jnp.einsum('i, j -> ij', _z, _z) != 0).astype(np.int16)  # shape: (n,n)
        Dij_x_msk_ij = Dij * msk_ij  # shape: (n,n)
        return jnp.where((Dij_x_msk_ij <= r_cut) & (Dij_x_msk_ij > 0), True, False)

    def get_idx(i):
        idx_ = idx[:, neigh_filter(z[i], R[i], cell[i])]  # shape: (2,n_pairs)
        # pad_idx = np.pad(idx_, ((0, 0), (0, int(pad_length[i]))), mode='constant', constant_values=((0, 0), (0, -1)))
        # shape: (2,n_pair+pad_length)
        return idx_

    logging.info('Generate neighborhood lists for {} geometries: '.format(n_data))
    idxs = list(map(get_idx, tqdm(range(n_data))))  # [(2,*), ..., (2,*)]  * is number of pairs per point

    n_pairs = [u.shape[-1] for u in idxs]
    n_pairs_max = max(n_pairs)
    pad_length = [n_pairs_max - u for u in n_pairs]

    def pad_idxs(t):
        u, l = t
        return np.pad(u, ((0, 0), (0, int(l))), mode='constant', constant_values=((0, 0), (0, -1)))

    pad_idx_i, pad_idx_j = map(np.squeeze,
                               np.split(np.array(list(map(pad_idxs, zip(idxs, pad_length)))),
                                        indices_or_sections=2,
                                        axis=-2))

    return {'idx_i': pad_idx_i, 'idx_j': pad_idx_j}

def pad_index_list(idx, n_pair_max, pad_value=-1):
    n_pair = idx.shape[-1]

    pad_length = n_pair_max - n_pair
    assert pad_length >= 0

    pad = partial(np.pad, pad_width=(0, pad_length), mode='constant',
                  constant_values=(0, pad_value))
    pad_idx = pad(idx)
    return pad_idx

def pad_shift(_shift, n_pair_max, pad_value=0):
    n_pair = _shift.shape[0]

    pad_length = n_pair_max - n_pair
    assert pad_length >= 0

    pad = partial(np.pad, pad_width=((0, pad_length), (0, 0)), mode='constant',
                  constant_values=((0, pad_value), (0, 0)))
    pad_s = pad(_shift)
    return pad_s

def get_pbc_neighbors(pos, node_mask, pbc, cell, cutoff) -> Dict:
    """
    Calculate the neighbors in the graph if periodic boundary conditions (PBC) are present. Under the hood, this
    function uses the ase.primitive_neighbors function from ASE to calculate the central index `idx_i`, the neighboring
    indices `idx_j` as well as shifts vectors `shifts`.

    Args:
        pos (Array): Positions that, if structures of different sizes are present, are already padded to the same
            length n, shape: (B,n,3)
        node_mask (Array): Mask that indicates which nodes are true nodes and which are only present due to padding,
            shape: (B,n)
        cell (Array): Cells are assumed to be row-wise (ASE default), shape: (B,3,3)
        pbc (Array): Directions of periodic boundary conditions, shape: (B,3)
        cutoff (float): Cutoff used to calculate the neighbors

    Returns: Dictionary, with central, neighboring indices as well as the shift vectors.

    """

    # Cells must be transposed in the following, since mlff neighbor list assume cells to be column wise to make it
    # consistent with GLP and jax_md neighbor list

    # TODO: check for 2*cutoff < min(||lattice_vector||_2)

    def estimate_capacity_multiplier():
        return 1.1 * node_mask.sum(-1).max() / node_mask.sum(-1).min()

    allocate_fn, update_fn = pbc_neighbor_list(cell=cell[0].T,
                                               cutoff=cutoff,
                                               capacity_multiplier=1.)

    neighbors = allocate_fn(pos=pos[0], node_mask=node_mask[0], pbc=pbc[0], new_cell=cell[0].T)

    tmp_idx_i_list = []
    tmp_idx_j_list = []
    tmp_shifts_list = []

    idx_i_list = []
    idx_j_list = []
    shifts_list = []

    print('Construct neighbors using minimal image convention ...')
    for (R, msk, pb, ce) in tqdm(zip(pos, node_mask, pbc, cell)):
        neighbors = update_fn(pos=R, node_mask=msk, pbc=pb, primitive_neighbors=neighbors, new_cell=ce.T)

        # check for overflow in the neighbor list
        if neighbors.overflow:
            # print('Overflow detected. Re-allocating neighbor list!')
            # re-allocate the neighborhood list
            # allocate_fn, update_fn = pbc_neighbor_list(cell=ce.T, cutoff=cutoff, capacity_multiplier=1.)
            neighbors = allocate_fn(pos=R, node_mask=msk, pbc=pb, new_cell=ce.T)

            # save the primitive neighbors calculated so far
            idx_i_list += [np.stack(tmp_idx_i_list, axis=0)]
            idx_j_list += [np.stack(tmp_idx_j_list, axis=0)]
            shifts_list += [np.stack(tmp_shifts_list, axis=0)]

            # clear the temporary lists
            tmp_idx_i_list = []
            tmp_idx_j_list = []
            tmp_shifts_list = []

        # append neighbors and shifts
        tmp_idx_i_list += [neighbors.idx_i]
        tmp_idx_j_list += [neighbors.idx_j]
        tmp_shifts_list += [neighbors.shifts]

    # append the remaining temporary lists
    idx_i_list += [np.stack(tmp_idx_i_list, axis=0)]
    idx_j_list += [np.stack(tmp_idx_j_list, axis=0)]
    shifts_list += [np.stack(tmp_shifts_list, axis=0)]

    n_edges_max = max([x.shape[-1] for x in idx_j_list])

    idx_i_list = [np.array(list(map(partial(pad_index_list, n_pair_max=n_edges_max), x))) for x in idx_i_list]
    idx_j_list = [np.array(list(map(partial(pad_index_list, n_pair_max=n_edges_max), x))) for x in idx_j_list]
    shifts_list = [np.array(list(map(partial(pad_shift, n_pair_max=n_edges_max), x))) for x in shifts_list]
    print('... done!')
    return {'idx_i': np.concatenate(idx_i_list, axis=0),
            'idx_j': np.concatenate(idx_j_list, axis=0),
            'shifts': np.concatenate(shifts_list, axis=0)}

def pbc_neighbor_list(cell, cutoff, skin: float = None, capacity_multiplier: float = 1.25):
    """

    Args:
        cell (Array): Lattice vectors, are assumed to be column-wise!! Differs from ASE default
        cutoff ():
        skin ():
        capacity_multiplier ():

    Returns:

    """

    def allocate_fn(pos, pbc, new_cell=None, node_mask=None):
        if new_cell is None:
            new_cell = cell

        if node_mask is None:
            node_mask = np.ones(pos.shape[0], dtype=pos.dtype).astype(bool)

        ce_ase = new_cell.T  # switch to ASE default, lattice vectors are now row-wise

        idx_i, idx_j, shifts = primitive_neighbor_list(quantities='ijS',
                                                       pbc=pbc,
                                                       cell=ce_ase,
                                                       positions=pos[node_mask],
                                                       cutoff=cutoff,
                                                       numbers=None,
                                                       self_interaction=False,
                                                       use_scaled_positions=False,
                                                       max_nbins=1000000.0)

        n_edges_max = int(np.ceil(capacity_multiplier * len(idx_i)))
        return PrimitiveNeighbors(idx_i=pad_index_list(idx_i, n_pair_max=n_edges_max),
                                  idx_j=pad_index_list(idx_j, n_pair_max=n_edges_max),
                                  shifts=pad_shift(shifts, n_pair_max=n_edges_max),
                                  overflow=False
                                  )

    def update_fn(pos, pbc, primitive_neighbors: PrimitiveNeighbors, new_cell, node_mask=None):
        """

        Args:
            pos (Array): Atomic positions, shape: (n,3)
            pbc (Array):
            primitive_neighbors (PrimitiveNeighbors)
            new_cell (Array): Unit cell, lattice vectors are assumed to be column-wise (transpose of ASE default),
                shape: (3,3)
            node_mask (Array):

        Returns:

        """
        if node_mask is None:
            node_mask = np.ones(pos.shape[0], dtype=pos.dtype).astype(bool)

        n_edges_max = len(primitive_neighbors.idx_i)

        ce_ase = new_cell.T  # switch to ASE default, lattice vectors are now row-wise

        idx_i, idx_j, shifts = primitive_neighbor_list(quantities='ijS',
                                                       pbc=pbc,
                                                       cell=ce_ase,
                                                       positions=pos[node_mask],
                                                       cutoff=cutoff,
                                                       numbers=None,
                                                       self_interaction=False,
                                                       use_scaled_positions=False,
                                                       max_nbins=1000000.0)

        if len(idx_i) > n_edges_max:
            overflow = True
        else:
            overflow = False
            idx_i = pad_index_list(idx_i, n_pair_max=n_edges_max)
            idx_j = pad_index_list(idx_j, n_pair_max=n_edges_max)
            shifts = pad_shift(shifts, n_pair_max=n_edges_max)

        return PrimitiveNeighbors(idx_i=idx_i, idx_j=idx_j, shifts=shifts, overflow=overflow)

    # note that the update function still returns correct neighbors and shifts, even when overflow is present.
    return allocate_fn, update_fn


import sys

import pytest
from pandas import Series

from src.modules import named_entity_recognition as ner

class TestRetrieveNamedEntities:
    @pytest.mark.skipif(
        sys.platform.startswith("linux"), reason="Unknown error during CI"
    )
    def test_retrieve_named_entities(self):
        test_data = Series(
            [
                "The ONS has just released an article on the UK Government's policy.",
                "my own care for nothing",
                "Hollywood actors now have their own statue",
            ]
        )
        actual = ner.retrieve_named_entities(test_data)
        expected = [["ONS", "the UK Government's"], [], ["Hollywood"]]
        assert actual == expected, "Did not successfully retrieve named entities"


import os
import cv2
import json
import time
import random
import string
import filetype
import numpy as np
import pandas as pd
from functools import wraps

from config import logger

def msgwrap(func):
    @wraps(func)
    def wrap(*args, **kwargs):
        ret = {
            'succeed':0,
            'message':'succeed',
        }
        try:
            result = func(*args, **kwargs)
            if type(result) is dict:
                ret.update(result)
        except Exception as e:
            logger.exception(e)
            ret = {
                'succeed':1,
                'message':str(e),
            }
        logger.debug('message : {}'.format(ret))
        return json.dumps(ret).encode('utf-8')
    return wrap

def gen_random_string(length=8, *, number=True, abc=True, upper_case=False, lower_case=False):
    charset = ''
    if number:
        charset += string.digits
    if abc:
        if upper_case==lower_case:
            charset += string.ascii_letters
        elif upper_case:
            charset += string.ascii_uppercase
        elif lower_case:
            charset += string.ascii_lowercase
    return ''.join(random.choice(charset) for _ in range(length))

def get_type(filepath=None, data=None):
    try:
        if filepath is not None:
            ans = filetype.guess_mime(filepath)
            logger.debug('filetype guess: {} {}'.format(filepath, ans))
            if ans is None:
                if filepath[-5:] == '.json':
                    with open(filepath, 'r') as f:
                        data = json.load(f)
                    if type(data) == list:
                        return 'Sequence'
                    return "Graph"
                elif filepath[-4:] == '.mat':
                    return 'Graph'
                elif filepath[-4:] == '.csv':
                    return "DataFrame"
                return "Text"
            if ans.find('image') != -1:
                return "Image"
            if ans.find('video') != -1:
                return 'Video'
            if ans.find('audio') != -1:
                return 'Audio'
        raise NotImplementedError
    except Exception as e:
        logger.exception('Guess Type Error : {} {}'.format(filepath, e))

def safepath(path):
    # TODO : change replace
    path = path.replace('..', '')
    path = path.lstrip('/')
    path = path.replace('/', os.path.sep)
    return path

def sample_data(data, type_=None, num=10):
    if type_ == 'path':
        type_ = get_type(data)
        if type_ == 'DataFrame':
            data = pd.read_csv(data, nrows=10)
        elif type_ == 'Image':
            data = [[cv2.imread(data), os.path.basename(data)]]
        elif type_ == 'Sequence':
            with open(data, 'r') as f:
                data = json.load(f)
        elif type_ == 'Text':
            with open(data, 'r') as f:
                data = f.read()
        elif type_ == 'Graph':
            data = ''

    ret = {}
    if type_ == 'DataFrame':
        num = max(num, 0)
        index = list(data.columns)
        df = data[0:num]
        df = df.round(3)
        types = [str(df[index[j]].dtype) for j in range(len(index))]
        df = df.fillna('NaN')
        df = np.array(df).tolist()
        df = list(map(lambda x:list(map(lambda y:str(y) if len(str(y))< 16 else str(y)[:13]+'...',x)), df))
        if len(index) > 20:
            index = index[:19] + ['More...']
            df = list(map(lambda x:x[:19]+['...'], df))

        ret = {
            'type':'DataFrame',
            'shape':list(data.shape),
            'col_num':len(index),
            'col_index':index,
            'col_type':types,
            'row_num':num,
            'data': df
        }
    elif type_ == 'Image':
        savename = gen_random_string() + '.png'
        savedir = os.path.join('app', 'static', 'cache')
        if not os.path.exists(savedir):
            os.mkdir(savedir)
        filename = data[1][0]
        data = data[0][0]
        shape = list(data.shape)
        shape[0], shape[1] = shape[1], shape[0]
        resize_shape = shape[:2]
        while resize_shape[0] > 640 or resize_shape[1] > 480:
            resize_shape[0] //= 2
            resize_shape[1] //= 2
        data = cv2.resize(data, tuple(resize_shape))
        cv2.imwrite(os.path.join(savedir, savename), data)
        ret = {
            'type':'Image',
            'data':{
                'url':'static/cache/'+savename,
                'shape':shape,
            }
        }
    elif type_ == 'Graph':
        ret = {
            'type':'Graph',
            'data':data,
        }
    elif type_ == 'Video':
        ret = {
            'type':'Video',
        }
    elif type_ == 'Sequence':
        data = data[:num]
        data = list(map(str, data))
        ret = {
            'type':'Sequence',
            'len':len(data),
            'data':data,
        }
    elif type_ == 'Text':
        ret = {
            'type':'Text',
            'data':data,
        }
    else:
        raise NotImplementedError
    logger.debug('sample data {}'.format(ret))
    return ret


import os

from datetime import date

cache_dir = 'cache'
conf_dir = 'conf'

cache_extensions = ('.json', '.png')

def get_cache_filename_generic(symbol, extension):
    prefix = Global.get_cache_filename_prefix()
    Global.try_create_cache_dir()
    cache_file = cache_dir + os.sep + '{}_{}{}'.format(prefix, symbol, extension)
    return cache_file

class Global:

    def try_create_cache_dir():
        if not os.path.exists(cache_dir):
            os.makedirs(cache_dir)

    def get_cache_filename_prefix():
        today = date.today()
        prefix = today.strftime('%Y-%m-%d')
        return prefix

    def get_cache_filename(symbol):
        return get_cache_filename_generic(symbol, '.json')

    def get_cache_chart_filename(symbol, chart_type):
        extension = '_chart_{}.png'.format(chart_type)
        return get_cache_filename_generic(symbol, extension)

    def clean_old_cached_files():
        Global.try_create_cache_dir()
        prefix = Global.get_cache_filename_prefix()
        count = 0
        for filename in os.listdir(cache_dir):
            if not filename.endswith(cache_extensions):
                continue
            if not filename.startswith(prefix):
                file_path = cache_dir + os.sep + filename
                os.remove(file_path)
                count += 1
        return count


import logging

import azure.functions as func

def main(req: func.HttpRequest, recommendations: func.DocumentList) -> func.HttpResponse:
    logging.info('Python HTTP trigger function processed a request.')

    if not recommendations or len(recommendations) == 0:
        logging.warning("Recommendation item not found")
        return func.HttpResponse(
            "Recommendation item not found",
            status_code=404,
        )

    return func.HttpResponse(
        recommendations[0].to_json(),
        mimetype="application/json",
        status_code=200,
    )


"""Data processing."""

import pandas as pd
import torch
import numpy as np
from torchvision import transforms

transformer = transforms.Compose([
    transforms.RandomRotation(10),
])

class MNISTDataset(torch.utils.data.Dataset):
    """Dataset."""

    def __init__(self, df: pd.DataFrame, transformer: transforms.Compose = None):
        """Init."""
        x_cols = [x for x in df.columns if x != "label"]
        df = df.reset_index()
        self.df_x = df[x_cols]
        self.df_y = df['label']
        self.transformer = transformer

    def __len__(self):
        return self.df_x.shape[0]

    def __getitem__(self, idx):
        """Get item."""
        y_values = self.df_y.loc[idx]
        x_values = self.df_x.loc[idx].to_numpy().astype("float")/255
        x_values = x_values.reshape((1, 28, 28))
        x_values = torch.tensor(x_values).float()
        if self.transformer and np.random.random() < 0.5:
            x_values = self.transformer(x_values)
        return x_values, y_values

if __name__ == "__main__":
    import os
    dataset = MNISTDataset(
        pd.read_csv(os.path.join('data', "train.csv")),
        transformer
    )
    dataset.__getitem__(0)


#!/usr/bin/python
import json
from bson import json_util
import bottle
from bottle import route, run, request, abort
import datetime
from pymongo import MongoClient

connection = MongoClient('localhost', 27017)
db = connection['market']
collection = db['stocks']
# set up URI paths for REST service

@route('/update', method='GET')
def update():
  id=request.query.id
  result=request.query.result
  findJson={"id":id}
  updateJson={"result":result}
  return update_document(findJson, updateJson)

def update_document(findDocument, newDocument):
  try:
    result=collection.update(findDocument, {"$set": newDocument})
  except Exception as ve:
      return False

if __name__ == '__main__':
 #app.run(debug=True)
 run(host='localhost', port=8080)


'''
Created on 24 de fev de 2017

@author: Thiago Nobrega
'''

import zipfile
import io

import csv

#head powershell
#gc log.txt | select -first 10 # head

from datetime import datetime
from parsers import base
import os

from datetime import datetime

def convertData(file,headers= 0,delimiter = '\,', quote=csv.QUOTE_ALL):
    """
        Read data from ziped file
    """
#     import codecs
#     f = codecs.iterdecode(file, "utf-8")
# #     f = codecs.iterdecode(file, "ISO-8859-1")
#
#     data = []
#
#     reader = csv.reader(f,delimiter=delimiter,quotechar='"',quoting=quote, skipinitialspace=True)
#
#     count = 0

    from util.csvutil import readQuoted
    data = readQuoted(file,delimiter = ',')

    dm = []

    if(headers != 0):
        dm.append(headers)

    count = 0
    for row in data:
        dm.append(filterData(row))
#         count+=1
#         if count % 10000 == 0:
#             print(count)

    return dm

def filterData(data):
    row = data
#     fdata = []

    ##
    ## Adress Basic Data : data about voter adress
    #
    ### adress
    ### res_addr, res_city, res_state , res_zip
    ead = [row[11] ,row[13] , row[14] , row[15] ]

    ##
    ## Personal Data : data about the voter (name,gender,bith,etc...)
    ##
    ### basic personal data
    ###  lastname , firstname , midlename ,  name_sufix
    bpd = [ row[3],row[4],row[5],row[6] ]

    ### extra personal data
    ### birth_date , register_date
    epd = [ row[7], row[8] ]
    ### party
    ### party_affiliation
    ext = [ row[10] ]

    entity = bpd + epd + ext + ead

    return entity

def listFiles(dir):
    import os
    lf = []
    for file in os.listdir(dir):
        if file.startswith("SWVF"):
            fr = os.path.join(dir, file)
            lf.append(fr)
    return lf

if __name__ == '__main__':

    dir = "F:\\etapa_02\\Voters\\Ohio\\"
    files = listFiles(dir)

    header = "lastname","firstname","middlename","namesufix","birth_date","register_date","party_affiliation","res_addr", "res_city", "res_state" , "res_zip"
    alldata = convertData(files[0],headers=header)
    print(len(alldata))
    for file in files[1:]:
        print(file)
        data = convertData(file)
        alldata += data

    print(len(alldata))
    print(alldata[0])
    print(alldata[1])

    file = dir + 'ohio_voters_striped_full.csv'
    from util.csvutil import write2csv2
    write2csv2(file,alldata)
    import sys
    sys.exit()

    print(file)

#     ofile  = open(outfile, "w" ,encoding="UTF-8")
#     writer = csv.writer(ofile, delimiter=out_delimiter, quotechar='"', quoting=quote ,lineterminator='\n')
#
#     data = convertData(zfile, fo)
#
    print("Done!")

#fast fix in data (remove later)
#"birth_date","register_date"
import zipfile
import io
import csv
from datetime import datetime
from parsers import base
import os

from datetime import datetime

def fix_date(din):
    d = str(din)
    dt = datetime.strptime(str(d),'%Y-%m-%d')
    newdate = str(dt.day) + "/" + str(dt.month) + "/" + str(dt.year)
    return newdate

in_dir ="F:"+os.path.sep+"z_dados"+os.path.sep+"us_voters"+os.path.sep
in_dir =os.path.sep+"home/thiago/dados"+os.path.sep+"us_voters"+os.path.sep
file = "ohio_voters_striped_full.csv"
data = base.readData(in_dir+file, sep=",")

data['birth_date'] = data['birth_date'].apply(fix_date)
data['register_date'] = data['register_date'].apply(fix_date)

base.writeData(data,in_dir+'ohio_voters.csv')

import sys
sys.exit()


from logbook import Logger
from collections import defaultdict
from copy import copy

from six import iteritems

from zipline.assets import Equity, Future, Asset
from .blotter import Blotter
from zipline.extensions import register
from zipline.finance.order import Order
from zipline.finance.slippage import (
    DEFAULT_FUTURE_VOLUME_SLIPPAGE_BAR_LIMIT,
    VolatilityVolumeShare,
    FixedBasisPointsSlippage,
)
from zipline.finance.commission import (
    DEFAULT_PER_CONTRACT_COST,
    FUTURE_EXCHANGE_FEES_BY_SYMBOL,
    PerContract,
    PerShare,
)
from zipline.utils.input_validation import expect_types

log = Logger('Blotter')
warning_logger = Logger('AlgoWarning')

@register(Blotter, 'default')
class SimulationBlotter(Blotter):
    def __init__(self,
                 equity_slippage=None,
                 future_slippage=None,
                 equity_commission=None,
                 future_commission=None,
                 cancel_policy=None):
        super(SimulationBlotter, self).__init__(cancel_policy=cancel_policy)

        # these orders are aggregated by asset
        self.open_orders = defaultdict(list)

        # keep a dict of orders by their own id
        self.orders = {}

        # holding orders that have come in since the last event.
        self.new_orders = []

        self.max_shares = int(1e+11)

        self.slippage_models = {
            Equity: equity_slippage or FixedBasisPointsSlippage(),
            Future: future_slippage or VolatilityVolumeShare(
                volume_limit=DEFAULT_FUTURE_VOLUME_SLIPPAGE_BAR_LIMIT,
            ),
        }
        self.commission_models = {
            Equity: equity_commission or PerShare(),
            Future: future_commission or PerContract(
                cost=DEFAULT_PER_CONTRACT_COST,
                exchange_fee=FUTURE_EXCHANGE_FEES_BY_SYMBOL,
            ),
        }

    def __repr__(self):
        return """
{class_name}(
    slippage_models={slippage_models},
    commission_models={commission_models},
    open_orders={open_orders},
    orders={orders},
    new_orders={new_orders},
    current_dt={current_dt})
""".strip().format(class_name=self.__class__.__name__,
                   slippage_models=self.slippage_models,
                   commission_models=self.commission_models,
                   open_orders=self.open_orders,
                   orders=self.orders,
                   new_orders=self.new_orders,
                   current_dt=self.current_dt)

    @expect_types(asset=Asset)
    def order(self, asset, amount, style, order_id=None):
        """Place an order.

        Parameters
        ----------
        asset : zipline.assets.Asset
            The asset that this order is for.
        amount : int
            The amount of shares to order. If ``amount`` is positive, this is
            the number of shares to buy or cover. If ``amount`` is negative,
            this is the number of shares to sell or short.
        style : zipline.finance.execution.ExecutionStyle
            The execution style for the order.
        order_id : str, optional
            The unique identifier for this order.

        Returns
        -------
        order_id : str or None
            The unique identifier for this order, or None if no order was
            placed.

        Notes
        -----
        amount > 0 :: Buy/Cover
        amount < 0 :: Sell/Short
        Market order:    order(asset, amount)
        Limit order:     order(asset, amount, style=LimitOrder(limit_price))
        Stop order:      order(asset, amount, style=StopOrder(stop_price))
        StopLimit order: order(asset, amount, style=StopLimitOrder(limit_price,
                               stop_price))
        """
        # something could be done with amount to further divide
        # between buy by share count OR buy shares up to a dollar amount
        # numeric == share count  AND  "$dollar.cents" == cost amount

        if amount == 0:
            # Don't bother placing orders for 0 shares.
            return None
        elif amount > self.max_shares:
            # Arbitrary limit of 100 billion (US) shares will never be
            # exceeded except by a buggy algorithm.
            raise OverflowError("Can't order more than %d shares" %
                                self.max_shares)

        is_buy = (amount > 0)
        order = Order(
            dt=self.current_dt,
            asset=asset,
            amount=amount,
            stop=style.get_stop_price(is_buy),
            limit=style.get_limit_price(is_buy),
            id=order_id
        )

        self.open_orders[order.asset].append(order)
        self.orders[order.id] = order
        self.new_orders.append(order)

        return order.id

    def cancel(self, order_id, relay_status=True):
        if order_id not in self.orders:
            return

        cur_order = self.orders[order_id]

        if cur_order.open:
            order_list = self.open_orders[cur_order.asset]
            if cur_order in order_list:
                order_list.remove(cur_order)

            if cur_order in self.new_orders:
                self.new_orders.remove(cur_order)
            cur_order.cancel()
            cur_order.dt = self.current_dt

            if relay_status:
                # we want this order's new status to be relayed out
                # along with newly placed orders.
                self.new_orders.append(cur_order)

    def cancel_all_orders_for_asset(self, asset, warn=False,
                                    relay_status=True):
        """
        Cancel all open orders for a given asset.
        """
        # (sadly) open_orders is a defaultdict, so this will always succeed.
        orders = self.open_orders[asset]

        # We're making a copy here because `cancel` mutates the list of open
        # orders in place.  The right thing to do here would be to make
        # self.open_orders no longer a defaultdict.  If we do that, then we
        # should just remove the orders once here and be done with the matter.
        for order in orders[:]:
            self.cancel(order.id, relay_status)
            if warn:
                # Message appropriately depending on whether there's
                # been a partial fill or not.
                if order.filled > 0:
                    warning_logger.warn(
                        'Your order for {order_amt} shares of '
                        '{order_sym} has been partially filled. '
                        '{order_filled} shares were successfully '
                        'purchased. {order_failed} shares were not '
                        'filled by the end of day and '
                        'were canceled.'.format(
                            order_amt=order.amount,
                            order_sym=order.asset.symbol,
                            order_filled=order.filled,
                            order_failed=order.amount - order.filled,
                        )
                    )
                elif order.filled < 0:
                    warning_logger.warn(
                        'Your order for {order_amt} shares of '
                        '{order_sym} has been partially filled. '
                        '{order_filled} shares were successfully '
                        'sold. {order_failed} shares were not '
                        'filled by the end of day and '
                        'were canceled.'.format(
                            order_amt=order.amount,
                            order_sym=order.asset.symbol,
                            order_filled=-1 * order.filled,
                            order_failed=-1 * (order.amount - order.filled),
                        )
                    )
                else:
                    warning_logger.warn(
                        'Your order for {order_amt} shares of '
                        '{order_sym} failed to fill by the end of day '
                        'and was canceled.'.format(
                            order_amt=order.amount,
                            order_sym=order.asset.symbol,
                        )
                    )

        assert not orders
        del self.open_orders[asset]

    def execute_cancel_policy(self, event):
        if self.cancel_policy.should_cancel(event):
            warn = self.cancel_policy.warn_on_cancel
            for asset in copy(self.open_orders):
                self.cancel_all_orders_for_asset(asset, warn,
                                                 relay_status=False)

    def reject(self, order_id, reason=''):
        """
        Mark the given order as 'rejected', which is functionally similar to
        cancelled. The distinction is that rejections are involuntary (and
        usually include a message from a broker indicating why the order was
        rejected) while cancels are typically user-driven.
        """
        if order_id not in self.orders:
            return

        cur_order = self.orders[order_id]

        order_list = self.open_orders[cur_order.asset]
        if cur_order in order_list:
            order_list.remove(cur_order)

        if cur_order in self.new_orders:
            self.new_orders.remove(cur_order)
        cur_order.reject(reason=reason)
        cur_order.dt = self.current_dt
        # we want this order's new status to be relayed out
        # along with newly placed orders.
        self.new_orders.append(cur_order)

    def hold(self, order_id, reason=''):
        """
        Mark the order with order_id as 'held'. Held is functionally similar
        to 'open'. When a fill (full or partial) arrives, the status
        will automatically change back to open/filled as necessary.
        """
        if order_id not in self.orders:
            return

        cur_order = self.orders[order_id]
        if cur_order.open:
            if cur_order in self.new_orders:
                self.new_orders.remove(cur_order)
            cur_order.hold(reason=reason)
            cur_order.dt = self.current_dt
            # we want this order's new status to be relayed out
            # along with newly placed orders.
            self.new_orders.append(cur_order)

    def process_splits(self, splits):
        """
        Processes a list of splits by modifying any open orders as needed.

        Parameters
        ----------
        splits: list
            A list of splits.  Each split is a tuple of (asset, ratio).

        Returns
        -------
        None
        """
        for asset, ratio in splits:
            if asset not in self.open_orders:
                continue

            orders_to_modify = self.open_orders[asset]
            for order in orders_to_modify:
                order.handle_split(ratio)

    def get_transactions(self, bar_data):
        """
        Creates a list of transactions based on the current open orders,
        slippage model, and commission model.

        Parameters
        ----------
        bar_data: zipline._protocol.BarData

        Notes
        -----
        This method book-keeps the blotter's open_orders dictionary, so that
         it is accurate by the time we're done processing open orders.

        Returns
        -------
        transactions_list: List
            transactions_list: list of transactions resulting from the current
            open orders.  If there were no open orders, an empty list is
            returned.

        commissions_list: List
            commissions_list: list of commissions resulting from filling the
            open orders.  A commission is an object with "asset" and "cost"
            parameters.

        closed_orders: List
            closed_orders: list of all the orders that have filled.
        """

        closed_orders = []
        transactions = []
        commissions = []

        if self.open_orders:
            for asset, asset_orders in iteritems(self.open_orders):
                slippage = self.slippage_models[type(asset)]

                for order, txn in \
                        slippage.simulate(bar_data, asset, asset_orders):
                    commission = self.commission_models[type(asset)]
                    additional_commission = commission.calculate(order, txn)

                    if additional_commission > 0:
                        commissions.append({
                            "asset": order.asset,
                            "order": order,
                            "cost": additional_commission
                        })

                    order.filled += txn.amount
                    order.commission += additional_commission

                    order.dt = txn.dt

                    transactions.append(txn)

                    if not order.open:
                        closed_orders.append(order)

        return transactions, commissions, closed_orders

    def prune_orders(self, closed_orders):
        """
        Removes all given orders from the blotter's open_orders list.

        Parameters
        ----------
        closed_orders: iterable of orders that are closed.

        Returns
        -------
        None
        """
        # remove all closed orders from our open_orders dict
        for order in closed_orders:
            asset = order.asset
            asset_orders = self.open_orders[asset]
            try:
                asset_orders.remove(order)
            except ValueError:
                continue

        # now clear out the assets from our open_orders dict that have
        # zero open orders
        for asset in list(self.open_orders.keys()):
            if len(self.open_orders[asset]) == 0:
                del self.open_orders[asset]


from os import environ
import requests
from flask import (Blueprint, render_template, redirect,
                   url_for, flash, request, abort)
from flask_login import login_required, current_user
from sqlalchemy.exc import IntegrityError
from sqlalchemy.sql import func
from application import db
from application.forms import AddEntryForm, UpdateEntryForm, SearchForm
from application.models import Word

main = Blueprint('main', __name__,
                 template_folder='templates',
                 static_folder='static')

@main.route('/')
def index():
    '''Index route'''

    words = Word.query.order_by(func.random()).limit(1)
    return render_template('main/index.html', words=words)

def query_api(word):
    '''Query API for translation'''

    api_key = environ.get('API_KEY')

    params = (
        ('l', 'enes'),
        ('q', word),
    )
    headers = {"X-Secret": api_key}
    response = requests.get('https://api.pons.com/v1/dictionary',
                            params=params, headers=headers)
    return response

@main.route('/translate', methods=['GET', 'POST'])
def translation():
    '''Translate ES => EN or EN => ES'''

    form = SearchForm()
    if form.validate_on_submit():
        if current_user.is_anonymous:
            flash('Please sign in to translate words.', 'warning')
            return redirect(url_for('main.translation'))
        else:
            query_api(form.word.data)
            return
    return render_template('main/translation.html', form=form)

@main.route('/add-word', methods=['GET', 'POST'])
@login_required
def add_word():
    '''Add a word to database'''

    form = AddEntryForm()
    if form.validate_on_submit():
        try:
            word = Word(word_es=form.word_es.data,
                        sentence1_es=form.sentence1_es.data,
                        sentence1_en=form.sentence1_en.data,
                        sentence2_es=form.sentence2_es.data,
                        sentence2_en=form.sentence2_en.data,
                        sentence3_es=form.sentence3_es.data,
                        sentence3_en=form.sentence3_en.data,
                        definition1_en=form.definition1_en.data,
                        definition2_en=form.definition2_en.data,
                        definition3_en=form.definition3_en.data,
                        definition4_en=form.definition4_en.data,
                        user_id=current_user.id)
            db.session.add(word)
            db.session.commit()
            db.session.remove()
            flash('Word added successfully', 'success')
            return redirect(url_for('main.add_word', _external=True))
        except IntegrityError:
            db.session.rollback()
            return redirect(url_for('main.add_word', _external=True))
    user_initials = current_user.firstname[0] + current_user.lastname[0]
    return render_template('main/add_word.html', form=form,
                           user_initials=user_initials)

@main.route('/word/update/<int:word_id>', methods=['GET', 'POST'])
@login_required
def update_word(word_id):
    '''Update a word'''

    word = Word.query.get_or_404(word_id)
    if word.author != current_user:
        abort(403)
    form = UpdateEntryForm()
    if request.method == 'GET':
        form.word_es.data = word.word_es
        form.sentence1_es.data = word.sentence1_es
        form.sentence1_en.data = word.sentence1_en
        form.sentence2_es.data = word.sentence2_es
        form.sentence2_en.data = word.sentence2_en
        form.sentence3_es.data = word.sentence3_es
        form.sentence3_en.data = word.sentence3_en
        form.definition1_en.data = word.definition1_en
        form.definition2_en.data = word.definition2_en
        form.definition3_en.data = word.definition3_en
        form.definition4_en.data = word.definition4_en
    elif form.validate_on_submit():
        try:
            word.word_es = form.word_es.data
            word.sentence1_es = form.sentence1_es.data
            word.sentence1_en = form.sentence1_en.data
            word.sentence2_es = form.sentence2_es.data
            word.sentence2_en = form.sentence2_en.data
            word.sentence3_es = form.sentence3_es.data
            word.sentence3_en = form.sentence3_en.data
            word.definition1_en = form.definition1_en.data
            word.definition2_en = form.definition2_en.data
            word.definition3_en = form.definition3_en.data
            word.definition4_en = form.definition4_en.data
            db.session.commit()
            db.session.remove()
            flash('Word updated successfully', 'success')
            return redirect(url_for('main.display_word',
                            word_id=word_id, _external=True))
        except IntegrityError:
            db.session.rollback()
            return redirect(url_for('main.update_word', _external=True))
    content = {
            'form': form,
            'word': word
            }
    return render_template('main/update_word.html', **content)

@main.route('/word/<int:word_id>', methods=['GET', 'POST'])
@login_required
def display_word(word_id):
    '''Display a vocabulary word'''

    word = Word.query.get_or_404(word_id)
    return render_template('main/word.html', word=word)

@main.route('/vocabulary')
@login_required
def vocabulary():
    '''Vocabulary route'''

    page = request.args.get('page', 1, type=int)
    words = Word.query.paginate(page=page, per_page=2)
    user_initials = current_user.firstname[0] + current_user.lastname[0]
    return render_template('main/vocabulary.html', words=words,
                           user_initials=user_initials)

@main.route('/mywords')
@login_required
def display_user_words():

    words = Word.query.filter(Word.user_id == current_user.id).all()
    return render_template('main/mywords.html', words=words)

@main.app_errorhandler(404)
def page_not_found(error):
    '''404 Page not found'''

    return render_template('404.html'), 404

@main.app_errorhandler(500)
def internal_server_error(error):
    '''500 Internal server error'''

    return render_template('500.html'), 500


import logging
import unittest
from unittest.mock import patch

import arrow

import omicron
from omicron.models.valuation import Valuation
from tests import init_test_env, start_omega

logger = logging.getLogger(__name__)

class ValuationTest(unittest.IsolatedAsyncioTestCase):
    async def asyncSetUp(self) -> None:
        self.cfg = init_test_env()
        self.omega = await start_omega()

        # hack: enable postgres during unittest
        self.cfg.postgres.enabled = True
        await omicron.init()

    async def asyncTearDown(self) -> None:
        await omicron.shutdown()
        if self.omega:
            self.omega.kill()

    async def test_crud(self):
        await Valuation.truncate()

        date = arrow.get("2020-10-26").date()
        # step 1, fetch from remote, and create/update into database
        logger.info("step 1: fetch from remote")
        with patch("omicron.has_db", return_value=False):
            recs = await Valuation.get("000001.XSHE", date)
            self.assertEqual(recs[0]["code"], "000001.XSHE")

        # step 2, now data should exist in database
        logger.info("step 2: get data from database")
        recs = await Valuation.get("000001.XSHE", date)
        self.assertEqual(recs[0]["code"], "000001.XSHE")

        # step 3, to fetch more than one securities
        date = arrow.get("2020-11-04").date()
        logger.info("step 3: fetch more than one securities")
        with patch("omicron.has_db", return_value=False):
            recs = await Valuation.get(["000001.XSHE", "600000.XSHG"], date)
            self.assertEqual(2, len(recs))
            self.assertEqual("000001.XSHE", recs[0]["code"])

        # step 4: fetch more than one day, from remote, make sure the order is asc
        logger.info("step 4: one sec many day")
        with patch("omicron.has_db", return_value=False):
            recs = await Valuation.get(["000001.XSHE"], date, n=3)
            self.assertEqual(3, len(recs))
            self.assertEqual(date, recs[2]["frame"])
            self.assertEqual(arrow.get("2020-11-2").date(), recs[0]["frame"])

        # step 5: fetch more than one day, from database, make sure the order is asc
        logger.info("step 5: more than one day")
        recs = await Valuation.get(["000001.XSHE"], date, n=3)
        self.assertEqual(3, len(recs))
        self.assertEqual(date, recs[2]["frame"])
        self.assertEqual(arrow.get("2020-11-2").date(), recs[0]["frame"])

        # step 6: more than one day, more than one sec
        logger.info("step 6: more than one day, more than one sec")
        recs = await Valuation.get(["000001.XSHE", "600000.XSHG"], date, n=3)
        self.assertEqual(6, len(recs))

    async def test_get_circulating_cap(self):
        sec = "000001.XSHE"
        date = arrow.get("2020-11-4").date()
        recs = await Valuation.get_circulating_cap(sec, date, 3)
        self.assertAlmostEqual(1940575.25, recs[0]["circulating_cap"], places=0)
        self.assertEqual(date, recs[-1]["frame"])


#! python

# Created by Aleksei Wan on 13.11.2019

# Imports
import os
import sys
import json
from argparse import ArgumentParser
import lightgbm as lgbm
import numpy as np
from random import shuffle

sys.path.append(os.path.join(os.path.dirname(__file__), "../scripts"))
from perturb_parameters import generate_bdst_param_pack

def train_lgbm_model(params, training_data, validation_data=None, save_model_path: str = 'lgbm-model.txt'):
    bst = lgbm.train(params, training_data, valid_sets=[validation_data])
    bst.save_model(save_model_path)
    return bst

def calculate_acc(predictions, labels):
    acc = 0
    for idx, val in enumerate(predictions):
        if np.argmax(val) == labels[idx]:
            acc += 1
    acc /= len(labels)
    return acc

def make_printable(x) -> str: return str(round(100 * float(x), 3))

if __name__ == "__main__":
    parser = ArgumentParser(description='Create a LightGBM tree based on provided data')
    parser.add_argument('--input_file', '-i', type=str, default='results.json', help='File containing results')
    parser.add_argument('--epochs', '-e', type=int, default=3, help='File containing results')
    args = parser.parse_args()
    path_to_check = os.path.abspath(args.input_file)
    if not os.path.exists(path_to_check):
        print('Provided path', path_to_check, 'is not a valid directory. Please try again')
        sys.exit(-1)

    # Read Input JSON file
    with open(path_to_check, 'r') as file:
        contents = json.load(file)

    # Create data sets
    shuffle(contents)
    data_list = []
    labels = []
    for element in contents:
        labels.append(element[0])
        data_list.append(element[1])
    idx1 = int(0.8 * len(data_list))
    idx2 = idx1 + int(0.1 * len(data_list))
    train_data = np.array(data_list[:idx1])
    train_labels = np.array(labels[:idx1])
    valid_data = np.array(data_list[idx1:idx2])
    valid_labels = np.array(labels[idx1:idx2])
    test_data = np.array(data_list[idx2:])
    test_labels = np.array(labels[idx2:])
    training_data = lgbm.Dataset(train_data, label=train_labels, params={'verbose': -1})
    validation_data = lgbm.Dataset(valid_data, label=valid_labels, params={'verbose': -1})

    # Run training
    acc_list = np.zeros((args.epochs, 3))
    param_list = []
    for i in range(args.epochs):
        param_list.append(generate_bdst_param_pack())

        bst = train_lgbm_model(params=param_list[i], training_data=training_data, validation_data=validation_data,
                               save_model_path='lgbm-model_' + str(i) + '.txt')

        train_acc = calculate_acc(bst.predict(train_data), train_labels)
        valid_acc = calculate_acc(bst.predict(valid_data), valid_labels)
        test_acc = calculate_acc(bst.predict(test_data), test_labels)

        acc_list[i][0] = calculate_acc(bst.predict(train_data), train_labels)
        acc_list[i][1] = calculate_acc(bst.predict(valid_data), valid_labels)
        acc_list[i][2] = calculate_acc(bst.predict(test_data), test_labels)

    np.savetxt('lgbm_model_accuracies.csv', acc_list, delimiter=",", fmt='%s')
    with open('lgbm_model_params.txt', 'w') as outfile:
        json.dump(param_list, outfile)

    vt_acc_list = np.array([0.5 * (a[1] + a[2]) for a in acc_list.tolist()])

    best_tree_idx = np.argmax(vt_acc_list)
    print('\nBest Tree:', best_tree_idx, '(with validation/test accuracy',
          make_printable(vt_acc_list[best_tree_idx]) + '%)')
    print('Best Tree Params:', param_list[best_tree_idx], '\n')

    vt_acc_list[best_tree_idx] = 0
    best_tree_idx = np.argmax(vt_acc_list)
    print('Second Best Tree:', best_tree_idx, '(with validation/test accuracy',
          make_printable(vt_acc_list[best_tree_idx]) + '%)')
    print('Second Best Tree Params:', param_list[best_tree_idx])

    vt_acc_list[best_tree_idx] = 0
    best_tree_idx = np.argmax(vt_acc_list)
    print('Third Best Tree:', best_tree_idx, '(with validation/test accuracy',
          make_printable(vt_acc_list[best_tree_idx]) + '%)')
    print('Third Best Tree Params:', param_list[best_tree_idx])

    print('Training Samples:', len(train_labels))
    print('Validation Samples:', len(valid_labels))
    print('Testing Samples:', len(test_labels))


import asyncio

from databases import Database

database = Database("sqlite+aiosqlite:///database.sqlite")

async def bootstrap():
    await database.connect()

    # Create a table.
    query = "CREATE TABLE IF NOT EXISTS HighScores (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"
    await database.execute(query=query)

    # Insert some data.
    query = "INSERT INTO HighScores(name, score) VALUES (:name, :score)"
    values = [
        {"name": "Daisy", "score": 92},
        {"name": "Neil", "score": 87},
        {"name": "Carol", "score": 43},
    ]
    await database.execute_many(query=query, values=values)

    # Run a database query.
    query = "SELECT * FROM HighScores"
    rows = await database.fetch_all(query=query)

    print("High Scores:")
    for _ in rows:
        print(_)

if __name__ == "__main__":
    asyncio.run(bootstrap())


import pygame
import typing

class EventHandler:
    def __init__(self) -> None:
        self.events = []
        self.mouse_rel = (0, 0)

        self._keypressed_callbacks: dict[int, list[typing.Callable]] = {}
        self._keydown_callbacks: dict[int, list[typing.Callable]] = {}
        self._keyup_callbacks: dict[int, list[typing.Callable]] = {}

        self._buttonpressed_callbacks: dict[int, list[typing.Callable]] = {}
        self._buttondown_callbacks: dict[int, list[typing.Callable]] = {}
        self._buttonup_callbacks: dict[int, list[typing.Callable]] = {}

        self._mouse_move_callbacks: list[typing.Callable[[tuple[int, int]], None]] = []

        self._quit_callbacks: list[typing.Callable] = []

    def remove_key_binding(self,
                                 key: int) -> None:
        """Removes all callbacks from the key."""

        if key in self._keypressed_callbacks:
            self._keypressed_callbacks.pop(key)
        if key in self._keydown_callbacks:
            self._keydown_callbacks.pop(key)
        if key in self._keyup_callbacks:
            self._keyup_callbacks.pop(key)

    def remove_button_binding(self,
                                    button: int) -> None:
        """Removes all callbacks from the button."""

        if button in self._buttonpressed_callbacks:
            self._buttonpressed_callbacks.pop(button)
        if button in self._buttondown_callbacks:
            self._buttondown_callbacks.pop(button)
        if button in self._buttonup_callbacks:
            self._buttonup_callbacks.pop(button)

    def remove_callback(self,
                              callback: typing.Callable) -> None:
        """Removes all callbacks from the callback."""

        for key in self._keypressed_callbacks:
            while callback in self._keypressed_callbacks[key]:
                self._keypressed_callbacks[key].remove(callback)
        for key in self._keydown_callbacks:
            while callback in self._keydown_callbacks[key]:
                self._keydown_callbacks[key].remove(callback)
        for key in self._keyup_callbacks:
            while callback in self._keyup_callbacks[key]:
                self._keyup_callbacks[key].remove(callback)
        for button in self._buttonpressed_callbacks:
            while callback in self._buttonpressed_callbacks[button]:
                self._buttonpressed_callbacks[button].remove(callback)
        for button in self._buttondown_callbacks:
            while callback in self._buttondown_callbacks[button]:
                self._buttondown_callbacks[button].remove(callback)
        for button in self._buttonup_callbacks:
            while callback in self._buttonup_callbacks[button]:
                self._buttonup_callbacks[button].remove(callback)
        while callback in self._mouse_move_callbacks:
            self._mouse_move_callbacks.remove(callback)
        while callback in self._quit_callbacks:
            self._quit_callbacks.remove(callback)

    def clear(self) -> None:
        """Removes all callbacks."""

        self._keypressed_callbacks = {}
        self._keydown_callbacks = {}
        self._keyup_callbacks = {}
        self._buttonpressed_callbacks = {}
        self._buttondown_callbacks = {}
        self._buttonup_callbacks = {}
        self._mouse_move_callbacks = []
        self._quit_callbacks = []

    def register_keypressed_callback(self,
                                     key: int,
                                     callback: typing.Callable) -> None:
        """Registers a callback to be called when the key is pressed."""

        if key not in self._keypressed_callbacks:
            self._keypressed_callbacks[key] = []
        self._keypressed_callbacks[key].append(callback)

    def register_keydown_callback(self,
                                  key: int,
                                  callback: typing.Callable) -> None:
        """Registers a callback to be called when the key is down."""

        if key not in self._keydown_callbacks:
            self._keydown_callbacks[key] = []
        self._keydown_callbacks[key].append(callback)

    def register_keyup_callback(self,
                                key: int,
                                callback: typing.Callable) -> None:
        """Registers a callback to be called when the key is up."""
        if key not in self._keyup_callbacks:
            self._keyup_callbacks[key] = []
        self._keyup_callbacks[key].append(callback)

    def register_buttonpressed_callback(self,
                                        button: int,
                                        callback: typing.Callable) -> None:
        """Registers a callback to be called when the button is pressed."""
        if button-1 not in self._buttonpressed_callbacks:
            self._buttonpressed_callbacks[button-1] = []
        self._buttonpressed_callbacks[button-1].append(callback)

    def register_buttondown_callback(self,
                                     button: int,
                                     callback: typing.Callable) -> None:
        """Registers a callback to be called when the button is down."""
        if button not in self._buttondown_callbacks:
            self._buttondown_callbacks[button] = []
        self._buttondown_callbacks[button].append(callback)

    def register_buttonup_callback(self,
                                   button: int,
                                   callback: typing.Callable) -> None:
        """Registers a callback to be called when the button is up."""
        if button not in self._buttonup_callbacks:
            self._buttonup_callbacks[button] = []
        self._buttonup_callbacks[button].append(callback)

    def register_mouse_move_callback(self,
                                     callback: typing.Callable[[tuple[int, int]], None]) -> None:
        """Registers a callback to be called when the cursor is dragging."""
        self._mouse_move_callbacks.append(callback)

    def register_quit_callback(self,
                               callback: typing.Callable) -> None:
        """Registers a callback to be called when the game is quit."""
        self._quit_callbacks.append(callback)

    async def handle_events(self) -> None:
        """Handles all events."""
        self.events = pygame.event.get()
        self.mouse_rel = pygame.mouse.get_rel()

        for event in self.events:
            if event.type == pygame.QUIT:
                await self._quit()
                continue

            if event.type == pygame.KEYDOWN:
                await self._keydown(event.key)
                continue

            if event.type == pygame.KEYUP:
                await self._keyup(event.key)
                continue

            if event.type == pygame.MOUSEBUTTONDOWN:
                await self._buttondown(event.button)
                continue

            if event.type == pygame.MOUSEBUTTONUP:
                await self._buttonup(event.button)
                continue

            if event.type == pygame.MOUSEMOTION:
                await self._mouse_move()

        key_pressed = pygame.key.get_pressed()
        button_pressed = pygame.mouse.get_pressed(5)

        for key in self._keypressed_callbacks:
            if key_pressed[key]:
                await self._keypressed(key)

        for button in self._buttonpressed_callbacks:
            if button_pressed[button]:
                await self._buttonpressed(button)

    async def _keypressed(self,
                          key: int) -> None:
        """Calls all callbacks registered to the key pressed."""
        if key not in self._keypressed_callbacks:
            return

        for callback in self._keypressed_callbacks[key]:
            await callback()

    async def _keydown(self,
                       key: int) -> None:
        """Calls all callbacks registered to the key down."""
        if key not in self._keydown_callbacks:
            return

        for callback in self._keydown_callbacks[key]:
            await callback()

    async def _keyup(self,
                     key: int) -> None:
        """Calls all callbacks registered to the key leave."""
        if key not in self._keyup_callbacks:
            return

        for callback in self._keyup_callbacks[key]:
            await callback()

    async def _buttonpressed(self,
                             button: int) -> None:
        """Calls all callbacks registered to the button pressed."""
        if button not in self._buttonpressed_callbacks:
            return

        for callback in self._buttonpressed_callbacks[button]:
            await callback()

    async def _buttondown(self,
                          button: int) -> None:
        """Calls all callbacks registered to the button down."""
        if button not in self._buttondown_callbacks:
            return

        for callback in self._buttondown_callbacks[button]:
            await callback()

    async def _buttonup(self,
                        button: int) -> None:
        """Calls all callbacks registered to the button up."""
        if button not in self._buttonup_callbacks:
            return

        for callback in self._buttonup_callbacks[button]:
            await callback()

    async def _mouse_move(self) -> None:
        """Calls all callbacks registered to the mouse move."""
        for callback in self._mouse_move_callbacks:
            await callback(self.mouse_rel)

    async def _quit(self) -> None:
        """Calls all callbacks registered to the quit event."""
        for callback in self._quit_callbacks:
            await callback()