from Mod import Mod
from taskScheduler import TaskScheduler
from presets import presets
from time import sleep
import datetime
import hashlib
import json
import requests
import os
import asyncio
from PyQt5.QtGui import QPixmap
import uuid

from TableModel import TableModel

class ModList():
	def __init__(self, form, settings, host, filename='modListData.json'):
		self.settings = settings
		self.cache = self.settings.cache
		self.presets = presets(form, settings)
		self.filename = self.settings.get("modListDataFile")
		self.form = form
		self.host = host
		self.moddir = self.settings.get("modDirPath")
		self.TaskScheduler = TaskScheduler(self.form, self.settings)
		sleep(0.1)
		form.progressBar.setValue(20)
		if (os.path.isfile(self.filename)):
			self.compareHash(form.progressBar)
		else:
			sleep(0.1)
			form.progressBar.setValue(40)
			self.downloadModList()
			sleep(0.05)
			form.progressBar.setValue(60)
		self.mods = []
		sleep(0.1)
		form.progressBar.setValue(90)
		self.generateModList()
		self.updateModList()
		self.defaultMod = self.mods[0]
		self.selectedMod = self.defaultMod
		self._local = False
		sleep(0.05)
		form.progressBar.setValue(100)

	def updateModList(self):
		self.TableModel.setNewData(self.renderList())
		self.form.modList.setModel(self.TableModel)
		self.cache.cleanBySize()
		self.cache.setSize()

	def compareHash(self, progressBar):
		response=""
		try:
			response = requests.request("POST", self.host, headers={}, verify=False, data={
				'hash': self.md5File(),
				'u': self.md5(str(uuid.getnode())),
				'v': self.settings.get('version')[0]
			})
		except:
			self.settings.local = True
		if not self.settings.local and response.status_code == 200:
			self.settings.local = False
			if json.loads(response.text)['success']:
				sleep(0.05)
				progressBar.setValue(40)
				sleep(0.05)
				progressBar.setValue(60)
				print("HASH matches")
				self.loadFile()
			else:
				sleep(0.1)
				progressBar.setValue(40)
				self.downloadModList()
				sleep(0.1)
				progressBar.setValue(60)
				print("HASH updated")
			return self
		print("unableToConnectToServer")
		self.settings.local = True
		sleep(0.2)
		progressBar.setValue(40)
		sleep(0.1)
		progressBar.setValue(60)
		self.loadFile()
		return self

	def downloadModList(self):
		try:
			response = requests.request(
				"GET", self.host, headers={}, data={}, verify=False)
			file = open(self.filename, 'w+', encoding='utf-8')
			self.rawData = json.loads(response.text)
			file.write(response.text)
		except:
			self.settings.local = True

	# self.rawData = json.load(open('mod.json', 'r', encoding='utf-8'))

	def loadFile(self):
		self.rawData = json.load(open(self.filename, 'r', encoding='utf-8'))
		return self.rawData

	def md5(self, string):
		hash_md5 = hashlib.md5()
		hash_md5.update(string.encode('utf-8'))
		return hash_md5.hexdigest()

	def md5File(self):
		hash_md5 = hashlib.md5()
		with open(self.filename, "rb") as f:
			for chunk in iter(lambda: f.read(4096), b""):
				hash_md5.update(chunk)
		return hash_md5.hexdigest()

	def generateModList(self):
		_, _, installed = next(os.walk(self.moddir))
		for mod in self.rawData:
			self.mods.append(Mod(mod, installed))

		def modListClicked(selected, deselected):
			index = self.form.modList.model().index(selected.row(), self.TableModel.columnCount() - 1)
			# print(index.data())
			self.setMod(index.data())

		self.TableModel = TableModel(self.form.modList, self.renderList(
		), ['', 'Name', 'Author', 'Raiting', 'Downloads', 'Version', 'gameversion', ''])
		self.form.modList.setModel(self.TableModel)
		self.form.modList.selectionModel().currentRowChanged.connect(modListClicked)
		self.form.modList.selectRow(0)

	def clearCache(self):
		print("MODLIST: clearCache")
		self.cache.cleanCache()

	def renderList(self):
		renderList = []
		for mod in self.mods:
			if mod.get("published") == 1:
				renderList.append(mod.render())

		localMods = []
		_, _, installed = next(os.walk(self.moddir))
		for name in installed:
			x = True
			for mod in self.mods:
				if x and mod.hasFile(name):
					x = False
			if x:
				localMods.append(name)
		for mod in localMods:
			renderList.append([True, f"_{mod}", '"Local"', "", "", "", "", f"Local?{mod}"])
		return renderList

	def getByID(self, id):
		for mod in self.mods:
			if mod.data['id'] == id:
				return mod
		return self.defaultMod

	def setMod(self, id=False):
		if not isinstance(id, bool) and not id.startswith("Local?"):
			self._local = False
			self.selectedMod = self.getByID(int(id))
			self.form.comboBox.clear()
			self.form.comboBox.addItem("Latest")
			self.form.comboBox.addItems(self.selectedMod.versions)
			asyncio.run(self.render())

		if not isinstance(id, bool) and id.startswith("Local?"):
			self._local = True
			self.renderLocalMod(id.split("?")[1])
		self.renderButtons()

	def renderLocalMod(self, mod):
		self.form.modName.setText(mod)
		self.form.modDesc.setText("")
		self.form.modInfo.setText("")
		self.form.image.setText(" ")
		self.form.deleteText.setText(f"Installed: {mod}
Cannot be restored after uninstall")
		self.form.comboBox.clear()

		pass

	async def render(self):
		self.form.modName.setText(self.selectedMod.get('name', 'Unknown'))
		self.form.modDesc.setText(self.selectedMod.get('description', self.settings.get('language')))

		def getLine(x, y):
			return f"{x}: {y}
"

		modInfo = ""
		modInfo += getLine(f"Author{len(self.selectedMod.data['authors']) != 1 and 's' or ''}",
						   self.selectedMod.get('authors', 'Unknown'))
		modInfo += getLine("Downloads", self.selectedMod.get('downloads', 'Unknown'))
		modInfo += getLine("Raiting", self.selectedMod.get('raiting', 'Unknown'))
		modInfo += getLine("Tags", self.selectedMod.get('tags'))

		modInfo += f"
{self.selectedMod.get('shotDescription', 'en')}"

		# modInfo += f"Views: {self.selectedMod.get('views', 'Unknown')}
"
		self.form.modInfo.setText(modInfo)
		if self.selectedMod.get("img") == False:
			self.form.image.setText(" ")
		else:
			imageFile = self.settings.get("imageDirPath") + "m" + str(self.selectedMod.get('id'))
			try:
				if not os.path.isfile(imageFile):
					url = self.selectedMod.get("img")
					response = requests.get(url, headers={}, allow_redirects=True, verify=False)
					open(imageFile, 'wb').write(response.content)

				width = self.form.image.frameGeometry().width()
				height = self.form.image.frameGeometry().height()

				pixmap = QPixmap(imageFile).scaled(width, height, 1)
				# pixmap = pixmap
				self.form.image.setPixmap(pixmap)
			except:
				self.form.image.setText(" ")

	def renderButtons(self):
		if self._local:
			self.form.comboBox.setEnabled(False)
			self.form.installButton.setText("Install")
			self.form.installButton.setEnabled(False)

			if self.form.modName.text() in self.TaskScheduler.task['remove']:
				self.form.deleteButton.setText("Cancel")
				self.form.deleteButton.setEnabled(True)
			else:
				self.form.deleteButton.setText("Uninstall")
				self.form.deleteButton.setEnabled(True)

			self.form.updateButton.setText('Upgrade')
			self.form.updateButton.setEnabled(False)
		else:
			_, selected = self.selectMod()
			version = self.selectedMod.get('installed')
			toBeInstalled = self.selectedMod.get('toInstall')
			toBeRemoved = self.selectedMod.get('toDelete')
			toBeUpdated = self.selectedMod.get('toUpdate')
			update = (toBeUpdated or selected).compare(version)
			latest = ""
			latestV = ""

			if toBeInstalled:
				self.form.installButton.setText("Cancel")
				self.form.installButton.setEnabled(True)
			elif version:
				self.form.installButton.setText("Install")
				self.form.installButton.setEnabled(False)
			else:
				self.form.installButton.setText("Install")
				self.form.installButton.setEnabled(True)

			if toBeRemoved:
				self.form.deleteButton.setText("Cancel")
				self.form.deleteButton.setEnabled(True)
			elif version:
				self.form.deleteButton.setText("Uninstall")
				self.form.deleteButton.setEnabled(True)
			else:
				self.form.deleteButton.setText("Uninstall")
				self.form.deleteButton.setEnabled(False)

			if not toBeUpdated and (not version or update == 0):
				self.form.updateButton.setText('Upgrade')
				self.form.updateButton.setEnabled(False)
			elif toBeUpdated:
				self.form.updateButton.setText('Cancel')
				self.form.updateButton.setEnabled(True)
			elif toBeRemoved or toBeInstalled:
				self.form.updateButton.setText(update == -1 and 'Downgrade' or 'Upgrade')
				self.form.updateButton.setEnabled(False)
			else:
				self.form.updateButton.setText(update == -1 and 'Downgrade' or 'Upgrade')
				self.form.updateButton.setEnabled(True)

			if self.settings.local:
				self.form.updateButton.setEnabled(False)
				self.form.installButton.setEnabled(False)

			if version:
				latest = version.latest and "(Latest)"
			if toBeInstalled or toBeUpdated:
				latestV = (toBeInstalled or toBeUpdated).latest and "(Latest)"

			text = f"Installed: {version or 'None'} {version and latest or ''}
"

			if self.settings.local:
				self.form.deleteText.setStyleSheet("background-color: #FF0000;color:#000000;")
				text += "unable connect to server"
			elif not (toBeInstalled or toBeRemoved or toBeUpdated):
				text += " "
			elif toBeUpdated:
				text += f"{update == -1 and 'Downgrade' or 'Upgrade'} To: {toBeUpdated} {latestV or ''}"
			else:
				text += f"To be {toBeRemoved and 'removed' or 'installed'}: {toBeInstalled or toBeRemoved} {toBeInstalled and latest or ''}"

			# 	self.form.deleteButton.setEnabled(True)
			# else:
			# 	self.form.deleteButton.setEnabled(False)

			# if version == toBeInstalled:
			# 	self.form.installButton.setText("Cancel")
			# else:
			# 	self.form.installButton.setText("Install")
			# if version != toBeInstalled and version and toBeInstalled:

			self.form.deleteText.setText(text)

			self.form.comboBox.setEnabled(True)

	def selectMod(self):
		select = self.form.comboBox.currentIndex()
		version = None
		if select <= 0:
			version = self.selectedMod.getLastVersion()
		else:
			version = self.selectedMod.getVersion(self.selectedMod.versions[select - 1])
		return (self.selectedMod, version)

	def installMod(self):
		self.TaskScheduler.installMod(self.selectMod())
		self.setMod()

	def deleteMod(self):
		if self._local:
			self.TaskScheduler.deleteLocalMod(self.form.modName.text())
		else:
			self.TaskScheduler.deleteMod(self.selectMod())
		self.setMod()

	def updateMod(self):
		self.TaskScheduler.updateMod(self.selectMod())
		self.setMod()

	def applyMods(self):
		self.form.deleteButton.setEnabled(False)
		self.form.installButton.setEnabled(False)
		self.form.updateButton.setEnabled(False)
		self.form.applyButton.setEnabled(False)
		self.form.comboBox.setEnabled(False)
		self.form.modList.setEnabled(False)
		self.TaskScheduler.Apply(self)

	def downloadMod(self):
		self.selectedMod.download()
		pass


#!python3
from kikit.panelize import Panel, fromMm, wxPointMM, Origin, fromDegrees, findBoundingBox
import shapely
from shapely.geometry import box
import pcbnew
import sys

startX = fromMm(50)
startY = fromMm(50)

frameThicknessMm = 5
frameThickness = fromMm(frameThicknessMm)
spacing = fromMm(3)
tabWidth = fromMm(4)

netRenamePattern="B{n}-{orig}"
refRenamePattern="B{n}-{orig}"
netRenamer = lambda x, y: netRenamePattern.format(n=x, orig=y)
refRenamer = lambda x, y: refRenamePattern.format(n=x, orig=y)

panel = Panel()

# size1, cuts = panel.makeTightGrid("../Encoder_SPI_MA730/Encoder_SPI_MA730.kicad_pcb", 1, 1, pcbnew.wxPoint(startX, startY), spacing, spacing,
#               spacing, 0, 0, sourceArea=None, tolerance=fromMm(100),
#               verTabWidth=tabWidth, horTabWidth=1, verTabCount=1, horTabCount=1,
#               rotation=0)
size1 = panel.appendBoard("../Encoder_AEAT_8800_Q24/Encoder_AEAT_8800_Q24.kicad_pcb", pcbnew.wxPoint(startX, startY), origin=Origin.TopLeft, tolerance=fromMm(50), rotationAngle=fromDegrees(90), netRenamer=netRenamer, refRenamer=refRenamer)
size2 = panel.appendBoard("../Encoder_SPI_TLE5012B/Encoder_SPI_TLE5012B.kicad_pcb", pcbnew.wxPoint(size1.x + size1.width + spacing, startY), origin=Origin.TopLeft, tolerance=fromMm(50), rotationAngle=fromDegrees(90), netRenamer=netRenamer, refRenamer=refRenamer)
size3 = panel.appendBoard("../Encoder_SPI_MA730/Encoder_SPI_MA730.kicad_pcb", pcbnew.wxPoint(size2.x + size2.width + spacing, startY), origin=Origin.TopLeft, tolerance=fromMm(50), rotationAngle=fromDegrees(90), netRenamer=netRenamer, refRenamer=refRenamer)
size4 = panel.appendBoard("../Encoder_SPI_MA730/Encoder_SPI_MA730.kicad_pcb", pcbnew.wxPoint(size3.x + size3.width + spacing, startY), origin=Origin.TopLeft, tolerance=fromMm(50), rotationAngle=fromDegrees(90), netRenamer=netRenamer, refRenamer=refRenamer)
size5 = panel.appendBoard("../Encoder/Encoder.kicad_pcb", pcbnew.wxPoint(size4.x + size4.width + spacing, startY), origin=Origin.TopLeft, tolerance=fromMm(50), rotationAngle=fromDegrees(90), netRenamer=netRenamer, refRenamer=refRenamer)
size6 = panel.appendBoard("../Encoder_SPI/Encoder.kicad_pcb", pcbnew.wxPoint(size5.x + size5.width + spacing, startY), origin=Origin.TopLeft, tolerance=fromMm(50), rotationAngle=fromDegrees(90), netRenamer=netRenamer, refRenamer=refRenamer)
minx, miny, maxx, maxy = panel.boardSubstrate.bounds()

tabs, cuts = panel.layerToTabs("Eco1.User", fromMm(4))
panel.appendSubstrate(tabs)
panel.makeMouseBites(cuts, diameter=fromMm(0.5), spacing=fromMm(0.9), offset=fromMm(-0.25))

gapFiller = box(size2.x, size1.y-spacing, size6.x + size6.width, size2.y-spacing)
panel.appendSubstrate(gapFiller)

topRail = box(minx, maxy + spacing, maxx, maxy + spacing + frameThickness)
bottomRail = box(minx, miny - spacing, maxx, miny - spacing - frameThickness)
panel.appendSubstrate(topRail)
panel.appendSubstrate(bottomRail)

panel.addMillFillets(fromMm(1))
panel.copperFillNonBoardAreas()
panel.addFiducials(horizontalOffset=fromMm(10), verticalOffset=fromMm(2.5), copperDiameter=fromMm(1), openingDiameter=fromMm(1))
panel.addTooling(horizontalOffset=fromMm(5), verticalOffset=fromMm(2.5), diameter=fromMm(1.5))

panel.save("Encoder_SPI_Panel_v1_1.kicad_pcb")

# size1 = panel.appendBoard("./i2c_jst_multi_for_kikit.kicad_pcb", pcbnew.wxPoint(startX, startY), origin=Origin.Center, tolerance=fromMm(50))
# panel.boardSubstrate.boundary
# minx, miny, maxx, maxy = panel.boardSubstrate.bounds()
# # topRail = box(minx, maxy + spacing - fromMm(0.001), maxx, maxy + spacing + frameThickness)
# # bottomRail = box(minx, miny - spacing + fromMm(0.001), maxx, miny - spacing - frameThickness)
# # gapFiller = box(minx, maxy, maxx-fromMm(18), maxy+spacing)
# # panel.appendSubstrate(topRail)
# # panel.appendSubstrate(bottomRail)
# # panel.appendSubstrate(gapFiller)
# #panel.makeRailsTb(frameThickness)
# tabs, cuts = panel.layerToTabs("Eco1.User", fromMm(4))
# panel.appendSubstrate(tabs)
# panel.makeMouseBites(cuts, diameter=fromMm(0.5), spacing=fromMm(0.9), offset=fromMm(-0.25))
# panel.addMillFillets(fromMm(1))
# #panel.copperFillNonBoardAreas()
# panel.addFiducials(horizontalOffset=fromMm(10), verticalOffset=fromMm(2.5), copperDiameter=fromMm(1), openingDiameter=fromMm(1))
# panel.addTooling(horizontalOffset=fromMm(5), verticalOffset=fromMm(2.5), diameter=fromMm(1.5))
# panel.save("i2c_jst_multi_panel.kicad_pcb")


import streamlit as st
import time
import numpy as np
import pandas as pd

progress_bar = st.sidebar.progress(0)
status_text = st.sidebar.empty()

data = pd.DataFrame([[0.0]], columns=["My bank account"])
chart = st.line_chart(data)
discount_rate = st.sidebar.slider("Discount rate", 0, 100, 2)
nums = st.text_input("Cash flow", value="-150 50 50 50 50 50 50 50 50 50 50 50")

cash = [float(y) for y in nums.split()]

def cash_flow(C, discount_rate, iteration):
    s = 0
    for i in range(iteration):
        s += C[i] / ((1. + discount_rate / 100.0) ** i)
    return s

for i in range(0, 10):
    status_text.text(f"{i+1} of 10 years complete")
    row = pd.DataFrame([[cash_flow(cash, discount_rate, i)]], columns=["My bank account"])
    data = data.append(row, ignore_index=True)
    print(data)
    chart.line_chart(data)
    progress_bar.progress(i * 10)
    time.sleep(0.05)

progress_bar.empty()

# Streamlit widgets automatically run the script from top to bottom. Since
# this button is not connected to any other logic, it just causes a plain
# rerun.
st.button("Re-run")


import requests

PROJECTS_URL = '/projects'
BRANCHES_URL = '/projects/{project_id}/repository/branches'
DELETE_BRANCH_URL = '/projects/{project_id}/repository/branches/{branch_name}'
DELETE_MERGED_BRANCHES = '/projects/{project_id}/repository/merged_branches'

class GitlabAPIConnector:
    def __init__(self, hostname, private_token):
        self.hostname = hostname
        self.headers = {
            'PRIVATE-TOKEN': private_token
        }

    def get_projects(self, name=None):
        if not name:
            return requests.get(f'{self.hostname}/api/v4{PROJECTS_URL}', headers=self.headers).json()
        return requests.get(f'{self.hostname}/api/v4{PROJECTS_URL}?search={name}', headers=self.headers).json()

    def get_branches(self, project_id):
        project_branches_url = BRANCHES_URL.format(project_id=project_id)
        return requests.get(f'{self.hostname}/api/v4{project_branches_url}', headers=self.headers).json()

    def get_branches_by_project_name(self, project_name):
        project_id = self.get_projects(project_name)[0]['id']
        return self.get_branches(project_id)

    def delete_branch(self, project_id, branch_name):
        print(f'Deleting branch {branch_name}')
        try:
            requests.delete(f'{self.hostname}/api/v4' + DELETE_BRANCH_URL.format(
                project_id=project_id,
                branch_name=branch_name
            ), headers=self.headers)
        except:
            print(f'An error happened when deleting branch {branch_name}')
            raise

    def delete_merged_branches(self, project_id):
        delete_url_for_project = DELETE_MERGED_BRANCHES.format(project_id=project_id)
        return requests.delete(f'{self.hostname}/api/v4{delete_url_for_project}')


import copy
import inspect
import itertools

from geomstats.vectorization import repeat_point

KNOWN_POINT_NAMES = (
    "point",
    "base_point",
    "point_a",
    "point_b",
    "fiber_point",
    "mat",
)
KNOWN_VECTOR_NAMES = (
    "vec",
    "vector",
    "tangent_vec",
    "tangent_vec_a",
    "tangent_vec_b",
    "tangent_vec_c",
    "tangent_vec_d",
    "cotangent_vec_a",
    "cotangent_vec_b",
    "direction",
)
KNOWN_ARGS = KNOWN_POINT_NAMES + KNOWN_VECTOR_NAMES

def _filter_combs(n_args, combs, vectorization_type):
    if vectorization_type == "sym" or n_args == 1:
        return combs

    repeat_indices = [int(val) for val in vectorization_type.split("-")[1:]]
    if len(repeat_indices) == n_args:
        return combs

    remove_indices = [val for val in range(n_args) if val not in repeat_indices]

    for comb in combs.copy():
        vals_remove = [comb[remove_index] for remove_index in remove_indices]
        if max(vals_remove) == 0 or min(vals_remove) == 1 and min(comb) == 1:
            continue

        combs.remove(comb)

    return combs

def _check_vectorization_type(vectorization_type, n_args):
    if vectorization_type in ("sym", "basic") or n_args == 1:
        return

    try:
        val = None
        for val in vectorization_type.split("-")[1:]:
            int(val)

        if val is None or int(val) > n_args:
            raise ValueError(
                "Unable to repeat unexisting args for vectorization type "
                f"`{vectorization_type}` and `n_args = {n_args}"
            )

    except ValueError:
        raise ValueError(
            f"Unable to understand vectorization type `{vectorization_type}`. "
            "Can handle `sym` and `repeat-(int)` format."
        )

def _generate_datum_vectorization_data(
    datum, comb_indices, arg_names, expected_name, n_reps=2
):
    if expected_name is not None:
        has_expected = True
        if type(expected_name) is str:
            expected_name = [expected_name]

        expected_combs = []
        for expected_name_ in expected_name:
            expected = datum.get(expected_name_)
            expected_combs.append([expected, repeat_point(expected, n_reps=n_reps)])
    else:
        has_expected = False

    args_combs = []
    for arg_name in arg_names:
        arg = datum.get(arg_name)
        args_combs.append([arg, repeat_point(arg, n_reps=n_reps, expand=True)])

    new_data = []
    for indices in comb_indices:
        new_datum = copy.copy(datum)

        if has_expected:
            rep = int(1 in indices)
            for expected_i, expected_name_ in enumerate(expected_name):
                new_datum[expected_name_] = expected_combs[expected_i][rep]

        for arg_i, (index, arg_name) in enumerate(zip(indices, arg_names)):
            new_datum[arg_name] = args_combs[arg_i][index]

        new_data.append(new_datum)

    return new_data

def generate_vectorization_data(
    data,
    arg_names,
    expected_name=None,
    n_reps=2,
    vectorization_type="sym",
):
    """Create new data with vectorized version of inputs.

    Parameters
    ----------
    data : list of dict
        Data. Each to vectorize.
    arg_names: list
        Name of inputs to vectorize.
    expected_name: str or list of str
        Output name in case it needs to be repeated.
    n_reps: int
        Number of times the input points should be repeated.
    vectorization_type: str
        Possible values are 'sym', 'basic', or the format 'repeat-(int)'
        (e.g. "repeat-0-2").
        'sym': tests all repetition combinations.
        'basic': tests only no repetition and repetition of all.
        'repeat-(int)': tests repetition of provided indices.
    """
    n_args = len(arg_names)
    _check_vectorization_type(vectorization_type, n_args)

    if vectorization_type == "basic":
        comb_indices = [tuple(i for _ in range(n_args)) for i in range(2)]
    else:
        comb_indices = list(itertools.product(*[range(2)] * n_args))
        comb_indices = _filter_combs(n_args, comb_indices, vectorization_type)
    comb_indices.pop(0)

    new_data = []
    for datum in data:
        new_data.extend(
            _generate_datum_vectorization_data(
                datum,
                comb_indices,
                arg_names,
                expected_name=expected_name,
                n_reps=n_reps,
            )
        )

    return new_data

def _generate_random_data(data_generator, arg_names):
    data = {}
    base_point = None
    for arg_name in arg_names:
        if arg_name in KNOWN_POINT_NAMES:
            base_point = data[arg_name] = data_generator.random_point()

    for arg_name in arg_names:
        if arg_name in KNOWN_VECTOR_NAMES:
            data[arg_name] = data_generator.random_tangent_vec(base_point)

    return data

def _get_vectorization_type(test_case, arg_names):
    if test_case.tangent_to_multiple:
        return "sym"

    tangent_vec_type = ""
    for k, arg_name in enumerate(arg_names):
        if arg_name in KNOWN_VECTOR_NAMES:
            tangent_vec_type += f"-{k}"

    return "repeat" + tangent_vec_type if tangent_vec_type else "sym"

def test_vectorization(self, test_func, n_reps, atol):
    # TODO: move this to decorator?
    # TODO: accept kwargs?
    arg_names = list(inspect.signature(test_func).parameters.keys())
    arg_names = list(filter(lambda x: x in KNOWN_ARGS, arg_names))
    data = _generate_random_data(self.data_generator, arg_names)

    geometry = (
        self.space.metric
        if hasattr(self, "is_metric") and self.is_metric
        else self.space
    )
    data["expected"] = getattr(geometry, test_func.__name__[5:])(**data)
    data["atol"] = atol

    vec_data = generate_vectorization_data(
        data=[data],
        arg_names=arg_names,
        expected_name="expected",
        n_reps=n_reps,
        vectorization_type=_get_vectorization_type(self, arg_names),
    )
    self._test_vectorization(vec_data, test_fnc_name=test_func.__name__)


import cv2
import time
import numpy as np
from hazard_detect import saturated_red_flash_count, luminance_flash_count
from collections import deque
import imutils
from multiprocessing import Pool

# Buffer size (in frames)
BUFFER_SIZE = 16

# Open the webcam
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_FPS, 100)

# Disable auto exposure
# TODO: This is not working

# Width and height of the webcam frame after resizing (without changing resolution)
ret, frame = cap.read()
frame = imutils.resize(frame, width = frame.shape[1] // 4)

WIDTH = frame.shape[1]
HEIGHT = frame.shape[0]

# Set the quarter area threshold (in pixels)
quarter_area_threshold = (WIDTH * HEIGHT) / 4

# Initialize the frame buffer
frame_buffer = deque()
time_buffer = deque()
luminous_flash_buffer = deque()
red_flash_buffer = deque()

luminous_flashes = np.zeros(frame.shape[:2])
red_flashes = np.zeros(frame.shape[:2])

# Initialize previous time
prev_time = time.time()

def thread_process_frame(proc_range):
    global luminous_flashes, red_flashes
    #print(proc_range, "Started", flush = True)
    cur_frame_part = cur_frame[proc_range[0]:proc_range[1]]
    prev_frame_part = prev_frame[proc_range[0]:proc_range[1]]

    luminous = luminance_flash_count(cur_frame_part, prev_frame_part)
    red = saturated_red_flash_count(cur_frame_part, prev_frame_part)

    #print(proc_range, "Done", flush = True)
    return np.array([luminous, red])

# Capture frames from the webcam
try:
    while True:
        # Read a frame from the webcam
        ret, frame = cap.read()
        #frame = np.zeros(frame.shape)

        #If the frame was successfully read
        if ret:
            s = time.time()
            #changing the resolution while keeping the aspect ratio
            frame = imutils.resize(frame, width=WIDTH)

            # Add the frame to the frame buffer
            frame_buffer.append(frame)
            time_buffer.append(time.time())
            s = time.time()

            # If a previous frame exists, check whether the transition has a luminous/red flash
            if len(frame_buffer) > 1:
                cur_frame = frame_buffer[-1]
                prev_frame = frame_buffer[-2]

                n = cur_frame.shape[0]
                per_thread = int(np.ceil(n/4))
                ranges = [(i, i+per_thread) for i in range(0, n, per_thread)]

                with Pool(4) as pool:
                    flashes = np.concatenate(pool.map(thread_process_frame, ranges), axis = 1)
                print(time.time() - s)
                #print(flashes.shape)
                #raise Exception

                # Update buffers and flash count
                luminous_flash_buffer.append(flashes[0])
                luminous_flashes += flashes[0]
                red_flash_buffer.append(flashes[1])
                red_flashes += flashes[1]

            # When there are enough frames and more than 1s has elapsed, check for flashing sequences
            interval = time_buffer[-1] - time_buffer[0]
            if len(frame_buffer) >= BUFFER_SIZE and interval >= 1:

                luminous_flash_freq = (luminous_flashes/2) / interval
                red_flash_freq = (red_flashes/2) / interval

                luminous_count = np.sum(luminous_flash_freq >= 3)
                red_count = np.sum(red_flash_freq >= 3)

                print(np.mean(luminous_flash_freq), np.mean(red_flash_freq))
                if luminous_count >= quarter_area_threshold or red_count >= quarter_area_threshold:
                    print("Flashing detected")

                # Pop first element to change sliding window
                frame_buffer.popleft()
                time_buffer.popleft()

                # Update flash count
                luminous_flashes -= luminous_flash_buffer.popleft()
                red_flashes -= red_flash_buffer.popleft()

                print(time.time() - s, len(time_buffer))

finally:
    cap.release()
    cv2.destroyAllWindows()


from flask import Flask
from flask import jsonify
from data_access import measure
import json

app = Flask(__name__)
app.config.from_json('config.json')

@app.route('/')
def hello_world():
    return 'godking'

@app.route('/api/top-list')
def latest_top_list():
    df = measure.get_top_list('2018-06-22')
    headerMap = measure.get_header_map()
    headers = [{'key': header, 'title': headerMap[header]['title'], 'type': headerMap[header]['type']} for header in df.columns.values]
    body = df.to_json(orient='records', force_ascii=False)
    res = {}
    res['header'] = headers
    res['body'] = json.loads(body)
    return jsonify(res)

if __name__ == '__main__':
    app.run()


import os
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
from flask import Flask, render_template, send_from_directory

app = Flask(__name__)

# Change this to the path of your custom image folder
IMAGE_FOLDER = '/home/peter/OpenDays/otherApp/static/images'

class ImageHandler(FileSystemEventHandler):
    def __init__(self, app):
        self.app = app
    def on_any_event(self, event):
        if event.is_directory:
            return None
        elif event.event_type == 'created':
            self.update_image_list()
        elif event.event_type == 'deleted':
            self.update_image_list()
        elif event.event_type == 'modified':
            self.update_image_list()
    def update_image_list(self):
        images = sorted(os.listdir(IMAGE_FOLDER))
        self.app.jinja_env.globals['images'] = images

@app.route('/')
def index():
    images = sorted(os.listdir(IMAGE_FOLDER))
    return render_template('index.html', images=images)

@app.route('/image/<filename>')
def get_image(filename):
    return send_from_directory(IMAGE_FOLDER, filename)

if __name__ == '__main__':
    event_handler = ImageHandler(app)
    observer = Observer()
    observer.schedule(event_handler, path=IMAGE_FOLDER)
    observer.start()
    app.jinja_env.globals['images'] = []
    app.run(debug=False, host='0.0.0.0', port=8000)


import glob
import os
import pandas as pd
from pdfminer.pdfdocument import PDFDocument
from pdfminer.pdfparser import PDFParser
from pdfminer.pdftypes import resolve1
from pdfminer.psparser import PSLiteral
from pdfminer.utils import decode_text

def parse_questionnaires(drop_cols, input_dir, output_dir, output_fname):
    data_list = read_forms_data(input_dir)

    df=pd.DataFrame(data_list)
    df=df.drop(columns=drop_cols)
    df=df.dropna(how='all')
    df.to_csv(os.path.join(output_dir,output_fname),index=False)

def get_filename_without_extension(path):
    filename_basename = os.path.basename(path)
    filename_without_extension = filename_basename.split('.')[0]
    return filename_without_extension

def read_forms_data(input_dir):
    data_list = {}
    for file in glob.glob(os.path.join(input_dir, "*.pdf")):

        with open(file, 'rb') as fp:
            parser = PDFParser(fp)
            doc = PDFDocument(parser)
            res = resolve1(doc.catalog)

            if 'AcroForm' not in res:
                raise ValueError("No AcroForm found")

            fields = resolve1(doc.catalog['AcroForm'])['Fields']

            for f in fields:
                field = resolve1(f)
                name, values = field.get('T'), field.get('V')

                name = decode_text(name)
                values = resolve1(values)

                if type(values) is bytes:
                    values=decode_text(values)
                elif values is None:
                    values=''
                elif type(values) is PSLiteral:
                    values=values.name
                else:
                    print('')
                if name not in data_list:
                    data_list[name] = []
                data_list[name].append(values)
    return data_list


import sys
from collections import deque

N, M, oil = map(int,sys.stdin.readline().split())
arr = [list(map(int,sys.stdin.readline().split())) for _ in range(N)]
r,c = map(int,sys.stdin.readline().split())
r -= 1; c -= 1
infos = [list(map(int,sys.stdin.readline().split())) for _ in range(M)]
persons = [[[] for _ in range(N)] for _ in range(N)]
person_arr = [[0 for _ in range(N)] for _ in range(N)]
dest_arr = [[0 for _ in range(N)] for _ in range(N)]

dir = [(1,0), (-1,0), (0,1), (0,-1)]
for info in infos:
    r1,c1,r2,c2 = info
    persons[r1-1][c1-1] = [r2-1,c2-1]
    person_arr[r1-1][c1-1] = 2
    dest_arr[r2-1][c2-1] = 3

def find(row,col,target, tr,tc):
    que = deque([[0,row,col]])
    visited = [[True for _ in range(N)] for _ in range(N)]
    visited[row][col] = False
    result = []
    while que:
        depth, y, x = que.popleft()
        if target == 2 and person_arr[y][x] == 2:
            if depth > 0:
                result.sort(key=lambda x: (x[1], x[2]))
                person_arr[result[0][1]][result[0][2]] = 0
                return result[0]
            elif depth == 0:
                person_arr[y][x] = 0
                return [0, y, x]
        elif target == 3 and dest_arr[y][x] == 3 and y == tr and x == tc:
            return [depth, y, x]

        for dy, dx in dir:
            yy = y + dy; xx = x + dx
            if 0 <= yy < N and 0 <= xx < N and visited[yy][xx] and arr[yy][xx] == 0:
                if target == 2 and person_arr[yy][xx] == 2 :
                    result.append([depth+1, yy, xx])
                visited[yy][xx] = False
                que.append([depth+1, yy,xx])
    return [-1, -1 , -1]

for _ in range(M):
    dist, y, x = find(r,c,2,-1,-1)
    if dist == -1 :
        print(-1)
        break
    r,c = y,x; oil -= dist
    if oil <= 0 :
        print(-1)
        break
    tr, tc = persons[r][c]
    dist, y, x = find(r,c,3,tr,tc)
    if dist == -1 :
        print(-1)
        break
    r,c = y,x; oil -= dist
    if oil < 0 :
        print(-1)
        break
    oil += dist * 2
else:
    print(oil)


__copyright__ = \
    """
    Copyright (C) 2022 University of Liège, Gembloux Agro-Bio Tech, Forest Is Life
    All rights reserved.

    This source code is under the CC BY-NC-SA-4.0 license (https://creativecommons.org/licenses/by-nc-sa/4.0/).
    It is to be used for academic research purposes only, no commercial use is permitted.

    Please contact the author Alexandre Delplanque (alexandre.delplanque@uliege.be) for any questions.

    Last modification: March 29, 2023
    """
__author__ = "Alexandre Delplanque"
__license__ = "CC BY-NC-SA 4.0"
__version__ = "0.2.0"

import torch

__all__ = ['Adaloss']

class Adaloss:

    def __init__(
        self,
        param: torch.Tensor,
        w: int = 3,
        rho: float = 0.9,
        delta_max: float = 5.
        ) -> None:

        assert isinstance(param, torch.Tensor), \
            f'param must be a torch.Tensor, got {type(param)}'

        assert isinstance(w, int) and w > 0, \
            'w must be a positive integer'

        assert 0. <= rho <= 1., \
            f'rho value must be between 0.0 and 1.0, got {rho}'

        self.param = param
        self.w = w
        self.rho = rho
        self.delta_max = delta_max

        self._step = 1
        self._losses = []
        self.loss_history = []
        self.var_history = []

        self.param_tracker = []

    def step(self) -> None:

        self._update_losses()
        self._update_vars()

        start = self.w

        if self._step > start:
            delta = self._cdelta()
            if abs(delta) < self.delta_max:
                self.param.add_(delta)
            elif delta < 0 :
                self.param.add_(- torch.tensor(self.delta_max))
            else:
                self.param.add_(torch.tensor(self.delta_max))

        self._step += 1
        self.param_tracker.append(torch.clone(self.param))

    def feed(self, loss: torch.Tensor) -> None:
        self._losses.append(loss)

    def _update_losses(self) -> None:
        self.loss_history.append(torch.tensor(self._losses))
        self._losses = []

    def _update_vars(self) -> None:
        window = torch.cat(self.loss_history[self._step - self.w : self._step])
        variance = torch.var(window)
        self.var_history.append(variance)

    def _cdelta(self):
        ratio = torch.div(self.var_history[-2], self.var_history[-1])
        return self.rho * torch.sub(1., ratio)


import argparse

def create_base_parser():
    parser = argparse.ArgumentParser(prog='viztree',
                                     add_help=True,
                                     description='generate a tree html file to visualize a directory structure')

    parser.add_argument('path', nargs='?',
                        type=str, default='.',
                        help='directory path')

    parser.add_argument('--save', '-s',
                        type=str, default='index.html',
                        help='html file name to save the result')

    parser.add_argument('--skin',
                        type=str, default='xp',
                        choices=['xp', 'vista', 'win7', 'win8', 'win8-n', 'win8-xxl', 'lion'],
                        help='selecte skin that applies to the visualization')

    return parser.parse_args()


#思路整理：给出源数据路径-给出被拆分表格名称-启动excel程序-打开源工作簿-选中要拆分的工资表-读取要拆分的数据-创建空字典存储数据-遍历读取数据，字典的键为商品名称，字典的值为各商品的项数据
#-新建工作簿-源数据表列标题复制过来-源数据数据放到列标题下A2开始的区域。保存表格-退出excel文件
import xlwings as xw

file_path = "f:\\新建文件夹\\源数据.xlsx"  # 源工资簿的文件路径
sheet_name = 'Sheet1'  # 源工作表名称
app = xw.App(visible=True, add_book=False)
workbook = app.books.open(file_path)  # 打开源数据工作簿
worksheet = workbook.sheets(sheet_name)  # 选中源数据工资表
value = worksheet.range('A2').expand('table').value  # 读取要拆分的工作表中的所有数据
data = dict()
# print(len(value))#列表的元素个数，也即是行数
# print(value)#显示得到信息value为一个列表，列表的每一个元素是存储每一行信息的列表
for i in range(len(value)):
    product_name = value[i][1]
    if product_name not in data:
        data[product_name] = []
    data[product_name].append((value[i]))
for key, value in data.items():
    new_workbook = xw.books.add()
    new_worksheet = new_workbook.sheets.add(key)
    new_worksheet["A1"].value=worksheet["A1:H1"].value#将源数据的列标题复制到新建簿中
    new_worksheet["A2"].value=value
    new_workbook.save("{}.xlsx".format(key))
app.quit()

条件拆分一个表为多个簿.py


import flask
import requests
import json
import re
import psycopg2
import uuid
import copy

from flask import request, jsonify
from requests.auth import HTTPBasicAuth
import urllib.parse
from shapely.geometry import shape, GeometryCollection
import shapely.wkt

# Properties regarding the various types of attributes to be extracted from profiles

attribute_tags = ['type','count','num_missing','theme','uniqueness','nesting_level']

series_tags = ['num_peaks','abs_energy','abs_sum_changes','len_above_mean','len_below_mean']

temporal_tags = ['start','end']

geometry_tags = ['mbr','centroid','crs']

textual_tags = ['ratio_uppercase','ratio_digits','ratio_special_characters']

# Properties regarding entire profiles

tabular_tags = ['num_rows','num_attributes']

raster_tags = ['format','height','width','crs','spatial_coverage','spatial_resolution','start_date','end_date','temporal_resolution','no_data_value']

rdfgraph_tags = ['num_nodes','num_edges','num_namespaces','num_classes','num_object_properties','num_datatype_properties','density','num_connected_components']

hierarchical_tags = ['num_records', 'num_attributes']

text_tags = ['language','num_sentences','num_words','num_distinct_words','num_characters','ratio_uppercase','ratio_digits','ratio_special_characters']

# Properties regarding the various types of distributions

numerical_distribution_tags = ['count','average','stddev','min','max','median','percentile10','percentile25','percentile75','percentile90','kurtosis','skewness','variance']

categorical_distribution_tags = ['type','class_name','count','percentage']

language_distribution_tags = ['language','percentage']

def create_CKAN_headers(API_TOKEN):
    """Create the headers required for publishing a package or a resource in CKAN.

    Args:
        API_TOKEN (string): The API_TOKEN issued by CKAN that is required to establish connection and submit the request.

    Returns:
        Two JSON obejcts: (i) headers for package and (ii) headers for resource.
    """

    package_headers = {'Authorization': API_TOKEN, 'Content-Type': 'application/json'}
    resource_headers = { 'X-CKAN-API-Key': API_TOKEN }
    return package_headers, resource_headers

def validate_spatial(geometry):
    """Prepares the value for spatial extent in GeoJSON as expected by CKAN.

    Args:
        geometry: A WKT string or GeoJSON object representing a 2-dimensional geometry.

    Returns:
        A GeoJSON as expected by CKAN for spatial indexing in SOLR.
    """
    try:  # First, consider it as a GeoJSON
        if isinstance(geometry, dict):
            return json.dumps(geometry)   # dictionary
        else:
            g = json.loads(geometry)      # string containing a JSON
    except:
        try:  # Then, assuming this is WKT
            wkt = shapely.wkt.loads(geometry)
            g = shapely.geometry.mapping(wkt)
        except:
            g = {"type":"Polygon","coordinates":[]}  # Empty polygon

    return json.dumps(g)

def calc_bbox(geometry):
    """Calculate the bounding box of a geometry.

    Args:
        geometry: A GeoJSON object representing a 2-dimensional geometry.

    Returns:
        A list of four coordinates representing a bounding box (left, bottom, right, top).
    """

#    print("GeoJSON:", geometry)
    # NOTE: buffer(0) is a trick for fixing scenarios where polygons have overlapping coordinates
    g = GeometryCollection([shape(geometry).buffer(0)])
    bbox = g.bounds

    return str(bbox[0])+","+str(bbox[1])+","+str(bbox[2])+","+str(bbox[3])

def handle_extras(json_metadata):
    """Convert key value pairs from the input JSON into the format required for extra metadata in CKAN.

    Args:
        A JSON object with key value pairs.

    Returns:
        A JSON array as required for extra metadata in CKAN.
    """

    extras = []
    for key,value in json_metadata.items():
        item = {}
        item["key"] = key
        if key=="spatial": # Special handling of GeoJSON or WKT for spatial extent
            item["value"] = validate_spatial(json_metadata['spatial'])   #json.dumps(json_metadata['spatial'])
        elif isinstance(value, list):  # Convert a list as required for extras in CKAN.
            item["value"] = json.dumps(value)
        else:
            item["value"] = value
        extras.append(item)
#    arr_json = json.dumps(extras)
    return extras

def handle_keywords(list_tags):
    """Convert a list of keywords from the input JSON into the format required for keywords (tags) in CKAN.

    Args:
        A JSON array with string values.

    Returns:
        A JSON array as required for keywords (tags) in CKAN.
    """

    tags = []
    for value in list_tags:
        item = {}
        item["name"] = value
        tags.append(item)

    return tags

def format_CKAN_filter(json_metadata):
    """Convert key value pairs from the input JSON into the format required for queries in CKAN by SOLR. SOLR syntax: https://gist.github.com/mankyKitty/5906859

    Args:
        json_metadata (dict): A JSON object with key value pairs.

    Returns:
        A string with the query specifications for submission to CKAN.
    """

    q = '?q='
    bbox = None
    for key,value in json_metadata.items():
        if key=="spatial": # Special handling of GeoJSON
            bbox = calc_bbox(value) # Replace given GeoJSON with its BBOX
#            print("bbox: " , bbox)
    if bbox:
#        json_metadata['ext_bbox'] = bbox
        ext_bbox = 'ext_bbox='+urllib.parse.quote(bbox)
        del json_metadata['spatial']  # Remove GeoJSON from the parameters
    else:
        ext_bbox = None

    # Encode parameters for the URL request, as required by SOLR
    filters = urllib.parse.urlencode(json_metadata, doseq=False)

    if filters:
        q += urllib.parse.quote(filters)

    if ext_bbox:
        q = '?' + ext_bbox

#        if q:
#            q += '&' + ext_bbox
#        else:
#            q = '?' + ext_bbox

    return q   #urllib.parse.quote(q) #.encode('iso-8859-1'))

def cleanupDict(mydict, keys):
    """Removes any elements in the given dictionary that are not tagged under the given keys.

    Args:
        mydict (dict): A dictionary with key value pairs.
        keys (list): List of keys to retain in the given dictionary.

    Returns:
        A copy of the input dictionary, holding only the specified keys (if present).
    """

    for k in mydict.copy().keys():
        if not k in keys:
            mydict.pop(k) # Does nothing if the key is not present

    return mydict

def cleanupListDict(mylist, keys):
    """Removes any elements in each dictionary in the given list that are not tagged under the given keys.

    Args:
        mylist (dict): A list of dictionaries, each with the same key value pairs.
        keys (list): List of keys to retain in each dictionary.

    Returns:
        A copy of the input list, holding only the specified keys (if present) in each dictionary.
    """

    newlist = []
    for d in mylist:
        newlist.append(cleanupDict(d, keys))

    return newlist

def processTabularResource(resource_id, metadata, sql):
    """Process metadata about a tabular resource in CKAN.

    Args:
        resource_id (String) : A unique identifier for this resource (assigned by CKAN).
        metadata (array): JSON array containing the the metadata of this tabular resource (according to KLMS ontology).
        sql (array): JSON array collecting the SQL commands from this resource.

    Returns:
        An updated collection of INSERT SQL statements to be executed for ingesting the metadata into PostgreSQL according to KLMS schema.
    """

    # Collect general info about this resource
    tabular_metadata = cleanupDict(copy.deepcopy(prof), tabular_tags)
    tabular_metadata['resource_id'] = resource_id
    # Rename property as required by the schema
    if 'num_columns' in tabular_metadata:
        tabular_metadata['num_columns'] = tabular_metadata.pop('num_attributes')
    sql.append(prepareInsertSql(tabular_metadata, 'klms.tabular'))

def processTabularProfile(resource_id, prof, sql):
    """Provides metadata extracted from the profile of a tabular/vector dataset.

    Args:
        resource_id (String) : A unique identifier for this profile.
        prof (array): JSON array containing the profile information.
        sql (array): JSON array collecting the SQL commands from this profile.

    Returns:
        An updated collection of INSERT SQL statements to be executed for ingesting profile metadata into PostgreSQL according to KLMS schema.
    """

    # Collect general info about this profile
    tabular_metadata = cleanupDict(copy.deepcopy(prof), tabular_tags)
    tabular_metadata['resource_id'] = resource_id
    # Rename property as required by the schema
    tabular_metadata['num_columns'] = tabular_metadata.pop('num_attributes')
    sql.append(prepareInsertSql(tabular_metadata, 'klms.tabular'))

def processRasterResource(resource_id, metadata, sql):
    """Process metadata regarding a raster resource in CKAN.

    Args:
        resource_id (String) : A unique identifier for this resource (assigned by CKAN).
        metadata (array): JSON array containing the metadata of this raster resource (according to KLMS ontology).
        sql (array): JSON array collecting the SQL commands from this resource.

    Returns:
        An updated collection of INSERT SQL statements to be executed for ingesting the metadata into PostgreSQL according to KLMS schema.
    """

    # Collect general info about this resource
    raster_metadata = cleanupDict(copy.deepcopy(metadata), raster_tags)
    raster_metadata['resource_id'] = resource_id
    sql.append(prepareInsertSql(raster_metadata, 'klms.raster'))

def processRasterProfile(resource_id, prof, sql):
    """Provides metadata extracted from the profile of a raster dataset.

    Args:
        resource_id (String) : A unique identifier for this profile.
        prof (array): JSON array containing the profile information.
        sql (array): JSON array collecting the SQL commands from this profile.

    Returns:
        An updated collection of INSERT SQL statements to be executed for ingesting profile metadata into PostgreSQL according to KLMS schema.
    """

    # Collect general info about this profile
    raster_metadata = cleanupDict(copy.deepcopy(prof), raster_tags)
    # CAUTION! Currently, a raster profile indicates spatial resolution per axis; KLMS schema assumes a common resolution
    raster_metadata['spatial_resolution'] = raster_metadata['spatial_resolution']['pixel_size_x']
    raster_metadata['resource_id'] = resource_id
    sql.append(prepareInsertSql(raster_metadata, 'klms.raster'))

    # Also ingest information about each of the bands
    for band in prof['bands']:
        # Numerical distribution of pixel values
        band_distribution = cleanupDict(copy.deepcopy(band), numerical_distribution_tags)
        band_distribution['distr_id'] =  str(uuid.uuid1())  # Generate a UUID for this distribution
        sql.append(prepareInsertSql(band_distribution, 'klms.numerical_distribution'))
        # Also handle this band as a numerical attribute
        attr_metadata = {}
        attr_metadata['attr_name'] = band['name']
        attr_metadata['type'] = 'Band'
        attr_metadata['attr_id'] = band['uuid']  # Reuse the UUID already included in the original profile
        attr_metadata['resource_id'] = resource_id
        sql.append(prepareInsertSql(attr_metadata, 'klms.attribute'))
        band_metadata = {}
        band_metadata['attr_id'] = band['uuid']  # Reuse the UUID already included in the original profile
        band_metadata['value_distribution'] = band_distribution['distr_id']
        sql.append(prepareInsertSql(band_metadata, 'klms.numerical_attribute'))

def processHierarchicalResource(resource_id, metadata, sql):
    """Process metadata about a hierarchical resource in CKAN.

    Args:
        resource_id (String) : A unique identifier for this resource (assigned by CKAN).
        metadata (array): JSON array containing the the metadata of this hierarchical resource (according to KLMS ontology).
        sql (array): JSON array collecting the SQL commands from this resource.

    Returns:
        An updated collection of INSERT SQL statements to be executed for ingesting the metadata into PostgreSQL according to KLMS schema.
    """

    # Collect general info about this resource
    hierarchical_metadata = cleanupDict(copy.deepcopy(prof), hierarchical_tags)
    hierarchical_metadata['resource_id'] = resource_id
    sql.append(prepareInsertSql(hierarchical_metadata, 'klms.hierarchical'))

def processHierarchicalProfile(resource_id, prof, sql):
    """Provides metadata extracted from the profile of a hierarchical dataset.

    Args:
        resource_id (String) : A unique identifier for this profile.
        prof (array): JSON array containing the profile information.
        sql (array): JSON array collecting the SQL commands from this profile.

    Returns:
        An updated collection of INSERT SQL statements to be executed for ingesting profile metadata into PostgreSQL according to KLMS schema.
    """

    # Get all values concerning depth distribution
    depth_distribution = cleanupDict(copy.deepcopy(prof['depth_distribution']), numerical_distribution_tags)
    depth_distribution['distr_id'] =  str(uuid.uuid1())  # Generate a UUID for this distribution
    sql.append(prepareInsertSql(depth_distribution, 'klms.numerical_distribution'))
    # Collect general info about this profile
    hierarchical_metadata = cleanupDict(copy.deepcopy(prof), hierarchical_tags)
    hierarchical_metadata['resource_id'] = resource_id
#    hierarchical_metadata['num_attributes'] = len(profile['variables'])   # CAUTION! property NOT currently available in the profile
    hierarchical_metadata['depth_distribution'] = depth_distribution['distr_id']
    sql.append(prepareInsertSql(hierarchical_metadata, 'klms.hierarchical'))

def processRdfGraphResource(resource_id, metadata, sql):
    """Process metadata about a RDF graph resource in CKAN.

    Args:
        resource_id (String) : A unique identifier for this resource (assigned by CKAN).
        metadata (array): JSON array containing the the metadata of this RDF graph resource (according to KLMS ontology).
        sql (array): JSON array collecting the SQL commands from this resource.

    Returns:
        An updated collection of INSERT SQL statements to be executed for ingesting the metadata into PostgreSQL according to KLMS schema.
    """

    # Collect general info about this resource
    rdfgraph_metadata = cleanupDict(copy.deepcopy(prof), rdfgraph_tags)
    rdfgraph_metadata['resource_id'] = resource_id
    sql.append(prepareInsertSql(rdfgraph_metadata, 'klms.rdfgraph'))

def processRdfGraphProfile(resource_id, prof, sql):
    """Provides metadata extracted from the profile of an RDF graph.

    Args:
        resource_id (String) : A unique identifier for this profile.
        prof (array): JSON array containing the profile information.
        sql (array): JSON array collecting the SQL commands from this profile.

    Returns:
        An updated collection of INSERT SQL statements to be executed for ingesting profile metadata into PostgreSQL according to KLMS schema.
    """

    # Collect general info about this profile
    rdfgraph_metadata = cleanupDict(copy.deepcopy(prof), rdfgraph_tags)
    rdfgraph_metadata['resource_id'] = resource_id

    # Also get all statistics concerning its various distributions
    degree_centrality_distribution = cleanupDict(copy.deepcopy(prof['degree_centrality_distribution']), numerical_distribution_tags)
    degree_centrality_distribution['distr_id'] = str(uuid.uuid1())  # Generate a UUID for this distribution
    rdfgraph_metadata['degree_centrality_distribution'] = degree_centrality_distribution['distr_id']
    sql.append(prepareInsertSql(degree_centrality_distribution, 'klms.numerical_distribution'))
    degree_distribution = cleanupDict(copy.deepcopy(prof['degree_distribution']), numerical_distribution_tags)
    degree_distribution['distr_id'] = str(uuid.uuid1())  # Generate a UUID for this distribution
    rdfgraph_metadata['degree_distribution'] = degree_distribution['distr_id']
    sql.append(prepareInsertSql(degree_distribution, 'klms.numerical_distribution'))
    in_degree_distribution = cleanupDict(copy.deepcopy(prof['in_degree_distribution']), numerical_distribution_tags)
    in_degree_distribution['distr_id'] = str(uuid.uuid1())  # Generate a UUID for this distribution
    rdfgraph_metadata['in_degree_distribution'] = in_degree_distribution['distr_id']
    sql.append(prepareInsertSql(in_degree_distribution, 'klms.numerical_distribution'))
    out_degree_distribution = cleanupDict(copy.deepcopy(prof['out_degree_distribution']), numerical_distribution_tags)
    out_degree_distribution['distr_id'] = str(uuid.uuid1())  # Generate a UUID for this distribution
    rdfgraph_metadata['out_degree_distribution'] = out_degree_distribution['distr_id']
    sql.append(prepareInsertSql(out_degree_distribution, 'klms.numerical_distribution'))
    if 'class_distribution' in prof:
        class_uuid = str(uuid.uuid1())  # Generate a UUID for this distribution
        class_distribution = cleanupListDict(copy.deepcopy(prof['class_distribution']), categorical_distribution_tags)
        for item in class_distribution:
            item['distr_id'] = class_uuid
            # Rename property as required by the schema
            item['type'] = item.pop('class_name')
            sql.append(prepareInsertSql(item, 'klms.categorical_distribution'))
        rdfgraph_metadata['class_distribution'] = class_uuid
    # Must have included foreign keys to the various distributions
    sql.append(prepareInsertSql(rdfgraph_metadata, 'klms.rdfgraph'))

def processTextualResource(resource_id, metadata, sql):
    """Process metadata about a textual resource in CKAN.

    Args:
        resource_id (String) : A unique identifier for this resource (assigned by CKAN).
        metadata (array): JSON array containing the the metadata of this textual resource (according to KLMS ontology).
        sql (array): JSON array collecting the SQL commands from this resource.

    Returns:
        An updated collection of INSERT SQL statements to be executed for ingesting the metadata into PostgreSQL according to KLMS schema.
    """

    # Collect general info about this resource
    text_metadata = cleanupDict(copy.deepcopy(prof), text_tags)
    text_metadata['resource_id'] = resource_id
    sql.append(prepareInsertSql(text_metadata, 'klms.text'))

def processTextualProfile(resource_id, prof, sql):
    """Provides metadata extracted from the profile of a text.

    Args:
        resource_id (String) : A unique identifier for this profile.
        prof (array): JSON array containing the profile information.
        sql (array): JSON array collecting the SQL commands from this profile.

    Returns:
        An updated collection of INSERT SQL statements to be executed for ingesting profile metadata into PostgreSQL according to KLMS schema.
    """

    # Collect general info about this profile
    text_metadata = cleanupDict(copy.deepcopy(prof), text_tags)
    text_metadata['resource_id'] = resource_id
    # Get all values concerning its various distributions
    if 'sentence_length_distribution' in prof:
        sentence_length_distribution = cleanupDict(copy.deepcopy(prof['sentence_length_distribution']), numerical_distribution_tags)
        sentence_length_distribution['distr_id'] =  str(uuid.uuid1())  # Generate a UUID for this distribution
        sql.append(prepareInsertSql(sentence_length_distribution, 'klms.numerical_distribution'))
        text_metadata['sentence_length_distribution'] = sentence_length_distribution['distr_id']
    if 'word_length_distribution' in prof:
        word_length_distribution = cleanupDict(copy.deepcopy(prof['word_length_distribution']), numerical_distribution_tags)
        word_length_distribution['distr_id'] =  str(uuid.uuid1())  # Generate a UUID for this distribution
        sql.append(prepareInsertSql(word_length_distribution, 'klms.numerical_distribution'))
        text_metadata['word_length_distribution'] = word_length_distribution['distr_id']
    if 'language_distribution' in prof:
        lang_uuid = str(uuid.uuid1())  # Generate a UUID for this distribution
        lang_distribution = cleanupListDict(copy.deepcopy(prof['language_distribution']), language_distribution_tags)
        for item in lang_distribution:
            item['distr_id'] = lang_uuid
            # Rename property as required by the schema
            item['type'] = item.pop('language')
            sql.append(prepareInsertSql(item, 'klms.categorical_distribution'))
        text_metadata['language_distribution'] = lang_uuid
    if 'special_characters_distribution' in prof:
        chars_uuid = str(uuid.uuid1())  # Generate a UUID for this distribution
        chars_distribution = cleanupListDict(copy.deepcopy(prof['special_characters_distribution']), categorical_distribution_tags)
        for item in chars_distribution:
            item['distr_id'] = chars_uuid
            sql.append(prepareInsertSql(item, 'klms.categorical_distribution'))
        text_metadata['special_characters_distribution'] = chars_uuid
    # Must have included foreign keys to the various distributions
    sql.append(prepareInsertSql(text_metadata, 'klms.text'))

def extractProfileProperties(resource_id, profile):
    """Provides metadata extracted from a data profile.

    Args:
        resource_id (String) : A unique identifier for this profile.
        profile (array): JSON array containing the profile information.

    Returns:
        A collection of INSERT SQL statements to be executed for ingesting profile metadata into PostgreSQL according to KLMS schema.
    """

    sql = []   		# Collects SQL commands to be executed

    # PHASE #1: Dataset-related information
    prof = profile['table']  # No need to generate a UUID for this dataset; it has already obtained one as a CKAN resource
    # Handle each profile according to its type
    if prof['profiler_type'] == 'Tabular':
        processTabularProfile(resource_id, prof, sql)
    elif prof['profiler_type'] == 'Raster':
        # IMPORTANT! Extract information about the first raster in the collection (the only one allowed in CKAN)
        if len(profile['variables']) > 0:
            processRasterProfile(resource_id, profile['variables'][0], sql)
            # IMPORTANT! Return the list of collected SQL commands for execution
            return sql
    elif prof['profiler_type'] == 'Hierarchical':
        processHierarchicalProfile(resource_id, prof, sql)
    elif prof['profiler_type'] == 'RDFGraph':
        processRdfGraphProfile(resource_id, prof, sql)
        # IMPORTANT! Return the list of collected SQL commands for execution
        return sql
    elif prof['profiler_type'] == 'Textual':
        # TODO: Textual profiling is applicable on text collections; but CKAN accepts each text file as a separate resource
        # OPTION #1: Extract information about the corpus
        #processTextualProfile(resource_id, prof, sql)
        # OPTION #2: Extract information about the first text in the corpus (the only one allowed in CKAN)
        if len(profile['variables']) > 0:
            processTextualProfile(resource_id, profile['variables'][0], sql)
            # IMPORTANT! Return the list of collected SQL commands for execution
            return sql

    # PHASE #2: Attribute-related information, value distributions
    for var in profile['variables']:
        # Generate a UUID for this attribute
        attr_uuid = str(uuid.uuid1())

        # Collect general info about this attribute
        attribute_metadata = cleanupDict(copy.deepcopy(var), attribute_tags)
        attribute_metadata['attr_name'] = var['name']
        attribute_metadata['attr_id'] = attr_uuid
        attribute_metadata['resource_id'] = resource_id
        sql.append(prepareInsertSql(attribute_metadata, 'klms.attribute'))

        # Handle each attribute according to its type
        if var['type'] == 'Numeric':
            # Get all values concerning numerical distribution
            numerical_distribution = cleanupDict(copy.deepcopy(var), numerical_distribution_tags)
#            numerical_distribution['attr_id'] = attr_uuid
            numerical_distribution['distr_id'] =  str(uuid.uuid1())  # Generate a UUID for this distribution
            sql.append(prepareInsertSql(numerical_distribution, 'klms.numerical_distribution'))
            # Compose values concerning the numerical attribute
            numerical_attribute = {}
            numerical_attribute['attr_id'] = attr_uuid
            numerical_attribute['value_distribution'] = numerical_distribution['distr_id']
            sql.append(prepareInsertSql(numerical_attribute, 'klms.numerical_attribute'))
        elif var['type'] == 'TimeSeries':
            # Get all values concerning numerical distribution
            numerical_distribution = cleanupDict(copy.deepcopy(var), numerical_distribution_tags)
#            numerical_distribution['attr_id'] = attr_uuid
            numerical_distribution['distr_id'] =  str(uuid.uuid1())  # Generate a UUID for this distribution
            sql.append(prepareInsertSql(numerical_distribution, 'klms.numerical_distribution'))
            # Get all values concerning time series statistics
            series_attribute = cleanupDict(copy.deepcopy(var), series_tags)
            series_attribute['attr_id'] = attr_uuid
            series_attribute['value_distribution'] = numerical_distribution['distr_id']
            sql.append(prepareInsertSql(series_attribute, 'klms.series_attribute'))
        elif var['type'] == 'DateTime':
            # Get all values concerning a temporal attribute
            temporal_attribute = cleanupDict(copy.deepcopy(var), temporal_tags)
            temporal_attribute['attr_id'] = attr_uuid
            # Rename properties with PostgreSQL-reserved words
            temporal_attribute['start_time'] = temporal_attribute.pop('start')
            temporal_attribute['end_time'] = temporal_attribute.pop('end')
            sql.append(prepareInsertSql(temporal_attribute, 'klms.temporal_attribute'))
        elif var['type'] == 'Categorical':
            # Get all values concerning frequency distribution
            freq_uuid = str(uuid.uuid1())  # Generate a UUID for this distribution
            frequency_distribution = cleanupListDict(copy.deepcopy(var['frequency_distribution']), categorical_distribution_tags)
            for item in frequency_distribution:
                item['distr_id'] = freq_uuid
#                item['attr_id'] = attr_uuid
                sql.append(prepareInsertSql(item, 'klms.categorical_distribution'))
            # Get all values concerning a categorical attribute
            categorical_attribute = {}
            categorical_attribute['attr_id'] = attr_uuid
            categorical_attribute['frequency_distribution'] = freq_uuid
            sql.append(prepareInsertSql(categorical_attribute, 'klms.categorical_attribute'))
        elif var['type'] == 'Textual':
            # Get all values concerning a textual attribute
            textual_attribute = cleanupDict(copy.deepcopy(var), textual_tags)
            textual_attribute['attr_id'] = attr_uuid
            # Collect any statistics about distributions
            if 'num_chars_distribution' in var:
                chars_uuid = str(uuid.uuid1())  # Generate a UUID for this distribution
                chars_distribution = cleanupDict(copy.deepcopy(var['num_chars_distribution']), numerical_distribution_tags)
                chars_distribution['distr_id'] = chars_uuid
#                chars_distribution['attr_id'] = attr_uuid
                textual_attribute['num_chars_distribution'] = chars_uuid
                sql.append(prepareInsertSql(chars_distribution, 'klms.numerical_distribution'))
            if 'num_words_distribution' in var:
                words_uuid = str(uuid.uuid1())  # Generate a UUID for this distribution
                words_distribution = cleanupDict(copy.deepcopy(var['num_words_distribution']), numerical_distribution_tags)
                words_distribution['distr_id'] = words_uuid
#                words_distribution['attr_id'] = attr_uuid
                textual_attribute['num_words_distribution'] = words_uuid
                sql.append(prepareInsertSql(words_distribution, 'klms.numerical_distribution'))
            # Must have included foreign keys to the various distributions
            sql.append(prepareInsertSql(textual_attribute, 'klms.textual_attribute'))
        elif var['type'] == 'Geometry':
            # Get all values concerning a geometry attribute
            geometry_attribute = cleanupDict(copy.deepcopy(var), geometry_tags)
            geometry_attribute['attr_id'] = attr_uuid
            # Collect any statistics about distributions
            if 'length_distribution' in var:
                length_uuid = str(uuid.uuid1())  # Generate a UUID for this distribution
                length_distribution = cleanupDict(copy.deepcopy(var['length_distribution']), numerical_distribution_tags)
                length_distribution['distr_id'] = length_uuid
#                length_distribution['attr_id'] = attr_uuid
                geometry_attribute['length_distribution'] = length_uuid
                sql.append(prepareInsertSql(length_distribution, 'klms.numerical_distribution'))
            if 'area_distribution' in var:
                area_uuid = str(uuid.uuid1())  # Generate a UUID for this distribution
                area_distribution = cleanupDict(copy.deepcopy(var['length_distribution']), numerical_distribution_tags)
                area_distribution['distr_id'] = area_uuid
#                area_distribution['attr_id'] = attr_uuid
                geometry_attribute['area_distribution'] = area_uuid
                sql.append(prepareInsertSql(area_distribution, 'klms.numerical_distribution'))
            if 'geom_type_distribution' in var:
                geomtype_uuid = str(uuid.uuid1())  # Generate a UUID for this distribution
                geom_type_distribution = cleanupListDict(copy.deepcopy(var['geom_type_distribution']), categorical_distribution_tags)
                for item in geom_type_distribution:
                    item['distr_id'] = geomtype_uuid
#                    item['attr_id'] = attr_uuid
                    sql.append(prepareInsertSql(item, 'klms.categorical_distribution'))
                geometry_attribute['geom_type_distribution'] = geomtype_uuid
            # Must have included foreign keys to the various distributions
            sql.append(prepareInsertSql(geometry_attribute, 'klms.geometry_attribute'))

    # Return the list of collected SQL commands for execution
    return sql

def extractResourceProperties(resource_id, metadata):
    """Provides metadata extracted from a resource that conform to the KLMS schema.

    Args:
        resource_id (String) : A unique identifier for this resource.
        metadata (dict): JSON containing the resource metadata.

    Returns:
        A collection of INSERT SQL statements to be executed for ingesting resource metadata into PostgreSQL according to KLMS schema.
    """

    sql = []   		# Collects SQL commands to be executed

    # PHASE #1: Dataset-related information
    # Handle each resource according to its type
    if metadata['resource_type'] == 'Tabular':
        processTabularResource(resource_id, metadata, sql)
        return sql
    elif metadata['resource_type'] == 'Raster':
        processRasterResource(resource_id, metadata, sql)
        return sql
    elif metadata['resource_type'] == 'Hierarchical':
        processHierarchicalResource(resource_id, metadata, sql)
        return sql
    elif metadata['resource_type'] == 'RDFGraph':
        processRdfGraphResource(resource_id, metadata, sql)
        return sql
    elif metadata['resource_type'] == 'Textual':
        processTextualResource(resource_id, metadata, sql)
        return sql

    #TODO: Vector ???
    #TODO: TimeSeries ???

    # PHASE #2: Attribute-related information NOT applicable

    # Return the list of collected SQL commands for execution
    return sql

def prepareInsertSql(metadata, table):
    """Prepares an INSERT statement in SQL for ingesting the metadata into the specified table.

    Args:
#        resource_id (String): The name of the table where this data will be inserted.
        table (String): The name of the table where this data will be inserted.
        metadata (dict): JSON dictionary containing key, value pairs.

    Returns:
        A string with the INSERT statement to be executed in PostgreSQL.
    """

    columns = ", ".join(list(metadata.keys()))
    values = ", ".join("'{0}'".format(item) for item in metadata.values())
    sql = "INSERT INTO " + table + "(" + columns + ")" + " VALUES (" + values + ");"

    return sql

def prepareZenodoMetadata(dataset, creator, creator_org, doi:None):
    """Prepares the metadata in JSON about a dataset as expected by Zenodo.

    Args:
        dataset (JSON): A JSON object representing metadata for a dataset (CKAN packege).
        creator (String): The name of the creator of the dataset as listed in CKAN.
        creator_org (String): The name of the owner organization of the dataset as listed in CKAN.
        doi (String): The Digital Object Identifier of the dataset, if already assigned by the publisher If not, leave the field empty and Zenodo will register a new DOI when the dataset gets published.

    Returns:
        A JSON as expected by Zenodo for describing a dataset.
    """

    # Extract specific metadata as required by Zenodo schema
    # (schema largely conforms with https://schema.datacite.org/meta/kernel-4.4/metadata.xsd)
    res_id = dataset['id']

    # Basic metadata
    title = dataset['title']
    description = dataset['notes']
    tags = [t['name'] for t in dataset['tags']]

    # List of creators/authors of the Zenodo deposition (dataset)
    author = dataset['author'] if dataset['author'] else None
    maintainer = dataset['maintainer'] if dataset['maintainer'] else None
    organization = dataset['organization']['description']
    creators = []
    if creator:
        creators.append({'name': creator, 'affiliation': creator_org})
    if author:
        creators.append({'name': author, 'affiliation': organization})
    if maintainer:
        creators.append({'name': maintainer, 'affiliation': organization})

    url = dataset['url'] if dataset['url'] else None
    version = dataset['version'] if dataset['version'] else None
    isopen = dataset['isopen']
    private = dataset['private']
    license_title = dataset['license_title'] if dataset['license_title'] else None

    # Handle some of the available extra metadata
    spatial = next((item['value'] for item in dataset['extras'] if item['key'] == 'spatial'), None)
    spatial_resolution_in_meters = next((item['value'] for item in dataset['extras'] if item['key'] == 'spatial_resolution_in_meters'), None)
    temporal_start = next((item['value'] for item in dataset['extras'] if item['key'] == 'temporal_start'), None)
    temporal_end = next((item['value'] for item in dataset['extras'] if item['key'] == 'temporal_end'), None)
    frequency = next((item['value'] for item in dataset['extras'] if item['key'] == 'frequency'), None)
    documentation = next((item['value'] for item in dataset['extras'] if item['key'] == 'documentation'), None)
    language = next((item['value'] for item in dataset['extras'] if item['key'] == 'language'), None)
    theme = next((item['value'] for item in dataset['extras'] if item['key'] == 'theme'), None)
    alternate_identifier = next((item['value'] for item in dataset['extras'] if item['key'] == 'alternate_identifier'), None)

    # locations : list of locations -> NOT always the BBOX specified in CKAN
    # * lat (double): latitude
    # * lon (double): longitude
    # * place (string): place’s name (required)
    # * description (string): place’s description (optional)
    # Example: [{"lat": 34.02577, "lon": -118.7804, "place": "Los Angeles"}, {"place": "Mt.Fuji, Japan", "description": "Sample found 100ft from the foot of the mountain."}]
    locations = None
    if spatial:
        locations = []
        loc = {}
        # Extract the centroid from the spatial extent in CKAN
        geom = json.loads(spatial)
        bbox = shape(geom)
        loc['lon'] = bbox.centroid.x
        loc['lat'] = bbox.centroid.y
        loc['place'] = 'N/A'
        locations.append(loc)

    # access_right -> Controlled vocabulary in Zenodo:
    # * open: Open Access
    # * embargoed: Embargoed Access
    # * restricted: Restricted Access
    # * closed: Closed Access
    if isopen:
        access_right = "open"
        license = license_title if license_title else None
    elif private:
        access_right = "closed"
    else:
        access_right = "restricted"

    # dates -> List of date intervals
    # * start (ISO date string): start date (*)
    # * end (ISO date string): end date (*)
    # * type (Collected, Valid, Withdrawn): The interval’s type (required)
    # * description (string): The interval’s description (optional)
    # (*) Note that you have to specify at least a start or end date. For an exact date, use the same value for both start and end.
    # Example: [{"start": "2018-03-21", "end": "2018-03-25", "type": "Collected", "description": "Specimen A5 collection period."}]
    dates = None
    if temporal_start or temporal_end:
        dates = []
        timespan = {'type' : 'Valid'}   # Assuming that timespan specified the time period when dataset is valid
        if temporal_start:
            timespan['start'] = temporal_start
        if temporal_end:
            timespan['end'] = emporal_end
        dates.append(timespan)

    # language: the main language of the record as ISO 639-2 or 639-3 code
    lang = None
    if language:
        list_lang = language.replace('{','').replace('}','').split(',')
        lang = list_lang[0]  # the first language specified in CKAN

    # https://developers.zenodo.org/#representation
    # IMPORTANT: By default, using EU project grant for STELAR. List of OpenAIRE-supported grants. Example: [{'id':'283595'}] (European Commission grants only) or funder DOI-prefixed: [{'id': '10.13039/501100000780::283595'}] (All grants, recommended)
    zenodo_metadata = { "upload_type":"dataset", "creators": creators, "title": title, "description": description, "keywords": tags, "access_right": access_right, "language":lang, "locations": locations, "dates" : dates,"license": license,"doi": doi, "grants": [{"id": "10.13039/501100000780::101070122"}] }

    return zenodo_metadata


from pyspark import SparkContext
STAT_FILE = "/user/x_robsl/data/stations-Ostergotland.csv"
PREC_FILE = "/user/x_robsl/data/precipitation-readings.csv"

sc = SparkContext(appName = "exercise 5")

stations_file = sc.textFile(STAT_FILE)
ostg_stations = stations_file.map(lambda line: line.split(";")[0]).collect()

precipitations_file = sc.textFile(PREC_FILE)
prec_data = precipitations_file.map(lambda line: line.split(";"))

# Filter out the data outside the wanted dates
prec_data = prec_data.filter(lambda x: int(x[1][0:4]) >= 1993 and int(x[1][0:4]) <= 2016)

# Filter only those station numbers that are in STAT_FILE.
prec_data = prec_data.filter(lambda x: x[0] in ostg_stations)

# map data like: (station_number;year-month, value)
prec_data = prec_data.map(lambda x: (x[0] + ";" + x[1][0:7], float(x[3])))
# calculate total monthly precipitation for each station
prec_data = prec_data.reduceByKey(lambda val1, val2: val1 + val2)

# We remove the stationnumber, since it is not required anymore
# The last 1 in the tuple is just used to track the amount of stations to
# divide with in the end.
prec_data = prec_data.map(lambda x: (x[0].split(';')[1], (x[1], 1)))
# Add all monthly values together
prec_data = prec_data.reduceByKey(lambda val1, val2: (val1[0] + val2[0], val1[1] + val2[1]))
# Divide the total monthly value to get average
prec_data = prec_data.map(lambda x: (x[0], x[1][0]/x[1][1]))

# Sort and save
output_precs = prec_data.sortBy(ascending = True, keyfunc = lambda k: (k[0], k[1]))
output_precs.saveAsTextFile("avg_prec")


# Implementar la funcion agregar_persona, que inserte un registro en la tabla Persona
# y devuelva los datos ingresados el id del nuevo registro.

from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, ForeignKey, Integer, String, Date
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

import datetime

from ejercicio_01 import Persona, session, crear_tabla, borrar_tabla

def agregar_persona(nombre, nacimiento, dni, altura):
    per = Persona()
    per.nombre = nombre
    per.fechaNacimiento = nacimiento
    per.dni = dni
    per.altura = altura
    session.add(per)
    session.commit()
    #p = session.query(Persona).all()
    #print("Lista de Personas: ")
    #for p in lp:
    #    print("Persona ",p.id,p.nombre,p.fechaNacimiento,p.dni,p.altura)
    return per.id

#@reset_tabla
def pruebas():
    crear_tabla()
    id_juan = agregar_persona('juan perez', datetime.date(1988, 5, 15), 32165498, 180)
    id_marcela = agregar_persona('marcela gonzalez', datetime.date(1980, 1, 25), 12164492, 195)
    assert id_juan > 0
    assert id_marcela > id_juan
    borrar_tabla()

if __name__ == '__main__':
    pruebas()

Soporte a la Gestión de Datos con Programación Visual/Trabajo Práctico 3A/ejercicio_02.py


import csv

import requests
import json

def getHTMLText(url):
    try:
        response = requests.get(url)
        response.raise_for_status()
        response.encoding = response.apparent_encoding
        return response.text
    except:
        return "error in getHTMLText"

base = "https://v0.yiketianqi.com/api?version=history&appid=23295957&appsecret=qD0oO1SD&city="
date = "&year=2021&month=3"
cities = ['杭州', '宁波', '温州', '嘉兴', '湖州', '绍兴', '金华', '衢州', '舟山', '台州', '丽水']
with open('data.csv', 'w', newline="") as file:
    writer = csv.writer(file)
    writer.writerow(
        ['city', 'ymd', 'bWendu', 'yWendu', 'tianqi', 'fengxiang', 'fengli', 'aqi', 'aqiInfo', 'aqiLevel'])
    for city in cities:
        url = str(base + city + date)
        data = json.loads(getHTMLText(url))
        for i in range(0, len(data['data'])):
            writer.writerow(
                [data['city'], data['data'][i]['ymd'], data['data'][i]['bWendu'], data['data'][i]['yWendu'],
                 data['data'][i]['tianqi'], data['data'][i]['fengxiang'], data['data'][i]['fengli'],
                 data['data'][i]['aqi'], data['data'][i]['aqiInfo'], data['data'][i]['aqiLevel']])
        print(data)


"""Initial migration

Revision ID: 8f20a09ef9f1
Revises:
Create Date: 2020-04-16 11:31:02.199129

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '8f20a09ef9f1'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    todo_table = op.create_table('todo',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('title', sa.String(length=80), nullable=False),
    sa.Column('description', sa.String(length=280), nullable=False),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('title')
    )
    print('Adding:', [{'id': i, 'title': f'{i} thing to do', 'description': f'The number {i} thing to do'} for i in range(5)])
    op.bulk_insert(todo_table,
    [{'id': i, 'title': f'{i} thing to do', 'description': f'The number {i} thing to do'} for i in range(5)]
    )

    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('todo')
    # ### end Alembic commands ###


import requests
import praw

def main():

    mdTablePath = 'csvAndMarkDown/markDownFiles/markdownTable.md'
    endOfPostPath = 'csvAndMarkDown/markDownFiles/endOfPost.md'

    with open(mdTablePath, 'r') as x:
        mainPost = x.read()

    with open(endOfPostPath, 'r') as x:
        ending = x.read()

    redditInstance = praw.Reddit('dwgBot')

    testSub = redditInstance.subreddit('XboxOne')

    testSub.submit('This Weeks Deals with Gold and Spotlight Sale! Formatted for Easy Reading!', selftext= mainPost + ending)
    print('\nSubmitted!')

if __name__ == '__main__':
    main()


# crwd.py
#
# Python library to query the CrowdStrike Falcon Custom Indicator API for data on file hashes and IP addresses.
# Responses are returned in JSON format. This library can be used independently or as part
# of the intel_collector threat intelligence aggregator project.
#
# github.com/jasonsford
# 7 September 2022

import json
import requests

crwd_client_id = 'your crowdstrike api client id'
crwd_client_secret = 'your crowdstrike api client secret'

crwd_base_url = 'https://api.crowdstrike.com'

def iocs(indicator: str, indicator_type: str):

    crwd_session = requests.session()
    crwd_session.verify = True

    crwd_payload = {'client_id': crwd_client_id, 'client_secret': crwd_client_secret}

    crwd_api_response = crwd_session.post(crwd_base_url + '/oauth2/token', data=crwd_payload)

    if(crwd_api_response.status_code == 201):

        headers =  {'Authorization': f'Bearer {crwd_api_response.json()["access_token"]}',
                    'token_type': 'bearer',
                    'Content-Type': 'application/json'}

        crwd_session.headers = headers

        crwd_params = {'type': indicator_type, 'value': indicator}

        crwd_getdetailedinfo = crwd_session.get(crwd_base_url + '/indicators/entities/iocs/v1', params=crwd_params)

        if(crwd_getdetailedinfo.status_code == 200):
            print('Response from CrowdStrike - Custom IOCs')
            return json.loads(crwd_getdetailedinfo.text)['resources']

    crwd_session.close()


import sys; sys.path.append('..')
import pickle
import numpy as np

import torch
from torch.utils.data import DataLoader
import torch.nn.functional as F

from sklearn.manifold import TSNE
import matplotlib.pyplot as plt

#from data.chem_preprocess import PIDataset
from data.torch_process import PIDataset

global_train_loss = []
global_test_loss = []

def make_dataloader(base_structures, batch_size = 32, device = None):
    trans = lambda x: x.reshape(1, int(np.sqrt(x.shape[0])), int(np.sqrt(x.shape[0])))

    dataset = PIDataset(root = '../data/xtb_data', base_structures = base_structures, transform = trans, device = device)

    splits = [int(0.7 * len(dataset)), int(0.3 * len(dataset))]

    train_data, test_data = torch.utils.data.random_split(dataset, [splits[0], splits[1] + (len(dataset) - sum(splits))])

    return DataLoader(train_data, batch_size = batch_size, shuffle=True), DataLoader(test_data, batch_size = 1, shuffle=False)

def r2_loss(output, target):
    target_mean = torch.mean(target)
    ss_tot = torch.sum((target - target_mean) ** 2)
    ss_res = torch.sum((target - output) ** 2)
    r2 = 1 - ss_res / ss_tot
    return r2

activation = {}
def get_activation(name):
    def hook(model, input, output):
        activation[name] = output.detach()
    return hook

class CNN1(torch.nn.Module):
    def __init__(self, PI_dim):
        super().__init__()
        self.conv1 = torch.nn.Conv2d(1, 8, 3, padding = 'same')
        self.batchnorm1 = torch.nn.BatchNorm2d(8)
        self.conv2 = torch.nn.Conv2d(8, 16, 3, padding = 'same')
        self.batchnorm2 = torch.nn.BatchNorm2d(16)
        self.fc1 = torch.nn.Linear(16 * PI_dim * PI_dim, 64)
        self.fc1.register_forward_hook(get_activation('fc1'))
        self.fc2 = torch.nn.Linear(64, 1)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        x = self.batchnorm1(x)
        x = F.relu(self.conv2(x))
        x = self.batchnorm2(x)
        x = torch.nn.Flatten()(x)
        x = F.relu(self.fc1(x))
        x = self.fc2(x) # No activation on final

        return x

def train(train_dataloader, testloader, model, epochs = 100):
    criterion = torch.nn.L1Loss()
    optimizer = torch.optim.Adam(model.parameters(), lr = 0.001)

    for ep in range(epochs):
        running_loss = 0.0
        for i, data in enumerate(train_dataloader):
            X, y = data
            optimizer.zero_grad()

            outputs = model(X)
            loss = criterion(outputs, y)
            loss.backward()

            optimizer.step()

            running_loss += loss.item()

            if i % 100 == 0:
                train_loss = running_loss / (100 * int(100 / i)) if i > 0 else running_loss
                test_loss = test(testloader, model)
                global_train_loss.append(train_loss)
                global_test_loss.append(test_loss)
                print(f'Epoch: {ep}, Loss: {train_loss}, Test Loss: {test_loss}')

        # for X, y in test_dataloader:

def test(test_dataloader, model):

    criterion = torch.nn.L1Loss()

    total_loss = 0
    n = 0

    with torch.no_grad():
        for X, y in test_dataloader:
            output = model(X)

            loss = criterion(output, y)

            total_loss += loss
            n += 1

    return total_loss / n

def get_embedding(loader, model):

    all_ys = []
    all_Xs = []

    with torch.no_grad():
        for X, y in loader:
            all_ys.append(y.detach())
            _ = model(X)
            all_Xs.append(activation['fc1'].detach())

    all_ys = torch.cat(all_ys, dim=0).tolist()
    all_Xs = torch.cat(all_Xs, dim = 0).numpy()

    xtsne = TSNE().fit_transform(all_Xs)
    plt.scatter(xtsne[:,0], xtsne[:,1], c = all_ys)
    plt.show()

if __name__ == '__main__':
    model = CNN1(PI_dim = 50)

    assert len(sys.argv) == 2, "usage: python3 CNN.py <epochs>"
    epochs = int(sys.argv[1])

    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

    train_loader, test_loader = make_dataloader([1], batch_size = 8, device = device)

    train(train_loader, test_loader, model, epochs = epochs)

    print('MAE Test', test(test_loader, model))

    pickle.dump(global_train_loss, open('global_train_loss.pickle', 'wb'))
    pickle.dump(global_test_loss, open('global_test_loss.pickle', 'wb'))

    # plt.plot(global_train_loss, label = 'train')
    # plt.plot(global_test_loss, label = 'test')
    # plt.legend()
    # plt.show()

    #get_embedding(train_loader, model)


from django.http.response import Http404
from django.shortcuts import render
from rest_framework.views import APIView
from .models import Subject, Question, Answer
from .serializers import SubjectSerializer, QuestionSerializer
from rest_framework.response import Response

class SubjectAPIView(APIView):

    def get_object(self, pk):
        try:
            return Subject.objects.get(pk=pk)
        except Subject.DoesNotExist:
            raise Http404

    def get(self, request, pk=None, format=None):
        if pk:
            data = self.get_object(pk)
            serializer = SubjectSerializer(data)

        else:
            data = Subject.objects.all()
            serializer = SubjectSerializer(data, many=True)

            return Response(serializer.data)

class QuestionAPIView(APIView):

    def get_object(self, pk):
        try:
            return QuestionAPIView.objects.get(pk=pk)
        except Question.DoesNotExist:
            raise Http404

    def get(self, request, pk=None, format=None):
        if pk:
            data = self.get_object(pk)
            serializer = QuestionSerializer(data)

        else:
            data = Question.objects.all()
            serializer = QuestionSerializer(data, many=True)

            return Response(serializer.data)
class AnswerAPIView(APIView):

    def get_object(self, pk):
        try:
            return AnswerAPIView.objects.get(pk=pk)
        except Answer.DoesNotExist:
            raise Http404

    def get(self, request, pk=None, format=None):
        if pk:
            data = self.get_object(pk)
            serializer = AnswerSerializer(data)

        else:
            data = Answer.objects.all()
            serializer = AnswerSerializer(data, many=True)

            return Response(serializer.data)

def post(self, request, format=None):
    data = request.data
    serializer = SubjectSerializer(data=data)

    serializer.is_valid(raise_exception=True)

    serializer.save()

    response = Response()

    response.data = {
        'question': 'what is newton 3rd law of motion',
        'data': serializer.data
    }

    return response

def put(self, request, pk=None, format=None):
    question_to_update = Question.objects.get(pk=pk)
    serializer = QuestionSerializer(instance=question_to_update,data=request.data, partial=True)
    serializer.is_valid(raise_exception=True)
    serializer.save()
    response = Response()
    response.data = {
        'question': 'what is the newton 2nd law',
        'data': serializer.data
    }

    return response

def delete(self, request, pk, format=None):
        question_to_delete =  Question.objects.get(pk=pk)

        question_to_delete.delete()

        return Response({
            'message': 'Todo Deleted Successfully'
        })


#!/usr/bin/python3

import vlc
import time
import sys
import signal
import asyncio

from PyQt5 import QtWidgets, QtCore, QtGui
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
from PyQt5.QtGui import *

signal.signal(signal.SIGINT, signal.SIG_DFL)

radioplayprefix: str = '?direct=true&listenerid=undefined&aw_0_1st.bauer_listenerid=undefined&aw_0_1st.playerid=BMUK_inpage_html5&aw_0_1st.skey=' + str(
	int(time.time()) - 4) + '&aw_0_1st.bauer_loggedin=false&aw_0_req.userConsentV2=false'

channels = [
	{
		"YLE Puhe": "https://yleradiolive.akamaized.net/hls/live/2027677/in-YlePuhe/abr.smil/chunklist_b256000_ao.m3u8",
		"YLE X": "https://yleradiolive.akamaized.net/hls/live/2027674/in-YleX/master.m3u8",
	},
	{
		'Top51': 'https://stream.radioplay.fi/top51/top51_64.aac',
		"Järviradio": "https://jarviradio.radiotaajuus.fi:9000/jr",
		"Radio Helsinki": "http://77.86.233.1:8004/",
	},
	{
		'SuomiRap': 'https://stream.radioplay.fi/suomirap/suomirap_64.aac',
		'NRJ': 'https://stream.radioplay.fi/nrj/nrj_64.aac',
	}
]

class MainWindow(QtWidgets.QMainWindow):

	async def update_volume(self) -> None:

		await asyncio.sleep(1)

		media_volume = self.media_player.audio_get_volume()
		self.sl.setValue(media_volume)
		self.label.setText(str(media_volume))

	def render_stop_button(self) -> None:

		mipmap = getattr(QtWidgets.QStyle, 'SP_MediaStop')
		icon = self.style().standardIcon(mipmap)

		stop_button = QtWidgets.QPushButton(None, self)
		stop_button.setIcon(icon)
		stop_button.clicked.connect(lambda: self.stop_radio())
		stop_button.resize(50, 32)
		stop_button.move(335, 155)

	def render_slider(self) -> None:

		self.sl = QtWidgets.QSlider(QtCore.Qt.Orientation.Horizontal, self)
		self.sl.setMinimum(0)
		self.sl.setMaximum(100)
		self.sl.setValue(25)
		self.sl.setTickPosition(QtWidgets.QSlider.TicksBelow)
		self.sl.setTickInterval(5)
		self.sl.setGeometry(50, 160, 270, 30)
		self.sl.valueChanged.connect(self.update_label)

		self.label = QtWidgets.QLabel('25', self)
		self.label.setMinimumWidth(80)
		self.label.move(13, 154)

	def update_label(self, value: int) -> None:

		self.media_player.audio_set_volume(value)
		self.label.setText(str(value))

	def __init__(self) -> None:

		QtWidgets.QMainWindow.__init__(self)

		self.setMinimumSize(QtCore.QSize(400, 200))
		self.setWindowTitle("r A d I o")

		self.sl = QtWidgets.QSlider(QtCore.Qt.Orientation.Horizontal, self)
		self.label = QtWidgets.QLabel('25', self)

		self.render_slider()
		self.render_stop_button()

		self.media_player = vlc.MediaPlayer()

		position_y = 10
		for channelGroups in channels:
			position_x = 10
			for name, url in channelGroups.items():
				self.build_button(name, position_x, position_y)
				position_x += 130
			position_y += 40

	def build_button(self, name: str, left: int, top: int) -> QtWidgets.QPushButton:

		button = QtWidgets.QPushButton(name, self)
		button.clicked.connect(lambda: self.play_radio(name))
		button.resize(120, 32)
		button.move(left, top)

		return button

	def stop_radio(self) -> None:

		self.media_player.stop()

	@staticmethod
	def get_station_url(channel: str) -> str:

		for channelGroups in channels:
			for name, url in channelGroups.items():
				if name == channel:
					if 'radioplay' in url:
						return url + radioplayprefix

					return url

	def play_radio(self, channel: str) -> None:

		self.stop_radio()

		media = vlc.Media(self.get_station_url(channel))
		self.media_player.set_media(media)
		self.media_player.play()

		print('selected channel: ', channel)

		asyncio.run(self.update_volume())

if __name__ == "__main__":
	app = QtWidgets.QApplication(sys.argv)
	mainWin = MainWindow()
	mainWin.show()
	sys.exit(app.exec_())


import cv2
import torch
import torch.hub

# Load the YOLOv5s model
model = torch.hub.load('ultralytics/yolov5', 'yolov5s')

# Create a VideoCapture object
cap = cv2.VideoCapture(0)

# Set the sharing strategy for PyTorch DataLoader
torch.multiprocessing.set_sharing_strategy('file_system')

while True:
    # Get a frame from the webcam
    ret, frame = cap.read()

    # Perform object detection on the frame
    results = model(frame)

    # Get the detections
    detections = results.pred[0]

    # Loop over the detections and draw rectangles on the frame
    for detection in detections:
        label = results.names[int(detection[5])]
        confidence = detection[4].item()
        bbox = detection[:4].cpu().numpy()

        if confidence > 0.3:  # Lower the confidence threshold to display more detections
            x1, y1, x2, y2 = bbox
            color = (0, 255, 0)  # Green color for the bounding box
            thickness = 2
            cv2.rectangle(frame, (int(x1), int(y1)), (int(x2), int(y2)), color, thickness)
            cv2.putText(frame, f"{label}: {confidence:.2f}", (int(x1), int(y1) - 10),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, thickness)

    # Show the frame with detections
    cv2.imshow('Object Detection', frame)

    # Press 'q' to quit
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

# Release the VideoCapture and close the OpenCV window
cap.release()
cv2.destroyAllWindows()