from selenium import webdriver
import bs4
import os
import re
from selenium.common.exceptions import NoSuchElementException
from time import sleep
import json
import winsound
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

#%%
def NYT_crawl(url, i = 10**9, path = os.getcwd(), return_ans = False):
    if re.search('^(([^:/?#]+):)?(//([^/?#]*))?',url).groups()[-1] != 'www.nytimes.com': #check url host matches
        print('Please input url from www.nytimes.com')
        return
    browser = webdriver.Chrome()
    browser.get(url)
    urls = []
    more_pages = True
    clicks = (i - 10)//10 #calculate number of times page needs to be extended
    j = 0 #initiate counter

#%%
    while more_pages and j < clicks:
        try:
            next_button = browser.find_element_by_css_selector('div.Search-showMoreWrapper--1Z88y') #get show more wrapper
            next_button.click() #click to show more urls
            j += 1 #incrment counter
            sleep(2.5)
        except NoSuchElementException: #if no button
            more_pages = False #no more pages
        except Exception as e: #any other errors
            innerHTML = browser.execute_script("return document.body.innerHTML") #get html
            soup = bs4.BeautifulSoup(innerHTML,'html.parser') #parse html
            for item in soup.select('div[class*="Item-wrapper--2ba8L"]'): #find elements with urls
                url_link = 'https://www.nytimes.com' + item.find_all('a')[0]['href'] #construct urls
                urls.append((url_link,'NYT')) #add url to list
            urls = list(set(urls)) #clean up urls
            print('Encountered unexpected problem')
            print(e)
            print(f'Scraped {len(urls)} urls')
            filename = path + f'\\{len(urls)}_nyt_urls.json' #conduct emergency save
            with open(filename, 'w') as outfile:
                json.dump(urls, outfile)
            if return_ans:
                return urls
            else:
                return

#%%
    innerHTML = browser.execute_script("return document.body.innerHTML") #get html
    soup = bs4.BeautifulSoup(innerHTML,'html.parser') #parse html
    for item in soup.select('div[class*="Item-wrapper--2ba8L"]'): #find elements with urls
        url_link = 'https://www.nytimes.com' + item.find_all('a')[0]['href'] #construct urls
        urls.append((url_link,'NYT')) #add url to list

#%%
    urls = list(set(urls)) #clean list
    browser.close()
    filename = path + '\\nyt_urls.json'
    with open(filename, 'w') as outfile:  #save data as json
        json.dump(urls, outfile)
    winsound.PlaySound('ding.wav', winsound.SND_FILENAME) #notify user
    if return_ans: #return answer if wanted
        return urls


from neutronclient.i18n import _
from neutronclient.neutron import v2_0 as neutronv20

class ListQoSPolicy(neutronv20.ListCommand):
    """List QoS policies that belong to a given tenant connection."""

    resource = 'policy'
    list_columns = ['id', 'name']
    _formatters = {}
    pagination_support = True
    sorting_support = True

class ShowQoSPolicy(neutronv20.ShowCommand):
    """Show information of a given qos policy."""

    resource = 'policy'

class CreateQoSPolicy(neutronv20.CreateCommand):
    """Create a qos policy."""

    resource = 'policy'

    def add_known_arguments(self, parser):
        parser.add_argument(
            'name', metavar='NAME',
            help=_('Name of QoS policy to create.'))
        parser.add_argument(
            '--description',
            help=_('Description of the QoS policy.'))
        parser.add_argument(
            '--shared',
            action='store_true',
            help=_('Accessible by other tenants. '
                   'Set shared to True (default is False).'))

    def args2body(self, parsed_args):
        body = {self.resource: {'name': parsed_args.name}, }
        if parsed_args.description:
            body[self.resource].update(
                {'description': parsed_args.description})
        if parsed_args.shared:
            body[self.resource].update(
                {'shared': parsed_args.shared})
        return body

class UpdateQoSPolicy(neutronv20.UpdateCommand):
    """Update a given qos policy."""

    resource = 'policy'

    def add_known_arguments(self, parser):
        parser.add_argument(
            '--description',
            help=_('Description of the QoS policy.'))

    def args2body(self, parsed_args):
        body = {self.resource: {}, }
        if parsed_args.description:
            body[self.resource].update(
                {'description': parsed_args.description})
        return body

class DeleteQoSPolicy(neutronv20.DeleteCommand):
    """Delete a given qos policy."""

    resource = 'policy'


from django.urls import path, include
from .views import home,base,acceso, Reserva,huesped,cliente, agendar, index, listarCliente, listarHuesped, funcionario, listarFuncionarios, modificarCliente, modificarHuesped,calcularTarifa, base2, nosotros, estadia, listaReserva, modificarReservas, vacunas, listar_vacunas, modificarVacuna
from django.contrib.auth.decorators import login_required

urlpatterns = [
    path('', index, name="index"),
    path('acceso/', login_required (acceso), name="acceso"),
    path('reserva/', login_required (Reserva), name="reserva"),
    path('huesped/', login_required(huesped), name="huesped"),
    path('cliente/', login_required(cliente), name="cliente"),
    path('base/', login_required(base), name='base'),
    path('agenda/', login_required (agendar), name='agenda'),
    path('home/', login_required(home), name='home'),
    path('listar/', login_required(listarCliente), name='listar'),
    path('listarHuesped/', login_required(listarHuesped), name='listarHuesped'),
    path('funcionario/', login_required(funcionario), name='funcionario'),
    path('listarFuncionarios/', login_required(listarFuncionarios), name='listarFuncionarios'),
    path('modificarCliente/<id>/', login_required(modificarCliente), name='modificarCliente'),
    path('modificarHuesped/<id>/', login_required(modificarHuesped), name='modificarHuesped'),
    path('calcular_tarifa/', login_required(calcularTarifa), name='calcular_tarifa'),
    path('resultado/', login_required(calcularTarifa), name='resultado'),
    path('nosotros/', nosotros, name='nosotros'),
    path('base2/',base2, name='base2'),
    path('estadia/', login_required(estadia), name='estadia'),
    path('lista_reserva/', login_required(listaReserva), name='lista_reserva'),
    path('modificarReserva/<id>/', login_required(modificarReservas), name='modificarReserva'),
    path('vacuna/', login_required(vacunas), name='vacuna'),
    path('listar_vacuna/', login_required(listar_vacunas), name='listar_vacuna'),
    path('modificarVacuna/<id>/', login_required(modificarVacuna), name='modificarVacuna'),

    ]


import subprocess, time, sys, os, tool, json, configparser, requests, logging
from threading import Thread, Lock
from PyQt6 import QtWidgets, QtCore, QtGui, uic, QtQml
from PyQt6.QtWidgets import QMessageBox, QFileDialog, QApplication
from PyQt6.QtCore import QRunnable, pyqtSlot, QThreadPool

logging.basicConfig(level=logging.INFO, filename="Logs\\Update.log",filemode="a+",
                    format="[%(asctime)s] %(levelname)s: %(message)s")

def LoadCfg():
    global Config, THREADS, HOME_FOLDER, SAVE_FOLDER, LANGUAGE, Speed, Tune, Codec, Langs, config, Check_ver_link, Version
    Config = configparser.ConfigParser()
    Config.read("Settings.ini")

    THREADS = Config["PERFORMANCE"].getint("THREADS", os.cpu_count())
    HOME_FOLDER = Config["OTHER"].get("HOME_FOLDER", os.path.dirname(os.path.realpath(__file__)))
    SAVE_FOLDER = Config["OTHER"].get("SAVE_FOLDER", os.path.dirname(os.path.realpath(__file__)))
    LANGUAGE = Config["OTHER"].get("LANGUAGE", "EN")

    with open("config.json") as f:
        config = json.load(f)
    Speed = config["Presets"]
    Tune = config["Tune"]
    Codec = config["Codecs"]
    Langs = config["Langs"]
    Check_ver_link = config["Update"]["check_url"]
    Version = config["Update"]["version"]

LoadCfg()

# monkey-patch/replace Popen
subprocess.Popen = tool.new_Popen

def check_for_updates():
    logging.info("Looking for updates")
    for i in range(3):
        try:
            logging.info("Trying to connect to update server...")
            r = requests.get(Check_ver_link,timeout=0.5)
            if r.json()["version"] != Version:
                logging.info("Found an update!")
                MainWindow.Ask_for_update(MainWindow,r.json()["version"])
                return
            else:
                logging.info("Update not founded")
                return
        except requests.exceptions.RequestException:
            logging.error(f"Connection failed. #{i} try")
    logging.error("Failed to reach server")

#Delay which not freezing program
def Delay(sec : int):
    for i in range(0, sec*10, 1):
        QApplication.processEvents()
        time.sleep(0.1)

#Main Conversion Function
def ConvertPrep():
    global args, done
    LoadCfg()
    args = [                             #Command line formation
        "ffmpeg\\bin\\ffmpeg.exe",
        "-i", "",
        "-c:v", "",
        "-tune", "",
        "-preset", "",
        "-crf", "",
        "-threads", "",
        ""
    ]
    try:args[2] = Screen.CurrentVideo[0].replace("/","\\") # Get path of selected input video
    except AttributeError: # If user not selected input video
        Screen.ErrorNoFileSelected()
        return
    if Screen.radioButton.isChecked(): args[4] = Codec[0]  # If user choose "Default" codec
    else: args[4] = Codec[1]                                  # If user choose HEVC
    args[6] = Tune[Screen.comboBox_2.currentIndex()]       # Insert tune in command line
    args[8] = Speed[Screen.comboBox.currentIndex()]        # Insert speed preset in command line
    args[10] = str(Screen.spinBox.value())                 # Insert quality level in command line
    args[12] = str(24)
    args[13] = Screen.Save_File()[0].replace("/","\\")        # Open Save As window
    if args[11].replace(" ", "") == "":                       # If user didn't choose destination
        Screen.ErrorNoSave()
        return

    #Start conversion process
    Screen.setDisabled(True)    # Disable all widgets on main window
    done = False
    thread = Converter(args)
    Screen.threadpool.start(thread) # Start ffmpeg thread
    TimePass = 0
    while True: # Loop checking
        if not done: Screen.statusbar.showMessage(f"Converting {Screen.CurrentVideo[0]}. Passed seconds: {str(TimePass)}")  # If conversion proceeding
        else:                                                                                # If conversion ended
            Screen.statusbar.showMessage(f"Converted {Screen.CurrentVideo[0]}. Conversion took: {str(TimePass)} seconds")
            Screen.EndConversionDialog(TimePass)
            break
        Delay(1)
        TimePass +=1

class Converter(QRunnable): # FFMpeg Thread
    def __init__(self, args):
        super(Converter, self).__init__()
        self.args = args

    @pyqtSlot()
    def run(self):
        global done
        p = subprocess.Popen(args=args, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) # Start FFMpeg
        p.wait()
        done = True

class Settings(QtWidgets.QWidget):
    def __init__(self):
        super(Settings, self).__init__()
        uic.loadUi("design\\Settings.ui", self)
        self.UpdateValues()
        self.pushButton_2.clicked.connect(self.Apply)
        self.pushButton.clicked.connect(self.close)
        self.pushButton_3.clicked.connect(self.close)
        self.trans = QtCore.QTranslator(self)
        self.ChangeLanguage(LANGUAGE)
        self.toolButton.clicked.connect(lambda e: self.lineEdit.setText(self.choose1()))
        self.toolButton_2.clicked.connect(lambda e: self.lineEdit_2.setText(self.choose2()))

    def Apply(self):
        if LANGUAGE != Langs[self.comboBox.currentIndex()]:
            MainWindow.ChangeLanguage(Screen, Langs[self.comboBox.currentIndex()])
            self.ChangeLanguage(Langs[self.comboBox.currentIndex()])

        Config["PERFORMANCE"]["THREADS"] = str(self.horizontalSlider.value())
        Config["OTHER"]["HOME_FOLDER"] = self.lineEdit.text()
        Config["OTHER"]["SAVE_FOLDER"] = self.lineEdit_2.text()
        Config["OTHER"]["LANGUAGE"] = Langs[self.comboBox.currentIndex()]
        with open("Settings.ini", "w") as configfile:
            Config.write(configfile)
        self.close()

    def UpdateValues(self):
        Config = configparser.ConfigParser()
        Config.read(".\\Settings.ini")

        THREADS = Config["PERFORMANCE"].getint("THREADS", os.cpu_count())
        HOME_FOLDER = Config["OTHER"].get("HOME_FOLDER", os.path.dirname(os.path.realpath(__file__)).replace("\\", "/"))
        SAVE_FOLDER = Config["OTHER"].get("SAVE_FOLDER", os.path.dirname(os.path.realpath(__file__)).replace("\\", "/"))
        LANGUAGE = Config["OTHER"].get("LANGUAGE", "EN")
        if LANGUAGE == "EN":
            LANGUAGE = 0
        elif LANGUAGE == "UA":
            LANGUAGE = 1

        self.horizontalSlider.setValue(int(THREADS))
        self.lineEdit.setText(HOME_FOLDER)
        self.lineEdit_2.setText(SAVE_FOLDER)
        self.comboBox.setCurrentIndex(LANGUAGE)

    def ChangeLanguage(self, lang):
        if lang == "UA":
            self.trans.load(config["Localization"]["UA"])
            QtWidgets.QApplication.installTranslator(self.trans)
        else:
            QtWidgets.QApplication.removeTranslator(self.trans)
        self.retranslateUi()

    def choose1(self): return QFileDialog.getExistingDirectory(self, "Open Default Video Directory",HOME_FOLDER)
    def choose2(self): return QFileDialog.getExistingDirectory(self, "Open Default Save Directory",SAVE_FOLDER)
    def retranslateUi(self):
        _translate = QtCore.QCoreApplication.translate
        self.setWindowTitle(_translate("Form", "Settings"))
        self.groupBox.setTitle(_translate("Form", "Performance"))
        self.label.setToolTip(_translate("Form", "
Count of CPUs used for conversion
"))
        self.label.setText(_translate("Form", "CPU count"))
        self.groupBox_2.setTitle(_translate("Form", "Other"))
        self.label_2.setToolTip(_translate("Form", "
Language of programm
"))
        self.label_2.setText(_translate("Form", "Language"))
        self.comboBox.setItemText(0, _translate("Form", "English"))
        self.comboBox.setItemText(1, _translate("Form", "Ukrainian"))
        self.toolButton.setToolTip(_translate("Form", "
Browse
"))
        self.toolButton.setText(_translate("Form", "..."))
        self.label_3.setToolTip(_translate("Form", "
Default path of videos
"))
        self.label_3.setText(_translate("Form", "Input Video Folder"))
        self.label_4.setToolTip(_translate("Form", "
Default path to save converted video
"))
        self.label_4.setText(_translate("Form", "Output Video Folder"))
        self.toolButton_2.setToolTip(_translate("Form", "
Browse
"))
        self.toolButton_2.setText(_translate("Form", "..."))
        self.pushButton.setText(_translate("Form", "Cancel"))
        self.pushButton_2.setToolTip(_translate("Form", "
Apply changes
"))
        self.pushButton_2.setText(_translate("Form", "Apply"))
        self.pushButton_3.setText(_translate("Form", "Ok"))

class MainWindow(QtWidgets.QMainWindow): #Main Window
    def __init__(self, parent=None):
        super(MainWindow, self).__init__(parent)
        uic.loadUi("design\\MainWindow.ui", self)
        self.pushButton.clicked.connect(ConvertPrep) # "Convert" button clicked
        self.actionOpen.triggered.connect(self.OpenFile) # "Open" menu button clicked
        self.threadpool = QThreadPool() #Create Thread pool
        self.actionSettings.triggered.connect(self.ShowSettings)
        self.trans = QtCore.QTranslator(self)
        self.ChangeLanguage(LANGUAGE)
        check_for_updates()
    def ShowSettings(self):
        LoadCfg()
        self.settings = Settings()
        self.settings.show()

    def EndConversionDialog(self, t): #End Conversion Dialog
        LoadCfg()
        Screen.setDisabled(False)
        msgBox = QMessageBox()
        msgBox.setIcon(QMessageBox.Icon.Information)
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap("Vid.ico"), QtGui.QIcon.Mode.Normal, QtGui.QIcon.State.Off)
        msgBox.setWindowIcon(icon)
        msgBox.setText(f"File {self.CurrentVideo[0]} was converted successfully!\nThe conversion took {t} seconds\nDo you want to open output folder?")
        msgBox.setWindowTitle("Conversion ended")
        msgBox.setStandardButtons(QMessageBox.StandardButton.Cancel | QMessageBox.StandardButton.Open)
        button = hex(msgBox.exec())
        f = self.SaveFile[0].replace("/", "\\").split("\\")
        f.pop()
        if button == "0x2000":
            os.system('explorer "{0}"'.format('\\'.join(f))) #Open Explorer where video saved
    def Save_File(self):
        LoadCfg()
        self.SaveFile = QFileDialog.getSaveFileName(self, "Save File As",SAVE_FOLDER, "Video Files (*.mp4 *.mov *.mkv)")
        return self.SaveFile
    def OpenFile(self):
        LoadCfg()
        self.CurrentVideo = QFileDialog.getOpenFileName(self, "Open File",HOME_FOLDER, "Video Files (*.mp4 *.mov *.mkv)")

    def ChangeLanguage(self, lang):
        if lang == "UA":
            self.trans.load(config["Localization"]["UA"])
            QtWidgets.QApplication.instance().installTranslator(self.trans)
        else: QtWidgets.QApplication.removeTranslator(self.trans)
        self.retranslateUi()

    def ErrorNoFileSelected(self):
        LoadCfg()
        msgBox = QMessageBox()
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap("Vid.ico"), QtGui.QIcon.Mode.Normal, QtGui.QIcon.State.Off)
        msgBox.setWindowIcon(icon)
        msgBox.setIcon(QMessageBox.Icon.Critical)
        msgBox.setText("Error! You don't selected any video to convert!")
        msgBox.setWindowTitle("Error! No Video Selected!")
        msgBox.setStandardButtons(QMessageBox.StandardButton.Cancel)
        msgBox.exec()

    def ErrorNoSave(self):
        LoadCfg()
        msgBox = QMessageBox()
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap("Vid.ico"), QtGui.QIcon.Mode.Normal, QtGui.QIcon.State.Off)
        msgBox.setWindowIcon(icon)
        msgBox.setIcon(QMessageBox.Icon.Critical)
        msgBox.setText("Dialog","Error! You must select destination and name of output video to conversion!")
        msgBox.setWindowTitle("Error! No Save Destination Selected!")
        msgBox.setStandardButtons(QMessageBox.StandardButton.Cancel)
        msgBox.exec()

    def Ask_for_update(self, ver):
        msgBox = QMessageBox()
        msgBox.setIcon(QMessageBox.Icon.Information)
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap("Vid.ico"), QtGui.QIcon.Mode.Normal, QtGui.QIcon.State.Off)
        msgBox.setWindowIcon(icon)
        msgBox.setText(f"New version available!\n{ver}\nInstall now?")
        msgBox.setWindowTitle("Update")
        msgBox.setStandardButtons(QMessageBox.StandardButton.Ok | QMessageBox.StandardButton.Cancel)

        returnValue = msgBox.exec()
        if returnValue == QMessageBox.StandardButton.Ok:
            DETACHED_PROCESS = 0x00000008
            subprocess.Popen(["Update.exe", f"{ver}", "-backup"], creationflags=DETACHED_PROCESS, close_fds=True, shell=True)
            exit()

    def retranslateUi(self):
        self.setWindowTitle(QtCore.QCoreApplication.translate("MainWindow", "Ultimate Video Converter"))
        self.groupBox_2.setTitle(QtCore.QCoreApplication.translate("MainWindow", "Video Quality"))
        self.spinBox.setToolTip(QtCore.QCoreApplication.translate("MainWindow", "
From 0 to 51.

(0 - best quality, 51 - the worst quality)
"))
        self.label.setText(QtCore.QCoreApplication.translate("MainWindow", "Ultimate Video Converter"))
        self.pushButton.setToolTip(QtCore.QCoreApplication.translate("MainWindow", "
Start Video Conversion
"))
        self.pushButton.setText(QtCore.QCoreApplication.translate("MainWindow", "Convert"))
        self.groupBox_4.setTitle(QtCore.QCoreApplication.translate("MainWindow", "Conversion Speed"))
        self.comboBox.setToolTip(QtCore.QCoreApplication.translate("MainWindow", "
The slower the conversion

the better the quality
"))
        self.comboBox.setPlaceholderText(QtCore.QCoreApplication.translate("MainWindow", "None"))
        self.comboBox.setItemText(0, QtCore.QCoreApplication.translate("MainWindow", "Ultra Fast"))
        self.comboBox.setItemText(1, QtCore.QCoreApplication.translate("MainWindow", "Super Fast"))
        self.comboBox.setItemText(2, QtCore.QCoreApplication.translate("MainWindow", "Very Fast"))
        self.comboBox.setItemText(3, QtCore.QCoreApplication.translate("MainWindow", "Faster"))
        self.comboBox.setItemText(4, QtCore.QCoreApplication.translate("MainWindow", "Fast"))
        self.comboBox.setItemText(5, QtCore.QCoreApplication.translate("MainWindow", "Medium"))
        self.comboBox.setItemText(6, QtCore.QCoreApplication.translate("MainWindow", "Slow"))
        self.comboBox.setItemText(7, QtCore.QCoreApplication.translate("MainWindow", "Slower"))
        self.comboBox.setItemText(8, QtCore.QCoreApplication.translate("MainWindow", "Very Slow"))
        self.groupBox_3.setTitle(QtCore.QCoreApplication.translate("MainWindow", "Video Tune"))
        self.comboBox_2.setToolTip(QtCore.QCoreApplication.translate("MainWindow", "
Choose Video Tune
"))
        self.comboBox_2.setPlaceholderText(QtCore.QCoreApplication.translate("MainWindow", "None"))
        self.comboBox_2.setItemText(0, QtCore.QCoreApplication.translate("MainWindow", "Film"))
        self.comboBox_2.setItemText(1, QtCore.QCoreApplication.translate("MainWindow", "Animation"))
        self.comboBox_2.setItemText(2, QtCore.QCoreApplication.translate("MainWindow", "Grain"))
        self.comboBox_2.setItemText(3, QtCore.QCoreApplication.translate("MainWindow", "Still image"))
        self.comboBox_2.setItemText(4, QtCore.QCoreApplication.translate("MainWindow", "Fast decode"))
        self.comboBox_2.setItemText(5, QtCore.QCoreApplication.translate("MainWindow", "Zero Latency"))
        self.groupBox.setTitle(QtCore.QCoreApplication.translate("MainWindow", "Encoding Codec"))
        self.radioButton.setToolTip(QtCore.QCoreApplication.translate("MainWindow", "
H264
"))
        self.radioButton.setText(QtCore.QCoreApplication.translate("MainWindow", "Default"))
        self.radioButton_2.setToolTip(QtCore.QCoreApplication.translate("MainWindow", "
H265
"))
        self.radioButton_2.setText(QtCore.QCoreApplication.translate("MainWindow", "HEVC"))
        self.menuChoose_file.setTitle(QtCore.QCoreApplication.translate("MainWindow", "File"))
        self.menuSettings.setTitle(QtCore.QCoreApplication.translate("MainWindow", "Settings"))
        self.menuHelp.setTitle(QtCore.QCoreApplication.translate("MainWindow", "Help"))
        self.actionOpen.setText(QtCore.QCoreApplication.translate("MainWindow", "Open"))
        self.actionOpen.setToolTip(QtCore.QCoreApplication.translate("MainWindow", "Load new file "))
        self.actionHelp.setText(QtCore.QCoreApplication.translate("MainWindow", "Help"))
        self.actionSettings.setText(QtCore.QCoreApplication.translate("MainWindow", "Settings"))

if __name__ == '__main__':
    app = QtWidgets.QApplication(sys.argv)
    Screen = MainWindow()
    Screen.show()
    sys.exit(app.exec())


from krita import *
import os
import pathlib

dirpath = pathlib.Path(__file__).parent.absolute()
required_layers = ['Line', 'Shadow', 'Fill']
files = ['player', 'wall', 'ground', 'slippery', 'target', 'target2', 'cube', 'furn', 'ground1', 'ground2', 'ground3', 'ground4', 'ice1', 'ice2', 'ice3']
files = ['ground4', 'ice1', 'ice2', 'ice3']
'''

def all_layers_present(doc):
    layers = []
    nodes = doc.topLevelNodes()
    for node in nodes:
        layers.append(node.name())
    for rl in required_layers:
        if rl not in layers:
            return False
    return True

def set_other_layers_transparent(doc, layer):
    print('setting transparetn')
    nodes = doc.topLevelNodes()
    for node in nodes:
        while node.visible():
            node.setVisible(False)
            node.setOpacity(255)
    node = doc.nodeByName(layer)
    while not node.visible():
        node.setVisible(True)
    print(f'{node.name()} visibility is {node.visible()}')

def save_all_layers(doc, name):
    if not all_layers_present(doc):
        print(f'{f} does not have all the required layers.')
        return
    doc.setBatchmode(True)
    print("set batchmode")
    for rl in required_layers:
        fname = os.path.join(dirpath, f'{name}_{rl}.png')
        if os.path.exists(fname):
            os.remove(fname)
        set_other_layers_transparent(doc, rl)
        doc.save()
        doc.refreshProjection()
        while not os.path.exists(fname):
            doc.exportImage(fname, InfoObject())
        print(f"exported image {fname}")

def main():
    for f in files:
        print('\n')
        print("get krita")
        doc = Krita.instance().openDocument(os.path.join(dirpath, f'{f}.kra'))
        print("open doc")
        Krita.instance().activeWindow().addView(doc)
        # Krita.instance().action('export_layers').trigger()
        print("set active doc")
        save_all_layers(doc, f)
        doc.close()
'''

export_all = False
export_slippery=False
export_player = False
export_main_menu = False
export_cube = False
export_wall = False
export_furn = False
export_target = True

if export_all:
    objects = ['player', 'cube', 'furn', 'ground', 'slippery', 'cold', 'hot', 'wall']
    for obj in objects:
        print(obj)
        doc = Krita.instance().openDocument(os.path.join(dirpath, f'{obj}.kra'))
        doc.setBatchmode(True)
        background = doc.nodeByName('Background')
        if background:
            background.remove()
            doc.save()
        doc.exportImage(os.path.join(dirpath, f'{obj}.png'), InfoObject())

if export_slippery:
    doc = Krita.instance().openDocument(os.path.join(dirpath, 'slippery.kra'))
    doc.setBatchmode(True)
    current_frame = doc.fullClipRangeStartTime()
    last_frame = doc.fullClipRangeEndTime()
    while current_frame <= last_frame:
        print(f'current_frame = {current_frame}')
        doc.setCurrentTime(current_frame)
        doc.exportImage(os.path.join(dirpath, f'slippery_{current_frame}.png'), InfoObject())
        current_frame += 1

if export_player:
    doc = Krita.instance().openDocument(os.path.join(dirpath, 'player3.kra'))
    doc.setBatchmode(True)
    current_frame = doc.fullClipRangeStartTime()
    last_frame = doc.fullClipRangeEndTime()
    while current_frame <= last_frame:
        print(f'current_frame = {current_frame}')
        doc.setCurrentTime(current_frame)
        fname = os.path.join(dirpath, f'anim_{current_frame}_Fill.png')
        doc.exportImage(fname, InfoObject())
        if os.path.exists(fname):
            current_frame += 1

if export_cube:
    doc = Krita.instance().openDocument(os.path.join(dirpath, 'cube.kra'))
    doc.setBatchmode(True)
    current_frame = doc.fullClipRangeStartTime()
    last_frame = doc.fullClipRangeEndTime()
    while current_frame <= last_frame:
        print(f'current_frame = {current_frame}')
        doc.setCurrentTime(current_frame)
        fname = os.path.join(dirpath, f'cube_{current_frame}_Fill.png')
        doc.exportImage(fname, InfoObject())
        if os.path.exists(fname):
            current_frame += 1

if export_wall:
    doc = Krita.instance().openDocument(os.path.join(dirpath, 'wall3.kra'))
    doc.setBatchmode(True)
    current_frame = doc.fullClipRangeStartTime()
    last_frame = doc.fullClipRangeEndTime()
    while current_frame <= last_frame:
        print(f'current_frame = {current_frame}')
        doc.setCurrentTime(current_frame)
        fname = os.path.join(dirpath, f'wall_{current_frame}_Fill.png')
        doc.exportImage(fname, InfoObject())
        if os.path.exists(fname):
            current_frame += 1

if export_furn:
    doc = Krita.instance().openDocument(os.path.join(dirpath, 'furn2.kra'))
    doc.setBatchmode(True)
    current_frame = doc.fullClipRangeStartTime()
    last_frame = doc.fullClipRangeEndTime()
    while current_frame <= last_frame:
        print(f'current_frame = {current_frame}')
        doc.setCurrentTime(current_frame)
        fname = os.path.join(dirpath, f'furn_{current_frame}_Fill.png')
        doc.exportImage(fname, InfoObject())
        if os.path.exists(fname):
            current_frame += 1

if export_target:
    doc = Krita.instance().openDocument(os.path.join(dirpath, 'target.kra'))
    doc.setBatchmode(True)
    current_frame = doc.fullClipRangeStartTime()
    last_frame = doc.fullClipRangeEndTime()
    while current_frame <= last_frame:
        print(f'current_frame = {current_frame}')
        doc.setCurrentTime(current_frame)
        fname = os.path.join(dirpath, f'target_{current_frame}_Fill.png')
        doc.exportImage(fname, InfoObject())
        if os.path.exists(fname):
            current_frame += 1

if export_main_menu:
    doc = Krita.instance().openDocument(os.path.join(dirpath, 'main_menu_anim.kra'))
    doc.setBatchmode(True)
    current_frame = doc.fullClipRangeStartTime()
    last_frame = doc.fullClipRangeEndTime()
    while current_frame <= last_frame:
        print(f'current_frame = {current_frame}')
        doc.setCurrentTime(current_frame)
        fname = os.path.join(dirpath, f'shadow_{current_frame}.png')
        doc.exportImage(fname, InfoObject())
        if os.path.exists(fname):
            current_frame += 1


import torch.nn as nn
import torch
import torch.nn.functional as F

class Classifier(nn.Module):

    def __init__(self, model, im_size, num_classes):
        super(Classifier, self).__init__()
        self.model = model
        sample_inp = torch.randn((1,3, im_size, im_size)).cuda()
        _, x = model(sample_inp)
        b_sz = x.shape[0]
        in_channel = x.view(b_sz, -1).shape[1]

        self.linear = nn.Linear(in_channel, num_classes)

    def forward(self, x):
        _, x = self.model(x)
        x = self.linear(x)
        return F.log_softmax(x, dim=1)


import argparse
import time
import numpy as np

import brainflow
from brainflow.board_shim import BoardShim, BrainFlowInputParams
from brainflow.data_filter import DataFilter, FilterTypes, AggOperations, DetrendOperations, WindowFunctions
from phue import Bridge

# Adam Luchjenbroers https://stackoverflow.com/questions/1969240/mapping-a-range-of-values-to-another
def translate(value, leftMin, leftMax, rightMin, rightMax):
    # Figure out how 'wide' each range is
    leftSpan = leftMax - leftMin
    rightSpan = rightMax - rightMin

    # Convert the left range into a 0-1 range (float)
    valueScaled = float(value - leftMin) / float(leftSpan)

    # Convert the 0-1 range into a value in the right range.
    return rightMin + (valueScaled * rightSpan)

def main(board, args):

    b = Bridge('192.168.1.9')

    b.connect()

    board_descr = board.get_board_descr(args.board_id)
    sampling_rate = int(board_descr['sampling_rate'])
    nfft = DataFilter.get_nearest_power_of_two(sampling_rate)
    board.start_stream()

    for _ in range(20):
        # board.start_stream () # use this for default options
        time.sleep(2)
        # data = board.get_current_board_data (256) # get latest 256 packages or less, doesnt remove them from internal buffer
        data = board.get_board_data()  # get all data and remove it from internal buffer

        eeg_channels = board_descr['eeg_channels']
        # second eeg channel of synthetic board is a sine wave at 10Hz, should see huge alpha
        eeg_channel = eeg_channels[1]
        # optional detrend
        DataFilter.detrend(data[eeg_channel], DetrendOperations.LINEAR.value)
        psd = DataFilter.get_psd_welch(data[eeg_channel], nfft, nfft // 2, sampling_rate,
                                    WindowFunctions.BLACKMAN_HARRIS.value)

        band_power_theta = DataFilter.get_band_power(psd, 3.0, 6.0)
        band_power_alpha = DataFilter.get_band_power(psd, 7.0, 13.0)
        band_power_beta = DataFilter.get_band_power(psd, 14.0, 30.0)
        print("theta/alpha/beta:%f", band_power_theta, band_power_alpha , band_power_beta)

        value = band_power_theta + band_power_alpha + band_power_beta
        print('1',value)
        value = int(translate(value, 0, 200, 0, 65535))
        print('2',value)
        b.set_light(4, 'hue', value)

    board.stop_stream()

def setup():
    BoardShim.enable_dev_board_logger()

    parser = argparse.ArgumentParser()
    # use docs to check which parameters are required for specific board, e.g. for Cyton - set serial port
    parser.add_argument('--timeout', type=int, help='timeout for device discovery or connection', required=False,
                        default=0)
    parser.add_argument('--ip-port', type=int, help='ip port', required=False, default=0)
    parser.add_argument('--ip-protocol', type=int, help='ip protocol, check IpProtocolType enum', required=False,
                        default=0)
    parser.add_argument('--ip-address', type=str, help='ip address', required=False, default='')
    parser.add_argument('--serial-port', type=str, help='serial port', required=False, default='')
    parser.add_argument('--mac-address', type=str, help='mac address', required=False, default='')
    parser.add_argument('--other-info', type=str, help='other info', required=False, default='')
    parser.add_argument('--streamer-params', type=str, help='streamer params', required=False, default='')
    parser.add_argument('--serial-number', type=str, help='serial number', required=False, default='')
    parser.add_argument('--board-id', type=int, help='board id, check docs to get a list of supported boards',
                        required=True)
    parser.add_argument('--file', type=str, help='file', required=False, default='')
    args = parser.parse_args()

    params = BrainFlowInputParams()
    params.ip_port = args.ip_port
    params.serial_port = args.serial_port
    params.mac_address = args.mac_address
    params.other_info = args.other_info
    params.serial_number = args.serial_number
    params.ip_address = args.ip_address
    params.ip_protocol = args.ip_protocol
    params.timeout = args.timeout
    params.file = args.file

    board = BoardShim(args.board_id, params)
    board.prepare_session()

    main(board, args)

    board.release_session()

if __name__ == "__main__":
    setup()


import util
from collections import defaultdict
from itertools import pairwise

grid = defaultdict(int)

lines = util.read_lines("inputs/day14.txt")

max_rock_y = 0
for line in lines:
    ints = util.extract_ints(line)
    coords = [(ints[i], ints[i+1]) for i in range(0, len(ints), 2)]
    for a, b in pairwise(coords):
        from_x, to_x = min(a[0], b[0]), max(a[0], b[0])
        from_y, to_y = min(a[1], b[1]), max(a[1], b[1])
        max_rock_y = max(max_rock_y, to_y)
        for x in range(from_x, to_x+1):
            for y in range(from_y, to_y+1):
                grid[(x, y)] = 1

ans = 0
sand_pos = (500, 0)
while True:
    if grid[(500, 0)] == 1:
        break

    if sand_pos[1] < max_rock_y + 1:
        test_pos = (sand_pos[0], sand_pos[1] + 1)
        if grid[test_pos] == 0:
            sand_pos = test_pos
            continue
        test_pos = (sand_pos[0] - 1, sand_pos[1] + 1)
        if grid[test_pos] == 0:
            sand_pos = test_pos
            continue
        test_pos = (sand_pos[0] + 1, sand_pos[1] + 1)
        if grid[test_pos] == 0:
            sand_pos = test_pos
            continue

    grid[sand_pos] = 1
    ans += 1
    sand_pos = (500, 0)

print(ans)


# source activate py35_qt4
# conda create -n py35_qt4 python=3.5 pyqt=4 anaconda
# conda install -c menpo mayavi=4.5.0
# conda install basemap

import pdb
import h5py
import numpy as np
import matplotlib
matplotlib.use('Qt4Agg')
#['GTK', 'GTKAgg', 'GTKCairo', 'MacOSX', 'Qt4Agg', 'Qt5Agg', 'TkAgg', 'WX', 'WXAgg', 'CocoaAgg', 'GTK3Cairo', 'GTK3Agg', 'WebAgg', 'nbAgg']
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
from mayavi import mlab

################################################################
# Parameters
path = '/Users/silver/Dropbox/phd/projects/87a/abs/data/B15.h5'

################################################################
# Load the data
dat = h5py.File(path, 'r')
keys = list(dat.keys())
print("Keys: %s" % keys)

########
# Read the geometry
rad = np.array(dat['radius'])
tht = np.array(dat['theta'])
phi = np.array(dat['phi'])

# Read data
vex = np.array(dat['vex'])
den = np.array(dat['den'])

# Read elements
ar36 = np.array(dat['ar36'])
c12  = np.array(dat['c12' ])
ca40 = np.array(dat['ca40'])
ca44 = np.array(dat['ca44'])
co56 = np.array(dat['co56'])
cr48 = np.array(dat['cr48'])
fe52 = np.array(dat['fe52'])
fe56 = np.array(dat['fe56'])
he4  = np.array(dat['he4' ])
mg24 = np.array(dat['mg24'])
nn   = np.array(dat['n'   ])
ne20 = np.array(dat['ne20'])
ni56 = np.array(dat['ni56'])
o16  = np.array(dat['o16' ])
pp   = np.array(dat['p'   ])
s32  = np.array(dat['s32' ])
sc44 = np.array(dat['sc44'])
si28 = np.array(dat['si28'])
ti44 = np.array(dat['ti44'])
x56  = np.array(dat['x56' ])

#for key in dat:
#    if key in ['phi', 'theta', 'radius']:
#        continue
#    print(np.sum(np.isnan(np.array(dat[key])[:,:,1199])), np.sum(np.isnan(np.array(dat[key])[:,:,:1199])))

plt.figure()
plt.loglog(rad[:-1], rad[:-1]**2*np.sum(den, axis=(0,1)))

#plt.figure()
#mm = Basemap(width=12000000,height=9000000,projection='hammer',
#            resolution='c',lat_1=45.,lat_2=55,lat_0=50,lon_0=-107.)
#mm.bluemarble()

plt.figure()
mm = Basemap(projection='hammer', lon_0 = 0, llcrnrlon=True, llcrnrlat=True)
plt_phi, plt_tht = np.rad2deg(phi), np.rad2deg(tht-np.pi/2)
plt_phi, plt_tht = np.meshgrid(plt_phi, plt_tht, indexing='ij')
plt_den = np.zeros((den.shape[0]+1, den.shape[1]+1))
plt_den[:-1,:-1] = np.sum(den[:,:,:-1], axis=2)
mm.pcolormesh(plt_phi, plt_tht, plt_den, latlon=True)

#plt.figure()
#sparse = 30
#tst_phi = phi[::sparse]
#tst_tht = tht[::sparse]
#mm = Basemap(projection='hammer', lon_0 = 0, llcrnrlon=True, llcrnrlat=True)
#plt_phi, plt_tht = np.meshgrid(np.rad2deg(tst_phi[:-1]), np.rad2deg(tst_tht[:-1]-np.pi/2), indexing='ij')
#mm.pcolormesh(plt_phi, plt_tht, np.sum(den[::sparse,::sparse,:-1], axis=2), latlon=True)

plt.show()

#  #from mayavi import mlab
#  #
#  #mlab.figure(bgcolor=(0, 0, 0), size=(400, 400))
#  #
#  #src = mlab.pipeline.scalar_field(den[:,:,:-1])
#  ## Our data is not equally spaced in all directions:
#  #src.spacing = [1, 1, 1.5]
#  #src.update_image_data = True
#  #
#  #
#  ## Extract some inner structures: the ventricles and the inter-hemisphere
#  ## fibers. We define a volume of interest (VOI) that restricts the
#  ## iso-surfaces to the inner of the brain. We do this with the ExtractGrid
#  ## filter.
#  #blur = mlab.pipeline.user_defined(src, filter='ImageGaussianSmooth')
#  #voi = mlab.pipeline.extract_grid(blur)
#  #voi.set(x_min=125, x_max=193, y_min=92, y_max=125, z_min=34, z_max=75)
#  #
#  #mlab.pipeline.iso_surface(voi, contours=[1610, 2480], colormap='Spectral')
#  #
#  ## Add two cut planes to show the raw MRI data. We use a threshold filter
#  ## to remove cut the planes outside the brain.
#  #thr = mlab.pipeline.threshold(src, low=1120)
#  #cut_plane = mlab.pipeline.scalar_cut_plane(thr,
#  #                                plane_orientation='y_axes',
#  #                                colormap='black-white',
#  #                                vmin=1400,
#  #                                vmax=2600)
#  #cut_plane.implicit_plane.origin = (136, 111.5, 82)
#  #cut_plane.implicit_plane.widget.enabled = False
#  #
#  #cut_plane2 = mlab.pipeline.scalar_cut_plane(thr,
#  #                                plane_orientation='z_axes',
#  #                                colormap='black-white',
#  #                                vmin=1400,
#  #                                vmax=2600)
#  #cut_plane2.implicit_plane.origin = (136, 111.5, 82)
#  #cut_plane2.implicit_plane.widget.enabled = False
#  #
#  ## Extract two views of the outside surface. We need to define VOIs in
#  ## order to leave out a cut in the head.
#  #voi2 = mlab.pipeline.extract_grid(src)
#  #voi2.set(y_min=112)
#  #outer = mlab.pipeline.iso_surface(voi2, contours=[1776, ],
#  #                                        color=(0.8, 0.7, 0.6))
#  #
#  #voi3 = mlab.pipeline.extract_grid(src)
#  #voi3.set(y_max=112, z_max=53)
#  #outer3 = mlab.pipeline.iso_surface(voi3, contours=[1776, ],
#  #                                         color=(0.8, 0.7, 0.6))
#  #
#  #
#  #mlab.view(-125, 54, 326, (145.5, 138, 66.5))
#  #mlab.roll(-175)
#  #
#  #mlab.show()

## ##from mayavi import mlab
## ##mlab.figure(1, bgcolor=(0.48, 0.48, 0.48), fgcolor=(0, 0, 0),
## ##               size=(400, 400))
## ##mlab.clf()
## ##
## #################################################################################
## ### Display points at city positions
## ##coords = np.array(coords)
## ### First we have to convert latitude/longitude information to 3D
## ### positioning.
## ##lat, long = coords.T * np.pi / 180
## ##x = np.cos(long) * np.cos(lat)
## ##y = np.cos(long) * np.sin(lat)
## ##z = np.sin(long)
## ##
## ##points = mlab.points3d(x, y, z,
## ##                     scale_mode='none',
## ##                     scale_factor=0.03,
## ##                     color=(0, 0, 1))
## ##
## #################################################################################
## ### Display connections between cities
## ##connections = np.array(connections)
## ### We add lines between the points that we have previously created by
## ### directly modifying the VTK dataset.
## ##points.mlab_source.dataset.lines = connections
## ##points.mlab_source.reset()
## ### To represent the lines, we use the surface module. Using a wireframe
## ### representation allows to control the line-width.
## ##mlab.pipeline.surface(points, color=(1, 1, 1),
## ##                              representation='wireframe',
## ##                              line_width=4,
## ##                              name='Connections')
## ##
## #################################################################################
## ### Display city names
## ##for city, index in cities.items():
## ##    label = mlab.text(x[index], y[index], city, z=z[index],
## ##                      width=0.016 * len(city), name=city)
## ##    label.property.shadow = True
## ##
## #################################################################################
## ### Display continents outline, using the VTK Builtin surface 'Earth'
## ##from mayavi.sources.builtin_surface import BuiltinSurface
## ##continents_src = BuiltinSurface(source='earth', name='Continents')
## ### The on_ratio of the Earth source controls the level of detail of the
## ### continents outline.
## ##continents_src.data_source.on_ratio = 2
## ##continents = mlab.pipeline.surface(continents_src, color=(0, 0, 0))
## ##
## #################################################################################
## ### Display a semi-transparent sphere, for the surface of the Earth
## ##
## ### We use a sphere Glyph, throught the points3d mlab function, rather than
## ### building the mesh ourselves, because it gives a better transparent
## ### rendering.
## ##sphere = mlab.points3d(0, 0, 0, scale_mode='none',
## ##                                scale_factor=2,
## ##                                color=(0.67, 0.77, 0.93),
## ##                                resolution=50,
## ##                                opacity=0.7,
## ##                                name='Earth')
## ##
## ### These parameters, as well as the color, where tweaked through the GUI,
## ### with the record mode to produce lines of code usable in a script.
## ##sphere.actor.property.specular = 0.45
## ##sphere.actor.property.specular_power = 5
## ### Backface culling is necessary for more a beautiful transparent
## ### rendering.
## ##sphere.actor.property.backface_culling = True
## ##
## #################################################################################
## ### Plot the equator and the tropiques
## ##theta = np.linspace(0, 2 * np.pi, 100)
## ##for angle in (- np.pi / 6, 0, np.pi / 6):
## ##    x = np.cos(theta) * np.cos(angle)
## ##    y = np.sin(theta) * np.cos(angle)
## ##    z = np.ones_like(theta) * np.sin(angle)
## ##
## ##    mlab.plot3d(x, y, z, color=(1, 1, 1),
## ##                        opacity=0.2, tube_radius=None)
## ##
## ##mlab.view(63.4, 73.8, 4, [-0.05, 0, 0])
## ##mlab.show()

# # #x, y, z = np.ogrid[-10:10:20j, -10:10:20j, -10:10:20j]
# # #s = np.sin(x*y*z)/(x*y*z)
# # #
# # #src = mlab.pipeline.scalar_field(s)
# # #mlab.pipeline.iso_surface(src, contours=[s.min()+0.1*s.ptp(), ], opacity=0.3)
# # #mlab.pipeline.iso_surface(src, contours=[s.max()-0.1*s.ptp(), ],)
# # #
# # #mlab.show()

####plt_den = np.log10(den[::4,::2,0:800:20])
####plt_tht = tht[:-1:2][np.newaxis, :, np.newaxis]-np.pi/2
####plt_phi = phi[:-1:4][:, np.newaxis, np.newaxis]
####plt_rad = np.log(rad[0:800:20][np.newaxis, np.newaxis, :])
####xx = plt_rad*np.cos(plt_tht) * np.cos(plt_phi)
####yy = plt_rad*np.cos(plt_tht) * np.sin(plt_phi)
####zz = plt_rad*np.sin(plt_tht)
####zz = np.repeat(zz, 45, axis=0)
####
####src = mlab.pipeline.scalar_scatter(xx, yy, zz, plt_den)
#####pts = mlab.pipeline.glyph(src, scale_mode='none', scale_factor=1)
####field = mlab.pipeline.delaunay3d(src)
####mlab.pipeline.iso_surface(field, contours=[plt_den.min()+0.1*plt_den.ptp(), ], opacity=0.3)
#####mlab.pipeline.iso_surface(field, contours=[plt_den.min()+0.3*plt_den.ptp(), ], opacity=0.3)
#####mlab.pipeline.iso_surface(field, contours=[plt_den.min()+0.5*plt_den.ptp(), ], opacity=0.3)
#####mlab.pipeline.iso_surface(field, contours=[plt_den.min()+0.7*plt_den.ptp(), ], opacity=0.3)
####mlab.pipeline.iso_surface(field, contours=[plt_den.min()+0.9*plt_den.ptp(), ], opacity=0.3)
####
####mlab.show()

#####x, y, z = np.random.random((3, 100))
#####data = x**2 + y**2 + z**2
#####src = mlab.pipeline.scalar_scatter(x, y, z, data)
#####pts = mlab.pipeline.glyph(src, scale_mode='none', scale_factor=.1)
#####field = mlab.pipeline.delaunay3d(src)


# Importing Modules
from pathlib import Path as pt
import sys

# Data analysis
import numpy as np
from scipy.optimize import curve_fit

from uncertainties import ufloat as uf

from felionlib.utils.FELion_constants import colors
from felionlib.utils.FELion_definitions import sendData, read_dat_file

def _2gaussian(x, amp1,cen1,sigma1, amp2,cen2,sigma2):

    return amp1*(np.exp((-1.0/2.0)*(((x-cen1)/sigma1)**2))) + \
            amp2*(np.exp((-1.0/2.0)*(((x-cen2)/sigma2)**2)))

def exp_fit(location, norm_method, start_wn, end_wn, output_filename, overwrite, fullfiles, amp1, amp2, cen1, cen2, sig1, sig2, tkplot=False):

    if location.name == "DATA": datfile_location = location.parent/"EXPORT"
    else: datfile_location = location/"EXPORT"
    readfile = f"{datfile_location}/{output_filename}.dat"
    wn, inten = read_dat_file(readfile, norm_method)

    if output_filename == "averaged": line_color = "black"
    else:
        index = fullfiles.index(output_filename)
        index = 2*index
        if index > len(colors):
            index = (index - len(colors)) - 1
            line_color = f"rgb{colors[index]}"
        else: line_color = f"rgb{colors[index]}"

    index = np.logical_and(wn > start_wn, wn < end_wn)
    wn = wn[index]
    inten = inten[index]

    _sig = "\u03C3"
    _del = "\u0394"

    data = wn, inten

    data =  np.take(data, data[1].argsort(), 1)

    if (amp1 == 0 or amp2 == 0): init = [data[1][-1], data[0][-1], 5, data[1][-5], data[0][-5], 5]
    else: init = [amp1,cen1,sig1, amp2,cen2,sig2]

    popt_2gauss, pcov_2gauss = curve_fit(_2gaussian, wn, inten, p0=init)
    perr_2gauss = np.sqrt(np.diag(pcov_2gauss))
    amp1, cen1, sig1, amp2, cen2, sig2 = popt_2gauss

    amp1_err, cen1_err, sig1_err = perr_2gauss[0:3]
    amp2_err, cen2_err, sig2_err = perr_2gauss[3:6]

    uamp1 = uf(amp1, amp1_err)
    uamp2 = uf(amp2, amp2_err)

    ucen1 = uf(cen1, cen1_err)
    ucen2 = uf(cen2, cen2_err)

    usig1 = uf(sig1, sig1_err)

    usig2 = uf(sig2, sig2_err)
    uFWHM1 = 2*np.sqrt(2*np.log(2))*usig1

    uFWHM2 = 2*np.sqrt(2*np.log(2))*usig2
    gauss_peak = _2gaussian(wn, *popt_2gauss)
    data = {
        "table": f"{ucen1:.2uP}, {uamp1:.2uP}, {usig1:.2uP}, {uFWHM1:.2uP}, {ucen2:.2uP}, {uamp2:.2uP}, {usig2:.2uP}, {uFWHM2:.2uP}",
        "peak": {"x":list(wn), "y":list(gauss_peak), "name":f"[{ucen1:.2uP}, {ucen2:.2uP}]", "mode": "lines", "line": {"color":line_color}},
        "for_weighted_error1":f"{cen1}, {cen1_err}, {amp1}, {amp1_err}, {uFWHM1.nominal_value}, {uFWHM1.std_dev}, {sig1}, {sig1_err}",
        "for_weighted_error2":f"{cen2}, {cen2_err}, {amp2}, {amp2_err}, {uFWHM2.nominal_value}, {uFWHM2.std_dev}, {sig2}, {sig2_err}",
        "annotations": [ {"x": cen1, "y": amp1, "xref": 'x', "yref": 'y', "text": f'{ucen1:.2uP}',
                "font":{"color":line_color}, "arrowcolor":line_color, "showarrow": True, "arrowhead": 2, "ax": -25, "ay": -40
            }, {"x": cen2, "y": amp2, "xref": 'x', "yref": 'y', "text": f'{ucen2:.2uP}',
                "font":{"color":line_color}, "arrowcolor":line_color, "showarrow": True, "arrowhead": 2, "ax": -25, "ay": -40
            }
        ]
    }

    filename = f"{output_filename}_{norm_method}.expfit"
    expfile = datfile_location/filename

    if not expfile.exists():
        with open(expfile, 'w+') as f:
            f.write(f"#Frequency\t#Freq_err\t#Sigma\t#Sigma_err\t#FWHM\t#FWHM_err\t#Amplitude\t#Amplitude_err\n")

    with open(expfile, ("a+", "w+")[overwrite]) as f:

        if overwrite: f.write(f"#Frequency\t#Freq_err\t#Sigma\t#Sigma_err\t#FWHM\t#FWHM_err\t#Amplitude\t#Amplitude_err\n")

        f.write(f"{cen1:.4f}\t{cen1_err:.4f}\t{sig1:.4f}\t{sig1_err:.4f}\t{uFWHM1.nominal_value:.4f}\t{uFWHM1.std_dev:.4f}\t{amp1:.4f}\t{amp1_err:.4f}\n")
        f.write(f"{cen2:.4f}\t{cen2_err:.4f}\t{sig2:.4f}\t{sig2_err:.4f}\t{uFWHM2.nominal_value:.4f}\t{uFWHM2.std_dev:.4f}\t{amp2:.4f}\t{amp2_err:.4f}\n")
    sendData(data)

if __name__ == "__main__":
    args = sys.argv[1:][0].split(",")
    print(f"Received args: {args}")

    fullfiles = [pt(i).stem for i in args[6:-6]]
    start_wn = float(args[-2])
    end_wn = float(args[-1])

    location = pt(args[-3])
    norm_method = args[-4]
    output_filename = args[-5]
    overwrite = args[-6]
    amp1, amp2, cen1, cen2, sig1, sig2 = [float(i) for i in args[:6]]

    if overwrite == "true": overwrite = True
    else: overwrite = False
    exp_fit(location, norm_method, start_wn, end_wn, output_filename, overwrite, fullfiles, amp1, amp2, cen1, cen2, sig1, sig2)


import datetime
import io
from django import http
from django.views import View
import qrcode
import json
import requests
import json
import locale
from urllib.parse import urljoin
from pprint import pprint

from django.http.response import HttpResponseNotFound, HttpResponseRedirect, JsonResponse
from django.urls import reverse_lazy
from django.shortcuts import render, redirect
from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt
from django.conf import settings
from django.contrib.auth import get_user_model
from django.db.models import F
from django.utils.decorators import method_decorator
from django.contrib.auth.mixins import UserPassesTestMixin
from django.core.exceptions import PermissionDenied
from core.models import PaymentKind

from kontorshund.settings import SWISH_PAYEEALIAS, SWISH_URL, SWISH_CERT, SWISH_ROOTCA, NGROK_URL

from common.prices import get_current_ad_price_as_int_and_string
from core.models import Advertisement, get_one_month_ahead_from_date_obj, get_one_month_ahead_from_today

User = get_user_model()

locale.setlocale(locale.LC_ALL,'sv_SE.UTF-8')

import logging
logger = logging.getLogger(__name__)

#################
# PAYMENT STATUS
#################

class CheckInitialPaymentStatus(View):
    """
    Support function for checking an initial payment status for an ad.
    """

    def post(self, request, pk):
        if request.user.is_authenticated:
            try:
                ad = Advertisement.objects.get(pk=pk)
            except Advertisement.DoesNotExist:
                logger.exception(f'User {request.user.pk} checked extended payment status for ad {pk}, which returned an doesnotexist error')
                return JsonResponse("Ad does not exist", status=404, safe=False)
            if request.user == ad.author:
                logger.debug(f'User {request.user.pk} checked initial payment status for ad {pk}')
                try:
                    ad = Advertisement.objects.get(pk=pk)
                except Advertisement.DoesNotExist:
                    logger.exception(f'User {request.user.pk} checked initial payment status for ad {pk}, which returned an doesnotexist error')
                    return JsonResponse("Ad does not exist", status=404, safe=False)

                if ad.has_initial_payment:
                    logger.debug(f'User {request.user.pk} checking initial payment status for ad {pk}, which returned True')
                    return JsonResponse("Payment is complete!", status=200, safe=False)
                else:
                    logger.debug(f'User {request.user.pk} checking initial payment status for ad {pk}, which returned False')
                    return JsonResponse("Payment is NOT complete", status=404, safe=False)
            else:
                logger.debug(f'User {request.user.pk} tried checking initial payment status for ad ({pk}) belonging to another user, redirecting to profile...')
                return redirect('profile')

        else:
            return redirect('account_login')

class CheckExtendedPaymentStatus(View):
    """
    Support function for checking an extended payment status for an ad.
    """

    def post(self, request, pk):
        if request.user.is_authenticated:
            try:
                ad = Advertisement.objects.get(pk=pk)
            except Advertisement.DoesNotExist:
                logger.exception(f'User {request.user.pk} checked extended payment status for ad {pk}, which returned an doesnotexist error')
                return JsonResponse("Ad does not exist", status=404, safe=False)
            if request.user == ad.author:
                logger.debug(f'User {request.user.pk} checked extended payment status for ad {pk}')

                if ad.has_extended_payment:
                    logger.debug(f'User {request.user.pk} checking initial payment status for ad {pk}, which returned True')
                    return JsonResponse("Payment is complete!", status=200, safe=False)
                else:
                    logger.debug(f'User {request.user.pk} checking initial payment status for ad {pk}, which returned False')
                    return JsonResponse("Payment is NOT complete", status=404, safe=False)
            else:
                logger.debug(f'User {request.user.pk} tried checking initial payment status for ad ({pk}) belonging to another user, redirecting to profile...')
                return redirect('profile')
        else:
            return redirect('account_login')

###################
# ANDROID SPECIFIC
###################

class AndroidSuccessPage(View):

    def get(self, request):
        logger.debug('Android success-page was requested and served')
        return render(request, 'core/payment/android_swish_success.html')

#############
# SWISH VIEWS
#############

class SwishCallback(View):
    """
    Callback view which recieves a POST request from Swish when a payment have been recieved.
    """

    @method_decorator(csrf_exempt)
    def dispatch(self, request, *args, **kwargs):
        return super(SwishCallback, self).dispatch(request, *args, **kwargs)

    def post(self, request):

        data=request.body
        data_dict = json.loads(data.decode("utf-8"))

        # Check if payment was successfull
        if data_dict['status'] == 'PAID':
            logger.debug('Swish callback view was called')

            ad_id = data_dict['payeePaymentReference']
            amount = data_dict['amount']
            date_paid_str = data_dict['datePaid']
            date_paid_obj = datetime.datetime.strptime(date_paid_str, '%Y-%m-%dT%H:%M:%S.%f%z')
            payment_reference = data_dict['paymentReference']
            payer_alias = data_dict['payerAlias']

            logger.debug(
                f'Swish callback view was called, ad id {ad_id} amount {amount} payment_reference {payment_reference} payer alias {payer_alias}'
            )
            try:
                ad_obj = Advertisement.objects.get(pk=ad_id)
            except Advertisement.DoesNotExist:
                logger.exception(f'Ad could not be found in Swish callback view, ad id {ad_id} amount {amount} payment_reference {payment_reference} payer alias {payer_alias}')
                return JsonResponse("Annonsen hittades ej", status=404, safe=False)

            # If payment is extended
            if ad_obj.has_initial_payment:

                payment_obj = ad_obj.create_payment(
                    payment_kind=PaymentKind.EXTENDED,
                    amount=amount,
                    payment_reference=payment_reference,
                    date_time_paid = date_paid_obj,
                    payer_alias = payer_alias
                    )

                logger.info(
                    f'Initial payment found - creating extended payment pk {payment_obj.pk}for when Swish callback view was requested, ad id {ad_obj.pk} amount {amount} payment_reference {payment_reference} payer alias {payer_alias}'
                )

                ad_obj.is_published = True
                ad_obj.is_deleted = False

                # Base case - add 30 day ahead from today
                new_deletion_date = get_one_month_ahead_from_today()

                # If deletion date is later than today, add 30 days ontop of that
                if get_one_month_ahead_from_date_obj(ad_obj.deletion_date) > new_deletion_date:
                    new_deletion_date = get_one_month_ahead_from_date_obj(ad_obj.deletion_date)

                logger.debug(
                    f'Setting deletion date for ad {ad_obj.pk} to {new_deletion_date} for when Swish callback view was requested, ad id {ad_obj.pk} amount {amount} payment_reference {payment_reference} payer alias {payer_alias}'
                )

                ad_obj.deletion_date = new_deletion_date
                ad_obj.save()

                return HttpResponse(status=200)

            # If payment is initial
            else:

                payment_obj = ad_obj.create_payment(
                    payment_kind=PaymentKind.INITIAL,
                    amount=amount,
                    payment_reference=payment_reference,
                    date_time_paid = date_paid_obj,
                    payer_alias = payer_alias
                    )

                ad_obj.is_published = True
                ad_obj.is_deleted = False
                ad_obj.save()

                logger.info(
                    f'Initial payment not found - creating initial payment pk {payment_obj.pk} for when Swish callback view was requested, ad id {ad_obj.pk} amount {amount} payment_reference {payment_reference} payer alias {payer_alias}'
                )
                return HttpResponse(status=200)

        else:
            error_code = data_dict['errorCode']
            error_message = data_dict['errorCode']

            if error_code != "TM01":
                logger.info(f'Problem creating payment: {error_code} {error_message}')
                return HttpResponse(status=200)

            return HttpResponse(status=200)

# For mobile payments
class GenerateSwishPaymentRequestToken(View):
    """
    Generate a PaymentRequestToken, used for users visiting from Mobile where they have the Swish application
    installed, when token have been generated, the Swish app opens at the users phone.
    """

    def post(self, request, pk):

        if request.user.is_authenticated:

            CURRENT_PRICE, CURRENT_PRICE_STRING = get_current_ad_price_as_int_and_string()

            PRICE_TO_PAY = CURRENT_PRICE

            try:
                ad_obj = Advertisement.objects.get(pk=pk)
            except Advertisement.DoesNotExist():
                logger.exception(f'Failed to find Advertisement with pk {pk} when User with pk {request.user.pk} was requesting swish Payment request token ')
                return HttpResponseNotFound("Annonsen kunde inte hittas")

            if ad_obj.has_initial_payment:
                PRICE_TO_PAY = CURRENT_PRICE

            ##########################
            # Enable for local testing
            #SWISH_CALLBACKURL = urljoin(NGROK_URL, "/swish/callback")
            ##########################

            url = request.build_absolute_uri('/')
            callback_path = f'swish/callback'

            ##################
            # Enable for prod
            SWISH_CALLBACKURL= f'{url}{callback_path}'
            ##################

            # Set-up variables for payment request
            payload = {
                "payeePaymentReference": pk,
                "callbackUrl": SWISH_CALLBACKURL,
                "payeeAlias": SWISH_PAYEEALIAS,
                "currency": "SEK",
                "amount": PRICE_TO_PAY,
                "message": f"Betalning för annons med ID {pk}"
            }

            resp = requests.post(urljoin(SWISH_URL, "v1/paymentrequests"), json=payload, cert=SWISH_CERT, verify=SWISH_ROOTCA, timeout=2)
            PaymentRequestToken = resp.headers['PaymentRequestToken']
            logger.info(f'Returning swish-payment-request-token to User with pk {request.user.pk}, for ad with pk {pk} for amount {PRICE_TO_PAY}')
            return JsonResponse({'token': PaymentRequestToken, 'callback_url': SWISH_CALLBACKURL}, status=201, safe=False)
        else:
            return redirect('account_login')

# Support function for below View
def get_qr_code(request, token):
    if request.user.is_authenticated:
        qr = qrcode.QRCode(
            version = 1,
            error_correction = qrcode.constants.ERROR_CORRECT_L,
            box_size = 5,
            border = 0,
        )

        qr.add_data("D" + token)
        qr.make(fit=True)
        qr_img = qr.make_image()

        bytesio_object = io.BytesIO()
        qr_img.save(bytesio_object)

        file_data = bytesio_object.getvalue()
        response = HttpResponse(content_type=f'image/png')
        response['Content-Disposition'] = f'inline; filename="qr-code-image"'
        response.write(file_data)
        return response
    else:
        return redirect('account_login')

class GenerateSwishPaymentQrCode(View):
    """
    Generate a QR code, for users visiting via a desktop computer for either providing initial or extended payment
    for a users ad.
    """

    def post(self, request, pk):

        if request.user.is_authenticated:
            CURRENT_PRICE, CURRENT_PRICE_STRING = get_current_ad_price_as_int_and_string()

            # Handle price depending on if it's an initial/extened payment
            PRICE_TO_PAY = CURRENT_PRICE

            try:
                ad_obj = Advertisement.objects.get(pk=pk)
            except Advertisement.DoesNotExist():
                logger.exception(f'Failed to find advertisement when user with pk {request.user.pk} requested payment QR code for ad with pk {pk}')
                return HttpResponseNotFound("Annonsen kunde inte hittas")

            if ad_obj.has_initial_payment:
                PRICE_TO_PAY = CURRENT_PRICE
            else:
                PRICE_TO_PAY = CURRENT_PRICE

            ##########################
            # Enable for local testing
            #SWISH_CALLBACKURL = urljoin(NGROK_URL, "/swish/callback")
            ##########################

            url = request.build_absolute_uri('/')
            callback_path = f'swish/callback'

            ##################
            # Enable for prod
            SWISH_CALLBACKURL= f'{url}{callback_path}'
            ###################

            # Set-up variables for payment request
            payload = {
                "payeePaymentReference": pk,
                "callbackUrl": SWISH_CALLBACKURL,
                "payeeAlias": SWISH_PAYEEALIAS,
                "currency": "SEK",
                "amount": PRICE_TO_PAY,
                "message": f"Betalning för annons med ID {pk}"
            }

            resp = requests.post(urljoin(SWISH_URL, "v1/paymentrequests"), json=payload, cert=SWISH_CERT, verify=SWISH_ROOTCA, timeout=2)
            PaymentRequestToken = resp.headers['PaymentRequestToken']

            qr_image_response = get_qr_code(request, PaymentRequestToken)
            logger.info(f'Returning swish-payment-QR-code to user with pk {request.user.pk}, for ad with pk {pk} for amount {PRICE_TO_PAY}')
            return qr_image_response
        else:
            return redirect('account_login')

###################
# PAYMENT TEMPLATES
###################

class PayForAdSwishTemplate(View):
    """
    Generate template for user for either providing intial or extened payment for their ad.
    """

    def get(self, request, pk):
        if request.user.is_authenticated:

            url = request.build_absolute_uri()
            CURRENT_PRICE, CURRENT_PRICE_STRING = get_current_ad_price_as_int_and_string()

            if 'swish-pay/initial' in url:

                try:
                    ad_obj = Advertisement.objects.get(pk=pk)
                except Advertisement.DoesNotExist():
                    logger.exception(f'Failed to return Advertisement which couldnt be found when user with PK {request.user-pk} requested initial swish payment template')
                    return HttpResponseNotFound("Annonsen kunde inte hittas")

                if ad_obj.has_initial_payment:
                    return HttpResponseRedirect(reverse_lazy('ad_detail', kwargs={'pk': pk}))

                ad_title = ad_obj.title

                url = request.build_absolute_uri('/')
                path = f'ads/{pk}'
                ad_path = f'{url}{path}'

                logger.info(f'Returning initial swish-payment template to user with pk {request.user.pk} for ad with pk {pk}')

                return render(
                    request,
                    'core/payment/swish_payment_template.html',
                    {
                        'pk': pk,
                        'title': ad_title,
                        'price': CURRENT_PRICE_STRING,
                        'ad_path': ad_path
                    }
                )

            if 'swish-pay/extend' in url:

                try:
                    ad_obj = Advertisement.objects.get(pk=pk)
                except Advertisement.DoesNotExist():
                    logger.exception(f'Failed to return Advertisement which couldnt be found when user with PK {request.user-pk} requested extended swish payment template')
                    return HttpResponseNotFound("Annonsen kunde inte hittas")

                if not ad_obj.has_initial_payment:
                    return HttpResponseRedirect(reverse_lazy('ad_detail', kwargs={'pk': pk}))

                ad_title = ad_obj.title

                url = request.build_absolute_uri('/')
                path = f'ads/{pk}'
                ad_path = f'{url}{path}'

                current_end_date = ad_obj.deletion_date
                new_end_date = current_end_date + datetime.timedelta(days=30)

                current_end_date_sv = current_end_date.strftime("%a, %d %b %Y")
                new_end_date_sv = new_end_date.strftime("%a, %d %b %Y")

                logger.info(f'Returning extended swish-payment template to user with pk {request.user.pk} for ad with pk {pk}')

                return render(
                    request,
                    'core/payment/swish_payment_template.html',
                    {
                        'pk': pk,
                        'title': ad_title,
                        'price': CURRENT_PRICE_STRING,
                        'ad_path': ad_path,
                        'current_end_date': current_end_date_sv,
                        'new_end_date': new_end_date_sv,
                    }
                )

        else:
            return redirect('account_login')

class PayForAdBg(View):
    """
    Generate template for user for paying with Bankgiro.
    """

    def get(self, request, pk):
        if request.user.is_authenticated:
            CURRENT_PRICE, CURRENT_PRICE_STRING = get_current_ad_price_as_int_and_string()

            # Handle price depending on if it's an initial/extened payment
            PRICE_TO_PAY = CURRENT_PRICE_STRING

            try:
                ad_obj = Advertisement.objects.get(pk=pk)
            except Advertisement.DoesNotExist():
                return HttpResponseNotFound("Annonsen kunde inte hittas")

            if ad_obj.has_initial_payment:
                PRICE_TO_PAY = CURRENT_PRICE_STRING

            url = request.build_absolute_uri('/')
            path = f'ads/{pk}'
            ad_path = f'{url}{path}'

            logger.info(f'Returning BG payment template to user with pk {request.user.pk} for ad with {pk}')
            return render(request, 'core/payment/bg_instructions.html', {'pk': pk, 'price': PRICE_TO_PAY, 'ad_path': ad_path})
        else:
            return redirect('account_login')


from django.http import HttpResponse
from django.http import Http404
from django.contrib.auth.models import User
from django.shortcuts import render, redirect, get_object_or_404
from .forms import NewTopicForm
from .models import Board, Topic, Post
from django import template
from django.contrib.auth.models import Group
from django.core.mail import BadHeaderError, EmailMessage
from django.core.mail import send_mail
from django.conf import settings

def new_topic(request, pk):
    board = get_object_or_404(Board, pk=pk)
    user = User.objects.first()  # TODO: get the currently logged in user
    if request.method == 'POST':
        form = NewTopicForm(request.POST)
        if form.is_valid():
            topic = form.save(commit=False)
            topic.board = board
            topic.starter = user
            topic.save()
            post = Post.objects.create(
                message=form.cleaned_data.get('message'),
                topic=topic,
                created_by=user
            )
            return redirect('board_topics', pk=board.pk)  # TODO: redirect to the created topic page
    else:
        form = NewTopicForm()
    return render(request, 'new_topic.html', {'board': board, 'form': form})

def home(request):
    board = Board.objects.all()
    return render(request, 'home.html', {'board': board})

def dashboard(request):
    boards = Board.objects.all()
    return render(request, 'dashboard.html', {'boards': boards})

def forum(request):
    boards = Board.objects.all()
    return render(request, 'forum.html', {'boards': boards})

def thank(request):
    boards = Board.objects.all()
    return render(request, 'thank.html', {'boards': boards})

def contact(request):
    #check for POST requests on load.
    request.method == 'POST'
    subject = request.POST.get('subject')
    message = request.POST.get('message')
    email = request.POST.get('email')

    if subject and message and email:
        try:
            send_mail(subject= subject,message= message,from_email=settings.DEFAULT_FROM_EMAIL,recipient_list = [email],fail_silently  = True,)
        except BadHeaderError:
            return HttpResponse('Invalid header found.')
        return redirect('thank')

    else:
        #loading contacts.html if no requests
        return render(request, 'contact.html')

def board_topics(request, pk):
    board = get_object_or_404(Board, pk=pk)
    return render(request, 'topics.html', {'board': board})

def new_topic(request, pk):
    board = get_object_or_404(Board, pk=pk)
    user = User.objects.first()  # TODO: get the currently logged in user
    if request.method == 'POST':
        form = NewTopicForm(request.POST)
        if form.is_valid():
            topic = form.save(commit=False)
            topic.board = board
            topic.starter = user
            topic.save()
            post = Post.objects.create(
                message=form.cleaned_data.get('message'),
                topic=topic,
                created_by=user
            )
            return redirect('board_topics', pk=board.pk)  # TODO: redirect to the created topic page
    else:
        form = NewTopicForm()
    return render(request, 'new_topic.html', {'board': board, 'form': form})

def orderdone(request):
    boards = Board.objects.all()
    return render(request, 'orderdone.html', {'boards': boards})

# def order(request):
#     #check for POST requests on load.
#     request.method == 'POST'
#     user = User.objects.first()
#     paquet = request.POST.get('paquet')
#     quantity = request.POST.get('quantity')
#     other = request.POST.get('other')
#     check = request.POST.get('check')

#     if paquet and quantity and check:
#         return redirect('order')

#     else:

#         return render(request, 'home.html')


from __future__ import absolute_import, division, print_function

import math
import os
import collections
import json
import numpy as np

import fire
import annotation
import io_util
import redis

from itertools import izip_longest

def fix_data_format_in_redis():
    """Convert from string key to list of floats, not used"""
    r_server = redis.StrictRedis(
        host='localhost', port=6379, db=2)

    # iterate a list in batches of size n
    def batcher(iterable, n):
        args = [iter(iterable)] * n
        return izip_longest(*args)

    # in batches of 500 delete keys matching user:*
    batch_idx = 0
    batch_size = 10000
    for keybatch in batcher(r_server.scan_iter('*'), 10000):
        print('{}'.format((batch_idx + 1) * batch_size))
        outputs = r_server.mget(keybatch)
        outputs = [json.loads(result_str) for result_str in outputs]
        r_server.delete(*keybatch)
        for idx in range(len(keybatch)):
            r_server.lpush(keybatch[idx], outputs[idx])

def _get_class_prediction_for_exp1(r_server, ids):
    return r_server.mget(ids)

def _get_class_prediction_for_exp2(r_server, ids):
    """For 2_more_test experiments, we store both the softmax values
    and the 1024 mobilenet feature vectors. This method retrieve
    class predictions.

    Args:
      r_server:
      ids:

    Returns:

    """
    batch_size = 10000
    total_num = len(ids)
    batch_num = int(math.ceil(total_num * 1.0 / batch_size))
    predictions = []
    for batch_idx in range(batch_num):
        print('[{}/{}]'.format(((batch_idx + 1) * batch_size), total_num))
        batch_ids = ids[batch_idx * batch_size:(batch_idx + 1) * batch_size]
        outputs = r_server.mget(batch_ids)
        outputs = [json.loads(result_str) for result_str in outputs]
        outputs = np.array(outputs)
        outputs = outputs[:, :2]
        predictions.extend(np.argmax(outputs, axis=1))
    return predictions

def print_stats(positive_dir,
                negative_dir,
                redis_db,
                get_prediction_fn=_get_class_prediction_for_exp2):
    r_server = redis.StrictRedis(
        host='localhost', port=6379, db=redis_db, socket_timeout=3600)
    class_label_map = {positive_dir: '1', negative_dir: '0'}
    tp, fn, tn, fp = 0, 0, 0, 0
    for class_dir in class_label_map.keys():
        print(class_dir)
        ids = [
            os.path.splitext(file_name)[0]
            for file_name in os.listdir(class_dir)
        ]
        predictions = get_prediction_fn(r_server, ids)
        assert all([prediction is not None for prediction in predictions])
        cnt = collections.Counter(predictions)
        print([(cls, cnt[cls],
                '{:.2f}%'.format(cnt[cls] / len(predictions) * 100.0))
               for cls in cnt])
        if class_dir == positive_dir:
            tp = cnt['1']
            fn = cnt['0']
        else:
            tn = cnt['0']
            fp = cnt['1']
    recall = tp / (tp + fn * 1.0)
    precision = tp / (tp + fp * 1.0)
    fpr = fp / (fp + tn * 1.0)
    tpr = recall
    print('recall: {:.2f}, precision: {:.2f}, fpr: {:.2f}, tpr: {:.2f}'.format(
        recall, precision, fpr, tpr))

def write_wrong_predictions_to_file(input_dir, label, output_file_path,
                                    redis_db):
    r_server = redis.StrictRedis(host='localhost', port=6379, db=redis_db)
    ids = [
        os.path.splitext(file_name)[0] for file_name in os.listdir(input_dir)
    ]
    predictions = r_server.mget(ids)
    assert all([prediction is not None for prediction in predictions])
    # find wrong ids
    ids_wrong = []
    for idx, prediction in enumerate(predictions):
        if prediction != label:
            ids_wrong.append('{}.jpg'.format(ids[idx]))
    io_util.write_list_to_file(ids_wrong, output_file_path)

def write_false_positives_and_false_negatives(positive_dir, negative_dir,
                                              redis_db):
    class_label_map = {
        positive_dir: ['1', 'false_negative.txt'],
        negative_dir: ['0', 'false_positive.txt']
    }
    for class_dir, [label, output_file_path] in class_label_map.iteritems():
        print(class_dir)
        write_wrong_predictions_to_file(class_dir, label, output_file_path,
                                        redis_db)

def _get_prediction_results(imageid, slice_annotations, tile_coords, redis_db):
    r_server = redis.StrictRedis(host='localhost', port=6379, db=redis_db)
    prediction_keys = [
        slice_annotations.get_tile_file_basename_without_ext(
            imageid, tile_coord) for tile_coord in tile_coords
    ]
    return r_server.mget(prediction_keys)

def print_metrics_on_videos(image_dir, annotation_dir, video_list_file_path,
                            redis_db):
    # # image dir should be just the sliced test images
    # dataset = 'stanford'
    # slice_annotations = (
    #     annotation.SliceAnnotationsFactory.get_annotations_for_slices(
    #         dataset, image_dir, annotation_dir))

    # with open(video_list_file_path, 'r') as f:
    #     video_names = f.read().splitlines()
    # videoids = [
    #     video_name.replace('_video.mov', '') for video_name in video_names
    # ]

    # # filter by video name
    # annotations = slice_annotations.annotations
    # annotations = annotations[annotations['videoid'].isin(videoids)]

    # # filter by label
    # mask = annotation.get_positive_annotation_mask(annotations)
    # target_annotations = annotations[mask].copy()

    # track_annotations_grp = target_annotations.groupby(['trackid'])

    annotations = io_util.load_stanford_campus_annotation(annotation_dir)
    if video_list_file_path:
        annotations = annotation.filter_annotation_by_video_name(annotations,
                                                       video_list_file_path)
    annotations = annotation.filter_annotation_by_label(annotations)
    track_annotations_grp = annotation.group_annotation_by_unique_track_ids(annotations)

    detected_tracks = {}
    all_tracks = []
    for track_id, track_annotations in track_annotations_grp:
        all_tracks.append(track_id)
        sorted_track_annotations = track_annotations.sort_values('frameid')
        start_frame = min(sorted_track_annotations['frameid'])
        end_frame = max(sorted_track_annotations['frameid'])
        videoid = list(set(sorted_track_annotations['videoid']))
        assert len(videoid) == 1
        videoid = videoid[0]
        print('video id: {}, track id: {}, start frame: {}, end frame: {}'.
              format(videoid, track_id, start_frame, end_frame))

        for index, row in sorted_track_annotations.iterrows():
            bx = (row['xmin'], row['ymin'], row['xmax'], row['ymax'])
            imageid = slice_annotations.get_imageid(row['videoid'],
                                                    row['frameid'])
            tile_coords = slice_annotations.get_tiles_contain_bounding_box(
                imageid, bx)
            predictions = _get_prediction_results(imageid, slice_annotations,
                                                  tile_coords, redis_db)
            # print('predictions for tiles {} are {}'.format(tile_coords,
            #                                                predictions))
            predictions_ints = [int(prediction) for prediction in predictions]
            # some positive is fired
            if sum(predictions_ints) != 0:
                print('first frame to be send: {}'.format(
                    start_frame, row['frameid']))
                detected_tracks[track_id] = row['frameid'] - start_frame
                break

    missed_tracks = set(all_tracks) - set(detected_tracks.keys())
    print('# missed tracks {}, percentage {:.2f}'.format(
        len(missed_tracks), len(missed_tracks) * 1.0 / len(all_tracks)))
    print('Average frame latency among detected tracks: {:.2f}({:.2f})'.format(
        np.mean(detected_tracks.values()), np.std(detected_tracks.values())))
    print(('Average time latency(s) among '
           'detected tracks (assuming 30FPS): {:.2f}({:.2f})').format(
               np.mean(detected_tracks.values()) / 30.0,
               np.std(detected_tracks.values()) / 30.0))

if __name__ == '__main__':
    fire.Fire()


import os
from bs4 import BeautifulSoup
import re
import csv

# HTML content (replace with your actual HTML content)

def clean_and_extract_dialogues(file_path):
    with open(file_path, "r", encoding="utf-8") as file:

        # Pass the filehandle to Beautiful Soup for parsing
        soup = BeautifulSoup(file, 'html.parser')
        character_dialogue_pairs = []
        formatted_output = []

    # Find all
 elements with class "npc"
    npc_elements = soup.find_all('div', class_='npc')

    # Iterate through the
 elements
    for div_element in npc_elements:
        # Extract the character name from the
 element
        character_name = div_element.get_text().strip()
        character_name = re.sub(r'\s+', ' ', character_name)

        # Find the  element with class "dialog" immediately following the

        next_span = div_element.find_next('span', class_='dialog')

        if next_span:
            # Extract and format the dialogue line
            dialogue_text = next_span.get_text().strip()
            dialogue_text = re.sub(r'
','',dialogue_text)
            dialogue_text = re.sub(r'\s+', ' ', dialogue_text)
            formatted_line = f"{character_name}: {dialogue_text}"
            if len(character_name) < 30:
                formatted_output.append(formatted_line)

    # Join the formatted lines with line breaks
    formatted_text = '\n'.join(formatted_output)
    return formatted_output

# Specify the root folder where your HTML files are located
root_folder = r"C:\Users\cemea\Documents\BG3_-_parsed_dialogue_1.0-20230915T195130Z-001\BG3-parsedDialogue\Dialogs"

# Create an empty list to store all transcripts
all_transcripts = []

# Recursively traverse the directory tree using os.walk
for root, _, files in os.walk(root_folder):
    for filename in files:
        if filename.endswith('.html'):
            file_path = os.path.join(root, filename)
            transcript = clean_and_extract_dialogues(file_path)
            all_transcripts.extend(transcript)

r"""
# Specify the output file path
output_file_path = 'output_transcript.txt'

# Write all transcripts to the output file
with open(r"C:\Users\cemea\Documents\BG3_-_parsed_dialogue_1.0-20230915T195130Z-001\BG3-parsedDialogue\output_text.txt", 'w', encoding='utf-8') as output_file:
    for transcript in all_transcripts:
   #     output_file.write(r"{transcript}\n00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvomb  ")
"""

# Specify the output CSV file path
output_csv_file = r'C:\Users\cemea\Documents\BG3_-_parsed_dialogue_1.0-20230915T195130Z-001\BG3-parsedDialogue\output_transcript.csv'

# Write all transcripts to the CSV file
with open(output_csv_file, 'w', newline='', encoding='utf-8') as csvfile:
    csvwriter = csv.writer(csvfile)
    for transcript in all_transcripts:
        character, dialogue = transcript.split(': ', 1)
        csvwriter.writerow([character, dialogue])


import os
import tensorflow as tf
import numpy as np
import argparse

SIGNAL_LENGTH = 2048
N_MELS = 80

class Denoiser:
    def __init__(self, model_path = 'denoising_model') -> None:
        self.model = tf.keras.models.load_model(model_path)

    def process(self, input_data):
        assert len(input_data.shape) == 2
        assert input_data.shape[1] == N_MELS
        if input_data.shape[0] > SIGNAL_LENGTH:
            input_arr = input_data[0: SIGNAL_LENGTH, :]
        elif input_data.shape[0] < SIGNAL_LENGTH:
            input_arr = np.zeros(shape=(SIGNAL_LENGTH, N_MELS))
            input_arr[0:input_data.shape[0], :] = input_data
        else:
            input_arr = input_data
        input_arr = np.expand_dims(input_arr, axis = 0)
        prediction = self.model(input_arr)[0].numpy()
        if input_data.shape[0] < input_arr.shape[1]:
            prediction = prediction[0: input_data.shape[0], :]
        return prediction

class Classificator:
    def __init__(self, model_path = 'classification') -> None:
        self.model = tf.keras.models.load_model(model_path)

    def process(self, input_data):
        assert len(input_data.shape) == 2
        assert input_data.shape[1] == N_MELS
        if input_data.shape[0] > SIGNAL_LENGTH:
            input_arr = input_data[0: SIGNAL_LENGTH, :]
        elif input_data.shape[0] < SIGNAL_LENGTH:
            input_arr = np.zeros(shape=(SIGNAL_LENGTH, N_MELS))
            input_arr[0:input_data.shape[0], :] = input_data
        else:
            input_arr = input_data
        input_arr = np.expand_dims(input_arr, axis = 0)
        prediction_arr = self.model(input_arr)[0]
        prediction = np.argmax(prediction_arr)
        if prediction == 0:
            pred_class = 'clear'
        else:
            pred_class = 'noise'
        confidence = prediction_arr[prediction]
        return pred_class, confidence

def get_args():
    parser = argparse.ArgumentParser(description='Description of your program')
    parser.add_argument('-m', '--mel_path', help='Path to .npy file with mel-spectrogram', required=True)
    parser.add_argument('-t', '--task', help = "Name of task: c - classification, d- denoising", default='d')
    parser.add_argument('-r', '--result_path', help = "Path to save result mel-spectrogram after denoising", default='.')
    args = parser.parse_args()
    return args

def load_data(path):
    try:
        data = np.load(path)
    except Exception as e:
        print(f"Can't load file {path}, error: {e}")
    return data

def save_data(data, data_path, path):
    fname = data_path.split(os.sep)[-1]
    fname_new = 'result_' + fname
    new_path = os.path.join(path, fname_new)
    try:
        with open(new_path, 'wb') as f:
            np.save(f, data)
        print(f"Data saved to {new_path}")
    except Exception as e:
        print(f"Can't save data to {path}, error: {e}")

if __name__ == "__main__":
    args = get_args()
    data = load_data(args.mel_path)
    if args.task == 'd':
        denoiser = Denoiser(model_path='denoising_model')
        denoized_data = denoiser.process(data)
        save_data(denoized_data, args.mel_path, args.result_path)
    if args.task == 'c':
        classificator = Classificator(model_path='classification')
        classification_results = classificator.process(data)
        print(f"Class: {classification_results[0]}, Confidence: {classification_results[1].numpy()[0]}")


from django.conf.urls import patterns, include, url

from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    (r"^admin/", include(admin.site.urls)),
    (r"^accounts/", include("userena.urls")),

    (r"^lists/", include("lists.urls", namespace="lists")),
    (r"", include("generic.urls", namespace="generic")),
)

from django.contrib.staticfiles.urls import staticfiles_urlpatterns
urlpatterns += staticfiles_urlpatterns()


"""Copy Artefact.source into track, store, series & repo

Revision ID: 38807948d269
Revises: cb121ad343dd
Create Date: 2023-10-25 12:33:03.206702+00:00

"""
import sqlalchemy as sa
from alembic import op

# revision identifiers, used by Alembic.
revision = "38807948d269"
down_revision = "cb121ad343dd"
branch_labels = None
depends_on = None

def upgrade() -> None:
    artefact_table = sa.table(
        "artefact",
        sa.column("id", sa.Integer()),
        sa.column("source", sa.JSON()),
        sa.column("track", sa.String()),
        sa.column("store", sa.String()),
        sa.column("series", sa.String()),
        sa.column("repo", sa.String()),
    )

    conn = op.get_bind()
    get_artefacts_id_source = sa.select(artefact_table.c.id, artefact_table.c.source)
    for id, source in conn.execute(get_artefacts_id_source):
        conn.execute(
            sa.update(artefact_table)
            .where(artefact_table.c.id == id)
            .values(
                track=source.get("track"),
                store=source.get("store"),
                series=source.get("series"),
                repo=source.get("repo"),
            )
        )

def downgrade() -> None:
    artefact_table = sa.table(
        "artefact",
        sa.column("track", sa.String()),
        sa.column("store", sa.String()),
        sa.column("series", sa.String()),
        sa.column("repo", sa.String()),
    )

    conn = op.get_bind()
    conn.execute(
        sa.update(artefact_table).values(
            track=None,
            store=None,
            series=None,
            repo=None,
        )
    )


from brl_gym.wrapper_envs.wrapper_maze import Expert, ExplicitBayesMazeEnv, simple_expert_actor
from brl_gym.envs.mujoco.maze10 import GOAL_POSE

import numpy as np
if __name__ == "__main__":

    expert = Expert(nenv=1, use_vf=True, maze_type=10)

    all_rewards = []
    completed = 0
    for i in range(500):
        env = ExplicitBayesMazeEnv(maze_type=10, difficulty='easy', entropy_weight=0.0)
        o = env.reset()
        idx = env.env.target
        print("target", idx)
        rewards = []
        for t in range(750):
            bel = o['zbel'].ravel()
            state = o['obs'][:2]
            idx = np.random.choice(np.arange(10), p=bel)

            # chosen_expert = expert.mps[idx]
            # action = simple_expert_actor(chosen_expert, state, GOAL_POSE[idx])
            # action = np.concatenate([action, np.array([0])])
            bel[:] = 0
            bel[idx] = 1.0
            action = expert.action(
                np.concatenate([o['obs'], bel]).reshape(1, -1)).ravel()

            action[2] = 0
            # action[2] = action[2] + np.random.normal()*0.1
            # action += np.random.normal(size=3)*0.1
            o, r, d, _ = env.step(action)
            # if r <= -50:
            #     r = 0 # temporarily remove the hardship
            # env.render()
            if t == 500:
                print(np.around(o['zbel'], 2))
            rewards += [r]
            if d:
                print(i, "complete")
                completed += 1
                break

        undiscounted_sum = np.sum(rewards)
        all_rewards += [undiscounted_sum]
        print('undiscounted sum', undiscounted_sum)

    print("stat", np.mean(all_rewards), np.std(all_rewards) / np.sqrt(len(all_rewards)))
    print("Completed", completed, "/", 500)
    # stat 512.4810662596318 14.01378592003006
    import IPython; IPython.embed();