'''MobileNetV2 in PyTorch.

See the paper "Inverted Residuals and Linear Bottlenecks:
Mobile Networks for Classification, Detection and Segmentation" for more details.
'''
import torch
import torch.nn as nn
import torch.nn.functional as F

from .deconv import *

class Block(nn.Module):
    '''expand + depthwise + pointwise'''
    def __init__(self, in_planes, out_planes, expansion, stride, deconv=None):
        super(Block, self).__init__()
        self.stride = stride

        planes = expansion * in_planes
        if deconv:
            self.deconv=True
            self.conv1 = deconv(in_planes, planes, kernel_size=1, stride=1, padding=0, bias=True)
            self.conv2 = deconv(planes, planes, kernel_size=3, stride=stride, padding=1, groups=planes, bias=True,n_iter=1)
            self.conv3 = deconv(planes, out_planes, kernel_size=1, stride=1, padding=0, bias=True,block=in_planes,n_iter=1)

            self.shortcut = nn.Sequential()
            if stride == 1 and in_planes != out_planes:
                self.shortcut = nn.Sequential(
                    deconv(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=True)
                )

        else:
            self.deconv=False
            self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, stride=1, padding=0, bias=False)
            self.bn1 = nn.BatchNorm2d(planes)
            self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, groups=planes, bias=False)
            self.bn2 = nn.BatchNorm2d(planes)
            self.conv3 = nn.Conv2d(planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False)
            self.bn3 = nn.BatchNorm2d(out_planes)

            self.shortcut = nn.Sequential()
            if stride == 1 and in_planes != out_planes:
                self.shortcut = nn.Sequential(
                    nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False),
                    nn.BatchNorm2d(out_planes),
                )

    def forward(self, x):
        if self.deconv:
            out = F.relu(self.conv1(x))
            out = F.relu(self.conv2(out))
            out = self.conv3(out)
        else:
            out = F.relu(self.bn1(self.conv1(x)))
            out = F.relu(self.bn2(self.conv2(out)))
            out = self.bn3(self.conv3(out))
        out = out + self.shortcut(x) if self.stride==1 else out
        return out

class MobileNetV2(nn.Module):
    # (expansion, out_planes, num_blocks, stride)
    cfg = [(1,  16, 1, 1),
           (6,  24, 2, 1),  # NOTE: change stride 2 -> 1 for CIFAR10
           (6,  32, 3, 2),
           (6,  64, 4, 2),
           (6,  96, 3, 1),
           (6, 160, 3, 2),
           (6, 320, 1, 1)]

    def __init__(self, num_classes=10, deconv=None,delinear=None,channel_deconv=None):
        super(MobileNetV2, self).__init__()
        # NOTE: change conv1 stride 2 -> 1 for CIFAR10
        if deconv:
            self.deconv=True
            self.conv1 = deconv(3, 32, kernel_size=3, stride=1, padding=1, bias=True,freeze=True,n_iter=10)
            self.conv2 = deconv(320, 1280, kernel_size=1, stride=1, padding=0, bias=True)

        else:
            self.conv1 = nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1, bias=False)
            self.bn1 = nn.BatchNorm2d(32)
            self.conv2 = nn.Conv2d(320, 1280, kernel_size=1, stride=1, padding=0, bias=False)
            self.bn2 = nn.BatchNorm2d(1280)

        self.layers = self._make_layers(in_planes=32, deconv=deconv)

        if delinear:
            self.linear = delinear(1280, num_classes)
        else:
            self.linear = nn.Linear(1280, num_classes)

        if channel_deconv:
            self.deconv1=channel_deconv()
    def _make_layers(self, in_planes, deconv=None):
        layers = []
        for expansion, out_planes, num_blocks, stride in self.cfg:
            strides = [stride] + [1]*(num_blocks-1)
            for stride in strides:
                layers.append(Block(in_planes, out_planes, expansion, stride, deconv=deconv))
                in_planes = out_planes
        return nn.Sequential(*layers)

    def forward(self, x):
        if hasattr(self,'bn1'):
            out = F.relu(self.bn1(self.conv1(x)))
        else:
            out = F.relu(self.conv1(x))

        out = self.layers(out)
        if hasattr(self,'bn2'):
            out = F.relu(self.bn2(self.conv2(out)))
        else:
            out = F.relu(self.conv2(out))
        # NOTE: change pooling kernel_size 7 -> 4 for CIFAR10
        if hasattr(self, 'deconv1'):
            out = self.deconv1(out)
        out = F.avg_pool2d(out, 4)
        out = out.view(out.size(0), -1)
        out = self.linear(out)
        return out

def test():
    net = MobileNetV2()
    x = torch.randn(2,3,32,32)
    y = net(x)
    print(y.size())

# test()


import logging
from typing import List, Dict

from explorerscript.ssb_converting.ssb_data_types import SsbOperation
from explorerscript.ssb_converting.ssb_special_ops import process_op_for_jump, SsbLabel
logger = logging.getLogger(__name__)

class OpsLabelJumpToResolver:
    def __init__(self, routines: list[list[SsbOperation]]):
        logger.debug("Constructing labels and jumps...")
        # A mapping of labels: {mem_location: label_name}
        self.labels: dict[int, SsbLabel] = {}
        self.routines = []
        for routine_id, rtn in enumerate(routines):
            new_rtn_ops = []
            self.routines.append(new_rtn_ops)
            for op in rtn:
                new_rtn_ops.append(process_op_for_jump(op, self.labels, routine_id))

    def __iter__(self):
        rtn_iterator = iter(self.routines)
        try:
            while True:
                # This will eventually raise StopIteration.
                yield list(self._iter_routine(next(rtn_iterator)))
        except StopIteration:
            return

    def _iter_routine(self, rtn):
        op_iterator = iter(rtn)
        try:
            while True:
                # This will eventually raise StopIteration.
                next_item = next(op_iterator)
                if next_item.offset in self.labels:
                    yield self.labels[next_item.offset]
                yield next_item
        except StopIteration:
            return


import time
import requests.auth

from .utils import check_connection

class API(object):
    def __init__(self, cfg_cls):
        self.cfg_cls = cfg_cls
        self.config = self.cfg_cls.config
        self.refresh_token = self.config.get('refresh_token') or False
        self.token_expires = self.config.get("token_expires") or False
        self.headers = self.cfg_cls.get_headers(self.config.get('access_token'))

    @check_connection
    def get_new_token(self):
        """
        Get new token using refresh token.
        """
        client_auth = requests.auth.HTTPBasicAuth(self.cfg_cls.CLIENT_ID, "")
        response = requests.post(
            'https://www.reddit.com/api/v1/access_token',
            auth=client_auth,
            data={
                'grant_type': 'refresh_token',
                'refresh_token': self.refresh_token
            },
            headers={
                "User-Agent": self.cfg_cls.USER_AGENT
            }
        )

        if response.json().get('error'):
            return False

        access_token = response.json()['access_token']
        self.token_expires = time.time() + 3600
        self.cfg_cls.set_key('token_expires', self.token_expires)
        self.cfg_cls.set_key('access_token', access_token)
        self.set_token(access_token)
        print("Token refreshed with %s, until %s" % (response.json()['access_token'], time.time() + 3600))

    def check_token(self):
        """
        Check if current token expired
        and get new one if it is.
        """
        if time.time() >= self.token_expires:
            self.get_new_token()

    def set_token(self, token):
        """
        Set token config value and update headers.
        """
        self.cfg_cls.set_key('access_token', token)
        self.headers = self.cfg_cls.get_headers(token)

    @check_connection
    def get_user_info(self):
        """
        Get current user info.
        """
        self.check_token()
        response = requests.get('https://oauth.reddit.com/api/v1/me', headers=self.headers)
        return response.json()

    @check_connection
    def get_subreddit(self, subreddit, posts_type, posts_limit='5'):
        """
        Get subreddit info.
        """
        self.check_token()
        posts_list = []

        response = requests.get(
            'https://oauth.reddit.com/r/%s/%s' % (subreddit, posts_type),
            headers=self.headers,
            params={'limit': posts_limit}
        )
        posts = response.json()['data']['children']

        for post in posts:
            posts_list.append({
                'link': post['data']['url'],
                'title': post['data']['title'],
                'upvotes': post['data']['ups'],
                'permalink': post['data']['permalink'],
                'gilded': post['data']['gilded']
            })

        return posts_list

    @check_connection
    def get_last_message(self):
        """
        Get contents of the last unread message.
        Should run only if notifications setting is 1.
        """
        response = requests.get(
            'https://oauth.reddit.com/message/unread',
            headers=self.headers,
            params={'limit': 1}
        )

        post_data = response.json()['data']['children'][0]['data']

        return {
            'body': post_data['body'],
            'author': post_data['author']
        }


import sys
import traceback
import os.path
import time
import webbrowser

from PyQt5 import QtWidgets, QtCore, QtGui

import toolbox.utils as ut
import acqGUI
from th260 import th260controller, th260sorter

# put here visual ressources
ICON_OK = ":/icons/ok.png"
ICON_WARNING = ":/icons/warning.png"
VERSION = '1.0'

class MainWindow(QtWidgets.QMainWindow, acqGUI.Ui_MainWindow):
    """ Main instance for the GUI of the hv controller panel """

    def __init__(self, parent=None):
        super(MainWindow, self).__init__(parent)
        self.setupUi(self)
        self.T2applySetBtn.setEnabled(False)
        self.T2startBtn.setEnabled(False)
        self.settings = QtCore.QSettings('Aalto-Antimatter', 'Pals3D')
        self.warnings = 'No warnings'

        self.th260 = th260controller.TH260Controller()
        self.sortingWorker = th260sorter.SortingWorker()
        self.sortingWorker.COINCRATE.connect(self.updateCoincRates)
        self.sortingThread = QtCore.QThread()
        self.sortingWorker.moveToThread(self.sortingThread)

        self.threadpool = QtCore.QThreadPool()

        # timer for picking counting rates when no acquisition running
        self.countRatesTimer = QtCore.QTimer()
        self.countRatesTimer.setInterval(100)
        self.countRatesTimer.timeout.connect(self.th260.getCountRates)
        self.countRatesTimer.timeout.connect(self.updateCountRates)

        # disconnect the default slots defined in TH260Controller
        self.th260.NEW_OUTPUT.disconnect()
        self.th260.WARNING.disconnect()
#        self.th260.PROGRESS.disconnect()
#        self.th260.DATA.disconnect()
#        self.th260.ACQ_ENDED.disconnect()
#        self.th260.DEVINIT.disconnect()
#        self.th260.UPDATECountRate.disconnect()
#        self.th260.ERROR.disconnect()

        # connecting signals to new slots:
        self.th260.NEW_OUTPUT.connect(self.printOutput)
        self.th260.WARNING.connect(self.updateWarning)
        self.th260.PROGRESS.connect(self.updateProgress)
        self.th260.DATA.connect(self.sortingWorker.sortBuffer,
                                type=QtCore.Qt.QueuedConnection)
        self.th260.ACQ_ENDED.connect(self.sortingWorker.processLastEvents,
                                     type=QtCore.Qt.QueuedConnection)
        self.th260.DEVINIT.connect(self.devInit)
        self.th260.UPDATECountRate.connect(self.updateCountRates)
#        self.th260.ERROR.connect()

        self.sortingWorker.NEW_OUTPUT.connect(self.printOutput)

        self.th260.searchDevices()
        self.setupWidgetLimits()

        self.T2settingDict = {}
        self.restaureSettings()
        self.fetchSettings("T2")

        self.initWk = TH260Thread(self.th260.initialization)
        self.statusbar.showMessage('Initialization of the device ...')
        self.threadpool.start(self.initWk)

        self.warningBtn = QtWidgets.QPushButton()
        self.warningBtn.setText('')
        self.warningBtn.setMaximumSize(100, 100)
        self.iconOk = QtGui.QIcon()
        self.iconOk.addPixmap(QtGui.QPixmap(ICON_OK))
        self.iconWarn = QtGui.QIcon()
        self.iconWarn.addPixmap(QtGui.QPixmap(ICON_WARNING))
        self.warningBtn.setIcon(self.iconOk)
        self.warningBtn.clicked.connect(self.on_warningBtn_clicked)

        self.statusbar.addPermanentWidget(self.warningBtn)

    # ------ Slots and GUI logic ------#
    @QtCore.pyqtSlot()
    def devInit(self):
        self.countRatesTimer.start()
        self.statusbar.clearMessage()
        self.statusbar.showMessage('Device initialized, ready to use', 30000)
        self.T2applySetBtn.setEnabled(True)
        self.T2startBtn.setEnabled(True)

    @QtCore.pyqtSlot(str)
    def updateWarning(self, text):
        self.warnings = text
        if text == 'No warning':
            self.warningBtn.setIcon(self.iconOk)
        else:
            self.warningBtn.setIcon(self.iconWarn)

    @QtCore.pyqtSlot()
    def updateCountRates(self):
        """Update the display of the count rate widgets"""
        self.rateSyncValue.display(self.th260.countRates[0])
        self.rateChn1Value.display(self.th260.countRates[1])
        self.rateChn2Value.display(self.th260.countRates[2])
        self.rateTotalValue.display(self.th260.countRates[3])

    @QtCore.pyqtSlot(int)
    def updateCoincRates(self, count):
        """Update the display of the coincidence count widgets"""
        if self.sortingWorker.sortingType == "2C":
            self.rateDoubleValue.display(self.rateDoubleValue.value() + count)
        else:
            self.rateTripleValue.display(self.rateTripleValue.value() + count)

    @QtCore.pyqtSlot(str, int)
    def updateProgress(self, mode, prog):
        """
        Update the display of the acquisition progress bars

        Parameters:
        -----------
        mode : str
            'file' or 'acq'
        prog : int
            for mode = 'file' time elapsed since the acquisition starts

            for mode = 'acq' number of file recorded up to now
        """
        if mode == "file":
            self.progFileTime = prog
            progRatio = prog*100/self.th260.tacq
            self.acqProgFileBar.setValue(progRatio)
        if mode == "acq":
            self.progAcqNumber = prog
            progRatio = prog*100/self.acqNoFiles
            self.acqProgBar.setValue(progRatio)
        self.progStatus.setText('Progress: {:.1f}/{:.0f}min'
                                ' of the file no {}/{}'
                                .format(self.progFileTime/60000,
                                        self.th260.tacq/60000,
                                        self.progAcqNumber,
                                        self.acqNoFiles))

    @QtCore.pyqtSlot()
    def on_warningBtn_clicked(self):
        self.showWarning(self.warnings)

    @QtCore.pyqtSlot()
    def on_T2filenameBtn_clicked(self):
        """Open a file dialog to select a destination file"""
        self.T2filename, _ = QtWidgets.QFileDialog.getSaveFileName(
                self, caption="choose a file",
                directory=self.T2defaultFileDir,
                filter="""Histogram files (*.hst);;Numpy files(*.npy);;
                          All files (*.*)""")
        self.T2filenameValue.setText(self.T2filename)

    @QtCore.pyqtSlot(int)
    def on_T2chn1Chk_stateChanged(self, state):
        """Enable/disable corresponding widgets when clicked"""
        pass
        # Remove the pass statement above and uncomment the following
        # lines when the code has been modified to handle the case when
        # only one of the two channels is used in the acquisition
#        if state == 0:   # Unchecked
#            ut.disableChildOf(self.T2chn1Frame)
#        if state == 2:  # Checked
#            ut.enableChildOf(self.T2chn1Frame)

    @QtCore.pyqtSlot(int)
    def on_T2chn2Chk_stateChanged(self, state):
        """Enable/disable corresponding widgets when clicked"""
        pass
        # Remove the pass statement above and uncomment the following
        # lines when the code has been modified to handle the case when
        # only one of the two channels is used in the acquisition
#        if state == 0:   # Unchecked
#            ut.disableChildOf(self.T2chn2Frame)
#        if state == 2:  # Checked
#            ut.enableChildOf(self.T2chn2Frame)

    @QtCore.pyqtSlot(bool)
    def on_T2modeDouble_toggled(self, checked):
        """Enable/disable the time resolution gate"""
        if checked:
            self.T2timeGateShortValue.setEnabled(False)
        else:
            self.T2timeGateShortValue.setEnabled(True)

    @QtCore.pyqtSlot()
    def on_T2saveDefaultPrmBtn_clicked(self):
        """Save the CFD settings as default"""
        self.saveSettings()

    @QtCore.pyqtSlot()
    def on_T2applySetBtn_clicked(self):
        """Fetch and apply the CFD settings"""
        self.fetchSettings("T2")
        self.printOutput(
                """Measurement settings:\n
                Sync CFDZeroCross  : {syncCFDZero}
                Sync CFDLevel      : {syncCFDLevel}
                Sync Offset        : {syncOffset}
                Chn1 CFDZeroCross  : {chn1CFDZero}
                Chn1 CFDLevel      : {chn1CFDLevel}
                Chn1 Offset        : {chn1Offset}
                Chn2 CFDZeroCross  : {chn2CFDZero}
                Chn2 CFDLevel      : {chn2CFDLevel}
                Chn2 Offset        : {chn2Offset}
                """.format(syncCFDZero=self.th260.syncCFDZeroCross,
                           syncCFDLevel=self.th260.syncCFDLevel,
                           syncOffset=self.th260.syncOffset,
                           chn1CFDZero=self.th260.inputCFDZeroCross[0],
                           chn1CFDLevel=self.th260.inputCFDLevel[0],
                           chn1Offset=self.th260.inputOffset[0],
                           chn2CFDZero=self.th260.inputCFDZeroCross[1],
                           chn2CFDLevel=self.th260.inputCFDLevel[1],
                           chn2Offset=self.th260.inputOffset[1]))
        self.th260.configureSetting()

    @QtCore.pyqtSlot()
    def on_T2startBtn_clicked(self):
        """Fetch settings for CFD and acquisition, then start measurement"""
        self.T2filename = self.T2filenameValue.text()
        try:
            self.checkExistingFilename(self.T2filename)
        except ValueError:
            return
        self.fetchSettings("T2")
        self.printOutput(
                """Measurement settings:\n
                Sync CFDZeroCross  : {syncCFDZero}
                Sync CFDLevel      : {syncCFDLevel}
                Sync Offset        : {syncOffset}
                Chn1 CFDZeroCross  : {chn1CFDZero}
                Chn1 CFDLevel      : {chn1CFDLevel}
                Chn1 Offset        : {chn1Offset}
                Chn2 CFDZeroCross  : {chn2CFDZero}
                Chn2 CFDLevel      : {chn2CFDLevel}
                Chn2 Offset        : {chn2Offset}
                """.format(syncCFDZero=self.th260.syncCFDZeroCross,
                           syncCFDLevel=self.th260.syncCFDLevel,
                           syncOffset=self.th260.syncOffset,
                           chn1CFDZero=self.th260.inputCFDZeroCross[0],
                           chn1CFDLevel=self.th260.inputCFDLevel[0],
                           chn1Offset=self.th260.inputOffset[0],
                           chn2CFDZero=self.th260.inputCFDZeroCross[1],
                           chn2CFDLevel=self.th260.inputCFDLevel[1],
                           chn2Offset=self.th260.inputOffset[1]))

        self.fetchAcqSettings("T2")
        self.printOutput(
            """Measurement settings:\n
            Acquisition time per file   : {acqTime} sec
            Number of files             : {noFiles}
            Time gate total             : {timeGate} ps
            Time gate (511 keV)         : {timeGate511} ps
            """.format(acqTime=self.th260.tacq,
                       noFiles=self.acqNoFiles,
                       timeGate=self.timeGate,
                       timeGate511=self.timeGate511
                       if self.T2modeTriple.isChecked()
                       else "None (2C mode)"))

        answer = self.showQuestion(""" Acquisition is about to start with:
                            Acquisition time per file   : {acqTime} sec
                            Number of files             : {noFiles}
                            Time gate total             : {timeGate} ps
                            Time gate (511 keV)         : {timeGate511} ps
                            """.format(acqTime=self.th260.tacq,
                                       noFiles=self.acqNoFiles,
                                       timeGate=self.timeGate,
                                       timeGate511=self.timeGate511
                                       if self.T2modeTriple.isChecked()
                                       else "None (2C mode)"))

        if answer == QtWidgets.QMessageBox.Ok:
            self.startAcquisition("T2")
            pass
        elif answer == QtWidgets.QMessageBox.Cancel:
            return

    @QtCore.pyqtSlot()
    def startAcquisition(self, mode):
        """Configure some variable of the TH260 controller and start"""
        # TODO: check param input
        self.saveAcqSettings()
        self.th260.countRates[3] = 0
        self.acqProgBar.setValue(0)
        self.acqProgFileBar.setValue(0)
        self.rateDoubleValue.display(0)
        self.rateTripleValue.display(0)
        self.progFileTime = 0
        self.progAcqNumber = 0
        if mode == "T2":
            self.th260.configureSetting()
            # sorting worker:
            self.sortingWorker.kwargs["nftot"] = self.acqNoFiles
            self.sortingWorker.kwargs["CFDset"] = self.T2settingDict
            self.sortingWorker.kwargs["acqTime"] = self.th260.tacq/60000
            self.sortingWorker.kwargs["filename"] = self.T2filename
            self.sortingWorker.kwargs["sortingType"] = "3C"\
                if self.T2modeTriple.isChecked()\
                else "2C"
            self.sortingWorker.kwargs["timeGate"] = self.timeGate
            if self.T2modeTriple.isChecked():
                self.sortingWorker.kwargs["timeRes"] = self.timeGate511
            else:
                self.sortingWorker.kwargs["timeRes"] = None

            # threads:
            self.acqThread = T2AcquisitionThread(self.th260, self.acqNoFiles)
            self.acqThread.globProgress.connect(self.updateProgress)
            self.acqThread.newMeas.connect(self.sortingWorker.newMeasurement)
            self.acqThread.fileDone.connect(self.sortingWorker.saveData)
            self.acqThread.measDone.connect(self.measEnded)

            self.sortingThread.start()
            self.countRatesTimer.stop()
            self.acqThread.start()

            self.statusbar.showMessage('Measurement running ...')
            self.progStatus = QtWidgets.QLabel('Progress: {}min/X'
                                               ' of the file no{}/N'
                                               .format(self.progFileTime,
                                                       self.progAcqNumber))
            self.statusbar.insertPermanentWidget(0, self.progStatus)
            ut.disableChildOf(self.T2acqGrp, self.T2stopBtn)
            ut.disableChildOf(self.T2settingsGrp)

    @QtCore.pyqtSlot()
    def measEnded(self):
        self.statusbar.removeWidget(self.progStatus)
        self.statusbar.showMessage("The measurement has ended normally",
                                   30000)
        ut.enableChildOf(self.T2acqGrp)
        ut.enableChildOf(self.T2settingsGrp)
        self.countRatesTimer.start()

    @QtCore.pyqtSlot()
    def on_T2stopBtn_clicked(self):
        """Stop the TTTR measurement and enable acq/settings widgets"""
        try:
            self.acqThread.requestInterruption()
        except AttributeError:
            self.showError("No thread to stop")
        else:
            self.th260.stoptttr()
            time.sleep(0.1)
            self.statusbar.removeWidget(self.progStatus)
            self.statusbar.showMessage('Last measurement stopped at {}'
                                       ' min of the file no {}'
                                       .format(self.progFileTime/60000,
                                               self.progAcqNumber+1))
        ut.enableChildOf(self.T2acqGrp)
        ut.enableChildOf(self.T2settingsGrp)
        self.countRatesTimer.start()

    @QtCore.pyqtSlot()
    def on_actionExit_triggered(self):
        self.th260.closeDevices()
        self.close()

    @QtCore.pyqtSlot()
    def on_actionDLL_version_triggered(self):
        self.showMessage("Currently installed library version: {}\n"
                         "\nPals3D has been developped using version: {}"
                         .format(self.th260.libVersion.value.decode("utf-8"),
                                 self.th260.LIB_VERSION))

    @QtCore.pyqtSlot()
    def on_actionPals3D_version_triggered(self):
        self.showMessage("Current version of Pals3D: {}\n"
                         .format(VERSION))

    @QtCore.pyqtSlot()
    def on_actionAbout_triggered(self):
        self.showMessage("Pals3D  Copyright (C) 2018-2019 "
                         "Aurélie Vancraeyenest"
                         "\nThis program comes with ABSOLUTELY NO WARRANTY."
                         "\nThis is free software, and you are welcome to "
                         "redistribute it under certain conditions. "
                         "See LICENSE file for details.")

    @QtCore.pyqtSlot()
    def on_actionPalss3D_help_triggered(self):
        webbrowser.open('http://pals3d.readthedocs.io/')

    def checkExistingFilename(self, filename):
        try:
            filebase, extension = filename.rsplit(sep=".", maxsplit=1)
        except ValueError:
            filebase = filename

        outputFileName = "".join((filebase,
                                  '_000.hst'))
        if os.path.isfile(outputFileName):
            answer = self.showQuestion("Output file {} will be overwritten!\n"
                                       "Do you want to continue?"
                                       .format(outputFileName))
            if answer == QtWidgets.QMessageBox.Cancel:
                raise ValueError()

    def fetchAcqSettings(self, mode):
        """Get the acquisition settings from the GUI widgets"""
        if mode == "T2":
            self.th260.tacq = self.T2acqTimePerFileValue.value()*60000
            self.acqNoFiles = self.T2acqNoFilesValue.value()
            self.timeGate = self.T2timeGateLongValue.value()
            if self.T2modeTriple.isChecked():
                self.timeGate511 = self.T2timeGateShortValue.value()
            else:
                self.timeGate511 = None

    def fetchSettings(self, mode):
        """Get the CFD settings for each channel from the GUI widgets"""
        if mode == "T2":
            self.th260.syncCFDLevel = self.T2syncLevelValue.value()
            self.th260.syncCFDZeroCross = self.T2syncZeroValue.value()
            self.th260.syncOffset = self.T2syncOffsetValue.value()
            self.th260.inputCFDLevel[0] = self.T2chn1LevelValue.value()
            self.th260.inputCFDLevel[1] = self.T2chn2LevelValue.value()
            self.th260.inputCFDZeroCross[0] = self.T2chn1ZeroValue.value()
            self.th260.inputCFDZeroCross[1] = self.T2chn2ZeroValue.value()
            self.th260.inputOffset[0] = self.T2chn1OffsetValue.value()
            self.th260.inputOffset[1] = self.T2chn2OffsetValue.value()
            self.T2settingDict['lev0'] = self.th260.syncCFDLevel
            self.T2settingDict['zero0'] = self.th260.syncCFDZeroCross
            self.T2settingDict['off0'] = self.th260.syncOffset
            self.T2settingDict['lev1'] = self.th260.inputCFDLevel[0]
            self.T2settingDict['zero1'] = self.th260.inputCFDZeroCross[0]
            self.T2settingDict['off1'] = self.th260.inputOffset[0]
            self.T2settingDict['lev2'] = self.th260.inputCFDLevel[1]
            self.T2settingDict['zero2'] = self.th260.inputCFDZeroCross[1]
            self.T2settingDict['off2'] = self.th260.inputOffset[1]

    def saveSettings(self):
        """Store the currents CFD settings as default values"""
        self.settings.setValue('T2chn1LevelValue',
                               self.T2chn1LevelValue.value())
        self.settings.setValue('T2chn1OffsetValue',
                               self.T2chn1OffsetValue.value())
        self.settings.setValue('T2chn1ZeroValue',
                               self.T2chn1ZeroValue.value())

        self.settings.setValue('T2chn2LevelValue',
                               self.T2chn2LevelValue.value())
        self.settings.setValue('T2chn2OffsetValue',
                               self.T2chn2OffsetValue.value())
        self.settings.setValue('T2chn2ZeroValue',
                               self.T2chn2ZeroValue.value())

        self.settings.setValue('T2syncLevelValue',
                               self.T2syncLevelValue.value())
        self.settings.setValue('T2syncOffsetValue',
                               self.T2syncOffsetValue.value())
        self.settings.setValue('T2syncZeroValue',
                               self.T2syncZeroValue.value())

    def saveAcqSettings(self):
        """Store the currents acquisition settings as default values"""
        self.settings.setValue('T2modeDouble',
                               self.T2modeDouble.isChecked())
        self.settings.setValue('T2acqTimePerFileValue',
                               self.T2acqTimePerFileValue.value())
        self.settings.setValue('T2acqNoFilesValue',
                               self.T2acqNoFilesValue.value())
        self.settings.setValue('T2timeGateLongValue',
                               self.T2timeGateLongValue.value())
        self.settings.setValue('T2timeGateShortValue',
                               self.T2timeGateShortValue.value())
        self.settings.setValue('T2filePath',
                               os.path.dirname(self.T2filename))
        self.settings.setValue('T2filename', self.T2filename)

    def restaureSettings(self):
        """Load the default values for CFD and acquisition settings"""
        self.T2chn1LevelValue.setValue(
                self.settings.value('T2chn1LevelValue', type=int))
        self.T2chn1OffsetValue.setValue(
                self.settings.value('T2chn1OffsetValue', type=int))
        self.T2chn1ZeroValue.setValue(
                self.settings.value('T2chn1ZeroValue', type=int))

        self.T2chn2LevelValue.setValue(
                self.settings.value('T2chn2LevelValue', type=int))
        self.T2chn2OffsetValue.setValue(
                self.settings.value('T2chn2OffsetValue', type=int))
        self.T2chn2ZeroValue.setValue(
                self.settings.value('T2chn2ZeroValue', type=int))

        self.T2syncLevelValue.setValue(
                self.settings.value('T2syncLevelValue', type=int))
        self.T2syncOffsetValue.setValue(
                self.settings.value('T2syncOffsetValue', type=int))
        self.T2syncZeroValue.setValue(
                self.settings.value('T2syncZeroValue', type=int))

        self.T2modeDouble.setChecked(
                self.settings.value('T2modeDouble', type=bool))

        self.T2acqTimePerFileValue.setValue(
                self.settings.value('T2acqTimePerFileValue', type=int))
        self.T2acqNoFilesValue.setValue(
                self.settings.value('T2acqNoFilesValue', type=int))

        self.T2timeGateLongValue.setValue(
                self.settings.value('T2timeGateLongValue', type=int))
        self.T2timeGateShortValue.setValue(
                self.settings.value('T2timeGateShortValue', type=int))

        self.T2filename = self.settings.value('T2filename', type=str)
        self.T2filenameValue.setText(self.T2filename)
        self.T2defaultFileDir = self.settings.value('T2filePath', type=str)

    def setupWidgetLimits(self):
        """Set the hardware limits to the corresponding widgets"""

        self.T2chn1LevelValue.setMaximum(self.th260.CFDLVLMAX)
        self.T2chn1LevelValue.setMinimum(self.th260.CFDLVLMIN)
        self.T2chn2LevelValue.setMaximum(self.th260.CFDLVLMAX)
        self.T2chn2LevelValue.setMinimum(self.th260.CFDLVLMIN)
        self.T2syncLevelValue.setMaximum(self.th260.CFDLVLMAX)
        self.T2syncLevelValue.setMinimum(self.th260.CFDLVLMIN)
        self.T2chn1OffsetValue.setMaximum(self.th260.CHANOFFSMAX)
        self.T2chn1OffsetValue.setMinimum(self.th260.CHANOFFSMIN)
        self.T2chn2OffsetValue.setMaximum(self.th260.CHANOFFSMAX)
        self.T2chn2OffsetValue.setMinimum(self.th260.CHANOFFSMIN)
        self.T2syncOffsetValue.setMaximum(self.th260.CHANOFFSMAX)
        self.T2syncOffsetValue.setMinimum(self.th260.CHANOFFSMIN)
        self.T2chn1ZeroValue.setMaximum(self.th260.CFDZCMAX)
        self.T2chn1ZeroValue.setMinimum(self.th260.CFDZCMIN)
        self.T2chn2ZeroValue.setMaximum(self.th260.CFDZCMAX)
        self.T2chn2ZeroValue.setMinimum(self.th260.CFDZCMIN)
        self.T2syncZeroValue.setMaximum(self.th260.CFDZCMAX)
        self.T2syncZeroValue.setMinimum(self.th260.CFDZCMIN)
        self.T2acqTimePerFileValue.setMaximum(self.th260.ACQTMAX/1000)  # in s
        self.T2acqTimePerFileValue.setMinimum(self.th260.ACQTMIN)  # always 1
        self.T2acqNoFilesValue.setMaximum(6000)
        self.T2acqNoFilesValue.setMinimum(0)
        self.T2timeGateLongValue.setMaximum(20000)      # in ps
        self.T2timeGateLongValue.setMinimum(0)          # in ps
        self.T2timeGateShortValue.setMaximum(20000)     # in ps
        self.T2timeGateShortValue.setMinimum(0)         # in ps

    def printOutput(self, text):
        """
        Append command outputs to the text box of the GUI

        Parameters:
        -----------
        text : 'str'
            Message to be printed in the command output widget
        """
        self.commandOutput.appendPlainText(text)

    def showMessage(self, message):
        """
        Display "message" in a "Information" message box with 'OK' button.

        Parameters:
        -----------
        message : 'str'
            Message to be printed in the command output widget
        """

        messageBox = QtWidgets.QMessageBox()
        messageBox.setText(message)
        messageBox.setWindowTitle("Info")
        messageBox.setIcon(QtWidgets.QMessageBox.Information)
        messageBox.setStandardButtons(QtWidgets.QMessageBox.Ok)
        messageBox.setTextInteractionFlags(QtCore.Qt.TextSelectableByMouse)

        # Show the window
        messageBox.raise_()
        messageBox.exec_()

    def showError(self, message):
        """
        Display "message" in a "Critical error" message box with 'OK' button.

        Parameters:
        -----------
        message : 'str'
            Message to be printed in the command output widget
        """

        messageBox = QtWidgets.QMessageBox()
        messageBox.setText(message)
        messageBox.setWindowTitle("Error")
        messageBox.setIcon(QtWidgets.QMessageBox.Critical)
        messageBox.setStandardButtons(QtWidgets.QMessageBox.Ok)
        messageBox.setTextInteractionFlags(QtCore.Qt.TextSelectableByMouse)

        # Show the window
        messageBox.raise_()
        messageBox.exec_()

    def showWarning(self, message):
        """
        Display "message" in a "Warning" message box with 'OK' button.

        Parameters:
        -----------
        message : 'str'
            Message to be printed in the command output widget
        """

        messageBox = QtWidgets.QMessageBox()
        messageBox.setText(message)
        messageBox.setWindowTitle("Warning")
        messageBox.setWindowIcon(QtGui.QIcon(QtGui.QPixmap(ICON_WARNING)))
        messageBox.setIcon(QtWidgets.QMessageBox.Warning)
        messageBox.setStandardButtons(QtWidgets.QMessageBox.Ok)
        messageBox.setTextInteractionFlags(QtCore.Qt.TextSelectableByMouse)

        # Show the window
        messageBox.raise_()
        messageBox.exec_()

    def showQuestion(self, message):
        """
        Display "message" in a "Warning question" message box with
        'OK' and 'Cancel' button

        Parameters:
        -----------
        message : 'str'
            Message to be printed in the command output widget
        """
        self.messageBox = QtWidgets.QMessageBox()
        self.messageBox.setText(message)
        self.messageBox.setWindowTitle("Question")
        self.messageBox.setWindowIcon(QtGui.QIcon(QtGui.QPixmap(ICON_WARNING)))
        self.messageBox.setIcon(QtWidgets.QMessageBox.Warning)
        self.messageBox.setStandardButtons(QtWidgets.QMessageBox.Cancel
                                           | QtWidgets.QMessageBox.Ok)
        self.messageBox.setTextInteractionFlags(
                                            QtCore.Qt.TextSelectableByMouse)

        # Show the window
        self.messageBox.raise_()
        answer = self.messageBox.exec_()
        return answer

# ---------- Thread workers ---------- #
class TH260Thread(QtCore.QRunnable):
    """
    Generic QRunnable that takes a function and its args/kwargs
    as arguments to run it in a threadpool
    """
    def __init__(self, fn, *args, **kwargs):
        """Constructor of the TH260Thread """
        super(TH260Thread, self).__init__()
        self.fn = fn
        self.args = args
        self.kwargs = kwargs

    @QtCore.pyqtSlot()
    def run(self):
        try:
            self.fn(*self.args, **self.kwargs)
        except Exception:
            traceback.print_exc()
            exctype, value = sys.exc_info()[:2]
        else:
            pass

class T2AcquisitionThread(QtCore.QThread):
    """
    Write docstring here

    Supported signals:
    ------------------
    globProgress : str, int
        Emitted at the end of each single acquisition
    fileDone : int
        Emitted when the nth acquisition has ended. Sent with n as
        argument
    newMeas : int
        Emmitted while starting a new measurment with as argument the
        numero of the starting acquisition
    measDone : None
        Not in use
    """
    globProgress = QtCore.pyqtSignal(str, int)
    fileDone = QtCore.pyqtSignal(int)
    newMeas = QtCore.pyqtSignal(int)
    measDone = QtCore.pyqtSignal()

    def __init__(self,  dev, noFiles):
        super(T2AcquisitionThread, self).__init__()
        self.th260device = dev
        self.noFiles = noFiles
        self.abort = False

    def run(self):
        for nof in range(self.noFiles):
            if self.isInterruptionRequested():
                return
            self.newMeas.emit(nof)
            self.th260device.startAcquisition()
            self.fileDone.emit(nof)
            self.globProgress.emit("acq", nof+1)
        self.measDone.emit()

        self.exec_()
        self.__init__(self.th260device, self.noFiles)

if __name__ == '__main__':
    app = QtWidgets.QApplication(sys.argv)
    form = MainWindow()
    form.show()
    app.exec()


from gym.wrappers.monitoring.video_recorder import VideoRecorder
from stable_baselines3 import PPO

from maze_envirement import MazeEnvironment
from model import train_agent

env = MazeEnvironment()
trained_agent = train_agent(env, total_timesteps=100)

trained_agent.save("trained_maze_agent")
loaded_agent = PPO.load("trained_maze_agent")

obs = env.reset()
done = False

video_recorder = VideoRecorder(env, "video_output.mp4")

if __name__ == "__main__":
    while not done:
        action, _ = loaded_agent.predict(obs)
        obs, reward, done, info = env.step(action)
        env.render()
        video_recorder.capture_frame()

video_recorder.close()
env.close()


from icecream import ic
from mofid.run_mofid import cif2mofid
import os
import pandas as pd
from pathlib import Path
import sys

# Defining global parameters
root_repo_folder = Path(os.path.realpath(__file__)).parents[2]
dataset_name = 'train'
dataset = pd.read_csv(os.path.join(root_repo_folder, 'data', '{}.csv'.format(dataset_name)))
mofid_table_presort = []

def collect_result(mofid):
    # Callback function (for parallelization later)
    mofid_table_presort.append(mofid)

def one_mof(MOFname):
    # Adjust the mof information to correspond to the round_idx given, and direct round_idx to the CIF file
    mof_name = "{}.cif".format(MOFname)
    ic(mof_name)
    mof_cif = os.path.join(root_repo_folder, 'data', 'mof_cif_{}'.format(dataset_name), mof_name)
    # Call the cif2mofid function as described in github.com/snurr-group/mofid
    mofid = cif2mofid(mof_cif, output_path=os.path.join(root_repo_folder, 'data', 'mofid'))
    return mofid

def main():
    # Load mofid_train_missing.csv
    missing_MOFs = pd.read_csv(os.path.join(root_repo_folder, 'output_cleaned', 'mofid_train_missing.csv'),
                               index_col=0, header=0, names=['MOFname'])
    ic(missing_MOFs)
    missing_MOFs.info()

    # Implement the extraction function
    for MOFname in missing_MOFs['MOFname']:
        collect_result(one_mof(MOFname))

    mofid_table_final = pd.DataFrame(mofid_table_presort).sort_values(by='cifname', ignore_index=True)
    mofid_table_final.to_csv(os.path.join(root_repo_folder, 'data', 'mofid_train_correction.csv'))

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
NAME:
===============================
Plot (plot.py)

BY:
===============================
Mark Gotham

LICENCE:
===============================
Creative Commons Attribution-ShareAlike 4.0 International License
https://creativecommons.org/licenses/by-sa/4.0/

ABOUT:
===============================
Plot

"""

import matplotlib.pyplot as plt
import numpy as np
from pathlib import Path
from scipy.optimize import curve_fit
from typing import Callable, Union
from music21 import roman

from . import CODE_FOLDER, data_by_heading, load_json

from .Pitch_profiles import chord_usage

ANTHOLOGY_PATH = CODE_FOLDER.parent / "Anthology"

# ------------------------------------------------------------------------------

# Linear, Polynomials (quadratic + cubic), and more

def linear(x, a, b):
    return a * x + b

def quadratic(x, a, b, c):
    return a * x ** 2 + b * x + c

def cubic(x, a, b, c, d):
    return a * x ** 3 + b * x ** 2 + c * x + d

def arc_tan(x, a, b, c, d):
    return a * np.arctan(b * x + c) + d

def exponential_decay(x, a, b, c):
    return a * np.exp(-b * x) + c

def log_decay(x, a, b, c):
    return -a * np.log(b * x) + c

# ------------------------------------------------------------------------------

def plot_dist(
        values_list_of_lists: list,
        name_list_of_lists: list,
        labels: list | None = None,
        plot_type: str = "bar",  # or "line"
        xlabel: str = "Position",
        ylabel: str = "Weight",
        title: str = "Profile weighting",
        width: float = 0.5,
        savefig: bool = False,
        out_path: Path | str | None = CODE_FOLDER / "test"
) -> plt:
    """
    Plot (bars or line) usage distribution
    whether for a source or a model (or anything else!)

    `values_list_of_lists` and `name_list_of_lists` are
    lists of lists to support any number of value / name pairs.
    There must be at least one sublist to each, and the same number as each other.
    """

    if not labels:
        labels = [str(x) for x in range(len(values_list_of_lists[0]))]
    elif len(labels) != len(values_list_of_lists[0]):
        raise ValueError("The length of variable labels "
                         f"(currently {len(labels)}) "
                         "must be equal to that of the values "
                         f"(currently {len(values_list_of_lists[0])}.")

    for x in values_list_of_lists:
        assert (len(x) == len(values_list_of_lists[0]))  # all the same length

    indexes = range(len(labels))

    # fig, ax = plt.subplots()
    # plt.figure()
    # plt.figure(figsize=(15, 6))

    # plot_type
    if plot_type == "bar":
        f = plt.bar
    elif plot_type == "line":
        f = plt.plot
    else:
        raise ValueError(f"plot_type {plot_type} not supported. Chose `bar` or `line`.")

    for i in range(len(values_list_of_lists)):
        f(indexes, values_list_of_lists[i], width, label=name_list_of_lists[i])

    plt.legend()  # loc="upper left")
    # plt.gcf().subplots_adjust(left=0.15)

    # plt.title(title)
    plt.xlabel(xlabel)  # , fontsize=14)
    plt.ylabel(ylabel)  # , fontsize=14)
    plt.xticks(indexes, labels)

    plt.ylim(0, 0.5)

    plt.tight_layout()

    if not title:
        title = "Plot"
    if savefig:
        plt.savefig(Path(out_path) / f"{title}.pdf", facecolor="w", edgecolor="w", format="pdf")
    else:
        return plt.figure

def plot_hist(
        data_list: list,
        bins: int | None = None,
        xlabel: str = "",
        ylabel: str = "Frequency",
        title: str = "",
        min_max_bin_from_data: bool = False,
        save_fig: bool = True
) -> plt:
    """
    Plotting data in the form of a list of integer values.
    Binning supported within the plot.
    """

    if min_max_bin_from_data:
        mn = min(data_list)
        mx = max(data_list)
        if bins is None:
            bins = int(mx) + 1 - int(mn)
    else:
        mn = 0
        mx = 100
        if bins is None:
            bins = 20

    incr = (mx - mn) / bins

    plt.figure(figsize=(15, 6))

    plt.hist(data_list,
             bins=bins,
             width=incr * 0.8,
             range=(mn, mx),
             align="mid",
             )

    plt.title(title, fontsize=20, family="serif")
    plt.xlabel(xlabel, fontsize=14, family="serif")
    plt.ylabel(ylabel, fontsize=14, family="serif")
    plt.xticks(np.arange(mn, mx, incr))
    plt.yticks(np.arange(0, 10, 1))

    if not title:
        title = "Plot"

    if save_fig:
        plt.savefig(title + ".pdf",
                    facecolor="w",
                    edgecolor="w",
                    format="pdf")
    else:
        return plt.figure

def plot_scatter_best_fit(
        xs: Union[list, np.array],
        ys: Union[list, np.array],
        fit_function: str | Callable = "exponential",
        p0: tuple | None = (2000, .1, 50),
        xscale: str | None = "log",
        xlabel: str = "Length (quarter notes)",
        ylabel: str = "Distance to prototype",
        title: str = "scatter_fit",
        savefig: bool = False
):
    """
    Make a scatter plot of 2D data and include a
    line (e.g. fit_function = "linear"), or
    curve (e.g. fit_function = "exponential"),
    of best fit.

    p0 argument is a tuple with starting estimate for fit function
    """

    if type(xs) == list:
        xs = np.array(xs)
    if type(ys) == list:
        ys = np.array(ys)

    # in case of multiple plots
    fig, ax = plt.subplots()
    ax.scatter(xs, ys, marker="x", linewidths=0.5)
    # plt.plot(xs, ys)  # , ".", label="data")

    if xscale:
        plt.xscale(xscale)

    x_fit = np.linspace(min(xs), max(xs), 10)  # issues with np.array(xs) for some reason

    squared_diffs = None

    if type(fit_function) == str:
        fit_function = fit_function.lower()

    if fit_function in ["lin", "linear", linear]:
        b, a = np.polynomial.polynomial.polyfit(xs, ys, 1)  # NB

        xs = np.array(xs)
        ys = np.array(ys)

        # params, cv = curve_fit(linear, xs, ys, p0)
        # a, b = params

        squared_diffs = np.square(ys - linear(xs, a, b))

        eq = f"{a}x + {b}"
        y_fit = a * x_fit + b

    elif fit_function in ["exp", "exponential", exponential_decay]:

        xs = np.array(xs)
        ys = np.array(ys)

        params, cv = curve_fit(exponential_decay, xs, ys, p0)
        a, b, c = params

        squared_diffs = np.square(ys - exponential_decay(xs, a, b, c))

        eq = f"y = {round(a, 3)}e^(-{round(b, 3)}x) + {round(c, 3)}"
        y_fit = exponential_decay(x_fit, a, b, c)

    elif fit_function in ["log", log_decay]:  # TODO compress

        xs = np.array(xs)
        ys = np.array(ys)

        # Fit
        params, cv = curve_fit(log_decay, xs, ys, p0)
        a, b, c = params

        # r^2 (quality of the fit from min 0 to max fit 1). TODO apply to all
        squared_diffs = np.square(ys - log_decay(xs, a, b, c))

        eq = f"y = -{round(a, 3)} * log({round(b, 3)}x) + {round(c, 3)}"
        y_fit = log_decay(x_fit, a, b, c)

    else:
        raise ValueError("Invalid fit_function")

    if squared_diffs is not None:
        r2_from_squared_diffs(ys, squared_diffs)

    plt.plot(x_fit, y_fit, "--", color="green", label=eq)
    plt.legend()

    plt.title(title, fontsize=16)
    plt.xlabel(xlabel, fontsize=12)
    plt.ylabel(ylabel, fontsize=12)

    if savefig:
        plt.savefig(title + ".pdf", facecolor="w", edgecolor="w", format="pdf")
    else:
        plt.show()

def r2_from_squared_diffs(
        ys: np.array,
        squared_diffs: np.square,
        print_r2: bool = True
):
    squared_diffs_from_mean = np.square(ys - np.mean(ys))
    r2 = 1 - np.sum(squared_diffs) / np.sum(squared_diffs_from_mean)
    if print_r2:
        print(f"R² = {round(r2 * 100, 3)}%")
    return r2

# ------------------------------------------------------------------------------

def plot_with_best_fit(
        x: list,
        y: list,
        fit_function=exponential_decay,
        a: float = None,  # Initial guesses for the variables
        b: float = None,  # ... as needed for the fx ...
        c: float = None,
        xlabel: str = "Position",
        ylabel: str = "Weight",
        title: str = "Distribution weighting",
        savefig: bool = False
):
    # Check fit_function legit + number of variables

    popt, pcov = curve_fit(fit_function, x, y, p0=(a, b, c))

    # Calculate new x and y values for the curve of best fit
    x_new = np.linspace(x[0], x[-1], 50)  # first, last, step
    # y_new = f(x_new)  # quad
    y_new = fit_function(x_new, *popt)  # exp

    # Plot
    plt.plot(x, y, "o", x_new, y_new)
    plt.xlim(x[0], x[-1])  # min / max
    # plt.plot(x, np.poly1d(np.polyfit(x, nums, 1))(y))
    # plt.plot(x, np.poly1d(np.polyfit(x, y, 1))(y))
    # plt.plot(np.unique(x), np.poly1d(np.polyfit(x, y, 1))(np.unique(x)))  # for unsorted
    plt.title(title, fontsize=16)
    plt.xlabel(xlabel, fontsize=12)
    plt.ylabel(ylabel, fontsize=12)

    if savefig:
        plt.savefig(title + ".pdf", facecolor="w", edgecolor="w", format="pdf")
    else:
        plt.show()
        return plt

# TODO integrate above ^
def exp_with_fit(
        xs: np.array,
        ys: np.array,
        title: str = "Test"
) -> plt:
    plt.plot(xs, ys, ".")
    plt.title(title)

    # Fit
    p0 = (2000, .1, 50)  # start with plausible values
    params, cv = curve_fit(exponential_decay, xs, ys, p0)
    a, b, c = params

    # r^2 (quality of the fit from min 0 to max fit 1)
    squared_diffs = np.square(ys - exponential_decay(xs, a, b, c))
    squared_diffsFromMean = np.square(ys - np.mean(ys))
    rSquared = 1 - np.sum(squared_diffs) / np.sum(squared_diffsFromMean)
    print(f"R² = {rSquared}")

    eq = f"Y = {round(a, 3)} * e^(-{round(b, 3)} * x) + {round(c, 3)}"

    # plot the results
    plt.plot(xs, ys, ".", label="data")
    plt.plot(xs, exponential_decay(xs, a, b, c), "--", label=f"fit: {eq}")
    plt.legend()

    plt.show()

# ------------------------------------------------------------------------------

def plot_prog_by_positions(
        corpus_name: str = "OpenScore-LiederCorpus",
        what: str = "Quiescenzas"
) -> plt:
    data = data_by_heading(ANTHOLOGY_PATH / corpus_name / (what + ".csv"))
    position_strings = [x["MEASURE"] for x in data]
    positions = []
    for i in range(len(position_strings)):
        x, y = position_strings[i].split("/")
        positions.append(100 * int(x) / int(y))

    plot_hist(
        positions,
        xlabel="Position in work (%, binned)",
        ylabel="Frequency",
        title=f"{what}_{corpus_name}",
    )

def plot_usage(
        corpus_name: str = "OpenScore-LiederCorpus",
        what: str = "Aug6",
        user_list: list | None = ["bII", "bII6", "bII64", "bII7", "bII65", "bII43", "bII2"],
        save_fig: bool = True,
        out_path: Path | None = None
) -> plt:
    """
    Plot the usage of single chords by category.
    E.g., if "what" = "Aug6",
    plots all augmented 6ths in a sub-corpus by mode (maj/min) and
    3 "Nationality" labels ignoring inversion.

    Args:
        corpus_name: one of the sub-corpora
        what: str. What chord type. Currently, "Aug6", "N6", "User" in which last case use ...
        user_list: list. Optional. A user-defined list of figures to plot (in order).
            This is ignored unless "what" is "User".
            The default provides an alternative way of plotting the Neapolitans.
        save_fig: bool. Save a copy
        out_path: Path | None. Path to write to. If None, use ANTHOLOGY_PATH / corpus_name.

    Returns: plt

    """
    minor_data = None
    major_data = None
    all_keys = None

    if what == "Aug6":
        minor_data = chord_usage.get_Aug6s(this_mode="minor", corpus_name=corpus_name)
        major_data = chord_usage.get_Aug6s(this_mode="major", corpus_name=corpus_name)
        all_keys = ("Ger", "It", "Fr")
    elif what == "N6":
        minor_data = chord_usage.get_N6s(this_mode="minor", corpus_name=corpus_name)
        major_data = chord_usage.get_N6s(this_mode="major", corpus_name=corpus_name)
        all_keys = list(minor_data.keys()) + list(major_data.keys())
    elif what == "User":
        chord_usage_dir = CODE_FOLDER / "Resources" / "chord_usage"
        minor_data = load_json(chord_usage_dir / f"minor_{corpus_name}_simple.json")
        major_data = load_json(chord_usage_dir / f"major_{corpus_name}_simple.json")
        all_keys = user_list
        pass

    for k in all_keys:
        if k not in minor_data:
            minor_data[k] = 0
        if k not in major_data:
            major_data[k] = 0

    by_mode = {"minor": tuple([round(minor_data[x], 2) for x in all_keys]),
               "major": tuple([round(major_data[x], 2) for x in all_keys])
               }
    max_val = max(list(by_mode["minor"]) + list(by_mode["major"]))

    x = np.arange(len(all_keys))
    bar_width = 0.25
    count = 0

    fig, ax = plt.subplots(layout='constrained')

    for k, v in by_mode.items():
        offset = bar_width * count
        rects = ax.bar(x + offset, v, bar_width, label=k)
        ax.bar_label(rects, padding=2)
        count += 1

    ax.set_ylabel("Frequency (%)")
    ax.set_xticks(x + bar_width / 2, all_keys)
    ax.legend(loc="upper right", ncols=2)

    ax.set_ylim(0, round(max_val, 2))  # top border = highest value; care if using with title
    ax.set_xlabel("Figure")

    if what == "Aug6":
        ax.set_xlabel('Augmented 6th by so-called "nationality" (any inversion)')

    if not out_path:
        out_path = ANTHOLOGY_PATH / corpus_name

    if save_fig:
        plt.savefig(out_path / (what + ".pdf"), facecolor="w", edgecolor="w", format="pdf")
    else:
        plt.show()

    return plt

def plot_counts(
        corpora: list | None = None,
        what: list | None = None,
        save_fig: bool = True,
        out_path: Path | None = None,
        title: str | None = None
) -> plt:
    """
    Plot the usage of chords and/or progressions by category.
    E.g., "what" defaults to
    ["ascending_fifths",
    "descending_fifths",
    "aufsteigender_Quintfall",
    "fallender_Quintanstieg"
    ],
    and plots the number of each appearing
    in each of the sub-corpora listed by `corpora`.

    Args:
        corpora: list of which sub-corpora
        what: list of what chord / progressions type.
        save_fig: bool. Save a copy
        out_path: Path | None. Path to write to. If None, use ANTHOLOGY_PATH.

    Returns: plt

    """
    if corpora is None:
        corpora = ["OpenScore-LiederCorpus",
                   "Piano_sonatas",
                   "Keyboard_Other",
                   "Early_Choral"
                   ]
    if what is None:
        what = ["ascending_fifths",
                "descending_fifths",
                "aufsteigender_Quintfall",
                "fallender_Quintanstieg"
                ]

    import csv

    by_topic = {}
    max_val = 0
    for corpus in corpora:
        this_list = []
        for w in what:
            this_path = str(ANTHOLOGY_PATH / corpus / f"{w}.csv")
            with open(this_path) as csvfile:
                data = csv.reader(csvfile)
                row_count = sum(1 for row in data)  # sum() w generator to avoid storing whole file
            this_list.append(row_count - 1)  # header
            if row_count > max_val:
                max_val = row_count
        by_topic[corpus] = this_list

    x = np.arange(len(what))
    c_and_gap = len(corpora) + 1
    bar_width = 1 / c_and_gap  # e.g., 0.25 for 3 corpora
    mid_bars = bar_width * c_and_gap / 2

    fig, ax = plt.subplots(layout='constrained')

    count = 1
    for k, v in by_topic.items():
        offset = bar_width * count
        rects = ax.bar(x + offset, v, bar_width, label=k)
        ax.bar_label(rects, padding=2)
        count += 1

    ax.set_ylabel("Count")
    ax.set_xticks(x + mid_bars,
                  [x.split("_")[0] for x in what]  # shorthand
                  )
    ax.legend(loc="upper right", ncols=1)

    ax.set_ylim(0, round(max_val, 2))

    if not out_path:
        out_path = ANTHOLOGY_PATH
    if not title:
        title = "multi_corpus_plot_count_example.pdf"

    if save_fig:
        plt.savefig(out_path / title, facecolor="w", edgecolor="w", format="pdf")
    else:
        plt.show()

    return plt

# ------------------------------------------------------------------------------

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()

    parser.add_argument("--plot_prog_by_positions", action="store_true")
    parser.add_argument("--plot_N6_usage", action="store_true")
    parser.add_argument("--plot_Aug6_usage", action="store_true")
    parser.add_argument("--plot_counts", action="store_true")

    parser.add_argument(
        "--corpus",
        type=str,
        required=False,
        default="OpenScore-LiederCorpus",
        help="Process all cases within this sub-corpus."
             "If set to `all` then run on all sub-corpora"
    )

    parser.add_argument(
        "--what",
        type=str,
        required=False,
        default="Quiescenzas",
        help="Which of the anthology cases.")

    args = parser.parse_args()

    if args.plot_prog_by_positions:
        plot_prog_by_positions(
            corpus_name=args.corpus,
            what=args.what
        )

    elif args.plot_N6_usage:
        if args.corpus == "all":
            for child in ANTHOLOGY_PATH.iterdir():
                if child.is_dir():
                    print(str(child.parts[-1]))
                    plot_usage(corpus_name=str(child.parts[-1]), what="N6")
        else:
            plot_usage(corpus_name=args.corpus, what="N6")

    elif args.plot_Aug6_usage:
        if args.corpus == "all":
            for child in ANTHOLOGY_PATH.iterdir():
                if child.is_dir():
                    print(str(child.parts[-1]))
                    plot_usage(corpus_name=str(child.parts[-1]), what="Aug6")
        else:
            plot_usage(corpus_name=args.corpus, what="Aug6")

    elif args.plot_counts:
        plot_counts()

    else:
        parser.print_help()


#!/usr/bin/env python
import numpy as np
import struct
import math
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

import rospy
import rospkg
from std_msgs.msg import Int32MultiArray
from sensor_msgs.msg import PointCloud2
import tf2_ros
import tf2_geometry_msgs
import geometry_msgs
from visualization_msgs.msg import Marker
from visualization_msgs.msg import MarkerArray
from darknet_rospy_msgs.msg import DarkBoxArray

class DarkPixelTo3DPointNode:

    def __init__(self):
        rospy.init_node('DarkPixelTo3DPointNode', anonymous=True)
        self.pointcloud_sub = rospy.Subscriber("/kinect_head/qhd/points", PointCloud2, self.pointcloud_callback, queue_size=1, buff_size=2**24)
        self.dark2d_sub = rospy.Subscriber("/darknet/recognition_result", DarkBoxArray, self.dark_callback, queue_size=1, buff_size=2**24)
        self.tf_buffer = tf2_ros.Buffer()
        self.tf_listner = tf2_ros.TransformListener(self.tf_buffer)
        self.pointcloud = None
        self.run_once_flg = False
        rospy.loginfo("Initialized DarkPixelTo3DPointNode.")

    def pixelTo3DPoint(self, u, v, pointcloud):
        point_step = pointcloud.point_step
        row_step = pointcloud.row_step
        array_pos = v*row_step + u*point_step

        bytesX = [ord(x) for x in pointcloud.data[array_pos:array_pos+4]]
        bytesY = [ord(x) for x in pointcloud.data[array_pos+4: array_pos+8]]
        bytesZ = [ord(x) for x in pointcloud.data[array_pos+8:array_pos+12]]

        byte_format=struct.pack('4B', *bytesX)
        X_sensor = struct.unpack('f', byte_format)[0]

        byte_format=struct.pack('4B', *bytesY)
        Y_sensor = struct.unpack('f', byte_format)[0]

        byte_format=struct.pack('4B', *bytesZ)
        Z_sensor = struct.unpack('f', byte_format)[0]

        nan_flg = False
        if math.isnan(X_sensor) or \
           math.isnan(Y_sensor) or \
           math.isnan(Z_sensor):
            nan_flg = True

        return [X_sensor, Y_sensor, Z_sensor], nan_flg

    def transform_3D_point_to_world_frame(self, xyz_point_from_sensor, sensor_frame_id):
        pose_stamped = geometry_msgs.msg.PoseStamped()
        pose_stamped.header.frame_id = sensor_frame_id
        pose_stamped.pose.orientation.w = 1.0
        pose_stamped.pose.position.x = xyz_point_from_sensor[0]
        pose_stamped.pose.position.y = xyz_point_from_sensor[1]
        pose_stamped.pose.position.z = xyz_point_from_sensor[2]

        transform = self.tf_buffer.lookup_transform("world",
                                                    pose_stamped.header.frame_id,
                                                    rospy.Time(0),
                                                    rospy.Duration(1.0))
        pose_transformed = tf2_geometry_msgs.do_transform_pose(pose_stamped, transform)

        X_world = pose_transformed.pose.position.x
        Y_world = pose_transformed.pose.position.y
        Z_world = pose_transformed.pose.position.z

        return [X_world, Y_world, Z_world]

    def extract_3d_points(self, dark_box):
        X = []
        Y = []
        Z = []
        for u in range(dark_box.xmin, dark_box.xmax+1):
            for v in range(dark_box.ymin, dark_box.ymax+1):
                xyz_point_from_sensor, nan_flg = self.pixelTo3DPoint(u,v,self.pointcloud)

                if nan_flg is not True:
                    xyz_point_from_world = self.transform_3D_point_to_world_frame(xyz_point_from_sensor,
                                                                                self.pointcloud.header.frame_id)
                    X.append(xyz_point_from_world[0])
                    Y.append(xyz_point_from_world[1])
                    Z.append(xyz_point_from_world[2])

        return X, Y, Z

    def plot_boxel(self, x, y, z, label, index):
        fig = plt.figure(1)
        ax = fig.gca(projection='3d')
        ax.set_xlabel(r'$x$ [m]',fontsize=14)
        ax.set_ylabel(r'$y$ [m]',fontsize=14)
        ax.set_zlabel(r'$z$ [m]',fontsize=14)
        # ax.set_xlim(0, 0.5)
        # ax.set_ylim(-0.25, 0.25)
        ax.set_zlim(-0.1, 0)

        ax.plot([0], [0], [0], 'o', color='y', ms=6 , label='world origin')
        ax.plot(x, y, z, "o", color='c', ms=3, mew=0.5, label='extracted pointcloud')
        ax.legend()

        fig.suptitle('detected object: '+label+' ['+str(index)+']' )
        plt.tight_layout()

    def pointcloud_callback(self, data):
        self.pointcloud = data

    def dark_callback(self, data):

        if self.pointcloud is None:
            return

        if self.run_once_flg is True:
            plt.pause(0.01)
            return
        else:
            self.run_once_flg = True
            index = 0
            dark_box = data.dark_array[index]
            X, Y, Z = self.extract_3d_points(dark_box)
            self.plot_boxel(X,Y,Z, dark_box.label, index)
            plt.pause(0.01)

if __name__ == '__main__':
    DarkPixelTo3DPointNode()
    try:
        rospy.spin()
    except KeyboardInterrupt:
        rospy.loginfo("Shutting Down.")


from django import forms

from TodoApp.Todo.models import Todo
from TodoApp.users.forms import BootsTrapMixin

class CreateTodoForm(BootsTrapMixin, forms.ModelForm):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.fields['description'].widget.attrs.update({
            'rows': 3,
        })

    class Meta:
        model = Todo
        fields = ['title', 'description']


import os
import numpy as np

import torch

# 데이터 로더를 구현하기
class Dataset(torch.utils.data.Dataset):
    def __init__(self, data_dir, transform=None):
        self.data_dir = data_dir
        self.transform = transform

        lst_data = os.listdir(self.data_dir)

        lst_label = [f for f in lst_data if f.startswith('label')]
        lst_input = [f for f in lst_data if f.startswith('input')]

        lst_label.sort()
        lst_input.sort()

        self.lst_label = lst_label
        self.lst_input = lst_input

    def __len__(self):
        return len(self.lst_label)

    def __getitem__(self, index):
        label = np.load(os.path.join(self.data_dir, self.lst_label[index]))
        input = np.load(os.path.join(self.data_dir, self.lst_input[index]))

        # 정규화
        label = label/255.0
        input = input/255.0

        # 이미지와 레이블의 차원 = 2일 경우(채널이 없을 경우, 흑백 이미지), 새로운 채널(축) 생성
        if label.ndim == 2:
            label = label[:, :, np.newaxis]
        if input.ndim == 2:
            input = input[:, :, np.newaxis]

        data = {'input': input, 'label': label}

        # transform이 정의되어 있다면 transform을 거친 데이터를 불러옴
        if self.transform:
            data = self.transform(data)

        return data

# # 데이터로더 잘 구현되었는지 확인(시각화)
# import matplotlib.pyplot as plt
# dir_data = './datasets'
# dir_save_train = os.path.join(dir_data, 'train')
# files_count = int(len(os.listdir(dir_save_train))/2)

# dataset_train = Dataset(data_dir=dir_save_train)
# data = dataset_train.__getitem__(np.random.randint(files_count))
# input = data['input']
# label = data['label']

# plt.subplot(122)
# plt.imshow(label, cmap='gray')
# plt.title('label')

# plt.subplot(121)
# plt.imshow(input, cmap='gray')
# plt.title('input')

# plt.show()


#%%
import scipy.io
import scipy.io
import numpy as np
import pandas as pd

##load matlab and make datafram to save label and file_id
path= '/Users/jessica/CMT309/MLCoursework2/lists/file_list.mat'
data = scipy.io.loadmat(path)
#%%
#data.keys()
#data.get("__header__") #null
#data.get("__version__") #null
#data.get("__globals__") #null
#data.get("file_list")
#data.get("annotation_list")
#data.get("label") #null
#%%
df = pd.DataFrame()
for i in data:
  if '__' not in i and 'readme' not in i:
    data_array = data[i]
    if df.empty:
      df = pd.DataFrame(data_array)
print(df.head())

#%%
full_list = pd.DataFrame(columns=['id','label'])
for i in range(len(df)):
    full_list.loc[i,'label']=str(str(df.loc[i,0]).split('/')[0]).split('-')[1] #label
    full_list.loc[i, 'id'] = str(str(df.loc[i, 0]).split("/")[1]).split("'")[0] # file name
#%%
full_list.head()

##create path and save all images
#%%
import os
import shutil
original_dataset_dir="/Users/jessica/CMT309/MLCoursework2/Images"
base_dir="/Users/jessica/CMT309/MLCoursework2/Images_New_Path"

#create train directory, validation_directory, test_directory

train_dir=os.path.join(base_dir,"train")
validation_dir=os.path.join(base_dir,"validation")
test_dir=os.path.join(base_dir,"test")
if not os.path.exists(base_dir):
    os.mkdir(base_dir)
if not os.path.exists(train_dir):
    os.mkdir(train_dir)
if not os.path.exists(validation_dir):
    os.mkdir(validation_dir)
if not os.path.exists(test_dir):
    os.mkdir(test_dir)

#%%
#copy 80% images of all folders to train directory and change names
#copy 10% images of all folders to validation...
#copy 10% images of all folders to test....
for i in os.listdir(original_dataset_dir):
    ori_path=os.path.join(original_dataset_dir,i)
    trainpath=os.path.join(train_dir,i)
    if not os.path.exists(trainpath):
        os.mkdir(trainpath)
    validpath = os.path.join(validation_dir, i)
    if not os.path.exists(validpath):
        os.mkdir(validpath)
    testpath=os.path.join(test_dir,i)
    if not os.path.exists(testpath):
        os.mkdir(testpath)
    #print(i)
    type=i.split("-")[0]
    #print(type)
    name=i.split("-")[1]
    #print(int(len(ori_path)*0.8),int(len(ori_path)*0.9),len(ori_path))
    #train_data
    for fname in os.listdir((ori_path))[:int(len(ori_path)*0.8)]:
        #print(fname)
        id=fname.split("_")[-1]
        #print(id)
        src=os.path.join(ori_path,fname)
        new_name=os.path.join(ori_path,name+'_'+id)
        dst = os.path.join(trainpath, name+'_'+id)
        #print(src)
        if os.path.exists(src):
            print("copy1")
            os.rename(src,new_name)
            shutil.copyfile(new_name,dst)
    #validation_data
    for fname in os.listdir((ori_path))[int(len(ori_path)*0.8):int(len(ori_path)*0.9)]:
        #print(fname)
        id=fname.split("_")[-1]
        #print(id)
        src=os.path.join(ori_path,fname)
        new_name=os.path.join(ori_path,name+'_'+id)
        dst = os.path.join(validpath, name+'_'+id)
        #print(src)
        if os.path.exists(src):
            print("copy2")
            os.rename(src,new_name)
            shutil.copyfile(new_name,dst)
    #test_data
    for fname in os.listdir((ori_path))[int(len(ori_path)*0.9):]:
        #print(fname)
        id=fname.split("_")[-1]
        #print(id)
        src=os.path.join(ori_path,fname)
        new_name=os.path.join(ori_path,name+'_'+id)
        dst = os.path.join(testpath, name+'_'+id)
        #print(src)
        if os.path.exists(src):
            print("copy3")
            os.rename(src,new_name)
            shutil.copyfile(new_name,dst)

#%%
#proprecessing of images
from keras.preprocessing.image import ImageDataGenerator
train_datagen=ImageDataGenerator(rescale=1./255)
test_datagen=ImageDataGenerator(rescale=1./255)
train_generator=train_datagen.flow_from_directory(train_dir,target_size=(150,150),batch_size=20,class_mode="categorical")
validation_generator=test_datagen.flow_from_directory(validation_dir,target_size=(150,150),batch_size=20,class_mode="categorical")

#%%
from keras import layers
from keras import models
import PIL
#%%

#create model
model=models.Sequential()
model.add(layers.Conv2D(32,(3,3),activation="relu",input_shape=(150,150,3))) #(148,148,32)
model.add(layers.MaxPool2D((2,2))) #(148/2=74,74,32)
model.add(layers.Conv2D(64,(3,3),activation="relu")) #(72,72,64)
model.add(layers.MaxPool2D(2,2)) #(36,36,64)
model.add(layers.Conv2D(128,(3,3),activation="relu")) #(34,34,128)
model.add(layers.MaxPool2D(2,2)) #(17,17,128)
model.add(layers.Conv2D(128,(3,3),activation="relu")) #(15,15,128)
model.add(layers.MaxPool2D(2,2)) #(7,7,128)
model.add(layers.Flatten()) #7*7*128=6272
model.add(layers.Dense(512,activation="relu"))
model.add(layers.Dense(120,activation="sigmoid"))
model.summary()
#%%
#fit model
from keras import optimizers
model.compile(loss="categorical_crossentropy",optimizer=optimizers.RMSprop(lr=1e-4),metrics=["acc"])
history=model.fit_generator(train_generator,steps_per_epoch=100,epochs=30,validation_data=validation_generator,validation_steps=50)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Book',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=25, blank=True)),
                ('author', models.CharField(max_length=25, blank=True)),
                ('end_date', models.DateField()),
                ('image', models.FileField(upload_to=b'static/img/uploads/%Y/%m/%d/')),
                ('description', models.TextField(blank=True)),
                ('is_no_draft', models.BooleanField(default=True)),
            ],
        ),
        migrations.CreateModel(
            name='Genre',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=25, blank=True)),
            ],
        ),
        migrations.AddField(
            model_name='book',
            name='genresbooks',
            field=models.ManyToManyField(to='books.Genre'),
        ),
    ]


# -*- coding: UTF-8 -*-
"""
title: 数组中数字出现的次数
一个整型数组 nums 里除两个数字之外，其他数字都出现了两次。请写程序找出这两个只出现一次的数字。要求时间复杂度是O(n)，空间复杂度是O(1)。

示例 1：
输入：nums = [4,1,4,6]
输出：[1,6] 或 [6,1]

示例 2：
输入：nums = [1,2,10,4,1,4,3,3]
输出：[2,10] 或 [10,2]

限制：
2 <= nums.length <= 10000
"""
from typing import List

class Solution:
    def singleNumbers(self, nums: List[int]) -> List[int]:
        """位运算"""
        xor = 0
        for num in nums:
            xor ^= num
        last_1 = xor & -xor
        num_1 = 0
        for num in nums:
            if num & last_1:
                num_1 ^= num
        return [num_1, num_1 ^ xor]

if __name__ == '__main__':
    print(Solution().singleNumbers([4, 1, 4, 6]))


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import json
import numpy as np
# import unicodedata
from tqdm import tqdm
import logging as log
log.basicConfig(format='%(asctime)s: %(message)s', level=log.INFO)

# 加载数据集合
def load_data(file_name, is_train=False, debug=False):
    D = []
    with open(file_name, 'r', encoding='utf-8') as f:
        for line in f:
            line = line.strip()
            l = json.loads(line)
            if is_train:
                text = l['text']
            else:
                text = l['text']
            D.append({'text': text,
                      'spo_list': [(spo['subject'], spo['predicate'], spo['object']) for spo in l['spo_list']]
                      })
    if debug:
        return D[:len(D)//4]
    return D

def search(pattern, sequence):
    n = len(pattern)
    for i in range(len(sequence)):
        if sequence[i:i + n] == pattern:
            return i
    return -1

class DataGenerator(object):
    def __init__(self, data, batch_size=32, buffer_size=None):
        self.data = data
        self.batch_size = batch_size
        if hasattr(self.data, '__len__'):
            self.steps = len(self.data) // self.batch_size
            if len(self.data) % self.batch_size != 0:
                self.steps += 1
        else:
            self.steps = None
        self.buffer_size = buffer_size or batch_size * 1000

    def __len__(self):
        return self.steps

    def sample(self, random=False):
        """采样函数，每个样本同时返回一个is_end标记
        """
        if random:
            if self.steps is None:

                def generator():
                    caches, isfull = [], False
                    for d in self.data:
                        caches.append(d)
                        if isfull:
                            i = np.random.randint(len(caches))
                            yield caches.pop(i)
                        elif len(caches) == self.buffer_size:
                            isfull = True
                    while caches:
                        i = np.random.randint(len(caches))
                        yield caches.pop(i)

            else:

                def generator():
                    indices = list(range(len(self.data)))
                    np.random.shuffle(indices)
                    for i in indices:
                        yield self.data[i]

            data = generator()
        else:
            data = iter(self.data)

        d_current = next(data)
        for d_next in data:
            yield False, d_current
            d_current = d_next

        yield True, d_current

    def __iter__(self, random=False):
        raise NotImplementedError

    def forfit(self):
        while True:
            for d in self.__iter__(True):
                yield d

class data_generator_v0(DataGenerator):
    """数据生成器
    """

    def __iter__(self, random=False, predicate2id=None, max_len=None, tokenizer=None):
        batch_text, batch_token_ids, batch_mask, batch_segment_ids = [], [], [], []
        batch_subject_labels, batch_subject_ids, batch_object_labels = [], [], []
        for is_end, d in self.sample(random):
            tokens_ids, segment_ids = tokenizer.encode(d['text'], maxlen=max_len)
            # 整理三元组 {s: [(o, p)]}
            spoes = {}
            for s, p, o in d['spo_list']:
                s = tokenizer.encode(s)[0][1:-1]
                p = predicate2id[p]
                o = tokenizer.encode(o)[0][1:-1]
                s_idx = search(s, tokens_ids)
                o_idx = search(o, tokens_ids)
                if s_idx != -1 and o_idx != -1:
                    s = (s_idx, s_idx + len(s) - 1)
                    o = (o_idx, o_idx + len(o) - 1, p)
                    if s not in spoes:
                        spoes[s] = []
                    spoes[s].append(o)
            if spoes:
                # subject标签
                input_mask = [1] * len(tokens_ids)    # TODO
                # segment_ids = [0] * len(token_ids)
                subject_labels = np.zeros((len(tokens_ids), 2))
                for s in spoes:
                    subject_labels[s[0], 0] = 1
                    subject_labels[s[1], 1] = 1
                # 随机选一个subject
                start, end = np.array(list(spoes.keys())).T
                start = np.random.choice(start)
                end = np.random.choice(end[end >= start])
                subject_ids = (start, end)
                # 对应的object标签
                object_labels = np.zeros((len(tokens_ids), len(predicate2id), 2))
                for o in spoes.get(subject_ids, []):
                    object_labels[o[0], o[2], 0] = 1
                    object_labels[o[1], o[2], 1] = 1
                # 构建batch
                batch_token_ids.append(tokens_ids)
                batch_text.append(d['text'])
                batch_mask.append(input_mask)
                batch_segment_ids.append(segment_ids)
                batch_subject_labels.append(subject_labels)
                batch_subject_ids.append(subject_ids)
                batch_object_labels.append(object_labels)
                if len(batch_token_ids) == self.batch_size or is_end:
                    batch_token_ids = sequence_padding(batch_token_ids, max_len)
                    batch_mask = sequence_padding(batch_mask, max_len)
                    batch_segment_ids = sequence_padding(batch_segment_ids, max_len)
                    batch_subject_labels = sequence_padding(
                        batch_subject_labels, max_len
                    )
                    batch_subject_ids = np.array(batch_subject_ids)
                    batch_object_labels = sequence_padding(batch_object_labels, max_len)
                    yield batch_text, batch_token_ids, batch_mask, batch_segment_ids, batch_subject_labels, \
                          batch_subject_ids, batch_object_labels

                    batch_text, batch_token_ids, batch_mask, batch_segment_ids = [], [], [], []
                    batch_subject_labels, batch_subject_ids, batch_object_labels = [], [], []

class data_generator(DataGenerator):
    """数据生成器
    """
    def __iter__(self, random=False, predicate2id=None, max_len=None, tokenizer=None):
        T0, T1, T2, M1, S1, S2, K3, O1, O2 = [], [], [], [], [], [], [], [], []
        for is_end, d in self.sample(random):
            token_ids, segment_ids = tokenizer.encode(d['text'], maxlen=max_len)
            # 整理三元组 {s: [(o, p)]}
            spoes = {}
            for s, p, o in d['spo_list']:
                s = tokenizer.encode(s)[0][1:-1]
                p = predicate2id[p]
                o = tokenizer.encode(o)[0][1:-1]
                s_idx = search(s, token_ids)
                o_idx = search(o, token_ids)
                if s_idx != -1 and o_idx != -1:
                    s = (s_idx, s_idx + len(s) - 1)
                    o = (o_idx, o_idx + len(o) - 1, p)
                    if s not in spoes:
                        spoes[s] = []
                    spoes[s].append(o)
            if spoes:
                # mask
                input_mask = [1] * len(token_ids)  # TODO

                s1, s2 = np.zeros(len(token_ids)), np.zeros(len(token_ids))
                for j in spoes:
                    s1[j[0]] = 1
                    s2[j[1]] = 1
                # 随机选一个subject
                start, end = np.array(list(spoes.keys())).T
                start = np.random.choice(start)
                end = np.random.choice(end[end >= start])
                subject_ids = (start, end)
                # 对应的object标签
                o1 = np.zeros((len(token_ids), len(predicate2id)))
                o2 = np.zeros((len(token_ids), len(predicate2id)))
                for o in spoes.get(subject_ids, []):
                    o1[o[0]][o[2]] = 1
                    o2[o[1]][o[2]] = 1
                # 构建batch
                T0.append(d['text'])
                T1.append(token_ids)
                T2.append(segment_ids)
                M1.append(input_mask)
                S1.append(s1)
                S2.append(s2)
                K3.append(subject_ids)
                # K1.append([start])
                # K2.append([end])
                O1.append(o1)
                O2.append(o2)
                if len(T1) == self.batch_size or is_end:
                    T1 = sequence_padding(T1)
                    T2 = sequence_padding(T2)
                    M1 = sequence_padding(M1)
                    S1 = sequence_padding(S1)
                    S2 = sequence_padding(S2)
                    O1 = sequence_padding(O1)
                    O2 = sequence_padding(O2)
                    # K1, K2, K3 = np.array(K1), np.array(K2), np.array(K3)
                    K3 = np.array(K3)
                    yield T0, T1, M1, T2,  S1, S2, K3, O1, O2
                    T0, T1, T2, M1, S1, S2, K3, O1, O2, = [], [], [], [], [], [], [], [], []

def closed_single(text, model, sess, tokenizer, max_len, id2predicate):
    tokens = tokenizer.tokenize(text, maxlen=max_len)
    mapping = tokenizer.rematch(text, tokens)
    token_ids, segment_ids = tokenizer.encode(text, maxlen=max_len)
    input_mask = [1] * len(token_ids)
    token_ids = sequence_padding([token_ids], max_len)
    input_mask = sequence_padding([input_mask], max_len)
    segment_ids = sequence_padding([segment_ids], max_len)
    feed_dict = {model.char_inputs: token_ids,
                 model.mask_inputs: input_mask,
                 model.segment_ids: segment_ids,
                 model.is_training: False}
    subject_s1_preds, subject_s2_preds = sess.run([model.prob_s1, model.prob_s2], feed_dict)
    start = np.where(subject_s1_preds[0] > 0.5)[0]
    end = np.where(subject_s2_preds[0] > 0.45)[0]
    subjects = []
    for i in start:
        j = end[end >= i]
        if len(j) > 0:
            j = j[0]
            subjects.append((i, j))
    if subjects:
        spoes = []
        token_ids = np.repeat(token_ids, len(subjects), 0)
        segment_ids = np.repeat(segment_ids, len(subjects), 0)
        input_mask = np.repeat(input_mask, len(subjects), 0)
        subjects = np.array(subjects)
        # 传入subject，抽取object和predicate
        feed_dict = {model.char_inputs: token_ids,
                     model.mask_inputs: input_mask,
                     model.segment_ids: segment_ids,
                     model.tag_s_ids: subjects,
                     model.is_training: False}
        object_o1_preds, object_o2_preds = sess.run([model.prob_o1, model.prob_o2], feed_dict)
        for subject, object_pred1, object_pred2 in zip(subjects, object_o1_preds, object_o2_preds):
            start = np.where(object_pred1 > 0.5)
            end = np.where(object_pred2 > 0.45)
            for _start, predicate1 in zip(*start):
                for _end, predicate2 in zip(*end):
                    if _start <= _end and predicate1 == predicate2 and subject[0] < len(mapping) \
                            and _start < len(mapping) and _end < len(mapping) and subject[1] < len(mapping) and _start != 0:
                        spoes.append(
                            ((mapping[subject[0]][0],
                              mapping[subject[1]][-1]), predicate1,
                             (mapping[_start][0], mapping[_end][-1]))
                        )
                        break
        return [(text[s[0]:s[1] + 1], id2predicate[p], text[o[0]:o[1] + 1])
                for s, p, o, in spoes]
    else:
        return []

class SPO(tuple):
    def __init__(self, spo):
        self.spox = (
            tuple(spo[0]),
            spo[1],
            tuple(spo[2]),
        )

    def __hash__(self):
        return self.spox.__hash__()

    def __eq__(self, spo):
        return self.spox == spo.spox

def sequence_padding(inputs, length=None, padding=0):
    if length is None:
        length = max([len(x) for x in inputs])

    pad_width = [(0, 0) for _ in np.shape(inputs[0])]
    outputs = []
    for x in inputs:
        x = x[:length]
        pad_width[0] = (0, length - len(x))
        x = np.pad(x, pad_width, 'constant', constant_values=padding)
        outputs.append(x)

    return np.array(outputs)

def evaluate(data, model, sess, tokenizer, max_len, id2predicate, limit=None):
    if limit:
        data = data[:limit]
    X, Y, Z = 1e-10, 1e-10, 1e-10
    pbar = tqdm()
    for d in data:
        R = set([SPO(spo) for spo in closed_single(d['text'], model, sess, tokenizer, max_len, id2predicate)])
        T = set([SPO(spo) for spo in d['spo_list']])
        X += len(R & T)
        Y += len(R)
        Z += len(T)
        f1, precision, recall = 2 * X / (Y + Z), X / Y, X / Z
        pbar.update()
        # pbar.set_description('f1: %.5f, precision: %.5f, recall: %.5f' % (f1, precision, recall))
        log.info('f1: %.5f, precision: %.5f, recall: %.5f' % (f1, precision, recall))

    pbar.close()
    return f1, precision, recall


# compare different ODE methods on the circular orbit problem
#
# M. Zingale (2013-02-19)

from __future__ import print_function

import matplotlib.pyplot as plt
from orbit import *

# circular orbit
o = Orbit(1.0, 0.0)   # eccentricity = 0

# orbital period
P = o.kepler_period()

hist_Euler = o.int_Euler(0.05, P)
hist_Euler_Cromer = o.int_Euler_Cromer(0.05, P)
hist_RK2 = o.int_RK2(0.05, P)
hist_RK4 = o.int_RK4(0.05, P)

plt.plot(hist_Euler.x, hist_Euler.y, label="Euler's method", color="k")

# mark the Sun
plt.scatter([0], [0], s=250, marker=(5,1), color="k")
plt.scatter([0], [0], s=200, marker=(5,1), color="y")

plt.plot(hist_Euler_Cromer.x, hist_Euler_Cromer.y, label="Euler-Cromer method",
           color="r")
plt.plot(hist_RK2.x, hist_RK2.y, label="R-K 2nd order",
           color="b")
plt.plot(hist_RK4.x, hist_RK4.y, label="R-K 4th order",
           color="g")

leg = plt.legend(frameon=False)
ltext = leg.get_texts()
plt.setp(ltext, fontsize='small')

plt.xlim(-2,2)
plt.ylim(-2,2)

ax = plt.gca()
ax.set_aspect("equal", "datalim")

plt.savefig("orbit-compare.png")


from pulumi import Output
from pulumi.resource import ResourceOptions
from pulumi_azure import monitoring
from pulumi_azure_native import insights

def _log_diagnostic_settings(
    log_analytics_workspace_id,
    resource_id,
    resource_name,
    log_config_obj,
    metrics_config_obj,
):

    # Instead of None, empty dictionary
    log_config_obj = log_config_obj or {}
    metrics_config_obj = metrics_config_obj or {}

    # Determine if we need to find what all the categories are
    if (
        log_config_obj.get("categories") is None
        or metrics_config_obj.get("categories") is None
    ):
        all_categories = monitoring.get_diagnostic_categories(resource_id=resource_id)
    else:
        all_categories = None

    # Set the categories if specified, all if not
    # Also set the setting name
    setting_name_log, setting_name_metric = "All", "All"
    if log_config_obj.get("categories") is None:
        log_categories = all_categories.logs
    else:
        log_categories = log_config_obj.get("categories")
        setting_name_log = "Configured"
    if metrics_config_obj.get("categories") is None:
        metric_categories = all_categories.metrics
    else:
        metric_categories = metrics_config_obj.get("categories")
        setting_name_metric = "Configured"

    # Change the categories from strings to objects
    if log_config_obj.get("enabled", True):
        log_objects = [
            insights.LogSettingsArgs(
                category=log,
                enabled=True,
                retention_policy=insights.RetentionPolicyArgs(
                    days=0,
                    enabled=False,
                ),
            )
            for log in sorted(log_categories)
        ]
    else:
        log_objects = []

    # Change the categories from strings to objects
    if metrics_config_obj.get("enabled", True):
        metric_objects = [
            insights.MetricSettingsArgs(
                category=metric,
                enabled=True,
                retention_policy=insights.RetentionPolicyArgs(
                    days=0,
                    enabled=False,
                ),
            )
            for metric in sorted(metric_categories)
        ]
    else:
        metric_objects = []

    insights.DiagnosticSetting(
        f"{resource_name}-diagnostic-setting",
        name=f"{setting_name_log} Logs, {setting_name_metric} Metrics",
        workspace_id=log_analytics_workspace_id,
        log_analytics_destination_type="Dedicated",
        resource_uri=resource_id,
        logs=log_objects,
        metrics=metric_objects,
        opts=ResourceOptions(ignore_changes=["log_analytics_destination_type"]),
    )

def log_diagnostic_settings(
    platform_config,
    log_analytics_workspace_id,
    resource_type,
    resource_id,
    resource_name,
    logs_config={},
    metrics_config={},
):

    # If any of these three are set, we will add the diagnostics
    # If the resource type is disabled, then the individual object must be
    # explicitly enabled
    if not any(
        [
            platform_config["logs"]["resource_types"].get(resource_type, True),
            logs_config.get("enabled", False),
            metrics_config.get("enabled", False),
        ]
    ):
        return

    # If logs and metrics both disabled, don't go any further
    if not logs_config.get("enabled", True) and not metrics_config.get("enabled", True):
        return

    if isinstance(resource_id, Output):
        resource_id.apply(lambda r_id:
            _log_diagnostic_settings(
                log_analytics_workspace_id, r_id, resource_name,
                logs_config, metrics_config)
        )
    else:
        _log_diagnostic_settings(
            log_analytics_workspace_id, resource_id, resource_name,
            logs_config, metrics_config)

def log_network_interfaces(
    platform_config,
    log_analytics_workspace_id,
    resource_name,
    network_interfaces,
    logs_config={},
    metrics_config={},
):
    network_interfaces.apply(
        lambda network_interfaces: [
            log_diagnostic_settings(
                platform_config,
                log_analytics_workspace_id,
                "Microsoft.Network/networkInterfaces",
                network_interface.id,
                f"{resource_name}-network_interface-{str(idx)}",
                logs_config=logs_config,
                metrics_config=metrics_config,
            )
            for idx, network_interface in enumerate(network_interfaces)
        ]
    )


import glob

import numpy as np
import xarray as xr
import matplotlib.pyplot as plt
from PIL import Image

def plot_temp_salt_contour(time_str, temp, salt, save_dir=None, counter=None):

    temp_levs = np.linspace(-4, 15, 60)
    salt_levs = np.linspace(0, 38, 100)

    fig, axs = plt.subplots(nrows=1, ncols=2, figsize=(14, 7))
    cf_temp = axs[0].contourf(temp[:,:], cmap='coolwarm', levels=temp_levs)
    cf_salt = axs[1].contourf(salt[:,:], cmap='PuBuGn', levels=salt_levs)

    names = [f'Temperature, {time_str}', 'Salinity']
    cf_top = [cf_temp, cf_salt]
    for i, ax in enumerate(axs.ravel()):
        ax.set_title(names[i])
    for cf, ax in zip(cf_top, axs):
        fig.colorbar(cf, ax=ax, location='bottom', pad=0.05)
    fig.tight_layout()

    if save_dir is not None:
        plt.savefig(save_dir + f'temp_salt_contours/{counter:03}', bbox_inches='tight', dpi=150)
        plt.close()
        print(f"Counter: {counter}")

def make_gif(frame_folder, gifname):
    frames = [Image.open(image) for image in sorted(glob.glob(f"{frame_folder}/*.png"))]
    frame_one = frames[0]
    frame_one.save(frame_folder + gifname, format="GIF", append_images=frames, save_all=True, duration=200, loop=0)

def save_plots():
    folder = '/cluster/projects/nn9297k/ROHO160+/OutputData/s_layers_25/1_dec2017-dec2018/'
    ds_temp = xr.open_dataset(folder + 'roho160_temp_0.nc')
    ds_salt = xr.open_dataset(folder + 'roho160_salt_0.nc')

    for i in range(ds_temp['ocean_time'].shape[0]):
        plot_temp_salt_contour(
            time_str=str(ds_temp['ocean_time'].isel(ocean_time=i).values),
            temp=ds_temp['temp'].isel(ocean_time=i, s_rho=-1).values,
            salt=ds_salt['salt'].isel(ocean_time=i, s_rho=-1).values,
            save_dir=folder,
            counter=i
            )

def save_gif():
    dir_name = '/cluster/projects/nn9297k/ROHO160+/OutputData/s_layers_25/1_dec2017-dec2018/temp_salt_contours'
    gif_name = "/temp_salt.gif"
    make_gif(dir_name, gif_name)

if __name__ == "__main__":
    save_plots()
    save_gif()


import requests
import shutil
from bs4 import BeautifulSoup
from imdb import IMDb

from DatabaseController import DatabaseController

class PosterBuilder:
	def __init__(self):
		print('New PosterBuilder')

	def build(self, service):

		db = DatabaseController('../res/mysql_data.cfg')
		films = db.get_all_films(service)

		for film in films:
			film_id = film[0]
			film_name = film[1]
			if(film[2] != None and film[3] == None):
				#imdb_id = self.get_image_from_imdb(film_name)
				#db.add_imdb_id(service, imdb_id, film_id)
				#imdb_id = film[2]
				imdb_id = self.get_imdb_id(film_name)
				if(imdb_id != None):
					imdb_id = self.pad_imdb_id(imdb_id)
					db.add_imdb_id(service, imdb_id, film_id)
					film_desc = self.get_film_desc(imdb_id)
					if(film_desc != None):
						db.add_film_desc(service, film_desc, film_id)

	def pad_imdb_id(self, imdb_id):
		imdb_id = str(imdb_id)
		return imdb_id.zfill(8)

	def get_image_from_imdb(self, film_name):

		imdb_id = self.get_imdb_id(film_name)
		if(imdb_id == None):
			return None

		print('IMDb ID: ', imdb_id)
		imdb_url = 'https://www.imdb.com/title/tt{}/'.format(imdb_id)
		print('IMDb URL: ', imdb_url)

		r = requests.get(imdb_url)
		html = BeautifulSoup(r.text, 'html.parser')
		div = html.find_all(class_='poster', limit=1)
		image_src = self.extract_src_url(div)

		if(image_src != None):
			self.write_image_to_file(image_src, '../res/posters/netflix/action/{}.jpg'.format(imdb_id))

		return imdb_id

	def get_imdb_id(self, film_name):
		imdb = IMDb()
		try:
			results = imdb.search_movie(film_name)
		except Exception as e:
			print('[x] get_imdb_id() - imdb.search_movie() failed')
		if(results == None):
			print('[x] No IMDB Results Found')
			return None
		if not results:
			print('[x] No IMDB Results Found')
			return None
		#For now, we'll assume the first result is always the correct result, since we're searching a specific Netflix-generated name
		return results[0].getID()

	def extract_src_url(self, div):
		src = None
		try:
			div = str(div)
			elements = div.split("=")
			src = elements[4].split(" ")[0]
			src = src.strip('\"')
		except:
			print('[x] extract_src_url() failed.')

		return src

	def write_image_to_file(self, image_url, file_path):
		try:
			response = requests.get(image_url, stream=True)
		except Exception as e:
			print('[x] Failed to get image url: {}'.format(image_url))
			return

		if(response.status_code != 200):
			print('[x] Could Not Connect to IMDb')
			return

		response.raw.decode_content = True

		file = open(file_path, "wb+")
		print('Saving {}'.format(file_path))
		shutil.copyfileobj(response.raw, file)
		#file.write(response.content)
		file.close()

	def get_film_desc(self, imdb_id):
		print('Getting Description For: {}'.format(imdb_id))
		imdb_url = 'https://www.imdb.com/title/tt{}/'.format(imdb_id)
		try:
			response = requests.get(imdb_url)
		except Exception as e:
			print('[x] Failed to retrieve IMDb Description for {}'.format(imdb_id))
			return None

		html = BeautifulSoup(response.text, 'html.parser')
		div = html.find_all(class_='summary_text', limit=1)
		desc = self.extract_desc(div)
		return desc

	def extract_desc(self, div):
		try:
			desc = str(div)
			desc = desc.split("\n")
			return desc[1].strip()
		except Exception as e:
			print('[x] Failed To Extract Description')
			return None

if __name__ == '__main__':
	pb = PosterBuilder()
	pb.build('netflix')