text:
import os
import numpy as np
import glob
import shutil
from pathlib import Path

from backend import Backend, States, Modes
from pointCloudGLWidget import PointCloudGLWidget
from linePlotWidget import LinePlotWidget

from PyQt5 import QtCore
from PyQt5.QtCore import QThread
from PyQt5.QtWidgets import QMainWindow, QGridLayout, QAction
from PyQt5.QtWidgets import QSpinBox, QLabel, QHBoxLayout, QFileDialog
from PyQt5.QtWidgets import QWidget, QApplication, QCheckBox
from PyQt5.QtWidgets import QPushButton, QTabWidget, QVBoxLayout
from PyQt5.QtWidgets import QDoubleSpinBox, QComboBox, QGroupBox

# Deletes all files in the content path.
def _clearContent(content):
    files = glob.glob(content)
    for f in files:
        if not os.path.isdir(f):
            os.remove(f)

    files = glob.glob(content)
    for f in files:
        if not os.path.isdir(f):
            os.remove(f)

# Calibration widget.
# the only reason it is created to catch the press 'n' event.
class CalibWidget(QWidget):
    takeImageTriggered = QtCore.pyqtSignal()

    def keyPressEvent(self, event):
        super(CalibWidget, self).keyPressEvent(event)
        if event.key() == QtCore.Qt.Key_N:
            self.takeImageTriggered.emit()

# Main Qt UI window
class MainWindow(QMainWindow):
    # Folder that contains the saved UI settings.
    SETTINGS_DIR = "settings"

    def __init__(self, *args, **kwargs):
        super(MainWindow, self).__init__(*args, **kwargs)
        # Initialize worker thread related members.
        self.workerThread = QThread(self)
        self.worker = Backend()
        self.worker.signals.framesSent.connect(self.updateVideo)
        self.worker.signals.finished.connect(self.thread_complete)
        self.worker.signals.error.connect(self.sigint_handler)

        # init UI.
        self.setMinimumSize(200, 100)
        mainLayout = QGridLayout()
        self._createMenu()

        # Create tabs
        tabwidget = QTabWidget()
        calibratorLayout = self._createCalibrationUI()
        self.calibratorLayoutWidget = CalibWidget()
        self.calibratorLayoutWidget.setLayout(calibratorLayout)
        tabwidget.addTab(self.calibratorLayoutWidget, "Sensor calibration")

        bmConfiguratorLayout = self._createBlockMatchingConfiguratorUI()
        bmConfiguratorLayoutWidget = QWidget()
        bmConfiguratorLayoutWidget.setLayout(bmConfiguratorLayout)
        tabwidget.addTab(
            bmConfiguratorLayoutWidget, "Block Matching Configurator")

        featureDetectorLayout = self._createFeatureDetectionUI()
        featureDetectionLayoutWidget = QWidget()
        featureDetectionLayoutWidget.setLayout(featureDetectorLayout)
        tabwidget.addTab(featureDetectionLayoutWidget, "Feature detection")

        motionEstimationLayout = self._createMotionEstimationUI()
        motionEstimationLayoutWidget = QWidget()
        motionEstimationLayoutWidget.setLayout(motionEstimationLayout)
        tabwidget.addTab(motionEstimationLayoutWidget, "Motion estimation")

        vSlamUILayout = self._createVSlamUI()
        vSlamUILayoutWidget = QWidget()
        vSlamUILayoutWidget.setLayout(vSlamUILayout)
        tabwidget.addTab(vSlamUILayoutWidget, "Visual SLAM")

        self._initUIElements()

        mainLayout.addWidget(tabwidget, 0, 0)
        mainWidget = QWidget()
        mainWidget.setLayout(mainLayout)
        self.setCentralWidget(mainWidget)

        desktop = QApplication.desktop()
        screenRect = desktop.screenGeometry()
        self.resize(screenRect.width(), screenRect.height())

        if not os.path.isdir(self.SETTINGS_DIR):
            os.mkdir(self.SETTINGS_DIR)

        self.show()

    # Saves all UI values into an npz file.
    # Saves all calibration images and chessboard.npz for each sensor
    # in the folder named identical to the settings npz file
    def _saveValues(self, settingsName):
        np.savez_compressed(
            settingsName,
            bm_textureThreshold=self.textureThreshold.value(),
            bm_min_disp=self.min_disp.value(),
            bm_num_disp=self.num_disp.value(),
            bm_blocksize=self.blockSize.value(),
            bm_uniquenessRatio=self.uniquenessRatio.value(),
            bm_speckleWindowSize=self.speckleWindowSize.value(),
            bm_speckleRange=self.speckleRange.value(),
            bm_disp12MaxDiff=self.disp12MaxDiff.value(),
            bm_preFilterType=self.preFilterType.value(),
            bm_preFilterSize=self.preFilterSize.value(),
            bm_preFilterCap=self.preFilterCap.value(),
            bm_smallerBlockSize=self.smallerBlockSize.value(),
            bm_mode=self.blockMatching.currentIndex(),
            bm_drawEpipolar=self.drawEpipolar.isChecked(),
            bm_resolution=self.resolutionBm.currentIndex(),
            bm_leftCameraIndex=self.bmCameraIndexLeft.currentIndex(),
            bm_rightCameraIndex=self.bmCameraIndexLeft.currentIndex(),
            pc_fov=self.fov.value(),
            pc_samplingRatio=self.samplingRatio.value(),
            pc_ignoreRendererMaxDepth=self.rendererMaxDepth.value(),
            cal_calib_image_index=self.calib_image_index.value(),
            cal_rms_limit=self.rms_limit.value(),
            cal_advanced=self.advanced.isChecked(),
            cal_ignoreExitingImageData=self
                                        .ignoreExistingImageData
                                        .isChecked(),
            cal_rms_increment=self.increment.value(),
            cal_max_rms=self.max_rms.value(),
            cal_resolution=self.resolutionCal.currentIndex(),
            cal_leftCameraIndex=self.calibCameraIndexLeft.currentIndex(),
            cal_rightCameraIndex=self.calibCameraIndexRight.currentIndex(),
            feat_featureDetector=self.featureDetector.currentIndex(),
            feat_featureMatcher=self.featureMatcher.currentIndex(),
            feat_maxDistance=self.maxDistance.value(),
            motion_inliers=self.inliers.value(),
            motion_maxDepth=self.maxDepth.value(),
            motion_reprojectionError=self.reprojectionError.value())

        settingsPath = Path(settingsName).with_suffix('')
        settingsPath = settingsPath.stem

        files = glob.glob(f"calibImages/left/{str(settingsPath)}/*")
        for f in files:
            os.remove(f)

        files = glob.glob(f"calibImages/right/{str(settingsPath)}/*")
        for f in files:
            os.remove(f)

        leftDirectory = f"calibImages/left/{str(settingsPath)}"
        if not os.path.isdir(leftDirectory):
            os.mkdir(leftDirectory)

        files = glob.glob("calibImages/left/*")
        for f in files:
            if not os.path.isdir(f):
                shutil.copy(f, leftDirectory)

        rightDirectory = f"calibImages/right/{str(settingsPath)}"
        if not os.path.isdir(rightDirectory):
            os.mkdir(rightDirectory)

        files = glob.glob("calibImages/right/*")
        for f in files:
            if not os.path.isdir(f):
                shutil.copy(f, rightDirectory)

    # Saves UI settings and calibration images/data
    # Also creates a lastSaved folder
    # for quick loading last saved info when the application starts.
    def saveSettings(self):
        options = QFileDialog.Options()
        options |= QFileDialog.DontUseNativeDialog
        fileName, _ = QFileDialog.getSaveFileName(
            self,
            "QFileDialog.getSaveFileName()", "", "npz (*.npz)",
            options=options)
        if fileName:
            self._saveValues(fileName)

            _clearContent("calibImages/left/lastSaved/*")
            _clearContent("calibImages/left/lastSaved/*")
            self._saveValues(f"{self.SETTINGS_DIR}/lastSaved.npz")

    # Loads and sets settings values from the npz file.
    # Also loads the appropriate calib images and data.
    def _setLoadedValues(self, settingsName):
        _clearContent("calibImages/left/*")
        _clearContent("calibImages/right/*")

        settings = np.load(settingsName)
        self.textureThreshold.setValue(settings["bm_textureThreshold"])
        self.min_disp.setValue(settings["bm_min_disp"])
        self.num_disp.setValue(settings["bm_num_disp"])
        self.blockSize.setValue(settings["bm_blocksize"])
        self.uniquenessRatio.setValue(settings["bm_uniquenessRatio"])
        self.speckleWindowSize.setValue(
            settings["bm_speckleWindowSize"])
        self.speckleRange.setValue(settings["bm_speckleRange"])
        self.disp12MaxDiff.setValue(settings["bm_disp12MaxDiff"])
        self.preFilterType.setValue(settings["bm_preFilterType"])
        self.preFilterSize.setValue(settings["bm_preFilterSize"])
        self.preFilterCap.setValue(settings["bm_preFilterCap"])
        self.blockMatching.setCurrentIndex(settings["bm_mode"])
        self.drawEpipolar.setChecked(bool(settings["bm_drawEpipolar"]))
        self.smallerBlockSize.setValue(settings["bm_smallerBlockSize"])
        self.resolutionBm.setCurrentIndex(settings["bm_resolution"])
        self.bmCameraIndexLeft.setCurrentIndex(settings["bm_leftCameraIndex"])
        self.bmCameraIndexRight.setCurrentIndex(settings["bm_rightCameraIndex"])
        self.fov.setValue(settings["pc_fov"])
        self.samplingRatio.setValue(settings["pc_samplingRatio"])
        self.rendererMaxDepth.setValue(settings["pc_ignoreRendererMaxDepth"])
        self.calib_image_index.setValue(settings["cal_calib_image_index"])
        self.rms_limit.setValue(settings["cal_rms_limit"])
        self.advanced.setChecked(bool(settings["cal_advanced"]))
        self.ignoreExistingImageData.setChecked(
            bool(settings["cal_ignoreExitingImageData"]))
        self.increment.setValue(settings["cal_rms_increment"])
        self.max_rms.setValue(settings["cal_max_rms"])
        self.resolutionCal.setCurrentIndex(settings["cal_resolution"])
        self.calibCameraIndexLeft.setCurrentIndex(settings["cal_leftCameraIndex"])
        self.calibCameraIndexRight.setCurrentIndex(settings["cal_rightCameraIndex"])
        self.featureDetector.setCurrentIndex(settings["feat_featureDetector"])
        self.featureMatcher.setCurrentIndex(settings["feat_featureMatcher"])
        self.maxDistance.setValue(settings["feat_maxDistance"])
        self.inliers.setValue(settings["motion_inliers"])
        self.maxDepth.setValue(settings["motion_maxDepth"])
        self.reprojectionError.setValue(settings["motion_reprojectionError"])

        settingsPath = Path(settingsName).with_suffix('')
        settingsPath = settingsPath.stem
        directory = f"calibImages/left/{str(settingsPath)}"
        if os.path.isdir(str(directory)):
            files = glob.glob(f"{directory}/*")
            for f in files:
                shutil.copy(f, "calibImages/left/")
        else:
            print(
                f"No left calib images to load \
(calibImages/left/{str(settingsPath)})")

        directory = f"calibImages/right/{str(settingsPath)}"
        if os.path.isdir(str(directory)):
            files = glob.glob(f"{directory}/*")
            for f in files:
                shutil.copy(f, "calibImages/right/")
        else:
            print(f"No left calib images to load \
(calibImages/left/{str(settingsPath)})")

    # Loads settings,calib images and data.
    def loadSettings(self):
        options = QFileDialog.Options()
        options |= QFileDialog.DontUseNativeDialog
        fileName, _ = QFileDialog.getOpenFileName(
            self,
            "QFileDialog.getOpenFileName()", "", "npz (*.npz)",
            options=options)
        if fileName:
            try:
                self._setLoadedValues(fileName)
            except IOError:
                print("Settings file at {0} not found"
                      .format(fileName))
                self.sigint_handler()

    # Creates the menu items.
    def _createMenu(self):
        saveAction = QAction('&Save Settings', self)
        saveAction.setShortcut('Ctrl+S')
        saveAction.setStatusTip('Save settings')
        saveAction.triggered.connect(self.saveSettings)

        loadAction = QAction('&Load Settings', self)
        loadAction.setShortcut('Ctrl+L')
        loadAction.setStatusTip('Load settings')
        loadAction.triggered.connect(self.loadSettings)

        exitAction = QAction('&Exit', self)
        exitAction.setShortcut('Ctrl+Q')
        exitAction.setStatusTip('Exit application')
        exitAction.triggered.connect(self.sigint_handler)

        # Create menu bar and add action
        menuBar = self.menuBar()
        fileMenu = menuBar.addMenu('&File')
        fileMenu.addAction(saveAction)
        fileMenu.addAction(loadAction)
        fileMenu.addAction(exitAction)

    # Creates the calibration tab UI.
    def _createCalibrationUI(self):
        layout = QGridLayout()
        helpLabel = QLabel("Quick user guide:\n\
    1. To start calibration interface, press start.\n\
    2. Once started, to capture a pair of frames, press 'n' or 'Take image'\
when your sensor's are in the desired position.\n\
    3. When enough calib images are create press process, to create the final\
calibration file.\n\n\
Use modes:\n\
    a, Simple (Default): capture as many calibration images as you want, \
when done press 'Process'.\n\
    b, Advanced: during capture the system will analyse the calibration RMS\
and will throw away if there is ahigh RMS result. See more in README.md\n\
       When finished press 'Process'")

        calibInfoLabel = QLabel("Calibration info")
        self.calibInfo = QLabel()

        RMSLabel = QLabel("RMS")
        self.RMSValue = QLabel()
        labelLayout = QVBoxLayout()
        labelLayout.addWidget(helpLabel)
        labelLayout.addWidget(calibInfoLabel)
        labelLayout.addWidget(self.calibInfo)
        labelLayout.addWidget(RMSLabel)
        labelLayout.addWidget(self.RMSValue)
        labelsLayoutWidget = QWidget()
        labelsLayoutWidget.setLayout(labelLayout)

        self.resolutionCal = QComboBox()
        self.resolutionCal.addItems(["480p", "720p"])
        ignoreExistingImageDataLabel = QLabel("Ignore existing image data")
        self.ignoreExistingImageData = QCheckBox()
        self.ignoreExistingImageData.setDisabled(True)
        advancedLabel = QLabel("Advanced use mode")
        self.advanced = QCheckBox()
        self.advanced.setDisabled(True)
        optionsLayout = QHBoxLayout()
        optionsLayout.addWidget(QLabel("Resolution"))
        optionsLayout.addWidget(self.resolutionCal)
        optionsLayout.addWidget(ignoreExistingImageDataLabel)
        optionsLayout.addWidget(self.ignoreExistingImageData)
        optionsLayout.addWidget(advancedLabel)
        optionsLayout.addWidget(self.advanced)
        optionsLayoutWidget = QWidget()
        optionsLayoutWidget.setLayout(optionsLayout)

        self.calib_image_index = QSpinBox()
        self.rms_limit = QDoubleSpinBox()
        self.increment = QDoubleSpinBox()
        self.max_rms = QDoubleSpinBox()
        configLayout = QHBoxLayout()
        configLayout.addWidget(QLabel("calib_image_index"))
        configLayout.addWidget(self.calib_image_index)
        configLayout.addWidget(QLabel("rms_limit"))
        configLayout.addWidget(self.rms_limit)
        configLayout.addWidget(QLabel("increment"))
        configLayout.addWidget(self.increment)
        configLayout.addWidget(QLabel("max_rms"))
        configLayout.addWidget(self.max_rms)
        configLayoutWidget = QWidget()
        configLayoutWidget.setLayout(configLayout)
        self.calib_image_index.setRange(0, 1000)
        self.calib_image_index.setSingleStep(1)
        self.rms_limit.setRange(0, 10.0)
        self.rms_limit.setDecimals(5)
        self.rms_limit.setSingleStep(0.005)
        self.increment.setRange(0, 1.0)
        self.increment.setDecimals(5)
        self.increment.setSingleStep(0.005)
        self.max_rms.setRange(0, 10.0)
        self.max_rms.setDecimals(5)
        self.max_rms.setSingleStep(0.005)

        cameraLayout = QGridLayout()
        displayGroupBox = QGroupBox("Visualisation")
        displayGroupBox.setLayout(cameraLayout)
        self.video0Calib = QLabel()
        self.calibSwapCameras = QPushButton("Swap lenses")
        self.calibCameraIndexLeft = QComboBox()
        cameraLayout.addWidget(self.video0Calib, 0, 0, 4, 3)
        cameraLayout.addWidget(self.calibSwapCameras, 5, 0, 1, 1)
        cameraLayout.addWidget(QLabel("Left camera indices"), 5, 1, 1, 1)
        cameraLayout.addWidget(self.calibCameraIndexLeft, 5, 2, 1, 1)
        self.video1Calib = QLabel()
        self.calibCameraIndexRight = QComboBox()
        cameraLayout.addWidget(self.video1Calib, 0, 3, 4, 3)
        cameraLayout.addWidget(QLabel("Right camera indices"), 5, 3, 1, 1)
        cameraLayout.addWidget(self.calibCameraIndexRight, 5, 4, 1, 1)
        layout.addWidget(displayGroupBox, 0, 0, 1, 1)

        start = QPushButton("Start")
        self.process = QPushButton("Process")
        self.process.hide()
        self.takeImage = QPushButton("Take image")
        self.takeImage.hide()
        start.clicked.connect(self._startCalibration)
        buttonLayout = QHBoxLayout()
        buttonLayout.addWidget(start)
        buttonLayout.addWidget(self.process)
        buttonLayout.addWidget(self.takeImage)
        buttonLayoutWidget = QWidget()
        buttonLayoutWidget.setLayout(buttonLayout)

        layout.addWidget(labelsLayoutWidget, 1, 0, 1, 2)
        layout.addWidget(optionsLayoutWidget, 2, 0, 1, 4)
        layout.addWidget(configLayoutWidget, 3, 0, 1, 4)
        layout.addWidget(buttonLayoutWidget, 4, 0, 1, 4)
        return layout

    # Creates the block matching UI.
    def _createBlockMatchingConfiguratorUI(self):
        self.win_sizeUpdated = QtCore.pyqtSignal(int)
        self.min_dispUpdated = QtCore.pyqtSignal(int)
        self.num_dispUpdated = QtCore.pyqtSignal(int)
        self.blockSizeUpdated = QtCore.pyqtSignal(int)
        self.uniquenessRatioUpdated = QtCore.pyqtSignal(int)
        self.speckleWindowSizeUpdated = QtCore.pyqtSignal(int)
        self.speckleRangeUpdated = QtCore.pyqtSignal(int)
        self.disp12MaxDiffUpdated = QtCore.pyqtSignal(int)
        layout = QGridLayout()

        self.textureThresholdLabel = QLabel("textureThreshold")
        self.textureThreshold = QSpinBox()
        self.min_disp = QSpinBox()
        self.num_disp = QSpinBox()
        self.blockSize = QSpinBox()
        self.uniquenessRatio = QSpinBox()
        self.speckleWindowSize = QSpinBox()
        self.speckleRange = QSpinBox()
        self.disp12MaxDiff = QSpinBox()
        self.preFilterSizeLabel = QLabel("preFilterSize")
        self.preFilterSize = QSpinBox()
        self.preFilterTypeLabel = QLabel("preFilterType")
        self.preFilterType = QSpinBox()
        self.preFilterCapLabel = QLabel("preFilterCap")
        self.preFilterCap = QSpinBox()
        self.smallerBlockSizeLabel = QLabel("smallerBlockSize")
        self.smallerBlockSize = QSpinBox()
        self.start = QPushButton("Start")
        self.start.clicked.connect(self._startBMConfiguration)

        # Init spin boxes
        self.textureThreshold.setRange(0, 10000)
        self.min_disp.setRange(-1, 1000)
        self.min_disp.setSingleStep(1)
        self.num_disp.setRange(16, 1000)
        self.num_disp.setSingleStep(16)
        self.blockSize.setRange(5, 255)
        self.blockSize.setSingleStep(2)
        self.smallerBlockSize.setRange(-1, 1000000)
        self.smallerBlockSize.setSingleStep(1)
        self.uniquenessRatio.setRange(0, 1000)
        self.speckleWindowSize.setRange(-1, 1000)
        self.speckleRange.setRange(-1, 1000)
        self.disp12MaxDiff.setRange(-1, 1000)
        self.preFilterType.setRange(0, 1)
        self.preFilterType.setSingleStep(1)
        self.preFilterSize.setRange(5, 255)
        self.preFilterSize.setSingleStep(2)
        self.preFilterCap.setRange(1, 63)
        self.preFilterCap.setSingleStep(1)

        self.resolutionBm = QComboBox()
        self.resolutionBm.addItems(["480p", "720p"])
        self.blockMatching = QComboBox()
        self.blockMatching.addItems(
            ["Block Matching", "Semi-Global Block Matching"])
        self.drawEpipolar = QCheckBox()

        self.video0Bm = QLabel()
        self.bmCameraIndexLeft = QComboBox()
        self.bmSwapCameras = QPushButton("Swap lenses")
        self.video1Bm = QLabel()
        self.bmCameraIndexRight = QComboBox()
        self.video_disp = QLabel()
        self.pointCloud = PointCloudGLWidget()
        self.fov = QSpinBox()
        self.fov.setRange(1, 360)
        self.fov.valueChanged.connect(
            self.pointCloud.setFov)
        self.rendererMaxDepth = QSpinBox()
        self.rendererMaxDepth.setRange(1, 5000)
        self.rendererMaxDepth.valueChanged.connect(
            self.pointCloud.setIgnoreDepthLimit)
        self.samplingRatio = QSpinBox()
        self.samplingRatio.setRange(1, 10000)
        self.samplingRatio.setSingleStep(50)
        self.samplingRatio.valueChanged.connect(
            self.pointCloud.setSamplingRatio)

        cameraLayout = QGridLayout()
        displayGroupBox = QGroupBox("Visualisation")
        displayGroupBox.setLayout(cameraLayout)
        pointCloudLayout = QGridLayout()
        pointCloudLayout.addWidget(self.pointCloud, 0, 0, 4, 4)
        pointCloudLayout.addWidget(QLabel("Field of view"), 0, 4, 1, 1)
        pointCloudLayout.addWidget(self.fov, 0, 5, 1, 1)
        pointCloudLayout.addWidget(QLabel("Sampling ratio"), 1, 4, 1, 1)
        pointCloudLayout.addWidget(self.samplingRatio, 1, 5, 1, 1)
        pointCloudLayout.addWidget(QLabel("Ignore depth"), 2, 4, 1, 1)
        pointCloudLayout.addWidget(self.rendererMaxDepth, 2, 5, 1, 1)
        pointCloudControl = QGroupBox("Point cloud")
        pointCloudControl.setLayout(pointCloudLayout)
        cameraLayout.addWidget(self.video0Bm, 0, 0, 4, 4)
        cameraLayout.addWidget(self.bmSwapCameras, 5, 0, 1, 1)
        cameraLayout.addWidget(QLabel("Left camera indices"), 5, 1, 1, 1)
        cameraLayout.addWidget(self.bmCameraIndexLeft, 5, 2, 1, 1)
        cameraLayout.addWidget(self.video1Bm, 0, 3, 4, 4)
        cameraLayout.addWidget(QLabel("Right camera indices"), 5, 3, 1, 1)
        cameraLayout.addWidget(self.bmCameraIndexRight, 5, 4, 1, 1)
        cameraLayout.addWidget(self.video_disp, 6, 0, 4, 4)
        cameraLayout.addWidget(pointCloudControl, 6, 3, 4, 4)
        layout.addWidget(displayGroupBox, 0, 0, 1, 6)

        bmControlLayout = QGridLayout()
        bmControlLayout.addWidget(QLabel("Block matching type"), 0, 0, 1, 1)
        bmControlLayout.addWidget(self.blockMatching, 0, 1, 1, 1)
        bmControlLayout.addWidget(QLabel("Draw epipolar lines"), 0, 2, 1, 1)
        bmControlLayout.addWidget(self.drawEpipolar, 0, 3, 1, 1)
        bmControlLayout.addWidget(QLabel("Resolution"), 0, 4, 1, 1)
        bmControlLayout.addWidget(self.resolutionBm, 0, 5, 1, 1)
        bmControlLayout.addWidget(self.textureThresholdLabel, 1, 0, 1, 1)
        bmControlLayout.addWidget(self.textureThreshold, 1, 1, 1, 1)
        bmControlLayout.addWidget(QLabel("min_disp"), 1, 2, 1, 1)
        bmControlLayout.addWidget(self.min_disp, 1, 3, 1, 1)
        bmControlLayout.addWidget(QLabel("num_disp"), 1, 4, 1, 1)
        bmControlLayout.addWidget(self.num_disp, 1, 5, 1, 1)
        bmControlLayout.addWidget(QLabel("blockSize"), 1, 6, 1, 1)
        bmControlLayout.addWidget(self.blockSize, 1, 7, 1, 1)
        bmControlLayout.addWidget(QLabel("uniquenessRatio"), 1, 8, 1, 1)
        bmControlLayout.addWidget(self.uniquenessRatio, 1, 9, 1, 1)
        bmControlLayout.addWidget(QLabel("speckleWindowSize"), 1, 10, 1, 1)
        bmControlLayout.addWidget(self.speckleWindowSize, 1, 11, 1, 1)
        bmControlLayout.addWidget(QLabel("speckleRange"), 1, 12, 1, 1)
        bmControlLayout.addWidget(self.speckleRange, 1, 13, 1, 1)
        bmControlLayout.addWidget(QLabel("disp12MaxDiff"), 1, 14, 1, 1)
        bmControlLayout.addWidget(self.disp12MaxDiff, 1, 15, 1, 1)
        bmControlLayout.addWidget(self.smallerBlockSizeLabel, 2, 0, 1, 1)
        bmControlLayout.addWidget(self.smallerBlockSize, 2, 1, 1, 1)
        bmControlLayout.addWidget(self.preFilterTypeLabel, 2, 2, 1, 1)
        bmControlLayout.addWidget(self.preFilterType, 2, 3, 1, 1)
        bmControlLayout.addWidget(self.preFilterCapLabel, 2, 4, 1, 1)
        bmControlLayout.addWidget(self.preFilterCap, 2, 5, 1, 1)
        bmControlLayout.addWidget(self.preFilterSizeLabel, 2, 6, 1, 1)
        bmControlLayout.addWidget(self.preFilterSize, 2, 7, 1, 1)
        bmControlGroup = QGroupBox("Block Matching control")
        bmControlGroup.setLayout(bmControlLayout)
        layout.addWidget(bmControlGroup, 2, 0, 1, 8)
        buttonLayout = QHBoxLayout()
        buttonLayout.addWidget(self.start)
        buttonLayoutWidget = QWidget()
        buttonLayoutWidget.setLayout(buttonLayout)
        layout.addWidget(buttonLayoutWidget, 4, 0, 1, 8)
        return layout

    ## @brief Creates the feature detection UI.
    def _createFeatureDetectionUI(self):
        layout = QGridLayout()
        self.videoFD = QLabel()

        messageTitle = QLabel("Logging:")
        messageLabel = QLabel()
        self.worker.signals.updateFeatureInfo.connect(messageLabel.setText)
        messageLayout = QVBoxLayout()
        messageLayout.addWidget(messageTitle)
        messageLayout.addWidget(messageLabel)
        messageLayoutWidget = QWidget()
        messageLayoutWidget.setLayout(messageLayout)

        featureDetectorLabel = QLabel("Feature detector")
        self.featureDetector = QComboBox()
        self.featureDetector.addItems(["sift", "orb", "surf"])
        self.featureDetector.currentTextChanged.connect(self.worker.updateFeatureDetector)
        featureMatcherLabel = QLabel("Feature matcher")
        self.featureMatcher = QComboBox()
        self.featureMatcher.addItems(["BF", "FLANN"])
        self.featureMatcher.currentTextChanged.connect(self.worker.updateFeatureMatcher)
        maxDistanceLabel = QLabel("Max allowed distance between best matches")
        self.maxDistance = QDoubleSpinBox()
        self.maxDistance.valueChanged.connect(self.worker.updateMatchDistanceThreshold)
        self.maxDistance.setRange(0.01, 100)
        self.maxDistance.setSingleStep(0.01)
        controlLayout = QGridLayout()
        controlLayout.addWidget(featureDetectorLabel, 0, 2, 1, 1)
        controlLayout.addWidget(self.featureDetector, 0, 3, 1, 1)
        controlLayout.addWidget(featureMatcherLabel, 0, 4, 1, 1)
        controlLayout.addWidget(self.featureMatcher, 0, 5, 1, 1)
        controlLayout.addWidget(maxDistanceLabel, 0, 6, 1, 1)
        controlLayout.addWidget(self.maxDistance, 0, 7, 1, 1)
        controlLayoutWidget = QWidget()
        controlLayoutWidget.setLayout(controlLayout)

        start = QPushButton("Start")
        start.clicked.connect(self._startFeatureDetection)
        buttonLayout = QHBoxLayout()
        buttonLayout.addWidget(start)
        buttonLayoutWidget = QWidget()
        buttonLayoutWidget.setLayout(buttonLayout)

        layout.addWidget(self.videoFD, 0, 0, 3, 1)
        layout.addWidget(messageLayoutWidget, 3, 0, 1, 1)
        layout.addWidget(controlLayoutWidget, 4, 0, 1, 1)
        layout.addWidget(buttonLayoutWidget, 5, 0, 1, 1)
        return layout

    ## @brief Creates the motion estimation UI.
    def _createMotionEstimationUI(self):
        layout = QGridLayout()
        self.videoDepthME = QLabel()
        self.motionDisplay = PointCloudGLWidget()
        self.trajectoryPlotDepth = LinePlotWidget()
        self.trajectoryPlotDepth.setAxisLabel("time", "z")
        self.trajectoryPlotXY = LinePlotWidget()
        self.trajectoryPlotXY.setAxisLabel("x", "y")

        start = QPushButton("Start")
        start.clicked.connect(self._startMotionEstimation)
        buttonLayout = QHBoxLayout()
        buttonLayout.addWidget(start)
        buttonLayoutWidget = QWidget()
        buttonLayoutWidget.setLayout(buttonLayout)

        inliersLabel = QLabel("inliers")
        self.inliers = QSpinBox()
        self.inliers.valueChanged.connect(self.worker.updateInlierLimit)
        self.inliers.setRange(1, 200)
        self.inliers.setSingleStep(1)
        maxDepthLabel = QLabel("maxDepth")
        self.maxDepth = QSpinBox()
        self.maxDepth.valueChanged.connect(self.worker.updateMaxDepth)
        self.maxDepth.setRange(1, 5000)
        self.maxDepth.setSingleStep(1)
        reprojectionErrorLabel = QLabel("reprojectionError")
        self.reprojectionError = QDoubleSpinBox()
        self.reprojectionError.valueChanged.connect(self.worker.updateReprojectionError)
        self.reprojectionError.setRange(0.01, 10)
        self.reprojectionError.setSingleStep(0.01)
        controlLayout = QGridLayout()
        controlLayout.addWidget(inliersLabel, 0, 0, 1, 1)
        controlLayout.addWidget(self.inliers, 0, 1, 1, 1)
        controlLayout.addWidget(maxDepthLabel, 0, 2, 1, 1)
        controlLayout.addWidget(self.maxDepth, 0, 3, 1, 1)
        controlLayout.addWidget(reprojectionErrorLabel, 0, 4, 1, 1)
        controlLayout.addWidget(self.reprojectionError, 0, 5, 1, 1)
        controlLayoutWidget = QWidget()
        controlLayoutWidget.setLayout(controlLayout)

        layout.addWidget(self.motionDisplay, 0, 0, 3, 4)
        layout.addWidget(self.videoDepthME, 3, 0, 1, 2)
        layout.addWidget(self.trajectoryPlotDepth, 3, 2, 2, 1)
        layout.addWidget(self.trajectoryPlotXY, 3, 3, 2, 1)
        layout.addWidget(controlLayoutWidget, 5, 0, 1, 4)
        layout.addWidget(buttonLayoutWidget, 6, 0, 1, 4)
        return layout

    ## @brief Creates the vSLAM UI.
    def _createVSlamUI(self):
        layout = QGridLayout()
        self.videoDepthVSlam = QLabel()
        self.motionDisplayVSlam = PointCloudGLWidget()

        start = QPushButton("Start")
        start.clicked.connect(self._startVSlam)
        buttonLayout = QHBoxLayout()
        buttonLayout.addWidget(start)
        buttonLayoutWidget = QWidget()
        buttonLayoutWidget.setLayout(buttonLayout)

        layout.addWidget(self.motionDisplayVSlam, 0, 0, 4, 4)
        layout.addWidget(self.videoDepthVSlam, 2, 0, 1, 1)
        layout.addWidget(buttonLayoutWidget, 4, 0, 1, 4)
        return layout

    # Updates the video stream labels.
    def updateVideo(
            self, v0, v1, v_depth_color, v_depth_gray, v_feature, trajectory, featureMap):
        if self.worker.mode == Modes.Calibration:
            self.process.show()
            self.takeImage.show()
            self.ignoreExistingImageData.setDisabled(False)
            self.advanced.setDisabled(False)
            self.video0Calib.setPixmap(v0)
            self.video1Calib.setPixmap(v1)
        elif self.worker.mode == Modes.BlockMatching:
            self.video0Bm.setPixmap(v0)
            self.video1Bm.setPixmap(v1)
            self.video_disp.setPixmap(v_depth_color)
            pointCloud = self.pointCloud.calculatePointCloud(v_depth_gray)
            self.pointCloud.setMapVBO(pointCloud)
            self.pointCloud.updateGL()
        elif self.worker.mode == Modes.FeatureDetection:
            self.videoFD.setPixmap(v_feature)
        elif self.worker.mode == Modes.MotionEstimation:
            self.videoDepthME.setPixmap(v_depth_color)
            self.motionDisplay.setTrajectoryVBO(trajectory)
            self.motionDisplay.updateGL()
            if trajectory is not None:
                self.trajectoryPlotDepth.plotData(
                    np.arange(len(trajectory)), trajectory[:, 2, 3])
            self.trajectoryPlotXY.plotData(trajectory[:, 0, 3], trajectory[:, 1, 3])
        elif self.worker.mode == Modes.Mapping:
            self.motionDisplayVSlam.setTrajectoryVBO(trajectory)
            self.motionDisplayVSlam.setMapVBO(featureMap)
            self.motionDisplayVSlam.updateGL()
            self.videoDepthVSlam.setPixmap(v_depth_color)

    # Shows/hides the appropriate controls for SGBM and BM.
    def updateBmType(self, index):
        if index == 1:
            self.textureThresholdLabel.hide()
            self.textureThreshold.hide()
            self.preFilterSizeLabel.hide()
            self.preFilterSize.hide()
            self.preFilterCapLabel.hide()
            self.preFilterCap.hide()
            self.preFilterTypeLabel.hide()
            self.preFilterType.hide()
            self.smallerBlockSizeLabel.hide()
            self.smallerBlockSize.hide()
        else:
            self.textureThresholdLabel.show()
            self.textureThreshold.show()
            self.preFilterSizeLabel.show()
            self.preFilterSize.show()
            self.preFilterCapLabel.show()
            self.preFilterCap.show()
            self.preFilterTypeLabel.show()
            self.preFilterType.show()
            self.smallerBlockSizeLabel.show()
            self.smallerBlockSize.show()
        if self.worker is not None:
            self.worker.updateBmType(index)

    def thread_complete(self):
        print("Worker thread stopped...")

    # Updates the calib UI control from the worker thread.
    def updateCalibInfo(self, rms, message):
        self.RMSValue.setText(f"{rms}")
        self.calibInfo.setText(message)

    # Updates the camera indices in all tabs.
    def updateCameraIndices(self, leftIndex, rightIndex, indicesList):
        if self.bmCameraIndexLeft.count() == 0:
            for index in indicesList:
                self.bmCameraIndexLeft.addItem(f"{index}")
                self.bmCameraIndexRight.addItem(f"{index}")
                self.calibCameraIndexLeft.addItem(f"{index}")
                self.calibCameraIndexRight.addItem(f"{index}")
        self.bmCameraIndexLeft.setCurrentIndex(leftIndex)
        self.bmCameraIndexRight.setCurrentIndex(rightIndex)
        self.calibCameraIndexLeft.setCurrentIndex(leftIndex)
        self.calibCameraIndexRight.setCurrentIndex(rightIndex)

    # Sets the ranges, limits and signal connections for each UI element.
    def _initUIElements(self):
        info = self.worker.getSensorIndices()
        self.updateCameraIndices(*info)

        # wire bm signals/slots
        self.blockMatching.currentIndexChanged.connect(
            self.updateBmType)
        self.resolutionBm.currentIndexChanged.connect(
            self.worker.updateResolution)
        self.drawEpipolar.toggled.connect(
            self.worker.updateDrawEpipolar)
        self.textureThreshold.valueChanged.connect(
            self.worker.updateTextureThreshold)
        self.min_disp.valueChanged.connect(self.worker.updateMin_disp)
        self.num_disp.valueChanged.connect(self.worker.updateNum_disp)
        self.blockSize.valueChanged.connect(self.worker.updateBlockSize)
        self.uniquenessRatio.valueChanged.connect(
            self.worker.updateUniquenessRatio)
        self.speckleWindowSize.valueChanged.connect(
            self.worker.updateSpeckleWindowSize)
        self.speckleRange.valueChanged.connect(self.worker.updateSpeckleRange)
        self.disp12MaxDiff.valueChanged.connect(self.worker.updateDisp12MaxDiff)
        self.smallerBlockSize.valueChanged.connect(self.worker.updateSmallerBlockSize)
        self.preFilterType.valueChanged.connect(self.worker.updatePreFilterType)
        self.preFilterSize.valueChanged.connect(self.worker.updatePrefilterSize)
        self.preFilterCap.valueChanged.connect(self.worker.updatePrefilterCap)
        self.worker.signals.cameraIndicesUpdated.connect(
            self.updateCameraIndices)
        self.bmCameraIndexLeft.currentIndexChanged.connect(
            self.worker.updateLeftCameraIndex)
        self.bmCameraIndexRight.currentIndexChanged.connect(
            self.worker.updateRightCameraIndex)
        self.bmSwapCameras.clicked.connect(self.worker.swapSensorIndices)

        # wire calib signals/slots
        self.calibratorLayoutWidget.takeImageTriggered.connect(
            self.worker.saveImage)
        self.process.clicked.connect(self.worker.calibrateSensor)
        self.takeImage.clicked.connect(self.worker.saveImage)
        self.resolutionCal.currentIndexChanged.connect(
            self.worker.updateResolution)
        self.ignoreExistingImageData.toggled.connect(
            self.worker.setIgnoreExistingImageData)
        self.advanced.toggled.connect(self.worker.enableAdvancedCalib)
        self.calib_image_index.valueChanged.connect(
            self.worker.updateCalib_image_index)
        self.rms_limit.valueChanged.connect(self.worker.updateRms_limit)
        self.increment.valueChanged.connect(self.worker.updateIncrement)
        self.max_rms.valueChanged.connect(self.worker.updateMax_rms)
        self.worker.signals.updateCalibInfo.connect(self.updateCalibInfo)
        self.worker.signals.rmsLimitUpdated.connect(self.rms_limit.setValue)
        self.worker.signals.calibImageIndexUpdated.connect(
            self.calib_image_index.setValue)
        self.calibCameraIndexLeft.currentIndexChanged.connect(
            self.worker.updateLeftCameraIndex)
        self.calibCameraIndexRight.currentIndexChanged.connect(
            self.worker.updateRightCameraIndex)
        self.calibSwapCameras.clicked.connect(self.worker.swapSensorIndices)

        # load values
        try:
            self._setLoadedValues(f"{self.SETTINGS_DIR}/lastSaved.npz")
        except IOError:
            print("Settings file at {0} not found"
                  .format(f"{self.SETTINGS_DIR}/lastSaved.npz"))
            self.sigint_handler()

        self.updateBmType(self.blockMatching.currentIndex())

    # Start the thread with calibration.
    # if the thread is already running, it will not restart it
    # just set to calibration mode, if it wasn't yet.
    def _startCalibration(self):
        print("Starting calibration...")
        if self.worker.mode == Modes.Calibration:
            print("Calibration is already running...")
            return

        self.worker.enableAdvancedCalib(self.advanced.isChecked())
        self.worker.setIgnoreExistingImageData(
            self.ignoreExistingImageData.isChecked())

        # Execute
        if self.worker.mode == Modes.NoMode:
            self.worker.moveToThread(self.workerThread)
            self.workerThread.finished.connect(self.worker.deleteLater)
            self.workerThread.started.connect(self.worker.run)
            self.workerThread.start()
        self.worker.state = States.Idle
        self.worker.mode = Modes.Calibration

    # @brief  Start the thread with block matching.
    #
    # If the thread is already running, it will not restart it
    # just set to block matching mode, if it wasn't yet.
    def _startBMConfiguration(self):
        print("Starting block matching...")
        if self.worker.mode == Modes.BlockMatching:
            print("Block Matching is already running...")
            return

        # Execute
        if self.worker.mode == Modes.NoMode:
            self.worker.moveToThread(self.workerThread)
            self.workerThread.finished.connect(self.worker.deleteLater)
            self.workerThread.started.connect(self.worker.run)
            self.workerThread.start()
        self.worker.state = States.Idle
        self.worker.mode = Modes.BlockMatching

    ## @brieft Start the thread with feature detection.
    #
    # If the thread is already running, it will not restart it
    # just set to block matching mode, if it wasn't yet.
    def _startFeatureDetection(self):
        print("Starting feature detection...")
        if self.worker.mode == Modes.FeatureDetection:
            print("Feature detection is already running...")
            return

        # Execute
        if self.worker.mode == Modes.NoMode:
            self.worker.moveToThread(self.workerThread)
            self.workerThread.finished.connect(self.worker.deleteLater)
            self.workerThread.started.connect(self.worker.run)
            self.workerThread.start()
        self.worker.state = States.Idle
        self.worker.mode = Modes.FeatureDetection

    ## @brieft Start the thread with motion estimation.
    #
    # If the thread is already running, it will not restart it
    # just set to block matching mode, if it wasn't yet.
    def _startMotionEstimation(self):
        print("Starting motion estimation...")
        if self.worker.mode == Modes.MotionEstimation:
            print("Motion estimation is already running...")
            return

        # Execute
        if self.worker.mode == Modes.NoMode:
            self.worker.moveToThread(self.workerThread)
            self.workerThread.finished.connect(self.worker.deleteLater)
            self.workerThread.started.connect(self.worker.run)
            self.workerThread.start()
        self.worker.state = States.Idle
        self.worker.mode = Modes.MotionEstimation

    ## @brieft Start the thread with mapping functionality.
    #
    # If the thread is already running, it will not restart it
    # just set to block matching mode, if it wasn't yet.
    def _startVSlam(self):
        print("Starting Visual SLAM...")
        if self.worker.mode == Modes.Mapping:
            print("Visual SLAM is already running...")
            return

        # Execute
        if self.worker.mode == Modes.NoMode:
            self.worker.moveToThread(self.workerThread)
            self.workerThread.finished.connect(self.worker.deleteLater)
            self.workerThread.started.connect(self.worker.run)
            self.workerThread.start()
        self.worker.state = States.Idle
        self.worker.mode = Modes.Mapping

    # Terminate UI and the threads appropriately.
    def sigint_handler(self):
        if self.worker is not None:
            self.worker.stop = True
            self.workerThread.quit()
            self.workerThread.wait()
        self.trajectoryPlotXY.clear()
        self.trajectoryPlotDepth.clear()
        print("Exiting app through GUI")
        QApplication.quit()


text:
import torch
import numpy as np
from itertools import groupby

np.seterr(divide="ignore")

def ctc_greedy_decode(outputBatch, inputLenBatch, eosIx, blank=0):

    """
    Greedy search technique for CTC decoding.
    This decoding method selects the most probable character at each time step. This is followed by the usual CTC decoding
    to get the predicted transcription.
    Note: The probability assigned to <EOS> token is added to the probability of the blank token before decoding
    to avoid <EOS> predictions in middle of transcriptions. Once decoded, <EOS> token is appended at last to the
    predictions for uniformity with targets.
    """

    outputBatch = outputBatch.cpu()
    inputLenBatch = inputLenBatch.cpu()
    outputBatch[:,:,blank] = torch.log(torch.exp(outputBatch[:,:,blank]) + torch.exp(outputBatch[:,:,eosIx]))
    reqIxs = np.arange(outputBatch.shape[2])
    reqIxs = reqIxs[reqIxs != eosIx]
    outputBatch = outputBatch[:,:,reqIxs]

    predCharIxs = torch.argmax(outputBatch, dim=2).T.numpy()
    inpLens = inputLenBatch.numpy()
    preds = list()
    predLens = list()
    for i in range(len(predCharIxs)):
        pred = predCharIxs[i]
        ilen = inpLens[i]
        pred = pred[:ilen]
        pred = np.array([x[0] for x in groupby(pred)])
        pred = pred[pred != blank]
        pred = list(pred)
        pred.append(eosIx)
        preds.extend(pred)
        predLens.append(len(pred))
    predictionBatch = torch.tensor(preds).int()
    predictionLenBatch = torch.tensor(predLens).int()
    return predictionBatch, predictionLenBatch

class BeamEntry:
    """
    Class for a single entry in the beam.
    """
    def __init__(self):
        self.logPrTotal = -np.inf
        self.logPrNonBlank = -np.inf
        self.logPrBlank = -np.inf
        self.logPrText = 0
        self.lmApplied = False
        self.lmState = None
        self.labeling = tuple()

class BeamState:

    """
    Class for the beam.
    """

    def __init__(self, alpha, beta):
        self.entries = dict()
        self.alpha = alpha
        self.beta = beta

    def score(self, entry):
        """
        Function to compute score of each entry in the beam.
        """
        labelingLen = len(entry.labeling)
        if labelingLen == 0:
            score = entry.logPrTotal + self.alpha*entry.logPrText
        else:
            score = (entry.logPrTotal + self.alpha*entry.logPrText)/(labelingLen**self.beta)
        return score

    def sort(self):
        """
        Function to sort all the beam entries in descending order depending on their scores.
        """
        beams = [entry for (key, entry) in self.entries.items()]
        sortedBeams = sorted(beams, reverse=True, key=self.score)
        return [x.labeling for x in sortedBeams]

def apply_lm(parentBeam, childBeam, spaceIx, lm):

    """
    Applying the language model to obtain the language model character probabilities at a time step
    given all the previous characters.
    """

    if not (childBeam.lmApplied):
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        if parentBeam.lmState == None:
            initStateBatch = None
            inputBatch = torch.tensor(spaceIx-1).reshape(1,1)
            inputBatch = inputBatch.to(device)
        else:
            initStateBatch = parentBeam.lmState
            inputBatch = torch.tensor(parentBeam.labeling[-1]-1).reshape(1,1)
            inputBatch = inputBatch.to(device)
        lm.eval()
        with torch.no_grad():
            outputBatch, finalStateBatch = lm(inputBatch, initStateBatch)
        logProbs = outputBatch.squeeze()
        logProb = logProbs[childBeam.labeling[-1]-1]
        childBeam.logPrText = parentBeam.logPrText + logProb
        childBeam.lmApplied = True
        childBeam.lmState = finalStateBatch
    return

def add_beam(beamState, labeling):
    """
    Function to add a new entry to the beam.
    """
    if labeling not in beamState.entries.keys():
        beamState.entries[labeling] = BeamEntry()

def log_add(a, b):
    """
    Addition of log probabilities.
    """
    result = np.log(np.exp(a) + np.exp(b))
    return result

def ctc_search_decode(outputBatch, inputLenBatch, beamSearchParams, spaceIx, eosIx, lm, blank=0):

    """
    Applies the CTC beam search decoding along with a character-level language model.
    Note: The probability assigned to <EOS> token is added to the probability of the blank token before decoding
    to avoid <EOS> predictions in middle of transcriptions. Once decoded, <EOS> token is appended at last to the
    predictions for uniformity with targets.
    """

    outputBatch = outputBatch.cpu()
    inputLenBatch = inputLenBatch.cpu()
    outputBatch[:,:,blank] = torch.log(torch.exp(outputBatch[:,:,blank]) + torch.exp(outputBatch[:,:,eosIx]))
    reqIxs = np.arange(outputBatch.shape[2])
    reqIxs = reqIxs[reqIxs != eosIx]
    outputBatch = outputBatch[:,:,reqIxs]

    beamWidth = beamSearchParams["beamWidth"]
    alpha = beamSearchParams["alpha"]
    beta = beamSearchParams["beta"]
    threshProb = beamSearchParams["threshProb"]

    outLogProbs = outputBatch.transpose(0, 1).numpy()
    inpLens = inputLenBatch.numpy()
    preds = list()
    predLens = list()

    for n in range(len(outLogProbs)):
        mat = outLogProbs[n]
        ilen = inpLens[n]
        mat = mat[:ilen,:]
        maxT, maxC = mat.shape

        #initializing the main beam with a single entry having empty prediction
        last = BeamState(alpha, beta)
        labeling = tuple()
        last.entries[labeling] = BeamEntry()
        last.entries[labeling].logPrBlank = 0
        last.entries[labeling].logPrTotal = 0

        #going over all the time steps
        for t in range(maxT):

            #a temporary beam to store all possible predictions (which are extensions of predictions
            #in the main beam after time step t-1) after time step t
            curr = BeamState(alpha, beta)
            #considering only the characters with probability above a certain threshold to speeden up the algo
            prunedChars = np.where(mat[t,:] > np.log(threshProb))[0]

            #keeping only the best predictions in the main beam
            bestLabelings = last.sort()[:beamWidth]

            #going over all the best predictions
            for labeling in bestLabelings:

                #same prediction (either blank or last character repeated)
                if len(labeling) != 0:
                    logPrNonBlank = last.entries[labeling].logPrNonBlank + mat[t, labeling[-1]]
                else:
                    logPrNonBlank = -np.inf

                logPrBlank = last.entries[labeling].logPrTotal + mat[t, blank]

                add_beam(curr, labeling)
                curr.entries[labeling].labeling = labeling
                curr.entries[labeling].logPrNonBlank = log_add(curr.entries[labeling].logPrNonBlank, logPrNonBlank)
                curr.entries[labeling].logPrBlank = log_add(curr.entries[labeling].logPrBlank, logPrBlank)
                curr.entries[labeling].logPrTotal = log_add(curr.entries[labeling].logPrTotal, log_add(logPrBlank, logPrNonBlank))
                curr.entries[labeling].logPrText = last.entries[labeling].logPrText
                curr.entries[labeling].lmApplied = True
                curr.entries[labeling].lmState = last.entries[labeling].lmState

                #extending the best prediction with all characters in the pruned set
                for c in prunedChars:

                    if c == blank:
                        continue

                    #extended prediction
                    newLabeling = labeling + (c,)

                    if (len(labeling) != 0)  and (labeling[-1] == c):
                        logPrNonBlank = mat[t, c] + last.entries[labeling].logPrBlank
                    else:
                        logPrNonBlank = mat[t, c] + last.entries[labeling].logPrTotal

                    add_beam(curr, newLabeling)
                    curr.entries[newLabeling].labeling = newLabeling
                    curr.entries[newLabeling].logPrNonBlank = log_add(curr.entries[newLabeling].logPrNonBlank, logPrNonBlank)
                    curr.entries[newLabeling].logPrTotal = log_add(curr.entries[newLabeling].logPrTotal, logPrNonBlank)

                    #applying language model
                    if lm is not None:
                        apply_lm(curr.entries[labeling], curr.entries[newLabeling], spaceIx, lm)

            #replacing the main beam with the temporary beam having extended predictions
            last = curr

        #output the best prediciton
        bestLabeling = last.sort()[0]
        bestLabeling = list(bestLabeling)
        bestLabeling.append(eosIx)
        preds.extend(bestLabeling)
        predLens.append(len(bestLabeling))

    predictionBatch = torch.tensor(preds).int()
    predictionLenBatch = torch.tensor(predLens).int()
    return predictionBatch, predictionLenBatch

ext: py

reason: import numpy


text:
import logging

import networkx as nx
import sqlalchemy as sa
from sqlalchemy.dialects.postgresql import insert

from models_library.projects import ProjectID
from models_library.projects_state import RunningState

from ....models.domains.comp_pipelines import CompPipelineAtDB
from ....utils.logging_utils import log_decorator
from ..tables import comp_pipeline
from ._base import BaseRepository

logger = logging.getLogger(__name__)

class CompPipelinesRepository(BaseRepository):
    @log_decorator(logger=logger)
    async def upsert_pipeline(
        self, project_id: ProjectID, dag_graph: nx.DiGraph, publish: bool
    ) -> None:

        pipeline_at_db = CompPipelineAtDB(
            project_id=project_id,
            dag_adjacency_list=nx.to_dict_of_lists(dag_graph),
            state=RunningState.PUBLISHED if publish else RunningState.NOT_STARTED,
        )
        insert_stmt = insert(comp_pipeline).values(**pipeline_at_db.dict(by_alias=True))
        on_update_stmt = insert_stmt.on_conflict_do_update(
            index_elements=[comp_pipeline.c.project_id],
            set_=pipeline_at_db.dict(by_alias=True, exclude_unset=True),
        )
        await self.connection.execute(on_update_stmt)

    @log_decorator(logger=logger)
    async def delete_pipeline(self, project_id: ProjectID) -> None:
        await self.connection.execute(
            sa.delete(comp_pipeline).where(
                comp_pipeline.c.project_id == str(project_id)
            )
        )

max_stars_repo_name: GitHK/osparc-simcore-forked

max_issues_repo_name: GitHK/osparc-simcore-forked

max_forks_repo_name: GitHK/osparc-simcore-forked

reason: import networkx


text:
import numpy as np
import pandas as pd
import torch
from physics.protein_os import Protein
import options
from utils import write_pdb, write_pdb_sample, transform_profile, load_protein
from physics.anneal import AnnealCoords, AnnealFrag
# from physics.move import SampleICNext
from physics.grad_minimizer import *
from physics.dynamics import *
import os
import mdtraj as md
from utils import test_setup
import h5py

#################################################
parser = options.get_fold_parser()
args = options.parse_args_and_arch(parser)

device, model, energy_fn, ProteinBase = test_setup(args)

# position_weights = torch.zeros((1, args.seq_len + 1), device=device)
# position_weights[:, 0:5] = 1
# energy_fn.energy_fn.position_weights = position_weights

#################################################
data_path = 'data/fold/cullpdb_val_deep'
protein_sample = pd.read_csv(f'{data_path}/sample.csv')
pdb_selected = protein_sample['pdb'].values
np.random.shuffle(pdb_selected)

fold_engine = args.fold_engine
mode = args.mode
# sample_ic = SampleICNext(mode)
exp_id = args.load_exp[-5:]
save_dir = args.save_dir
# if not os.path.exists(f'data/fold/{exp_id}'):
#     os.mkdir(f'data/fold/{exp_id}')
if not os.path.exists(f'data/fold/{save_dir}'):
    os.mkdir(f'data/fold/{save_dir}')

for pdb_id in pdb_selected:

    seq, coords_native, profile = load_protein(data_path, pdb_id, mode, device, args)

    protein_native = Protein(seq, coords_native, profile)
    energy_native = protein_native.get_energy(energy_fn).item()
    print('energy_native:', energy_native)
    rg2, collision = protein_native.get_rad_gyration(coords_native)
    print('native radius of gyration square:', rg2.item())
    # residue_energy = protein_native.get_residue_energy(energy_fn)
    # print(residue_energy)
    # write_pdb(seq, coords_native, pdb_id, 'native', exp_id)

    protein = Protein(seq, coords_native.clone(), profile.clone())
    if args.random_init:
        # random_coords_int = sample_ic.random_coords_int(len(seq)-3).to(device)
        # protein.update_coords_internal(random_coords_int)
        # extend_coords_int = torch.tensor([[5.367, 1.6, 0.0]], device=device).repeat((len(seq)-3, 1))
        extend_coords_int = torch.tensor([[5.367, 0.1, 0.0]], device=device).repeat((len(seq)-3, 1))
        protein.update_coords_internal(extend_coords_int)
        protein.update_cartesian_from_internal()
    coords_init = protein.coords

    energy_init = protein.get_energy(energy_fn).item()
    print('energy_init:', energy_init)
    # write_pdb(seq, coords_init, pdb_id, f'init_{mode}', exp_id)

    if fold_engine == 'anneal':
        # simulated annealing
        torch.set_grad_enabled(False)
        if args.anneal_type == 'int_one':
            annealer = AnnealCoords(energy_fn, protein, mode=mode, ic_move_std=args.ic_move_std,
                                    T_max=args.T_max, T_min=args.T_min, L=args.L)
        elif args.anneal_type == 'frag':
            frag_file = h5py.File(f'data/fragment/{pdb_id}/{pdb_id}_int.h5', 'r')
            query_pos = torch.tensor(frag_file['query_pos'][()], device=device)
            frag_int = torch.tensor(frag_file['coords_int'][()], device=device)
            annealer = AnnealFrag(energy_fn, protein, frag=(query_pos, frag_int), use_rg=args.use_rg,
                                  T_max=args.T_max, T_min=args.T_min, L=args.L)
        else:
            raise ValueError('anneal_type should be int_one / frag.')
        annealer.run()
        coords_best = annealer.x_best
        energy_best = annealer.energy_best
        sample = annealer.sample
        sample_energy = annealer.sample_energy
    elif fold_engine == 'grad':
        if args.x_type == 'cart':
            minimizer = GradMinimizerCartesian(energy_fn, protein, lr=args.lr, num_steps=args.L)
        elif args.x_type == 'internal':
            minimizer = GradMinimizerInternal(energy_fn, protein, lr=args.lr, num_steps=args.L, momentum=0.0)
        elif args.x_type == 'int_fast':
            minimizer = GradMinimizerIntFast(energy_fn, protein, lr=args.lr, num_steps=args.L)
        elif args.x_type == 'mixed':
            minimizer = GradMinimizerMixed(energy_fn, protein, lr=args.lr, num_steps=args.L)
        elif args.x_type == 'mix_fast':
            minimizer = GradMinimizerMixFast(energy_fn, protein, lr=args.lr, num_steps=args.L)
        else:
            raise ValueError('x_type should be cart / internal / mixed / int_fast / mix_fast.')

        minimizer.run()
        coords_best = minimizer.x_best
        energy_best = minimizer.energy_best
        sample = minimizer.sample
        sample_energy = minimizer.sample_energy
    elif fold_engine == 'dynamics':
        if args.x_type == 'cart':
            minimizer = Dynamics(energy_fn, protein, num_steps=args.L, lr=args.lr, t_noise=args.T_max)
        elif args.x_type == 'internal':
            minimizer = DynamicsInternal(energy_fn, protein, num_steps=args.L, lr=args.lr, t_noise=args.T_max)
        elif args.x_type == 'int_fast':
            minimizer = DynamicsIntFast(energy_fn, protein, num_steps=args.L, lr=args.lr, t_noise=args.T_max)
        elif args.x_type == 'mixed':
            minimizer = DynamicsMixed(energy_fn, protein, num_steps=args.L, lr=args.lr, t_noise=args.T_max)
        elif args.x_type == 'mix_fast':
            minimizer = DynamicsMixFast(energy_fn, protein, num_steps=args.L, lr=args.lr, t_noise=args.T_max)
        else:
            raise ValueError('x_type should be cart / internal / mixed / int_fast / mix_fast.')

        minimizer.run()
        coords_best = minimizer.x_best
        energy_best = minimizer.energy_best
        sample = minimizer.sample
        sample_energy = minimizer.sample_energy
    else:
        raise ValueError('fold_engine should be anneal / grad / dynamics')

    # protein.update_coords(coords_best)
    # residue_energy = protein.get_residue_energy(energy_fn)
    # print(residue_energy)
    # write_pdb(seq, coords_best, pdb_id, f'best_{mode}', exp_id)

    # save sampled structures
    sample = [coords_native.cpu(), coords_best.cpu(), coords_init.cpu()] + sample
    sample_energy = [energy_native, energy_best, energy_init] + sample_energy
    # write_pdb_sample(seq, sample, pdb_id, 'sample', exp_id)
    # pd.DataFrame({'sample_energy': sample_energy}).to_csv(f'data/fold/{exp_id}/{pdb_id}_energy.csv', index=False)
    write_pdb_sample(seq, sample, pdb_id, 'sample', save_dir)

    # compute RMSD,
    sample_xyz = torch.stack(sample, 0).cpu().detach().numpy()
    print(sample_xyz.shape)
    t = md.Trajectory(xyz=sample_xyz, topology=None)
    t = t.superpose(t, frame=0)
    write_pdb_sample(seq, t.xyz, pdb_id, 'sample2', save_dir)
    sample_rmsd = md.rmsd(t, t, frame=0)  # computation will change sample_xyz;

    print(f'best RMSD: {sample_rmsd[1]}')
    df = pd.DataFrame({'sample_energy': sample_energy,
                       'sample_rmsd': sample_rmsd})
    df.to_csv(f'data/fold/{save_dir}/{pdb_id}_energy.csv', index=False)

max_stars_repo_name: lahplover/nnef

max_issues_repo_name: lahplover/nnef

max_forks_repo_name: lahplover/nnef


text:
import sys

import streamlit as st
import matplotlib.pyplot as plt
import numpy as np
from dpu_utils.utils import RichPath

from data.edits import Edit
from dpu_utils.ptutils import BaseComponent

'''
# Copy Span Visualization
'''
model_path = sys.argv[1]

@st.cache
def get_model(filename):
    path = RichPath.create(filename)
    model = BaseComponent.restore_model(path, device='cpu')
    model.eval()

    return model

st.markdown(f'> Using model from {model_path}')
before_tokens = st.text_area('Input (space) tokenized before version.').strip().split()
after_tokens = st.text_area('Input (space) tokenized after version.').strip().split()

'''
#### Input Data
'''
edit = Edit(input_sequence=before_tokens, output_sequence=after_tokens, provenance='', edit_type='')
st.write(edit)

model = get_model(model_path)

tensorized_data = [model.load_data_from_sample(edit)]
mb_data, is_full, num_elements = model.create_minibatch(tensorized_data, max_num_items=10)
assert num_elements == 1

ground_input_sequence = [edit.input_sequence]
predicted_outputs = model.beam_decode(input_sequences=mb_data['input_sequences'],
                                                        ground_input_sequences=ground_input_sequence)[0]

ll, debug_info = model.compute_likelihood(**mb_data, return_debug_info=True)
ll = ll.cpu().numpy()
st.markdown(f' > Likelihood of target edit {ll[0]:.2f}')

copy_span_logprobs = debug_info['copy_span_logprobs'][0]
gen_logprobs = debug_info['generation_logprobs'][0]
vocabulary = debug_info['vocabulary']
before_tokens = ['<s>'] + before_tokens + ['</s>']
after_tokens = after_tokens + ['</s>']

for i in range(copy_span_logprobs.shape[0]):
    st.markdown(f'### At position {i}: "{after_tokens[i]}"')
    st.markdown(f'Current context `{["<s>"] + after_tokens[:i]}`')
    plt.figure(figsize=[1, 1])
    current_copy_span_probs = np.exp(copy_span_logprobs[i])
    plt.matshow(current_copy_span_probs, cmap='Greys')
    plt.xticks(range(copy_span_logprobs.shape[1]), before_tokens, fontsize=8, rotation=90)
    plt.xlabel('Start Span Pos')
    plt.yticks(range(copy_span_logprobs.shape[2]), before_tokens, fontsize=8, rotation=0)
    plt.ylabel('End Span Pos')
    plt.colorbar()
    st.pyplot()

    max_idx = np.argmax(current_copy_span_probs)
    from_idx, to_idx = max_idx // current_copy_span_probs.shape[1], max_idx % current_copy_span_probs.shape[1],
    st.markdown(f'* Best copy suggestion: `Copy({from_idx}:{to_idx+1})` with prob {np.max(current_copy_span_probs)*100:.1f}%, _i.e._ `Copy({before_tokens[from_idx: to_idx+1]})`.')
    st.markdown(f'* Best generation suggestion: `Gen("{vocabulary.get_name_for_id(np.argmax(gen_logprobs[i]))}")` with prob {np.exp(np.max(gen_logprobs[i]))*100:.1f}%')

'''### Beam decoding results '''
for i, (prediction, logprob) in enumerate(zip(predicted_outputs[0], predicted_outputs[1])):
    if i > 2:
        break
    st.markdown(f'* {" ".join(prediction)} ({np.exp(logprob)*100:.1f}%)')


text:
#!/usr/bin/env python
# encoding: utf-8

"""
@Author: yangwenhao
@Contact: 874681044@qq.com
@Software: PyCharm
@File: output_visual_script.py
@Time: 2020/3/21 10:43 PM
@Overview:
"""
import argparse
import os
import pathlib
import pickle

import matplotlib.pyplot as plt
import numpy as np
from matplotlib import animation

parser = argparse.ArgumentParser(description='PyTorch Speaker Recognition')
# Model options
parser.add_argument('--train-dir', type=str,
                    default='/home/yangwenhao/local/project/lstm_speaker_verification/data/Vox1_aug_spect/dev_org',
                    help='path to dataset')
parser.add_argument('--test-dir', type=str,
                    default='/home/yangwenhao/local/project/lstm_speaker_verification/data/Vox1_aug_spect/test',
                    help='path to voxceleb1 test dataset')
parser.add_argument('--sitw-dir', type=str,
                    default='/home/yangwenhao/local/project/lstm_speaker_verification/data/sitw_spect',
                    help='path to voxceleb1 test dataset')

parser.add_argument('--check-path', default='Data/checkpoint/SuResCNN10/spect/aug',
                    help='folder to output model checkpoints')
parser.add_argument('--extract-path', default='Lime/SuResCNN10/data',
                    help='folder to output model checkpoints')

# Training options
parser.add_argument('--cos-sim', action='store_true', default=True,
                    help='using Cosine similarity')
parser.add_argument('--embedding-size', type=int, default=1024, metavar='ES',
                    help='Dimensionality of the embedding')
parser.add_argument('--sample-utt', type=int, default=120, metavar='ES',
                    help='Dimensionality of the embedding')
parser.add_argument('--batch-size', type=int, default=1, metavar='BS',
                    help='input batch size for training (default: 128)')
parser.add_argument('--test-batch-size', type=int, default=1, metavar='BST',
                    help='input batch size for testing (default: 64)')
parser.add_argument('--input-per-spks', type=int, default=192, metavar='IPFT',
                    help='input sample per file for testing (default: 8)')
parser.add_argument('--test-input-per-file', type=int, default=1, metavar='IPFT',
                    help='input sample per file for testing (default: 8)')

args = parser.parse_args()
cValue_1 = ['#15b01a', '#ff796c', '#7e1e9c', '#ff81c0', '#96f97b', '#e50000', '#ffff14', '#653700', '#95d0fc',
            '#029386', '#f97306',
            '#c20078', '#75bbfd', '#929591', '#89fe05', '#bf77f6', '#9a0eea', '#033500', '#06c2ac', '#c79fef',
            '#00035b', '#d1b26f', '#00ffff', '#13eac9', '#06470c', '#ae7181', '#35063e', '#01ff07', '#650021',
            '#6e750e', '#e6daa6', '#0504aa', '#001146', '#cea2fd', '#000000', '#ff028d', '#ad8150', '#c7fdb5',
            '#ffb07c', '#677a04', '#cb416b', '#8e82fe', '#53fca1', '#aaff32', '#380282', '#ceb301', '#ffd1df',
            '#cf6275', '#0165fc', '#0cff0c', '#c04e01', '#04d8b2', '#01153e', '#3f9b0b', '#d0fefe', '#840000',
            '#be03fd', '#c0fb2d', '#a2cffe', '#dbb40c', '#8fff9f', '#580f41', '#4b006e', '#8f1402', '#014d4e',
            '#610023', '#aaa662', '#137e6d', '#7af9ab', '#02ab2e', '#9aae07', '#8eab12', '#b9a281', '#341c02',
            '#36013f', '#c1f80a', '#fe01b1', '#fdaa48', '#9ffeb0', '#b0ff9d', '#e2ca76', '#c65102', '#a9f971',
            '#a57e52', '#80f9ad', '#6b8ba4', '#4b5d16', '#363737', '#d5b60a', '#fac205', '#516572', '#0343df',
            '#90e4c1', '#a83c09', '#040273', '#ffcfdc', '#0485d1', '#ff474c', '#d2bd0a', '#bf9005', '#ffff84',
            '#8c000f', '#ed0dd9', '#0b4008', '#607c8e', '#5b7c99', '#b790d4', '#047495', '#d648d7', '#a5a502',
            '#d8dcd6', '#5ca904', '#fffe7a', '#380835', '#5a7d9a', '#658b38', '#98eff9', '#ffffff', '#789b73',
            '#87ae73', '#a03623', '#b04e0f', '#7f2b0a', '#ffffc2', '#fc5a50', '#03719c', '#40a368', '#960056',
            '#fd3c06', '#703be7', '#020035', '#d6b4fc', '#c0737a', '#2c6fbb', '#cdfd02', '#b0dd16', '#601ef9',
            '#5e819d', '#6c3461', '#acbf69', '#5170d7', '#f10c45', '#ff000d', '#069af3', '#5729ce', '#045c5a',
            '#0652ff', '#ffffe4', '#b1d1fc', '#80013f', '#74a662', '#76cd26', '#7ef4cc', '#bc13fe', '#1e488f',
            '#d46a7e', '#6f7632', '#0a888a', '#632de9', '#34013f', '#856798', '#154406', '#a2a415', '#ffa756',
            '#0b8b87', '#af884a', '#06b48b', '#10a674', '#a2bffe', '#769958', '#5cac2d', '#cb0162', '#980002',
            '#88b378', '#02d8e9', '#ca6641', '#caa0ff', '#a9561e', '#373e02', '#c9ff27', '#be0119', '#82a67d',
            '#3d1c02', '#5d06e9', '#6a79f7', '#ffb7ce', '#343837', '#0a481e', '#e17701', '#696112', '#8b2e16',
            '#6a6e09', '#ff9408', '#fe7b7c', '#12e193', '#b00149', '#887191', '#f7879a', '#fe019a', '#030aa7',
            '#be6400', '#9a0200', '#fd411e', '#cdc50a']
marker = ['.', '*']

def main():
    # conv1s = np.array([]).reshape((0, 64, 5, 5))
    # grads = np.array([]).reshape((0, 2, 161))
    model_set = ['kaldi_5wd', 'aug']
    epochs = np.arange(0, 31)

    if os.path.exists(args.extract_path + '/conv1s_means.npy'):
        conv1s_means = np.load(args.extract_path + '/conv1s_means.npy')
        conv1s_std = np.load(args.extract_path + '/conv1s_std.npy')
        input_grads = np.load(args.extract_path + '/input_grads.npy')
    else:
        conv1s_means = []
        conv1s_std = []
        input_grads = []

        for m in model_set:
            extract_paths = os.path.join(args.extract_path, m)
            conv1s = np.array([]).reshape((0, 64, 5, 5))
            grads = np.array([]).reshape((0, 2, 161))
            print('\nProcessing data in %s.' % extract_paths)

            for i in epochs:
                save_path = pathlib.Path(extract_paths + '/epoch_%d' % i)

                if not save_path.exists():
                    # pdb.set_trace()
                    print(str(save_path) + ' ERROR!')
                    continue

                print('\rReading: ' + str(save_path), end='')
                # pdb.set_trace()
                grads_abs = np.array([]).reshape((0, 161))

                for name in ['train', 'valid']:
                    sets_files = list(save_path.glob('vox1_%s.*.bin' % name))
                    grad_abs = np.zeros((161))
                    num_utt = 0
                    for f in sets_files:
                        with open(str(f), 'rb') as f:
                            sets = pickle.load(f)
                            for (uid, orig, conv1, bn1, relu1, grad) in sets:
                                grad_abs += np.mean(np.abs(grad), axis=0)
                                num_utt += 1
                    grads_abs = np.concatenate((grads_abs, grad_abs[np.newaxis, :] / num_utt), axis=0)

                grads_abs = grads_abs[np.newaxis, :]
                grads = np.concatenate((grads, grads_abs), axis=0)

                cs = list(save_path.glob('model.conv1.npy'))

                conv1_epoch = np.load(str(cs[0])).squeeze()
                conv1_epoch = conv1_epoch[np.newaxis, :]
                # pdb.set_trace()
                conv1s = np.concatenate((conv1s, conv1_epoch), axis=0)

            means = np.mean(np.abs(conv1s), axis=(2, 3))
            stds = np.std(conv1s, axis=(2, 3))

            conv1s_means.append(means)
            conv1s_std.append(stds)
            input_grads.append(grads)

        for x in conv1s_means, conv1s_std:
            while x[0].shape[0] < x[1].shape[0]:
                x[0] = np.concatenate((x[0], x[0][-1, :].reshape(1, x[0].shape[1])), axis=0)

        while input_grads[0].shape[0] < input_grads[1].shape[0]:
            input_grads[0] = np.concatenate(
                (input_grads[0], input_grads[0][-1, :, :].reshape(1, input_grads[0].shape[1], input_grads[0].shape[2])),
                axis=0)

        conv1s_means = np.array(conv1s_means)  # [[2,21,64]; [2,30,64]]
        conv1s_std = np.array(conv1s_std)  # 2,21,64
        input_grads = np.array(input_grads)  # 2,21,64

        np.save(args.extract_path + '/conv1s_means.npy', conv1s_means)
        np.save(args.extract_path + '/conv1s_std.npy', conv1s_std)
        np.save(args.extract_path + '/input_grads.npy', input_grads)

    # plotting filters distributions
    # fig = plt.figure(figsize=(10, 10))
    # plt.title('Convergence of 16 Filters')
    # # pdb.set_trace()
    #
    # max_x = max(np.max(conv1s_means[0][1]), np.max(conv1s_means[1][1]))
    # min_x = min(np.min(conv1s_means[0][1]), np.min(conv1s_means[1][1]))
    # max_y = max(np.max(conv1s_std[0][1]), np.max(conv1s_std[1][1]))
    # min_y = min(np.min(conv1s_std[0][1]), np.min(conv1s_std[1][1]))
    #
    # plt.xlim(min_x - 0.15 * np.abs(max_x), max_x + 0.15 * np.abs(max_x))
    # plt.ylim(min_y - 0.15 * np.abs(max_y), max_y + 0.15 * np.abs(max_y))
    # plt.xlabel('Means of Abs')
    # plt.ylabel('Std')

    # fig, ax = plt.subplots()
    # means_shape = conv1s_means.shape  # 2,21,16
    # set_dots = []
    # text_e = plt.text(max_x, max_y, 'Epoch 0')
    #
    # for i in range(means_shape[0]):  # aug, kaldi
    #     dots = []
    #     for j in range(means_shape[2]):
    #         dot_x = conv1s_means[i][0][j]
    #         dot_y = conv1s_std[i][0][i]
    #
    #         dot, = plt.plot(dot_x, dot_y, color=cValue_1[j], marker=marker[i])
    #         text_p = plt.text(dot_x, dot_y, '%d' % j)
    #         dots.append([dot, text_p])
    #     set_dots.append(dots)
    #
    # plt.legend([set_dots[0][0][0], set_dots[1][0][0]], ['aug', 'kaldi'], loc='lower right', scatterpoints=1)
    #
    # def gen_dot():
    #     for i in range(means_shape[1]):
    #         text_e.set_text('Epoch %2s' % str(i))
    #         newdot = [conv1s_means[:, i], conv1s_std[:, i]]
    #         yield newdot
    #
    # def update_dot(newd):
    #     # pdb.set_trace()
    #     for i in range(means_shape[0]):
    #         dots = set_dots[i]
    #         for j in range(means_shape[2]):
    #             dot, text_p = dots[j]
    #
    #             dot_x = newd[0][i][j]
    #             dot_y = newd[1][i][j]
    #
    #             dot.set_data(dot_x, dot_y)
    #             text_p.set_position((dot_x, dot_y))
    #
    #     return set_dots
    #
    # ani = animation.FuncAnimation(fig, update_dot, frames=gen_dot, interval=800)
    # ani.save(args.extract_path + "/conv1s.gif", writer='pillow', fps=2)
    # print('Saving %s' % args.extract_path + "/conv1s.gif")

    fig = plt.figure(figsize=(10, 8))
    plt.title('Filting over 8000Hz')
    plt.xlabel('Frequency')
    plt.ylabel('Weight')

    x = np.arange(161) * 8000 / 161  # [0-8000]
    y = np.nan_to_num(input_grads)  # 2, 21, 2, 161
    # pdb.set_trace()
    max_x = np.max(x)
    min_x = np.min(x)
    max_y = np.max(y[0][0][0] / y[0][0][0].sum())
    min_y = np.min(y)
    plt.xlim(min_x - 0.15 * np.abs(max_x), max_x + 0.15 * np.abs(max_x))
    plt.ylim(min_y, max_y + 0.15 * np.abs(max_y))
    # pdb.set_trace()
    # print(y.shape)
    text_e = plt.text(min_x, max_y, 'Epoch 0')
    y_shape = y.shape  # 2, 21, 2, 161
    set_dots = []

    for j in range(y_shape[0]):  # aug and kaldi
        dots = []
        for h in range(y_shape[2]):  # train and valid
            # dot, = plt.plot(x, y[j][0][h] / y[j][0][h].sum(), marker=marker[j], color=cValue_1[j + h * 4])
            # dot, = plt.plot(x, y[j][0][h] / y[j][0][h].sum(), marker=marker[j], alpha=0.8)
            dot, = plt.plot(x, y[j][0][h] / y[j][0][h].sum(), alpha=0.7)
            dots.append(dot)

        set_dots.append(dots)

    plt.legend([set_dots[0][0], set_dots[0][1], set_dots[1][0], set_dots[1][1]],
               ['kaldi_Train', 'kaldi_Valid', 'aug_Train', 'aug_Valid'], loc='upper right')

    def gen_line():
        for i in range(1, y_shape[1]):
            newdot = [x, y[:, i]]  # 2,2,161
            text_e.set_text('Epoch %2s' % str(i))

            yield newdot

    def update_line(newd):
        for i in range(y_shape[0]):
            dots = set_dots[i]
            for j in range(y_shape[2]):
                dots[j].set_data(newd[0], newd[1][i][j] / newd[1][i][j].sum())

        return set_dots

    ani = animation.FuncAnimation(fig, update_line, frames=gen_line, interval=800)
    ani.save(args.extract_path + "/grads.gif", writer='pillow', fps=2)

if __name__ == '__main__':
    main()

max_stars_repo_name: Wenhao-Yang/DeepSpeaker-pytorch

max_issues_repo_name: Wenhao-Yang/DeepSpeaker-pytorch

max_forks_repo_name: Wenhao-Yang/DeepSpeaker-pytorch


text:
"""
Predict labels with a model.
"""
import logging
import zipfile
from os import makedirs, walk
from os.path import exists, join, basename
from shutil import rmtree

import jams
import numpy as np
from sklearn.externals.joblib import dump, load
from tensorflow.python.keras import backend as K

def predict_from_models(features, input_shape, model_loaders, normalizer, ground_truth, job_dir='./job',
                        write_jams=True, overwrite=False):
    """
    Create predictions for a test or validation dataset (i.e., a ground truth) for
    multiple models (passed via model loaders).

    :param job_dir:
    :param overwrite: overwrite prediction
    :param features: dict with id :: features
    :param input_shape: network input shape
    :param log: log function
    :param model_loaders: dict with model names as keys and lists of model loaders as values (different runs)
    :param normalizer: normalization function
    :param ground_truth: ground truth to predict for
    :return: a dict for each model kind with lists of results for multiple runs
    """

    sorted_names = list(model_loaders.keys())
    sorted_names.sort()

    dataset_description = ground_truth.name
    logging.info('Predicting for dataset \'{}\'.'.format(dataset_description))

    results = {}
    for model_name in sorted_names:
        same_kind_models = model_loaders[model_name]
        same_kind_results = results[model_name] = []

        for run, model_loader in enumerate(same_kind_models):
            file = prediction_file(model_loader, dataset_description)
            if overwrite or not exists(file):
                model = model_loader.load()
                if write_jams:
                    jams_dir = join(job_dir, 'jams/jams_' + dataset_description + '_'
                                    + basename(model_loader.file).replace('.h5', ''))
                else:
                    jams_dir = None
                windowed = True
                predictions = global_predict(model, input_shape, windowed, ground_truth, features, normalizer, jams_dir)
                dump_predictions(predictions, model_loader, dataset_description)
                # don't keep all models in memory
                del model
                K.clear_session()
            else:
                logging.info('{}. run. Predictions exist. Loading from file {}'.format(run, file))
                predictions = load_predictions(model_loader, dataset_description)
            same_kind_results.append(predictions)
    return results

def prediction_file(model_loader, testset_description):
    """
    Standardized name for a prediction.

    :param model_loader: model loader
    :param testset_description: test set description
    :return: file name for predictions
    """
    return model_loader.file.replace('.h5', '_pred_{}.joblib'.format(testset_description))

def load_predictions(model_loader, testset_description):
    """
    Load predictions for a model and a test set.

    :param model_loader: model loader
    :param testset_description: test set description
    :return: predictions
    """
    return load(prediction_file(model_loader, testset_description))

def dump_predictions(predictions, model_loader, testset_description):
    """
    Store predictions for a model and a test set.

    :param predictions: predictions
    :param model_loader: model loader
    :param testset_description: test set description
    """
    dump(predictions, prediction_file(model_loader, testset_description))

def global_predict(model, input_shape, windowed, ground_truth, features, normalizer, jams_dir=None):
    """
    Predict global tempo values using the given model, spectrogram input shape, ground truth,
    features and normalizer function.

    :param model: model
    :param input_shape: single spectrogram shape
    :param windowed: if ``True``, predict for multiple windows
    :param ground_truth: ground truth necessary to convert indices to actual labels
    :param features: dict, mapping keys to spectrograms
    :param normalizer: function that takes a numpy array and normalizes it (needed before prediction)
    :param jams_dir: if not None, dump predictions as jams zip file into the given directory
    :return: dict, mapping keys to predicted labels
    """
    results = {}
    model_input_length = input_shape[1]
    for key in ground_truth.labels.keys():
        if key not in features:
            logging.warning('Prediction for dataset \'{}\': Spectrogram not found. Id=\'{}\''
                            .format(ground_truth.name, key))
            continue

        # make sure we don't modify the original!
        spectrogram = np.copy(features[key])

        if windowed:
            length = spectrogram.shape[1]
            samples = []

            # half overlapping
            hop_size = input_shape[1] // 2
            pos = 0
            while pos + model_input_length <= length:
                sample = spectrogram[:, pos:pos + model_input_length, :]
                if normalizer is not None:
                    sample = normalizer(sample)
                samples.append(np.reshape(sample, (1, *input_shape)))
                pos += hop_size
            X = np.vstack(samples)
        else:
            # this assumes that we can predict spectrograms of arbitrary lengths (dim=1)
            if normalizer is not None:
                spectrogram = normalizer(spectrogram)
            X = np.expand_dims(spectrogram, axis=0)

        predictions = model.predict(X, X.shape[0])
        predictions = np.sum(predictions, axis=0)
        discrete_label = ground_truth.get_label(np.argmax(predictions))
        interpolated_label = ground_truth.get_label(quad_interpol_argmax(predictions))

        results[key] = discrete_label, interpolated_label, predictions

        # dump interpolated labels as JAMS
        if jams_dir is not None:
            create_jam_file(jams_dir, key, ground_truth.name, interpolated_label)

    if jams_dir is not None:
        def zipdir(path, zip_handle):
            for root, dirs, files in walk(path):
                for file in files:
                    zip_handle.write(join(root, file), file)

        if exists(jams_dir):
            zip_file = zipfile.ZipFile(jams_dir + '.zip', 'w', zipfile.ZIP_DEFLATED)
            zipdir(jams_dir, zip_file)
            rmtree(jams_dir, ignore_errors=True)
            logging.info('Exported predictions as jams to {}'.format(zip_file))
        else:
            logging.warning('Failed to create jams zip file. Jams dir does not exist: {}'.format(jams_dir))

    return results

def quad_interpol_argmax(y):
    """
    Find argmax for quadratic interpolation around argmax of y.

    :param y: array
    :return: float (index) of interpolated max
    """
    x = np.argmax(y)
    if x == 0 or x == y.shape[0]-1:
        return x
    z = np.polyfit([x - 1, x, x + 1], [y[x - 1], y[x], y[x + 1]], 2)
    # find (float) x value for max
    argmax = -z[1] / (2. * z[0])
    return argmax

def create_jam_file(jams_dir, key, corpus, estimate):
    """
    Create a jam file for the given key, corpus, and estimate.

    :param jams_dir: dir
    :param key: file id
    :param corpus: corpus (e.g. Ballroom)
    :param estimate: tempo estimate
    """
    makedirs(jams_dir, exist_ok=True)
    if estimate < 0:
        logging.warning('Attempting to store tempo estimate for {} that is less than 0: {}'.format(key, estimate))
        estimate = 0
    tempo = jams.Annotation(namespace='tempo')
    tempo.append(time=0.0,
                 duration='nan',
                 value=estimate,
                 confidence=1.0)
    tempo.annotation_metadata = jams.AnnotationMetadata(
        corpus=corpus,
        version='1.0',
        data_source='')
    jam = jams.JAMS()
    jam.annotations.append(tempo)
    jam.file_metadata.duration = 5000  # bogus value to please JAMS
    jam.save(join(jams_dir, '{}.jams'.format(key)))


text:
"""
"""
import numpy as np
from ..ellipticity_evolution import ellipticity_vs_time
from ..ellipticity_evolution import PARAM_BOUNDS
from ..fit_ellipticity_helpers import fit_ellipticity, get_loss_data

SEED = 32

def test_ellipticity_fitter():
    """Pick a random point in parameter space and demonstrate that the fitter
    recovers the correct result.
    """
    t_sim = np.linspace(0.1, 14, 100)
    lo = [b[0] for b in PARAM_BOUNDS.values()]
    hi = [b[1] for b in PARAM_BOUNDS.values()]
    rng = np.random.RandomState(SEED)
    p_target = rng.uniform(lo, hi)
    e_sim = ellipticity_vs_time(t_sim, *p_target)
    log_mah_sim = np.zeros_like(e_sim) + 100
    lgm_min = 0
    u_p0, _loss_data = get_loss_data(t_sim, e_sim, log_mah_sim, lgm_min)
    res = fit_ellipticity(t_sim, e_sim, log_mah_sim, lgm_min)
    p_best, loss, method, loss_data = res
    e_best = ellipticity_vs_time(t_sim, *p_best)
    assert np.allclose(e_best, e_sim, atol=0.01)
    assert np.allclose(p_best, p_target, atol=0.01)

    # Enforce that the returned loss_data contains the expected information
    for a, b in zip(_loss_data, loss_data):
        assert np.allclose(a, b)

max_stars_repo_name: BaryonPasters/diffprof

max_issues_repo_name: BaryonPasters/diffprof

max_forks_repo_name: BaryonPasters/diffprof


text:
"""Print a table of how accurate IAU 2000B is with fewer coefficients."""

import matplotlib.pyplot as plt
from numpy import sqrt
from skyfield.api import load
from skyfield.nutationlib import iau2000a, iau2000b

ts = load.timescale()
#t = ts.utc(2000, 1, range(366 * 50))
t = ts.utc(1995, 1, range(366 * (2050 - 1995)))
print(t[0].utc_jpl())
print(t[-1].utc_jpl())

dpsi_a, depsilon_a = iau2000a(t.tt)

for i in list(range(0, 10)) + list(range(10, 71, 10)) + [77]:
    dpsi_b, depsilon_b = iau2000b(t.tt, i)
    diff1 = abs(dpsi_b - dpsi_a).max()
    diff2 = abs(depsilon_b - depsilon_a).max()
    diff_mas = sqrt(diff1 * diff1 + diff2 * diff2) * 1e-4
    if diff_mas > 1000:
        print(i, '%.3f' % (diff_mas / 1e3), 'arcseconds')
    elif diff_mas > 100:
        print(i, int(diff_mas + 0.999), 'mas')
    elif diff_mas > 10:
        print(i, '%.1f' % diff_mas, 'mas')
    else:
        print(i, '%.2f' % diff_mas, 'mas')

max_stars_repo_name: facorazza/python-skyfield

max_issues_repo_name: facorazza/python-skyfield

max_forks_repo_name: facorazza/python-skyfield

reason: from numpy


text:
############################
# turning off the warnings #

import warnings
warnings.filterwarnings('ignore')

import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

import tensorflow.python.util.deprecation as deprecation
deprecation._PRINT_DEPRECATION_WARNINGS = False

#####################
# importing modules #

import os
import numpy as np
import tensorflow as tf

def select_indices(num_items, num_to_select, random_seed=None):
    np.random.seed(random_seed)
    return np.random.choice(range(num_items), num_to_select)

# Both 'items' and 'indices' must be numpy arrays.
def select_items(items, indices):
    return items[indices]

CIFAR10_CLASSES = np.array(['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'])

CIFAR100_CLASSES = ['apple', 'aquarium_fish', 'baby', 'bear', 'beaver', 'bed', 'bee', 'beetle',
'bicycle', 'bottle', 'bowl', 'boy', 'bridge', 'bus', 'butterfly', 'camel',
'can', 'castle', 'caterpillar', 'cattle', 'chair', 'chimpanzee', 'clock',
'cloud', 'cockroach', 'couch', 'crab', 'crocodile', 'cup', 'dinosaur',
'dolphin', 'elephant', 'flatfish', 'forest', 'fox', 'girl', 'hamster',
'house', 'kangaroo', 'keyboard', 'lamp', 'lawn_mower', 'leopard', 'lion',
'lizard', 'lobster', 'man', 'maple_tree', 'motorcycle', 'mountain', 'mouse',
'mushroom', 'oak_tree', 'orange', 'orchid', 'otter', 'palm_tree', 'pear',
'pickup_truck', 'pine_tree', 'plain', 'plate', 'poppy', 'porcupine',
'possum', 'rabbit', 'raccoon', 'ray', 'road', 'rocket', 'rose',
'sea', 'seal', 'shark', 'shrew', 'skunk', 'skyscraper', 'snail', 'snake',
'spider', 'squirrel', 'streetcar', 'sunflower', 'sweet_pepper', 'table',
'tank', 'telephone', 'television', 'tiger', 'tractor', 'train', 'trout',
'tulip', 'turtle', 'wardrobe', 'whale', 'willow_tree', 'wolf', 'woman', 'worm']

def to_classes(y, dataset_name):
    if dataset_name == 'CIFAR10':
        return CIFAR10_CLASSES[np.argmax(y, axis=-1)]
    elif dataset_name == 'CIFAR100':
        return CIFAR100_CLASSES[np.argmax(y, axis=-1)]

def show_structure(model, file_name=None):
    model.summary()
    file_name = file_name.split('.')[0]
    file_name += '.png'
    tf.keras.utils.plot_model(model, to_file=file_name, show_shapes=True, show_layer_names=True)

def create_directory(path):
    try:
        if not os.path.exists(path):
            os.makedirs(path)
    except OSError:
        print('Failed to create directory:', path)

def get_bounding_box_2D(points):
    xMin, xMax = min(points[:, 0]), max(points[:, 0])
    yMin, yMax = min(points[:, 1]), max(points[:, 1])
    return xMin, xMax, yMin, yMax

def generate_random_points(num_points, xMin, xMax, yMin, yMax):
    x = np.random.uniform(xMin, xMax, size=num_points)
    y = np.random.uniform(yMin, yMax, size=num_points)
    return np.array(list(zip(x, y)))

def generate_uniform_grid_points(nx, ny, xMin, xMax, yMin, yMax):
    x = np.linspace(xMin, xMax, nx)
    y = np.linspace(yMin, yMax, ny)
    xv, yv = np.meshgrid(x, y)
    xv = xv.flatten()
    yv = yv.flatten()
    return np.array(list(zip(xv, yv)))

max_stars_repo_name: zelosdev/TensorflowKerasTutorial

max_issues_repo_name: zelosdev/TensorflowKerasTutorial

max_forks_repo_name: zelosdev/TensorflowKerasTutorial


text:
# Script to generate testing GMM data
import sys
import numpy as np
from sklearn import mixture

if len(sys.argv) != 5:
    print "Usage: python scripts/generateData.py <numDims> <numClusters> <numPoints> <numDataPacks>"
    sys.exit(-1)

version = 1		# version of the datapack

numDims = int(sys.argv[1])	# number of dimensions
numClusters = int(sys.argv[2])	# number of clusters
numPoints = int(sys.argv[3])	# per cluster points
numDataPacks = int(sys.argv[4])	# number of datapacks

# Generate the cluster proportion, means and variance first, try not to use Gaussians!!!
print "Generating cluster parameters"
pis = (1./numClusters) * np.ones(numClusters)  # currently fixed
means = 200 * np.random.rand(numClusters, numDims) - 100
# Variable stds
#stds =  2 * abs(np.random.rand(numClusters, numDims))
# Fixed stds
stds = 64 * np.ones((numClusters, numDims)) # abs(np.random.rand(numClusters, numDims))
variances = np.square(stds)

# writing down the ground truth
with open('data/synthetic_%d_%d_%d-gt.dat' % (numDims, numClusters, numPoints), 'w') as filePt:
    # Headers (version, number of clusters and number of dimensions)
    np.array(version, dtype='int32').tofile(filePt)
    np.array(numDims, dtype='int32').tofile(filePt)
    np.array(numClusters, dtype='int32').tofile(filePt)

    # now write the pis, means, vars
    pis.tofile(filePt)
    means.tofile(filePt)
    variances.tofile(filePt)

# Train data generation
for j in xrange(numDataPacks):
    print "Generating data pack %d" % j
    train = []
    for i in xrange(numClusters):
        train.append( means[i] + stds[i] * np.random.randn(numPoints, numDims) )

    train = np.vstack(train)

    # shuffle the points
    np.random.shuffle(train)

    # just a hack: first point from unique clusters
    # for i in xrange(numClusters):
    #     train[i] =  means[i] #+ 2 * np.random.randn(numDims)

    # Open a file and write the points
    with open('data/synthetic_%d_%d_%d-%d.dat' % (numDims, numClusters, numPoints, j), 'w') as filePt:
        # Headers (version, number of dimensions and number of points)
        np.array(version, dtype='int32').tofile(filePt)
        np.array(numDims, dtype='int32').tofile(filePt)
        np.array(numPoints * numClusters, dtype='int32').tofile(filePt)

        # now write the points
        train.tofile(filePt);

# Generate test data
print "Generating test data set"
numTest = int(numPoints)#Train-test split
test = []
for i in xrange(numClusters):
    test.append(  means[i] + stds[i] * np.random.randn(numTest, numDims) );
test = np.vstack(test)

# Open a file and write the points
with open('data/synthetic_%d_%d_%d-test.dat' % (numDims, numClusters, numPoints), 'w') as filePt:
    # Headers (version, number of dimensions and number of points)
    np.array(version, dtype='int32').tofile(filePt)
    np.array(numDims, dtype='int32').tofile(filePt)
    np.array(numTest * numClusters, dtype='int32').tofile(filePt)

    # now write the points
    test.tofile(filePt);

# Try scikit-learn GMM to have point of reference
g = mixture.GMM(n_components=numClusters, verbose=1)
#g.fit(train)
#skProb = np.sum(g.score(test))
#print 'Loglikelihood (sklearn) : %f' % skProb

g.weights_ = pis
g.means_ = means
g.covars_ = variances
gtProb = np.sum(g.score(test))

print 'Loglikelihood (true) : %f' % gtProb

max_stars_repo_name: autonlab/fastlvm

max_issues_repo_name: autonlab/fastlvm

max_forks_repo_name: autonlab/fastlvm


text:
"""
Helper for converting obj (vt, ft) texture to NMR texture format.

To use NMR, we need to supply a uv_map (F x T x T x 2).
T is a sampling factor on each triangle.
Given this w can sample from an UV image and texture the map.

SMAL already has it's own custom UV map (set via blender).
This is loaded in 'template_w_tex_uv.obj'.
This stores the UV map in the standard ft and vt format of obj, where
    vt: for each vertex, it stores the uv coordinate.
    ft: for each face, this holds the index into vt
i.e.
uv_map_for_verts = mesh.vt[mesh.ft]

This is same as F x T x T x 2 value where T = 1 (only the vertices have color).

So we need to compute the UV coordinates of the samples on each triangle.
We do this by interpolating with barycentric coordinates.

Other small things to note are that obj reads y [1-0],
not [0-1] like in image space.
"""

import numpy as np

def obj2nmr_uvmap(ft, vt, tex_size=6):
    """
    Converts obj uv_map to NMR uv_map (F x T x T x 2),
    where tex_size (T) is the sample rate on each face.
    """
    # This is F x 3 x 2
    uv_map_for_verts = vt[ft]

    # obj's y coordinate is [1-0], but image is [0-1]
    uv_map_for_verts[:, :, 1] = 1 - uv_map_for_verts[:, :, 1]

    # range [0, 1] -> [-1, 1]
    uv_map_for_verts = (2 * uv_map_for_verts) - 1

    alpha = np.arange(tex_size, dtype=np.float) / (tex_size - 1)
    beta = np.arange(tex_size, dtype=np.float) / (tex_size - 1)
    import itertools
    # Barycentric coordinate values
    coords = np.stack([p for p in itertools.product(*[alpha, beta])])

    # Compute alpha, beta (this is the same order as NMR)
    v2 = uv_map_for_verts[:, 2]
    v0v2 = uv_map_for_verts[:, 0] - uv_map_for_verts[:, 2]
    v1v2 = uv_map_for_verts[:, 1] - uv_map_for_verts[:, 2]
    # Interpolate the vertex uv values: F x 2 x T*2
    uv_map = np.dstack([v0v2, v1v2]).dot(coords.T) + v2.reshape(-1, 2, 1)

    # F x T*2 x 2  -> F x T x T x 2
    uv_map = np.transpose(uv_map, (0, 2, 1)).reshape(-1, tex_size, tex_size, 2)

    return uv_map

max_stars_repo_name: silviazuffi/smalst

max_issues_repo_name: silviazuffi/smalst

max_forks_repo_name: silviazuffi/smalst


text:
# First we import numpy and matplotlib (to plot for the example)

import numpy as np
import matplotlib.pyplot as plt

# Then import the base seg1d and the seg1d processing module

import seg1d
import seg1d.processing as proc

# ##########################  SEG1D Processing #######################

# To start, we load an array of dictionaries provided as sample data in seg1d.
# raw_r is the set of references, and raw_t is a set of targets

raw_r, raw_t = seg1d.sample_input_data()

# First we can take a look at the reference datasets, noticing they are all different lengths

plt.figure(figsize=(3,3))  # doctest: +SKIP
for r in raw_r:
    plt_r = np.asarray( [ x for x in r.values()  ]  ).T
    plt.plot(plt_r,alpha=0.3)  # doctest: +SKIP
plt.show()  # doctest: +SKIP


text:
from os.path import join
from os import listdir
import numpy as np

json_ids = listdir("/data/honesty/corpora/Twitter/US_politician_twitter_conversations")
json_ids = set([i.split(".")[0] for i in json_ids])

csv_ids = listdir("/data/honesty/corpora/Twitter/US_politician_twitter_conversations_csv")
csv_ids = set([i.split(".")[0] for i in csv_ids])
print(f"{len(json_ids)} json conversations, {len(csv_ids)} csv conversations")

diff = json_ids.difference(csv_ids)
dst = "/home/jlasser/Honesty-project/code/corpus_twitter"
print(f"{len(diff)} conversations to convert")
np.savetxt(join(dst, "conversations_to_convert.txt"), np.asarray(list(diff)), fmt='%s')

max_stars_repo_name: StephanLewandowsky/Honesty-project

max_issues_repo_name: StephanLewandowsky/Honesty-project

max_forks_repo_name: StephanLewandowsky/Honesty-project