text:
from __future__ import annotations
from typing import Sequence
import pyqtgraph as pg
from qtpy.QtCore import Qt
import numpy as np

from ._utils import convert_color_code, to_rgba

# compatibility with matplotlib
_LINE_STYLE = {"-": Qt.SolidLine,
              "--": Qt.DashLine,
              ":": Qt.DotLine,
              "-.": Qt.DashDotLine,
              }

_SYMBOL_MAP = {
        "*": "star",
        "D": "d",
        "^": "t1",
        "<": "t3",
        "v": "t",
        ">": "t2",
    }

class LayerItem:
    native: pg.GraphicsItem

    def __repr__(self) -> str:
        return f"{self.__class__.__name__} '{self.name}'"

    @property
    def visible(self):
        """Visibility of data."""
        return self.native.isVisible()

    @visible.setter
    def visible(self, value: bool):
        self.native.setVisible(value)

    @property
    def zorder(self) -> float:
        """Z-order of item. Item with larger z will be displayed on the top."""
        return self.native.zValue()

    @zorder.setter
    def zorder(self, value: float):
        self.native.setZValue(value)

class PlotDataLayer(LayerItem):
    native: pg.PlotCurveItem | pg.ScatterPlotItem

    @property
    def xdata(self) -> np.ndarray:
        return self.native.getData()[0]

    @xdata.setter
    def xdata(self, value: Sequence[float]):
        self.native.setData(value, self.ydata)

    @property
    def ydata(self) -> np.ndarray:
        return self.native.getData()[1]

    @ydata.setter
    def ydata(self, value: Sequence[float]):
        self.native.setData(self.xdata, value)

    @property
    def ndata(self) -> int:
        return self.native.getData()[0].size

    @property
    def name(self) -> str:
        return self.native.opts["name"]

    @name.setter
    def name(self, value: str):
        value = str(value)
        self.native.opts["name"] = value
        # TODO: now name is not linked to label item

    def add(self, points: np.ndarray | Sequence, **kwargs):
        """Add new points to the plot data item."""
        points = np.atleast_2d(points)
        if points.shape[1] != 2:
            raise ValueError("Points must be of the shape (N, 2).")
        self.native.setData(np.concatenate([self.xdata, points[:, 0]]),
                            np.concatenate([self.ydata, points[:, 1]]),
                            **kwargs
                            )
        return None

    def remove(self, i: int | Sequence[int]):
        """Remove the i-th data."""
        if isinstance(i, int):
            i = [i]
        sl = list(set(range(self.ndata)) - set(i))
        xdata = self.xdata[sl]
        ydata = self.ydata[sl]
        self.native.setData(xdata, ydata)
        return None

    @property
    def edge_color(self) -> np.ndarray:
        """Edge color of the data."""
        return to_rgba(self.native.opts["pen"])

    @edge_color.setter
    def edge_color(self, value: str | Sequence):
        value = convert_color_code(value)
        self.native.setPen(value, width=self.lw, style=self.ls)

    @property
    def face_color(self) -> np.ndarray:
        """Face color of the data."""
        return to_rgba(self.native.opts["brush"])

    @face_color.setter
    def face_color(self, value: str | Sequence):
        value = convert_color_code(value)
        self.native.setBrush(value)

    color = property()

    @color.setter
    def color(self, value: str | Sequence):
        """Set face color and edge color at the same time."""
        self.face_color = value
        self.edge_color = value

    @property
    def lw(self):
        """Line width."""
        return self.native.opts["pen"].width()

    @lw.setter
    def lw(self, value: float):
        self.native.opts["pen"].setWidth(value)

    linewidth = lw # alias

    @property
    def ls(self):
        """Line style."""
        return self.native.opts["pen"].style()

    @ls.setter
    def ls(self, value: str):
        _ls = _LINE_STYLE[value]
        self.native.opts["pen"].setStyle(_ls)

    linestyle = ls # alias

class Scatter(PlotDataLayer):
    native: pg.ScatterPlotItem

    def __init__(self,
                 x,
                 y,
                 face_color = None,
                 edge_color = None,
                 size: float = 7,
                 name: str | None = None,
                 lw: float = 1,
                 ls: str = "-",
                 symbol="o"
                 ):
        face_color, edge_color = _set_default_colors(
            face_color, edge_color, "white", "white"
            )
        pen = pg.mkPen(edge_color, width=lw, style=_LINE_STYLE[ls])
        brush = pg.mkBrush(face_color)
        symbol = _SYMBOL_MAP.get(symbol, symbol)
        self.native = pg.ScatterPlotItem(x=x, y=y, pen=pen, brush=brush, size=size, symbol=symbol)
        self.name = name

    @property
    def symbol(self):
        return self.native.opts["symbol"]

    @symbol.setter
    def symbol(self, value):
        value = _SYMBOL_MAP.get(value, value)
        self.native.setSymbol(value)

    @property
    def size(self):
        return self.native.opts["symbolSize"]

    @size.setter
    def size(self, size: float):
        self.native.setSymbolSize(size)

class Curve(PlotDataLayer):
    native: pg.PlotDataItem

    def __init__(self,
                 x,
                 y,
                 face_color = None,
                 edge_color = None,
                 size: float = 7,
                 name: str | None = None,
                 lw: float = 1,
                 ls: str = "-",
                 symbol=None
                 ):
        face_color, edge_color = _set_default_colors(
            face_color, edge_color, "white", "white"
            )
        pen = pg.mkPen(edge_color, width=lw, style=_LINE_STYLE[ls])
        brush = pg.mkBrush(face_color)
        symbol = _SYMBOL_MAP.get(symbol, symbol)
        self.native = pg.PlotDataItem(x=x, y=y, pen=pen, brush=brush, symbolSize=size,
                                      symbol=symbol, symbolPen=pen, symbolBrush=brush)
        self.name = name

    @property
    def symbol(self):
        return self.native.opts["symbol"]

    @symbol.setter
    def symbol(self, value):
        value = _SYMBOL_MAP.get(value, value)
        self.native.setSymbol(value)

    @property
    def size(self):
        return self.native.opts["symbolSize"]

    @size.setter
    def size(self, size: float):
        self.native.setSymbolSize(size)

    @property
    def edge_color(self) -> np.ndarray:
        rgba = self.native.opts["pen"].color().getRgb()
        return np.array(rgba)/255

    @edge_color.setter
    def edge_color(self, value: str | Sequence):
        value = convert_color_code(value)
        self.native.setPen(value)
        self.native.setSymbolPen(value)

    @property
    def face_color(self) -> np.ndarray:
        rgba = self.native.opts["symbolBrush"].color().getRgb()
        return np.array(rgba)/255

    @face_color.setter
    def face_color(self, value: str | Sequence):
        value = convert_color_code(value)
        self.native.setBrush(value)
        self.native.setSymbolBrush(value)

class Histogram(PlotDataLayer):
    native: pg.ScatterPlotItem

    def __init__(self,
                 data,
                 bins: int | Sequence | str = 10,
                 range=None,
                 density: bool = False,
                 face_color = None,
                 edge_color = None,
                 name: str | None = None,
                 lw: float = 1,
                 ls: str = "-",
                 ):
        face_color, edge_color = _set_default_colors(
            face_color, edge_color, "white", "white"
            )
        pen = pg.mkPen(edge_color, width=lw, style=_LINE_STYLE[ls])
        brush = pg.mkBrush(face_color)
        y, x = np.histogram(data, bins=bins, range=range, density=density)
        self.native = pg.PlotCurveItem(x=x, y=y, pen=pen, brush=brush,
                                       stepMode="center", fillLevel=0)
        self.name = name

    def set_hist(self, data, bins=10, range=None, density=False):
        y, x = np.histogram(data, bins=bins, range=range, density=density)
        self.native.setData(x=x, y=y)

class BarPlot(PlotDataLayer):
    native: pg.BarGraphItem

    def __init__(self,
                 x,
                 y,
                 face_color = None,
                 edge_color = None,
                 width: float = 0.6,
                 name: str | None = None,
                 lw: float = 1,
                 ls: str = "-"):
        face_color, edge_color = _set_default_colors(
            face_color, edge_color, "white", "white"
            )
        pen = pg.mkPen(edge_color, width=lw, style=_LINE_STYLE[ls])
        brush = pg.mkBrush(face_color)
        self.native = pg.BarGraphItem(x=x, height=y, width=width, pen=pen, brush=brush)
        self.name = name

    @property
    def edge_color(self) -> np.ndarray:
        rgba = self.native.opts["pen"].color().getRgb()
        return np.array(rgba)/255

    @edge_color.setter
    def edge_color(self, value: str | Sequence):
        value = convert_color_code(value)
        self.native.setOpts(pen=pg.mkPen(value))

    @property
    def face_color(self) -> np.ndarray:
        rgba = self.native.opts["brush"].color().getRgb()
        return np.array(rgba)/255

    @face_color.setter
    def face_color(self, value: str | Sequence):
        value = convert_color_code(value)
        self.native.setOpts(brush=pg.mkBrush(value))

    @property
    def xdata(self) -> np.ndarray:
        return self.native.getData()[0]

    @xdata.setter
    def xdata(self, value: Sequence[float]):
        self.native.setOpts(x=value)

    @property
    def ydata(self) -> np.ndarray:
        return self.native.getData()[1]

    @ydata.setter
    def ydata(self, value: Sequence[float]):
        self.native.setOpts(height=value)

class InfLine(LayerItem):
    native: pg.InfiniteLine

    def __init__(self,
                 pos,
                 angle,
                 edge_color = None,
                 name: str | None = None,
                 lw: float = 1,
                 ls: str = "-"
                 ):
        if edge_color is None:
            edge_color = "yellow"

        pen = pg.mkPen(edge_color, width=lw, style=_LINE_STYLE[ls])
        self.native = pg.InfiniteLine(pos, angle, pen=pen, name=name)
        self.name = name

    @property
    def slope(self) -> float:
        """Slope of the line."""
        return np.tan(np.deg2rad(self.native.angle))

    @slope.setter
    def slope(self, value: float):
        self.native.setAngle(np.rad2deg(np.arctan(value)))

    @property
    def intercept(self) -> float:
        """Y-intercept of the line."""
        a = self.slope
        x0, y0 = self.native.getPos()
        return y0 - a * x0

    @intercept.setter
    def intercept(self, value: float):
        value = float(value)
        self.native.setPos((0, value))

    @property
    def pos(self) -> np.ndarray:
        return np.array(self.native.getPos())

    @pos.setter
    def pos(self, value):
        self.native.setPos(value)

    @property
    def angle(self) -> float:
        """Angle of the line in degree."""
        return self.native.angle

    @angle.setter
    def angle(self, value: float):
        self.native.setAngle(value)

    @property
    def edge_color(self) -> np.ndarray:
        return to_rgba(self.native.pen)

    @edge_color.setter
    def edge_color(self, value):
        value = convert_color_code(value)
        self.native.setPen(value, width=self.lw, style=self.ls)

    color = edge_color

    @property
    def name(self):
        return self.native._name

    @name.setter
    def name(self, value: str):
        self.native.setName(value)

    @property
    def lw(self):
        """Line width."""
        return self.native.pen.width()

    @lw.setter
    def lw(self, value: float):
        self.native.pen.setWidth(value)

    linewidth = lw # alias

    @property
    def ls(self):
        """Line style."""
        return self.native.pen.style()

    @ls.setter
    def ls(self, value: str):
        _ls = _LINE_STYLE[value]
        self.native.pen.setStyle(_ls)

    linestyle = ls # alias

class FillBetween(PlotDataLayer):
    native: pg.FillBetweenItem

    def __init__(self,
                 x,
                 y1,
                 y2,
                 face_color = None,
                 edge_color = None,
                 name: str | None = None,
                 lw: float = 1,
                 ls: str = "-"):
        face_color, edge_color = _set_default_colors(
            face_color, edge_color, "white", "white"
            )
        pen = pg.mkPen(edge_color, width=lw, style=_LINE_STYLE[ls])
        brush = pg.mkBrush(face_color)
        curve1 = pg.PlotCurveItem(x=x, y=y1, pen=pen)
        curve2 = pg.PlotCurveItem(x=x, y=y2, pen=pen)
        self.native = pg.FillBetweenItem(curve1, curve2, brush=brush, pen=pen)
        self.name = name

    @property
    def edge_color(self) -> np.ndarray:
        rgba = self.native.curves[0].opts["pen"].color().getRgb()
        return np.array(rgba)/255

    @edge_color.setter
    def edge_color(self, value: str | Sequence):
        value = convert_color_code(value)
        self.native.setPen(pg.mkPen(value))

    @property
    def face_color(self) -> np.ndarray:
        rgba = self.native.curves[0].opts["brush"].color().getRgb()
        return np.array(rgba)/255

    @face_color.setter
    def face_color(self, value: str | Sequence):
        value = convert_color_code(value)
        self.native.setBrush(pg.mkBrush(value))

    @property
    def name(self) -> str:
        return self.native.curves[0].opts["name"]

    @name.setter
    def name(self, value: str):
        value = str(value)
        self.native.curves[0].opts["name"] = value

    @property
    def lw(self):
        """Line width."""
        return self.native.curves[0].opts["pen"].width()

    @lw.setter
    def lw(self, value: float):
        self.native.curves[0].opts["pen"].setWidth(value)
        self.native.curves[1].opts["pen"].setWidth(value)

    linewidth = lw # alias

    @property
    def ls(self):
        """Line style."""
        return self.native.curves[0].opts["pen"].style()

    @ls.setter
    def ls(self, value: str):
        _ls = _LINE_STYLE[value]
        self.native.curves[0].opts["pen"].setStyle(_ls)
        self.native.curves[1].opts["pen"].setStyle(_ls)

    linestyle = ls  # alias

# WIP!
# How to update a subset of properties? item.text[2:5] = "new" or item[2:5].text = "new"
class TextGroup(LayerItem):
    def __init__(self,
                 x: Sequence[float],
                 y: Sequence[float],
                 texts: Sequence[str],
                 color = None,
                 name: str = None):
        self.native = pg.ItemGroup()
        if color is None:
            color = "white"
        for x_, y_, text_ in zip(x, y, texts):
            item = pg.TextItem(text_, color=convert_color_code(color))
            item.setPos(x_, y_)
            self.native.addItem(item)

        self.name = name

    @property
    def text_items(self) -> list[pg.TextItem]:
        return self.native.childItems()

    def __getitem__(self, key: int | slice) -> TextItemView:
        return TextItemView(self.text_items[key])

    @property
    def xdata(self) -> np.ndarray:
        return np.array([item.pos().x() for item in self.text_items])

    @property
    def ydata(self) -> np.ndarray:
        return np.array([item.pos().y() for item in self.text_items])

    @property
    def color(self) -> np.ndarray:
        """Text color."""
        rgba = np.stack([item.color.getRgb() for item in self.text_items])
        return rgba/255

    @color.setter
    def color(self, value):
        value = convert_color_code(value)
        for item in self.text_items:
            item.setText(item.toPlainText(), value)

    @property
    def background_color(self) -> np.ndarray:
        """Text background color."""
        return np.stack([to_rgba(item.fill) for item in self.text_items])

    @background_color.setter
    def background_color(self, value):
        value = convert_color_code(value)
        brush = pg.mkBrush(value)
        for item in self.text_items:
            item.fill = brush
            item._updateView()

    @property
    def border(self) -> np.ndarray:
        """Border color of text bounding box."""
        if isinstance(self.native, list):
            return np.stack([to_rgba(item.border) for item in self.native])
        else:
            return to_rgba(self.native.border)

    @border.setter
    def border(self, value):
        value = convert_color_code(value)
        pen = pg.mkPen(value)

        for item in self.text_items:
            item.border = pen
            item._updateView()

    @property
    def text(self) -> str | list[str]:
        """Text string."""
        return [item.toPlainText() for item in self.text_items]

    @text.setter
    def text(self, value: str):
        for item in self.text_items:
            item.setText(value)

    @property
    def anchor(self) -> np.ndarray:
        """Text anchor position."""
        out = []
        for item in self.text_items:
            anchor = item.anchor
            out.append([anchor.x(), anchor.y()])
        return np.array(out)

    @anchor.setter
    def anchor(self, value):
        for item in self.text_items:
            item.setAnchor(value)

class TextItemView:
    def __init__(self, textitem: pg.TextItem | list[pg.TextItem]):
        self.native = textitem

    @property
    def color(self) -> np.ndarray:
        """Text color."""
        if isinstance(self.native, list):
            rgba = np.stack([item.color.getRgb() for item in self.native])
            arr = rgba/255
        else:
            rgba = self.native.color.getRgb()
            arr = np.array(rgba)/255
        return arr

    @color.setter
    def color(self, value):
        value = convert_color_code(value)
        if isinstance(self.native, list):
            for item in self.native:
                item.setText(item.text, value)
        else:
            self.native.setText(self.text, value)

    @property
    def background_color(self) -> np.ndarray:
        """Text background color."""
        if isinstance(self.native, list):
            return np.stack([to_rgba(item.fill) for item in self.native])
        else:
            return to_rgba(self.native.fill)

    @background_color.setter
    def background_color(self, value):
        value = convert_color_code(value)
        brush = pg.mkBrush(value)
        if isinstance(self.native, list):
            for item in self.native:
                item.fill = brush
                item._updateView()
        else:
            self.native.fill = brush
            self.native._updateView()

    @property
    def border(self) -> np.ndarray:
        """Border color of text bounding box."""
        if isinstance(self.native, list):
            return np.stack([to_rgba(item.border) for item in self.native])
        else:
            return to_rgba(self.native.border)

    @border.setter
    def border(self, value):
        value = convert_color_code(value)
        pen = pg.mkPen(value)

        if isinstance(self.native, list):
            for item in self.native:
                item.border = pen
                item._updateView()
        else:
            self.native.border = pen
            self.native._updateView()

    @property
    def text(self) -> str | list[str]:
        """Text string."""
        if isinstance(self.native, list):
            return [item.toPlainText() for item in self.native]
        else:
            return self.native.toPlainText()

    @text.setter
    def text(self, value: str):

        if isinstance(self.native, list):
            for item in self.native:
                item.setText(value)
        else:
            self.native.setText(value)

    @property
    def anchor(self) -> np.ndarray:
        """Text anchor position."""
        if isinstance(self.native, list):
            out = []
            for item in self.native:
                anchor = item.anchor
                out.append([anchor.x(), anchor.y()])
            return np.array(out)
        else:
            anchor = self.native.anchor
            return np.array([anchor.x(), anchor.y()])

    @anchor.setter
    def anchor(self, value):
        if isinstance(self.native, list):
            for item in self.native:
                item.setAnchor(value)
        else:
            self.native.setAnchor(value)

def _set_default_colors(face_color, edge_color, default_f, default_e):
    if face_color is None:
        face_color = default_f
    if edge_color is None:
        edge_color = default_e
    return face_color, edge_color

max_stars_repo_name: hanjinliu/magic-class

max_issues_repo_name: hanjinliu/magic-class

max_forks_repo_name: hanjinliu/magic-class


text:
import os
import sys
import shutil
import csv
import numpy as np
import pandas as pd
from datetime import datetime
import pytz

import warnings
import matplotlib.pyplot as plt
import matplotlib
from PyQt5 import QtCore, QtGui, QtWidgets
import pyqtgraph as pg

from Controller import Controller
from MainConfig import MainConfig
from ConfigFile import ConfigFile
from HDFRoot import HDFRoot
from Utilities import Utilities
from FieldPhotos import FieldPhotos

class AnomAnalWindow(QtWidgets.QDialog):

    def __init__(self, inputDirectory, parent=None):
        super().__init__(parent)
        self.inputDirectory = inputDirectory
        self.setModal(True)

        pg.setConfigOption('background', 'w')
        pg.setConfigOption('foreground', 'k')

        self.minBand = ConfigFile.minDeglitchBand
        self.maxBand = ConfigFile.maxDeglitchBand

        # If there is an AnomAnal file, open it for later access
        # to file-specific parameters
        anomAnalFileName = os.path.splitext(ConfigFile.filename)[0]
        self.anomAnalFileName = anomAnalFileName + '_anoms.csv'
        fp = os.path.join('Config',self.anomAnalFileName)
        if os.path.exists(fp):
            self.params = Utilities.readAnomAnalFile(fp)
        else:
            self.params = {}

        # Initially, set these to generic values in ConfigFile
        for sensor in ["ES","LI","LT"]:
            setattr(self,f'{sensor}WindowDark', ConfigFile.settings[f'fL1d{sensor}WindowDark'])
            setattr(self,f'{sensor}WindowLight', ConfigFile.settings[f'fL1d{sensor}WindowLight'] )
            setattr(self,f'{sensor}SigmaDark', ConfigFile.settings[f'fL1d{sensor}SigmaDark'] )
            setattr(self,f'{sensor}SigmaLight', ConfigFile.settings[f'fL1d{sensor}SigmaLight'] )
            setattr(self,f'{sensor}MinDark', ConfigFile.settings[f'fL1d{sensor}MinDark'] )
            setattr(self,f'{sensor}MaxDark', ConfigFile.settings[f'fL1d{sensor}MaxDark'] )
            setattr(self,f'{sensor}MinMaxBandDark', ConfigFile.settings[f'fL1d{sensor}MinMaxBandDark'] )
            setattr(self,f'{sensor}MinLight', ConfigFile.settings[f'fL1d{sensor}MinLight'] )
            setattr(self,f'{sensor}MaxLight', ConfigFile.settings[f'fL1d{sensor}MaxLight'] )
            setattr(self,f'{sensor}MinMaxBandLight', ConfigFile.settings[f'fL1d{sensor}MinMaxBandLight'] )

        setattr(self,'Threshold', ConfigFile.settings['bL1dThreshold'] )

        # Set up the User Interface
        self.initUI()

    def initUI(self):

        intValidator = QtGui.QIntValidator()
        doubleValidator = QtGui.QDoubleValidator()

        # Put up the metadata at the top of the window
        self.fileDateLabel = QtWidgets.QLabel(self)
        self.windSpeedLabel = QtWidgets.QLabel(self)
        self.cloudsLabel = QtWidgets.QLabel(self)
        self.relAzLabel = QtWidgets.QLabel(self)
        self.szaLabel = QtWidgets.QLabel(self)
        self.wavesLabel = QtWidgets.QLabel(self)
        self.speedLabel = QtWidgets.QLabel(self)

        # Add a button to launch photo method
        # photoLabel = QtWidgets.QLabel("Photo",self)
        self.photoButton = QtWidgets.QPushButton("Photo")
        self.photoButton.clicked.connect(self.photoButtonPressed)
        photoFormatLabel = QtWidgets.QLabel(\
            "InputDir/Photos naming (+timezone), e.g. IMG_%Y%m%d_%H%M%S.jpg-0400:", self)
        self.photoFormat = QtWidgets.QLineEdit(self)
        if 'sL1dphotoFormat' in ConfigFile.settings:
            self.photoFormat.setText(ConfigFile.settings["sL1dphotoFormat"])
        else:
            self.photoFormat.setText('IMG_%Y%m%d_%H%M%S.jpg-0400')
            # Adds to Config, and saves
            ConfigFile.settings['sL1dphotoFormat'] = 'IMG_%Y%m%d_%H%M%S.jpg-0400'
            ConfigFile.saveConfig(ConfigFile.filename)

        # These will be adjusted on the slider once a file is loaded
        interval = 10
        self.waveBands = list(range(380, 780 +1, interval))
        # self.sLabel = QtWidgets.QLabel(f'{self.waveBands[0]}')
        self.sLabel = QtWidgets.QLabel(\
            'Use autoscaled slider to select the waveband keeping in mind deglitching is only performed from 350 - 850 nm:')
        self.slider = QtWidgets.QSlider(QtCore.Qt.Horizontal)
        self.slider.valueChanged.connect(self.sliderMove)
        # self.slider.setMinimum(min(self.waveBands))
        # self.slider.setMaximum(max(self.waveBands))
        self.slider.setMinimum(self.minBand)
        self.slider.setMaximum(self.maxBand)
        self.slider.setTickPosition(QtWidgets.QSlider.TicksBelow)
        self.slider.setTickInterval(interval)
        self.slider.setValue(self.waveBands[0])
        # self.slider.setSingleStep(1)

        self.radioButton1 = QtWidgets.QRadioButton('ES')
        self.radioButton1.setChecked(True)
        self.sensor = 'ES'
        self.radioButton1.toggled.connect(self.radioClick)

        self.radioButton2 = QtWidgets.QRadioButton('LI')
        self.radioButton2.toggled.connect(self.radioClick)

        self.radioButton3 = QtWidgets.QRadioButton('LT')
        self.radioButton3.toggled.connect(self.radioClick)

        AnomalyStepLabel = QtWidgets.QLabel("Waveband interval to save plots to disk: ", self)
        self.AnomalyStepLineEdit = QtWidgets.QLineEdit(self)
        self.AnomalyStepLineEdit.setText(str(ConfigFile.settings["fL1dAnomalyStep"]))
        self.AnomalyStepLineEdit.setValidator(intValidator)

        self.loadButton = QtWidgets.QPushButton('Load L1C', self, clicked=self.loadL1Cfile)

        self.updateButton = QtWidgets.QPushButton('***  Update  ***', self, clicked=self.updateButtonPressed)
        self.updateButton.setToolTip('Updates all but the Min/Max Bands')
        self.updateButton.setDefault(True)

        self.photoFormat.returnPressed.connect(self.updateButton.click)

        self.saveButton = QtWidgets.QPushButton('Save Sensor Params', self, clicked=self.saveButtonPressed)
        self.saveButton.setToolTip('Save these params to Configuration and file')

        self.plotButton = QtWidgets.QPushButton('Save Anomaly Plots', self, clicked=self.plotButtonPressed)

        self.processButton = QtWidgets.QPushButton('Process to L1D', self, clicked=self.processButtonPressed)

        self.closeButton = QtWidgets.QPushButton('Close', self, clicked=self.closeButtonPressed)

        self.WindowDarkLabel = QtWidgets.QLabel("Window(odd)", self)
        self.WindowDarkLineEdit = QtWidgets.QLineEdit(self)
        self.WindowDarkLineEdit.setText(str(ConfigFile.settings[f'fL1d{self.sensor}WindowDark']))
        self.WindowDarkLineEdit.setValidator(intValidator)
        # self.WindowDarkLineEdit.setValidator(oddValidator)
        self.WindowDarkLineEdit.returnPressed.connect(self.updateButton.click)

        self.SigmaDarkLabel = QtWidgets.QLabel("Sigma", self)
        self.SigmaDarkLineEdit = QtWidgets.QLineEdit(self)
        self.SigmaDarkLineEdit.setText(str(ConfigFile.settings[f'fL1d{self.sensor}SigmaDark']))
        self.SigmaDarkLineEdit.setValidator(doubleValidator)
        self.SigmaDarkLineEdit.returnPressed.connect(self.updateButton.click)

        self.MinDarkLabel = QtWidgets.QLabel("Min", self)
        self.MinDarkLineEdit = QtWidgets.QLineEdit(self)
        self.MinDarkLineEdit.setText(str(ConfigFile.settings[f'fL1d{self.sensor}MinDark']))
        self.MinDarkLineEdit.returnPressed.connect(self.updateButton.click)

        self.MinMaxDarkButton = QtWidgets.QPushButton('Set Band:', self, clicked=self.MinMaxDarkButtonPressed)
        self.MinMaxDarkButton.setToolTip('Select this band for Min/Max thresholding')
        self.MinMaxDarkLabel = QtWidgets.QLabel("", self)

        self.MaxDarkLabel = QtWidgets.QLabel("Max", self)
        self.MaxDarkLineEdit = QtWidgets.QLineEdit(self)
        self.MaxDarkLineEdit.setText(str(ConfigFile.settings[f'fL1d{self.sensor}MaxDark']))
        self.MaxDarkLineEdit.returnPressed.connect(self.updateButton.click)

        self.pLossDarkLabel = QtWidgets.QLabel('% Loss (all bands)', self)
        self.pLossDarkLineEdit = QtWidgets.QLineEdit(self)
        self.pLossDarkLineEdit.setText('0')

        self.WindowLightLabel = QtWidgets.QLabel("Window(odd)", self)
        self.WindowLightLineEdit = QtWidgets.QLineEdit(self)
        self.WindowLightLineEdit.setText(str(ConfigFile.settings[f'fL1d{self.sensor}WindowLight']))
        self.WindowLightLineEdit.setValidator(intValidator)
        # self.WindowLightLineEdit.setValidator(oddValidator)
        self.WindowLightLineEdit.returnPressed.connect(self.updateButton.click)

        self.SigmaLightLabel = QtWidgets.QLabel("Sigma Factor", self)
        self.SigmaLightLineEdit = QtWidgets.QLineEdit(self)
        self.SigmaLightLineEdit.setText(str(ConfigFile.settings[f'fL1d{self.sensor}SigmaLight']))
        self.SigmaLightLineEdit.setValidator(doubleValidator)
        self.SigmaLightLineEdit.returnPressed.connect(self.updateButton.click)

        self.MinLightLabel = QtWidgets.QLabel("Min", self)
        self.MinLightLineEdit = QtWidgets.QLineEdit(self)
        self.MinLightLineEdit.setText(str(ConfigFile.settings[f'fL1d{self.sensor}MinLight']))
        self.MinLightLineEdit.returnPressed.connect(self.updateButton.click)

        self.MinMaxLightButton = QtWidgets.QPushButton('Set Band:', self, clicked=self.MinMaxLightButtonPressed)
        self.MinMaxLightButton.setToolTip('Select this band for Min/Max thresholding')
        self.MinMaxLightLabel = QtWidgets.QLabel("", self)

        self.MaxLightLabel = QtWidgets.QLabel("Max", self)
        self.MaxLightLineEdit = QtWidgets.QLineEdit(self)
        self.MaxLightLineEdit.setText(str(ConfigFile.settings[f'fL1d{self.sensor}MaxLight']))
        self.MaxLightLineEdit.returnPressed.connect(self.updateButton.click)

        self.pLossLightLabel = QtWidgets.QLabel('% Loss (all bands)', self)
        self.pLossLightLineEdit = QtWidgets.QLineEdit(self)
        self.pLossLightLineEdit.setText('0')

        # self.ThresholdLabel = QtWidgets.QLabel("Threshold", self)
        self.ThresholdCheckBox = QtWidgets.QCheckBox("Threshold", self)
        if int(ConfigFile.settings["bL1dThreshold"]) == 1:
            self.ThresholdCheckBox.setChecked(True)
        else:
            self.ThresholdCheckBox.setChecked(False)
        self.ThresholdCheckBoxUpdate()

        # Set up datetime axis objects
        #   https://stackoverflow.com/questions/49046931/how-can-i-use-dateaxisitem-of-pyqtgraph
        class TimeAxisItem(pg.AxisItem):
            def tickStrings(self, values, scale, spacing):
                return [datetime.fromtimestamp(value, pytz.timezone("UTC")) for value in values]

        date_axis_Dark = TimeAxisItem(orientation='bottom')
        date_axis_Light = TimeAxisItem(orientation='bottom')

        # Set up realtime plot widgets
        self.plotWidgetDark = pg.PlotWidget(self)
        self.plotWidgetLight = pg.PlotWidget(self)

        guideLabel = QtWidgets.QLabel(\
            'Left-click-hold to pan, right-click-hold to zoom, or right-click-release for more options.\
                IF PLOT IS BLANK, CLICK THE "A" IN THE BOTTOM LEFT CORNER TO RESET ZOOM.')

        self.ThresholdCheckBox.clicked.connect(self.ThresholdCheckBoxUpdate)

        # Opening plot
        x=[0,0]
        y=[0,0]
        self.phDark = self.plotWidgetDark.plot(x,y, symbolPen='b',\
                 symbol='o', name='time series', pen=None)
        self.phTimeAveDark = self.plotWidgetDark.plot(x,y, pen='g', name='running')
        self.ph1stDark = self.plotWidgetDark.plot(x,y, symbolPen='r',\
                 symbol='x', name='1st pass', pen=None)
        self.ph2ndDark = self.plotWidgetDark.plot(x,y, symbolPen='r',\
                 symbol='+', name='2nd pass', pen=None)
        self.ph3rdDark = self.plotWidgetDark.plot(x,y, symbolPen='r',\
                 symbol='o', name='2nd pass', pen=None)
        # Add datetime object to x axis
        self.plotWidgetDark.setAxisItems({'bottom': date_axis_Dark})

        self.phLight = self.plotWidgetLight.plot(x,y, symbolPen='b',\
                 symbol='o', name='time series', pen=None)
        self.phTimeAveLight = self.plotWidgetLight.plot(x,y, pen='g', name='running')
        self.ph1stLight = self.plotWidgetLight.plot(x,y, symbolPen='r',\
                 symbol='x', name='1st pass', pen=None)
        self.ph2ndLight = self.plotWidgetLight.plot(x,y, symbolPen='r',\
                 symbol='+', name='2nd pass', pen=None)
        self.ph3rdLight = self.plotWidgetLight.plot(x,y, symbolPen='r',\
                 symbol='o', name='2nd pass', pen=None)
        # Add datetime object to x axis
        self.plotWidgetLight.setAxisItems({'bottom': date_axis_Light})

        # Layout ######################################################
        self.VBox = QtWidgets.QVBoxLayout()
        self.HBoxMeta1 = QtWidgets.QHBoxLayout()
        self.HBoxMeta2 = QtWidgets.QHBoxLayout()
        self.HBoxMeta1.addWidget(self.fileDateLabel)
        self.HBoxMeta1.addSpacing(45)
        self.HBoxMeta1.addWidget(photoFormatLabel)
        self.HBoxMeta1.addWidget(self.photoFormat)
        self.HBoxMeta1.addWidget(self.photoButton)

        self.HBoxMeta2.addWidget(self.windSpeedLabel)
        self.HBoxMeta2.addWidget(self.cloudsLabel)
        self.HBoxMeta2.addWidget(self.relAzLabel)
        self.HBoxMeta2.addWidget(self.szaLabel)
        self.HBoxMeta2.addWidget(self.wavesLabel)
        self.HBoxMeta2.addWidget(self.speedLabel)
        self.VBox.addLayout(self.HBoxMeta1)
        self.VBox.addLayout(self.HBoxMeta2)

        self.VBox.addWidget(self.sLabel)
        HBox1 = QtWidgets.QHBoxLayout()
        HBox1.addWidget(self.slider)
        self.VBox.addLayout(HBox1)

        HBox2 = QtWidgets.QHBoxLayout()
        HBox2.addWidget(self.loadButton)
        # RBox = QtWidgets.QGridLayout()
        self.buttonGroup = QtWidgets.QButtonGroup()
        self.buttonGroup.addButton(self.radioButton1)
        self.buttonGroup.addButton(self.radioButton2)
        self.buttonGroup.addButton(self.radioButton3)
        HBox2.addWidget(self.radioButton1)
        HBox2.addWidget(self.radioButton2)
        HBox2.addWidget(self.radioButton3)

        HBox2.addSpacing(55)
        HBox2.addWidget(self.updateButton)
        HBox2.addSpacing(55)
        HBox2.addStretch()

        HBox2.addWidget(AnomalyStepLabel)
        HBox2.addWidget(self.AnomalyStepLineEdit)
        HBox2.addWidget(self.saveButton)
        HBox2.addWidget(self.plotButton)
        HBox2.addWidget(self.processButton)
        HBox2.addWidget(self.closeButton)
        self.VBox.addLayout(HBox2)

        HBox3 = QtWidgets.QHBoxLayout()
        HBox3.addWidget(self.WindowDarkLabel)
        HBox3.addWidget(self.WindowDarkLineEdit)
        HBox3.addWidget(self.SigmaDarkLabel)
        HBox3.addWidget(self.SigmaDarkLineEdit)

        HBox3.addWidget(self.pLossDarkLabel)
        HBox3.addWidget(self.pLossDarkLineEdit)

        HBox3.addWidget(self.WindowLightLabel)
        HBox3.addWidget(self.WindowLightLineEdit)
        HBox3.addWidget(self.SigmaLightLabel)
        HBox3.addWidget(self.SigmaLightLineEdit)

        HBox3.addWidget(self.pLossLightLabel)
        HBox3.addWidget(self.pLossLightLineEdit)

        self.VBox.addLayout(HBox3)

        HBox4 = QtWidgets.QHBoxLayout()
        # HBox4.addWidget(self.ThresholdLabel)
        HBox4.addWidget(self.ThresholdCheckBox)
        HBox4.addWidget(self.MinDarkLabel)
        HBox4.addWidget(self.MinDarkLineEdit)
        HBox4.addWidget(self.MaxDarkLabel)
        HBox4.addWidget(self.MaxDarkLineEdit)
        HBox4.addWidget(self.MinMaxDarkButton)
        HBox4.addWidget(self.MinMaxDarkLabel)
        HBox4.addSpacing(35)
        HBox4.addStretch()
        HBox4.addWidget(self.MinLightLabel)
        HBox4.addWidget(self.MinLightLineEdit)
        HBox4.addWidget(self.MaxLightLabel)
        HBox4.addWidget(self.MaxLightLineEdit)
        HBox4.addWidget(self.MinMaxLightButton)
        HBox4.addWidget(self.MinMaxLightLabel)
        self.VBox.addLayout(HBox4)

        HBox5 = QtWidgets.QHBoxLayout()
        HBox5.addWidget(self.plotWidgetDark)
        HBox5.addWidget(self.plotWidgetLight)
        self.VBox.addLayout(HBox5)

        self.VBox.addWidget(guideLabel)

        self.setLayout(self.VBox)
        self.setGeometry(100, 70, 1400, 700)

        self.sliderWave = float(self.slider.value())

        # Set up the photo path
        # format = self.photoFormat.text()
        # tz = format[-5:] # clumsy hardcoding of TZ format: Must be the last 5 characters
        # format = format[0:-5]
        self.photoFP = os.path.join(self.inputDirectory,'Photos')
        if os.path.isdir(self.photoFP) is False:
            os.mkdir(self.photoFP)
        # photoList, self.photoDT = FieldPhotos.photoSetup(self.photoFP, self.start, self.end, format, tz)

        # Run this on first opening the GUI
        self.loadL1Cfile()

        ##############################################

    def photoButtonPressed(self):
        print("Photo button pressed")

        photoWidget = FieldPhotos(self.photoList, self.photoDT, self)
        photoWidget.show()

    def sliderMove(self):
        self.sliderWave = float(self.slider.value())
        ''' This fails to update the label '''
        self.sLabel.setText(f'Deglitching only performed from 350-850 nm: {self.sliderWave}')
        # print(self.sliderWave)

    def loadL1Cfile(self):
        inputDirectory = self.inputDirectory
        # Open L1C HDF5 file for Deglitching
        inLevel = "L1C"
        subInputDir = os.path.join(inputDirectory + '/' + inLevel + '/')
        if os.path.exists(subInputDir):
            inFilePath,_ = QtWidgets.QFileDialog.getOpenFileNames(self, "Open L1C HDF5 file for Deglitching", \
                subInputDir)
        else:
            inFilePath,_ = QtWidgets.QFileDialog.getOpenFileNames(self, "Open L1C HDF5 file for Deglitching", \
                inputDirectory)
        try:
            print(inFilePath[0])
            if not "hdf" in inFilePath[0] and not "HDF" in inFilePath[0]:
                msg = "This does not appear to be an HDF file."
                Utilities.errorWindow("File Error", msg)
                print(msg)
                return
        except:
            print('No file returned')
            return

        root = HDFRoot.readHDF5(inFilePath[0])
        if root.attributes["PROCESSING_LEVEL"] != "1c":
            msg = "This is not a Level 1C file."
            Utilities.errorWindow("File Error", msg)
            print(msg)
            return

        Utilities.rootAddDateTime(root)

        self.fileName = os.path.basename(os.path.splitext(inFilePath[0])[0])
        self.setWindowTitle(self.fileName)
        self.root = root

        # If a parameterization has been saved in the AnomAnalFile, set the properties in the local object
        # for all sensors
        if self.fileName in self.params.keys():
            ref = 0
            for sensor in ['ES','LI','LT']:
                setattr(self,f'{sensor}WindowDark', self.params[self.fileName][ref+0] )
                setattr(self,f'{sensor}WindowLight', self.params[self.fileName][ref+1] )
                setattr(self,f'{sensor}SigmaDark', self.params[self.fileName][ref+2] )
                setattr(self,f'{sensor}SigmaLight', self.params[self.fileName][ref+3] )
                setattr(self,f'{sensor}MinDark', self.params[self.fileName][ref+4] )
                setattr(self,f'{sensor}MaxDark', self.params[self.fileName][ref+5] )
                setattr(self,f'{sensor}MinMaxBandDark', self.params[self.fileName][ref+6] )
                setattr(self,f'{sensor}MinLight', self.params[self.fileName][ref+7] )
                setattr(self,f'{sensor}MaxLight', self.params[self.fileName][ref+8] )
                setattr(self,f'{sensor}MinMaxBandLight', self.params[self.fileName][ref+9] )
                ref += 10
            setattr(self,'Threshold', self.params[self.fileName][30])
            if getattr(self,'Threshold') == 1:
                self.ThresholdCheckBox.setChecked(True)
                # if getattr(self,f'{self.sensor}MinMaxBandDark'):
                #     self.waveBand = getattr(self,f'{self.sensor}MinMaxBandDark')
                #     self.slider.setValue(self.waveBand)
                # if getattr(self,f'{self.sensor}MinMaxBandLight'):
                #     self.waveBand = getattr(self,f'{self.sensor}MinMaxBandLight')
                #     self.slider.setValue(self.waveBand)
            else:
                self.ThresholdCheckBox.setChecked(False)
            self.ThresholdCheckBoxUpdate()

        # Set the GUI parameters for the current sensor from the local object
        self.WindowDarkLineEdit.setText(str(getattr(self,f'{self.sensor}WindowDark')))
        self.WindowLightLineEdit.setText(str(getattr(self,f'{self.sensor}WindowLight')))
        self.SigmaDarkLineEdit.setText(str(getattr(self,f'{self.sensor}SigmaDark')))
        self.SigmaLightLineEdit.setText(str(getattr(self,f'{self.sensor}SigmaLight')))
        self.MinDarkLineEdit.setText(str(getattr(self,f'{self.sensor}MinDark')))
        self.MinLightLineEdit.setText(str(getattr(self,f'{self.sensor}MinLight')))
        self.MaxDarkLineEdit.setText(str(getattr(self,f'{self.sensor}MaxDark')))
        self.MaxLightLineEdit.setText(str(getattr(self,f'{self.sensor}MaxLight')))

        # Add an information bar based on metadata
        for group in root.groups:
            if group.id == 'ANCILLARY_METADATA':
                ancGroup = group
            if group.id.startswith('GP'):
                gpsGroup = group
                self.start = gpsGroup.datasets['DATETIME'].data[0]
                self.end = gpsGroup.datasets['DATETIME'].data[-1]
            if group.id == "SOLARTRACKER" or group.id == "SOLARTRACKER_UM":
                    trackerGroup = group

        with warnings.catch_warnings():
            warnings.filterwarnings('ignore', r'All-NaN (slice|axis) encountered')
            if 'WINDSPEED' in ancGroup.datasets:
                windSpeed = np.nanmedian(ancGroup.datasets['WINDSPEED'].data.tolist())
            else:
                windSpeed = np.nan
            if 'CLOUD' in ancGroup.datasets:
                cloud = np.nanmedian(ancGroup.datasets['CLOUD'].data.tolist())
            else:
                cloud = np.nan
            if 'REL_AZ' in ancGroup.datasets:
                relAz = np.nanmedian(ancGroup.datasets['REL_AZ'].data.tolist())
            else:
                relAz = np.nanmedian(trackerGroup.datasets['REL_AZ'].data.tolist())
            sza = np.nanmedian(ancGroup.datasets['SZA'].data.tolist())
            if 'WAVE_HT' in ancGroup.datasets:
                waves = np.nanmedian(ancGroup.datasets['WAVE_HT'].data.tolist())
            else:
                waves = np.nan
            if 'SPEED_F_W' in ancGroup.datasets:
                speed = np.nanmedian(ancGroup.datasets['SPEED_F_W'].data.tolist())
            else:
                # Could resort to GPS SOG here, but that makes less sense than NaN
                speed = np.nan

        # self.fileDateLabel.setText(f"Begin: {root.attributes['TIME-STAMP']}")
        self.fileDateLabel.setText(f"FROM: {self.start:%Y-%m-%d %H:%M} TO: {self.end:%Y-%m-%d %H:%M} UTC")
        self.windSpeedLabel.setText(f' (Median->) WIND: {windSpeed:.1f} m/s')
        self.cloudsLabel.setText(f' CLOUD: {cloud:.0f} %')
        self.relAzLabel.setText(f' REL.AZ: {relAz:.0f} deg.')
        self.szaLabel.setText(f' SZA: {sza:.0f} deg.')
        self.wavesLabel.setText(f' WAVES: {waves:.1f} m')
        self.speedLabel.setText(f' SPEED: {speed:.1f} m/s')

        # Match data to photo, if possible
        format = self.photoFormat.text()
        tz = format[-5:] # clumsy hardcoding of TZ format: Must be the last 5 characters
        format = format[0:-5]
        self.photoList, self.photoDT = FieldPhotos.photoSetup(self.photoFP, self.start, self.end, format, tz)
        if self.photoList is not None:
            print('Matching photo found')
            self.photoButton.setText(os.path.split(self.photoList[0])[-1])
            self.photoButton.setDisabled(0)
        else:
            self.photoButton.setText('No Photo Found')
            self.photoButton.setDisabled(1)

        self.updateButtonPressed()

    def radioClick(self):
        # Before changing to the new sensor, locally save the parameters
        # from the last sensor in case it was changed.

        # Test the window sizes
        if int(self.WindowDarkLineEdit.text())%2 == 0 or int(self.WindowLightLineEdit.text())%2 ==0:
            alert = QtWidgets.QMessageBox()
            alert.setText('Deglitching windows must be odd integers.')
            alert.exec_()
            return

        # Set parameters in the local object
        setattr(self,f'{self.sensor}WindowDark', int(self.WindowDarkLineEdit.text()) )
        setattr(self,f'{self.sensor}WindowLight', int(self.WindowLightLineEdit.text()) )
        setattr(self,f'{self.sensor}SigmaDark', float(self.SigmaDarkLineEdit.text()) )
        setattr(self,f'{self.sensor}SigmaLight', float(self.SigmaLightLineEdit.text()) )
        x = None if self.MinDarkLineEdit.text()=='None' else float(self.MinDarkLineEdit.text())
        setattr(self,f'{self.sensor}MinDark', x )
        x = None if self.MinLightLineEdit.text()=='None' else float(self.MinLightLineEdit.text())
        setattr(self,f'{self.sensor}MinLight', x )
        x = None if self.MaxDarkLineEdit.text()=='None' else float(self.MaxDarkLineEdit.text())
        setattr(self,f'{self.sensor}MaxDark', x )
        x = None if self.MaxLightLineEdit.text()=='None' else float(self.MaxLightLineEdit.text())
        setattr(self,f'{self.sensor}MaxLight', x )

        radioButton = self.sender()
        if radioButton.isChecked():
            self.sensor = radioButton.text()
            print("Sensor is %s" % (self.sensor))
            # Now update the LineEdits with saved values in the local object
            self.WindowDarkLineEdit.setText(str(getattr(self,f'{self.sensor}WindowDark')))
            self.WindowLightLineEdit.setText(str(getattr(self,f'{self.sensor}WindowLight')))
            self.SigmaDarkLineEdit.setText(str(getattr(self,f'{self.sensor}SigmaDark')))
            self.SigmaLightLineEdit.setText(str(getattr(self,f'{self.sensor}SigmaLight')))
            self.MinDarkLineEdit.setText(str(getattr(self,f'{self.sensor}MinDark')))
            self.MinLightLineEdit.setText(str(getattr(self,f'{self.sensor}MinLight')))
            self.MaxDarkLineEdit.setText(str(getattr(self,f'{self.sensor}MaxDark')))
            self.MaxLightLineEdit.setText(str(getattr(self,f'{self.sensor}MaxLight')))

        self.updateButtonPressed()

    def updateButtonPressed(self):
        print("Update pressed")
        # print(self.sliderWave)

        sensorType = self.sensor
        print(sensorType)

        # Update photo format
        ConfigFile.settings['sL1dphotoFormat'] = self.photoFormat.text()
        ConfigFile.saveConfig(ConfigFile.filename)

        # Test for root
        if not hasattr(self, 'root'):
            note = QtWidgets.QMessageBox()
            note.setText('You must load L1C file before plotting')
            note.exec_()
            return

        # Test the window sizes
        if int(self.WindowDarkLineEdit.text())%2 == 0 or int(self.WindowLightLineEdit.text())%2 ==0:
            alert = QtWidgets.QMessageBox()
            alert.setText('Deglitching windows must be odd integers.')
            alert.exec_()
            return

        # Set parameters in the local object from the GUI
        setattr(self,f'{self.sensor}WindowDark', int(self.WindowDarkLineEdit.text()) )
        setattr(self,f'{self.sensor}WindowLight', int(self.WindowLightLineEdit.text()) )
        setattr(self,f'{self.sensor}SigmaDark', float(self.SigmaDarkLineEdit.text()) )
        setattr(self,f'{self.sensor}SigmaLight', float(self.SigmaLightLineEdit.text()) )
        x = None if self.MinDarkLineEdit.text()=='None' else float(self.MinDarkLineEdit.text())
        setattr(self,f'{self.sensor}MinDark', x )
        x = None if self.MinLightLineEdit.text()=='None' else float(self.MinLightLineEdit.text())
        setattr(self,f'{self.sensor}MinLight', x )
        x = None if self.MaxDarkLineEdit.text()=='None' else float(self.MaxDarkLineEdit.text())
        setattr(self,f'{self.sensor}MaxDark', x )
        x = None if self.MaxLightLineEdit.text()=='None' else float(self.MaxLightLineEdit.text())
        setattr(self,f'{self.sensor}MaxLight', x )

        darkData = None
        lightData = None
        # Extract Light & Dark datasets from the sensor group
        for gp in self.root.groups:
            if gp.attributes["FrameType"] == "ShutterDark" and sensorType in gp.datasets:
                darkData = gp.getDataset(sensorType)
                darkDateTime = Utilities.getDateTime(gp)
            if gp.attributes["FrameType"] == "ShutterLight" and sensorType in gp.datasets:
                lightData = gp.getDataset(sensorType)
                lightDateTime = Utilities.getDateTime(gp)

        # Deglitch and plot Dark from selected band
        if darkData is None:
            print("Error: No dark data to deglitch")
        else:
            print("Dark data anomaly analysis")
            lightDark = 'Dark'
            darkData.datasetToColumns()
            dark2D = darkData.columns

            # Same for Dark and Light... here taking from Dark
            waveBands = []
            waveBandStrings = []
            for key in dark2D.keys():
                waveBands.append(float(key))
                waveBandStrings.append(key)
            self.waveBands = waveBands
            whr = Utilities.find_nearest(self.waveBands, self.sliderWave)
            self.waveBand = self.waveBands[whr]
            # print(self.waveBand)

            radiometry1D = dark2D[f'{self.waveBand:03.2f}']

            self.realTimePlot(self, radiometry1D, darkDateTime, sensorType,lightDark)

        # Update the slider
        self.sLabel.setText(f'Deglitching only performed from 350-850 nm: {self.waveBand}')
        # self.slider.setMinimum(min(waveBands))
        # self.slider.setMaximum(max(waveBands))
        # self.slider.setTickInterval(10)
        self.slider.setValue(self.sliderWave)

        # Update minmax text
        self.MinMaxDarkLabel.setText(str(getattr(self,f'{self.sensor}MinMaxBandDark')) +'nm' )
        self.MinMaxLightLabel.setText(str(getattr(self,f'{self.sensor}MinMaxBandLight')) +'nm')

        # # Try to use the Threshold bands to reset the slider and display
        # self.ThresholdCheckBoxUpdate()

        # Deglitch and plot Light from selected band
        if lightData is None:
            print("Error: No light data to deglitch")
        else:
            print("Light data anomaly analysis")
            lightDark = 'Light'
            lightData.datasetToColumns()
            light2D = lightData.columns
            radiometry1D = light2D[f'{self.waveBand:03.2f}']
            # print(self.waveBand)

            self.realTimePlot(self, radiometry1D, lightDateTime, sensorType, lightDark)

        # Now run the deglitcher for all wavebands light and dark to calculate the % loss to the data from this sensor
        # Darks
        columns = darkData.columns
        globalBadIndex = []
        for dark1D in columns.items():
            band = float(dark1D[0])
            if band > self.minBand and band < self.maxBand:
                radiometry1D = dark1D[1]
                window = int(self.WindowDarkLineEdit.text())
                sigma = float(self.SigmaDarkLineEdit.text())
                minDark = None if self.MinDarkLineEdit.text()=='None' else float(self.MinDarkLineEdit.text())
                maxDark = None if self.MaxDarkLineEdit.text()=='None' else float(self.MaxDarkLineEdit.text())
                MinMaxDarkBand = getattr(self,f'{self.sensor}MinMaxBandDark')
                lightDark = 'Dark'
                badIndex, badIndex2, badIndex3 = Utilities.deglitchBand(band,radiometry1D, window, sigma, lightDark, minDark, maxDark,MinMaxDarkBand)
                globalBadIndex.append(badIndex)
                globalBadIndex.append(badIndex2)
                globalBadIndex.append(badIndex3)

        # Collapse the badIndexes from all wavebands into one timeseries
        # Must be done seperately for dark and light as they are different length time series
        # Convert to an array and test along the columns (i.e. each timestamp)
        gIndex = np.any(np.array(globalBadIndex), 0)
        percentLoss = 100*(sum(gIndex)/len(gIndex))
        pLabel = f'Data reduced by {sum(gIndex)} ({percentLoss:.1f}%)'
        print(pLabel)
        # self.plotWidgetLight.TextItem(pLabel,anchor=(0.2,0.2))
        self.pLossDarkLineEdit.setText(f'{percentLoss:.1f}')

        # Lights
        columns = lightData.columns
        globalBadIndex = []
        for light1D in columns.items():
            band = float(light1D[0])
            if band > self.minBand and band < self.maxBand:
                radiometry1D = light1D[1]
                window = int(self.WindowLightLineEdit.text())
                sigma = float(self.SigmaLightLineEdit.text())
                minLight = None if self.MinLightLineEdit.text()=='None' else float(self.MinLightLineEdit.text())
                maxLight = None if self.MaxLightLineEdit.text()=='None' else float(self.MaxLightLineEdit.text())
                MinMaxLightBand = getattr(self,f'{self.sensor}MinMaxBandLight')
                lightDark = 'Light'
                badIndex, badIndex2, badIndex3 = Utilities.deglitchBand(band,radiometry1D, window, sigma, lightDark, minLight, maxLight,MinMaxLightBand)
                globalBadIndex.append(badIndex)
                globalBadIndex.append(badIndex2)
                globalBadIndex.append(badIndex3)

        # Collapse the badIndexes from all wavebands into one timeseries
        # Convert to an array and test along the columns (i.e. each timestamp)
        gIndex = np.any(np.array(globalBadIndex), 0)
        percentLoss = 100*(sum(gIndex)/len(gIndex))
        pLabel = f'Data reduced by {sum(gIndex)} ({percentLoss:.1f}%)'
        print(pLabel)
        self.pLossLightLineEdit.setText(f'{percentLoss:.1f}')

    def saveButtonPressed(self):
        # Saves local parameterizations to the ConfigFile.settings
        # Triggered only when button pressed
        print("Save params pressed")

        params = []
        for sensor in ["ES","LI","LT"]:
            ConfigFile.settings[f'fL1d{sensor}WindowDark'] = getattr(self,f'{sensor}WindowDark')
            ConfigFile.settings[f'fL1d{sensor}WindowLight'] = getattr(self,f'{sensor}WindowLight')
            ConfigFile.settings[f'fL1d{sensor}SigmaDark'] = getattr(self,f'{sensor}SigmaDark')
            ConfigFile.settings[f'fL1d{sensor}SigmaLight'] = getattr(self,f'{sensor}SigmaLight')
            ConfigFile.settings[f'fL1d{sensor}MinDark'] = getattr(self,f'{sensor}MinDark')
            ConfigFile.settings[f'fL1d{sensor}MinLight'] = getattr(self,f'{sensor}MinLight')
            ConfigFile.settings[f'fL1d{sensor}MaxDark'] = getattr(self,f'{sensor}MaxDark')
            ConfigFile.settings[f'fL1d{sensor}MaxLight'] = getattr(self,f'{sensor}MaxLight')
            ConfigFile.settings[f'fL1d{sensor}MinMaxBandDark'] = getattr(self,f'{sensor}MinMaxBandDark')
            ConfigFile.settings[f'fL1d{sensor}MinMaxBandLight'] = getattr(self,f'{sensor}MinMaxBandLight')

            params.append(getattr(self,f'{sensor}WindowDark'))
            params.append(getattr(self,f'{sensor}WindowLight'))
            params.append(getattr(self,f'{sensor}SigmaDark'))
            params.append(getattr(self,f'{sensor}SigmaLight'))
            params.append(getattr(self,f'{sensor}MinDark'))
            params.append(getattr(self,f'{sensor}MaxDark'))
            params.append(getattr(self,f'{sensor}MinMaxBandDark'))
            params.append(getattr(self,f'{sensor}MinLight'))
            params.append(getattr(self,f'{sensor}MaxLight'))
            params.append(getattr(self,f'{sensor}MinMaxBandLight'))

        ConfigFile.settings[f'bL1dThreshold'] = getattr(self,f'Threshold')
        params.append(getattr(self,f'Threshold'))

        self.params[self.fileName] = params

        # Steps in wavebands used for plots
        ConfigFile.settings["fL1dAnomalyStep"] = int(self.AnomalyStepLineEdit.text())

        # Save file-specific parameters to CSV file
        fp = os.path.join('Config',self.anomAnalFileName)
        self.writeAnomAnalFile(fp)

    def writeAnomAnalFile(self, filePath):
        header = ['filename','ESWindowDark','ESWindowLight','ESSigmaDark','ESSigmaLight','ESMinDark','ESMaxDark',\
            'ESMinMaxBandDark','ESMinLight','ESMaxLight','ESMinMaxBandLight',
            'LIWindowDark','LIWindowLight','LISigmaDark','LISigmaLight','LIMinDark','LIMaxDark',
            'LIMinMaxBandDark','LIMinLight','LIMaxLight','LIMinMaxBandLight',
            'LTWindowDark','LTWindowLight','LTSigmaDark','LTSigmaLight','LTMinDark','LTMaxDark',
            'LTMinMaxBandDark','LTMinLight','LTMaxLight','LTMinMaxBandLight','Threshold']
        # fieldnames = params.keys()

        with open(filePath, 'w', newline='') as csvfile:
            paramwriter = csv.writer(csvfile, delimiter=',',
                                quotechar='|', quoting=csv.QUOTE_MINIMAL)
            paramwriter.writerow(header)
            for key, values in self.params.items():
                values = [-999 if v==None else v for v in values]
                paramwriter.writerow([key]+values)

    def plotButtonPressed(self):
        print("Save plots pressed")
        # Steps in wavebands used for plots
        step = float(ConfigFile.settings["fL1dAnomalyStep"])

        outDir = MainConfig.settings["outDir"]
        # If default output path is used, choose the root HyperInSPACE path, and build on that
        if os.path.abspath(outDir) == os.path.join('./','Data'):
            outDir = './'

        if not os.path.exists(os.path.join(outDir,'Plots','L1C_Anoms')):
            os.makedirs(os.path.join(outDir,'Plots','L1C_Anoms'))
        plotdir = os.path.join(outDir,'Plots','L1C_Anoms')
        # fp = os.path.join(plotdir,self.fileName)

        sensorTypes = ["ES","LT","LI"]

        for sensorType in sensorTypes:
            print(sensorType)
            darkData = None
            lightData = None

            for gp in self.root.groups:
                if gp.attributes["FrameType"] == "ShutterDark" and sensorType in gp.datasets:
                    darkData = gp.getDataset(sensorType)
                    darkDateTime = Utilities.getDateTime(gp)
                    # print(type(darkData))
                    # print(type(darkData.data))
                if gp.attributes["FrameType"] == "ShutterLight" and sensorType in gp.datasets:
                    lightData = gp.getDataset(sensorType)
                    lightDateTime = Utilities.getDateTime(gp)

            if darkData is None:
                print("Error: No dark data to deglitch")
            else:
                print("Dark data anomaly analysis")
                lightDark = 'Dark'
                darkData.datasetToColumns()
                columns = darkData.columns
                dateTime = darkDateTime
                window = getattr(self,f'{sensorType}WindowDark')
                sigma = getattr(self,f'{sensorType}SigmaDark')
                minDark = getattr(self,f'{sensorType}MinDark')
                maxDark = getattr(self,f'{sensorType}MaxDark')
                minMaxDarkBand = getattr(self,f'{sensorType}MinMaxBandDark')

                index = 0
                # Loop over bands to populate globBads
                for timeSeries in columns.items():
                    if index==0:
                        # Initialize boolean lists for capturing global badIndex conditions across all wavebands
                        globBad = [False]*len(timeSeries[1])
                        globBad2 = [False]*len(timeSeries[1])
                        globBad3 = [False]*len(timeSeries[1])
                    band = float(timeSeries[0])
                    if band > self.minBand and band < self.maxBand:
                        # if index % step == 0:
                        radiometry1D = timeSeries[1]
                        badIndex, badIndex2, badIndex3 = Utilities.deglitchBand(band,radiometry1D, window, sigma, lightDark, minDark, maxDark,minMaxDarkBand)
                        globBad[:] = (True if val2 else val1 for (val1,val2) in zip(globBad,badIndex))
                        globBad2[:] = (True if val2 else val1 for (val1,val2) in zip(globBad2,badIndex2))
                        globBad3[:] = (True if val2 else val1 for (val1,val2) in zip(globBad3,badIndex3))
                            ##It would be good to use an aggregated badIndex on these plots for all bands combined...
                            # self.savePlots(self.fileName,plotdir,timeSeries,sensorType,lightDark,window,sigma,badIndex,badIndex2,badIndex3)
                    index +=1
                # Now plot a selection of these USING UNIVERSALLY EXCLUDED INDEXES
                index =0
                for timeSeries in columns.items():
                    band = float(timeSeries[0])
                    if band > self.minBand and band < self.maxBand:
                        if index % step == 0:
                            # self.savePlots(self.fileName,plotdir,timeSeries,sensorType,lightDark,window,sigma,globBad,globBad2,globBad3)
                            Utilities.saveDeglitchPlots(self.fileName,plotdir,timeSeries,dateTime,sensorType,lightDark,window,sigma,globBad,globBad2,globBad3)
                    index +=1

            if lightData is None:
                print("Error: No light data to deglitch")
            else:
                print("Light data anomaly analysis")
                lightDark = 'Light'
                lightData.datasetToColumns()
                columns = lightData.columns
                dateTime = lightDateTime
                window = getattr(self,f'{sensorType}WindowLight')
                sigma = getattr(self,f'{sensorType}SigmaLight')
                minLight = getattr(self,f'{sensorType}MinLight')
                maxLight = getattr(self,f'{sensorType}MaxLight')
                minMaxLightBand = getattr(self,f'{sensorType}MinMaxBandLight')

                index = 0
                for timeSeries in columns.items():
                    if index==0:
                        # Initialize boolean lists for capturing global badIndex conditions across all wavebands
                        globBad = [False]*len(timeSeries[1])
                        globBad2 = [False]*len(timeSeries[1])
                        globBad3 = [False]*len(timeSeries[1])
                    band = float(timeSeries[0])
                    if band > self.minBand and band < self.maxBand:
                        # if index % step == 0:
                        radiometry1D = timeSeries[1]
                        badIndex, badIndex2, badIndex3 = Utilities.deglitchBand(band,radiometry1D, window, sigma, lightDark, minLight, maxLight,minMaxLightBand)
                        globBad[:] = (True if val2 else val1 for (val1,val2) in zip(globBad,badIndex))
                        globBad2[:] = (True if val2 else val1 for (val1,val2) in zip(globBad2,badIndex2))
                        globBad3[:] = (True if val2 else val1 for (val1,val2) in zip(globBad3,badIndex3))

                            # self.savePlots(self.fileName,plotdir,timeSeries,sensorType,lightDark,window,sigma,badIndex,badIndex2,badIndex3)
                    index += 1
                # Now plot a selection of these USING UNIVERSALLY EXCLUDED INDEXES
                index =0
                for timeSeries in columns.items():
                    band = float(timeSeries[0])
                    if band > self.minBand and band < self.maxBand:
                        if index % step == 0:
                            # self.savePlots(self.fileName,plotdir,timeSeries,sensorType,lightDark,window,sigma,globBad,globBad2,globBad3)
                            Utilities.saveDeglitchPlots(self.fileName,plotdir,timeSeries,dateTime,sensorType,lightDark,window,sigma,globBad,globBad2,globBad3)
                    index +=1

                print('Complete')

    def processButtonPressed(self):
        # Run L1D processing for this file

        inFilePath = os.path.join(self.inputDirectory, 'L1C', self.fileName+'.hdf')
        fileBaseName = self.fileName.split('_L1C')[0]
        outFilePath = os.path.join(self.inputDirectory, 'L1D', fileBaseName+'_L1D.hdf')

        # self.processL1d(inFilePath, outFilePath)
        # Jumpstart the logger:
        msg = "Process Single Level from Anomaly Analysis"
        os.environ["LOGFILE"] = (fileBaseName + '_L1C_L1D.log')
        Utilities.writeLogFile(msg,mode='w') # <<---- Logging initiated here
        root = Controller.processL1d(inFilePath, outFilePath)

        # In case of processing failure, write the report at this Process level, unless running stations
        #   Use numeric level for writeReport
        pathOut = outFilePath.split('L1D')[0]

        if root is None and ConfigFile.settings["bL2WriteReport"] == 1:
            Controller.writeReport(fileBaseName, pathOut, outFilePath, 'L1D', inFilePath)
        print('Process L1D complete')

    def closeButtonPressed(self):
        print('Done')
        self.close()

    def ThresholdCheckBoxUpdate(self):
        print("ThresholdCheckBoxUpdate")

        disabled = (not self.ThresholdCheckBox.isChecked())
        self.MinDarkLineEdit.setDisabled(disabled)
        self.MaxDarkLineEdit.setDisabled(disabled)
        self.MinLightLineEdit.setDisabled(disabled)
        self.MaxLightLineEdit.setDisabled(disabled)
        self.MinMaxDarkButton.setDisabled(disabled)
        self.MinMaxLightButton.setDisabled(disabled)

        if disabled:
            ConfigFile.settings["bL1dThreshold"] = 0
            setattr(self,"Threshold", 0)
        else:
            ConfigFile.settings["bL1dThreshold"] = 1
            setattr(self,"Threshold", 1)

            # # Problem: can either set to the threshold waveband or to the slider position, not both
            # if getattr(self,f'{self.sensor}MinMaxBandDark'):
            #     self.waveBand = getattr(self,f'{self.sensor}MinMaxBandDark')
            #     self.slider.setValue(self.waveBand)
            # if getattr(self,f'{self.sensor}MinMaxBandLight'):
            #     self.waveBand = getattr(self,f'{self.sensor}MinMaxBandLight')
            #     self.slider.setValue(self.waveBand)

    def MinMaxDarkButtonPressed(self):
        print('Updating waveband for Dark thresholds')
        # Test for root
        if not hasattr(self, 'root'):
            note = QtWidgets.QMessageBox()
            note.setText('You must load L1C file before continuing')
            note.exec_()
            return
        sensorType = self.sensor
        print(sensorType)

         # Set parameters in the local object from the GUI
        setattr(self,f'{self.sensor}MinMaxBandDark', self.waveBand)
        self.MinMaxDarkLabel.setText(str(self.waveBand) +'nm' )

    def MinMaxLightButtonPressed(self):
        print('Updating waveband for Light thresholds')
        # Test for root
        if not hasattr(self, 'root'):
            note = QtWidgets.QMessageBox()
            note.setText('You must load L1C file before continuing')
            note.exec_()
            return
        sensorType = self.sensor
        print(sensorType)

         # Set parameters in the local object from the GUI
        setattr(self,f'{self.sensor}MinMaxBandLight', self.waveBand)
        self.MinMaxLightLabel.setText(str(self.waveBand) +'nm' )

    @staticmethod
    def realTimePlot(self, radiometry1D, dateTime, sensorType,lightDark):
        # Radiometry at this point is 1D 'column' from the appropriate group/dataset/waveband
        #   in time (radiometry1D)

        styles = {'font-size': '18px'}
        # text_xlabel="Time Series"

        # Initialize the plot
        if lightDark == 'Dark':
            ph =  self.phDark
            phAve = self.phTimeAveDark
            ph1st = self.ph1stDark
            ph2nd = self.ph2ndDark
            ph3rd = self.ph3rdDark
            window = getattr(self,f'{sensorType}WindowDark')
            sigma = getattr(self,f'{sensorType}SigmaDark')
            minRad = getattr(self,f'{sensorType}MinDark')
            maxRad = getattr(self,f'{sensorType}MaxDark')
            minMaxBand = getattr(self,f'{sensorType}MinMaxBandDark')
            # Round/truncate waveband and add units
            if sensorType == 'ES':
                radUnits = self.root.attributes['ES_UNITS']
            else:
                radUnits = self.root.attributes['LI_UNITS']

            text_ylabel=f'[DARKS]  {sensorType}({self.waveBand:.0f}) [{radUnits}]'
            figTitle = f'Band: {self.waveBand} Window: {window} Sigma: {sigma}'
            # self.plotWidgetDark.setWindowTitle(figTitle)
            print(f'{figTitle} Dark')
            # self.plotWidgetDark.setWindowTitle(figTitle, **styles)
            self.plotWidgetDark.setLabel('left', text_ylabel,**styles)
            # self.plotWidgetDark.setLabel('bottom', text_xlabel,**styles)
            self.plotWidgetDark.showGrid(x=True, y=True)
            # ''' legend not working '''
            # self.plotWidgetDark.addLegend()
        else:
            ph = self.phLight
            phAve = self.phTimeAveLight
            ph1st = self.ph1stLight
            ph2nd = self.ph2ndLight
            ph3rd = self.ph3rdLight
            window = getattr(self,f'{sensorType}WindowLight')
            sigma = getattr(self,f'{sensorType}SigmaLight')
            minRad = getattr(self,f'{sensorType}MinLight')
            maxRad = getattr(self,f'{sensorType}MaxLight')
            minMaxBand = getattr(self,f'{sensorType}MinMaxBandLight')
            # Round/truncate waveband and add units
            if sensorType == 'ES':
                radUnits = self.root.attributes['ES_UNITS']
            else:
                radUnits = self.root.attributes['LI_UNITS']

            text_ylabel=f'[LIGHTS]  {sensorType}({self.waveBand:.0f}) [{radUnits}]'
            figTitle = f'Band: {self.waveBand} Window: {window} Sigma: {sigma}'
            print(f'{figTitle} Dark')
            # self.plotWidgetLight.setWindowTitle(figTitle, **styles)
            self.plotWidgetLight.setLabel('left', text_ylabel, **styles)
            # self.plotWidgetLight.setLabel('bottom', text_xlabel, **styles)
            self.plotWidgetLight.showGrid(x=True, y=True)
            # self.plotWidgetLight.addLegend()

        badIndex, badIndex2, badIndex3 = Utilities.deglitchBand(self.waveBand,radiometry1D, window, sigma, lightDark, minRad, maxRad,minMaxBand)
        avg = Utilities.movingAverage(radiometry1D, window).tolist()

        # ''' Use numeric series (x) for now in place of datetime '''
        # x = np.arange(0,len(radiometry1D),1)
        x=np.array([x.timestamp() for x in dateTime])

        # First Pass
        y_anomaly = np.array(radiometry1D)[badIndex]
        x_anomaly = x[badIndex]

        # Second Pass
        y_anomaly2 = np.array(radiometry1D)[badIndex2]
        x_anomaly2 = x[badIndex2]

        # Thresholds
        y_anomaly3 = np.array(radiometry1D)[badIndex3]
        x_anomaly3 = x[badIndex3]

        #Plot results
        try:
            ph.setData(x, radiometry1D, symbolPen='k', symbolBrush='w', \
                symbol='o', name=sensorType, pen=None)
            phAve.setData(x[3:-3], avg[3:-3], name='rMean', \
                pen=pg.mkPen('g', width=3))
            ph1st.setData(x_anomaly, y_anomaly, symbolPen=pg.mkPen('r', width=3),\
                symbol='x', name='1st pass')
            ph2nd.setData(x_anomaly2, y_anomaly2, symbolPen='r',\
                symbol='+', name='2nd pass')
            ph3rd.setData(x_anomaly3, y_anomaly3, symbolPen='r',\
                symbol='o', name='thresholds')

        except:
            e = sys.exc_info()[0]
            print("Error: %s" % e)

ext: py

max_stars_repo_name: doizuc/HyperInSPACE

max_issues_repo_name: doizuc/HyperInSPACE

max_forks_repo_name: doizuc/HyperInSPACE

reason: import numpy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright 2017    Ke Wang

"""Build the LSTM neural networks.
This module provides an example of definiting compute graph with tensorflow.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import sys

import numpy as np
import tensorflow as tf

from tensorflow.contrib.layers import batch_norm, fully_connected
from tensorflow.contrib.layers import xavier_initializer, l2_regularizer

sys.path.append(os.path.dirname(sys.path[0]))
from utils.ops import leakyrelu

class RES_LSTM_I(object):

    def __init__(self, lstm):
        self.lstm = lstm

    def __call__(self, inputs, labels, lengths, reuse=False):
        """Build LSTM model. On first pass will make vars."""
        self.inputs = inputs
        self.labels = labels
        self.lengths = lengths

        outputs = self.infer(reuse)

        return outputs

    def infer(self, reuse):
        lstm = self.lstm
        lstm_cell_size = 760
        num_projection = 257
        lstm_num_layer = 3
        in_dims = self.inputs.get_shape().as_list()
        assert len(in_dims) == 3

        if lstm.cross_validation:
            is_training = False
        else:
            is_training = True

        with tf.variable_scope("g_model") as scope:
            if reuse:
                scope.reuse_variables()

            if lstm.batch_norm:
                normalizer_fn = batch_norm
                normalizer_params = {
                    "is_training": is_training,
                    "scale": True,
                    "renorm": True
                }
            else:
                normalizer_fn = None
                normalizer_params = None

            if not is_training:
                lstm.keep_prob = 1.0

            if not reuse:
                print("****************************************")
                print("*** Generator summary ***")
                print("G inputs shape: {}".format(self.inputs.get_shape()))
            sys.stdout.flush()

            inputs = self.inputs
            # h = fully_connected(inputs, num_projection,
            #                     activation_fn=leakyrelu,
            #                     normalizer_fn=normalizer_fn,
            #                     normalizer_params=normalizer_params,
            #                     weights_initializer=xavier_initializer(),
            #                     biases_initializer=tf.zeros_initializer())

            def lstm_cell():
                return tf.contrib.rnn.LSTMCell(
                    lstm_cell_size, use_peepholes=True,
                    initializer=xavier_initializer(),
                    num_proj=num_projection,
                    forget_bias=1.0, state_is_tuple=True,
                    activation=tf.tanh,
                    reuse=reuse)
            attn_cell = lstm_cell

            if is_training and lstm.keep_prob < 1.0:
                def attn_cell():
                    return tf.contrib.rnn.DropoutWrapper(
                        lstm_cell(), output_keep_prob=lstm.keep_prob)

            with tf.variable_scope("lstm_cell_1"):
                cell1 = attn_cell()
                initial_states = cell1.zero_state(lstm.batch_size, tf.float32)
                outputs1, states1 = tf.nn.dynamic_rnn(cell1, self.inputs,
                                              sequence_length=self.lengths,
                                              initial_state=initial_states,
                                              dtype=tf.float32,
                                              time_major=False)

            with tf.variable_scope("lstm_cell_2"):
                inputs2 = outputs1 + self.inputs
                cell2 = attn_cell()
                initial_states = cell2.zero_state(lstm.batch_size, tf.float32)
                outputs2, states2 = tf.nn.dynamic_rnn(cell2, inputs2,
                                              sequence_length=self.lengths,
                                              initial_state=initial_states,
                                              dtype=tf.float32,
                                              time_major=False)

            # with tf.variable_scope("lstm_cell_3"):
            #     inputs3 = outputs2 + self.inputs
            #     cell3 = attn_cell()
            #     initial_states = cell3.zero_state(lstm.batch_size, tf.float32)
            #     outputs3, states3 = tf.nn.dynamic_rnn(cell3, inputs3,
            #                                   sequence_length=self.lengths,
            #                                   initial_state=initial_states,
            #                                   dtype=tf.float32,
            #                                   time_major=False)

            # with tf.variable_scope("lstm_cell_4"):
            #     inputs4 = outputs3 + self.inputs
            #     cell4 = attn_cell()
            #     initial_states = cell4.zero_state(lstm.batch_size, tf.float32)
            #     outputs4, states4 = tf.nn.dynamic_rnn(cell4, inputs4,
            #                                   sequence_length=self.lengths,
            #                                   initial_state=initial_states,
            #                                   dtype=tf.float32,
            #                                   time_major=False)

            # with tf.variable_scope("lstm_cell_5"):
            #     inputs5 = outputs4 + self.inputs
            #     cell5 = attn_cell()
            #     initial_states = cell5.zero_state(lstm.batch_size, tf.float32)
            #     outputs5, states5 = tf.nn.dynamic_rnn(cell5, inputs5,
            #                                   sequence_length=self.lengths,
            #                                   initial_state=initial_states,
            #                                   dtype=tf.float32,
            #                                   time_major=False)

            # with tf.variable_scope("lstm_cell_6"):
            #     inputs6 = outputs5 + self.inputs
            #     cell6 = attn_cell()
            #     initial_states = cell6.zero_state(lstm.batch_size, tf.float32)
            #     outputs6, states6 = tf.nn.dynamic_rnn(cell6, inputs6,
            #                                   sequence_length=self.lengths,
            #                                   initial_state=initial_states,
            #                                   dtype=tf.float32,
            #                                   time_major=False)

            # with tf.variable_scope("lstm_cell_7"):
            #     inputs7 = outputs6 + self.inputs
            #     cell7 = attn_cell()
            #     initial_states = cell7.zero_state(lstm.batch_size, tf.float32)
            #     outputs7, states7 = tf.nn.dynamic_rnn(cell7, inputs7,
            #                                   sequence_length=self.lengths,
            #                                   initial_state=initial_states,
            #                                   dtype=tf.float32,
            #                                   time_major=False)

            # with tf.variable_scope("lstm_cell_8"):
            #     inputs8 = outputs7 + self.inputs
            #     cell8 = attn_cell()
            #     initial_states = cell8.zero_state(lstm.batch_size, tf.float32)
            #     outputs8, states8 = tf.nn.dynamic_rnn(cell8, inputs8,
            #                                   sequence_length=self.lengths,
            #                                   initial_state=initial_states,
            #                                   dtype=tf.float32,
            #                                   time_major=False)

            if not reuse:
                print("G hidden layer number is {}".format(lstm_num_layer))
                print("G cell size is {}".format(lstm_cell_size))
                print("G projection num is {}".format(num_projection))
            sys.stdout.flush()

            # Linear output
            with tf.variable_scope("forward_out"):
                # inputs9 = outputs8 + self.inputs
                # inputs9 = outputs4 + self.inputs
                inputs9 = outputs2 + self.inputs
                y = fully_connected(inputs9, lstm.output_dim,
                                    activation_fn=None,
                                    weights_initializer=xavier_initializer(),
                                    biases_initializer=tf.zeros_initializer())
            if not reuse:
                print("G output shape: {}".format(y.get_shape()))
                sys.stdout.flush()

        return y

max_stars_repo_name: wangkenpu/rsrgan

max_issues_repo_name: wangkenpu/rsrgan

max_forks_repo_name: wangkenpu/rsrgan


text:
import glob, os, time, sys
import numpy as np
import scipy.linalg as sl, scipy.stats, scipy.special

class OutlierGibbs(object):

    """Gibbs-based pulsar-timing outlier analysis.

    Based on:

        Article by Tak, Ellis, Ghosh:
        "Robust and Accurate Inference via a Mixture
        of Gaussian and Student's t Errors",
        https://doi.org/10.1080/10618600.2018.1537925
        arXiv:1707.03057.

        Article by Wang, Taylor:
        "Controlling Outlier Contamination In Multimessenger
        Time-domain Searches For Supermasssive Binary Black Holes",
        (In prep. 2021)

        Code from https://github.com/jellis18/gibbs_student_t

    Authors:

        J. A. Ellis, S. R. Taylor, J. Wang

    Example usage:

        > gibbs = OutlierGibbs(pta, model='mixture', vary_df=True,
                        theta_prior='beta', vary_alpha=True)
        > params = np.array([p.sample() for p in gibbs.params]).flatten()
        > gibbs.sample(params, outdir='./outlier/',
                       niter=10000, resume=False)
        > poutlier = np.mean(gibbs.poutchain, axis = 0)
        # Gives marginalized outlier probability of each TOA

    """

    def __init__(self, pta, model='mixture',
                 m=0.01, tdf=4, vary_df=True,
                 theta_prior='beta',
                 alpha=1e10, vary_alpha=True,
                 pspin=None):
        """
        Parameters
        -----------
        pta : object
            instance of a pta object for a single pulsar
        model : str
            type of outlier model
            [default = mixture of Gaussian and Student's t]
        tdf : int
            degrees of freedom for Student's t outlier distribution
            [default = 4]
        m : float
            a-priori proportion of observations that are outliers
            [default = 0.01]
        vary_df : boolean
            vary the Student's t degrees of freedom
            [default = True]
        theta_prior : str
            prior outlier probability
            [default = beta distribution]
        alpha : float
            relative width of outlier to inlier distribution
            [default = 1e10]
        vary_alpha : boolean
            vary the relative outlier to inlier width
            [default = True]
        pspin : float
            pulsar spin period for vvh17 model, arXiv:1609.02144
            [default = None]
        """

        self.pta = pta
        if np.any(['basis_ecorr' in key for
                   key in self.pta._signal_dict.keys()]):
            pass
        else:
            print('ERROR: Gibbs outlier analysis must use basis_ecorr, not kernel ecorr')

        # a-priori proportion of observations that are outliers
        self.mp = m
        # a-priori outlier probability distribution
        self.theta_prior = theta_prior

        # spin period
        self.pspin = pspin

        # vary t-distribution d.o.f
        self.vary_df = vary_df

        # vary alpha
        self.vary_alpha = vary_alpha

        # For now assume one pulsar
        self._residuals = self.pta.get_residuals()[0]

        # which likelihood model
        self._lmodel = model

        # auxiliary variable stuff
        xs = [p.sample() for p in pta.params]
        self._b = np.zeros(self.pta.get_basis(xs)[0].shape[1])

        # for caching
        self.TNT = None
        self.d = None

        # outlier detection variables
        self._pout = np.zeros_like(self._residuals)
        self._z = np.zeros_like(self._residuals)
        if not vary_alpha:
            self._alpha = np.ones_like(self._residuals) * alpha
        else:
            self._alpha = np.ones_like(self._residuals)
        self._theta = self.mp
        self.tdf = tdf
        if model in ['t', 'mixture', 'vvh17']:
            self._z = np.ones_like(self._residuals)

    @property
    def params(self):
        ret = []
        for param in self.pta.params:
            ret.append(param)
        return ret

    def map_params(self, xs):
        return {par.name: x for par,
                x in zip(self.params, xs)}

    def get_hyper_param_indices(self):
        ind = []
        for ct, par in enumerate(self.params):
            if 'ecorr' in par.name or 'log10_A' in par.name or 'gamma' in par.name:
                ind.append(ct)
        return np.array(ind)

    def get_white_noise_indices(self):
        ind = []
        for ct, par in enumerate(self.params):
            if 'efac' in par.name or 'equad' in par.name:
                ind.append(ct)
        return np.array(ind)

    def update_hyper_params(self, xs):

        # get hyper parameter indices
        hind = self.get_hyper_param_indices()

        # get initial log-likelihood and log-prior
        lnlike0, lnprior0 = self.get_lnlikelihood(xs), self.get_lnprior(xs)
        xnew = xs.copy()
        for ii in range(10):

            # standard gaussian jump (this allows for different step sizes)
            q = xnew.copy()
            sigmas = 0.05 * len(hind)
            probs = [0.1, 0.15, 0.5, 0.15, 0.1]
            sizes = [0.1, 0.5, 1.0, 3.0, 10.0]
            scale = np.random.choice(sizes, p=probs)
            par = np.random.choice(hind, size=1) # only one hyper param at a time
            q[par] += np.random.randn(len(q[par])) * sigmas * scale

            # get log-like and log prior at new position
            lnlike1, lnprior1 = self.get_lnlikelihood(q), self.get_lnprior(q)

            # metropolis step
            diff = (lnlike1 + lnprior1) - (lnlike0 + lnprior0)
            if diff > np.log(np.random.rand()):
                xnew = q
                lnlike0 = lnlike1
                lnprior0 = lnprior1
            else:
                xnew = xnew

        return xnew

    def update_white_params(self, xs):

        # get white noise parameter indices
        wind = self.get_white_noise_indices()

        xnew = xs.copy()
        lnlike0, lnprior0 = self.get_lnlikelihood_white(xnew), self.get_lnprior(xnew)
        for ii in range(20):

            # standard gaussian jump (this allows for different step sizes)
            q = xnew.copy()
            sigmas = 0.05 * len(wind)
            probs = [0.1, 0.15, 0.5, 0.15, 0.1]
            sizes = [0.1, 0.5, 1.0, 3.0, 10.0]
            scale = np.random.choice(sizes, p=probs)
            par = np.random.choice(wind, size=1)
            q[par] += np.random.randn(len(q[par])) * sigmas * scale

            # get log-like and log prior at new position
            lnlike1, lnprior1 = self.get_lnlikelihood_white(q), self.get_lnprior(q)

            # metropolis step
            diff = (lnlike1 + lnprior1) - (lnlike0 + lnprior0)
            if diff > np.log(np.random.rand()):
                xnew = q
                lnlike0 = lnlike1
                lnprior0 = lnprior1
            else:
                xnew = xnew
        return xnew

    def update_b(self, xs):

        # map parameter vector
        params = self.map_params(xs)

        # start likelihood calculations
        loglike = 0

        # get auxiliaries
        Nvec = self._alpha**self._z * self.pta.get_ndiag(params)[0]
        phiinv = self.pta.get_phiinv(params, logdet=False)[0]
        residuals = self._residuals

        T = self.pta.get_basis(params)[0]
        if self.TNT is None and self.d is None:
            self.TNT = np.dot(T.T, T / Nvec[:,None])
            self.d = np.dot(T.T, residuals/Nvec)
        #d = self.pta.get_TNr(params)[0]
        #TNT = self.pta.get_TNT(params)[0]

        # Red noise piece
        Sigma = self.TNT + np.diag(phiinv)

        try:
            u, s, _ = sl.svd(Sigma)
            mn = np.dot(u, np.dot(u.T, self.d)/s)
            Li = u * np.sqrt(1/s)
        except np.linalg.LinAlgError:

            Q, R = sl.qr(Sigma)
            Sigi = sl.solve(R, Q.T)
            mn = np.dot(Sigi, self.d)
            u, s, _ = sl.svd(Sigi)
            Li = u * np.sqrt(1/s)

        b = mn + np.dot(Li, np.random.randn(Li.shape[0]))

        return b

    def update_theta(self, xs):

        if self._lmodel in ['t', 'gaussian']:
            return self._theta
        elif self._lmodel in ['mixture', 'vvh17']:
            n = len(self._residuals)
            if self.theta_prior == 'beta':
                mk = n * self.mp
                k1mm = n * (1-self.mp)
            else:
                mk, k1mm = 1.0, 1.0
            # from Tak, Ellis, Ghosh (2018): k = sample size, m = 0.01
            ret = scipy.stats.beta.rvs(np.sum(self._z) + mk,
                                       n - np.sum(self._z) + k1mm)
            return ret

    def update_z(self, xs):

        # map parameters
        params = self.map_params(xs)

        if self._lmodel in ['t', 'gaussian']:
            return self._z
        elif self._lmodel in ['mixture', 'vvh17']:
            Nvec0 = self.pta.get_ndiag(params)[0]
            Tmat = self.pta.get_basis(params)[0]

            Nvec = self._alpha * Nvec0
            theta_mean = np.dot(Tmat, self._b)
            top = self._theta * scipy.stats.norm.pdf(self._residuals,
                                                     loc=theta_mean,
                                                     scale=np.sqrt(Nvec))
            if self._lmodel == 'vvh17':
                top = self._theta / self.pspin

            bot = top + (1-self._theta) * scipy.stats.norm.pdf(self._residuals,
                                                               loc=theta_mean,
                                                               scale=np.sqrt(Nvec0))
            q = top / bot
            q[np.isnan(q)] = 1
            self._pout = q

            return scipy.stats.binom.rvs(1, list(map(lambda x: min(x, 1), q)))

    def update_alpha(self, xs):

        # map parameters
        params = self.map_params(xs)

        # equation 12 of Tak, Ellis, Ghosh
        if np.sum(self._z) >= 1 and self.vary_alpha:
            Nvec0 = self.pta.get_ndiag(params)[0]
            Tmat = self.pta.get_basis(params)[0]
            theta_mean = np.dot(Tmat, self._b)
            top = ((self._residuals - theta_mean)**2 *
                   self._z / Nvec0 + self.tdf) / 2
            bot = scipy.stats.gamma.rvs((self._z +
                                         self.tdf) / 2)
            return top / bot
        else:
            return self._alpha

    def update_df(self, xs):

        if self.vary_df:
            # 1. evaluate the log conditional posterior of df for 1, 2, ..., 30.
            log_den_df = np.array(list(map(self.get_lnlikelihood_df,
                                           np.arange(1,31))))

            # 2. normalize the probabilities
            den_df = np.exp(log_den_df - log_den_df.max())
            den_df /= den_df.sum()

            # 3. sample one of values (1, 2, ..., 30) according to the probabilities
            df = np.random.choice(np.arange(1, 31), p=den_df)

            return df
        else:
            return self.tdf

    def get_lnlikelihood_white(self, xs):

        # map parameters
        params = self.map_params(xs)
        matrix = self.pta.get_ndiag(params)[0]

        # Nvec and Tmat
        Nvec = self._alpha**self._z * matrix
        Tmat = self.pta.get_basis(params)[0]

        # whitened residuals
        mn = np.dot(Tmat, self._b)
        yred = self._residuals - mn

        # log determinant of N
        logdet_N = np.sum(np.log(Nvec))

        # triple product in likelihood function
        rNr = np.sum(yred**2/Nvec)

        # first component of likelihood function
        loglike = -0.5 * (logdet_N + rNr)

        return loglike

    def get_lnlikelihood(self, xs):

        # map parameter vector
        params = self.map_params(xs)

        # start likelihood calculations
        loglike = 0

        # get auxiliaries
        Nvec = self._alpha**self._z * self.pta.get_ndiag(params)[0]
        phiinv, logdet_phi = self.pta.get_phiinv(params,
                                                 logdet=True)[0]
        residuals = self._residuals

        T = self.pta.get_basis(params)[0]
        if self.TNT is None and self.d is None:
            self.TNT = np.dot(T.T, T / Nvec[:,None])
            self.d = np.dot(T.T, residuals/Nvec)

        # log determinant of N
        logdet_N = np.sum(np.log(Nvec))

        # triple product in likelihood function
        rNr = np.sum(residuals**2/Nvec)

        # first component of likelihood function
        loglike += -0.5 * (logdet_N + rNr)

        # Red noise piece
        Sigma = self.TNT + np.diag(phiinv)

        try:
            cf = sl.cho_factor(Sigma)
            expval = sl.cho_solve(cf, self.d)
        except np.linalg.LinAlgError:
            return -np.inf

        logdet_sigma = np.sum(2 * np.log(np.diag(cf[0])))
        loglike += 0.5 * (np.dot(self.d, expval) -
                          logdet_sigma - logdet_phi)

        return loglike

    def get_lnlikelihood_df(self, df):
        n = len(self._residuals)
        ll = -(df/2) * np.sum(np.log(self._alpha)+1/self._alpha) + \
            n * (df/2) * np.log(df/2) - n*scipy.special.gammaln(df/2)
        return ll

    def get_lnprior(self, xs):

        return sum(p.get_logpdf(x) for p, x
                   in zip(self.params, xs))

    def sample(self, xs, outdir='./', niter=10000, resume=False):

        print(f'Creating chain directory: {outdir}')
        os.system(f'mkdir -p {outdir}')

        self.chain = np.zeros((niter, len(xs)))
        self.bchain = np.zeros((niter, len(self._b)))
        self.thetachain = np.zeros(niter)
        self.zchain = np.zeros((niter, len(self._residuals)))
        self.alphachain = np.zeros((niter, len(self._residuals)))
        self.poutchain = np.zeros((niter, len(self._residuals)))
        self.dfchain = np.zeros(niter)

        self.iter = 0
        startLength = 0
        xnew = xs
        if resume:
            print('Resuming from previous run...')
            # read in previous chains
            tmp_chains = []
            tmp_chains.append(np.loadtxt(f'{outdir}/chain.txt'))
            tmp_chains.append(np.loadtxt(f'{outdir}/bchain.txt'))
            tmp_chains.append(np.loadtxt(f'{outdir}/thetachain.txt'))
            tmp_chains.append(np.loadtxt(f'{outdir}/zchain.txt'))
            tmp_chains.append(np.loadtxt(f'{outdir}/alphachain.txt'))
            tmp_chains.append(np.loadtxt(f'{outdir}/poutchain.txt'))
            tmp_chains.append(np.loadtxt(f'{outdir}/dfchain.txt'))

            # find minimum length
            minLength = np.min([tmp.shape[0] for tmp in tmp_chains])

            # take only the minimum length entries of each chain
            tmp_chains = [tmp[:minLength] for tmp in tmp_chains]

            # pad with zeros if shorter than niter
            self.chain[:tmp_chains[0].shape[0]] = tmp_chains[0]
            self.bchain[:tmp_chains[1].shape[0]] = tmp_chains[1]
            self.thetachain[:tmp_chains[2].shape[0]] = tmp_chains[2]
            self.zchain[:tmp_chains[3].shape[0]] = tmp_chains[3]
            self.alphachain[:tmp_chains[4].shape[0]] = tmp_chains[4]
            self.poutchain[:tmp_chains[5].shape[0]] = tmp_chains[5]
            self.dfchain[:tmp_chains[6].shape[0]] = tmp_chains[6]

            # set new starting point for sampling
            startLength = minLength
            xnew = self.chain[startLength-1]

        tstart = time.time()
        for ii in range(startLength, niter):
            self.iter = ii
            self.chain[ii, :] = xnew
            self.bchain[ii,:] = self._b
            self.zchain[ii,:] = self._z
            self.thetachain[ii] = self._theta
            self.alphachain[ii,:] = self._alpha
            self.dfchain[ii] = self.tdf
            self.poutchain[ii, :] = self._pout

            self.TNT = None
            self.d = None

            # update white parameters
            xnew = self.update_white_params(xnew)

            # update hyper-parameters
            xnew = self.update_hyper_params(xnew)

            # if accepted update quadratic params
            if np.all(xnew != self.chain[ii,-1]):
                self._b = self.update_b(xnew)

            # update outlier model params
            self._theta = self.update_theta(xnew)
            self._z = self.update_z(xnew)
            self._alpha = self.update_alpha(xnew)
            self.tdf = self.update_df(xnew)

            if ii % 100 == 0 and ii > 0:
                sys.stdout.write('\r')
                sys.stdout.write('Finished %g percent in %g seconds.'%(ii / niter * 100,
                                                                       time.time()-tstart))
                sys.stdout.flush()
                np.savetxt(f'{outdir}/chain.txt', self.chain[:ii+1, :])
                np.savetxt(f'{outdir}/bchain.txt', self.bchain[:ii+1, :])
                np.savetxt(f'{outdir}/thetachain.txt', self.thetachain[:ii+1])
                np.savetxt(f'{outdir}/zchain.txt', self.zchain[:ii+1, :])
                np.savetxt(f'{outdir}/alphachain.txt', self.alphachain[:ii+1, :])
                np.savetxt(f'{outdir}/poutchain.txt', self.poutchain[:ii+1, :])
                np.savetxt(f'{outdir}/dfchain.txt', self.dfchain[:ii+1])

    def marg_outlierprob(self, burn=None):

        if burn is None:
            burn = int(0.25*self.poutchain.shape[0])

        return np.mean(self.poutchain[burn:,:self.iter], axis = 0)

reason: import numpy,import scipy