# Repository: https://gitlab.com/quantify-os/quantify-core
# Licensed according to the LICENCE file on the main branch
"""Module containing the MeasurementControl."""
from __future__ import annotations

import itertools
import math
import signal
import tempfile
import threading
import time
import types
from collections.abc import Iterable
from itertools import chain
from pathlib import Path
from typing import Any, Dict, Optional
from tqdm.auto import tqdm

import adaptive
import numpy as np
import xarray as xr
from filelock import FileLock
from qcodes import validators as vals
from qcodes.instrument import Instrument, InstrumentChannel
from qcodes.parameters import InstrumentRefParameter, ManualParameter

from quantify_core import __version__ as _quantify_version
from quantify_core.data.experiment import QuantifyExperiment
from quantify_core.data.handling import (
    DATASET_NAME,
    _is_uniformly_spaced_array,
    create_exp_folder,
    grow_dataset,
    initialize_dataset,
    snapshot,
    trim_dataset,
)
from quantify_core.measurement.types import Gettable, Settable, is_batched
from quantify_core.utilities.general import call_if_has_method

# Intended for plotting monitors that run in separate processes
_DATASET_LOCKS_DIR = Path(tempfile.gettempdir())

class MeasurementControl(Instrument):  # pylint: disable=too-many-instance-attributes
    """
    Instrument responsible for controlling the data acquisition loop.

    MeasurementControl (MC) is based on the notion that every experiment consists of
    the following steps:

        1. Set some parameter(s)            (settable_pars)
        2. Measure some other parameter(s)  (gettable_pars)
        3. Store the data.

    Example:

        .. code-block:: python

            meas_ctrl.settables(mw_source1.freq)
            meas_ctrl.setpoints(np.arange(5e9, 5.2e9, 100e3))
            meas_ctrl.gettables(pulsar_QRM.signal)
            dataset = meas_ctrl.run(name='Frequency sweep')

    MC exists to enforce structure on experiments. Enforcing this structure allows:

        - Standardization of data storage.
        - Providing basic real-time visualization.

    MC imposes minimal constraints and allows:

    - Iterative loops, experiments in which setpoints are processed step by step.
    - Batched loops, experiments in which setpoints are processed in batches.
    - Adaptive loops, setpoints are determined based on measured values.

    .. seealso:: :ref:`Measurement Control How-To <howto-measurement-control>`

    Parameters
    ----------
    name
        name of this instrument.
    """

    def __init__(self, name: str):
        super().__init__(name=name)

        # Parameters are attributes included in logging and which the user can change.

        self.lazy_set = ManualParameter(
            vals=vals.Bool(),
            initial_value=False,
            name="lazy_set",
            instrument=self,
        )
        """If set to ``True``, only set any settable if the setpoint differs
        from the previous setpoint. Note that this parameter is overridden by the
        ``lazy_set`` argument passed to the :meth:`.run` and :meth:`.run_adaptive`
        methods."""

        self.verbose = ManualParameter(
            vals=vals.Bool(),
            initial_value=True,
            instrument=self,
            name="verbose",
        )
        """If set to ``True``, prints to ``std_out`` during experiments."""

        self.on_progress_callback = ManualParameter(
            vals=vals.Callable(),
            instrument=self,
            name="on_progress_callback",
        )
        """A callback to communicate progress. This should be a callable accepting
        floats between 0 and 100 indicating the percentage done."""

        self.instr_plotmon = InstrumentRefParameter(
            vals=vals.MultiType(vals.Strings(), vals.Enum(None)),
            instrument=self,
            name="instr_plotmon",
        )
        """Instrument responsible for live plotting. Can be set to ``None`` to disable
        live plotting."""

        self.update_interval = ManualParameter(
            initial_value=0.5,
            vals=vals.Numbers(min_value=0.1),
            instrument=self,
            name="update_interval",
        )
        """Interval for updates during the data acquisition loop, every time more than
        :attr:`.update_interval` time has elapsed when acquiring new data points, data
        is written to file (and the live monitoring detects updated)."""

        # Add experiment_data submodule to allow user to save custom metadata
        experiment_data = InstrumentChannel(self, "experiment_data")
        self.add_submodule("experiment_data", experiment_data)

        self._soft_avg_validator = vals.Ints(1, int(1e8)).validate

        # variables that are set before the start of any experiment.
        self._settable_pars = []
        self._setpoints = []
        self._setpoints_input = []
        self._gettable_pars = []

        # variables used for book keeping during acquisition loop.
        self._soft_avg = 1
        self._nr_acquired_values = 0
        self._loop_count = 0
        self._begintime = time.time()
        self._last_upd = time.time()
        self._batch_size_last = None
        self._dataarray_cache: Optional[Dict[str, Any]] = None

        # variables used for persistence, plotting and data handling
        self._dataset = None
        self._exp_folder: Path = None
        self._experiment = None
        self._plotmon_name = ""
        # attributes named as if they are python attributes, e.g. dset.drid_2d == True
        self._plot_info = {
            "grid_2d": False,
            "grid_2d_uniformly_spaced": False,
            "1d_2_settables_uniformly_spaced": False,
        }

        # properly handling KeyboardInterrupts
        self._thread_data = threading.local()
        # counter for KeyboardInterrupts to allow forced interrupt
        self._thread_data.events_num = 0

    def __repr__full__(self):
        str_out = super().__repr__() + "\n"

        # hasattr is necessary in case the instrument was closed
        if hasattr(self, "_settable_pars"):
            settable_names = [p.name for p in self._settable_pars]
            str_out += f"    settables: {settable_names}\n"

        if hasattr(self, "_gettable_pars"):
            gettable_names = [p.name for p in self._gettable_pars]
            str_out += f"    gettables: {gettable_names}\n"

        if hasattr(self, "_setpoints_input") and self._setpoints_input is not None:
            input_shapes = [
                np.asarray(points).shape for points in self._setpoints_input
            ]
            str_out += f"    setpoints_grid input shapes: {input_shapes}\n"

        if hasattr(self, "_setpoints") and self._setpoints is not None:
            str_out += f"    setpoints shape: {np.asarray(self._setpoints).shape}\n"

        return str_out

    def __repr__(self):
        """
        Returns a string containing a summary of this object regarding settables,
        gettables and setpoints.

        Intended, for example, to give a more useful representation in interactive
        shells.
        """
        return self.__repr__full__()

    def get_idn(self) -> dict[str, str | None]:
        return {
            "vendor": "Quantify",
            "model": f"{self.__module__}.{self.__class__.__name__}",
            "serial": self.name,
            "firmware": _quantify_version,
        }

    def show(self):
        """Print short representation of the object to stdout."""
        print(self.__repr__full__())

    def set_experiment_data(
        self, experiment_data: Dict[str, Any], overwrite: bool = True
    ):
        """
        Populates the experiment_data submodule with experiment_data parameters

        Parameters
        -----------
        experiment_data:
            Dict specifying the names of the experiment_data parameters and their
            values. Follows the format:

            .. code-block:: python

                {
                    "parameter_name": {
                        "value": 10.2
                        "label": "parameter label"
                        "unit": "Hz"
                    }
                }

        overwrite:
            If True, clear all previously saved experiment_data parameters and save new
            ones.
            If False, keep all previously saved experiment_data parameters and change
            their values if necessary
        """
        if overwrite:
            self.clear_experiment_data()

        for name, parameter in experiment_data.items():
            if name not in self.experiment_data.parameters:
                self.experiment_data.add_parameter(
                    name=name, parameter_class=ManualParameter
                )

            self.experiment_data.parameters[name](parameter.get("value"))
            self.experiment_data.parameters[name].label = parameter.get("label", name)
            self.experiment_data.parameters[name].unit = parameter.get("unit", "")

    def clear_experiment_data(self):
        """
        Remove all experiment_data parameters from the experiment_data submodule
        """
        self.experiment_data.parameters = {}

    ############################################
    # Methods used to control the measurements #
    ############################################

    def _reset(self, save_data=True):
        """
        Resets all experiment specific variables for a new run.
        """
        self._nr_acquired_values = 0
        self._loop_count = 0
        self._begintime = time.time()
        self._batch_size_last = None
        # Reset KeyboardInterrupt counter
        self._thread_data.events_num = 0
        # Assign handler to interrupt signal
        signal.signal(signal.SIGINT, self._interrupt_handler)
        self._save_data = save_data
        self._dataarray_cache = None

    def _reset_post(self):
        """
        Resets specific variables that can change before `.run()`.
        """
        self._plot_info = {
            "grid_2d": False,
            "grid_2d_uniformly_spaced": False,
            "1d_2_settables_uniformly_spaced": False,
        }
        # Reset to default interrupt handler
        signal.signal(signal.SIGINT, signal.default_int_handler)

        # Make sure tqdm progress bar attribute is closed and removed if mc is interrupted and shot down gracefully
        if self.verbose() and hasattr(self, "pbar"):
            self.pbar.close()
            del self.pbar

    def _init(self, name):
        """
        Initializes MC, such as creating the Dataset, experiment folder and such.
        """
        # needs to be calculated here because we need the settables' `.batched`
        if self._setpoints is None:
            self._setpoints = grid_setpoints(self._setpoints_input, self._settable_pars)

        # initialize an empty dataset
        self._dataset = initialize_dataset(
            self._settable_pars, self._setpoints, self._gettable_pars
        )

        self._dataset.attrs["name"] = name
        # cannot add it as a separate (nested) dict so make it flat.
        self._dataset.attrs.update(self._plot_info)

        tuid = self._dataset.attrs["tuid"]

        self._experiment = QuantifyExperiment(tuid=tuid)
        if self._save_data:
            self._exp_folder = Path(create_exp_folder(tuid=tuid, name=name))
            self._safe_write_dataset()  # Write the empty dataset

            snap = snapshot(update=False, clean=True)  # Save a snapshot of all
            self._experiment.save_snapshot(snap)
        else:
            self._exp_folder = None

        if self.instr_plotmon():
            # Tell plotmon to start monitoring the new dataset
            self.instr_plotmon.get_instr().update(tuid=tuid)

    def run(
        self,
        name: str = "",
        soft_avg: int = 1,
        lazy_set: Optional[bool] = None,
        save_data: bool = True,
    ) -> xr.Dataset:
        """
        Starts a data acquisition loop.

        Parameters
        ----------
        name
            Name of the measurement. It is included in the name of the data files.
        soft_avg
            Number of software averages to be performed by the measurement control.
            E.g. if `soft_avg=3` the full dataset will be measured 3 times and the
            measured values will be averaged element-wise, the averaged dataset is then
            returned.
        lazy_set
            If ``True`` and a setpoint equals the previous setpoint, the ``.set`` method
            of the settable will not be called for that iteration.
            If this argument is ``None``, the ``.lazy_set()`` ManualParameter is used
            instead (which by default is ``False``).

            .. warning:: This feature is not available yet when running in batched mode.
        save_data
            If ``True`` that the measurement data is stored.
        """
        lazy_set = lazy_set if lazy_set is not None else self.lazy_set()
        self._soft_avg_validator(soft_avg)  # validate first
        self._soft_avg = soft_avg
        self._reset(save_data=save_data)
        self._init(name)

        self._prepare_settables()

        try:
            if self._get_is_batched():
                if self.verbose():
                    print("Starting batched measurement...")
                self._run_batched()
            else:
                if self.verbose():
                    print("Starting iterative measurement...")
                self._run_iterative(lazy_set)
        except KeyboardInterrupt:
            print("\nInterrupt signaled, exiting gracefully...")

        if self._save_data:
            self._safe_write_dataset()  # Wrap up experiment and store data
        self._finish()
        self._reset_post()

        self._check_interrupt()  # Propagate interruption

        return self._dataset

    def run_adaptive(self, name, params, lazy_set: Optional[bool] = None) -> xr.Dataset:
        """
        Starts a data acquisition loop using an adaptive function.

        .. warning ::
            The functionality of this mode can be complex - it is recommended to read
            the relevant long form documentation.

        Parameters
        ----------
        name
            Name of the measurement. This name is included in the name of the data
            files.
        params
            Key value parameters describe the adaptive function to use, and any further
            parameters for that function.
        lazy_set
            If ``True`` and a setpoint equals the previous setpoint, the ``.set`` method
            of the settable will not be called for that iteration.
            If this argument is ``None``, the ``.lazy_set()`` ManualParameter is used
            instead (which by default is ``False``).
        """
        lazy_set = lazy_set if lazy_set is not None else self.lazy_set()

        def measure(vec) -> float:
            """
            This function executes the measurement and is passed to the adaptive
            function (often a minimization algorithm) to be evaluated many times.

            Although the measure function acquires (and stores) all gettable parameters,
            only the first value is returned to match the function signature for a valid
            measurement function.
            """
            if len(self._dataset["y0"]) == self._nr_acquired_values:
                self._dataset = grow_dataset(self._dataset)

            #  1D sweeps return single values, wrap in a list
            if np.isscalar(vec):
                vec = [vec]

            self._iterative_set_and_get(vec, self._nr_acquired_values, lazy_set)
            # only y0 is returned so as to match the function signature for a valid
            # measurement function.
            ret = self._dataset["y0"].values[self._nr_acquired_values]
            self._nr_acquired_values += 1
            self._update(".")
            self._check_interrupt()
            return ret

        def subroutine():
            self._prepare_settables()
            self._prepare_gettables()

            adaptive_function = params.get("adaptive_function")
            af_pars_copy = dict(params)

            # if the adaptive function is part of the python adaptive library
            if isinstance(adaptive_function, type) and issubclass(
                adaptive_function, adaptive.learner.BaseLearner
            ):
                goal = af_pars_copy["goal"]
                unusued_pars = ["adaptive_function", "goal"]
                for unusued_par in unusued_pars:
                    af_pars_copy.pop(unusued_par, None)
                learner = adaptive_function(measure, **af_pars_copy)
                adaptive.runner.simple(learner, goal)

            # any adaptive function with the right signature
            elif isinstance(adaptive_function, types.FunctionType):
                unused_pars = ["adaptive_function"]
                for unused_par in unused_pars:
                    af_pars_copy.pop(unused_par, None)
                adaptive_function(measure, **af_pars_copy)

        self._reset()
        self.setpoints(
            np.empty((64, len(self._settable_pars)))
        )  # block out some space in the dataset
        self._init(name)

        try:
            print("Running adaptively...")
            subroutine()
        except KeyboardInterrupt:
            print("\nInterrupt signaled, exiting gracefully...")

        self._finish()
        self._dataset = trim_dataset(self._dataset)
        self._safe_write_dataset()  # Wrap up experiment and store data

        self._check_interrupt()  # Propagate interruption

        return self._dataset

    def _run_iterative(self, lazy_set: bool = False):
        while self._get_fracdone() < 1.0:
            self._prepare_gettables()

            self._dataarray_cache = {}
            for row in self._setpoints:
                self._iterative_set_and_get(row, self._curr_setpoint_idx(), lazy_set)
                self._nr_acquired_values += 1
                self._update()
                self._check_interrupt()
            self._dataarray_cache = None
            self._loop_count += 1

    def _run_batched(self):  # pylint: disable=too-many-locals
        batch_size = self._get_batch_size()
        where_batched = self._get_where_batched()
        where_iterative = self._get_where_iterative()
        batched_settables = self._get_batched_settables()
        iterative_settables = self._get_iterative_settables()

        if self.verbose():
            print(
                "Iterative settable(s) [outer loop(s)]:\n\t",
                ", ".join(par.name for par in iterative_settables) or "--- (None) ---",
                "\nBatched settable(s):\n\t",
                ", ".join(par.name for par in batched_settables),
                f"\nBatch size limit: {batch_size:d}\n",
            )
        while self._get_fracdone() < 1.0:
            setpoint_idx = self._curr_setpoint_idx()
            self._batch_size_last = batch_size
            slice_len = setpoint_idx + self._batch_size_last
            for i, spar in enumerate(iterative_settables):
                # Here ensure that all setpoints of each iterative settable are the same
                # within each batch
                val, iterator = next(
                    itertools.groupby(
                        self._setpoints[setpoint_idx:slice_len, where_iterative[i]]
                    )
                )
                spar.set(val)
                # We also determine the size of each next batch
                self._batch_size_last = min(self._batch_size_last, len(tuple(iterator)))

            slice_len = setpoint_idx + self._batch_size_last
            for i, spar in enumerate(batched_settables):
                pnts = self._setpoints[setpoint_idx:slice_len, where_batched[i]]
                spar.set(pnts)
            # Update for `print_progress`
            self._batch_size_last = min(self._batch_size_last, len(pnts))

            self._prepare_gettables()

            y_off = 0
            for gpar in self._gettable_pars:
                new_data = gpar.get()  # can return (N, M)
                # if we get a simple array, shape it to (1, M)
                if len(np.shape(new_data)) == 1:
                    new_data = new_data.reshape(1, (len(new_data)))

                for row in new_data:
                    yi_name = f"y{y_off}"
                    slice_len = setpoint_idx + len(row)  # the slice we will be updating
                    old_vals = self._dataset[yi_name].values[setpoint_idx:slice_len]
                    old_vals[
                        np.isnan(old_vals)
                    ] = 0  # will be full of NaNs on the first iteration, change to 0
                    self._dataset[yi_name].values[
                        setpoint_idx:slice_len
                    ] = self._build_data(row, old_vals)
                    y_off += 1

            self._nr_acquired_values += np.shape(new_data)[1]
            self._update()
            self._check_interrupt()

    def _build_data(self, new_data, old_data):
        if self._soft_avg == 1:
            return old_data + new_data

        return (new_data + old_data * self._loop_count) / (1 + self._loop_count)

    def _iterative_set_and_get(
        self, setpoints: np.ndarray, idx: int, lazy_set: bool = False
    ):
        """
        Processes one row of setpoints. Sets all settables, gets all gettables, encodes
        new data in dataset.

        If lazy_set==True and any setpoint equals the corresponding previous setpoint,
        that setpoint is not set in its corresponding settable.

        .. note ::

            Note: some lines in this function are redundant depending on mode (sweep vs
            adaptive). Specifically:

                - in sweep, the x dimensions are already filled
                - in adaptive, soft_avg is always 1
        """
        assert self._dataset is not None

        # set all individual setparams
        for setpar_idx, (spar, spt) in enumerate(zip(self._settable_pars, setpoints)):
            xi_name = f"x{setpar_idx}"
            if self._dataarray_cache is None:
                xi_dataarray_values = self._dataset[xi_name].values
            else:
                if not xi_name in self._dataarray_cache:
                    self._dataarray_cache[xi_name] = self._dataset[xi_name].values
                xi_dataarray_values = self._dataarray_cache[xi_name]
            xi_dataarray_values[idx] = spt
            prev_spt = xi_dataarray_values[idx - 1] if idx else None
            # if lazy_set==True and the setpoint equals the previous setpoint, do not
            # set the setpoint.
            if not (lazy_set and spt == prev_spt):
                spar.set(spt)

        # get all data points
        y_offset = 0
        for gpar in self._gettable_pars:
            new_data = gpar.get()
            # if the gettable returned a float, cast to list
            if np.isscalar(new_data):
                new_data = [new_data]
            # iterate through the data list, each element is different y for these
            # x coordinates
            for val in new_data:
                yi_name = f"y{y_offset}"
                if self._dataarray_cache is None:
                    yi_dataarray_values = self._dataset[yi_name].values
                else:
                    if not yi_name in self._dataarray_cache:
                        self._dataarray_cache[yi_name] = self._dataset[yi_name].values
                    yi_dataarray_values = self._dataarray_cache[yi_name]
                old_val = yi_dataarray_values[idx]
                if self._soft_avg == 1 or np.isnan(old_val):
                    if isinstance(val, np.ndarray) and val.size == 1:
                        # This branch avoids usage of deprecated code
                        yi_dataarray_values[idx] = val.item()
                    else:
                        # This is deprecated if val is an np.ndarray with ndim > 0
                        yi_dataarray_values[idx] = val
                else:
                    averaged = (val + old_val * self._loop_count) / (
                        1 + self._loop_count
                    )
                    yi_dataarray_values[idx] = averaged
                y_offset += 1

    ############################################
    # Methods used to control the measurements #
    ############################################

    def _interrupt_handler(self, signal, frame):
        """
        A proper handler for signal.SIGINT (a.k.a. KeyboardInterrupt).

        Used to avoid affecting any other code, e.g. instruments drivers, and be able
        safely stop the MC after an iteration finishes.
        """
        self._thread_data.events_num += 1
        if self._thread_data.events_num >= 5:
            raise KeyboardInterrupt
        print(
            f"\n\n[!!!] {self._thread_data.events_num} interruption(s) signaled. "
            "Stopping after this iteration/batch.\n"
            f"[Send {5 -  self._thread_data.events_num} more interruptions to force"
            f"stop (not safe!)].\n"
        )

    def _check_interrupt(self):
        """
        Verifies if the user has signaled the interruption of the experiment.

        Intended to be used after each iteration or after each batch of data.
        """
        if self._thread_data.events_num >= 1:
            # It is safe to raise the KeyboardInterrupt here because we are guaranteed
            # To be running MC code. The exception can be handled in a try-except
            raise KeyboardInterrupt("Measurement interrupted")

    def _update(self, print_message: str = None):
        """
        Do any updates to/from external systems, such as saving, plotting, etc.
        """
        update = (
            time.time() - self._last_upd > self.update_interval()
            or self._nr_acquired_values == self._get_max_setpoints()
        )
        if update:
            self.print_progress(print_message)

            if self._save_data:
                self._safe_write_dataset()

            self._last_upd = time.time()

    def _prepare_gettables(self) -> None:
        """
        Call prepare() on the Gettable, if prepare() exists
        """
        for getpar in self._gettable_pars:
            call_if_has_method(getpar, "prepare")

    def _prepare_settables(self) -> None:
        """
        Call prepare() on all Settable, if prepare() exists
        """
        for setpar in self._settable_pars:
            call_if_has_method(setpar, "prepare")

    def _finish(self) -> None:
        """
        Call finish() on all Settables and Gettables, if finish() exists
        """
        for par in self._gettable_pars + self._settable_pars:
            call_if_has_method(par, "finish")

    def _get_batched_mask(self):
        return tuple(is_batched(spar) for spar in self._settable_pars)

    def _get_where_batched(self):
        # Indices to select correct entries in results data
        return np.where(self._get_batched_mask())[0]

    def _get_where_iterative(self):
        return np.where(tuple(not m for m in self._get_batched_mask()))[0]

    def _get_iterative_settables(self):
        return tuple(spar for spar in self._settable_pars if not is_batched(spar))

    def _get_batched_settables(self):
        return tuple(spar for spar in self._settable_pars if is_batched(spar))

    def _get_batch_size(self):
        # np.inf is not supported by the JSON schema, but we keep the code robust
        min_with_inf = min(
            getattr(par, "batch_size", np.inf)
            for par in chain.from_iterable((self._settable_pars, self._gettable_pars))
        )
        return min(min_with_inf, len(self._setpoints))

    def _get_is_batched(self) -> bool:
        if any(
            is_batched(gpar) for gpar in chain(self._gettable_pars, self._settable_pars)
        ):
            if not all(is_batched(gpar) for gpar in self._gettable_pars):
                raise RuntimeError(
                    "Control mismatch; all Gettables must have batched Control Mode, "
                    "i.e. all gettables must have `.batched=True`."
                )
            if not any(is_batched(spar) for spar in self._settable_pars):
                raise RuntimeError(
                    "Control mismatch; At least one settable must have "
                    "`settable.batched=True`, if the gettables are batched."
                )
            return True

        return False

    def _get_max_setpoints(self) -> int:
        """
        The total number of setpoints to examine
        """
        return len(self._setpoints) * self._soft_avg

    def _curr_setpoint_idx(self) -> int:
        """
        Current position through the sweep

        Returns
        -------
        int
            setpoint_idx
        """
        acquired = self._nr_acquired_values
        setpoint_idx = acquired % len(self._setpoints)
        self._loop_count = acquired // len(self._setpoints)
        return setpoint_idx

    def _get_fracdone(self) -> float:
        """
        Returns the fraction of the experiment that is completed.
        """
        return self._nr_acquired_values / self._get_max_setpoints()

    def print_progress(self, progress_message: str = None):
        """
        Prints the provided `progress_messages` or displays tqdm progress bar; and calls the
        callback specified by `on_progress_callback`.

        NB: if called with no progress message (progress bar is used),
            `self.pbar` attribute should be closed and removed.

        Printing and progress bar display can be suppressed with `.verbose(False)`.
        """

        # There are no points initialized, progress does not make sense
        if self._get_max_setpoints() == 0:
            raise ValueError("No setpoints available, progress cannot be defined")

        # by checking if `progress_message` is None we make sure we change print behavior for adaptive run
        if self.verbose() and not hasattr(self, "pbar") and progress_message is None:
            # when you use `unit` instead of `postfix` it removes unintended comma
            # see https://github.com/tqdm/tqdm/issues/712
            custom_bar_format = "{l_bar}{bar} [ elapsed time: {elapsed} | time left: {remaining} ] {unit}"
            self.pbar = tqdm(total=100, desc="Completed", bar_format=custom_bar_format)

        progress_percent = self._get_fracdone() * 100

        if self.verbose() and progress_message is None:
            progress_diff = math.floor(progress_percent) - self.pbar.n
            if self._batch_size_last is not None:
                self.pbar.unit = " last batch size: %s" % self._batch_size_last
            else:
                # if no unit attribute provided `custom_bar_format` breaks with extra `it` output
                self.pbar.unit = ""
            self.pbar.update(progress_diff)

        if (
            self.verbose()
            and hasattr(self, "pbar")
            and math.floor(progress_percent) >= 100
        ):
            self.pbar.close()
            del self.pbar

        if self.on_progress_callback() is not None:
            self.on_progress_callback()(progress_percent)

        if self.verbose() and progress_message is not None:
            print(progress_message, end="")

    def _safe_write_dataset(self):
        """
        Uses a lock when writing the file to stay safe for multiprocessing.
        Locking files are written into a temporary dir to avoid polluting
        the experiment container.
        """
        # Multiprocess safe
        lockfile = (
            _DATASET_LOCKS_DIR / f"{self._dataset.attrs['tuid']}-{DATASET_NAME}.lock"
        )
        with FileLock(lockfile, 5):
            self._experiment.write_dataset(self._dataset)

    ####################################
    # Non-parameter get/set functions  #
    ####################################

    def settables(self, settable_pars):
        """
        Define the settable parameters for the acquisition loop.

        The :class:`.Settable` helper class defines the
        requirements for a Settable object.

        Parameters
        ---------
        settable_pars
            parameter(s) to be set during the acquisition loop, accepts a list or tuple
            of multiple Settable objects or a single Settable object.
        """
        # for native nD compatibility we treat this like a list of settables.
        if not isinstance(settable_pars, (list, tuple)):
            settable_pars = [settable_pars]

        self._settable_pars = []
        for settable in settable_pars:
            self._settable_pars.append(Settable(settable))

    def setpoints(self, setpoints: np.ndarray):
        """
        Set setpoints that determine values to be set in acquisition loop.

        .. tip::

            Use :func:`~numpy.column_stack` to reshape multiple
            1D arrays when setting multiple settables.

        Parameters
        ----------
        setpoints :
            An array that defines the values to loop over in the experiment.
            The shape of the array has to be either (N,) or (N,1) for a 1D loop;
            or (N, M) in the case of an MD loop.
        """
        if len(np.shape(setpoints)) == 1:
            setpoints = setpoints.reshape((len(setpoints), 1))
        elif len(np.shape(setpoints)) == 2:
            # used in plotmon to detect need for interpolation in 2d plot
            is_uniform = all(
                _is_uniformly_spaced_array(setpoints_i) for setpoints_i in setpoints.T
            )
            self._plot_info["1d_2_settables_uniformly_spaced"] = is_uniform

        self._setpoints = setpoints
        # `.setpoints()` and `.setpoints_grid()` cannot be used at the same time
        self._setpoints_input = None

    def setpoints_grid(self, setpoints: Iterable[np.ndarray]):
        """
        Makes a grid from the provided `setpoints` assuming each array element
        corresponds to an orthogonal dimension.
        The resulting gridded points determine values to be set in the acquisition loop.

        The gridding is such that the inner most loop corresponds to the batched
        settable with the smallest `.batch_size`.

        .. seealso:: :ref:`Measurement Control How-To <howto-measurement-control>`

        Parameters
        ----------
        setpoints
            The values to loop over in the experiment. The grid is reshaped in the same
            order.
        """
        self._setpoints = None  # assigned later in the `._init()`
        self._setpoints_input = setpoints

        if len(setpoints) == 2:
            self._plot_info["xlen"] = len(setpoints[0])
            self._plot_info["ylen"] = len(setpoints[1])
            self._plot_info["grid_2d"] = True
            is_uniform = all(  # used in plotmon to detect need for interpolation
                _is_uniformly_spaced_array(setpoints[i]) for i in (0, 1)
            )
            self._plot_info["grid_2d_uniformly_spaced"] = is_uniform

    def gettables(self, gettable_pars):
        """
        Define the parameters to be acquired during the acquisition loop.

        The :class:`.Gettable` helper class defines the
        requirements for a Gettable object.

        Parameters
        ----------
        gettable_pars
            parameter(s) to be get during the acquisition loop, accepts:
                 - list or tuple of multiple Gettable objects
                 - a single Gettable object
        """
        if not isinstance(gettable_pars, (list, tuple)):
            gettable_pars = [gettable_pars]

        self._gettable_pars = []
        for gpar in gettable_pars:
            self._gettable_pars.append(Gettable(gpar))

    def measurement_description(self) -> Dict[str, Any]:
        """Return a serializable description of the latest measurement

        Users can add additional information to the description manually.

        Returns
        -------
        :
            Dictionary with description of the measurement
        """
        experiment_description = {
            "name": self._dataset.attrs["name"],
            "settables": [str(s) for s in self._settable_pars],
        }
        experiment_description["gettables"] = [str(s) for s in self._gettable_pars]
        experiment_description["setpoints_shape"] = self._setpoints.shape
        experiment_description["soft_avg"] = self._soft_avg
        experiment_description["acquired_dataset"] = {"tuid": self._dataset.tuid}

        return experiment_description

def grid_setpoints(setpoints: Iterable, settables: Iterable = None) -> np.ndarray:
    """
    Makes gridded setpoints. If `settables` is provided, the gridding is such that the
    inner most loop corresponds to the batched settable with the smallest `.batch_size`.

    .. warning ::

        Using this method typecasts all values into the same type.
        This may lead to validator errors when setting
        e.g., a `float` instead of an `int`.

    Parameters
    ----------
    setpoints
        A list of arrays that defines the values to loop over in the experiment for each
        orthogonal dimension. The grid is reshaped in the same order.
    settables
        A list of settable objects to which the elements in the `setpoints` correspond
        to. Used to correctly grid data when mixing batched and iterative settables.

    Returns
    -------
    :class:`~numpy.ndarray`
        An array where the first numpy axis correspond to individual setpoints.
    """

    if settables is None:
        settables = [None] * len(setpoints)

    coordinates_names = [f"x{i}" for i, pnts in enumerate(setpoints)]
    dataset_coordinates = xr.Dataset(coords=dict(zip(coordinates_names, setpoints)))
    coordinates_batched = [
        name for name, spar in zip(coordinates_names, settables) if is_batched(spar)
    ]
    coordinates_iterative = sorted(
        (
            name
            for name, spar in zip(coordinates_names, settables)
            if not is_batched(spar)
        ),
        reverse=True,
    )

    if len(coordinates_batched):
        batch_sizes = [
            getattr(spar, "batch_size", np.inf)
            for spar in settables
            if is_batched(spar)
        ]
        coordinates_batched_set = set(coordinates_batched)
        inner_coord_name = coordinates_batched[np.argmin(batch_sizes)]
        coordinates_batched_set.remove(inner_coord_name)
        # The inner most coordinate must correspond to the batched settable with
        # min `.batch_size`
        stack_order = (
            coordinates_iterative
            + sorted(coordinates_batched_set, reverse=True)
            + [inner_coord_name]
        )
    else:
        stack_order = coordinates_iterative + sorted(coordinates_batched, reverse=True)

    # Internally the xarray's stack mechanism is used to achieve the desired grid
    stacked_dset = dataset_coordinates.stack(dim_0=stack_order)

    # Return numpy array in the original order
    return np.column_stack([stacked_dset[name].values for name in coordinates_names])


import numpy as np
import pandas as pd
import datetime

from obspy import read, Stream
from obspy.clients.fdsn import Client
from earthscopestraintools.timeseries import Timeseries

inv_client = Client("IRIS")
from requests.exceptions import HTTPError

import logging

logger = logging.getLogger(__name__)

def load_mseed_to_df(
    net: str,
    sta: str,
    loc: str,
    cha: str,
    start: str,
    end: str,
    save_as: str = None,
    print_url: bool = False,
    print_traces: bool = True
):
    """
    Load miniseed data from fdsnws-dataselect a pandas dataframe, with time as the index and
    each channel as a column.  Uses obspy as an intermediate step.  Enforces completeness by
    building an index of all timestamps in date range and filling gaps with nans.

    :param net: FDSN 2 character network code, accepts wildcards
    :type net: str
    :param sta: FDSN 4 character station code, accepts wildcards
    :type sta: str
    :param loc: FDSN 2 character location code, accepts wildcards
    :type loc: str
    :param cha: FDSN 3 character channel code, accepts wildcards
    :type cha: str
    :param start: start time of query as ISO formatted string
    :type start: str
    :param end: end time of query as ISO formatted string
    :type end: str
    :param save_as: filename to save miniseed data, defaults to None
    :type save_as: str, optional
    :param print_url: show dataselect url, defaults to False
    :type print_url: bool, optional
    :param print_traces: show traces loaded into obspy, defaults to True
    :type print_traces: bool, optional
    :return: data loaded into pandas dataframe
    :rtype: pd.DataFrame
    """
    st = download_mseed(
        net=net, sta=sta, loc=loc, cha=cha, start=start, end=end, print_url=print_url
    )
    if save_as:
        st.write(filename=save_as)

    df = mseed2pandas(st, print_traces=print_traces)
    #
    #freq = df.index.to_series().diff().median()
    #timestamps = pd.date_range(start=start, end=end, freq=freq, inclusive='left')
    #df2 = df.reindex(timestamps, method='nearest')
    #df2 = df.reindex(timestamps, method=None)
    #df2.index.name = "time"
    return df

def load_mseed_file_to_df(filename: str):
    """read miniseed file from local disk into pandas dataframe, with time as the index and
    each channel as a column.  Uses obspy as an intermediate step.

    :param filename: miniseed filename
    :type filename: str
    :return: data loaded into pandas dataframe
    :rtype: pd.DataFrame
    """
    st = read(filename)
    df = mseed2pandas(st)
    return df

def save_mseed_file(
    net: str, sta: str, loc: str, cha: str, start: str, end: str, filename: str
):
    """Load miniseed data from fdsnws-dataselect into obspy, and save as miniseed.

    :param net: FDSN 2 character network code, accepts wildcards
    :type net: str
    :param sta: FDSN 4 character station code, accepts wildcards
    :type sta: str
    :param loc: FDSN 2 character location code, accepts wildcards
    :type loc: str
    :param cha: FDSN 3 character channel code, accepts wildcards
    :type cha: str
    :param start: start time of query as ISO formatted string
    :type start: str
    :param end: end time of query as ISO formatted string
    :type end: str
    :param filename: filename to save miniseed data
    :type filename: str
    """
    st = download_mseed(net=net, sta=sta, loc=loc, cha=cha, start=start, end=end)
    st.write(filename)

def download_mseed(
    net: str,
    sta: str,
    loc: str,
    cha: str,
    start: str,
    end: str,
    print_url: bool = False,
):
    """Load miniseed data from fdsnws-dataselect into obspy Stream object.

    :param net: FDSN 2 character network code, accepts wildcards
    :type net: str
    :param sta: FDSN 4 character station code, accepts wildcards
    :type sta: str
    :param loc: FDSN 2 character location code, accepts wildcards
    :type loc: str
    :param cha: FDSN 3 character channel code, accepts wildcards
    :type cha: str
    :param start: start time of query as ISO formatted string
    :type start: str
    :param end: end time of query as ISO formatted string
    :type end: str
    :param print_url: show dataselect url, defaults to False
    :type print_url: bool, optional
    :return: data as Stream object
    :rtype: obspy.Core.Stream
    """
    logger.info(
        f"{net} {sta} Loading {loc} {cha} from {start} to {end} from Earthscope DMC miniseed"
    )
    url = (
        f"https://service.iris.edu/fdsnws/dataselect/1/query?net={net}"
        f"&sta={sta}&loc={loc}&cha={cha}&starttime={start}&endtime={end}"
        f"&format=miniseed&nodata=404"
    )
    if print_url:
        logger.info(f"{net} {sta} Reading from {url}")
    try:
        st = read(url)
    except HTTPError as e:
        logger.error(f"{net} {sta} No data found for specified query {url}")
        st = Stream()
    return st.merge(fill_value=999999)

def mseed2pandas(st: Stream, print_traces=True):
    """Restructure data from obspy.Core.Stream to pandas.DataFrame with time as the index and
    each channel as a column.

    :param st: Stream object containing one or more Traces
    :type st: obspy.Core.Stream
    :param print_traces: show traces loaded into obspy, defaults to True
    :type print_traces: bool, optional
    :return: data loaded into pandas dataframe
    :rtype: pandas.DataFrame
    """
    df = pd.DataFrame()
    dfs = {}
    for i, tr in enumerate(st):
        channel = fdsn2bottlename(tr.stats.channel)
        start = tr.stats["starttime"]
        stop = tr.stats["endtime"]
        step = datetime.timedelta(seconds=tr.stats["delta"])
        if print_traces:
            logger.info(
                f"    Trace {i + 1}. {start}:{stop} mapping {tr.stats.channel} to {channel}"
            )
        time_buffer = np.arange(start, stop + step, step, dtype=np.datetime64)
        df2 = pd.DataFrame(index=time_buffer)
        df2[channel] = tr.data
        if channel not in dfs.keys():
            dfs[channel] = df2
        else:
            dfs[channel] = pd.concat([dfs[channel], df2])
    for i, key in enumerate(dfs.keys()):
        df[key] = dfs[key]
    df.index.name = "time"
    return df

def fdsn2bottlename(channel):
    """convert FDSN channel code into bottlename

    :param channel: FDSN channel
    :type channel: str
    :return: bottlename
    :rtype: str
    """

    codes = {
        "RS1": "CH0",
        "LS1": "CH0",
        "BS1": "CH0",
        "RS2": "CH1",
        "LS2": "CH1",
        "BS2": "CH1",
        "RS3": "CH2",
        "LS3": "CH2",
        "BS3": "CH2",
        "RS4": "CH3",
        "LS4": "CH3",
        "BS4": "CH3",
        "RDO": "atmp",
        "LDO": "atmp",
        "RRO": "rain",
        "RK1": "LoggerDegC",
        "RKD": "DownholeDegC",
        "RE1": "BatteryVolts",
        "REO": "SolarAmps",
        "RE2": "SystemAmps",
        "RK2": "PowerBoxDegC",
        "LDD": "PorePressure",
        "LKD": "PoreDegC",
        "VEP": "Q330Volts"
    }

    return codes[channel]

def ts_from_mseed(
    network: str,
    station: str,
    location: str,
    channel: str,
    start: str,
    end: str,
    name: str = None,
    period: float = None,
    series: str = "raw",
    units: str = "unknown",
    fill_value: int = 999999,
    to_nan: bool = True,
    scale_factor: float = None,
):
    """Load miniseed data from fdsnws-dataselect into Timeseries object.

    :param network: FDSN 2 character network code, accepts wildcards
    :type network: str
    :param station: FDSN 4 character station code, accepts wildcards
    :type station: str
    :param location: FDSN 2 character location code, accepts wildcards
    :type location: str
    :param channel: FDSN 3 character channel code, accepts wildcards
    :type channel: str
    :param start: start time of query as ISO formatted string
    :type start: str
    :param end: end time of query as ISO formatted string
    :type end: str
    :param name: name of timeseries, including station name.  useful for showing stats. , defaults to None
    :type name: str, optional
    :param period: sample period of data, defaults to None
    :type period: float, optional
    :param series: description of timeseries, ie 'raw', 'microstrain', 'atmp_c', 'tide_c', 'offset_c', 'trend_c', defaults to "raw"
    :type series: str, optional
    :param units: units of timeseries, defaults to "counts" or "unknown"
    :type units: str, optional
    :param fill_value: gap fill value to replace if to_nan is set to True, defaults to 999999
    :type fill_value: int, optional
    :param to_nan: option to convert gap fill values to numpy.nan, defaults to True
    :type to_nan: bool, optional
    :param scale_factor: scale factor to apply to miniseed data, defaults to None
    :type scale_factor: float, optional
    :return: Timeseries object containing data loaded from miniseed
    :rtype: earthscopestraintools.timeseries.Timeseries
    """
    df = load_mseed_to_df(
        net=network,
        sta=station,
        loc=location,
        cha=channel,
        start=start,
        end=end,
    )
    level = "0"
    series = series
    if channel.startswith("RS") or channel.startswith("LS") or channel.startswith("BS"):
        units = "counts"
    if name is None:
        name = f"{network}.{station}.{location}.{channel}"
    ts = Timeseries(
        data=df, series=series, units=units, level=level, name=name
    )
    if to_nan:
        ts = ts.remove_fill_values(fill_value=fill_value)
    if scale_factor:
        ts.data = ts.data * scale_factor
    return ts


#!/usr/bin/python3
# -*- coding: utf-8 -*-
import re
import random
import requests
import time
import threading
import os
import json

import vk_api
from vk_api import VkUpload
from vk_api.bot_longpoll import VkBotLongPoll

class Bot:

    def __init__(self, vk):

        self.spamDetected = {}
        self.commandsUsed = {}
        self.badwordsDetected = {}
        self.messagesParsed = {}
        self.totalInfo = {}
        self.spamming = 0
        self.session = requests.Session()
        self.vk_session = vk
        self.vk = vk.get_api()
        self.helloWorlds = ['Console.WriteLine("Hello World!");',
                            'System.out.println("Hello World!");',
                            'writeln("Hello World!");',
                            'print("Hello World")',
                            'cout<<"Hello World!"<<endl;',
                            'Hello World!',
                            'echo "Hello World!";',
                            'printf("Hello World!");']
        self.patterns = [r"([\w/ !,\\?\.\(\)]*)п[еeи]д[оoaа]*р([\w/ !,\\?\.\(\)]*)", r"([\w/ !,\\?\.\(\)]*)[оoаa ][xх][*уy][йеeяuи]([\w/ !,\\?\.\(\)]*)",
        r"([\w/ !,\\?\.\(\)]*)[еёиe]б[аauи]([нhтcс])([\w/ !,\\?\.\(\)]*)",
        r"([\w/ !,\\?\.\(\)]*)[пn][иueе\*]+" + r"[сc3з]+" + r"д([\w/ !,\\?\.\(\)]*)", r"([\w/ !,\\?\.\(\)]*) бля([\w/ !,\\?\.\(\)]*)",
        r"([\w/ !,\\?\.\(\)]*) бля([\w/ !,\\?\.\(\)]*)", r"([\w/ !,\\?\.\(\)]*)бля[т]([\w/ !,\\?\.\(\)]*)",
        r"([\w/ ]*)[ ь][eе]б[aаyу] ([\w\\\ ]*)", r"([\w\\\ ]*)д[oо]лб[aа][ёeе]б([\w\\\ ]*)"]
        self.help = 'команды:\n/hw - Сказать "Привет мир"\n' \
                    '/helloWorld - Привет мир на разных ЯП\n' \
                    '/сбор - призвать всех в беседу (только для админов)\n' \
                    '/онлайн - проверить, кто онлайн в беседе\n' \
                    '/кто <фраза, имя, прозвище, и т.д.> - рандомно выбирает\n' \
                    'человека\n' \
                    '/ или /help - показать команды.\n'
        self.startTime = time.time()

    def sendMsgToChat(self, event, message, image = False, url = 'https://pbs.twimg.com/media/D4aTJ8yWAAE4GJu.jpg', video = False, videoref = 'video213946714_456239065'):
        try:
            upload = VkUpload(self.vk_session)
            attachments = []
            if image:
                upload = VkUpload(self.vk_session)
                image_url = url
                image = self.session.get(image_url, stream=True)
                photo = upload.photo_messages(photos=image.raw)[0]
                attachments.append(
                'photo{}_{}'.format(photo['owner_id'], photo['id']))
            if video:
                attachments.append('video-' + videoref)
            self.vk.messages.send(
                        attachment=videoref if video else '',
                        chat_id=event.chat_id,
                        message=message,
                        random_id=1
                    )
        except Exception as e:
            print(e)

    def parseChat(self, event, msg):
        ################### INFO-COLLECT ####################
        try:
            self.messagesParsed[event.chat_id] += 1
        except:
            self.messagesParsed[event.chat_id] = 1
        try:
            self.totalInfo['messagesParsed'] += 1
        except:
            self.totalInfo['messagesParsed'] = 1
        ################### INFO-COLLECT ####################
        if "ааааааа" in msg.lower():
            self.sendMsgToChat(event, "Не кричи на меня! Я очень ранимый естественный разум.")
            return
        badwords = False
        for p in self.patterns:
            if re.match(p, msg.lower()):
                badwords = True
        '''if random.randint(1, 10) in (1, 3, 5, 7, 9):
            badwords = False'''
        if msg.startswith('/') and badwords:
            self.sendMsgToChat(event, "Команды с матами выполнять не буду!", image = True)
            return
        if badwords:
            ################### INFO-COLLECT ####################
            try:
                self.badwordsDetected[event.chat_id] += 1
            except:
                self.badwordsDetected[event.chat_id] = 1
            try:
                self.totalInfo['badwordsDetected'] += 1
            except:
                self.totalInfo['badwordsDetected'] = 1
            ################### INFO-COLLECT ####################
            self.sendMsgToChat(event, "Ребята, общайтесь без мата, пожалуйста", image = True)
        if not msg.startswith('/'):
            return
        ################### INFO-COLLECT ####################
        try:
            self.commandsUsed[event.chat_id] += 1
        except:
            self.commandsUsed[event.chat_id] = 1
        try:
            self.totalInfo['commandsUsed'] += 1
        except:
            self.totalInfo['commandsUsed'] = 1
        ################### INFO-COLLECT ####################
        args = msg.split()
        if not hasattr(self, 'lastCommandTime'):
            self.lastCommandTime = time.time() - 10
        currentTime = time.time()
        #print(str(currentTime - self.lastCommandTime))
        if currentTime - self.lastCommandTime < 4:
            self.spamming += 1
            if self.spamming > 4:
                ################### INFO-COLLECT ####################
                try:
                    self.spamDetected[event.chat_id] += 1
                except:
                    self.spamDetected[event.chat_id] = 1
                try:
                    self.totalInfo['spamDetected'] += 1
                except:
                    self.totalInfo['spamDetected'] = 1
                ################### INFO-COLLECT ####################
                self.sendMsgToChat(event, 'Хватит спамить командами!!!', video = True)
                self.lastCommandTime = time.time()
                return
        else:
            self.spamming = 0
        self.lastCommandTime = time.time()
        if args[0] == '/hw':
            self.sendMsgToChat(event, 'Привет мир!')
            #self.sendMsgToChat(event, '', video = True)
        elif args[0] == '/helloWorld':
            i = random.randint(0, len(self.helloWorlds) - 1)
            self.sendMsgToChat(event, self.helloWorlds[i])
        elif args[0] == '/сбор':
            cmembers = self.vk.messages.getConversationMembers(peer_id = 2000000000 + event.chat_id)
            items = cmembers['items']
            useSbor = self.isAdmin(items, event)
            if not useSbor:
                return
            m = ''
            if len(args) > 1:
                for i in range(1, len(args)):
                    m += args[i] + ' '
            else:
                m = 'Общий сбор!\n'

            members = self.vk.messages.getConversationMembers(peer_id = 2000000000 + event.chat_id)['profiles']
            for member in members:
                #print(member)
                if not 'screen_name' in member.keys():
                    continue
                m += '[' + member['screen_name'] + '|' + member['first_name'] + '], '
            self.sendMsgToChat(event, m)
        elif args[0] == '/кто':
            if len(args) == 1:
                args.append('Это ')
            members = self.vk.messages.getConversationMembers(peer_id = 2000000000 + event.chat_id)['profiles']
            i = random.randint(0, len(members) - 1)
            member = members[i]
            who = ''
            for word in args[1:]:
                who += word + ' '
            self.sendMsgToChat(event, who + 'естественно ' + \
                               member['first_name'] + ' ' + member['last_name'] + '!')
        elif args[0] == '/онлайн':
            online = []
            members = self.vk.messages.getConversationMembers(peer_id = 2000000000 + event.chat_id)['profiles']
            for member in members:
                if member['online']:
                    online.append(member['first_name'] + ' ' + member['last_name'])
            txt = 'Сейчас онлайн:' if len(online) > 1 else 'Никого нет онлайн, кроме вас.'
            for user in online:
                if len(online) < 2:
                    break
                txt += user + ', '
            txt += 'и бот.' if len(online) > 1 else ''
            self.sendMsgToChat(event, txt)
        elif msg in ('/help', '/'):
            self.sendMsgToChat(event, self.help)
        elif args[0] in ("/ъеь", "/ъуъ", "/ьеъ", "/ьуъ"):
            self.sendMsgToChat(event, "Логикаааа зашкаливает...")
        elif args[0] == "/exec":
            if event.object['from_id'] != 423904265:
                self.sendMsgToChat(event, "Доступ запрещен.")
                return
            if len(args) == 1:
                self.sendMsgToChat(event, "Неправильное использование. Используйте так: /bash [python/bash] [command]")
                return
            if args[1] == "python":
                err = False
                try:
                    res = eval(" ".join(args[2:]))
                except Exception  as e:
                    res = e
                    err = True
                if err:
                    self.sendMsgToChat(event, "Произошла ошибка: " + str(res))
                else:
                    self.sendMsgToChat(event, "Результат: " + str(res))
            elif args[1] == "bash":
                try:
                    self.sendMsgToChat(event, os.popen(" ".join(args[2:])))
                except:
                    self.sendMsgToChat(event, "Ошибка.")
            else:
                self.sendMsgToChat(event, "Неожиданный тип исполнения: " + args[1] + ".")
        elif args[0] == "/easterEgg":
            self.sendMsgToChat(event, "Автор сейчас ждет Соник Йожыг: кооператив злодеев, часть 2\n:3")
        elif args[0] == "/отчет":
            lastResetTime = (time.time() - self.startTime)/60
            if lastResetTime > 60:
                minutes = int(lastResetTime % 60)
                lastResetTime //= 60
                lastResetTime = int(lastResetTime)
                timeFromLastReset = str(lastResetTime)
                timeFromLastReset += " ч"
                timeFromLastReset += f", {minutes}" if minutes != 0 else ""
                timeFromLastReset += " мин."
            else:
                minutes = round(lastResetTime)
                timeFromLastReset = str(minutes)
                timeFromLastReset += " мин."

            #print(lastResetTime)
            if len(args) > 1:
                if args[1] == 'total':
                    msg = "Отчет с последнего сброса (все беседы):\n"
                    msg += "Время с последнего сброса: " + timeFromLastReset + "\n"
                    msg += "Всего матов обнаружено: " + str(self.totalInfo.get("badwordsDetected", 0)) + ".\n"
                    msg += "Всего реакций на спам: " + str(self.totalInfo.get("spamDetected", 0)) + ".\n"
                    msg += "Всего команд (или сообщений со знаком '/' в начале) пропарсено: " + str(self.totalInfo.get("commandsUsed", 0)) + ".\n"
                    msg += "Всего сообщений обработано: " + str(self.totalInfo.get("messagesParsed", 0)) + ".\n"
                else:
                    msg = "Неправильное использование команды. Использование: /отчет, или /отчет total"
            else:
                msg = "Отчет с последнего сброса (эта беседа):\n"
                msg += "Время с последнего сброса: " + timeFromLastReset + "\n"
                msg += "Всего матов обнаружено: " + str(self.badwordsDetected.get(event.chat_id, 0)) + ".\n"
                msg += "Всего реакций на спам: " + str(self.spamDetected.get(event.chat_id, 0)) + ".\n"
                msg += "Всего команд (или сообщений со знаком '/' в начале) пропарсено: " + str(self.commandsUsed.get(event.chat_id, 0)) + ".\n"
                msg += "Всего сообщений обработано: " + str(self.messagesParsed.get(event.chat_id, 0)) + ".\n"
            self.sendMsgToChat(event, msg)

    def isAdmin(self, items, event):
        admin = False
        for item in items:
            if item['member_id'] == event.object['from_id']:
                admin = item.get('is_admin')
        return admin

def main():
    try:
        file = open("data.json")
        data = json.load(file)
        token = data['token']
        group_id = data['groupId']
    except FileNotFoundError:
        print("Data file not found, make it by starting setup.py")
        input("Press <enter> to exit. ")
        return
    fileForErrors = open('errorLog.txt', 'w')
    while True:
        try:
            vk_session = vk_api.VkApi(token)

            longpoll = VkBotLongPoll(vk_session, group_id)

            bot = Bot(vk_session)

            for event in longpoll.listen():
                if event.from_chat:
                    t = threading.Thread(target=bot.parseChat, args=(event, event.obj.text))
                    t.start()
        except Exception as e:
            print(e)
            fileForErrors.write(str(type(e)) + ':' + str(e) + '\n')
            print("Happened error. Restarting...")
if __name__ == "__main__":
    main()


import itertools

def gen_secs():
    return (x for x in range(60))

def gen_minutes():
    return (x for x in range(60))

def gen_hours():
    return (x for x in range(24))

def gen_time():
    secs_generator = gen_secs()
    mins_generator = gen_minutes()
    hours_generator = gen_hours()

    sec = next(secs_generator)
    min = next(mins_generator)
    hour = next(hours_generator)

    while True:
        yield '%s:%s:%s' % (str(hour).zfill(2), str(min).zfill(2), str(sec).zfill(2))
        try:
            sec = next(secs_generator)
        except:
            try:
                min = next(mins_generator)
            except:
                try:
                    hour = next(hours_generator)
                except:
                    break
                else:
                    mins_generator = gen_minutes()
                    min = next(mins_generator)
            finally:
                secs_generator = gen_secs()
                sec = next(secs_generator)

# for gt in gen_time():
#     print(gt)

def gen_years(start=2021):
    return itertools.count(start)

def gen_months():
    return (x for x in range(1, 13))

def gen_days(month, leap_year=True):
    if month in [1, 3, 5, 7, 8, 10, 12]:
        return (x for x in range(1, 32))
    elif month in [4, 6, 9, 11]:
        return (x for x in range(1, 31))
    elif month == 2:
        if leap_year:
            return (x for x in range(1, 30))
        else:
            return (x for x in range(1, 29))

def is_leap(year):
    if year % 4 == 0:
        if year % 100 == 0:
            if year % 400 == 0:
                return True
            else:
                return False
        else:
            return True
    else:
        return False

def gen_date():
    months_generator = gen_months()
    years_generator = gen_years()
    time_generator = gen_time()

    month = next(months_generator)
    year = next(years_generator)
    is_leap_year = is_leap(year)
    time = next(time_generator)

    days_generator = gen_days(month, is_leap_year)
    day = next(days_generator)

    while True:
        yield '%s/%s/%s %s' % (str(day).zfill(2), str(month).zfill(2), str(year), time)
        try:
            time = next(time_generator)
        except:
            try:
                day = next(days_generator)
            except:
                try:
                    month = next(months_generator)
                except:
                    year = next(years_generator)
                    is_leap_year = is_leap(year)
                    months_generator = gen_months()
                    month = next(months_generator)
                finally:
                    days_generator = gen_days(month, is_leap_year)
                    day = next(days_generator)
            finally:
                time_generator = gen_time()
                time = next(time_generator)

counter = 0
for gd in gen_date():
    if counter == 1000000:
        print(gd)
        counter = 0
    counter += 1


import datetime
import os
from glob import glob

import ee
import numpy as np
import plotly.express as px
import rasterio
import yaml
from google.cloud import storage
from matplotlib import pyplot as plt

from ParamsFetching import ParamsFetching
from run_cnn_model_gedi import create_model_cpu

class Gedi:

    def qualityMask(self, img):
        return img.updateMask(img.select('quality_flag').eq(1)).updateMask(img.select('degrade_flag').eq(0))

    def qualityMask_l4a(self, img):
        return img.updateMask(img.select('l4_quality_flag').eq(1)).updateMask(img.select('degrade_flag').eq(0))

    def read_tiff(self, file_path):
        with rasterio.open(file_path, 'r') as reader:
            profile = reader.profile
            tif_as_array = reader.read().astype(np.float32).transpose((1,2,0))
        return tif_as_array, profile

    def write_tiff(self, file_path, arr, profile):
        with rasterio.Env():
            with rasterio.open(file_path, 'w', **profile) as dst:
                dst.write(arr.astype(rasterio.float32))

    def random_blind(self, array, percentage):
        '''
        This function is used in test section, to randomly mask out some percentage of GEDI L2A RH data in the input.
        :param array: Input Array.
        :param percentage: Percentage of footprints to be masked.
        :return:
        '''
        sample_filter = np.random.binomial(1, percentage, array.shape)
        filter = np.logical_and(sample_filter == 1, np.logical_not(np.isnan(array)))
        return np.where(filter, array, np.nan)

    def download_to_gcloud(self, region_ids=['na'], mode='test', year = 2019, custom_region=None):
        '''
        This function is to process GEDI and PALSAR-2 data in Google Earth Engine。
        :param region_ids: Regions to be processed ['na', 'sa', 'af', 'eu', 'au', 'sas', 'nas']
        :param mode: Download test or train samples if custom_region is set to None
        :param year: The year of PALSAR-2 and GEDI data
        :param custom_region: When custom_region is assigned, only images of given region will be downloaded.
        :return:
        '''
        dataset = ee.ImageCollection('JAXA/ALOS/PALSAR/YEARLY/SAR_EPOCH') \
            .filter(ee.Filter.date(str(year)+'-01-01', str(year + 1)+'-01-01')) # PALSAR-2 data import
        sarHh_log = dataset.select('HH').first().pow(2).log10().multiply(10).subtract(83) # SAR Digital Numebr to Backscatter
        sarHv_log = dataset.select('HV').first().pow(2).log10().multiply(10).subtract(83)
        sarhvhh = sarHv_log.subtract(sarHh_log).rename('HV-HH') # HV-HH is considered as the third channel
        composite = ee.Image([sarHh_log, sarHv_log, sarhvhh])
        lc = ee.ImageCollection("ESA/WorldCover/v100").first() # landcover data
        l4b = ee.Image('LARSE/GEDI/GEDI04_B_002').select(['PS', 'MU']) # GEDI L4B data (Not in use but included as reference)
        if custom_region == None:
            for region_id in region_ids:
                # Select test region or train region
                if mode=='test':
                    roi_col = ee.FeatureCollection('users/zhaoyutim/GEDI_Test_' + region_id.upper())
                else:
                    roi_col = ee.FeatureCollection('users/zhaoyutim/GEDI_SAMPLE_'+region_id.upper())
                size = roi_col.size().getInfo()
                roi_col = roi_col.toList(size)
                for i in range(size):
                    # For each polygon getch data from GEE to Google cloud storage
                    roi = ee.Feature(roi_col.get(i).getInfo())
                    class_id = roi.args['metadata'].get('class')
                    date_pre = dataset.select('epoch').median().clip(roi).reduceRegion(
                        reducer=ee.Reducer.max(),
                        geometry=roi.geometry(),
                        scale=1000,
                        crs='EPSG:4326').getNumber('epoch')
                    date_pre = ee.Date(ee.Date.fromYMD(1970,1,1).advance(ee.Number(date_pre).divide(1000), 'second')) # calculate capture date of PALSAR-2 image
                    gedi = ee.ImageCollection('LARSE/GEDI/GEDI02_A_002_MONTHLY')\
                        .filterDate(date_pre.difference(15, "day"), date_pre.advance(15, "day"))\
                        .map(self.qualityMask)\
                        .select(['rh40', 'rh50', 'rh60', 'rh70', 'rh98']).mosaic()
                    # get GEDI images within 15 days before and after date of PALSAR-2 image
                    agbd_l4a = ee.ImageCollection('LARSE/GEDI/GEDI04_A_002_MONTHLY').filterDate(date_pre.difference(15, "day"), date_pre.advance(15, "day")).map(self.qualityMask_l4a).select(['agbd']).mosaic()
                    output = ee.Image([composite, lc, l4b, gedi, agbd_l4a])
                    dir = 'gedi_palsar_data' + '/' + region_id.upper() + str(year) + '/' + 'year'+ str(year)+ 'class_' + class_id + '_' + str(i)
                    if mode=='test':
                        mode_str=mode
                    else:
                        mode_str=''
                    # Export to cloud storage
                    image_task = ee.batch.Export.image.toCloudStorage(
                        image=output.toFloat(),
                        description='Image Export:' + 'GEDI_PALSAR_' + region_id.upper() + mode_str + str(year)+'_CLASS_'+class_id,
                        fileNamePrefix=dir,
                        bucket='ai4wildfire',
                        scale=25,
                        maxPixels=1e11,
                        region=roi.geometry(),
                        fileDimensions=256*5
                    )
                    image_task.start()
                    if mode=='test':
                        print('Start with image task (id: {}).'.format(
                            'GEDI-PALSAR Image Export:' + 'GEDI_Test_'+region_id.upper() + str(year)+'_INDEX_'+str(i)+'_CLASS_'+class_id))

                    else:
                        print('Start with image task (id: {}).'.format(
                            'GEDI-PALSAR Image Export:' + 'GEDI_SAMPLE_'+region_id.upper() + str(year)+'_INDEX_'+str(i)+'_CLASS_'+class_id))
        else:
            # initiate an polygon with given coordinates
            region = ee.Geometry.Rectangle(custom_region)
            roi = ee.Feature(region)
            date_pre = dataset.select('epoch').median().clip(roi).reduceRegion(
                reducer=ee.Reducer.max(),
                geometry=roi.geometry(),
                scale=1000,
                crs='EPSG:4326').getNumber('epoch')
            date_pre = ee.Date(ee.Date.fromYMD(1970,1,1).advance(ee.Number(date_pre).divide(1000), 'second'))
            gedi = ee.ImageCollection('LARSE/GEDI/GEDI02_A_002_MONTHLY')\
                .filterDate(date_pre.difference(15, "day"), date_pre.advance(15, "day"))\
                .map(self.qualityMask)\
                .select(['rh40', 'rh50', 'rh60', 'rh70', 'rh98']).mosaic()
            agbd_l4a = ee.ImageCollection('LARSE/GEDI/GEDI04_A_002_MONTHLY').filterDate(date_pre.difference(15, "day"), date_pre.advance(15, "day")).map(self.qualityMask_l4a).select(
                ['agbd']).mosaic()
            output = ee.Image([composite, lc, l4b, gedi, agbd_l4a])
            dir = 'gedi_palsar_data' + '/' + 'custom_region' + str(year) + '/' + 'year'+ str(year)
            image_task = ee.batch.Export.image.toCloudStorage(
                image=output.toFloat(),
                description='Image Export:' + 'GEDI_PALSAR_' + 'custom_region' + str(year),
                fileNamePrefix=dir,
                bucket='ai4wildfire',
                scale=25,
                maxPixels=1e11,
                region=roi.geometry(),
                fileDimensions=256*5
            )
            image_task.start()
            print('Start with image task (id: {}).'.format(
                'GEDI-PALSAR Image Export:' + 'GEDI_SAMPLE_' + 'custom_region' + str(year)))

    def download_to_local_proj4(self, create_time='2022-06-18'):
        '''
        Download images from cloud storage to local with images created later than given date
        :param create_time: Only files created after this time will be downloaded.
        :return:
        '''
        storage_client = storage.Client()
        bucket = storage_client.bucket('ai4wildfire')
        blobs = bucket.list_blobs(prefix='gedi_palsar_data')
        for blob in blobs:
            if blob.time_created.date() < datetime.datetime.strptime(create_time, '%Y-%m-%d').date():
                continue
            filename = blob.name
            path = os.path.dirname(filename)
            if not os.path.exists(path):
                os.makedirs(path)
            blob.download_to_filename(filename)
            print(
                "Blob {} downloaded to {}.".format(
                    filename, filename
                )
            )

    def slice_into_small_tiles(self, array, new_shape, concat=False):
        '''
        Clip GeoTIFF into the wanted shape.
        :param array: Input array
        :param new_shape: Cliped shape
        :param concat: Concatenation or stack operation
        :return:
        '''
        shape_x = array.shape[0]
        shape_y = array.shape[1]
        loop_x = shape_x // new_shape
        loop_y = shape_y // new_shape
        new_array = []
        for i in range(loop_x):
            for j in range(loop_y):
                piece = array[new_shape*i:new_shape*(i+1), new_shape*j:new_shape*(j+1), :]
                new_array.append(piece)
        if concat==False:
            array = np.stack(new_array, axis=0)
        else:
            array = np.concatenate(new_array, axis=0)
        return array

    def slice_into_small_tiles_inference(self, array, new_shape=64, overlap=32):
        '''
        Clip and stack inference images into smaller shape with overlap between each other. During the inference time, to
        have better qualitative results, only the center part of the image is used.
        :param array: Input array
        :param new_shape: Cliped shape
        :param overlap: Overlap between two samples
        :return:
        '''
        shape = array.shape[0]
        center_size = new_shape-overlap
        loop_size = (shape-overlap)//center_size
        new_array = []
        for i in range(loop_size):
            for j in range(loop_size):
                piece = array[center_size*i:center_size*i+new_shape, center_size*j:center_size*j+new_shape, :]
                if np.nanmean(piece[:,:,8])==-1.0:
                    continue
                new_array.append(piece)
        array = np.stack(new_array, axis=0)
        return array, loop_size

    def combine_images_inference(self, array, loop_size, overlap=32):
        '''
        Combine images after the inference into the orginal shape
        :param array: input array
        :param loop_size: Number of horizontal loops and vertical loops
        :param overlap: overlap between patches
        :return:
        '''
        current_shape = array.shape[1]
        desired_shape = (current_shape-overlap)*loop_size
        center_shape = current_shape-overlap
        edge_shape=overlap//2
        new_array = np.zeros((desired_shape, desired_shape, array.shape[3]))
        for i in range(loop_size):
            for j in range(loop_size):
                new_array[center_shape*i:center_shape*(i+1), center_shape*j:center_shape*(j+1), :] = array[i*loop_size+j, edge_shape:current_shape-edge_shape, edge_shape:current_shape-edge_shape, :]
        return new_array

    def remove_outliers(self, x, outlierConstant):
        upper_quartile = np.nanpercentile(x, 75)
        lower_quartile = np.nanpercentile(x, 25)
        IQR = (upper_quartile - lower_quartile) * outlierConstant
        quartileSet = (lower_quartile - IQR, upper_quartile + IQR)
        result = x * (x >= quartileSet[0]) * (x <= quartileSet[1])
        return result

    def remove_outliers_nan(self, x, outlierConstant):
        upper_quartile = np.nanpercentile(x, 75)
        lower_quartile = np.nanpercentile(x, 25)
        IQR = (upper_quartile - lower_quartile) * outlierConstant
        quartileSet = (lower_quartile - IQR, upper_quartile + IQR)
        x = np.nan_to_num(x)
        result = x * (x >= quartileSet[0]) * (x <= quartileSet[1])
        result = np.where(result == 0, np.nan, result)
        return result

    def standardization(self, x):
        x = (x - x.mean()) / x.std()
        return x

    def normalization(self, x):
        return 255 * (x - np.nanmin(x)) / (np.nanmax(x) - np.nanmin(x))

    def generate_dataset_proj4(self, region_ids = ['na', 'sa', 'af', 'eu', 'au', 'sas', 'nas'], random_blind=False, year=2020, custom_region=None, mode='test', rb=0.5):
        #params_fetching = ParamsFetching()
        if custom_region!=None:
            region_ids=['custom_region']
        for region_id in region_ids:
            if year == 2019:
                path = os.path.join('gedi_palsar_data', region_id.upper(), '*.tif')
            else:
                path = os.path.join('gedi_palsar_data', region_id.upper()+str(year), '*.tif')
            file_list = glob(path)
            dataset_list = []
            print('region_id:', region_id)
            index=0
            # Read file
            for file in file_list:
                if mode=='test' and 'Test' not in file and region_id!='custom_region':
                    continue
                elif mode!='test' and 'Test' in file and region_id!='custom_region':
                    continue
                array, _ = self.read_tiff(file)
                index += 1
                if array.shape[0]<64 or array.shape[1]<64 or array.shape[2]!=12:
                    continue
                for i in range(5):
                    rh = array[:, :, 6+i]
                    array[:, :, 6+i] = self.remove_outliers_nan(rh, 1)
                output_array = np.zeros((array.shape[0], array.shape[1], 10)).astype(np.float32)
                # Previous solution we neet to transfer L2A to L4A since L4A is not availble in GEE
                # agbd_l2a = params_fetching.get_agbd(array[:, :, 4:])
                # agbd_l2a = np.where(agbd_l2a==-1, np.nan, agbd_l2a)

                # Remove outlier and nan for PALSAR-2 images
                for i in range(3):
                    output_array[:, :, i] = self.remove_outliers(array[:, :, i], 1)
                    output_array[:, :, i] = np.nan_to_num(output_array[:, :, i])

                # Randomly drop GEDI L2A RH at input
                if random_blind:
                    output_array[:, :, 4:9] = np.nan_to_num(self.random_blind(array[:, :, 6:11], rb), nan=-1)
                else:
                    output_array[:, :, 4:9] = np.nan_to_num(array[:, :, 6:11], nan=-1)

                # Remove nan for GEDI L4A for masked MSE
                agbd_l4a = np.nan_to_num(array[:, :, 11], nan=-1)
                output_array[:, :, 9] = agbd_l4a
                print(index)
                output_array[:, :, 3] = array[:, :, 3]
                output_array = self.slice_into_small_tiles(output_array, 64)
                dataset_list.append(output_array)
                del output_array
                del array
                del rh

                if index % 10==0:
                    print('{:.2f}% completed'.format(index*100/len(file_list)))

            if len(dataset_list)==1:
                dataset = dataset_list[0]
                del dataset_list
            else:
                dataset = np.concatenate(dataset_list, axis=0)
                del dataset_list

            np.save('dataset/proj4_train_'+region_id+str(year)+mode+str(rb)+'.npy', dataset)
            del dataset
            print('finish')

    def evaluate_and_plot(self, test_array_path='dataset/proj4_train_na2020.npy', model_path='model/proj4_unet_pretrained_resnet18_nchannels_', nchannels=4):
        import segmentation_models as sm
        region_id='custom_region'
        sm.set_framework('tf.keras')
        test_array= np.load(test_array_path)
        agbd = test_array[:,:,:,[9]]
        agbd = np.where(np.isnan(agbd), -1, agbd)

        # Inference
        if not os.path.exists('dataset_pred/'+region_id+'agbd_resnet18_unet_nchannels_'+str(nchannels)+'.npy'):
            model = create_model_cpu('unet', 'resnet18', 0.0003, nchannels=nchannels, nclass=1)
            model.load_weights(model_path+str(nchannels))
            agbd_pred = model.predict(test_array[:, :, :, :nchannels])
        else:
            agbd_pred = np.load('dataset_pred/'+region_id+'agbd_resnet18_unet_nchannels_'+str(nchannels)+'.npy')
        x_scatter = agbd[np.squeeze(agbd) != -1]
        y_scatter = agbd_pred[np.squeeze(agbd) != -1]
        from scipy import stats
        # Calculate R squared
        slope, intercept, r_value, p_value, std_err = stats.linregress(x_scatter.flatten(), y_scatter.flatten())
        res = stats.linregress(x_scatter.flatten(), y_scatter.flatten())
        random_id = np.random.choice(x_scatter.shape[0], 10000)
        fig = px.scatter(x=x_scatter[random_id, 0], y=y_scatter[random_id, 0] * 100, range_x=[0, 1000], range_y=[0, 1000], trendline='ols')
        fig.update_layout(title='Correlation with ' + str(nchannels) + ' channels. R-squared: {0:.2f}'.format(r_value ** 2), xaxis_title="AGB Groundtruth (GEDI L4A AGB)", yaxis_title="AGB Predicted")
        fig.show()

    def inference(self, rb, path='gedi_palsar_data/CUSTOM_REGION2020/*.tif', random_blind=False, model_path='model/proj4_unet_pretrained_resnet18_nchannels_', nchannels=9, overlap=32):
        # Inference and generate image with original scale
        file_list=glob(path)
        import segmentation_models as sm
        sm.set_framework('tf.keras')
        model = create_model_cpu('unet', 'resnet18', 0.0003, nchannels, 1)
        model.load_weights(model_path+str(nchannels))
        for file_dir in file_list:
            array, pf = self.read_tiff(file_dir)
            if array.shape[0] != 1280 or array.shape[1] != 1280 or array.shape[2] != 11:
                print('invalid shape')
                continue
            output_array = np.zeros((array.shape[0], array.shape[1], 9)).astype(np.float32)
            for i in range(3):
                output_array[:, :, i] = self.remove_outliers(array[:, :, i], 1)
                output_array[:, :, i] = np.nan_to_num(output_array[:, :, i])
            output_array[:, :, 3] = array[:, :, 3]
            if not random_blind:
                output_array[:, :, 4:] = np.nan_to_num(array[:, :, 6:])
            else:
                output_array[:, :, 4:] = np.nan_to_num(self.random_blind(array[:, :, 6:], rb))
            input, loop_size = self.slice_into_small_tiles_inference(output_array, new_shape=64, overlap=overlap)
            agbd_pred = model.predict(input[:,:,:,:nchannels])
            agbd_pred = self.combine_images_inference(array=agbd_pred, loop_size=loop_size, overlap=overlap)
            pf.data['count'] = 1
            self.write_tiff(file_dir.replace('gedi_palsar_data', 'recon'), agbd_pred.transpose((2,0,1)), pf)
            print('successfully reconstruct agbd predicted')

if __name__=='__main__':

    array=np.array([[np.nan,np.nan,np.nan,np.nan,4],
                   [np.nan,2,np.nan,np.nan,np.nan],
                   [np.nan,np.nan,1,np.nan,np.nan],
                   [np.nan,3,np.nan,np.nan,np.nan],
                   [np.nan,np.nan,np.nan,5,np.nan]])
    gedi=Gedi()
    print(array)
    print(gedi.random_blind(array, 1))

{'sm': 'segmentation_models', 'stats': 'scipy.stats'}


from flask import Flask, redirect, url_for
from flask_admin import Admin
from flask_migrate import Migrate
from flask_admin.contrib.sqla import ModelView
from . settings import *
from . extensions import db, bcrypt, csrf, login_manager, mail

from apps.accounts.views import accounts_router
from apps.chat.views import chat_router
from apps.status.views import status_router

from apps.common.models import TimeStampedUUIDModel
from apps.accounts.models import User, Timezone, BlockedContact, Otp
from apps.chat.models import Message

def register_extensions(app):
    db.init_app(app)
    with app.app_context():
        db.create_all()
    Migrate(app, db)
    bcrypt.init_app(app)
    login_manager.init_app(app)
    csrf.init_app(app)
    app.config['MAIL_SERVER'] = MAIL_SERVER
    app.config['MAIL_PORT'] = MAIL_PORT
    app.config['MAIL_USERNAME'] = MAIL_USERNAME
    app.config['MAIL_PASSWORD'] = MAIL_PASSWORD
    # app.config['MAIL_USE_SSL'] = MAIL_USE_SSL
    mail.init_app(app)

def create_app(config):
    app = Flask(__name__, root_path=ROOT_PATH)
    app.config.from_object(config)
    app.config['SECRET_KEY'] = SECRET_KEY
    app.config["SQLALCHEMY_DATABASE_URI"] = f"postgresql://{DATABASES['USER']}:{DATABASES['PASSWORD']}@{DATABASES['HOST']}:{DATABASES['PORT']}/{DATABASES['DB_NAME']}"
    app.config['FLASK_ADMIN_SWATCH'] = 'cerulean'
    # app.wsgi_app = middleware(app.wsgi_app)

    # REGISTER ADMIN
    admin = Admin(app, name='WWC V4', template_mode='bootstrap3')
    admin.add_view(ModelView(User, db.session))
    admin.add_view(ModelView(Timezone, db.session))
    admin.add_view(ModelView(BlockedContact, db.session))
    admin.add_view(ModelView(Otp, db.session))
    admin.add_view(ModelView(Message, db.session))

    # REGISTER BLUEPRINTS
    app.register_blueprint(accounts_router, url_prefix="/accounts")
    app.register_blueprint(chat_router, url_prefix="/chat")
    app.register_blueprint(status_router, url_prefix="/status")
    register_extensions(app)

    return app

app = create_app(config)
app.app_context().push()

@app.route('/', methods=['GET'])
def index():
    return redirect(url_for('chat_router.home'))


import sys
sys.stdin = open('input.txt')
from collections import deque

dx = [-1, 1, 0, 0]
dy = [0, 0, -1, 1]

def bfs():
    q = deque()
    q.append([0, 0, 0])      # 시작이 1,1 인데 인덱스 맞추려고 0,0 함

    while q:
        x, y, z = q.popleft()

        # 도착
        if x == n-1 and y == m-1:
            return visited[x][y][z]

        for k in range(4):
            nx = x + dx[k]
            ny = y + dy[k]

            # 범위 내에 있을 때
            if 0 <= nx < n and 0 <= ny < m:
                # 벽을 만나고, 안 부술 경우 > +1 인 부분이 cnt += 1 같은 역할
                if load[nx][ny] == 1 and z == 0:
                    visited[nx][ny][1] = visited[x][y][0] + 1
                    q.append([nx, ny, 1])

                # 벽 아니고, 방문 안 했다면 > +1 인 부분이 cnt += 1 같은 역할
                elif load[nx][ny] == 0 and visited[nx][ny][z] == 0:
                    visited[nx][ny][z] = visited[x][y][z] + 1
                    q.append([nx, ny, z])

    return -1

n, m = map(int, input().split())
load = [list(map(int, input())) for _ in range(n)]

# 3차원 행렬로 벽 부수는 거 판단, visited[x][y][0] > 부수기 가능, [1]은 불가능
visited = [[[0] * 2 for _ in range(m)] for _ in range(n)]
visited[0][0][0] = 1

print(bfs())


from django import forms
from .models import Cases, CaseComments, CasePresentations, Documents
from django.contrib.auth.models import User
from .widgets import DatePickerInput
from django.contrib.auth import get_user_model
from django.contrib.auth.forms import UserCreationForm

class CasesForm(forms.ModelForm):
    lawyers=forms.ModelMultipleChoiceField(queryset=User.objects.all(), widget=forms.CheckboxSelectMultiple, label="Abogados propios" )

    class Meta:
        model = Cases
        fields = ['name', 'description', 'start_date', 'case_type', 'demandante', 'demandado',
                  'juzgado', 'partido_judicial', 'phase', 'demandado', 'lawyers', 'cont_lawyer_one', 'cont_lawyer_two', 'cont_lawyer_three', 'procurador_one',
                  'procurador_two',
                  'procurador_three','pc_one','pc_two','pc_three']
        CASE_TYPES_CHOICES = (
            ("", "Seleccione un tipo de juicio"),
            ("Juicio ordinario", "Juicio ordinario"),
            ("Juicio verbal", "Juicio verbal"),
            ("Juicio Monitorio", "Juicio Monitorio"),
            ("Juicio Cambiario", "Juicio Cambiario"),
        )

        PHASE_CHOICES = (
            ('', 'Seleccione la fase actual del procedimiento'),
            ('Reclamación Extrajudicial', 'Reclamación Extrajudicial'),
            ('Demanda', 'Demanda'),
            ('Poder Apud Acta', 'Poder Apud Acta'),
            ('Contestación a la demanda', 'Contestación a la demanda'),
            ('Audiencia previa', 'Audiencia previa'),
            ('Juicio oral', 'Juicio oral'),
            ('Recurso apelación', 'Recurso apelación'),
            ('Recurso casación', 'Recurso casación'),
        )
        widgets = {
            'name': forms.TextInput(attrs={
                'class': "form-control",
                'placeholder': "Introduce el título del procedimiento"
            }),
            'start_date': DatePickerInput(),
            'description': forms.Textarea(attrs={'class': "form-control", 'placeholder': "Introduce la descripción del procedimiento"}),
            'case_type': forms.Select(choices=CASE_TYPES_CHOICES, attrs={'class': 'form-control'}),
            'demandante': forms.TextInput(attrs={
                'class': "form-control",
                'placeholder': "Introduce el nombre del Demandante"
            }),
            'demandado': forms.TextInput(attrs={
                'class': "form-control",
                'placeholder': "Introduce el nombre del Demandado"
            }),
            'juzgado': forms.TextInput(attrs={
                'class': "form-control",
                'placeholder': "Introduce el juzgado"
            }),
            'partido_judicial': forms.TextInput(attrs={
                'class': "form-control",
                'placeholder': "Introduce el partido judicial"
            }),
            'phase': forms.Select(choices=PHASE_CHOICES, attrs={'class': 'form-control'}),
            'cont_lawyer_one': forms.TextInput(attrs={
                'class': "form-control",
                'placeholder': "Introduce el nombre de un abogado contrario",
                'required': True
            }),
            'cont_lawyer_two': forms.TextInput(attrs={
                'class': "form-control",
                'placeholder': "Introduce el nombre de un abogado contrario",
                'required': False
            }),
            'cont_lawyer_three': forms.TextInput(attrs={
                'class': "form-control",
                'placeholder': "Introduce el nombre de un abogado contrario",
                'required': False
            }),
            'procurador_one': forms.TextInput(attrs={
                'class': "form-control",
                'placeholder': "Introduce el nombre de un procurador propio",
                'required': True
            }),
            'procurador_two': forms.TextInput(attrs={
                'class': "form-control",
                'placeholder': "Introduce el nombre de un procurador propio",
                'required': False
            }),
            'procurador_three': forms.TextInput(attrs={
                'class': "form-control",
                'placeholder': "Introduce el nombre de un procurador propio",
                'required': False
            }),
            'pc_one': forms.TextInput(attrs={
                'class': "form-control",
                'placeholder': "Introduce el nombre de un procurador contrario",
                'required': True
            }),
            'pc_two': forms.TextInput(attrs={
                'class': "form-control",
                'placeholder': "Introduce el nombre de un procurador contrario",
                'required': False
            }),
            'pc_three': forms.TextInput(attrs={
                'class': "form-control",
                'placeholder': "Introduce el nombre de un procurador contrario",
                'required': False
            }),

        }
        labels = {
            'name':'Título del procedimiento',
            'description':'Descripción',
            'phase': 'Fase',
            'lawyers': 'Abogados propios',
            'start_date': 'Fecha de comienzo',
            'cont_lawyer_one': 'Abogado contrario 1',
            'cont_lawyer_two': 'Abogado contrario 2',
            'cont_lawyer_three': 'Abogado contrario 3',
            'case_type': 'Tipo de procedimiento',
            'procurador_one': "Procurador propio 1",
            'procurador_two': "Procurador propio 2",
            'procurador_three': "Procurador propio 3",
            'pc_one': "Procurador contrario 1",
            'pc_two': "Procurador contrario 2",
            'pc_three': "Procurador contrario 3",

        }

class SignUpForm(UserCreationForm):
    class Meta:
        model = get_user_model()
        fields = ("first_name", "last_name", "email",
                  "username", "password1", "password2")

class CreateCommentForm(forms.ModelForm):
    class Meta:
        model = CaseComments
        fields = ['comment']
        widgets = {
            'comment': forms.Textarea(attrs={'class': "form-control", 'placeholder': "Introduce tu comentario"}),
        }
        labels = {
            'comment': "Comentario"
        }

class CreatePresentationForm(forms.ModelForm):
    class Meta:
        model = CasePresentations
        fields = ['description', 'presentation_date']
        widgets = {
            'description': forms.Textarea(attrs={'class': "form-control", 'placeholder': "Introduce tu comentario"}),
            'presentation_date':DatePickerInput()
        }
        labels = {
            'description': "Descripción de presentación",
            'presentation_date':"Fecha de presentación"
        }

class DocForm(forms.ModelForm):
    class Meta:
        model=Documents
        fields=('title','doc')

        labels={
            'title':'Título',
            'doc':'Documento (PNG,JPG)'
        }