text:
# Copyright (c) 2015-2020, Swiss Federal Institute of Technology (ETH Zurich)
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
"""Analyse whether ML techniques are capable of determining the currently running application in the input (thermal) sequences."""

import abc
import math
import os
import pickle as pk
import random
import time
import typing as t
from functools import partial
from pathlib import Path

import editdistance
import numpy as np
import pandas as pd
import scipy.stats as st
import tensorflow as tf
import tensorflow_addons as tfa
import toml
from fastdtw import fastdtw as FDTW
from scipy import interpolate
from scipy.spatial.distance import euclidean
from sklearn.utils import shuffle

from exot.channel._base import Analysis
from exot.channel.mixins.mldataset import (
    TestSampleLen,
    TestWeights,
    TestX,
    TestY,
    TrainSampleLen,
    TrainWeights,
    TrainX,
    TrainY,
    VerifSampleLen,
    VerifX,
    VerifY,
)
from exot.exceptions import *
from exot.util.attributedict import AttributeDict, LabelMapping
from exot.util.logging import get_root_logger
from exot.util.misc import validate_helper
from exot.util.wrangle import Matcher

from ._mixins import ThermalAppDecectionDataSetHandler

__all__ = ("AppDetection", "ADLSTM", "ADCNNLSTM", "ADConvLSTM", "ADBiDiLSTM")

class AppDetection(
    Analysis,
    ThermalAppDecectionDataSetHandler,
    TrainX,
    TrainY,
    TrainSampleLen,
    TrainWeights,
    TestX,
    TestY,
    TestSampleLen,
    TestWeights,
    VerifX,
    VerifY,
    VerifSampleLen,
):
    def __init__(self, *args, **kwargs):
        Analysis.__init__(self, *args, **kwargs)

        tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)

        validate = partial(validate_helper, self.config, msg="Analysis")

        # top-level configuration
        validate("name", str)
        validate("type", str)

        validate(("DATASET"), AttributeDict)
        validate(("DATASET", "load_from"), (str))
        self.read_dataset()

        validate(("TRAIN"), AttributeDict)
        validate(("TRAIN", "n_epochs"), (int))
        validate(("TRAIN", "debug_step"), (int))
        validate(("TRAIN", "early_stopping_threshold"), (int))

        validate(("MODEL"), AttributeDict)
        validate(("MODEL", "type"), str)
        validate(("MODEL", "max_grad_norm"), (float, int))
        validate(("MODEL", "p_dropout"), (float, int))
        validate(("MODEL", "num_layers"), (int))
        validate(("MODEL", "num_hidden"), (int))
        validate(("MODEL", "learning_rate"), (float, int))

    @property
    def n_epochs(self):
        return getattr(self.config.TRAIN, "n_epochs", 0)

    @n_epochs.setter
    def n_epochs(self, value):
        self.config.TRAIN.n_epochs = value

    @property
    def start_epoch(self):
        if hasattr(self.config.TRAIN, "start_epoch") and not hasattr(self, "_start_epoch"):
            self._start_epoch = self.config.TRAIN["start_epoch"]
        return getattr(self, "_start_epoch", 0)

    @start_epoch.setter
    def start_epoch(self, value: int) -> None:
        self._start_epoch = value

    @property
    def debug_step(self):
        return getattr(self.config.TRAIN, "debug_step", 0)

    @property
    def early_stopping_threshold(self):
        return getattr(self.config.TRAIN, "early_stopping_threshold", 0)

    @property
    def model_type(self):
        return getattr(self.config.MODEL, "type", None)

    @property
    def max_grad_norm(self):
        return getattr(self.config.MODEL, "max_grad_norm", None)

    @property
    def p_dropout(self):
        return getattr(self.config.MODEL, "p_dropout", None)

    @property
    def p_keep(self):
        return 1 - self.model_p_dropout

    @property
    def num_layers(self):
        return getattr(self.config.MODEL, "num_layers", None)

    @property
    def num_hidden(self):
        return getattr(self.config.MODEL, "num_hidden", None)

    @property
    def learning_rate(self):
        return getattr(self.config.MODEL, "learning_rate", None)

    @property
    def model_files(self):
        return getattr(self.config.MODEL, "model_files", None)

    @model_files.setter
    def model_files(self, value: str) -> None:
        setattr(self.config.MODEL, "model_files", value)

    @property
    def filtering_window_size(self):
        return getattr(self.config.MODEL, "filtering_window_size", 25)

    @property
    def unknown_label(self):
        return getattr(self.config.DATASET, "unknown_label", False)

    @unknown_label.setter
    def unknown_label(self, value: bool) -> None:
        self.config.DATASET.unknown_label = value

    @property
    def max_timesteps(self):
        return getattr(self.config.MODEL, "max_timesteps", 0)

    @property
    def batch_size(self):
        if not hasattr(self, "_batch_size"):
            self._batch_size = max(
                [
                    math.ceil(self.train_actual_batch_len.max() / self.max_timesteps),
                    math.ceil(self.test_actual_batch_len.max() / self.max_timesteps),
                    math.ceil(self.verif_actual_batch_len.max() / self.max_timesteps),
                ]
            )
        return self._batch_size

    @batch_size.setter
    def batch_size(self, value):
        pass

    def reshuffle_data(self, phase: str, epoch: int) -> None:
        X, Y, actual_batch_len, weights = shuffle(
            getattr(self, phase + "_X"),
            getattr(self, phase + "_Y"),
            getattr(self, phase + "_actual_batch_len"),
            getattr(self, phase + "_weights"),
        )
        setattr(self, phase + "_X", X)
        setattr(self, phase + "_Y", Y)
        setattr(self, phase + "_actual_batch_len", actual_batch_len)
        setattr(self, phase + "_weights", weights)

    @property
    def train_X_shape(self):
        return (self.batch_size, self.max_timesteps, self.num_feature_dims)

    @property
    def train_Y_shape(self):
        return (self.batch_size, self.max_timesteps)

    @property
    def train_actual_batch_len_shape(self):
        return self.batch_size

    @property
    def train_weights_shape(self):
        return self.train_Y_shape

    @property
    def test_X_shape(self):
        return self.train_X_shape

    @property
    def test_Y_shape(self):
        return self.train_Y_shape

    @property
    def test_actual_batch_len_shape(self):
        return self.batch_size

    @property
    def test_weights_shape(self):
        return self.test_Y_shape

    @property
    def verif_X_shape(self):
        return self.train_X_shape

    @property
    def verif_Y_shape(self):
        return train_Y_shape

    @property
    def verif_actual_batch_len_shape(self):
        return self.batch_size

    def get_data(self, phase: str, batch: int) -> tuple:
        X = np.zeros([self.batch_size, self.max_timesteps, self.num_feature_dims])
        Y = np.zeros([self.batch_size, self.max_timesteps])
        w = np.ones([self.batch_size, self.max_timesteps])

        full_samples, last_sample_len = divmod(
            getattr(self, phase + "_actual_batch_len")[batch, 0], self.max_timesteps
        )
        full_samples = int(full_samples)
        last_sample_len = int(last_sample_len)

        if phase == "verif":
            if self.batch_size_timesteps > getattr(self, phase + "_X").shape[1]:
                X[:full_samples, :, :] = getattr(self, phase + "_X")[
                    batch, : full_samples * self.max_timesteps, :
                ].reshape([full_samples, self.max_timesteps, self.num_feature_dims])
                X[full_samples, :last_sample_len, :] = getattr(self, phase + "_X")[
                    batch, full_samples * self.max_timesteps :, :
                ].reshape([last_sample_len, self.num_feature_dims])

                Y[:full_samples, :] = (
                    getattr(self, phase + "_Y")[batch, : full_samples * self.max_timesteps, :]
                    .squeeze()
                    .reshape([full_samples, self.max_timesteps])
                )
                Y[full_samples, :last_sample_len] = (
                    getattr(self, phase + "_Y")[batch, full_samples * self.max_timesteps :, :]
                    .squeeze()
                    .reshape([last_sample_len])
                )

                w[full_samples, :last_sample_len] = 0
            else:
                X = getattr(self, phase + "_X")[batch, : self.batch_size_timesteps, :].reshape(
                    [self.batch_size, self.max_timesteps, self.num_feature_dims]
                )
                Y = (
                    getattr(self, phase + "_Y")[batch, : self.batch_size_timesteps, :]
                    .squeeze()
                    .reshape([self.batch_size, self.max_timesteps])
                )
        else:
            cur_position = 0
            for sample_idx in range(self.batch_size):
                if sample_idx == self.batch_size - 1:
                    # For training, make sure the last sample has between 1% and 10% of zero padding at the end
                    actual_sample_len = random.randint(
                        math.floor(0.9 * self.max_timesteps),
                        math.ceil(0.95 * self.max_timesteps),
                    )
                else:
                    actual_sample_len = self.max_timesteps
                X[sample_idx, :actual_sample_len, :] = getattr(self, phase + "_X")[
                    batch, cur_position : cur_position + actual_sample_len, :
                ].reshape([actual_sample_len, self.num_feature_dims])
                Y[sample_idx, :actual_sample_len] = getattr(self, phase + "_Y")[
                    batch, cur_position : cur_position + actual_sample_len, :
                ].reshape([actual_sample_len])
                w[sample_idx, :actual_sample_len] = getattr(self, phase + "_weights")[
                    batch, cur_position : cur_position + actual_sample_len, :
                ].reshape([actual_sample_len])
                w[sample_idx, actual_sample_len:] = 0

                cur_position += actual_sample_len

        return X.astype(np.float32), Y.astype(int), w.astype(np.float32)

    @property
    def path(self) -> Path:
        return self.experiment.path.joinpath(self.name)

    @property
    def path_dataset(self) -> Path:
        """Get the save path"""
        return self.experiment.path.joinpath(self.config.DATASET.load_from)

    def read(self):
        pass

    def write(self):
        pass

    def _execute_handler(self, *args, **kwargs):
        self.init_app_detector()
        best_model = self.train()
        self.verify(best_model)

    def write_debug_files(self, phase: str, outputs=None, prefix="", as_txt=False):
        """Serialise a checkpoint"""
        debug_path = self.path.joinpath(phase)
        debug_path.mkdir(exist_ok=True)
        # Take advantage of the writing methods
        if outputs is not None:
            for name, value in outputs.items():
                if isinstance(value, np.ndarray):
                    if as_txt:
                        np.savetxt(
                            debug_path / (prefix + "_" + name + ".dat"), value, fmt="%.3f"
                        )
                    else:
                        self._save_npz(
                            debug_path / (prefix + "_" + name + ".npz"), {name: value}
                        )
                elif isinstance(value, (pd.DataFrame, pd.Series, pd.Panel)):
                    self._save_hdf(debug_path / (prefix + "_" + name + ".h5"), {name: value})
                elif isinstance(value, (dict, AttributeDict)):
                    with debug_path.joinpath(prefix + "_" + name + ".toml").open(
                        "w"
                    ) as toml_file:
                        toml.dump(value, toml_file)
                else:
                    with open(debug_path.joinpath(prefix + "_" + name + ".pkl"), "wb") as f:
                        pk.dump(value, f)

    def read_debug_files(self, phase: str, prefix=""):
        """Load a checkpoint"""
        debug_path = self.path.joinpath(phase)
        # Take advantage of the writing methods
        loaded_data = AttributeDict()
        for _data_file_path in os.listdir(debug_path):
            _data_file_path = debug_path.joinpath(_data_file_path)
            if _data_file_path.name.startswith(prefix):
                _fileextension = _data_file_path.name.split(".")[-1]
                key = _data_file_path.name.replace(prefix + "_", "")
                key = key.replace("." + _fileextension, "")
                if _fileextension == "npz":
                    with np.load(_data_file_path) as _np_store:
                        for data_key in _np_store:
                            loaded_data[data_key] = _np_store[data_key]
                elif _fileextension == "dat":
                    loaded_data[key] = np.loadtxt(_data_file_path)
                elif _fileextension == "h5":
                    with pd.HDFStore(_data_file_path, "r") as _pd_store:
                        for data_key in _pd_store:
                            loaded_data[data_key] = _pd_store[data_key]
                elif _fileextension == "toml":
                    with _data_file_path.open("r") as toml_file:
                        loaded_data[key] = toml.load(toml_file)
                elif _fileextension == "pkl":
                    with open(_data_file_path, "rb") as f:
                        loaded_data[key] = pk.load(f)
                else:
                    raise Exception(f"Unknown file extension for file {_data_file_path}")
        return loaded_data

    def init_app_detector(self):
        appdetector_args = {
            "batch_size": self.batch_size,
            "num_hidden": self.num_hidden,
            "num_layers": self.num_layers,
            "num_labels": self.num_labels,
            "p_dropout": self.p_dropout,
            "max_timesteps": self.max_timesteps,
            "num_features": self.max_timesteps,
        }
        if self.model_type == "LSTM":
            self.appdetector = ADLSTM(**appdetector_args)
        elif self.model_type == "CNNLSTM":
            self.appdetector = ADCNNLSTM(**appdetector_args)
        else:
            raise Exception(NotImplemented)
        self.optimizer = tf.keras.optimizers.RMSprop(
            learning_rate=self.learning_rate,
            rho=0.9,
            momentum=0.1,
            epsilon=1e-07,
            centered=False,
            name="RMSprop",
        )
        self.checkpoint = tf.train.Checkpoint(
            optimizer=self.optimizer, appdetector=self.appdetector
        )

    def train(self):
        get_root_logger().info(
            "Epoch X | loss: (TRAIN/TEST) | accuracy: (TRAIN/TEST) | lev_abs: (TRAIN/TEST) | lev_len: (TRAIN/TEST) | lev_norm: (TRAIN/TEST) | timing: (TRAIN/TEST)"
        )
        self.best_lev_norm = np.inf
        self.best_loss = np.inf
        best_model = None
        early_stop = False

        trace_loss = {
            "train": np.empty((math.ceil(self.n_epochs / self.debug_step))),
            "test": np.empty((math.ceil(self.n_epochs / self.debug_step))),
        }
        trace_accuracy = {
            "train": np.empty((math.ceil(self.n_epochs / self.debug_step))),
            "test": np.empty((math.ceil(self.n_epochs / self.debug_step))),
        }
        trace_lev_abs = {
            "train": np.empty((math.ceil(self.n_epochs / self.debug_step))),
            "test": np.empty((math.ceil(self.n_epochs / self.debug_step))),
        }
        trace_lev_norm = {
            "train": np.empty((math.ceil(self.n_epochs / self.debug_step))),
            "test": np.empty((math.ceil(self.n_epochs / self.debug_step))),
        }
        trace_lev_len = {
            "train": np.empty((math.ceil(self.n_epochs / self.debug_step))),
            "test": np.empty((math.ceil(self.n_epochs / self.debug_step))),
        }
        trace_timing = {
            "train": np.empty((math.ceil(self.n_epochs / self.debug_step))),
            "test": np.empty((math.ceil(self.n_epochs / self.debug_step))),
        }

        loss = {"train": None, "test": None}
        accuracy = {"train": None, "test": None}
        lev_abs = {"train": None, "test": None}
        lev_norm = {"train": None, "test": None}
        lev_len = {"train": None, "test": None}
        timing = {"train": None, "test": None}
        pred = {"train": None, "test": None}
        Y = {"train": None, "test": None}
        X = {"train": None, "test": None}
        weights = {"train": None, "test": None}
        dbg_cnt = 0
        for epoch in range(self.start_epoch + 1, self.n_epochs + 1):
            appdetector_state = self.appdetector.initial_hidden_state
            # Training
            self.reshuffle_data("train", epoch)
            debug = (epoch % self.debug_step) == 0 or epoch == self.n_epochs or epoch == 1
            (
                pred["train"],
                Y["train"],
                X["train"],
                weights["train"],
                loss["train"],
                accuracy["train"],
                lev_abs["train"],
                lev_norm["train"],
                lev_len["train"],
                timing["train"],
            ) = self.inference(epoch, train=True, debug=debug)
            if (epoch % self.debug_step) == 0 or epoch == self.n_epochs or epoch == 1:
                (
                    pred["test"],
                    Y["test"],
                    X["test"],
                    weights["test"],
                    loss["test"],
                    accuracy["test"],
                    lev_abs["test"],
                    lev_norm["test"],
                    lev_len["test"],
                    timing["test"],
                ) = self.inference(
                    epoch, train=False, debug=debug
                )  # Test inference

                for phase in ["train", "test"]:
                    trace_loss[phase][dbg_cnt] = np.nansum(loss[phase])
                    trace_accuracy[phase][dbg_cnt] = np.nanmean(accuracy[phase])
                    trace_lev_abs[phase][dbg_cnt] = np.nansum(lev_abs[phase])
                    trace_lev_norm[phase][dbg_cnt] = np.nanmean(lev_norm[phase])
                    trace_lev_len[phase][dbg_cnt] = np.nansum(lev_len[phase])
                    trace_timing[phase][dbg_cnt] = np.nanmean(timing[phase])

                get_root_logger().info(
                    "Epoch %03i | loss: (%03.3f/%03.3f) | accuracy: (%2.2f%%/%2.2f%%) | lev_abs: (%4i/%4i) | lev_len: (%4i/%4i) | lev_norm: (%.3f/%.3f) | timing: (%.3f/%.3f)"
                    % (
                        epoch,
                        trace_loss["train"][dbg_cnt],
                        trace_loss["test"][dbg_cnt],
                        trace_accuracy["train"][dbg_cnt],
                        trace_accuracy["test"][dbg_cnt],
                        trace_lev_abs["train"][dbg_cnt],
                        trace_lev_abs["test"][dbg_cnt],
                        trace_lev_len["train"][dbg_cnt],
                        trace_lev_len["test"][dbg_cnt],
                        trace_lev_norm["train"][dbg_cnt],
                        trace_lev_norm["test"][dbg_cnt],
                        trace_timing["train"][dbg_cnt],
                        trace_timing["test"][dbg_cnt],
                    )
                )

                if trace_loss["test"][dbg_cnt] == 0.0:
                    train_violations = self.early_stopping_threshold + 1
                if trace_loss["test"][dbg_cnt] < self.best_loss:
                    train_violations = 0
                    self.best_loss = trace_loss["test"][dbg_cnt]
                    for phase in ["train", "test"]:
                        self.write_debug_files(
                            phase=phase, outputs=pred[phase], prefix="prediction"
                        )
                        self.write_debug_files(phase=phase, outputs=Y[phase], prefix="Y")
                        self.write_debug_files(
                            phase=phase,
                            outputs={"X": X[phase], "weights": weights[phase]},
                            prefix="data",
                        )
                        self.write_debug_files(
                            phase=phase,
                            outputs={
                                "accuracy": accuracy[phase],
                                "lev_abs": lev_abs[phase],
                                "lev_norm": lev_norm[phase],
                                "lev_len": lev_len[phase],
                                "timing": timing[phase],
                                "results": {
                                    "epoch": epoch,
                                    "loss": trace_loss[phase][dbg_cnt],
                                    "accuracy": trace_accuracy[phase][dbg_cnt],
                                    "lev_abs": trace_lev_abs[phase][dbg_cnt],
                                    "lev_norm": trace_lev_norm[phase][dbg_cnt],
                                    "timing": trace_timing[phase][dbg_cnt],
                                },
                            },
                            prefix="metric",
                            as_txt=True,
                        )
                    best_model = self.checkpoint.save(file_prefix=self.path.joinpath("tf_ckpt"))
                    self.config.MODEL["best_model"] = best_model
                    with self.path.joinpath("model.toml").open("w") as toml_file:
                        toml.dump(self.config.MODEL, toml_file)
                else:
                    train_violations += self.debug_step if epoch >= self.debug_step else 1
                dbg_cnt += 1
            if train_violations >= self.early_stopping_threshold:
                if not early_stop:
                    new_lr = self.learning_rate / 10
                    get_root_logger().info(
                        "Starting second phase of training at %i epochs with learning rate %f!"
                        % (epoch, new_lr)
                    )
                    get_root_logger().info(
                        "Restoring previously best model from %s!" % str(best_model)
                    )
                    self.checkpoint.restore(best_model)
                    config = self.optimizer.get_config()
                    config["learning_rate"] = self.learning_rate / 10
                    self.optimizer = self.optimizer.from_config(config)
                    early_stop = True
                    train_violations = 0
                else:
                    get_root_logger().info("Early stopping triggered after %i epochs!" % epoch)
                    break

        for phase in ["train", "test"]:
            self.write_debug_files(
                phase=phase,
                outputs={
                    "loss": trace_loss[phase],
                    "accuracy": trace_accuracy[phase],
                    "lev_abs": trace_lev_abs[phase],
                    "lev_norm": trace_lev_norm[phase],
                    "lev_len": trace_lev_len[phase],
                    "timing": trace_timing[phase],
                },
                prefix="trace",
                as_txt=True,
            )
        get_root_logger().info(
            "Training finished, saved files to "
            + str(self.path.joinpath("train"))
            + " and "
            + str(self.path.joinpath("test"))
        )
        return best_model

    def debug_restore_checkpoint(self, phase: str = "train"):
        pred = self.read_debug_files(phase=phase, prefix="prediction")
        Y = self.read_debug_files(phase=phase, prefix="Y")
        data = self.read_debug_files(phase=phase, prefix="data")
        metrics = self.read_debug_files(phase=phase, prefix="metric")
        traces = self.read_debug_files(phase=phase, prefix="trace")
        return pred, Y, data, metrics, traces

    def verify(self, best_model=None):
        if best_model is not None:
            self.checkpoint.restore(best_model)
            get_root_logger().info("Restored model from " + str(best_model))
        if self.verif_X is None:
            get_root_logger().info("No Inference data, nothing to do...")
        else:
            (
                pred,
                Y,
                X,
                weights,
                loss,
                accuracy,
                lev_abs,
                lev_norm,
                lev_len,
                timing,
            ) = self.inference(train=False, epoch=None, debug=True)
            overall_loss = np.nansum(loss)
            overall_accuracy = np.nanmean(accuracy)
            overall_lev_abs = np.nansum(lev_abs)
            overall_lev_norm = np.nanmean(lev_norm)
            overall_lev_len = np.nansum(lev_len)
            overall_timing = np.nanmean(timing)
            get_root_logger().info(
                "Verification | loss: %03.3f | accuracy: %2.2f%% | lev_abs: %4i | lev_len: %4i | lev_norm: %.3f | timing: %.3f"
                % (
                    overall_loss,
                    overall_accuracy,
                    overall_lev_abs,
                    overall_lev_len,
                    overall_lev_norm,
                    overall_timing,
                )
            )
            self.write_debug_files(phase="verif", outputs=pred, prefix="prediction")
            self.write_debug_files(phase="verif", outputs=Y, prefix="Y")
            self.write_debug_files(phase="verif", outputs=self.test_set, prefix="")
            self.write_debug_files(
                phase="verif", outputs={"X": X, "weights": weights}, prefix="data"
            )
            self.write_debug_files(
                phase="verif",
                outputs={
                    "accuracy": accuracy,
                    "lev_abs": lev_abs,
                    "lev_norm": lev_norm,
                    "lev_len": lev_len,
                    "timing": timing,
                    "results": {
                        "loss": overall_loss,
                        "accuracy": overall_accuracy,
                        "lev_abs": overall_lev_abs,
                        "lev_norm": overall_lev_norm,
                        "timing": overall_timing,
                    },
                },
                prefix="metric",
                as_txt=True,
            )
            self.checkpoint.save(file_prefix=self.path.joinpath("verif").joinpath("tf_ckpt"))
            get_root_logger().info(
                "Inference finished, saved files to " + str(self.path.joinpath("verif"))
            )

    def inference(self, epoch: int = None, train: bool = True, debug: bool = True):
        if train:
            phase = "train"
        else:
            phase = "verif" if epoch is None else "test"
        num_batches = getattr(self, "num_" + phase + "_batches")
        pred = {
            "raw": [None for _ in range(num_batches)],
            "filtered": [None for _ in range(num_batches)],
            "collapse": [None for _ in range(num_batches)],
            "timingCX": [None for _ in range(num_batches)],
        }
        Y = {
            "raw": [None for _ in range(num_batches)],
            "collapse": [None for _ in range(num_batches)],
            "timingCX": [None for _ in range(num_batches)],
        }
        X = [None for _ in range(num_batches)]
        weights = [None for _ in range(num_batches)]

        loss = np.zeros((num_batches))
        accuracy = np.zeros((num_batches))
        lev_abs = np.zeros((num_batches))
        lev_norm = np.zeros((num_batches))
        lev_len = np.zeros((num_batches))
        timing = np.zeros((num_batches))

        if epoch is not None:
            self.reshuffle_data(phase, epoch)
        appdetector_state = self.appdetector.initial_hidden_state
        for batch in range(num_batches):
            X[batch], Y["raw"][batch], weights[batch] = self.get_data(phase, batch)

            # Use application detector to get per timestep predictions
            loss[batch], tf_pred, appdetector_state = self.process(
                X[batch],
                Y["raw"][batch][:, 0 :: self.appdetector.oversampling_rate],
                weights[batch][:, 0 :: self.appdetector.oversampling_rate],
                appdetector_state,
                train=train,
            )
            if debug:
                pred["raw"][batch] = tf_pred.numpy()

                if pred["raw"][batch].size != Y["raw"][batch].size:
                    timesteps_new = np.arange(0, Y["raw"][batch].shape[1])
                    timesteps_old = np.linspace(
                        0, Y["raw"][batch].shape[1], pred["raw"][batch].shape[1]
                    )
                    pred_resampled = np.empty(Y["raw"][batch].shape)
                    for sample_idx in range(Y["raw"][batch].shape[0]):
                        label_interpolation = interpolate.interp1d(
                            timesteps_old, pred["raw"][batch][sample_idx, :], kind="nearest"
                        )
                        pred_resampled[sample_idx, :] = label_interpolation(timesteps_new)
                    pred["raw"][batch] = pred_resampled

                # Apply majority voting to compensate for jittery predictions
                pred["filtered"][batch] = self.filter_batch(
                    pred["raw"][batch], window_size=self.filtering_window_size
                )

                # Collapse labels such that only the sequence without duplicates remains
                pred["collapse"][batch], pred["timingCX"][batch] = self.get_sequences(
                    pred["filtered"][batch], weights[batch]
                )
                Y["collapse"][batch], Y["timingCX"][batch] = self.get_sequences(
                    Y["raw"][batch], weights[batch]
                )

                # Calculate metrics
                accuracy[batch] = self.batch_accuracy(
                    pred["filtered"][batch], Y["raw"][batch], weights[batch]
                )
                lev_abs[batch], lev_norm[batch], lev_len[
                    batch
                ] = self.batch_levensthein_distance(
                    pred["collapse"][batch], Y["collapse"][batch]
                )
                timing[batch] = self.batch_timing_accuracy(
                    Y["timingCX"][batch], pred["timingCX"][batch]
                )
        return pred, Y, X, weights, loss, accuracy, lev_abs, lev_norm, lev_len, timing

    @tf.function
    def process(self, inp, targ=None, weights=None, appdetector_state=None, train=True):
        loss = 0
        batch_loss = 0

        with tf.GradientTape() as tape:
            enc_output, appdetector_state = self.appdetector(inp, appdetector_state)
            if targ is not None and weights is not None:
                loss = tf.reduce_sum(
                    tfa.seq2seq.sequence_loss(
                        enc_output,
                        targ,
                        weights,
                        average_across_timesteps=False,
                        average_across_batch=True,
                        sum_over_timesteps=False,
                        sum_over_batch=False,
                        softmax_loss_function=None,
                        name="Seq2SeqLoss",
                    )
                )
        if targ is not None:
            batch_loss = loss / int(targ.shape[1])

        if train:
            # Train with gradient clipping
            gradients = tape.gradient(loss, self.appdetector.trainable_variables)
            gradients_no_nans = [
                tf.where(tf.math.is_nan(x), tf.zeros_like(x), x) for x in gradients
            ]
            gradients, _ = tf.clip_by_global_norm(gradients, self.max_grad_norm)
            self.optimizer.apply_gradients(zip(gradients, self.appdetector.trainable_variables))

        predictions = tf.argmax(enc_output, 2, output_type=tf.int32)

        return batch_loss, predictions, appdetector_state

    def filter_batch(self, data, window_size=25):
        """Sliding window over the data. Majority voting is performed to replace outliers and reduce noise."""
        filtered = np.empty((data.shape[0], data.shape[1]))
        for iteration in range(data.shape[0]):
            filtered[iteration] = self.majority_voting(data[iteration], window_size)
        return filtered

    def majority_voting(self, data, window_size=25):
        if window_size == 0:
            out_data = data.copy()
        else:
            out_data = np.empty(data.shape)

            if len(data.shape) == 1:

                def majority_filter(idx):
                    out_data[idx] = st.mode(
                        data[
                            max([0, math.floor(idx - window_size / 2)]) : min(
                                [math.ceil(idx + window_size / 2), data.shape[0]]
                            )
                        ],
                        axis=0,
                    )[0]

            else:

                def majority_filter(idx):
                    out_data[idx, :] = st.mode(
                        data[
                            max([0, math.floor(idx - window_size / 2)]) : min(
                                [math.ceil(idx + window_size / 2), data.shape[0]]
                            ),
                            :,
                        ],
                        axis=0,
                    )[0]

            filtering_function = np.vectorize(majority_filter)
            filtering_function(np.arange(0, data.shape[0]))
        return out_data

    def get_sequences(self, labels, weights=None):
        sequences = [None for _ in range(labels.shape[0])]
        timings = [None for _ in range(labels.shape[0])]
        for iteration in range(labels.shape[0]):
            tf_labels_collapsed, tf_labels_change, tf_labels_masked = self.tf_collapse_labels(
                labels[iteration, :], weights[iteration, :]
            )
            sequences[iteration] = tf_labels_collapsed.numpy()
            timings[iteration] = tf_labels_change.numpy()
            if tf_labels_masked.numpy().size > 0:
                sequences[iteration] = np.concatenate(
                    [sequences[iteration], np.array([tf_labels_masked.numpy()[0]])]
                )
        return sequences, timings

    @tf.function
    def tf_collapse_labels(self, labels, weights=None):
        if weights is not None:
            labels_masked = tf.cast(
                tf.boolean_mask(labels, tf.greater(weights, tf.constant(0, dtype=tf.float32))),
                tf.int32,
            )
        else:
            labels_masked = tf.cast(labels, tf.int32)

        labels_diff = tf.subtract(labels_masked[:-1], labels_masked[1:])
        labels_change = tf.where(labels_diff != tf.constant(0, dtype=tf.int32))
        labels_collapsed = tf.boolean_mask(
            labels_masked, tf.not_equal(labels_diff, tf.constant(0, dtype=tf.int32))
        )

        return labels_collapsed, labels_change, labels_masked

    def batch_accuracy(self, predictions, target, weights=None):
        accuracy = np.empty((target.shape[0]))
        if weights is not None:
            for iteration in range(target.shape[0]):
                accuracy[iteration] = self.tf_accuracy(
                    predictions[weights > 0], target[weights > 0]
                )
        else:
            for iteration in range(target.shape[0]):
                accuracy[iteration] = self.tf_accuracy(predictions, target)
        return np.nanmean(accuracy) * 100

    @tf.function
    def tf_accuracy(self, predictions, target):
        return tf.divide(
            tf.size(
                tf.boolean_mask(
                    predictions,
                    tf.equal(tf.cast(predictions, tf.int32), tf.cast(target, tf.int32)),
                )
            ),
            tf.size(target),
        )

    def batch_levensthein_distance(self, predictions, target):
        lev_abs = np.empty((len(target)))
        lev_norm = np.empty((len(target)))
        lev_len = np.empty((len(target)))
        for iteration in range(len(target)):
            lev_abs[iteration] = editdistance.eval(predictions[iteration], target[iteration])
            lev_norm[iteration] = lev_abs[iteration] / target[iteration].size
            lev_len[iteration] = target[iteration].size
        return np.nansum(lev_abs), np.nanmean(lev_norm), np.nansum(lev_len)

    def batch_timing_accuracy(self, predictions, target):
        timings = np.empty((len(target)))
        for iteration in range(len(target)):
            timings[iteration], _ = FDTW(
                target[iteration], predictions[iteration], dist=euclidean
            )
        return timings.mean()

class AppDetector(tf.keras.Model):
    def __init__(self, *args, **kwargs):
        super(AppDetector, self).__init__()
        self._init_handler(*args, **kwargs)

    @property
    def oversampling_rate(self):
        return 1

    @abc.abstractmethod
    def _init_handler(self, *args, **kwargs):
        pass

    @abc.abstractmethod
    def call(self, x, state):
        pass

    @property
    def initial_hidden_state(self):
        return (
            [tf.zeros(self.state_size) for _ in range(self.num_layers)],
            [tf.zeros(self.state_size) for _ in range(self.num_layers)],
        )

    @property
    def state_size(self):
        return (self.batch_sz, self.enc_units)

class ADBiDiLSTM(AppDetector):
    pass

class ADConvLSTM(AppDetector):
    pass

class ADCNNLSTM(AppDetector):
    @property
    def oversampling_rate(self):
        return np.prod(self.kernel_size)

    def _init_handler(self, *args, **kwargs):
        self.batch_sz = kwargs["batch_size"]
        self.enc_units = kwargs["num_hidden"]
        self.num_layers = kwargs["num_layers"]
        self.num_labels = kwargs["num_labels"]
        self.max_timesteps = kwargs["max_timesteps"]

        self.num_cnn_layers = 1
        filters = [64]
        self.kernel_size = [25]  # Results in an output samping step of 1 Seconds

        self.cnn = [
            tf.keras.layers.Conv1D(
                filters=filters[layer_idx],
                kernel_size=self.kernel_size[layer_idx],
                activation="relu",
                input_shape=(
                    self.batch_sz,
                    int(self.max_timesteps / np.prod(self.kernel_size[: layer_idx + 1])),
                    None,
                ),
                padding="causal",
                kernel_initializer="he_uniform",
                data_format="channels_last",
            )
            for layer_idx in range(self.num_cnn_layers)
        ]
        self.pooling = [
            tf.keras.layers.MaxPool1D(
                pool_size=self.kernel_size[layer_idx],
                padding="same",
                data_format="channels_last",
                input_shape=(
                    self.batch_sz,
                    int(self.max_timesteps / np.prod(self.kernel_size[: layer_idx + 1])),
                    None,
                ),
            )
            for layer_idx in range(self.num_cnn_layers)
        ]

        self.lstm = [
            tf.keras.layers.LSTM(
                self.enc_units,
                activation="tanh",
                kernel_initializer="glorot_uniform",
                recurrent_initializer="glorot_uniform",
                bias_initializer="zeros",
                dropout=kwargs["p_dropout"],
                return_sequences=True,
                return_state=True,
                recurrent_dropout=0,
                go_backwards=False,
                stateful=True,
                unroll=False,
            )
            for _ in range(self.num_layers)
        ]
        self.dense = tf.keras.layers.TimeDistributed(
            tf.keras.layers.Dense(
                self.num_labels,
                activation="linear",
                kernel_initializer="glorot_uniform",
                bias_initializer="zeros",
            ),
            input_shape=(None, kwargs["max_timesteps"], self.enc_units),
        )

    def call(self, x, state):
        output = x
        for layer_idx in range(self.num_cnn_layers):
            output = self.cnn[layer_idx](output)
            output = self.pooling[layer_idx](output)
        output_state = (
            [None for _ in range(self.num_layers)],
            [None for _ in range(self.num_layers)],
        )
        for layer_idx in range(self.num_layers):
            output, output_state[0][layer_idx], output_state[1][layer_idx] = self.lstm[
                layer_idx
            ](output, initial_state=(state[0][layer_idx], state[1][layer_idx]))
        output = self.dense(output)
        return output, output_state

class ADLSTM(AppDetector):
    def _init_handler(self, *args, **kwargs):
        self.batch_sz = kwargs["batch_size"]
        self.enc_units = kwargs["num_hidden"]
        self.num_layers = kwargs["num_layers"]
        self.num_labels = kwargs["num_labels"]

        self.lstm = [
            tf.keras.layers.LSTM(
                self.enc_units,
                activation="tanh",
                kernel_initializer="glorot_uniform",
                recurrent_initializer="glorot_uniform",
                bias_initializer="zeros",
                dropout=kwargs["p_dropout"],
                return_sequences=True,
                return_state=True,
                recurrent_dropout=0,
                go_backwards=False,
                stateful=True,
                unroll=False,
            )
            for _ in range(self.num_layers)
        ]
        self.dense = tf.keras.layers.TimeDistributed(
            tf.keras.layers.Dense(
                self.num_labels,
                activation="linear",
                kernel_initializer="glorot_uniform",
                bias_initializer="zeros",
            ),
            input_shape=(None, kwargs["max_timesteps"], self.enc_units),
        )

    def call(self, x, state):
        output = x
        output_state = (
            [None for _ in range(self.num_layers)],
            [None for _ in range(self.num_layers)],
        )
        for layer_idx in range(self.num_layers):
            output, output_state[0][layer_idx], output_state[1][layer_idx] = self.lstm[
                layer_idx
            ](output, initial_state=(state[0][layer_idx], state[1][layer_idx]))
        output = self.dense(output)
        return output, output_state

reason: import numpy,import scipy,from scipy


text:
import numpy
from collections import deque

from zmq.backend.cython import context

from huffman import HuffmanTree
numpy.random.seed(12345)

class InputData:
    """Store data for word2vec, such as word map, huffman tree, sampling table and so on.

    Attributes:
        word_frequency: Count of each word, used for filtering low-frequency words and sampling table
        word2id: Map from word to word id, without low-frequency words.
        id2word: Map from word id to word, without low-frequency words.
        sentence_count: Sentence count in files.
        word_count: Word count in files, without low-frequency words.
    """

    def __init__(self, file_name, min_count):
        self.get_words(file_name, min_count)
        print(" ")
        self.cbow_count = []
        self.word_pair_catch = deque()
        self.cbow_word_pair_catch = deque()
        self.init_sample_table()
        tree = HuffmanTree(self.word_frequency)
        print("tree ", tree)
        self.huffman_positive, self.huffman_negative = tree.get_huffman_code_and_path()
        print('Word Count: %d' % len(self.word2id))
        print('Sentence Length: %d' % (self.sentence_length))

    def get_words(self, file_name, min_count):
        self.input_file_name = file_name
        # self.input_file = open(self.input_file_name)
        self.input_file = open(self.input_file_name, encoding="UTF-8")
        self.sentence_length = 0
        self.sentence_count = 0
        word_frequency = dict()
        for line in self.input_file:
            self.sentence_count += 1
            line = line.strip().split(' ')
            self.sentence_length += len(line)
            for w in line:
                try:
                    word_frequency[w] += 1
                except:
                    word_frequency[w] = 1
        self.word2id = dict()
        self.id2word = dict()
        wid = 0
        self.word_frequency = dict()
        for w, c in word_frequency.items():
            if c < min_count:
                self.sentence_length -= c
                continue
            self.word2id[w] = wid
            self.id2word[wid] = w
            self.word_frequency[wid] = c
            wid += 1
        self.word_count = len(self.word2id)

    def init_sample_table(self):
        self.sample_table = []
        sample_table_size = 1e8
        # pow_frequency = numpy.array(self.word_frequency.values())**0.75
        pow_frequency = numpy.array(list(self.word_frequency.values()))**0.75
        words_pow = sum(pow_frequency)
        ratio = pow_frequency / words_pow
        count = numpy.round(ratio * sample_table_size)
        for wid, c in enumerate(count):
            self.sample_table += [wid] * int(c)
        self.sample_table = numpy.array(self.sample_table)

    # @profile
    def get_batch_pairs(self, batch_size, window_size):
        while len(self.word_pair_catch) < batch_size:
            for _ in range(10000):
                sentence = self.input_file.readline()
                if sentence is None or sentence == '':
                    self.input_file = open(self.input_file_name, encoding="utf-8")
                    sentence = self.input_file.readline()
                word_ids = []
                for word in sentence.strip().split(' '):
                    try:
                        word_ids.append(self.word2id[word])
                    except:
                        continue
                for i, u in enumerate(word_ids):
                    for j, v in enumerate(word_ids[max(i - window_size, 0):i + window_size]):
                        assert u < self.word_count
                        assert v < self.word_count
                        if i == j:
                            continue
                        self.word_pair_catch.append((u, v))
        batch_pairs = []
        for _ in range(batch_size):
            batch_pairs.append(self.word_pair_catch.popleft())
        return batch_pairs

    def get_cbow_batch_all_pairs(self, batch_size, context_size):
        while len(self.cbow_word_pair_catch) < batch_size:
            for _ in range(10000):
                self.input_file = open(self.input_file_name, encoding="utf-8")
                sentence = self.input_file.readline()
                if sentence is None or sentence == '':
                    continue
                    # self.input_file = open(self.input_file_name, encoding="utf-8")
                    # sentence = self.input_file.readline()
                # if sentence is not None or sentence != "":
                word_ids = []
                for word in sentence.strip().split(' '):
                    try:
                        word_ids.append(self.word2id[word])
                    except:
                        continue
                # for i, u in enumerate(word_ids):
                #     con = []
                #     for j, v in enumerate(word_ids[max(i - window_size, 0):i + window_size]):
                #        assert u < self.word_count
                #        assert v < self.word_count
                #        if i == j:
                #            continue
                #        elif j >= max(0, i - window_size + 1) and j <= min(len(word_ids), i + window_size - 1):
                #            con.append(v)
                #     if len(con) == 0:
                #         continue
                #     self.cbow_word_pair_catch.append((con, u))

                # for i in range(2, len(word_ids) - 2):
                #     bow = ([word_ids[i - 2], word_ids[i - 1], word_ids[i + 1], word_ids[i + 2]], word_ids[i])
                #     self.cbow_word_pair_catch.append(bow)

                for i, u in enumerate(word_ids):
                    contentw = []
                    for j, v in enumerate(word_ids):
                        assert u < self.word_count
                        assert v < self.word_count
                        if i == j:
                            continue
                        # elif j >= max(0, i - self.args.window_size + 1) and j <= min(len(word_ids), i + self.args.window_size-1):
                        elif j >= max(0, i - context_size + 1) and j <= min(len(word_ids), i + context_size-1):
                            contentw.append(v)
                    if len(contentw) == 0:
                        continue
                    self.cbow_word_pair_catch.append((contentw, u))

                # for i in range(context_size, len(word_ids) - context_size):
                #     context = []
                #     for j in range(context_size, 0, -1):
                #         context.append(word_ids[i - j])
                #     for j in range(1, context_size + 1):
                #         context.append(word_ids[i + j])
                #     bow = (context, word_ids[i])
                #     self.cbow_word_pair_catch.append(bow)
        batch_pairs = []
        for _ in range(batch_size):
            batch_pairs.append(self.cbow_word_pair_catch.popleft())
        return batch_pairs

    def get_cbow_batch_pairs(self, batch_size, window_size):
        batch_pairs = []
        for _ in range(batch_size):
            batch_pairs.append(self.cbow_word_pair_catch.popleft())
        return batch_pairs

    # @profile
    def get_pairs_by_neg_sampling(self, pos_word_pair, count):
        neg_word_pair = []
        a = len(self.word2id) - 1
        for pair in pos_word_pair:
            i = 0
            neg_v = numpy.random.choice(self.sample_table, size=count)
            neg_word_pair += zip([pair[0]] * count, neg_v)
        return pos_word_pair, neg_word_pair

    def get_cbow_pairs_by_neg_sampling(self, pos_word_pair, count):
        neg_word_pair = []
        # print("get_cbow_pairs_by_neg_sampling", pos_word_pair)
        for pair in pos_word_pair:
            neg_v = numpy.random.choice(self.sample_table, size=count)
            neg_word_pair += zip([pair[0]] * count, neg_v)
        return pos_word_pair, neg_word_pair

    def get_pairs_by_huffman(self, pos_word_pair):
        neg_word_pair = []
        a = len(self.word2id) - 1
        for i in range(len(pos_word_pair)):
            pair = pos_word_pair[i]
            pos_word_pair += zip([pair[0]] *
                                 len(self.huffman_positive[pair[1]]),
                                 self.huffman_positive[pair[1]])
            neg_word_pair += zip([pair[0]] *
                                 len(self.huffman_negative[pair[1]]),
                                 self.huffman_negative[pair[1]])

        return pos_word_pair, neg_word_pair

    def get_cbow_pairs_by_huffman(self, pos_word_pair):
        neg_word_pair = []
        a = len(self.word2id) - 1
        for i in range(len(pos_word_pair)):
            pair = pos_word_pair[i]
            pos_word_pair += zip([pair[0]] *
                                 len(self.huffman_positive[pair[1]]),
                                 self.huffman_positive[pair[1]])
            neg_word_pair += zip([pair[0]] *
                                 len(self.huffman_negative[pair[1]]),
                                 self.huffman_negative[pair[1]])
        # print("hufman pos", pos_word_pair)
        # print("hufman nef", neg_word_pair)
        return pos_word_pair, neg_word_pair

    def evaluate_pair_count(self, window_size):
        return self.sentence_length * (2 * window_size - 1) - (
            self.sentence_count - 1) * (1 + window_size) * window_size

def test():
    a = InputData('./zhihu.txt')

if __name__ == '__main__':
    test()

ext: py

reason: import numpy


text:
import numpy as np
from scipy.stats import norm

from edge import Simulation
from edge.agent import SafetyLearner
from edge.envs import Hovership
from edge.model.safety_models import SafetyTruth, MaternSafety
from edge.graphics.plotter import SafetyPlotter, DetailedSafetyPlotter

class OptimisticSimulation(Simulation):
    def __init__(self, max_samples, gamma_optimistic, gamma_cautious,
                 lambda_cautious, shape, every):
        self.x_seed = np.array([1.45, 0.5])
        self.y_seed = np.array([.8])
        dynamics_parameters = {
            'shape': shape
        }
        self.env = Hovership(
            random_start=True,
            dynamics_parameters=dynamics_parameters,
            default_initial_state=self.x_seed[:1]
        )

        self.ground_truth = SafetyTruth(self.env)
        self.ground_truth.from_vibly_file(
            '../data/ground_truth/from_vibly/hover_map.pickle'
        )

        self.hyperparameters = {
            'outputscale_prior': (0.4, 2),
            'lengthscale_prior': (0.1, 0.1),
            'noise_prior': (0.001, 0.002)
        }
        self.agent = SafetyLearner(
            env=self.env,
            gamma_optimistic=gamma_optimistic,
            gamma_cautious=gamma_cautious,
            lambda_cautious=lambda_cautious,
            x_seed=self.x_seed,
            y_seed=self.y_seed,
            gp_params=self.hyperparameters,
        )
        plotters = {
            'DetailedSafety': DetailedSafetyPlotter(self.agent, self.ground_truth)
        }

        super(OptimisticSimulation, self).__init__(
            'results', 'optimistic', plotters
        )

        self.max_samples = max_samples
        self.every = every
        self.samples_path = self.output_directory / 'samples'
        self.samples_path.mkdir(parents=True, exist_ok=True)
        self.model_path = self.output_directory / 'model'
        self.model_path.mkdir(parents=True, exist_ok=True)

        failure_indexes = np.argwhere(self.ground_truth.failure_set == 1)
        self.failure_set = np.array([
            self.ground_truth.stateaction_space[tuple(index)]
            for index in failure_indexes[::3]
        ])

    def run_optim(self):
        train_x, train_y = self.ground_truth.get_training_examples(
            n_examples=2000,
            from_viable=True,
            from_failure=False
        )
        self.agent.fit_models(train_x, train_y, epochs=20)

    def save_samples(self, name):
        self.agent.safety_model.save_samples(str(self.samples_path / name))

    def load_samples(self, name):
        self.agent.safety_model.load_samples(str(self.samples_path / name))

    def save_model(self):
        self.agent.safety_model.save(str(self.model_path))

    def load_model(self):
        self.agent.safety_model = MaternSafety.load(str(self.model_path),
            self.env, self.agent.safety_model.gamma_measure,
            self.x_seed, self.y_seed
        )

    def check_failure_set(self):
        model = self.agent.safety_model

        measure_slice, covar_slice = model._query(
            self.failure_set, return_covar=True)
        level_value = norm.cdf(
                (measure_slice - 0) / np.sqrt(covar_slice)
            )
        failure_levels = level_value > model.gamma_measure

        if failure_levels.any():
            print('Nonzero value in the failure set !')

    def run_learning(self):
        n_samples = 0
        self.save_figs(prefix='0')
        while n_samples < self.max_samples:
            failed = self.agent.failed
            n_steps = 0
            while not failed and n_steps < 50:
                #self.check_failure_set()
                n_samples += 1
                n_steps += 1
                old_state = self.agent.state
                new_state, reward, failed, _ = self.agent.step()
                action = self.agent.last_action

                self.on_run_iteration(
                    n_samples,
                    old_state, action, new_state, reward, failed
                )

                if n_samples >= self.max_samples:
                    break

            reset_state = self.agent.get_random_safe_state()
            if reset_state is None:
                raise Exception('The whole measure is 0. There is no safe '
                                'action.')
            self.agent.reset(reset_state)

    def on_run_iteration(self, n_samples, old_state, action, new_state,
                         reward, failed):
        super(OptimisticSimulation, self).on_run_iteration(
            old_state, action, new_state, reward, failed
        )
        print(f'Step {n_samples}/{self.max_samples} - {old_state} '
              f' -> {action} -> {new_state} ({failed})')
        if n_samples % self.every == 0:
            self.save_figs(prefix=f'{n_samples}')

if __name__ == '__main__':
    np.random.seed(2)
    sim = OptimisticSimulation(
        max_samples=500,
        gamma_optimistic=0.52,
        gamma_cautious=0.8,
        lambda_cautious=0.0,
        shape=(201, 151),
        every=500
    )
    optimize = True
    load_hyper = False
    samples_to_load = None #'1500.npz'

    if optimize:
        print('optimizing hyperparameters...')
        sim.run_optim()
        print('saving hyperparameters...')
        sim.save_model()
    elif load_hyper:
        print('loading hyperparameters...')
        sim.load_model()
    else:
        print(f'keeping default hyperparameters: {sim.hyperparameters}... ')
    if samples_to_load is not None:
        print('loading samples...')
        sim.load_samples(samples_to_load)

    print('learning...')
    sim.run_learning()
    print('saving outputs...')
    sim.save_samples('last')
    sim.compile_gif()

reason: import numpy,from scipy


text:
# Advent of Code 2021
# Day 3: Part 1 and Part 2
# Author: Nico Van den Hooff

import numpy as np

def read_data(path):
    with open(path, "r") as f:
        data = f.read().splitlines()

    return data

def part_1(data):

    # split the binary digits into a 2D matrix
    digits = np.array([[int(d) for d in binary] for binary in data])

    # get gamma and epsilon based on digit counts in each row
    gamma = np.array(digits.sum(axis=0) > (len(data) / 2), dtype=int)
    epsilon = np.array(digits.sum(axis=0) <= (len(data) / 2), dtype=int)

    # join digits together
    gamma = "".join(gamma.astype(str).tolist())
    epsilon = "".join(epsilon.astype(str).tolist())

    # convert to decimal from binary and solve
    solution1 = int(gamma, 2) * int(epsilon, 2)

    return solution1

def get_rating(data, type):

    # vector of binary numbers
    binaries = np.array([int(d) for d in data]).reshape(-1, 1)

    # maxtrix of individual binary digits
    digits = np.array([[int(d) for d in binary] for binary in data])

    count = 0

    # continuing pruning digits until we are left with one
    while digits.shape[0] != 1:

        # current column to count 0s and 1s for
        column = digits[:, count].reshape(-1, 1)

        # based on the rating filter binary numbers that we want
        if type == "oxygen":
            if (np.sum(column == 1)) >= np.sum((column == 0)):
                digits = np.where(column == 1, digits, 0)
            else:
                digits = np.where(column == 0, digits, 0)

        # co2 rating swaps the two where clauses, the rest is the same
        else:
            if (np.sum(column == 1)) >= np.sum((column == 0)):
                digits = np.where(column == 0, digits, 0)
            else:
                digits = np.where(column == 1, digits, 0)

        # remove zero values
        digits = digits[~np.all(digits == 0, axis=1)]

        count += 1

    # get binary rating and convert to decimal
    rating = int("".join(digits.flatten().astype(str).tolist()), 2)

    return rating

def part_2(data):

    oxygen = get_rating(data, "oxygen")
    co2 = get_rating(data, "co2")
    solution2 = oxygen * co2

    return solution2

def main(path):
    data = read_data(path)
    solution1 = part_1(data)
    solution2 = part_2(data)

    print(f"Part 1 Solution: {solution1}")
    print(f"Part 2 Solution: {solution2}")

if __name__ == "__main__":
    path = "problems/day-3-binary-diagnostic/input.txt"
    main(path)


text:
#!/usr/bin/env python2

'''
Description:    Shared functionality:
                    * start_logging(filename, level=DEFAULT_LOG_LEVEL)
                    * progressbar(it, prefix="", size=60)
                    * is_number(s)
                    * wind_components(wind_speed, wind_direction)
                    * ismember(a, b)
Author:         Ronald van Haren, NLeSC (r.vanharen@esciencecenter.nl)
Created:
Last Modified:
License:        Apache 2.0
Notes:
'''

import logging
import sys
from numpy import sin as npsin
from numpy import cos as npcos
from numpy import radians as npradians
import csv
from math import radians, cos, sin, asin, sqrt

# define global LOG variables
DEFAULT_LOG_LEVEL = 'debug'
LOG_LEVELS = {'debug': logging.DEBUG,
              'info': logging.INFO,
              'warning': logging.WARNING,
              'error': logging.ERROR,
              'critical': logging.CRITICAL}
LOG_LEVELS_LIST = LOG_LEVELS.keys()
LOG_FORMAT = '%(asctime)s - %(levelname)s - %(message)s'
DATE_FORMAT = "%Y/%m/%d/%H:%M:%S"

def start_logging(filename, level=DEFAULT_LOG_LEVEL):
    "Start logging with given filename and level."
    try:
        logger
    except UnboundLocalError:
        logger = logging.getLogger()
    else:  # wish there was a logger.close()
        for handler in logger.handlers[:]:  # make a copy of the list
            logger.removeHandler(handler)
    logger.setLevel(LOG_LEVELS[level])
    formatter = logging.Formatter(LOG_FORMAT, datefmt=DATE_FORMAT)
    fh = logging.FileHandler(filename)
    fh.setFormatter(formatter)
    logger.addHandler(fh)
    return logger

def progressbar(it, prefix="", size=60):
    '''
    progressbar for a loop
    '''
    count = len(it)

    def _show(_i):
        x = int(size*_i/count)
        sys.stdout.write("%s[%s%s] %i/%i\r" % (prefix, "#"*x, "."*(size-x),
                                               _i, count))
        sys.stdout.flush()
    _show(0)
    for i, item in enumerate(it):
        yield item
        _show(i+1)
    sys.stdout.write("\n")
    sys.stdout.flush()

def is_number(s):
    '''
    check if the value in the string is a number and return True or False
    '''
    try:
        float(s)
        return True
    except ValueError:
        pass
    return False

def wind_components(wind_speed, wind_direction):
    '''
    return U and V wind components from wind speed and
    wind direction (in degrees)
    '''
    U = wind_speed * npsin(npradians(wind_direction)) * -1
    V = wind_speed * npcos(npradians(wind_direction)) * -1
    return U, V

def ismember(a, b):
    '''
    return items in a that are also in b
    '''
    bind = {}
    for i, elt in enumerate(b):
        if elt not in bind:
            bind[elt] = i
    # None can be replaced by any other "not in b" value
    return [bind.get(itm, None) for itm in a]

def write_csvfile(csvfile, data_out):
    # TODO: check if csv file exists already
    with open(csvfile, 'w') as fp:
        a = csv.writer(fp, delimiter=',')
        a.writerows(data_out)

def merge_two_dicts(x, y):
    '''
    Given two dicts, merge them into a new dict as a shallow copy.
    '''
    z = x.copy()
    z.update(y)
    return z

def fitem(item):
    try:
        item = item.strip()
    except AttributeError:
        pass
    try:
        item = float(item)
    except ValueError:
        pass
    return item

def progressbar(it, prefix="", size=60):
    '''
    progressbar for a loop
    '''
    count = len(it)

    def _show(_i):
        x = int(size*_i/count)
        sys.stdout.write("%s[%s%s] %i/%i\r" % (prefix, "#"*x, "."*(size-x),
                                               _i, count))
        sys.stdout.flush()
    _show(0)
    for i, item in enumerate(it):
        yield item
        _show(i+1)
    sys.stdout.write("\n")
    sys.stdout.flush()

def progressbar2(_i, count, prefix="", size=60):
    '''
    progressbar for a loop
    '''
    x = int(size*_i/count)
    sys.stdout.write("%s[%s%s] %i/%i\r" % (prefix, "#"*x, "."*(size-x),
                                            _i, count))
    sys.stdout.flush()

reason: from numpy


text:
__author__ = 'Richard Lincoln, r.w.lincoln@gmail.com'

""" Functions common to all experiments. """

import os
import sys
import logging

from numpy import zeros, array, c_, vectorize, copy
from scipy.io import mmwrite

import pylon

from pylon.util import pickle_matpower_cases

import pyreto.discrete
import pyreto.continuous

from pybrain.rl.learners.directsearch.directsearch import DirectSearchLearner
from pybrain.rl.learners.valuebased.valuebased import ValueBasedLearner

from pybrain.rl.agents import LearningAgent
from pybrain.rl.learners.valuebased import ActionValueTable
from pybrain.tools.shortcuts import buildNetwork
from pybrain.structure import TanhLayer, LinearLayer #@UnusedImport

def setup_logging():
    logger = logging.getLogger()
    for handler in logger.handlers:
        logger.removeHandler(handler) # rm pybrain
    logger.addHandler(logging.StreamHandler(sys.stdout))
    logger.setLevel(logging.DEBUG)

def get_case6ww():
    """ Returns the 6 bus case from Wood & Wollenberg PG&C.
    """
    path = os.path.dirname(pylon.__file__)
    path = os.path.join(path, "test", "data")
    path = os.path.join(path, "case6ww", "case6ww.pkl")

    case = pylon.Case.load(path)
    case.generators[0].p_cost = (0.0, 4.0, 200.0)
    case.generators[1].p_cost = (0.0, 3.0, 200.0)

#    case.generators[0].p_cost = (0.0, 5.1, 200.0) # 10%
#    case.generators[1].p_cost = (0.0, 4.5, 200.0) # 30%

    case.generators[2].p_cost = (0.0, 6.0, 200.0) # passive

#    case.generators[0].c_shutdown = 100.0
#    case.generators[1].c_shutdown = 100.0
#    case.generators[2].c_shutdown = 100.0

    case.generators[0].p_min = 0.0 # TODO: Unit-decommitment.
    case.generators[1].p_min = 0.0
    case.generators[2].p_min = 0.0

    case.generators[0].p_max = 110.0
    case.generators[1].p_max = 110.0
    case.generators[2].p_max = 220.0 # passive

    # FIXME: Correct generator naming order.
    for g in case.generators:
        g.name

    #pyreto.util.plotGenCost(case.generators)

    return case

def get_case6ww2():
    """ Returns the 6 bus case from Wood & Wollenberg PG&C.
    """
    path = os.path.dirname(pylon.__file__)
    path = os.path.join(path, "test", "data")
    path = os.path.join(path, "case6ww", "case6ww.pkl")

    case = pylon.Case.load(path)

    case.generators[0].p_cost = (0.0, 5.1, 200.0) # 10%
    case.generators[1].p_cost = (0.0, 4.5, 200.0) # 30%
    case.generators[2].p_cost = (0.0, 6.0, 200.0) # passive

    case.generators[0].c_shutdown = 100.0
    case.generators[1].c_shutdown = 100.0
    case.generators[2].c_shutdown = 100.0

    case.generators[0].p_min = 0.0 # TODO: Unit-decommitment.
    case.generators[1].p_min = 0.0
    case.generators[2].p_min = 0.0

    case.generators[0].p_max = 110.0
    case.generators[1].p_max = 110.0
    case.generators[2].p_max = 220.0 # passive

    # FIXME: Correct generator naming order.
    for g in case.generators:
        g.name

    #pyreto.util.plotGenCost(case.generators)

    return case

def get_case24_ieee_rts():
    """ Returns the 24 bus IEEE Reliability Test System.
    """
    path = os.path.dirname(pylon.__file__)
    path = os.path.join(path, "test", "data")
    path = os.path.join(path, "case24_ieee_rts", "case24_ieee_rts.pkl")

    case = pylon.Case.load(path)

    # FIXME: Correct generator naming order.
    for g in case.generators:
        g.name

    return case

def get_case24_ieee_rts2():
    """ Returns the 24 bus IEEE Reliability Test System with U12 and U20
    generators removed.
    """
    path = os.path.dirname(__file__)
    path = os.path.join(path, 'data', "case24_ieee_rts2.pkl")

    case = pylon.Case.load(path)

    # FIXME: Correct generator naming order.
    for g in case.generators:
        g.name

    return case

def get_case24_ieee_rts3():
    """ Returns the 24 bus IEEE Reliability Test System with U12 and U20
    generators removed and generators of the same type at the same bus
    aggregated.
    """
    path = os.path.dirname(__file__)
    path = os.path.join(path, 'data', "case24_ieee_rts3.pkl")

    case = pylon.Case.load(path)

    # FIXME: Correct generator naming order.
    for g in case.generators:
        g.name

        g.p = 0.0

    return case

def pickle_cases():
    data_dir = os.path.dirname(__file__)
    case_paths = [os.path.join(data_dir, 'data', 'case24_ieee_rts2.m'),
                  os.path.join(data_dir, 'data', 'case24_ieee_rts3.m')]

    pickle_matpower_cases(case_paths)

def get_discrete_task_agent(generators, market, nStates, nOffer, markups,
        withholds, maxSteps, learner, Pd0=None, Pd_min=0.0):
    """ Returns a tuple of task and agent for the given learner.
    """
    env = pyreto.discrete.MarketEnvironment(generators, market,
                                            numStates=nStates,
                                            numOffbids=nOffer,
                                            markups=markups,
                                            withholds=withholds,
                                            Pd0=Pd0,
                                            Pd_min=Pd_min)
    task = pyreto.discrete.ProfitTask(env, maxSteps=maxSteps)

    nActions = len(env._allActions)
    module = ActionValueTable(numStates=nStates, numActions=nActions)

    agent = LearningAgent(module, learner)

    return task, agent

def get_continuous_task_agent(generators, market, nOffer, maxMarkup,
        maxWithhold, maxSteps, learner):
    env = pyreto.continuous.MarketEnvironment(generators, market, nOffer,
                                              maxMarkup, maxWithhold)

    task = pyreto.continuous.ProfitTask(env, maxSteps=maxSteps)

    net = buildNetwork(env.outdim,
#                       4,
                       env.indim,
                       bias=False,
#                       outputbias=True,
#                       hiddenclass=TanhLayer,
#                       outclass=TanhLayer
                       )

#    net._setParameters(([0.0]))

    agent = LearningAgent(net, learner)
#    agent.name = generators[0].name

    return task, agent

def get_zero_task_agent(generators, market, nOffer, maxSteps):
    """ Returns a task-agent tuple whose action is always zero.
    """
    env = pyreto.discrete.MarketEnvironment(generators, market, nOffer)
    task = pyreto.discrete.ProfitTask(env, maxSteps=maxSteps)
    agent = pyreto.util.ZeroAgent(env.outdim, env.indim)
    return task, agent

def get_neg_one_task_agent(generators, market, nOffer, maxSteps):
    """ Returns a task-agent tuple whose action is always minus one.
    """
    env = pyreto.discrete.MarketEnvironment(generators, market, nOffer)
    task = pyreto.discrete.ProfitTask(env, maxSteps=maxSteps)
    agent = pyreto.util.NegOneAgent(env.outdim, env.indim)
    return task, agent

def get_pd_min(case, profile):
    l = min(profile)
    Pd_min = sum([b.p_demand * l for b in case.buses])
    return Pd_min

def get_pd_max(case, profile):
    u = max(profile)
    Pd_max = sum([b.p_demand * u for b in case.buses])
    return Pd_max

def run_experiment(experiment, roleouts, episodes, in_cloud=False,
                   dynProfile=None):
    """ Runs the given experiment and returns the results.
    """
    def run():
        if dynProfile is None:
            maxsteps = len(experiment.profile) # episode length
        else:
            maxsteps = dynProfile.shape[1]
        na = len(experiment.agents)
        ni = roleouts * episodes * maxsteps

        all_action = zeros((na, 0))
        all_reward = zeros((na, 0))
        epsilon = zeros((na, ni)) # exploration rate

        # Converts to action vector in percentage markup values.
        vmarkup = vectorize(get_markup)

        for roleout in range(roleouts):
            if dynProfile is not None:
                # Apply new load profile before each roleout (week).
                i = roleout * episodes # index of first profile value
                experiment.profile = dynProfile[i:i + episodes, :]

#            print "PROFILE:", experiment.profile, episodes

            experiment.doEpisodes(episodes) # number of samples per learning step

            nei = episodes * maxsteps # num interactions per role
            epi_action = zeros((0, nei))
            epi_reward = zeros((0, nei))

            for i, (task, agent) in \
            enumerate(zip(experiment.tasks, experiment.agents)):
                action = copy(agent.history["action"])
                reward = copy(agent.history["reward"])

                for j in range(nei):
                    if isinstance(agent.learner, DirectSearchLearner):
                        action[j, :] = task.denormalize(action[j, :])
                        k = nei * roleout
                        epsilon[i, k:k + nei] = agent.learner.explorer.sigma[0]
                    elif isinstance(agent.learner, ValueBasedLearner):
                        action[j, :] = vmarkup(action[j, :], task)
                        k = nei * roleout
                        epsilon[i, k:k + nei] = agent.learner.explorer.epsilon
                    else:
                        action = vmarkup(action, task)

                # FIXME: Only stores action[0] for all interactions.
                epi_action = c_[epi_action.T, action[:, 0].flatten()].T
                epi_reward = c_[epi_reward.T, reward.flatten()].T

                if hasattr(agent, "module"):
                    print "PARAMS:", agent.module.params

                agent.learn()
                agent.reset()

            all_action = c_[all_action, epi_action]
            all_reward = c_[all_reward, epi_reward]

        return all_action, all_reward, epsilon

    if in_cloud:
        import cloud
        job_id = cloud.call(run, _high_cpu=False)
        result = cloud.result(job_id)
        all_action, all_reward, epsilon = result
    else:
        all_action, all_reward, epsilon = run()

    return all_action, all_reward, epsilon

def get_markup(a, task):
    i = int(a)
    m = task.env._allActions[i]
    return m[0]

def save_results(results, name, version="1_1"):

    expt_action_mean, expt_action_std, \
        expt_reward_mean, expt_reward_std, epsilon = results

    if expt_action_mean is not None:
        mmwrite("./out/ex%s_%s_action_mean.mtx" % (version, name.lower()),
                expt_action_mean,
                "Experiment %s %s actions mean." % (version, name))

    if expt_action_std is not None:
        mmwrite("./out/ex%s_%s_action_std.mtx" % (version, name.lower()),
                expt_action_std,
                "Experiment %s %s actions SD." % (version, name))

    if expt_reward_mean is not None:
        mmwrite("./out/ex%s_%s_reward_mean.mtx" % (version, name.lower()),
                expt_reward_mean,
                "Experiment %s %s rewards mean." % (version, name))

    if expt_reward_std is not None:
        mmwrite("./out/ex%s_%s_reward_std.mtx" % (version, name.lower()),
                expt_reward_std,
                "Experiment %s %s rewards SD." % (version, name))

    if epsilon is not None:
        mmwrite("./out/ex%s_%s_epsilon.mtx" % (version, name.lower()),
                epsilon,
                "Experiment %s %s exploration rates." % (version, name))

def save_rewards(rewards, name, version):

    mmwrite("./out/ex%s_%s_all_rewards.mtx" % (version, name.lower()),
            rewards,
            "Experiment %s %s all rewards." % (version, name))

def get_weekly():
    """ Returns the percent of annual peak for eack week of a year starting the
    first week of January.  Data from the IEEE RTS.
    """
    weekly = array([
        86.2, 90.0, 87.8, 83.4, 88.0, 84.1, 83.2, 80.6, 74.0, 73.7, 71.5,
        72.7, 75.0, 72.1, 80.0, 70.4, 87.0, 88.0, 75.4, 83.7, 85.6, 81.1,
        90.0, 88.7, 89.6, 86.1, 75.5, 81.6, 80.1, 88.0, 72.2, 80.0, 72.9,
        77.6, 72.6, 70.5, 78.0, 69.5, 72.4, 72.4, 74.3, 74.4, 80.0, 88.1,
        88.5, 90.9, 94.0, 89.0, 94.2, 97.0, 100.0, 95.2
    ])
    return weekly

def get_daily():
    """ Retruns the percent of weekly peak. Week beginning Monday.
    """
    daily = array([93, 100, 98, 96, 94, 77, 75])
    return daily

def get_winter_hourly():
    """ Return the percentage of daily peak, starting at midnight.
    Weeks 1-8 and 44-52.
    """
    hourly_winter_wkdy = array([
        67, 63, 60, 59, 59, 60, 74, 86, 95, 96, 96, 95, 95,
        95, 93, 94, 99, 100, 100, 96, 91, 83, 73, 63
    ])
    hourly_winter_wknd = array([
        78, 72, 68, 66, 64, 65, 66, 70, 80, 88, 90, 91, 90,
        88, 87, 87, 91, 100, 99, 97, 94, 92, 87, 81
    ])
    return hourly_winter_wkdy, hourly_winter_wknd

def get_summer_hourly():
    """ Return the percentage of daily peak, starting at midnight. Weeks 18-30.
    """
    hourly_summer_wkdy = array([
        64, 60, 58, 56, 56, 58, 64, 76, 87, 95, 99, 100, 99,
        100, 100, 97, 96, 96, 93, 92, 92, 93, 87, 72
    ])
    hourly_summer_wknd = array([
        74, 70, 66, 65, 64, 62, 62, 66, 81, 86, 91, 93, 93,
        92, 91, 91, 92, 94, 95, 95, 100, 93, 88, 80
    ])
    return hourly_summer_wkdy, hourly_summer_wknd

def get_spring_autumn_hourly():
    """ Return the percentage of daily peak, starting at midnight.
    Weeks 9-17 and 31-43.
    """
    hourly_spring_autumn_wkdy = array([
        63, 62, 60, 58, 59, 65, 72, 85, 95, 99, 100,
        99, 93, 92, 90, 88, 90, 92, 96, 98, 96, 90, 80, 70
    ])
    hourly_spring_autumn_wknd = array([
        75, 73, 69, 66, 65, 65, 68, 74, 83, 89, 92,
        94, 91, 90, 90, 86, 85, 88, 92, 100, 97, 95, 90, 85
    ])
    return hourly_spring_autumn_wkdy, hourly_spring_autumn_wknd

def get_full_year():
    """ Returns percentages of peak load for all hours of the year.

    @return:
        Numpy array of doubles with length 8736.
    """
    weekly = get_weekly()
    daily = get_daily()
    hourly_winter_wkdy, hourly_winter_wknd = get_winter_hourly()
    hourly_summer_wkdy, hourly_summer_wknd = get_summer_hourly()
    hourly_spring_autumn_wkdy, hourly_spring_autumn_wknd = \
        get_spring_autumn_hourly()

    fullyear = zeros(364 * 24)
    c = 0
    l = [(0, 7, hourly_winter_wkdy, hourly_winter_wknd),
         (8, 16, hourly_spring_autumn_wkdy, hourly_spring_autumn_wknd),
         (17, 29, hourly_summer_wkdy, hourly_summer_wknd),
         (30, 42, hourly_spring_autumn_wkdy, hourly_spring_autumn_wknd),
         (43, 51, hourly_winter_wkdy, hourly_winter_wknd)]

    for start, end, wkdy, wknd in l:
        for w in weekly[start:end + 1]:
            for d in daily[:5]:
                for h in wkdy:
                    fullyear[c] = w * (d / 100.0) * (h / 100.0)
                    c += 1
            for d in daily[5:]:
                for h in wknd:
                    fullyear[c] = w * (d / 100.0) * (h / 100.0)
                    c += 1
    return fullyear

def get_all_days():
    """ Returns percentages of peak load for all days of the year.
    Data from the IEEE RTS.
    """
    weekly = get_weekly()
    daily = get_daily()

    return [w * (d / 100.0) for w in weekly for d in daily]

def get_outages():
    # Outage rate (outages/year).
    rate = [0.24, 0.51, 0.33, 0.39, 0.48, 0.38, 0.02, 0.36, 0.34, 0.33, 0.3,
            0.44, 0.44, 0.02, 0.02, 0.02, 0.02, 0.4, 0.39, 0.4, 0.52,
            0.49, 0.38, 0.33, 0.41, 0.41, 0.41, 0.35, 0.34, 0.32, 0.54,
            0.35, 0.35, 0.38, 0.38, 0.34, 0.34, 0.45]

    per = 365
    outage_rate = [r / per for r in rate]
    return outage_rate

max_stars_repo_name: rwl/pylon

max_issues_repo_name: rwl/pylon

max_forks_repo_name: rwl/pylon

reason: from numpy,from scipy