text:
# ------------------------------------------
# LiNGAM Likelihood Ratios
# ------------------------------------------
#
#
#
# Hyvarinen & Smith, JMLR, 2013

import numpy as np
from scipy.stats import kurtosis
from sklearn.preprocessing import scale

def DiffEntropyHyv1998(x):
    """
    approximate the differential entropy as described in Hyvarinen (1998)
    """

    # define constants
    K1 = 79.047
    K2 = 7.4129
    gamma = .37457

    diffE = -1 * K1 * (np.log(np.cosh(x)).mean(0) - gamma) ** 2
    diffE -= K2 * (x * np.exp(-0.5 * x * x)).mean(0) ** 2

    return np.sum(1 * diffE)

def DiffEntropyKraskov2004(x):
    """
    approximate the differential entropy using equation (4) of Kraskov et al (2004)

    we note there are additive constants which we ignore as we are only interested in likelihood ratios !
    """

    x = scale(x)
    x = np.unique(x)  # computing log of differences, hence need to remove repeated values (other computing log 0)
    N = len(x)

    x = np.sort(x)

    return np.mean(np.log(N * np.diff(x)))

def DiffEntropyWG1999(x, m=1):
    """
    approximate the differential entropy using equation (3) of Wieczorkowski and Grzegorzewski (1999)

    """

    x = scale(x)
    x = np.unique(x)
    N = len(x)

    x = np.sort(x)

    return (1. / (N - m)) * np.sum(np.log(((N + 1.) / m) * (x[m:] - x[:(len(x) - m)])))

def cumulant_hyv13_ratio(x, y):
    """
    determine causal direction based on cumulants as described in
    section 2.6 of Hyvarinen & Smith (2013)
    """
    rho = np.corrcoef(x, y)[0, 1]  # estimate correlation coef

    kurtx = kurtosis(x)

    R = np.sign(kurtx) * rho * (np.mean(np.power(x, 3) * y - x * np.power(y, 3)))

    predictCausalDir = 'x->y' if R > 0 else 'y->x'
    return R, predictCausalDir

class EntropyLR:
    """
    determine causal direction based on LR as described in Hyvarinen & Smith (2013)
    """

    def __init__(self, entropy='Hyv98', normalize=True):
        assert entropy in ['Hyv98', 'Kraskov04', 'WG98']
        self.entropy = entropy
        self.normalize = normalize

    def predict_proba(self, data):
        d = data.shape[1] // 2
        if d > 1:
            # only Hyv98 entropy supports high D variables for now
            assert self.entropy == 'Hyv98'
        x, y = data[:, :d], data[:, d:]
        # first scale
        x = scale(x)
        y = scale(y)
        # estimate correlation coef
        rho = np.corrcoef(x, y, rowvar=False)[:d, d:]

        functionDict = {'Hyv98': DiffEntropyHyv1998,
                        'Kraskov04': DiffEntropyKraskov2004,
                        'WG98': DiffEntropyWG1999}

        entFunc = functionDict[self.entropy]

        # compute LR of x->y
        resid = y - np.dot(rho, x.T).T
        if self.normalize:
            resid /= np.std(resid, axis=0)
        L_xy = -1 * entFunc(x) - entFunc(resid)

        # compute LR of x<-y
        resid = x - np.dot(rho, y.T).T
        if self.normalize:
            resid /= np.std(resid, axis=0)
        L_yx = -1 * entFunc(y) - entFunc(resid)

        LR = (L_xy - L_yx)
        causal_dir = 'x->y' if LR > 0 else 'y->x'

        return LR, causal_dir

max_stars_repo_name: mrateike/carefl

max_issues_repo_name: mrateike/carefl

max_forks_repo_name: mrateike/carefl


text:
import collections
import datetime
import logging
from typing import Any, Callable, Dict, List, Optional, Tuple, Union

import numpy as np
import pandas as pd
import scipy as sp
import sklearn as sklear

import core.config as cconfig
import core.data_adapters as cdataa
import core.dataflow.utils as cdu
import core.finance as cfinan
import core.signal_processing as csigna
import core.statistics as cstati
import helpers.dbg as dbg
from core.dataflow.core import DAG, Node
from core.dataflow.nodes.base import (
    ColModeMixin,
    FitPredictNode,
    SeriesToDfColProcessor,
)
from core.dataflow.nodes.sources import ReadDataFromDf
from core.dataflow.nodes.transformers import ColumnTransformer
from core.dataflow.visitors import extract_info

_LOG = logging.getLogger(__name__)

_COL_TYPE = Union[int, str]
_PANDAS_DATE_TYPE = Union[str, pd.Timestamp, datetime.datetime]
_TO_LIST_MIXIN_TYPE = Union[List[_COL_TYPE], Callable[[], List[_COL_TYPE]]]

class SmaModel(FitPredictNode, ColModeMixin):
    """
    Fit and predict a smooth moving average (SMA) model.
    """

    def __init__(
        self,
        nid: str,
        col: _TO_LIST_MIXIN_TYPE,
        steps_ahead: int,
        tau: Optional[float] = None,
        min_tau_periods: Optional[float] = 2,
        col_mode: Optional[str] = None,
        nan_mode: Optional[str] = None,
    ) -> None:
        """
        Specify the data and SMA modeling parameters.

        :param nid: unique node id
        :param col: name of column to model
        :param steps_ahead: as in `ContinuousSkLearnModel`
        :param tau: as in `csigna.compute_smooth_moving_average`. If `None`,
            learn this parameter. Will be re-learned on each `fit` call.
        :param min_tau_periods: similar to `min_periods` as in
            `csigna.compute_smooth_moving_average`, but expressed in units of
            tau
        :param col_mode: `merge_all` or `replace_all`, as in `ColumnTransformer()`
        :param nan_mode: as in `ContinuousSkLearnModel`
        """
        super().__init__(nid)
        self._col = cdu.convert_to_list(col)
        dbg.dassert_eq(len(self._col), 1)
        self._steps_ahead = steps_ahead
        dbg.dassert_lte(
            0, self._steps_ahead, "Non-causal prediction attempted! Aborting..."
        )
        if nan_mode is None:
            self._nan_mode = "raise"
        else:
            self._nan_mode = nan_mode
        self._col_mode = col_mode or "replace_all"
        dbg.dassert_in(self._col_mode, ["replace_all", "merge_all"])
        # Smooth moving average model parameters to learn.
        self._must_learn_tau = tau is None
        self._tau = tau
        self._min_tau_periods = min_tau_periods or 0
        self._min_depth = 1
        self._max_depth = 1
        self._metric = sklear.metrics.mean_absolute_error

    def fit(self, df_in: pd.DataFrame) -> Dict[str, pd.DataFrame]:
        idx = df_in.index[: -self._steps_ahead]
        x_vars = self._col
        y_vars = self._col
        df = cdu.get_x_and_forward_y_fit_df(
            df_in, x_vars, y_vars, self._steps_ahead
        )
        forward_y_cols = df.drop(x_vars, axis=1).columns
        # Handle presence of NaNs according to `nan_mode`.
        self._handle_nans(idx, df.index)
        # Define and fit model.
        if self._must_learn_tau:
            forward_y_df = df[forward_y_cols]
            # Prepare forward y_vars in sklearn format.
            forward_y_fit = cdataa.transform_to_sklearn(
                forward_y_df, forward_y_df.columns.tolist()
            )
            # Prepare `x_vars` in sklearn format.
            x_fit = cdataa.transform_to_sklearn(df, self._col)
            self._tau = self._learn_tau(x_fit, forward_y_fit)
        _LOG.debug("tau=%s", self._tau)
        return self._predict_and_package_results(df_in, idx, df.index, fit=True)

    def predict(self, df_in: pd.DataFrame) -> Dict[str, pd.DataFrame]:
        cdu.validate_df_indices(df_in)
        df = df_in.copy()
        idx = df.index
        # Restrict to times where col has no NaNs.
        non_nan_idx = df.loc[idx][self._col].dropna().index
        # Handle presence of NaNs according to `nan_mode`.
        self._handle_nans(idx, non_nan_idx)
        # Use trained model to generate predictions.
        dbg.dassert_is_not(
            self._tau,
            None,
            "Parameter tau not found! Check if `fit` has been run.",
        )
        return self._predict_and_package_results(
            df_in, idx, non_nan_idx, fit=False
        )

    def get_fit_state(self) -> Dict[str, Any]:
        fit_state = {"_tau": self._tau, "_info['fit']": self._info["fit"]}
        return fit_state

    def set_fit_state(self, fit_state: Dict[str, Any]) -> None:
        self._tau = fit_state["_tau"]
        self._info["fit"] = fit_state["_info['fit']"]

    def _predict_and_package_results(
        self,
        df_in: pd.DataFrame,
        idx: pd.Index,
        non_nan_idx: pd.Index,
        fit: bool = True,
    ) -> Dict[str, pd.DataFrame]:
        data = cdataa.transform_to_sklearn(df_in.loc[non_nan_idx], self._col)
        fwd_y_hat = self._predict(data)
        forward_y_df = cdu.get_forward_cols(df_in, self._col, self._steps_ahead)
        forward_y_df = forward_y_df.loc[non_nan_idx]
        # Put predictions in dataflow dataframe format.
        fwd_y_hat_vars = [f"{y}_hat" for y in forward_y_df.columns]
        fwd_y_hat = cdataa.transform_from_sklearn(
            non_nan_idx, fwd_y_hat_vars, fwd_y_hat
        )
        # Return targets and predictions.
        df_out = forward_y_df.reindex(idx).merge(
            fwd_y_hat.reindex(idx), left_index=True, right_index=True
        )
        dbg.dassert_no_duplicates(df_out.columns)
        # Select columns for output.
        df_out = self._apply_col_mode(
            df_in, df_out, cols=self._col, col_mode=self._col_mode
        )
        # Update `info`.
        info = collections.OrderedDict()
        info["tau"] = self._tau
        info["min_periods"] = self._get_min_periods(self._tau)
        info["df_out_info"] = cdu.get_df_info_as_string(df_out)
        method = "fit" if fit else "predict"
        self._set_info(method, info)
        return {"df_out": df_out}

    def _handle_nans(
        self, idx: pd.DataFrame.index, non_nan_idx: pd.DataFrame.index
    ) -> None:
        if self._nan_mode == "raise":
            if idx.shape[0] != non_nan_idx.shape[0]:
                nan_idx = idx.difference(non_nan_idx)
                raise ValueError(f"NaNs detected at {nan_idx}")
        elif self._nan_mode == "drop":
            pass
        elif self._nan_mode == "leave_unchanged":
            pass
        else:
            raise ValueError(f"Unrecognized nan_mode `{self._nan_mode}`")

    def _learn_tau(self, x: np.array, y: np.array) -> float:
        def score(tau: float) -> float:
            x_srs = pd.DataFrame(x.flatten())
            sma = csigna.compute_smooth_moving_average(
                x_srs,
                tau=tau,
                min_periods=0,
                min_depth=self._min_depth,
                max_depth=self._max_depth,
            )
            min_periods = self._get_min_periods(tau)
            return self._metric(sma[min_periods:], y[min_periods:])

        tau_lb, tau_ub = 1, 1000
        # Satisfy 2 * tau_ub * min_tau_periods = len(x).
        # This ensures that no more than half of the `fit` series is burned.
        if self._min_tau_periods > 0:
            tau_ub = int(len(x) / (2 * self._min_tau_periods))
        opt_results = sp.optimize.minimize_scalar(
            score, method="bounded", bounds=[tau_lb, tau_ub]
        )
        return opt_results.x

    def _get_min_periods(self, tau: float) -> int:
        """
        Return burn-in period.

        Multiplies `tau` by `min_tau_periods` and converts to an integer.

        :param tau: kernel tau (approximately equal to center of mass)
        :return: minimum number of periods required to generate a prediction
        """
        return int(np.rint(self._min_tau_periods * tau))

    def _predict(self, x: np.array) -> np.array:
        x_srs = pd.DataFrame(x.flatten())
        # TODO(*): Make `min_periods` configurable.
        min_periods = int(np.rint(self._min_tau_periods * self._tau))
        _LOG.debug("min_periods=%f", min_periods)
        x_sma = csigna.compute_smooth_moving_average(
            x_srs,
            tau=self._tau,
            min_periods=min_periods,
            min_depth=self._min_depth,
            max_depth=self._max_depth,
        )
        return x_sma.values

class SingleColumnVolatilityModel(FitPredictNode):
    def __init__(
        self,
        nid: str,
        steps_ahead: int,
        col: _COL_TYPE,
        p_moment: float = 2,
        progress_bar: bool = False,
        tau: Optional[float] = None,
        nan_mode: Optional[str] = None,
        out_col_prefix: Optional[str] = None,
    ) -> None:
        """
        Parameters have the same meaning as `SmaModel`.
        """
        super().__init__(nid)
        self._col = col
        self._steps_ahead = steps_ahead
        dbg.dassert_lte(1, p_moment)
        self._p_moment = p_moment
        self._progress_bar = progress_bar
        self._tau = tau
        self._learn_tau_on_fit = tau is None
        self._nan_mode = nan_mode
        self._out_col_prefix = out_col_prefix

    def get_fit_state(self) -> Dict[str, Any]:
        fit_state = {
            "_col": self._col,
            "_tau": self._tau,
            "_info['fit']": self._info["fit"],
            "_out_col_prefix": self._out_col_prefix,
        }
        return fit_state

    def set_fit_state(self, fit_state: Dict[str, Any]):
        self._col = fit_state["_col"]
        self._tau = fit_state["_tau"]
        self._info["fit"] = fit_state["_info['fit']"]
        self._out_col_prefix = fit_state["_out_col_prefix"]

    def fit(self, df_in: pd.DataFrame) -> Dict[str, pd.DataFrame]:
        return {"df_out": self._fit_predict_helper(df_in, fit=True)}

    def predict(self, df_in: pd.DataFrame) -> Dict[str, pd.DataFrame]:
        return {"df_out": self._fit_predict_helper(df_in, fit=False)}

    def _fit_predict_helper(self, df_in: pd.DataFrame, fit: bool) -> pd.DataFrame:
        info = collections.OrderedDict()
        name = self._out_col_prefix or self._col
        name = str(name)
        dbg.dassert_not_in(name + "_vol", df_in.columns)
        if self._learn_tau_on_fit and fit:
            tau = None
        else:
            tau = self._tau
        config = self._get_config(col=self._col, out_col_prefix=name, tau=tau)
        dag = self._get_dag(df_in[[self._col]], config)
        mode = "fit" if fit else "predict"
        df_out = dag.run_leq_node(
            "demodulate_using_vol_pred", mode, progress_bar=self._progress_bar
        )["df_out"]
        info[self._col] = extract_info(dag, [mode])
        if self._learn_tau_on_fit and fit:
            self._tau = info[self._col]["compute_smooth_moving_average"]["fit"][
                "tau"
            ]
        df_out = df_out.reindex(df_in.index)
        self._set_info(mode, info)
        return df_out

    def _get_config(
        self,
        col: _COL_TYPE,
        out_col_prefix: _COL_TYPE,
        tau: Optional[float] = None,
    ) -> cconfig.Config:
        """
        Generate a DAG config.

        :param col: column whose volatility is to be modeled
        :param tau: tau for SMA; if `None`, then to be learned
        :return: a complete config to be used with `_get_dag()`
        """
        config = cconfig.get_config_from_nested_dict(
            {
                "calculate_vol_pth_power": {
                    "cols": [col],
                    "col_rename_func": lambda x: out_col_prefix + "_vol",
                    "col_mode": "merge_all",
                },
                "compute_smooth_moving_average": {
                    "col": [out_col_prefix + "_vol"],
                    "steps_ahead": self._steps_ahead,
                    "tau": tau,
                    "col_mode": "merge_all",
                    "nan_mode": self._nan_mode,
                },
                "calculate_vol_pth_root": {
                    "cols": [
                        out_col_prefix + "_vol",
                        out_col_prefix + "_vol_" + str(self._steps_ahead),
                        out_col_prefix
                        + "_vol_"
                        + str(self._steps_ahead)
                        + "_hat",
                    ],
                    "col_mode": "replace_selected",
                },
                "demodulate_using_vol_pred": {
                    "signal_cols": [col],
                    "volatility_col": out_col_prefix
                    + "_vol_"
                    + str(self._steps_ahead)
                    + "_hat",
                    "signal_steps_ahead": 0,
                    "volatility_steps_ahead": self._steps_ahead,
                    "col_rename_func": lambda x: out_col_prefix + "_vol_adj",
                    "col_mode": "replace_selected",
                    "nan_mode": self._nan_mode,
                },
            }
        )
        return config

    def _get_dag(self, df_in: pd.DataFrame, config: cconfig.Config) -> DAG:
        """
        Build a DAG from data and config.

        :param df_in: data over which to run DAG
        :param config: config for configuring DAG nodes
        :return: ready-to-run DAG
        """
        dag = DAG(mode="strict")
        _LOG.debug("%s", config)
        # Load `df_in`.
        nid = "load_data"
        node = ReadDataFromDf(nid, df_in)
        tail_nid = self._append(dag, None, node)
        # Raise volatility columns to pth power.
        nid = "calculate_vol_pth_power"
        node = ColumnTransformer(
            nid,
            transformer_func=lambda x: np.abs(x) ** self._p_moment,
            **config[nid].to_dict(),
        )
        tail_nid = self._append(dag, tail_nid, node)
        # Predict pth power of volatility using smooth moving average.
        nid = "compute_smooth_moving_average"
        node = SmaModel(nid, **config[nid].to_dict())
        tail_nid = self._append(dag, tail_nid, node)
        # Calculate the pth root of volatility columns.
        nid = "calculate_vol_pth_root"
        node = ColumnTransformer(
            nid,
            transformer_func=lambda x: np.abs(x) ** (1.0 / self._p_moment),
            **config[nid].to_dict(),
        )
        tail_nid = self._append(dag, tail_nid, node)
        # Divide returns by volatilty prediction.
        nid = "demodulate_using_vol_pred"
        node = VolatilityModulator(
            nid, mode="demodulate", **config[nid].to_dict()
        )
        self._append(dag, tail_nid, node)
        return dag

    # TODO(gp): This code has several copies. Move it to the base class.
    @staticmethod
    def _append(dag: DAG, tail_nid: Optional[str], node: Node) -> str:
        dag.add_node(node)
        if tail_nid is not None:
            dag.connect(tail_nid, node.nid)
        return node.nid

class _MultiColVolatilityModelMixin:
    def _fit_predict_volatility_model(
        self, df: pd.DataFrame, fit: bool, out_col_prefix: Optional[str] = None
    ) -> Tuple[Dict[str, pd.DataFrame], collections.OrderedDict]:
        dfs = {}
        info = collections.OrderedDict()
        for col in df.columns:
            local_out_col_prefix = out_col_prefix or col
            scvm = SingleColumnVolatilityModel(
                "volatility",
                steps_ahead=self._steps_ahead,
                col=col,
                p_moment=self._p_moment,
                progress_bar=self._progress_bar,
                tau=self._tau,
                nan_mode=self._nan_mode,
                out_col_prefix=local_out_col_prefix,
            )
            if fit:
                df_out = scvm.fit(df[[col]])["df_out"]
                info_out = scvm.get_info("fit")
                self._col_fit_state[col] = scvm.get_fit_state()
            else:
                scvm.set_fit_state(self._col_fit_state[col])
                df_out = scvm.predict(df[[col]])["df_out"]
                info_out = scvm.get_info("predict")
            dfs[col] = df_out
            info[col] = info_out
        return dfs, info

class VolatilityModel(
    FitPredictNode,
    ColModeMixin,
    _MultiColVolatilityModelMixin,
):
    """
    Fit and predict a smooth moving average volatility model.

    Wraps `SmaModel` internally, handling calculation of volatility from
    returns and column appends.
    """

    def __init__(
        self,
        nid: str,
        steps_ahead: int,
        cols: Optional[_TO_LIST_MIXIN_TYPE] = None,
        p_moment: float = 2,
        progress_bar: bool = False,
        tau: Optional[float] = None,
        col_rename_func: Callable[[Any], Any] = lambda x: f"{x}_zscored",
        col_mode: Optional[str] = None,
        nan_mode: Optional[str] = None,
    ) -> None:
        """
        Specify the data and smooth moving average (SMA) modeling parameters.

        :param nid: unique node id
        :param cols: name of columns to model
        :param steps_ahead: as in ContinuousSkLearnModel
        :param p_moment: exponent to apply to the absolute value of returns
        :param tau: as in `csigna.compute_smooth_moving_average`. If `None`,
            learn this parameter
        :param col_rename_func: renaming function for z-scored column
        :param col_mode:
            - If "merge_all" (default), merge all columns from input dataframe and
              transformed columns
            - If "replace_selected", merge unselected columns from input dataframe
              and transformed selected columns
            - If "replace_all", leave only transformed selected columns
        :param nan_mode: as in ContinuousSkLearnModel
        """
        super().__init__(nid)
        self._cols = cols
        self._steps_ahead = steps_ahead
        #
        dbg.dassert_lte(1, p_moment)
        self._p_moment = p_moment
        #
        self._progress_bar = progress_bar
        #
        dbg.dassert(tau is None or tau > 0)
        self._tau = tau
        self._col_rename_func = col_rename_func
        self._col_mode = col_mode or "merge_all"
        self._nan_mode = nan_mode
        # State of the model to serialize/deserialize.
        self._fit_cols: List[_COL_TYPE] = []
        self._col_fit_state = {}

    def fit(self, df_in: pd.DataFrame) -> Dict[str, pd.DataFrame]:
        return self._fit_predict_helper(df_in, fit=True)

    def predict(self, df_in: pd.DataFrame) -> Dict[str, pd.DataFrame]:
        return self._fit_predict_helper(df_in, fit=False)

    def get_fit_state(self) -> Dict[str, Any]:
        fit_state = {
            "_fit_cols": self._fit_cols,
            "_col_fit_state": self._col_fit_state,
            "_info['fit']": self._info["fit"],
        }
        return fit_state

    def set_fit_state(self, fit_state: Dict[str, Any]):
        self._fit_cols = fit_state["_fit_cols"]
        self._col_fit_state = fit_state["_col_fit_state"]
        self._info["fit"] = fit_state["_info['fit']"]

    def _fit_predict_helper(self, df_in: pd.DataFrame, fit: bool):
        cdu.validate_df_indices(df_in)
        # Get the columns.
        self._fit_cols = cdu.convert_to_list(self._cols or df_in.columns.tolist())
        df = df_in[self._fit_cols]
        dfs, info = self._fit_predict_volatility_model(df, fit=fit)
        df_out = pd.concat(dfs.values(), axis=1)
        df_out = self._apply_col_mode(
            df_in.drop(df_out.columns.intersection(df_in.columns), 1),
            df_out,
            cols=self._fit_cols,
            col_mode=self._col_mode,
        )
        method = "fit" if fit else "predict"
        self._set_info(method, info)
        return {"df_out": df_out}

class MultiindexVolatilityModel(FitPredictNode, _MultiColVolatilityModelMixin):
    """
    Fit and predict a smooth moving average volatility model.

    Wraps SmaModel internally, handling calculation of volatility from
    returns and column appends.
    """

    def __init__(
        self,
        nid: str,
        in_col_group: Tuple[_COL_TYPE],
        steps_ahead: int,
        p_moment: float = 2,
        progress_bar: bool = False,
        tau: Optional[float] = None,
        nan_mode: Optional[str] = None,
    ) -> None:
        """
        Specify the data and sma modeling parameters.

        :param nid: unique node id
        :param steps_ahead: as in ContinuousSkLearnModel
        :param p_moment: exponent to apply to the absolute value of returns
        :param tau: as in `csigna.compute_smooth_moving_average`. If `None`,
            learn this parameter
        :param nan_mode: as in ContinuousSkLearnModel
        """
        super().__init__(nid)
        dbg.dassert_isinstance(in_col_group, tuple)
        self._in_col_group = in_col_group
        self._out_col_group = in_col_group[:-1]
        self._out_col_prefix = str(in_col_group[-1])
        #
        self._steps_ahead = steps_ahead
        dbg.dassert_lte(1, p_moment)
        self._p_moment = p_moment
        #
        self._progress_bar = progress_bar
        #
        self._tau = tau
        self._nan_mode = nan_mode
        #
        self._col_fit_state = {}

    def fit(self, df_in: pd.DataFrame) -> Dict[str, pd.DataFrame]:
        return self._fit_predict_helper(df_in, fit=True)

    def predict(self, df_in: pd.DataFrame) -> Dict[str, pd.DataFrame]:
        return self._fit_predict_helper(df_in, fit=False)

    def get_fit_state(self) -> Dict[str, Any]:
        fit_state = {
            "_col_fit_state": self._col_fit_state,
            "_info['fit']": self._info["fit"],
        }
        return fit_state

    def set_fit_state(self, fit_state: Dict[str, Any]):
        self._col_fit_state = fit_state["_col_fit_state"]
        self._info["fit"] = fit_state["_info['fit']"]

    def _fit_predict_helper(self, df_in: pd.DataFrame, fit: bool):
        cdu.validate_df_indices(df_in)
        df = SeriesToDfColProcessor.preprocess(df_in, self._in_col_group)
        dfs, info = self._fit_predict_volatility_model(
            df, fit=fit, out_col_prefix=self._out_col_prefix
        )
        df_out = SeriesToDfColProcessor.postprocess(dfs, self._out_col_group)
        df_out = cdu.merge_dataframes(df_in, df_out)
        method = "fit" if fit else "predict"
        self._set_info(method, info)
        return {"df_out": df_out}

class VolatilityModulator(FitPredictNode, ColModeMixin):
    """
    Modulate or demodulate signal by volatility.

    Processing steps:
      - shift volatility to align it with signal
      - multiply/divide signal by volatility

    Usage examples:
      - Z-scoring
        - to obtain volatility prediction, pass in returns into `SmaModel` with
          a `steps_ahead` parameter
        - to z-score, pass in signal, volatility prediction, `signal_steps_ahead=0`,
          `volatility_steps_ahead=steps_ahead`, `mode='demodulate'`
      - Undoing z-scoring
        - Let's say we have
          - forward volatility prediction `n` steps ahead
          - prediction of forward z-scored returns `m` steps ahead. Z-scoring
            for the target has been done using the volatility prediction above
        - To undo z-scoring, we need to pass in the prediction of forward
          z-scored returns, forward volatility prediction, `signal_steps_ahead=n`,
          `volatility_steps_ahead=m`, `mode='modulate'`
    """

    def __init__(
        self,
        nid: str,
        signal_cols: _TO_LIST_MIXIN_TYPE,
        volatility_col: _COL_TYPE,
        signal_steps_ahead: int,
        volatility_steps_ahead: int,
        mode: str,
        col_rename_func: Optional[Callable[[Any], Any]] = None,
        col_mode: Optional[str] = None,
        nan_mode: Optional[str] = None,
    ) -> None:
        """
        :param nid: node identifier
        :param signal_cols: names of columns to (de)modulate
        :param volatility_col: name of volatility column
        :param signal_steps_ahead: steps ahead of the signal columns. If signal
            is at `t_0`, this value should be `0`. If signal is a forward
            prediction of z-scored returns indexed by knowledge time, this
            value should be equal to the number of steps of the prediction
        :param volatility_steps_ahead: steps ahead of the volatility column. If
            volatility column is an output of `SmaModel`, this corresponds to
            the `steps_ahead` parameter
        :param mode: "modulate" or "demodulate"
        :param col_rename_func: as in `ColumnTransformer`
        :param col_mode: as in `ColumnTransformer`
        """
        super().__init__(nid)
        self._signal_cols = cdu.convert_to_list(signal_cols)
        self._volatility_col = volatility_col
        dbg.dassert_lte(0, signal_steps_ahead)
        self._signal_steps_ahead = signal_steps_ahead
        dbg.dassert_lte(0, volatility_steps_ahead)
        self._volatility_steps_ahead = volatility_steps_ahead
        dbg.dassert_in(mode, ["modulate", "demodulate"])
        self._mode = mode
        self._col_rename_func = col_rename_func or (lambda x: x)
        self._col_mode = col_mode or "replace_all"
        self._nan_mode = nan_mode or "leave_unchanged"

    def fit(self, df_in: pd.DataFrame) -> Dict[str, pd.DataFrame]:
        df_out = self._process_signal(df_in)
        info = collections.OrderedDict()
        info["df_out_info"] = cdu.get_df_info_as_string(df_out)
        self._set_info("fit", info)
        return {"df_out": df_out}

    def predict(self, df_in: pd.DataFrame) -> Dict[str, pd.DataFrame]:
        df_out = self._process_signal(df_in)
        info = collections.OrderedDict()
        info["df_out_info"] = cdu.get_df_info_as_string(df_out)
        self._set_info("predict", info)
        return {"df_out": df_out}

    def _process_signal(self, df_in: pd.DataFrame) -> pd.DataFrame:
        """
        Modulate or demodulate signal by volatility prediction.

        :param df_in: dataframe with `self._signal_cols` and
            `self._volatility_col` columns
        :return: adjusted signal indexed in the same way as the input signal
        """
        dbg.dassert_is_subset(self._signal_cols, df_in.columns.tolist())
        dbg.dassert_in(self._volatility_col, df_in.columns)
        fwd_signal = df_in[self._signal_cols]
        fwd_volatility = df_in[self._volatility_col]
        # Shift volatility to align it with signal.
        volatility_shift = self._volatility_steps_ahead - self._signal_steps_ahead
        if self._nan_mode == "drop":
            fwd_volatility = fwd_volatility.dropna()
        elif self._nan_mode == "leave_unchanged":
            pass
        else:
            raise ValueError(f"Unrecognized `nan_mode` {self._nan_mode}")
        volatility_aligned = fwd_volatility.shift(volatility_shift)
        # Adjust signal by volatility.
        if self._mode == "demodulate":
            adjusted_signal = fwd_signal.divide(volatility_aligned, axis=0)
        elif self._mode == "modulate":
            adjusted_signal = fwd_signal.multiply(volatility_aligned, axis=0)
        else:
            raise ValueError(f"Invalid mode=`{self._mode}`")
        df_out = self._apply_col_mode(
            df_in,
            adjusted_signal,
            cols=self._signal_cols,
            col_rename_func=self._col_rename_func,
            col_mode=self._col_mode,
        )
        return df_out

class VolatilityNormalizer(FitPredictNode, ColModeMixin):
    def __init__(
        self,
        nid: str,
        col: str,
        target_volatility: float,
        col_mode: Optional[str] = None,
    ) -> None:
        """
        Normalize series to target annual volatility.

        :param nid: node identifier
        :param col: name of column to rescale
        :param target_volatility: target volatility as a proportion
        :param col_mode: `merge_all` or `replace_all`. If `replace_all`, return
            only the rescaled column, if `merge_all`, append the rescaled
            column to input dataframe
        """
        super().__init__(nid)
        self._col = col
        self._target_volatility = target_volatility
        self._col_mode = col_mode or "merge_all"
        dbg.dassert_in(
            self._col_mode,
            ["merge_all", "replace_all"],
            "Invalid `col_mode`='%s'",
            self._col_mode,
        )
        self._scale_factor: Optional[float] = None

    def fit(self, df_in: pd.DataFrame) -> Dict[str, pd.DataFrame]:
        dbg.dassert_in(self._col, df_in.columns)
        self._scale_factor = cfinan.compute_volatility_normalization_factor(
            df_in[self._col], self._target_volatility
        )
        rescaled_y_hat = self._scale_factor * df_in[self._col]
        df_out = self._apply_col_mode(
            df_in,
            rescaled_y_hat.to_frame(),
            cols=[self._col],
            col_rename_func=lambda x: f"rescaled_{x}",
            col_mode=self._col_mode,
        )
        # Store info.
        info = collections.OrderedDict()
        info["scale_factor"] = self._scale_factor
        self._set_info("fit", info)
        return {"df_out": df_out}

    def predict(self, df_in: pd.DataFrame) -> Dict[str, pd.DataFrame]:
        dbg.dassert_in(self._col, df_in.columns)
        rescaled_y_hat = self._scale_factor * df_in[self._col]
        df_out = self._apply_col_mode(
            df_in,
            rescaled_y_hat.to_frame(),
            cols=[self._col],
            col_rename_func=lambda x: f"rescaled_{x}",
            col_mode=self._col_mode,
        )
        return {"df_out": df_out}

ext: py

reason: import numpy,import scipy


text:
import numpy as np
import tensorflow as tf
import os,time
import warp

def load(opt, test=False):
	"""
	load data
	데이터 불러오는 함수
	:param opt: 옵션
	:param test: 테스트를 위해 데이터를 불러오는지
	:return: 데이터셋이 담긴 디렉토리
	"""
	# 불러올 디렉토리 경로 설정
	main_folder = "/content/gdrive/My Drive/Colab Notebooks/spatial-transformer-GAN/glasses/"
	path = main_folder + "dataset"

	# 테스트 데이터셋를 불러와야 한다면,
	if test:
		# 테스트 이미지 데이터가 담긴 .npy 파일 불러오기
		images = np.load("{0}/image_test.npy".format(path))    # 귀걸이 안 낀 이미지 npy 파일
		# 귀걸이 착용 유무 속성정보 불러오기
		hasGlasses = np.load("{0}/attribute_test.npy".format(path))[:, 35]

		images_0 = images[~hasGlasses]
		images_1 = images[hasGlasses]

	# 학습 데이터셋을 불러와야 한다면,
	else:
		not_wearing_images_0 = np.load("{0}/not_wearing_earing_test_0.npy".format(path))
		not_wearing_images_1 = np.load("{0}/not_wearing_earing_test_1.npy".format(path))
		wearing_images_0 = np.load("{0}/earing_image_test_0.npy".format(path))
		wearing_images_1 = np.load("{0}/earing_image_test_1.npy".format(path))
		images_0 = np.vstack([not_wearing_images_0, not_wearing_images_1])  # 귀걸이 안 낀 이미지 npy 파일
		images_1 = np.vstack([wearing_images_0, wearing_images_1])  # 귀걸이 낀 이미지 npy 파일
	"""

	# 테스트 이미지 데이터가 담긴 .npy 파일 불러오기
	images = np.load("{0}/image_test.npy".format(path))  # 귀걸이 안 낀 이미지 npy 파일
	# 귀걸이 착용 유무 속성정보 불러오기
	hasGlasses = np.load("{0}/attribute_test.npy".format(path))[:, 35]

	images_0 = images[~hasGlasses]
	images_1 = images[hasGlasses]

	"""

	# 안경 데이터셋 불러오기
	glasses = np.load("{0}/earring_train.npy".format(path))

	# 디렉토리 생성
	D = {
		"image0": images_0,
		"image1": images_1,
		"glasses": glasses,
	}
	return D

def makeBatch(opt, data, PH):
	"""
	make training batch
	학습을 위한 배치(학습 데이터 덩어리) 만드는 함수
	:param opt: 옵션
	:param data: 배치를 가져올 전체 데이터셋
	:param PH: placeholder
	:return: 배치 데이터
	"""
	# 데이터셋에서 각 데이터들의 개수를 구함
	N0 = len(data["image0"])
	N1 = len(data["image1"])
	NG = len(data["glasses"])

	# 배치 사이즈 만큼의 데이터 인덱스를 배치 사이즈 만큼 생성
	randIdx0 = np.random.randint(N0, size=[opt.batchSize])
	randIdx1 = np.random.randint(N1, size=[opt.batchSize])
	randIdxG = np.random.randint(NG, size=[opt.batchSize])

	# put data in placeholders
	[imageBGfakeData, imageRealData, imageFGfake] = PH

	# 각 픽셀이 0~255로 되어있는 상태를 0~1로 전처리해주며 배치 사이즈만큼 가져옴
	batch = {
		imageBGfakeData: data["image0"][randIdx0]/255.0,
		imageRealData: data["image1"][randIdx1]/255.0,
		imageFGfake: data["glasses"][randIdxG]/255.0,
	}
	return batch

# make test batch
def makeBatchEval(opt, testImage, glasses, PH):
	"""
	make test batch
	테스트를 위한 배치 만드는 함수
	:param opt: 옵션
	:param testImage: 테스트할 인물 이미지 데이터셋
	:param glasses: 안경 이미지 데이터셋
	:param PH: placeholder
	:return: 배치 데이터
	"""
	# 배치 사이즈만큼의 1차원 배열 생성 0, 1, 2, ..., batchSize-1
	idxG = np.arange(opt.batchSize)

	print(idxG)

	# put data in placeholders
	[imageBG, imageFG] = PH

	batch = {
		# 인물 이미지 배치 사이즈 만큼 반복하여 연결
		imageBG: np.tile(testImage, [opt.batchSize, 1, 1, 1]),
		# 배치 사이즈 만큼의 안경 데이터 가져옴
		imageFG: glasses[idxG]/255.0,
	}
	return batch

def perturbBG(opt, imageData):
	"""
	generate pereturbed image
	교란된 이미지 생성하는 함수
	:param opt: 옵션
	:param imageData: 전체 이미지 데이터셋
	:return: 이미지 데이터
	"""
	# batch 사이즈 만큼의 랜덤한 값을 가진 배열들을 생성하여 각 원소에 pertBG 값 곱해줌
	rot = opt.pertBG*tf.random_normal([opt.batchSize])
	tx = opt.pertBG*tf.random_normal([opt.batchSize])
	ty = opt.pertBG*tf.random_normal([opt.batchSize])
	# 배치 사이즈 만큼의 0으로만 이루어진 배열 생성
	O = tf.zeros([opt.batchSize])
	# 교란된 백그라운드 이미지(인물 사진) 생성을 위한 warp 매개변수 생성
	# 왜곡 타입이 homography(가로, 세로간의 평행을 유지하지 않고 이미지를 왜곡시킴)일 때와
	# 왜곡 타입이 affine(가로, 세로간의 평행을 유지하며 이미지를 왜곡시킴)으로 나누어 생성
	pPertBG = \
		tf.stack([tx, rot, O, O, ty, -rot, O, O], axis=1) if opt.warpType == "homography" else\
		tf.stack([O, rot, tx, -rot, O, ty], axis=1) if opt.warpType == "affine" else None
	# 벡터 형태의 warp 매개변수를 matrix로 변환
	pPertBGmtrx = warp.vec2mtrx(opt, pPertBG)
	# warp 파라미터에 따라 이미지 데이터셋을 변형 및 크롭하여 생성
	image = warp.transformCropImage(opt, imageData, pPertBGmtrx)
	return image

history = [None, 0, True]

# update history and group fake samples
def updateHistory(opt, newFake):
	"""
	히스토리를 업데이트하고 가짜 샘플을 그룹핑하는 함수
	:param opt: 옵션들
	:param newFake: 새로운 가짜 이미지
	:return: 페이크 이미지
	"""
	if history[0] is None:
		history[0] = np.ones([opt.histQsize, opt.H, opt.W, 3], dtype=np.float32)
		history[0][:opt.batchSize] = newFake
		history[1] = opt.batchSize
		return newFake
	else:
		randIdx = np.random.permutation(opt.batchSize)
		storeIdx = randIdx[:opt.histSize]
		useIdx = randIdx[opt.histSize:]
		# group fake samples
		hi, growing = history[1], history[2]
		extractIdx = np.random.permutation(hi if growing else opt.histQsize)[:opt.histSize]
		groupFake = np.concatenate([history[0][extractIdx], newFake[useIdx]], axis=0)
		hinew = hi+opt.batchSize-opt.histSize
		history[0][hi:hinew] = newFake[storeIdx]
		history[1] = hinew
		if hinew == opt.histQsize:
			history[1] = 0
			history[2] = False
		return groupFake

reason: import numpy


text:
from pyFHE.mulfft import TwistFFT, TwistIFFT, TwistGen
import numpy as np

a = [-1, 2 ** 31, 0, 1]
b = [2, 3, 1, 2]
twist = TwistGen(4)

print(TwistIFFT(np.sum([TwistFFT(a, twist), TwistFFT(b, twist)], axis=0), twist))

max_stars_repo_name: virtualsecureplatform/pyFHE

max_issues_repo_name: virtualsecureplatform/pyFHE

max_forks_repo_name: virtualsecureplatform/pyFHE


text:
'''
PyTorch has its own implementation of backward function for SVD https://github.com/pytorch/pytorch/blob/291746f11047361100102577ce7d1cfa1833be50/tools/autograd/templates/Functions.cpp#L1577
We reimplement it with a safe inverse function in light of degenerated singular values
'''

import numpy as np
import torch
import torch.nn.functional as Functional
import scipy.sparse.linalg

def safe_inverse(x, epsilon=1E-12):
    return x/(x**2 + epsilon)

class ARNOLDISVD(torch.autograd.Function):

    # INPUT:
    # M - input matrix
    # k [int] - desired rank
    # p [int] - oversampling rank. Total rank sampled k+p
    # vnum [int] - ?
    # q [int] - number of matrix-vector multiplications for power scheme
    # s [int] - reorthogonalization
    # OUTPUT
    # U, S, V - truncated svd decomposition M \approx U S V^dag
    @staticmethod
    def forward(self, M, k):
        # check symmetry of M
        #print("norm(0.5*(M-M^t)) "+str(torch.norm(M - M.t())))

        # get M as numpy ndarray
        M_nograd = M.clone().detach().numpy()
        M_nograd = M_nograd @ M_nograd
        #print("FWD k: "+ str(k) + " M: " + str(M_nograd.shape))

        # we get, say (since M is symmetric) left singular vectors of M
        # M = U.S.Vt -> M.Mt = U.S.S.Ut
        s, u = scipy.sparse.linalg.eigsh(M_nograd, k=k)

        # Debug info about properties of output tensors from eigsh
        #print("u "+str(u.shape))
        #print("u "+str(u.strides))

        # find range
        # print(s)
        # if (s[0] is np.nan) or (s[0] != s[0]):
        #     raise Exception('nan')
        # rankS = len(list(filter(lambda e: e > 0., s)))
        # print(rankS)

        # properly order the result (singular values are given in ascending
        # order) by ARPACK
        u = np.copy(u[:,::-1])
        s = np.copy(s[::-1])
        s = np.sqrt(np.abs(s))

        U = torch.as_tensor(u)
        S = torch.as_tensor(s)

        # compute right singular vectors as Mt = V.S.Ut /.U => Mt.U = V.S
        # since M = Mt, M.U = V.S
        V = M @ U
        V = Functional.normalize(V, p=2, dim=0)

        # print("U "+str(U.shape))
        # print(U.stride())
        # for i in range(k):
        #     print(str(i)+": "+str(np.linalg.norm(U[:,i])))
        #print(S.shape)
        #print(S)
        # print("V "+str(V.shape))
        # print(V.stride())
        # for i in range(k):
        #     print(str(i)+": "+str(np.linalg.norm(V[:,i])))

        self.save_for_backward(U, S, V)
        return U, S, V

    @staticmethod
    def backward(self, dU, dS, dV):
        U, S, V = self.saved_tensors
        Vt = V.t()
        Ut = U.t()
        M = U.size(0)
        N = V.size(0)
        NS = len(S)

        F = (S - S[:, None])
        F = safe_inverse(F)
        F.diagonal().fill_(0)

        G = (S + S[:, None])
        G.diagonal().fill_(np.inf)
        G = 1/G

        UdU = Ut @ dU
        VdV = Vt @ dV

        Su = (F+G)*(UdU-UdU.t())/2
        Sv = (F-G)*(VdV-VdV.t())/2

        dA = U @ (Su + Sv + torch.diag(dS)) @ Vt
        if (M>NS):
            dA = dA + (torch.eye(M, dtype=dU.dtype, device=dU.device) - U@Ut) @ (dU/S) @ Vt
        if (N>NS):
            dA = dA + (U/S) @ dV.t() @ (torch.eye(N, dtype=dU.dtype, device=dU.device) - V@Vt)
        return dA, None

# def test_svd():
#     M, N = 50, 40
#     torch.manual_seed(2)
#     input = torch.rand(M, N, dtype=torch.float64, requires_grad=True)
#     assert(torch.autograd.gradcheck(SVD.apply, input, eps=1e-6, atol=1e-4))
#     print("Test Pass!")

# if __name__=='__main__':
#     test_svd()

max_stars_repo_name: jurajHasik/tensorgrad

max_issues_repo_name: jurajHasik/tensorgrad

max_forks_repo_name: jurajHasik/tensorgrad


text:
import argparse
import os
import pickle
import sys

import numpy as np
from joblib import Parallel, delayed

os.environ["OPENBLAS_NUM_THREADS"] = "1"
sys.path.append("../")

from environments.GeneralEnvironment import PricingAdvertisingJointEnvironment
from environments.Settings.EnvironmentManager import EnvironmentManager
from utils.experiments_helper import build_discrete_bandit
from utils.folder_management import handle_folder_creation

# Basic default settings
N_ROUNDS = 100000
BASIC_OUTPUT_FOLDER = "../report/project_point_4/"

# Pricing settings
SCENARIO_NAME = "linear_scenario"  # corresponds to the name of the file in "resources"
MIN_PRICE = 15
MAX_PRICE = 25
N_ARMS = 11
DEFAULT_DISCRETIZATION = "UNIFORM"
FIXED_BUDGET = 1000 / 3
UNIT_COST = 12

CRP_UPPER_BOUND = 0.3
GAMMA_EXP_3 = 0.1

def get_arguments():
    """
    Defining the arguments available for the script

    :return: argument parser
    """
    parser = argparse.ArgumentParser()
    # Pricing settings
    parser.add_argument("-n_arms", "--n_arms", help="Number of arms for the prices", type=int, default=N_ARMS)
    parser.add_argument("-d", "--discretization", help="Discretization type", type=str, default=DEFAULT_DISCRETIZATION)
    parser.add_argument("-c", "--unit_cost", help="Unit cost for each sold unit", type=float, default=UNIT_COST)

    # Ads setting
    parser.add_argument("-bud", "--budget", default=FIXED_BUDGET, help="Fixed budget to be used for the simulation",
                        type=float)

    # Scenario settings
    parser.add_argument("-scenario_name", "--scenario_name", default=SCENARIO_NAME,
                        type=str, help="Name of the setting of the experiment")

    # Experiment run details
    parser.add_argument("-n_runs", "--n_runs", default=1, help="Number of runs of the experiments", type=int)
    parser.add_argument("-n_jobs", "--n_jobs", default=1, help="Number of jobs to be run", type=int)
    parser.add_argument("-t", "--n_rounds", default=N_ROUNDS, help="Number of rounds", type=int)

    # Bandit hyper-parameters
    parser.add_argument("-b", "--bandit_name", help="Name of the bandit to be used in the experiment")
    parser.add_argument("-gamma", "--gamma",
                        help="Parameter for tuning the desire to pick an action uniformly at random",
                        type=float, default=GAMMA_EXP_3)
    parser.add_argument("-crp_ub", "--crp_upper_bound", help="Upper bound of the conversion rate probability",
                        type=float, default=CRP_UPPER_BOUND)

    # Store results
    parser.add_argument("-s", "--save_result", help="Whether to store results or not", type=lambda x: int(x) != 0,
                        default=1)
    parser.add_argument("-o", "--output_folder", default=BASIC_OUTPUT_FOLDER, help="Basic folder where"
                                                                                "to store the output",
                        type=str)

    return parser.parse_args()

def get_prices(args):
    if args.discretization == "UNIFORM":
        return np.linspace(start=MIN_PRICE, stop=MAX_PRICE, num=args.n_arms)
    else:
        raise NotImplemented("Not implemented discretization method")

def main(args):
    scenario = EnvironmentManager.load_scenario(args.scenario_name)
    env = PricingAdvertisingJointEnvironment(scenario)
    env.set_budget_allocation([args.budget] * scenario.get_n_subcampaigns())

    prices = get_prices(args=args)
    arm_profit = prices - args.unit_cost
    bandit = build_discrete_bandit(bandit_name=args.bandit_name, n_arms=len(arm_profit),
                                   arm_values=arm_profit, args=args)

    iterate = not env.next_day()
    while iterate:
        iterate = not env.next_day()

    for _ in range(0, args.n_rounds):
        # Choose arm
        price_idx = bandit.pull_arm()

        # Observe reward
        env.next_user()
        reward, elapsed_day = env.round(price=prices[price_idx])
        reward = reward * arm_profit[price_idx]

        if elapsed_day:
            iterate = not env.next_day()
            while iterate:
                iterate = not env.next_day()

        # Update bandit
        bandit.update(pulled_arm=price_idx, reward=reward)

    return bandit.collected_rewards, env.get_day_breakpoints()

def run(id, seed, args):
    """
    Run a task to carry out the experiment

    :param id: id of the task
    :param seed: random seed that is used in this execution
    :param args: arguments given to the experiment
    :return: collected rewards
    """
    # Eventually fix here the seeds for additional sources of randomness (e.g. tensorflow)
    np.random.seed(seed)
    print("Starting run {}".format(id))
    rewards, day_breakpoints = main(args=args)
    print("Done run {}".format(id))
    return rewards, day_breakpoints

# Scheduling runs: ENTRY POINT
args = get_arguments()

seeds = [np.random.randint(1000000) for _ in range(args.n_runs)]
if args.n_jobs == 1:
    results = [run(id=id, seed=seed, args=args) for id, seed in zip(range(args.n_runs), seeds)]
else:
    results = Parallel(n_jobs=args.n_jobs, backend='loky')(
        delayed(run)(id=id, seed=seed, args=args) for id, seed in zip(range(args.n_runs), seeds))

rewards = [res[0] for res in results]
day_breakpoint = [res[1] for res in results]

if args.save_result:
    # Set up writing folder and file
    fd, folder_path_with_date = handle_folder_creation(result_path=args.output_folder)

    # Writing results and experiment details
    print("Storing results on file...", end="")
    with open("{}reward_{}.pkl".format(folder_path_with_date, args.bandit_name), "wb") as output:
        pickle.dump(rewards, output)
    with open("{}day_{}.pkl".format(folder_path_with_date, args.bandit_name), "wb") as output:
        pickle.dump(day_breakpoint, output)
    print("Done")

    fd.write("Bandit experiment\n")
    fd.write("Number of arms: {}\n".format(args.n_arms))
    fd.write("Number of runs: {}\n".format(args.n_runs))
    fd.write("Horizon: {}\n".format(args.n_rounds))
    fd.write("Bandit algorithm: {}\n\n".format(args.bandit_name))
    fd.write("Scenario name {}\n".format(args.scenario_name))

    fd.write("Discretization type {}\n\n".format(args.discretization))

    fd.write("Bandit parameters \n")
    fd.write("Gamma parameter (EXP3) {}\n".format(args.gamma))
    fd.write("CRP Upper bound {}\n".format(args.crp_upper_bound))

    fd.close()


text:
#This file contains functions to find information on the complexity of the methods.
import numpy as np
import scipy as sp
import R3
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from mpl_toolkits.mplot3d.art3d import Poly3DCollection
import time
from scipy.optimize import root
from skimage import measure

#Plot the time of an approximation and a marching cubes sampling as we increase the amount of input points:
#Parameters: f, the domain, weighing function theta, the range of amount of equidistant points per axis and the order.
def time_input(f, domain, theta, sizes, order):
	X, Y, Z = domain
	approx_times = [None] * len(sizes)
	sample_times = [None] * len(sizes)
	inputs = [None] * len(sizes)
	i = 0
	for s in sizes:
		test = np.mgrid[-4:4:s*1j, -4:4:s*1j, -4:4:s*1j].reshape(3,-1).T #Grid size can be changed here if needed.
		inputs[i] = len(test)
		start = time.time()
		#Get the aproximation (comment/uncomment to choose the method).
		approx = R3.least_squares_R3(test, f(test), order)
		#approx = R3.weighted_least_squares_R3(test, f(test), order, theta)
		#approx = R3.moving_least_squares_R3(test, f(test), order, theta)
		approx_times[i] = time.time() - start

		#Take samples:
		start = time.time()
		vertices, simplices,normals, values = measure.marching_cubes_lewiner(approx(X,Y,Z), -0.3)
		sample_times[i] = time.time() - start
		i+=1

	total_times = [None] * len(sizes)
	for i in range(len(sizes)):
		total_times[i] = approx_times[i] + sample_times[i]

	plt.plot(inputs, approx_times, color='b', label="Approximation time")
	plt.scatter(inputs, approx_times, color='b')
	plt.plot(inputs, sample_times, color='r', label="Sample time")
	plt.scatter(inputs, sample_times, color='r')
	plt.plot(inputs, total_times, color='#000000', label="Total time")
	plt.scatter(inputs, total_times, color='#000000')
	plt.xlabel('Amount of input points')
	plt.ylabel('Time')
	plt.legend(loc='upper left')
	plt.savefig('input.png')
	plt.show()

#Plot the time of an approximation and a marching cubes sampling as we increase the order:
#Parameters: input points, f, the domain, weighing function theta and the range of orders of the polynomial space.
def time_order(points, f, domain, theta, ms):
	X, Y, Z = domain
	approx_times = [None] * len(ms)
	sample_times = [None] * len(ms)
	i = 0
	for m in ms:
		start = time.time()
		#Get the aproximation (comment/uncomment to choose the method).
		approx = R3.least_squares_R3(points, f(points), m)
		#approx = R3.weighted_least_squares_R3(points, f(points), m, theta)
		#approx = R3.moving_least_squares_R3(points, f(points), m, theta)
		approx_times[i] = time.time() - start

		#Take samples:
		start = time.time()
		vertices, simplices,normals, values = measure.marching_cubes_lewiner(approx(X,Y,Z), -0.3)
		sample_times[i] = time.time() - start
		i+=1

	width = 0.2
	plt.bar(ms, approx_times, width, color='b', label="Approximation time")
	plt.bar(ms, sample_times, width, color='r', label="Sample time", bottom=approx_times)
	plt.xlabel('Order of the polynomial space')
	plt.ylabel('Time')
	plt.legend(loc='upper left')
	plt.savefig('order.png')
	plt.show()

domain = np.mgrid[-4:4:50j, -4:4:50j, -4:4:50j]
test = np.mgrid[-4:4:6j, -4:4:6j, -4:4:6j].reshape(3,-1).T
ms = range(2, 10)
theta = lambda d : ((1-d/8)**4)*(4*d/8 + 1)

#Define the 3D function:
def func(arr):
	res = np.zeros((len(arr),1))
	i = 0
	for p in arr:
		res[i] = np.cos(p[0]) + np.cos(p[1]) + np.cos(p[2])
		i+=1
	return res

# time_order(test, func, domain, theta, ms)

# sizes = range(4, 16)
# time_input(func, domain, theta, sizes, 3)

max_stars_repo_name: desparito/comparing-ls-methods

max_issues_repo_name: desparito/comparing-ls-methods

max_forks_repo_name: desparito/comparing-ls-methods

reason: import numpy,import scipy,from scipy


text:
import numpy as np
import ubelt as ub
import torchvision
import pandas as pd
from torchvision.datasets import cifar
from clab.torch.transforms import (RandomWarpAffine, RandomGamma, RandomBlur,)
from clab import util

class Task(object):
    def __init__(task, labelnames=None, ignore_labelnames=[], alias={}):
        if labelnames is not None:
            task.set_labelnames(labelnames, ignore_labelnames, alias)

    def set_labelnames(task, labelnames, ignore_labelnames=[], alias={}):
        task.labelnames = list(labelnames)
        task.labelname_alias = alias
        task.ignore_labelnames = ignore_labelnames

        # Remove aliased classes
        for k in alias.keys():
            if k in task.labelnames:
                task.labelnames.remove(k)

        # Assign an integer label to each labelname
        task.labelname_to_id = ub.invert_dict(dict(enumerate(task.labelnames)))

        # Map aliased classes to a different label
        for k, v in alias.items():
            task.labelname_to_id[k] = task.labelname_to_id[v]

        task.ignore_labelnames = ignore_labelnames
        task.ignore_labels = np.array(list(ub.take(task.labelname_to_id, task.ignore_labelnames)))

        task.labels = np.arange(len(task.labelnames))
        task.relevant_labels = np.setdiff1d(task.labels, task.ignore_labels)

class CIFAR100_Task(Task):
    """
    task = CIFAR100_Task()
    task._initialize()
    ignore_labelnames = []
    alias = {}
    """
    def __init__(task, root=None):
        if root is None:
            root = ub.ensure_app_cache_dir('clab')
        task.root = root
        task._initialize()

    def _initialize(task):
        from os.path import join
        import pickle
        train_dset = cifar.CIFAR100(root=task.root, download=False, train=True)

        fpath = join(train_dset.root, cifar.CIFAR100.base_folder, 'meta')
        with open(fpath, 'rb') as fo:
            entry = pickle.load(fo, encoding='latin1')
            labelnames = entry['fine_label_names']
        task.set_labelnames(labelnames)

def mutex_clf_gt_info(gt_labels, task):
    """
    gt_labels = train_dset.train_labels
    """
    index = pd.Index(task.labels, name='label')
    gtstats = pd.DataFrame(0, index=index, columns=['freq'], dtype=np.int)

    label_freq = pd.value_counts(gt_labels)
    gtstats.freq = pd.to_numeric(label_freq)

    gtstats['classname'] = list(ub.take(task.labelnames, gtstats.index))
    gtstats['mf_weight'] = gtstats.freq.median() / gtstats.freq
    gtstats.loc[~np.isfinite(gtstats.mf_weight), 'mf_weight'] = 1

    # Clip weights, so nothing gets crazy high weights, low weights are ok
    gtstats = gtstats.sort_index()
    gtstats.index.name = 'label'
    gtstats = gtstats.reset_index().set_index('classname', drop=False)
    return gtstats

class InMemoryInputs(object):
    """
    Change inputs.Inputs to OnDiskInputs
    """
    def __init__(inputs, tag=''):
        inputs.tag = tag
        inputs.im = None
        inputs.gt = None
        inputs.colorspace = None

    def __nice__(inputs):
        n = len(inputs)
        return '{} {}'.format(inputs.tag, n)

    def __len__(inputs):
        if inputs.im is not None:
            n = len(inputs.im)
        elif inputs.gt is not None:
            n = len(inputs.gt)
        else:
            n = 0
        return n

    @classmethod
    def from_bhwc_rgb(cls, bhwc, labels=None, **kw):
        # convert to bhwc
        inputs = cls(**kw)
        inputs.im = bhwc
        inputs.gt = labels
        inputs.colorspace = 'rgb'
        return inputs

    def convert_colorspace(inputs, colorspace):
        if colorspace.lower() == inputs.colorspace.lower():
            return
        im_out = np.empty_like(inputs.im)
        dst = np.ascontiguousarray(np.empty_like(inputs.im[0]))
        for ix, im in enumerate(inputs.im):
            util.convert_colorspace(im, src_space=inputs.colorspace,
                                    dst_space=colorspace, dst=dst)
            im_out[ix] = dst
        inputs.im = im_out
        inputs.colorspace = colorspace

    def take(inputs, idxs, **kw):
        new_inputs = inputs.__class__(**kw)
        new_inputs.im = inputs.im.take(idxs, axis=0)
        new_inputs.gt = inputs.gt.take(idxs, axis=0)
        new_inputs.colorspace = inputs.colorspace
        return new_inputs

    def prepare_images(self, force=False):
        """
        If not already done, loads paths to images into memory and constructs a
        unique id for that set of im/gt images.

        It the paths are already set, then only the input-id is constructed.
        """
        if self.n_input is not None and not force:
            return

        self.prepare_image_paths()
        print('Preparing {} images'.format(self.tag))

        if self.aux_paths:
            # new way
            depends = sorted(self.paths.items())
        else:
            depends = []
            depends.append(self.im_paths)
            depends.append(self.gt_paths)
            if self.gt_paths:
                # HACK: We will assume image data depends only on the filename
                # HACK: be respectful of gt label changes (ignore aug)
                # stride>1 is faster but might break
                # stride=1 is the safest
                hashes = [
                    util.hash_file(p, stride=32)
                    for p in ub.ProgIter(self.gt_paths, label='hashing')
                    if 'aug' not in basename(p) and 'part' not in basename(p)
                ]
                label_hashid = util.hash_data(hashes)
                depends.append(label_hashid)
        n_im = None if self.im_paths is None else len(self.im_paths)
        n_gt = None if self.gt_paths is None else len(self.gt_paths)
        self.n_input = n_im or n_gt

        hashid = hashutil.hash_data(depends)[:self.abbrev]
        self.input_id = '{}-{}'.format(self.n_input, hashid)

        print(' * n_images = {}'.format(n_im))
        print(' * n_groundtruth = {}'.format(n_gt))
        print(' * input_id = {}'.format(self.input_id))

def cifar_training_datasets():
    root = ub.ensure_app_cache_dir('clab')
    train_dset = cifar.CIFAR100(root=root, download=True, train=True)
    bchw = (train_dset.train_data).astype(np.float32) / 255.0
    labels = np.array(train_dset.train_labels)
    inputs = InMemoryInputs.from_bhwc_rgb(bchw, labels=labels)
    # inputs.convert_colorspace('lab')

    vali_frac = .1
    n_vali = int(len(inputs) * vali_frac)
    input_idxs = np.arange(len(inputs))
    rng = np.random.RandomState(0)
    rng.shuffle(input_idxs)
    train_idxs = input_idxs[:-n_vali]
    vali_idxs = input_idxs[-n_vali:]

    train_inputs = inputs.take(train_idxs, tag='train')
    vali_inputs = inputs.take(vali_idxs, tag='vali')

class CIFAR_Wrapper(cifar.CIFAR10):
    def __init__(self, inputs, task, workdir):
        self.inputs = inputs
        self.task = task

        self.rng = np.random.RandomState(432432)

        inputs_base = ub.ensuredir((workdir, 'inputs'))
        inputs.base_dpath = inputs_base
        if len(inputs):
            inputs.prepare_images(force=True)
            inputs.prepare_input()
            self.input_id = inputs.input_id
            self.with_gt = self.inputs.gt_paths
        else:
            self.input_id = ''

        self.augment = None
        self.im_augment = torchvision.transforms.Compose([
            RandomGamma(rng=self.rng),
            RandomBlur(rng=self.rng),
        ])
        self.rand_aff = RandomWarpAffine(self.rng)

        if self.inputs.aux_paths:
            self.aux_keys = sorted(self.inputs.aux_paths.keys())
        else:
            self.aux_keys = []

        self.center_inputs = None

    def _make_normalizer(self):
        from .torch.transforms import (ImageCenterScale, DTMCenterScale,
                                             ZipTransforms)
        transforms = []
        nan_value = -32767.0  # hack: specific number for DTM
        if len(self.inputs):
            self.center_stats = self.inputs.prepare_center_stats(
                self.task, nan_value=nan_value, colorspace=self.colorspace)
            # self.center_stats['image'].pop('detail')
            # if self.aux_keys:
            #     self.center_stats['aux'].pop('detail')

            if self.colorspace == 'LAB':
                # Do per-channel mean / std centering independently for LAB images
                channel_stats = self.center_stats['image']['simple']['channel']
                im_mean = channel_stats['mean']
                im_scale = channel_stats['std']
            elif self.colorspace == 'RGB':
                # Normalize across channels for RGB
                scalar_stats = self.center_stats['image']['simple']['image']
                im_mean = scalar_stats['mean']
                im_scale = scalar_stats['std']
                # self.im_center = ub.identity
            else:
                raise Exception()

            im_center = ImageCenterScale(im_mean, im_scale)
            transforms.append(im_center)

            # im_scale = np.ceil(channel_stats['max']) - np.floor(channel_stats['min'])

            if self.aux_keys:
                # Note: Internal stats are not gaurenteed to make sense outside
                # the DTM domain.
                internal_aux_stats = self.center_stats['aux']['internal']
                mean_deviation = internal_aux_stats['image']['mean_absdev_from_median']['mean']
                # zero the median on a per-chip basis, but use
                # the global internal_std to normalize extent
                # aux_std =
                aux_center = DTMCenterScale(mean_deviation,
                                            nan_value=nan_value, fill='median')
                transforms.append(aux_center)

        center_inputs = ZipTransforms(transforms)
        self.center_inputs = center_inputs
        return center_inputs

    def __len__(self):
        return len(self.inputs)

    def load_inputs(self, index):
        im_fpath = self.inputs.im_paths[index]
        if self.inputs.gt_paths:
            gt_fpath = self.inputs.gt_paths[index]
            gt = self.loader(gt_fpath, colorspace=None)
        else:
            gt = None

        # Load in RGB for now, we will convert right before we center the data
        im = self.loader(im_fpath, colorspace='RGB')

        aux_channels = []
        if self.aux_keys:
            aux_paths = [self.inputs.aux_paths[k][index]
                         for k in self.aux_keys]
            aux_channel = np.dstack([
                self.loader(p, colorspace=None)
                for p in aux_paths
            ])
            aux_channels = [aux_channel]

        if self.augment:
            # Image augmentation must be done in RGB
            # Augment intensity independently
            im = self.im_augment(im)
            # Augment geometry consistently
            im, aux_channels, gt = self.rand_aff.sseg_warp(
                im, aux_channels, gt)

        im = util.convert_colorspace(im, src_space='RGB',
                                     dst_space=self.colorspace)

        # Do centering of inputs
        input_tuple = [im] + aux_channels
        input_tuple = self.center_inputs(input_tuple)

        return input_tuple, gt

    def from_tensor(self, im, gt=None):
        if len(im.shape) == 3:
            im = im.cpu().numpy().transpose(2, 0, 1)
        else:
            im = im.cpu().numpy().transpose(0, 2, 3, 1)
        if gt is not None:
            gt = gt.cpu().numpy()
        return im, gt

    def to_tensor(self, input_tuple, gt):
        # NHWC -> NCHW
        input_tuple = [im_loaders.image_to_float_tensor(data)
                       for data in input_tuple]
        if gt is None:
            gt_tensor = None
        else:
            gt_tensor = im_loaders.label_to_long_tensor(gt)
        return input_tuple, gt_tensor

    def __getitem__(self, index):
        """

        Ignore:
            >>> from clab.live.sseg_train import *
            >>> self = load_task_dataset('urban_mapper_3d')['train']
            >>> self.augment = True
            >>> index = 0
            >>> self.center_inputs = self._make_normalizer()
            >>> im, gt = self[0]
        """
        input_tuple, gt = self.load_inputs(index)
        input_tuple, gt_tensor = self.to_tensor(input_tuple, gt)

        data_tensor = torch.cat(input_tuple, dim=0)

        if self.with_gt:
            # print('gotitem: ' + str(data_tensor.shape))
            # print('gt_tensor: ' + str(gt_tensor.shape))
            return data_tensor, gt_tensor
        else:
            return data_tensor

    @property
    def n_channels(self):
        if self.aux_keys:
            return 3 + len(self.aux_keys)
        else:
            return 3

    @property
    def n_classes(self):
        return int(self.task.labels.max() + 1)

    @property
    def ignore_label(self):
        return self.task.ignore_label

    def class_weights(self):
        """
            >>> from clab.live.sseg_train import *
            >>> self = load_task_dataset('urban_mapper_3d')['train']
            >>> self.class_weights()
        """
        # Handle class weights
        print('prep class weights')
        gtstats = self.inputs.prepare_gtstats(self.task)
        gtstats = self.inputs.gtstats
        # Take class weights (ensure they are in the same order as labels)
        mfweight_dict = gtstats['mf_weight'].to_dict()
        class_weights = np.array(list(ub.take(mfweight_dict, self.task.classnames)))
        class_weights[self.task.ignore_labels] = 0
        # HACK
        # class_weights[0] = 1.0
        # class_weights[1] = 0.7
        print('class_weights = {!r}'.format(class_weights))
        print('class_names   = {!r}'.format(self.task.classnames))
        return class_weights

def train():
    import ubelt as ub
    root = ub.ensure_app_cache_dir('clab')
    train_dset = cifar.CIFAR100(root=root, download=True, train=True)
    test_dest = cifar.CIFAR100(root=root, download=True, train=False)

max_stars_repo_name: chengjianglong/clab

max_issues_repo_name: chengjianglong/clab

max_forks_repo_name: chengjianglong/clab


text:
from typing import Any, Callable, Final, Optional, Tuple

import cv2
import numpy as np
from PIL import ImageFont, ImageDraw, Image

from state import Face, State
from winter2021_recognition.amazon_rekognition import MaskStatus

class Window:
    def __init__(
        self,
        state: State,
        window_name: str,
        width: int,
        height: int,
        capture: Callable[[], Tuple[bool, np.ndarray]],
        on_lbutton_down: Optional[Callable[[
            int, int, Any, Any, ], None]] = None,
    ) -> None:
        self.state: Final = state
        self.window_name: Final = window_name
        self.width: Final = width
        self.height: Final = height
        self.capture: Final = capture
        self.on_lbutton_down: Final = on_lbutton_down
        self.stop_flag = False

    def set(self) -> None:
        cv2.namedWindow(self.window_name, cv2.WINDOW_NORMAL)
        cv2.resizeWindow(self.window_name, self.width, self.height)
        cv2.moveWindow(self.window_name, 0, 0)
        cv2.setWindowTitle(self.window_name, self.window_name)
        cv2.setMouseCallback(
            self.window_name, lambda *args: self.on_mouse_click(*args))

    def start(self) -> None:
        self.__draw()

    def close(self) -> None:
        self.stop_flage = True
        cv2.destroyWindow(self.window_name)

    def on_mouse_click(self, event, x, y, flags, params):
        if event == cv2.EVENT_LBUTTONDOWN and self.on_lbutton_down:
            self.on_lbutton_down(x, y, flags, params)

    def is_opened(self) -> bool:
        return cv2.getWindowProperty(self.window_name, 0) >= 0

    def __draw(self) -> None:
        while True:
            if self.stop_flag:
                raise Exception('stop flag set.')

            if not self.is_opened():
                raise Exception('window is not opened')

            _, img = self.capture()

            if not self.state.is_core_running:
                img = self.__show_help_text(img)
            else:
                img = self.__draw_faces(img)

            cv2.imshow(self.window_name, img)
            key = cv2.waitKey(10)

            if key == 27:  # Press ESC
                raise Exception('Press esc')

    def __show_help_text(self, img: np.ndarray) -> np.ndarray:
        text = 'Tap screen to start rekognition'
        font = cv2.FONT_HERSHEY_SIMPLEX
        text_size = cv2.getTextSize(text, font, 1, 6)[0]
        org = (int((img.shape[1] - text_size[0]) / 2),
               int((img.shape[0] + text_size[1]) / 2))
        cv2.putText(img, text, org, font, 1, (255, 255, 255), 6,)

        return img

    def __draw_faces(self, img: np.ndarray) -> np.ndarray:
        for face in self.state.faces:
            img = self.__draw_face(face, img)

        return img

    def __draw_face(self, face: Face, img: np.ndarray) -> np.ndarray:
        mask_color_dict = {
            MaskStatus.NOT_WEARED: (0, 0, 255),
            MaskStatus.UNKNOWN: (0, 0, 0),
            MaskStatus.WEARED: (0, 255, 0),
        }

        color = mask_color_dict[face.mask_status]
        cv2.rectangle(
            img,
            (face.box.left, face.box.top),
            (face.box.right, face.box.bottom),
            color,
            thickness=4,
        )

        mask_text_dict = {
            MaskStatus.NOT_WEARED: '마스크 미착용',
            MaskStatus.UNKNOWN: '',
            MaskStatus.WEARED: '마스크 착용'
        }

        mask_text = mask_text_dict[face.mask_status]

        text = ''

        if face.name:
            text += face.name

        if mask_text:
            text += f': {mask_text}'

        org = (face.box.left, face.box.bottom)

        pil_image = Image.fromarray(img)
        draw = ImageDraw.Draw(pil_image)
        font = ImageFont.truetype("fonts/gulim.ttc", 22)
        draw.text(org, text, font=font, fill=color)

        return np.array(pil_image)


text:
import argparse
import torch
import numpy as np
import os
import sys
import time
import math
import json

import logging
from datetime import datetime

from dataloader import DataLoader
from model import CaptionModel, CrossEntropyCriterion
from train import test

import utils
import opts

import requests

logger = logging.getLogger(__name__)

def progress_bar(some_iter):
    try:
        from tqdm import tqdm
        return tqdm(some_iter)
    except ModuleNotFoundError:
        return some_iter

def download_file_from_google_drive(file_id, destination):
    print("Trying to fetch {}".format(destination))

    def get_confirm_token(response):
        for key, value in response.cookies.items():
            if key.startswith('download_warning'):
                return value

        return None

    def save_response_content(response, destination):
        CHUNK_SIZE = 32768

        with open(destination, "wb") as f:
            for chunk in progress_bar(response.iter_content(CHUNK_SIZE)):
                if chunk: # filter out keep-alive new chunks
                    f.write(chunk)

    URL = "https://docs.google.com/uc?export=download"

    session = requests.Session()

    response = session.get(URL, params = { 'id' : file_id }, stream = True)
    token = get_confirm_token(response)

    if token:
        params = { 'id' : file_id, 'confirm' : token }
        response = session.get(URL, params = params, stream = True)

    save_response_content(response, destination)

ID_DICT = {'GL-RG_XE_msrvtt': '1xaAW-hUbOiXv5kdMxO-gCLAgl-wkre8q',
           'GL-RG_DXE_msrvtt': '1Jx1sCU2aQt0AA5-dRfRZallsCnbYB1Ud',
           'GL-RG_DR_msrvtt': '1x8Mh7HJuCmAWjwNExOR8MXqFCNyYttyJ',
           'GL-RG_XE_msvd': '1J4-I9bf2nB1_HlOLNq8aUpLfuSRvTlq3',
           'GL-RG_DXE_msvd': '1HixyH_LOT-3HtcsehQT_c9PAlPwFCTd-',
           'GL-RG_DR_msvd': '1cCisyMpp1mUS9NQPHSn4iCiVmqMfJeL5'}

if __name__ == '__main__':

    opt = opts.parse_opts()

    logging.basicConfig(level=getattr(logging, opt.loglevel.upper()),
                        format='%(asctime)s:%(levelname)s: %(message)s')

    logger.info('Input arguments: %s', json.dumps(vars(opt), sort_keys=True, indent=4))

    start = datetime.now()

    test_opt = {'label_h5': opt.test_label_h5,
                'batch_size': opt.test_batch_size,
                'feat_h5': opt.test_feat_h5,
                'cocofmt_file': opt.test_cocofmt_file,
                'seq_per_img': opt.test_seq_per_img,
                'num_chunks': opt.num_chunks,
                'use_resnet_feature': opt.use_resnet_feature,
                'use_c3d_feature': opt.use_c3d_feature,
                'use_audio_feature': opt.use_audio_feature,
                'use_sem_tag_feature': opt.use_sem_tag_feature,
                'use_long_range': opt.use_long_range,
                'use_short_range': opt.use_short_range,
                'use_local': opt.use_local,
                'mode': 'test'
                }

    test_loader = DataLoader(test_opt)

    if not os.path.exists(opt.model_file):
        logger.info('downloading model: %s', opt.model_file)
        model_name = opt.model_file.split('/')[-2]
        model_dir = os.path.dirname(opt.model_file)
        if not os.path.exists(model_dir):
            os.makedirs(model_dir)
        download_file_from_google_drive(ID_DICT[model_name], opt.model_file)

    logger.info('Loading model: %s', opt.model_file)
    checkpoint = torch.load(opt.model_file)
    checkpoint_opt = checkpoint['opt']

    opt.model_type = checkpoint_opt.model_type
    opt.vocab = checkpoint_opt.vocab
    opt.vocab_size = checkpoint_opt.vocab_size
    opt.seq_length = checkpoint_opt.seq_length
    opt.feat_dims = checkpoint_opt.feat_dims

    assert opt.seq_length == test_loader.get_seq_length()
    assert opt.feat_dims == test_loader.get_feat_dims()

    logger.info('Building model...')
    model = CaptionModel(opt)
    logger.info('Loading state from the checkpoint...')
    model.load_state_dict(checkpoint['model'])

    xe_criterion = CrossEntropyCriterion()

    if torch.cuda.is_available():
        model.cuda()
        xe_criterion.cuda()

    logger.info('Start testing...')
    test(model, xe_criterion, test_loader, opt)
    logger.info('Time: %s', datetime.now() - start)


text:
import argparse
import os
import sys

import torch
import numpy as np
import random
from torch.optim.lr_scheduler import LambdaLR

from dataloader import dataloader
from model.config import cfg
from model.model import GSCAN_model
from model.utils import *

def train(
    train_data_path: str,
    val_data_paths: dict, use_cuda: bool,
    model_name: str, is_baseline: bool,
    resume_from_file=None,
    is_wandb=False,
):
    logger.info("Loading Training set...")
    logger.info(model_name)
    train_iter, train_input_vocab, train_target_vocab = dataloader(train_data_path,
                                                                   batch_size=cfg.TRAIN.BATCH_SIZE,
                                                                   use_cuda=use_cuda)
    val_iters = {}
    for split_name, path in val_data_paths.items():
        val_iters[split_name], _, _ = dataloader(path, batch_size=cfg.VAL_BATCH_SIZE, use_cuda=use_cuda,
                                                 input_vocab=train_input_vocab, target_vocab=train_target_vocab)

    pad_idx, sos_idx, eos_idx = train_target_vocab.stoi['<pad>'], train_target_vocab.stoi['<sos>'], \
                                train_target_vocab.stoi['<eos>']

    train_input_vocab_size, train_target_vocab_size = len(train_input_vocab.itos), len(train_target_vocab.itos)

    '''
    Input (command) [0]: batch_size x max_cmd_len       [1]: batch_size x 0 (len for each cmd)
    Situation: batch_size x grid x grid x feat_size
    Target (action) [0]: batch_size x max_action_len    [1]: batch_size x 0 (len for each action sequence)

    max_cmd_len = 6, max_action_len = 16
    '''
    logger.info("Done Loading Training set.")

    # if generate_vocabularies:
    #     training_set.save_vocabularies(input_vocab_path, target_vocab_path)
    #     logger.info("Saved vocabularies to {} for input and {} for target.".format(input_vocab_path, target_vocab_path))

    logger.info("Loading Dev. set...")

    # val_input_vocab_size, val_target_vocab_size = train_input_vocab_size, train_target_vocab_size

    # Shuffle the test set to make sure that if we only evaluate max_testing_examples we get a random part of the set.

    # val_set.shuffle_data()
    logger.info("Done Loading Dev. set.")

    model = GSCAN_model(pad_idx, eos_idx, train_input_vocab_size, train_target_vocab_size, is_baseline=is_baseline,
                        output_directory=os.path.join(os.getcwd(), cfg.OUTPUT_DIRECTORY, model_name), use_cuda=use_cuda)

    model = model.cuda() if use_cuda else model

    log_parameters(model)
    trainable_parameters = [parameter for parameter in model.parameters() if parameter.requires_grad]
    optimizer = torch.optim.Adam(trainable_parameters, lr=cfg.TRAIN.SOLVER.LR,
                                 betas=(cfg.TRAIN.SOLVER.ADAM_BETA1, cfg.TRAIN.SOLVER.ADAM_BETA2))
    scheduler = LambdaLR(optimizer,
                         lr_lambda=lambda t: cfg.TRAIN.SOLVER.LR_DECAY ** (t / cfg.TRAIN.SOLVER.LR_DECAY_STEP))

    start_iteration = 1
    best_exact_match = 0

    if resume_from_file:
        assert os.path.isfile(resume_from_file), "No checkpoint found at {}".format(resume_from_file)
        logger.info("Loading checkpoint from file at '{}'".format(resume_from_file))
        optimizer_state_dict = model.load_model(resume_from_file)
        optimizer.load_state_dict(optimizer_state_dict)
        start_iteration = model.trained_iterations
        logger.info("Loaded checkpoint '{}' (iter {})".format(resume_from_file, start_iteration))

    logger.info("Training starts..")
    training_iteration = start_iteration
    while training_iteration < cfg.TRAIN.MAX_EPOCH:  # iterations here actually means "epoch"

        # Shuffle the dataset and loop over it.
        # training_set.shuffle_data()
        num_batch = 0
        for x in train_iter:
            is_best = False
            model.train()
            target_scores, target_position_scores = model(x.input, x.situation,
                                                          x.target)

            loss = model.get_loss(target_scores, x.target[0])

            target_loss = 0
            if cfg.AUXILIARY_TASK:
                target_loss = model.get_auxiliary_loss(target_position_scores,
                                                       x.target)
            loss += cfg.TRAIN.WEIGHT_TARGET_LOSS * target_loss

            # Backward pass and update model parameters.
            loss.backward()
            optimizer.step()
            scheduler.step()
            optimizer.zero_grad()
            model.update_state(is_best=is_best)

            # Print current metrics.
            if num_batch % cfg.PRINT_EVERY == 0:
                accuracy, exact_match = model.get_metrics(target_scores, x.target[0])
                if cfg.AUXILIARY_TASK:
                    auxiliary_accuracy_target = model.get_auxiliary_accuracy(target_position_scores,
                                                                             x.target)
                else:
                    auxiliary_accuracy_target = 0.
                learning_rate = scheduler.get_lr()[0]
                logger.info("Iteration %08d, batch_num %03d, loss %8.4f, accuracy %5.2f, exact match %5.2f, learning_rate %.5f,"
                            " aux. accuracy target pos %5.2f" % (training_iteration, num_batch, loss, accuracy, exact_match,
                                                                 learning_rate, auxiliary_accuracy_target))
                if is_wandb:
                    wandb.log({'train/training_iteration': training_iteration})
                    wandb.log({'train/task_loss': loss})
                    wandb.log({'train/task_accuracy': accuracy})
                    wandb.log({'train/task_exact_match': exact_match})
                    wandb.log({'train/learning_rate': learning_rate})

            num_batch += 1

        if training_iteration % cfg.EVALUATE_EVERY == 0:
            with torch.no_grad():
                model.eval()
                logger.info("Evaluating..")
                test_exact_match = 0
                test_accuracy = 0
                try:
                    for split_name, val_iter in val_iters.items():
                        accuracy, exact_match, target_accuracy = evaluate(
                            val_iter, model=model,
                            max_decoding_steps=30, pad_idx=pad_idx,
                            sos_idx=sos_idx,
                            eos_idx=eos_idx,
                            max_examples_to_evaluate=None)
                        if split_name == 'dev':
                            test_exact_match = exact_match
                            test_accuracy = accuracy

                        logger.info(" %s Accuracy: %5.2f Exact Match: %5.2f "
                                    " Target Accuracy: %5.2f " % (split_name, accuracy, exact_match, target_accuracy))
                        if is_wandb:
                            wandb.log({'eval/accuracy': accuracy})
                            wandb.log({'eval/exact_match': exact_match})
                except:
                    print("Exception!")

                if test_exact_match > best_exact_match:
                    is_best = True
                    best_accuracy = test_accuracy
                    best_exact_match = test_exact_match
                    model.update_state(accuracy=test_accuracy, exact_match=test_exact_match, is_best=is_best)
                file_name = model_name + "checkpoint.{}th.tar".format(str(training_iteration))
                # file_name = os.path.join(os.getcwd(), cfg.OUTPUT_DIRECTORY, model_name, file_name)
                if is_best:
                    logger.info("saving best model...")
                    model.save_checkpoint(file_name=file_name, is_best=is_best,
                                          optimizer_state_dict=optimizer.state_dict())

        if training_iteration % cfg.SAVE_EVERY == 0:
            logger.info("forcing to save model every several epochs...")
            file_name = model_name + " checkpoint_force.{}th.tar".format(str(training_iteration))
            # file_name = os.path.join(os.getcwd(), cfg.OUTPUT_DIRECTORY, model_name, file_name)
            model.save_checkpoint(file_name=file_name, is_best=False, optimizer_state_dict=optimizer.state_dict())

        training_iteration += 1  # warning: iteratin represents epochs here
    logger.info("Finished training.")

def main(flags, use_cuda):
    if not os.path.exists(os.path.join(cfg.OUTPUT_DIRECTORY, flags.run)):
        os.mkdir(os.path.join(os.getcwd(), cfg.OUTPUT_DIRECTORY, flags.run))

    # Some checks on the flags
    if cfg.GENERATE_VOCABULARIES:
        assert cfg.INPUT_VOCAB_PATH and cfg.TARGET_VOCAB_PATH, "Please specify paths to vocabularies to save."

    train_data_path = os.path.join(flags.data_dir, "train.json")

    test_splits = [
        'test',
        'dev',
    ]
    val_data_paths = {split_name: os.path.join(flags.data_dir, split_name + '.json') for split_name in test_splits}

    if cfg.MODE == "train":
        if flags.is_baseline:
            logger.info("Running baseline + embedding...")
        else:
            logger.info("Running full model...")
        train(
            train_data_path=train_data_path,
            val_data_paths=val_data_paths,
            use_cuda=use_cuda,
            model_name=flags.run,
            resume_from_file=flags.load,
            is_baseline=flags.is_baseline,
            is_wandb=flags.is_wandb,
        )

    elif cfg.MODE == "predict":
        raise NotImplementedError()

    else:
        raise ValueError("Wrong value for parameters --mode ({}).".format(cfg.MODE))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="LGCN models for GSCAN")
    parser.add_argument('--run', type=str, help='Define the run name')
    parser.add_argument('--txt', dest='redirect_output', action='store_true')
    parser.add_argument('--baseline', dest='is_baseline', action='store_true')
    parser.add_argument('--load', type=str, help='Path to model')
    parser.add_argument('--data_dir', type=str, help='Path to dataset')
    parser.add_argument('--seed', type=int, help='random seeds')
    parser.add_argument("--is_wandb", dest="is_wandb", default=False,
                        action="store_true", help="Whether to report metrics to weights and bias.")
    parser.add_argument("--use_cuda", dest="use_cuda", default=False,
                        action="store_true", help="Whether to use cuda if available.")
    parser.set_defaults(redirect_output=False, is_baseline=False)
    args = parser.parse_args()
    FORMAT = "%(asctime)-15s %(message)s"

    random.seed(args.seed)
    torch.manual_seed(args.seed)
    np.random.seed(args.seed)

    # reconstruct the run name to be consistent with other models.
    dataset_name = args.data_dir.strip("/").split("/")[-1]
    run_name = f"gcnlstm_{dataset_name}_seed_{args.seed}_lr_{cfg.TRAIN.SOLVER.LR}"
    args.run = run_name
    if args.is_wandb:
        logger.warning("Enabling wandb for tensorboard logging...")
        import wandb
        run = wandb.init(
            project="ReaSCAN-Causal",
            entity="wuzhengx",
            name=args.run,
        )
        wandb.config.update(args)

    if args.redirect_output:
        output_file = open(os.path.join('exp/', args.run + '.txt'), 'w')
        sys.stdout = output_file
        sys.stderr = sys.stdout
        logging.basicConfig(format=FORMAT, level=logging.DEBUG,
                            datefmt="%Y-%m-%d %H:%M", filename=os.path.join('exp/', args.run + '.txt'))
    else:
        logging.basicConfig(format=FORMAT, level=logging.DEBUG,
                            datefmt="%Y-%m-%d %H:%M")

    logger = logging.getLogger(__name__)
    use_cuda = args.use_cuda
    logger.info("Initialize logger")

    if use_cuda:
        logger.info("Using CUDA.")
        logger.info("Cuda version: {}".format(torch.version.cuda))

    logger.info("Training arguments:")
    logger.info(args)

    main(args, use_cuda)

max_stars_repo_name: frankaging/ReaSCAN-Causal

max_issues_repo_name: frankaging/ReaSCAN-Causal

max_forks_repo_name: frankaging/ReaSCAN-Causal


text:
# -*- coding: utf-8 -*-
"""
Functions and classes to enable bruteforce solution of the TSP.
Note Bruteforce is inefficient after tours exceed 5 cities
"""

import itertools as ite
import numpy as np

class BruteForceSolver(object):
    """
    Enumerates all permutations of a given list of cities and calculates
    waits. Note for n cities there are n! permutations!  BruteForceSolver is
    too slow for tours beyond 5 cities.
    """
    def __init__(self, init_solution, objective, maximisation=False):
        """
        Constructor Method
        @init_solution = initial tour
        @matrix = matrix of travel costs
        """
        self._objective = objective
        self._init_solution = init_solution

        if maximisation:
            self._negate = 1.0
        elif not maximisation:
            self._negate = -1.0
        else:
            raise ValueError('parameter maximisation must be bool True|False')

        #list of best solutions in case of multiple optima
        self.best_solutions = [init_solution]
        self._best_cost = self._objective.evaluate(init_solution) * self._negate

    def _get_best_cost(self):
        return self._best_cost * self._negate

    def solve(self):
        """
        Enumerate all costs to find the minimum.
        Store solution(s)
        """
        #perms = self._all_permutations(self._init_solution)

        #print(p) for p in perms]  #uncomment if want to see all perms

        #better to pick a random start city and then permute
        #the rest and rejoin.
        for current_tour in ite.permutations(self._init_solution):
            cost = self._objective.evaluate(current_tour) * self._negate

            if self._best_cost == cost:
                self._best_cost = cost
                self.best_solutions.append(current_tour)

            elif cost > self._best_cost:
                self._best_cost = cost
                self.best_solutions = [current_tour]

    def _all_permutations(self, tour):
        """
        Returns a list of lists containing all permutations of a
        tour.  The base_city is appended to each_list
        """
        return [np.array(x) for x in ite.permutations(tour)]

    best_cost = property(_get_best_cost)

class RandomSearch(object):
    """
    A simple global optimisation algorithm - encapsulates Random Search.
    The algorithm is completely explorative and randomly
    samples a tour and compares if it is better than the current
    best.
    """
    def __init__(self, init_solution, objective, max_iter=1000, maximisation=True):
        """
        Constructor Method

        Parameters:
        ---------
        init_solution -- initial tour
        matrix -- matrix of travel costs
        """
        self._objective = objective
        self._init_solution = init_solution
        self._max_iter = max_iter

        if maximisation:
            self._negate = 1.0
        elif not maximisation:
            self._negate = -1.0
        else:
            raise ValueError('parameter maximisation must be bool True|False')

        #list of best solutions in case of multiple optima
        self.best_solutions = [init_solution]
        self._best_cost = self._objective.evaluate(init_solution) * self._negate

    def _get_best_cost(self):
        return self._best_cost * self._negate

    def solve(self):
        '''
        Random search.

        Loop until all iterations are complete.
        Sample a random tour on each iterations and compare to best.

        '''

        for _ in range(self._max_iter):
            sample_tour = np.random.permutation(self._init_solution)
            sample_cost = self._objective.evaluate(sample_tour) * self._negate

            if self.best_cost == sample_cost:
                self.best_solutions.append(sample_tour)

            elif sample_cost > self._best_cost:
                self.best_solutions = [sample_tour]
                self._best_cost = sample_cost

    best_cost = property(_get_best_cost)


text:
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision
from torchvision import transforms
from torchvision.io import read_image, ImageReadMode
from torch.utils.data import Dataset, DataLoader

import cv2
import os
import time
from copy import deepcopy
import matplotlib.pyplot as plt
import os
from PIL import Image
from .data_utils import *

'''
@todo customized kitti transform
      more augmentation
      quantization
'''
class KITTI_SEM(Dataset):
    '''
    @note assume that label image's name and corresponding rgb image's name are the same
    just in different folders. e.g. labels/1.jpg <---> rgbs/1.jpg
    '''
    rgb_preprocess = transforms.Compose([
        transforms.ToTensor(),
        transforms.Resize( (200, 640) ),
        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
    ])
    label_preprocess = transforms.Compose([transforms.Resize( (200, 640) )])

    def __init__(self, rgb_dir_paths, label_dir_paths, transform=None, target_transform=None,
                 device=torch.device("cuda" if torch.cuda.is_available() else "cpu"), original=False):
        self.rgb_dir_paths = rgb_dir_paths
        self.label_dir_paths = label_dir_paths

        self.data_pair_paths = []
        for rgb_dir_path, label_dir_path in zip(rgb_dir_paths, label_dir_paths):
            rgb_data_names = sorted(os.listdir(rgb_dir_path))
            label_data_names = sorted(os.listdir(label_dir_path))
            for rgb_data_name, label_data_name  in zip(rgb_data_names, label_data_names):
                img_path = os.path.join(rgb_dir_path, rgb_data_name)
                label_path = os.path.join(label_dir_path, label_data_name)
                self.data_pair_paths.append((img_path, label_path))

        self.transform = transform if transform != None else self.rgb_preprocess
        self.target_transform = target_transform if target_transform != None else self.label_preprocess

        self.device = device
        self.qmark = False
        self.original = original

    def __len__(self):
        return len(self.data_pair_paths)

    def __getitem__(self, idx):
        img_path, label_path = self.data_pair_paths[idx]

        img = Image.open(img_path)
        label = read_image(label_path, ImageReadMode.GRAY)

        img = self.transform(img)
        label = self.target_transform(label)

        img = img.to(self.device)
        label = label.squeeze().to(torch.long).to(self.device)

        if self.original:
            original_img = cv2.imread(img_path)
            original_img = cv2.resize(original_img, (1241, 376))
            original_label = cv2.imread(label_path, cv2.IMREAD_GRAYSCALE)
            original_label = cv2.resize(original_label, (1241, 376))
        else:
            original_img = torch.empty(1)
            original_label = torch.empty(1)

        data = {}
        data['rgb'] = img
        data['label'] = label
        data['original_rgb'] = original_img
        data['original_label'] = original_label
        data['rgb_path'] = img_path
        data['label_path'] = label_path

        return data

    def example(self, idx=10):
        data = self.__getitem__(idx)
        image = data['rgb']
        label = data['label']
        original_image = data['original_rgb']
        original_label = data['original_label']

        image = image.permute(1, 2, 0).to("cpu").to(torch.long)
        label = label.to("cpu").numpy()

        plt.figure(figsize=(20, 10))
        plt.imshow(image)
        plt.savefig(f"example image {idx}.png")

        plt.figure(figsize=(20, 10))
        plt.imshow(label)
        plt.savefig(f"example original image {idx}.png")

        plt.figure(figsize=(20, 10))
        plt.imshow(original_image)
        plt.savefig(f"example original image {idx}.png")

        plt.figure(figsize=(20, 10))
        plt.imshow(original_label)
        plt.savefig(f"example original label {idx}.png")

class KITTI_DEP(Dataset):
    '''
    @note assume that label image's name and corresponding rgb image's name are the same
    just in different folders. e.g. labels/1.jpg <---> rgbs/1.jpg
    '''
    rgb_preprocess = transforms.Compose([
        transforms.ToTensor(),
        transforms.Resize( (200, 640) ),
        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
    ])
    label_preprocess = transforms.Compose([#transforms.ToTensor(),
                                           transforms.GaussianBlur(15, sigma=4.0),
                                           transforms.Resize( (200, 640) )])

    def __init__(self, rgb_dir_paths, label_dir_paths, transform=None, target_transform=None,
                 device=torch.device("cuda" if torch.cuda.is_available() else "cpu"), original=False):
        self.rgb_dir_paths = rgb_dir_paths
        self.label_dir_paths = label_dir_paths

        self.data_pair_paths = []
        for rgb_dir_path, label_dir_path in zip(rgb_dir_paths, label_dir_paths):
            rgb_data_names = sorted(os.listdir(rgb_dir_path))
            label_data_names = sorted(os.listdir(label_dir_path))
            for rgb_data_name, label_data_name  in zip(rgb_data_names, label_data_names):
                img_path = os.path.join(rgb_dir_path, rgb_data_name)
                label_path = os.path.join(label_dir_path, label_data_name)
                self.data_pair_paths.append((img_path, label_path))

        self.transform = transform if transform != None else self.rgb_preprocess
        self.target_transform = target_transform if target_transform != None else self.label_preprocess

        self.device = device
        self.qmark = False
        self.original = original

    def __len__(self):
        return len(self.data_pair_paths)

    def __getitem__(self, idx):
        img_path, label_path = self.data_pair_paths[idx]

        img = Image.open(img_path)
        label = cv2.imread(label_path)
        label = torch.tensor(label[:,:,0]).unsqueeze(0)

        img = self.transform(img)
        label = self.target_transform(label)

        img = img.to(self.device)
        label = label.to(self.device).to(torch.float32)

        if self.original:
            original_img = cv2.imread(img_path)
            original_label = cv2.imread(label_path)[:,:,0]
        else:
            original_img = torch.empty(1)
            original_label = torch.empty(1)

        data = {}
        data['rgb'] = img
        data['label'] = label
        data['original_rgb'] = original_img
        data['original_label'] = original_label
        data['rgb_path'] = img_path
        data['label_path'] = label_path

        return data

    def example(self, idx=10):
        data = self.__getitem__(idx)
        image = data['rgb']
        label = data['label']
        original_image = data['original_rgb']
        original_label = data['original_label']

        image = image.permute(1, 2, 0).to("cpu").to(torch.uint8)
        label = label.permute(1, 2, 0).to("cpu").to(torch.uint8).squeeze(2).numpy()

        plt.figure(figsize=(20, 10))
        plt.imshow(image)
        plt.savefig(f"example image {idx}.png")

        plot_depth_map(label, np.ones_like(label), ".")

        plt.figure(figsize=(20, 10))
        plt.imshow(original_image)
        plt.savefig(f"example original image {idx}.png")

        h = depth2Heatmap(original_label)
        plt.figure(figsize=(20, 10))
        plt.imshow(h)
        plt.savefig(f"example original label {idx}.png")