text:
"""
This script is for running compression of convolutional networks using tucker decomposition.

Usage:
    script.py tucker [-h] [-v|-vv] [--lr-configuration-file path] [--seed int] [--only-dense] [--rank-percentage-dense float] [--batch-size int] [--train-val-split float] [--rank-percentage-conv float] [--rank-percentage float] [--keep-first-layer] [--keep-last-layer] [--lr float] [--nb-epoch int] [--use-clr] [--min-lr float] [--max-lr float] [--epoch-step-size int] (--mnist|--svhn|--cifar10|--cifar100|--test-data) [--cifar100-resnet50-new|--cifar100-resnet20-new|--cifar100-resnet50|--cifar100-resnet20|--mnist-500|--mnist-lenet|--test-model|--cifar10-vgg19|--cifar100-vgg19|--svhn-vgg19]
    script.py tensortrain [-h] [-v|-vv] [--lr-configuration-file path] [--seed int] [--rank0-1-conv] [--only-dense] [--batch-size int] [--use-pretrained] [--train-val-split float] [--rank-value int] [--order int] [--keep-first-layer] [--keep-last-layer] [--lr float] [--nb-epoch int] [--use-clr] [--min-lr float] [--max-lr float] [--epoch-step-size int] (--mnist|--svhn|--cifar10|--cifar100|--test-data) [--cifar100-resnet50-new|--cifar100-resnet20-new|--cifar100-resnet50|--cifar100-resnet20|--mnist-500|--mnist-lenet|--test-model|--cifar10-vgg19|--cifar100-vgg19|--svhn-vgg19]
    script.py deepfried [-h] [-v|-vv] [--lr-configuration-file path] [--seed int] [--only-dense] [--keep-last-layer] [--batch-size int] [--train-val-split float] [--keep-first-layer] [--nb-stack int] [--lr float] [--nb-epoch int] [--use-clr] [--min-lr float] [--max-lr float] [--epoch-step-size int] (--mnist|--svhn|--cifar10|--cifar100|--test-data) [--cifar100-resnet50-new|--cifar100-resnet20-new|--cifar100-resnet50|--cifar100-resnet20|--mnist-500|--mnist-lenet|--test-model|--cifar10-vgg19|--cifar100-vgg19|--svhn-vgg19]
    script.py magnitude [-h] [-v|-vv] [--lr-configuration-file path] [--seed int] --final-sparsity float [--only-dense] [--batch-size int] [--train-val-split float] [--keep-last-layer] [--keep-first-layer] [--lr float] [--nb-epoch int] [--use-clr] [--min-lr float] [--max-lr float] [--epoch-step-size int] (--mnist|--svhn|--cifar10|--cifar100|--test-data) [--cifar100-resnet50-new|--cifar100-resnet20-new|--cifar100-resnet50|--cifar100-resnet20|--mnist-500|--mnist-lenet|--test-model|--cifar10-vgg19|--cifar100-vgg19|--svhn-vgg19]
    script.py magnitude_hard [-h] [-v|-vv] [--lr-configuration-file path] [--seed int] --final-sparsity float [--only-dense] [--batch-size int] [--train-val-split float] [--keep-last-layer] [--keep-first-layer] [--lr float] [--nb-epoch int] [--use-clr] [--min-lr float] [--max-lr float] [--epoch-step-size int] (--mnist|--svhn|--cifar10|--cifar100|--test-data) [--cifar100-resnet50-new|--cifar100-resnet20-new|--cifar100-resnet50|--cifar100-resnet20|--mnist-500|--mnist-lenet|--test-model|--cifar10-vgg19|--cifar100-vgg19|--svhn-vgg19]
    script.py random [-h] [-v|-vv] [--lr-configuration-file path] [--seed int] --sparsity-factor int [--nb-factor int] [--batch-size int] [--train-val-split float] [--only-dense] [--keep-last-layer] [--keep-first-layer] [--lr float] [--nb-epoch int] [--use-clr] [--min-lr float] [--max-lr float] [--epoch-step-size int] (--mnist|--svhn|--cifar10|--cifar100|--test-data) [--cifar100-resnet50-new|--cifar100-resnet20-new|--cifar100-resnet50|--cifar100-resnet20|--mnist-500|--mnist-lenet|--test-model|--cifar10-vgg19|--cifar100-vgg19|--svhn-vgg19]
    script.py dummy [-h] [-v|-vv] [--lr-configuration-file path] [--seed int]  [--batch-size int] [--train-val-split float] [--only-dense] [--keep-last-layer] [--keep-first-layer] [--lr float] [--nb-epoch int] [--use-clr] [--min-lr float] [--max-lr float] [--epoch-step-size int] (--mnist|--svhn|--cifar10|--cifar100|--test-data) [--cifar100-resnet50-new|--cifar100-resnet20-new|--cifar100-resnet50|--cifar100-resnet20|--mnist-500|--mnist-lenet|--test-model|--cifar10-vgg19|--cifar100-vgg19|--svhn-vgg19]

Options:
  -h --help                             Show this screen.
  -vv                                   Set verbosity to debug.
  -v                                    Set verbosity to info.
  --train-val-split float               Tells the proportion of validation data. If not specified, validation data is test data.
  --seed int                            Tell the seed to use in the experiment.
  --lr-configuration-file path          Tell the path to the lr configuration file

Dataset:
  --mnist                               Use Mnist dataset.
  --svhn                                Use svhn dataset.
  --cifar10                             Use cifar10 dataset.
  --cifar100                            Use cifar100 dataset.
  --test-data                           Use test datasset (that is actually mnist).

Model:
  --mnist-lenet                         Use model lenet pretrained for mnist.
  --test-model                          Use test, small, model.
  --cifar10-vgg19                       Use model vgg19 pretrained on cifar10.
  --cifar100-vgg19                      Use model vgg19 pretrained on cifar100.
  --svhn-vgg19                          Use model vgg19 pretrained on svhn.
  --mnist-500                           Use model fc 500 hidden units pretrained on mnist.
  --cifar100-resnet50                   Use model resnet50 pretrained on cifar100.
  --cifar100-resnet50-new               Use model resnet50 pretrained on cifar100.
  --cifar100-resnet20                   Use model resnet20 pretrained on cifar100.
  --cifar100-resnet20-new               Use model resnet50 pretrained on cifar100.

Compression specific options:
    --keep-first-layer                  Tell the replacer to keep the first layer of the network
    --keep-last-layer                   Tell the replacer to keep the last layer (classification) of the network
    --only-dense                        Tell the replacer to replace only dense layers.

Tucker specific option:
    --rank-percentage-dense float       Tell tucker to replace Dense layers by low rank decomposition with rank equal percentage of base rank
    --rank-percentage-conv float        Tell tucker to use a percentage of in_channel and out_channel to determine the rank in convolution and not VBMF.
    --rank-percentage float             Tell tucker to use the same rank percentage for dense and conv

Tensortrain specific option:
    --rank-value int                    The values for r0, r1 r... rk (exemple: 2, 4, 6)
    --order int                         The value for k (number of cores)
    --use-pretrained                    Tell the layer replacer to use the decomposition of the initial weights.
    --rank0-1-conv                      Tell to use a r0 value equal to 1 in convolutional layers. For fair comparison in term of input size.

DeepFried specific option:
    --nb-stack int                      The values for r0, r1 r... rk (exemple: 2, 4, 6)

Magnitude pruning specific option:
    --final-sparsity float              The final sparsity ratio: proportion of zero parameters.

Random Sparse Facto options:
    --sparsity-factor int                 Integer coefficient from which is computed the number of value in each factor.
    --nb-factor int                       Tells the number of sparse factor for palm

Finetuning options:
    --lr float                          Overide learning rate for optimization
    --min-lr float                      Set min lr if use-clr is used learning rate for optimization
    --max-lr float                      Set max lr if use-clr is used learning rate for optimization
    --epoch-step-size int               Number of epoch before an half cycle.
    --use-clr                           Tells to use cyclical learning rate instead of standard learning rate, lr and clr can't be set together
    --nb-epoch int                      Overide the number of epochs
    --batch-size int                    Use specified batch size
"""
import logging

from palmnet.core.layer_replacer_dummy import LayerReplacerDummy

mpl_logger = logging.getLogger("matplotlib")
mpl_logger.setLevel(logging.WARNING)
from tensorflow import set_random_seed
import yaml
from sklearn.model_selection import train_test_split

import sys
import time
import numpy as np
import keras as base_keras
import docopt
import os
from collections import defaultdict
import pandas as pd
from pathlib import Path
from keras.models import Model
from keras.layers import Conv2D, Dense
import zlib
import tensorflow as tf
from tensorflow_model_optimization.python.core.sparsity.keras.pruning_wrapper import PruneLowMagnitude

from palmnet.core.layer_replacer_TT import LayerReplacerTT
from palmnet.core.layer_replacer_deepfried import LayerReplacerDeepFried
from palmnet.core.layer_replacer_magnitude_pruning import LayerReplacerMagnitudePruning
from palmnet.core.layer_replacer_random_sparse_facto import LayerReplacerRandomSparseFacto
from palmnet.core.layer_replacer_tucker import LayerReplacerTucker
from palmnet.core.randomizer import Randomizer
from palmnet.data import Mnist, Cifar10, Cifar100, Svhn, Test, categorical_cross_entropy_from_logits
from palmnet.experiments.utils import ResultPrinter, ParameterManager
from palmnet.layers.fastfood_layer_conv import FastFoodLayerConv
from palmnet.layers.fastfood_layer_dense import FastFoodLayerDense
from palmnet.layers.low_rank_dense_layer import LowRankDense
from palmnet.layers.sparse_facto_conv2D_masked import SparseFactorisationConv2D
from palmnet.layers.sparse_facto_dense_masked import SparseFactorisationDense
from palmnet.layers.tt_layer_conv import TTLayerConv
from palmnet.layers.tt_layer_dense import TTLayerDense
from palmnet.layers.tucker_layer import TuckerLayerConv
from palmnet.utils import CyclicLR, CSVLoggerByBatch, get_nb_learnable_weights_from_model, get_nb_learnable_weights, SafeModelCheckpoint, translate_keras_to_tf_model, DummyWith, load_function_lr
from palmnet.layers.sparse_conv2D_masked import SparseConv2D
from palmnet.layers.sparse_dense_masked import SparseDense
from skluc.utils import logger, log_memory_usage
import keras.backend as K
from tensorflow_model_optimization.sparsity import keras as sparsity
# from tensorflow_model_optimization.python.core.api.sparsity import keras

lst_results_header = [
    "decomposition_time",
    "test_accuracy_base_model",
    "test_accuracy_compressed_model",
    "test_accuracy_finetuned_model",
    "test_loss_base_model",
    "test_loss_compressed_model",
    "test_loss_finetuned_model",
    "val_accuracy_base_model",
    "val_accuracy_compressed_model",
    "val_accuracy_finetuned_model",
    "val_loss_base_model",
    "val_loss_compressed_model",
    "val_loss_finetuned_model",
    "train_accuracy_base_model",
    "train_accuracy_compressed_model",
    "train_accuracy_finetuned_model",
    "train_loss_base_model",
    "train_loss_compressed_model",
    "train_loss_finetuned_model",
    "base_model_nb_param",
    "new_model_nb_param",
    "actual-lr",
    "actual-batch-size",
    "actual-nb-epochs",
    "actual-min-lr",
    "actual-max-lr",
]

class ParameterManagerTensotrainAndTuckerDecomposition(ParameterManager):
    def __init__(self, dct_params, **kwargs):
        super().__init__(self, **dct_params, **kwargs)
        self.__init_seed()

        self["--min-lr"] = float(self["--min-lr"]) if self["--min-lr"] is not None else None
        self["--max-lr"] = float(self["--max-lr"]) if self["--max-lr"] is not None else None
        self["--lr"] = float(self["--lr"]) if self["--lr"] is not None else None
        self["--nb-epoch"] = int(self["--nb-epoch"]) if self["--nb-epoch"] is not None else None
        self["--epoch-step-size"] = int(self["--epoch-step-size"]) if self["--epoch-step-size"] is not None else None

        # tensortrain parameters
        self["--rank-value"] = int(self["--rank-value"]) if self["--rank-value"] is not None else None
        self["--order"] = int(self["--order"]) if self["--order"] is not None else None

        # tucker parameters
        self["--rank-percentage-dense"] = float(self["--rank-percentage-dense"]) if self["--rank-percentage-dense"] is not None else None
        self["--rank-percentage-conv"] = float(self["--rank-percentage-conv"]) if self["--rank-percentage-conv"] is not None else None
        self["--rank-percentage"] = float(self["--rank-percentage"]) if self["--rank-percentage"] is not None else None

        assert (self["--rank-percentage-conv"] is None and self["--rank-percentage-dense"] is None) or self["--rank-percentage"] is None, "--rank-percentage and --rank-percentage-conv or dense can't be set together"

        if self["--rank-percentage"] is not None:
            self["actual-rank-percentage-dense"] = self["--rank-percentage"]
            self["actual-rank-percentage-conv"] = self["--rank-percentage"]
        else:
            self["actual-rank-percentage-dense"] = self["--rank-percentage-dense"]
            self["actual-rank-percentage-conv"] = self["--rank-percentage-conv"]

        self["--nb-stack"] = int(self["--nb-stack"]) if self["--nb-stack"] is not None else None

        self["--final-sparsity"] = float(self["--final-sparsity"]) if self["--final-sparsity"] is not None else None

        self["--sparsity-factor"] = int(self["--sparsity-factor"]) if self["--sparsity-factor"] is not None else None
        self["--nb-factor"] = int(self["--nb-factor"]) if self["--nb-factor"] is not None else None

        if "--train-val-split" in self.keys() and self["--train-val-split"] is not None:
            self["--train-val-split"] = float(self["--train-val-split"]) if self["--train-val-split"] is not None else None
            assert 0 <= self["--train-val-split"] <= 1, f"Train-val split should be comprise between 0 and 1. {self['--train-val-split']}"

        self["--batch-size"] = int(self["--batch-size"]) if self["--batch-size"] is not None else None

        self["--lr-configuration-file"] = Path(self["--lr-configuration-file"]) if self["--lr-configuration-file"] is not None else None

        self.__init_hash_expe()
        self.__init_output_file()

    def __init_output_file(self):
        self["output_file_resprinter"] = Path(self["hash"] + "_results.csv")
        self["output_file_modelprinter"] = Path(self["hash"] + "_model.h5")
        self["output_file_notfinishedprinter"] = Path(self["hash"] + ".notfinished")
        self["output_file_csvcbprinter"] = Path(self["hash"] + "_history.csv")
        self["output_file_layerbylayer"] = Path(self["hash"] + "_layerbylayer.csv")

    def __init_seed(self):
        if not "--seed" in self.keys():
            self["seed"] = np.random.randint(0, 2 ** 32 - 2)

        if self["--seed"] is not None:
            self["seed"] = int(self["--seed"])
            np.random.seed(self["seed"])
        else:
            self["seed"] = np.random.randint(0, 2 ** 32 - 2)

    def __init_hash_expe(self):
        lst_elem_to_remove_for_hash = [
            'identifier',
            '-v',
            '--help',
            "output_file_resprinter",
            "output_file_modelprinter",
            "output_file_notfinishedprinter",
            "output_file_csvcbprinter",
            "output_file_layerbylayer",
        ]

        keys_expe = sorted(self.keys())
        any(keys_expe.remove(item) for item in lst_elem_to_remove_for_hash if item in keys_expe)
        val_expe = [self[k] for k in keys_expe]
        str_expe = [str(val) for pair in zip(keys_expe, val_expe) for val in pair]
        self["hash"] = hex(zlib.crc32(str.encode("".join(str_expe))))

def get_params_optimizer():

    if paraman["--mnist-lenet"]:
        param_train_dataset = Mnist.get_model_param_training()

        str_data_param = "mnist"
        str_model_param = "lenet"
    elif paraman["--mnist-500"]:
        param_train_dataset = Mnist.get_model_param_training("mnist_500")
        str_data_param = ""
        str_model_param = ""
    elif paraman["--cifar10-vgg19"]:
        param_train_dataset = Cifar10.get_model_param_training()
        str_data_param = "cifar10"
        str_model_param = "vgg19"
    elif paraman["--cifar100-vgg19"]:
        param_train_dataset = Cifar100.get_model_param_training()
        str_data_param = "cifar100"
        str_model_param = "vgg19"
    elif paraman["--cifar100-resnet20"]:
        param_train_dataset = Cifar100.get_model_param_training("cifar100_resnet")
        str_data_param = "cifar100"
        str_model_param = "resnet20"
    elif paraman["--cifar100-resnet50"]:
        param_train_dataset = Cifar100.get_model_param_training("cifar100_resnet")
        str_data_param = "cifar100"
        str_model_param = "resnet50"
    elif paraman["--cifar100-resnet20-new"]:
        param_train_dataset = Cifar100.get_model_param_training("cifar100_resnet")
        str_data_param = "cifar100"
        str_model_param = "resnet20"
    elif paraman["--cifar100-resnet50-new"]:
        param_train_dataset = Cifar100.get_model_param_training("cifar100_resnet")
        str_data_param = "cifar100"
        str_model_param = "resnet50"
    elif paraman["--svhn-vgg19"]:
        param_train_dataset = Svhn.get_model_param_training()
        str_data_param = "svhn"
        str_model_param = "vgg19"
    elif paraman["--test-model"]:
        param_train_dataset = Test.get_model_param_training()
        str_data_param = ""
        str_model_param = ""
    else:
        raise NotImplementedError("No dataset specified.")

    if paraman["tucker"]:
        str_method = "tucker"
    elif paraman["tensortrain"]:
        str_method = "tensortrain"
    elif paraman["deepfried"]:
        str_method = "deepfried"
    elif paraman["magnitude"]:
        str_method = "magnitude"
    elif paraman["magnitude_hard"]:
        str_method = "magnitude_hard"
    elif paraman["random"]:
        str_method = "random"
    elif paraman["dummy"]:
        str_method = "dummy"
    else:
        raise ValueError("Unknown compression method")

    params_optimizer = param_train_dataset.params_optimizer

    # str_keep_first = "-keep_first" if paraman["--keep-first-layer"] else ""
    # str_config_for_lr = f"{str_data_param}-{str_model_param}-{str_method}" + str_keep_first

    if paraman["--lr-configuration-file"] is not None:
        with open(str(paraman["--lr-configuration-file"]), 'r') as f:
            dct_config_lr = yaml.full_load(f)

        params_optimizer["lr"] = load_function_lr(dct_res_new=dct_config_lr,
                                                  dataset=str_data_param,
                                                  model=str_model_param,
                                                  compression=str_method,
                                                  sparsity_value_magnitude=paraman["--final-sparsity"],
                                                  sparsity_value_random=paraman["--sparsity-factor"],
                                                  nb_fac_random=paraman["--nb-factor"],
                                                  order_value_tensortrain=paraman["--order"],
                                                  rank_value_tensortrain=paraman["--rank-value"],
                                                  rank_value_tucker=paraman["--rank-percentage-dense"])
    else:
        params_optimizer["lr"] = paraman["--lr"] if paraman["--lr"] is not None else params_optimizer["lr"]

    dct_optimizer = {
        "RMSProp": keras.optimizers.RMSprop,
        "RMSprop": keras.optimizers.RMSprop,
        "Adam": keras.optimizers.Adam,
        "SGD": keras.optimizers.SGD
    }

    model_compilation_params = {
        "loss": param_train_dataset.loss,
        "optimizer": dct_optimizer[param_train_dataset.optimizer.__name__](**params_optimizer),
        "metrics": ['categorical_accuracy']
    }
    return model_compilation_params, param_train_dataset

def get_dataset():
    (x_train, y_train), (x_test, y_test) = paraman.get_dataset().load_data()
    if paraman["--mnist-500"]:
        x_test = np.reshape(x_test, (-1, 784))
        x_train = np.reshape(x_train, (-1, 784))

    if paraman["--train-val-split"] is not None:
        x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=paraman["--train-val-split"], random_state=paraman["seed"])
    else:
        x_val, y_val = x_test, y_test

    return (x_train, y_train), (x_val, y_val), (x_test, y_test)

def define_callbacks(param_train_dataset, x_train):
    call_backs = []

    model_checkpoint_callback = SafeModelCheckpoint(str(paraman["output_file_modelprinter"]), monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=False, mode='auto',
                                                    period=1)
    call_backs.append(model_checkpoint_callback)

    actual_min_lr = param_train_dataset.min_lr if paraman["--min-lr"] is None else paraman["--min-lr"]
    actual_max_lr = param_train_dataset.max_lr if paraman["--max-lr"] is None else paraman["--max-lr"]

    if paraman["--use-clr"]:
        clr_cb = CyclicLR(base_lr=actual_min_lr,
                          max_lr=actual_max_lr,
                          step_size=(paraman["--epoch-step-size"] * (x_train.shape[0] // param_train_dataset.batch_size)),
                          logrange=True)
        call_backs.append(clr_cb)

    csvcallback = CSVLoggerByBatch(str(paraman["output_file_csvcbprinter"]), n_batch_between_display=100, separator=',', append=True)
    call_backs.append(csvcallback)

    if paraman["magnitude"]:
        call_backs.append(sparsity.UpdatePruningStep())

    dct_results = {
        "actual-batch-size": param_train_dataset.batch_size,
        "actual-nb-epochs": param_train_dataset.epochs if paraman["--nb-epoch"] is None else paraman["--nb-epoch"],
        "actual-min-lr": actual_min_lr,
        "actual-max-lr": actual_max_lr,
    }

    resprinter.add(dct_results)

    return call_backs

def get_and_evaluate_base_model(model_compilation_params, x_train, y_train, x_val, y_val, x_test, y_test):
    base_model = paraman.get_model()
    if paraman["magnitude"]:
        base_model = translate_keras_to_tf_model(base_model)
    base_model.compile(**model_compilation_params)

    test_score_base, test_acc_base = base_model.evaluate(x_test, y_test, verbose=0)
    train_score_base, train_acc_base = base_model.evaluate(x_train, y_train, verbose=0)
    val_score_base, val_acc_base = base_model.evaluate(x_val, y_val, verbose=0)

    dct_results = {
        "test_accuracy_base_model": test_acc_base,
        "test_loss_base_model": test_score_base,
        "train_accuracy_base_model": train_acc_base,
        "train_loss_base_model": train_score_base,
        "val_accuracy_base_model": val_acc_base,
        "val_loss_base_model": val_score_base,
    }
    resprinter.add(dct_results)

    return base_model

def compress_and_evaluate_model(base_model, model_compilation_params, param_train_dataset, x_train, y_train, x_val, y_val, x_test, y_test):
    if paraman["tucker"]:
        layer_replacer = LayerReplacerTucker(keep_last_layer=paraman["--keep-last-layer"], keep_first_layer=paraman["--keep-first-layer"], only_dense=paraman["--only-dense"],
                                             rank_percentage_dense=paraman["actual-rank-percentage-dense"], rank_percentage_conv=paraman["actual-rank-percentage-conv"])
    elif paraman["tensortrain"]:
        layer_replacer = LayerReplacerTT(keep_last_layer=paraman["--keep-last-layer"], keep_first_layer=paraman["--keep-first-layer"], only_dense=paraman["--only-dense"],
                                         rank_value=paraman["--rank-value"], order=paraman["--order"], use_pretrained=paraman["--use-pretrained"], tt_rank0_conv_1=paraman["--rank0-1-conv"])
    elif paraman["deepfried"]:
        layer_replacer = LayerReplacerDeepFried(keep_last_layer=paraman["--keep-last-layer"], keep_first_layer=paraman["--keep-first-layer"],
                                                nb_stack=paraman["--nb-stack"], only_dense=paraman["--only-dense"])
    elif paraman["dummy"]:
        layer_replacer = LayerReplacerDummy(keep_last_layer=paraman["--keep-last-layer"], keep_first_layer=paraman["--keep-first-layer"], only_dense=paraman["--only-dense"])
    elif paraman["random"]:
        randomizer = Randomizer(sparsity_fac=paraman["--sparsity-factor"],
                                nb_factor=paraman["--nb-factor"])

        layer_replacer = LayerReplacerRandomSparseFacto(only_mask=True, sparse_factorizer=randomizer,
                                                        keep_last_layer=paraman["--keep-last-layer"],
                                                        keep_first_layer=paraman["--keep-first-layer"],
                                                        only_dense=paraman["--only-dense"]
                                                        )
    elif paraman["magnitude"]:
        # base_model = translate_keras_to_tf_model(base_model)
        epochs = param_train_dataset.epochs if paraman["--nb-epoch"] is None else paraman["--nb-epoch"]
        end_step = np.ceil(1.0 * x_train.shape[0] / param_train_dataset.batch_size).astype(np.int32) * epochs
        layer_replacer = LayerReplacerMagnitudePruning(final_sparsity=paraman["--final-sparsity"], end_step=end_step,
                                                       keep_last_layer=paraman["--keep-last-layer"], keep_first_layer=paraman["--keep-first-layer"],
                                                       keras_module=tf.keras, only_dense=paraman["--only-dense"])
    elif paraman["magnitude_hard"]:
        # base_model = translate_keras_to_tf_model(base_model)
        layer_replacer = LayerReplacerMagnitudePruning(final_sparsity=paraman["--final-sparsity"], hard=True,
                                                       keep_last_layer=paraman["--keep-last-layer"], keep_first_layer=paraman["--keep-first-layer"],
                                                       only_dense=paraman["--only-dense"])
    else:
        raise ValueError("Unknown compression method.")

    start_replace = time.time()
    new_model = layer_replacer.fit_transform(base_model)
    # new_model = base_model
    stop_replace = time.time()
    new_model.compile(**model_compilation_params)
    test_score_compressed, test_acc_compressed = new_model.evaluate(x_test, y_test, verbose=0)
    train_score_compressed, train_acc_compressed = new_model.evaluate(x_train, y_train, verbose=0)
    val_score_compressed, val_acc_compressed = new_model.evaluate(x_val, y_val, verbose=0)
    actual_learning_rate = K.eval(new_model.optimizer.lr)

    dct_results = {
        "actual-lr": actual_learning_rate,
        "decomposition_time": stop_replace - start_replace,
        "test_accuracy_compressed_model": test_acc_compressed,
        "test_loss_compressed_model": test_score_compressed,
        "train_accuracy_compressed_model": train_acc_compressed,
        "train_loss_compressed_model": train_score_compressed,
        "val_accuracy_compressed_model": val_acc_compressed,
        "val_loss_compressed_model": val_score_compressed,
    }
    resprinter.add(dct_results)

    return new_model

def count_models_parameters(new_model, base_model=None):
    if base_model is None:
        df_layerby_layer = pd.read_csv(paraman["output_file_layerbylayer"])
        new_model_nb_param = get_nb_learnable_weights_from_model(new_model, nnz=True)
        dct_results_param = {
            "new_model_nb_param": new_model_nb_param
        }
        resprinter.add(dct_results_param)
    else:
        df_layerby_layer = None
        new_model_nb_param = get_nb_learnable_weights_from_model(new_model, nnz=True)
        base_model_nb_param = get_nb_learnable_weights_from_model(base_model, nnz=True)
        dct_results_param = {
            "base_model_nb_param": base_model_nb_param,
            "new_model_nb_param": new_model_nb_param
        }
        resprinter.add(dct_results_param)

        if len(base_model.layers) < len(new_model.layers):
            base_model = Model(inputs=base_model.inputs, outputs=base_model.outputs)

    dct_results_matrices = defaultdict(lambda: [])

    for idx_layer, compressed_layer in enumerate(new_model.layers):

        if any(isinstance(compressed_layer, _class) for _class in (PruneLowMagnitude, TuckerLayerConv,
                                                                   TTLayerConv, TTLayerDense, LowRankDense, FastFoodLayerDense, FastFoodLayerConv,
                                                                   SparseFactorisationDense, SparseFactorisationConv2D, tf.keras.layers.Conv2D, tf.keras.layers.Dense,
                                                                   Conv2D, Dense)):
            if df_layerby_layer is None:
                base_layer = base_model.layers[idx_layer]
                row_layer = None
            else:
                base_layer = None
                row_layer = df_layerby_layer[df_layerby_layer["idx-layer"] == idx_layer]
            log_memory_usage("Start secondary loop")

            # get informations to identify the layer (and do cross references)
            dct_results_matrices["idx-expe"].append(paraman["identifier"])
            dct_results_matrices["hash"].append(paraman["hash"])
            dct_results_matrices["layer-name-compressed"].append(compressed_layer.name)
            dct_results_matrices["layer-name-base"].append(base_layer.name if base_layer is not None else row_layer["layer-name-base"].values[0])
            dct_results_matrices["idx-layer"].append(idx_layer)
            dct_results_matrices["layer-class-name"].append(base_layer.__class__.__name__ if base_layer is not None else row_layer["layer-class-name"].values[0])
            dct_results_matrices["layer-compressed-class-name"].append(compressed_layer.__class__.__name__)

            # complexity analysis #
            # get nb val base layer and comrpessed layer
            if base_layer is not None:
                nb_weights_base_layer = get_nb_learnable_weights(base_layer, nnz=True)
            else:
                nb_weights_base_layer = row_layer["nb-non-zero-base"].values[0]

            dct_results_matrices["nb-non-zero-base"].append(nb_weights_base_layer)

            nb_weights_compressed_layer = get_nb_learnable_weights(compressed_layer, nnz=True)
            dct_results_matrices["nb-non-zero-compressed"].append(nb_weights_compressed_layer)
            dct_results_matrices["nb-non-zero-compression-rate"].append(nb_weights_base_layer / nb_weights_compressed_layer)

    df_results_layers = pd.DataFrame.from_dict(dct_results_matrices)
    df_results_layers.to_csv(paraman["output_file_layerbylayer"])

def load_model_from_disc():
    if paraman["magnitude"]:
        with_obj = sparsity.prune_scope
    else:
        with_obj = DummyWith

    with with_obj():
        new_model = keras.models.load_model(paraman["output_file_modelprinter"], custom_objects={
            'TuckerLayerConv': TuckerLayerConv,
            'LowRankDense': LowRankDense,
            'TTLayerConv': TTLayerConv,
            "TTLayerDense": TTLayerDense,
            'FastFoodLayerDense': FastFoodLayerDense,
            'FastFoodLayerConv': FastFoodLayerConv,
            "SparseFactorisationDense": SparseFactorisationDense,
            "SparseFactorisationConv2D": SparseFactorisationConv2D,
            'categorical_cross_entropy_from_logits': categorical_cross_entropy_from_logits,
            'SparseConv2D': SparseConv2D,
            'SparseDense': SparseDense
        })

    return new_model

def get_or_load_new_model(model_compilation_params, param_train_dataset, x_train, y_train, x_val, y_val, x_test, y_test):
    if os.path.exists(paraman["output_file_notfinishedprinter"]):
        df_history = pd.read_csv(paraman["output_file_csvcbprinter"])
        init_nb_epoch = df_history["epoch"].max() - 1
        logger.info(f"Restart from iteration {init_nb_epoch}")

        df = pd.read_csv(paraman["output_file_resprinter"])

        # get all results from before
        dct_results = {}
        for header in lst_results_header:
            if not header in df.columns:
                continue
            dct_results[header] = df[header].values[0]
        resprinter.add(dct_results)

        # if paraman["tucker"]:
        #     new_model = keras.models.load_model(paraman["output_file_modelprinter"],custom_objects={'TuckerLayerConv': TuckerLayerConv, 'LowRankDense': LowRankDense})
        # elif paraman["tensortrain"]:
        #     new_model = keras.models.load_model(paraman["output_file_modelprinter"], custom_objects={'TTLayerConv': TTLayerConv, "TTLayerDense": TTLayerDense})
        # else:
        #     raise ValueError("Unknown compression.")
    else:
        # Base Model #
        base_model = get_and_evaluate_base_model(model_compilation_params,
                                                x_train=x_train,
                                                y_train=y_train,
                                                x_val=x_val,
                                                y_val=y_val,
                                                x_test=x_test,
                                                y_test=y_test)

        # New model compression #
        new_model = compress_and_evaluate_model(base_model, model_compilation_params, param_train_dataset,
                                                x_train=x_train,
                                                y_train=y_train,
                                                x_val=x_val,
                                                y_val=y_val,
                                                x_test=x_test,
                                                y_test=y_test)

        # count the number of parameter
        count_models_parameters(new_model, base_model)

        new_model.save(str(paraman["output_file_modelprinter"]))
        del new_model
        del base_model
        keras.backend.clear_session()
        init_nb_epoch = 0

    new_model = load_model_from_disc()

    return new_model, init_nb_epoch

def fit_new_model(new_model, param_train_dataset, init_nb_epoch, call_backs, x_train, y_train, x_test, y_test, x_val, y_val):
    open(paraman["output_file_notfinishedprinter"], 'w').close()
    batch_size = param_train_dataset.batch_size if paraman["--batch-size"] is None else paraman["--batch-size"]

    new_model.fit(param_train_dataset.image_data_generator.flow(x_train, y_train,
                                                                batch_size=batch_size),
                  epochs=(param_train_dataset.epochs if paraman["--nb-epoch"] is None else paraman["--nb-epoch"]) - init_nb_epoch,
                  # epochs=2 - init_nb_epoch,
                  # verbose=1,
                  verbose=2,
                  # validation_data=(x_test, y_test),
                  callbacks=param_train_dataset.callbacks + call_backs)

    if paraman["magnitude"]:
        count_models_parameters(new_model)
        # new_model = sparsity.strip_pruning(new_model)

    test_score_finetuned, test_acc_finetuned = new_model.evaluate(x_test, y_test, verbose=0)
    train_score_finetuned, train_acc_finetuned = new_model.evaluate(x_train, y_train, verbose=0)
    val_score_finetuned, val_acc_finetuned = new_model.evaluate(x_val, y_val, verbose=0)

    if os.path.exists(paraman["output_file_notfinishedprinter"]):
        os.remove(paraman["output_file_notfinishedprinter"])

    dct_results = {
        "test_accuracy_finetuned_model": test_acc_finetuned,
        "test_loss_finetuned_model": test_score_finetuned,
        "train_accuracy_finetuned_model": train_acc_finetuned,
        "train_loss_finetuned_model": train_score_finetuned,
        "val_accuracy_finetuned_model": val_acc_finetuned,
        "val_loss_finetuned_model": val_score_finetuned,
        "actual_batch_size": batch_size
    }

    resprinter.add(dct_results)

def main():

    # Params optimizer #
    model_compilation_params, param_train_dataset = get_params_optimizer()
    # Dataset #
    (x_train, y_train), (x_val, y_val), (x_test, y_test) = get_dataset()
    # Do compression or load #
    new_model, init_nb_epoch = get_or_load_new_model(model_compilation_params, param_train_dataset,
                                                x_train=x_train,
                                                y_train=y_train,
                                                x_val=x_val,
                                                y_val=y_val,
                                                x_test=x_test,
                                                y_test=y_test)
    # Write results before finetuning #
    resprinter.print()
    # Callbacks definition #
    call_backs = define_callbacks(param_train_dataset, x_train)
    # New model Finetuning #
    fit_new_model(new_model, param_train_dataset, init_nb_epoch, call_backs,
                                                x_train=x_train,
                                                y_train=y_train,
                                                x_val=x_val,
                                                y_val=y_val,
                                                x_test=x_test,
                                                y_test=y_test)

if __name__ == "__main__":
    logger.info("Command line: " + " ".join(sys.argv))
    log_memory_usage("Memory at startup")
    arguments = docopt.docopt(__doc__)
    paraman = ParameterManagerTensotrainAndTuckerDecomposition(arguments)
    set_random_seed(paraman["seed"])
    initialized_results = dict((v, None) for v in lst_results_header)
    if paraman["magnitude"]:
        keras = tf.keras
    else:
        keras = base_keras

    resprinter = ResultPrinter(output_file=paraman["output_file_resprinter"])
    resprinter.add(initialized_results)
    resprinter.add(paraman)
    if paraman["-v"] >= 2:
        logger.setLevel(level=logging.DEBUG)
    elif paraman["-v"] >= 1:
        logger.setLevel(level=logging.INFO)
    else:
        logger.setLevel(level=logging.WARNING)

    logger.warning("Verbosity set to warning")
    logger.info("Verbosity set to info")
    logger.debug("Verbosity set to debug")

    if not os.path.exists(paraman["output_file_notfinishedprinter"]) and \
        os.path.exists(paraman["output_file_resprinter"]) and \
        os.path.exists(paraman["output_file_modelprinter"]):
        sys.exit("Expe {} already executed. Exit".format(paraman["hash"]))

    has_failed = False
    try:
        main()
    except Exception as e:
        has_failed = True
        raise e

    finally:
        failure_dict = {
            "failure": has_failed
        }

        resprinter.add(failure_dict)
        resprinter.print()

max_stars_repo_name: lucgiffon/psm-nets

max_issues_repo_name: lucgiffon/psm-nets

max_forks_repo_name: lucgiffon/psm-nets


text:
from sensor_msgs.msg import NavSatFix
from .translator import Translator, VectorTranslatorMixin
from bff_msgs.msg import FloatStatus, SurfaceVesselStatus, SupportVesselStatus
import math
# from tf.transformations import euler_from_quaternion
import numpy as np
import quaternion
from geojson import LineString

class FloatStatusTranslator(Translator, VectorTranslatorMixin):

    messageType = FloatStatus
    geomType = Translator.GeomTypes.Point
    crsName = 'wgs84'

    @staticmethod
    def translate(msg):
        return [{
            'type': 'Feature',
            'geometry': {
                'type': 'Point',
                'coordinates': [msg.gnss_position.longitude, msg.gnss_position.latitude]
            },
            'properties': {
                'stamp': msg.header.stamp.to_sec(),
                'state': msg.state,
                'voltage': msg.main_battery_state.voltage,
            }
        }]

class FloatStatusHistoryTranslator(Translator, VectorTranslatorMixin):
    messageType = FloatStatus
    geomType = Translator.GeomTypes.Point
    crsName = 'wgs84'
    def __init__(self):
        self.ll_history = []
        self.history_length = 100

    def translate(self, msg):
        self.ll_history.append([msg.gnss_position.longitude, msg.gnss_position.latitude])
        if len(self.ll_history) > self.history_length:
            self.ll_history = self.ll_history[len(self.ll_history) - self.history_length:]

        return [{
            'type': 'Feature',
            'geometry': LineString(self.ll_history),
            'properties': {
                'stamp': msg.header.stamp.to_sec(),
                'state': msg.state,
                'voltage': msg.main_battery_state.voltage,
            }
        }]

class SurfaceVesselStatusTranslator(Translator, VectorTranslatorMixin):

    messageType = SurfaceVesselStatus
    geomType = Translator.GeomTypes.Point
    crsName = 'wgs84'

    @staticmethod
    def translate(msg):

        q = np.quaternion(msg.gnss_pose.orientation.x, msg.gnss_pose.orientation.y, msg.gnss_pose.orientation.z, msg.gnss_pose.orientation.w)
        rpy = quaternion.as_euler_angles(q)
        yaw = rpy[2]

        heading = math.pi - yaw
        heading_d = heading*180.0/np.pi
        return [{
            'type': 'Feature',
            'geometry': {
                'type': 'Point',
                'coordinates': [msg.gnss_pose.position.longitude, msg.gnss_pose.position.latitude]
            },
            'properties': {
                'stamp': msg.header.stamp.to_sec(),
                'state': msg.state,
                'voltage': msg.battery_state.voltage,
                'heading': heading_d,
                'yaw': yaw,
                'yaw_x': math.cos(yaw),
                'yaw_y': math.sin(yaw)
            }
        }]

ext: py

reason: import numpy


text:
#!/usr/bin/env python3

# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.

# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.

import numpy as np
import torch
import torch.nn as nn

from ss_baselines.common.utils import Flatten
from habitat_sim.utils.common import d3_40_colors_rgb

class VisualCNN(nn.Module):
    r"""A Simple 3-Conv CNN followed by a fully connected layer

    Takes in observations and produces an embedding of the rgb and/or depth components

    Args:
        observation_space: The observation_space of the agent
        output_size: The size of the embedding vector
    """

    def __init__(self, observation_space, output_size, extra_rgb=False):
        super().__init__()
        self._output_size = output_size
        if "rgb" in observation_space.spaces and not extra_rgb:
            self._n_input_rgb = observation_space.spaces["rgb"].shape[2]
        else:
            self._n_input_rgb = 0

        if "depth" in observation_space.spaces:
            self._n_input_depth = observation_space.spaces["depth"].shape[2]
        else:
            self._n_input_depth = 0

        if "semantic" in observation_space.spaces:
            self._n_input_semantic = 6
        else:
            self._n_input_semantic = 0

        # kernel size for different CNN layers
        self._cnn_layers_kernel_size = [(8, 8), (4, 4), (3, 3)]

        # strides for different CNN layers
        self._cnn_layers_stride = [(4, 4), (2, 2), (2, 2)]

        if self._n_input_rgb > 0:
            cnn_dims = np.array(
                observation_space.spaces["rgb"].shape[:2], dtype=np.float32
            )
        elif self._n_input_depth > 0:
            cnn_dims = np.array(
                observation_space.spaces["depth"].shape[:2], dtype=np.float32
            )
        elif self._n_input_semantic > 0:
            cnn_dims = np.array(
                observation_space.spaces["semantic"].shape[:2], dtype=np.float32
            )

        if self.is_blind:
            self.cnn = nn.Sequential()
        else:
            self._input_shape = (self._n_input_rgb + self._n_input_depth + self._n_input_semantic,
                                 int(cnn_dims[0]), int(cnn_dims[1]))
            for kernel_size, stride in zip(
                self._cnn_layers_kernel_size, self._cnn_layers_stride
            ):
                cnn_dims = self._conv_output_dim(
                    dimension=cnn_dims,
                    padding=np.array([0, 0], dtype=np.float32),
                    dilation=np.array([1, 1], dtype=np.float32),
                    kernel_size=np.array(kernel_size, dtype=np.float32),
                    stride=np.array(stride, dtype=np.float32),
                )

            self.cnn = nn.Sequential(
                nn.Conv2d(
                    in_channels=self._n_input_rgb + self._n_input_depth + self._n_input_semantic,
                    out_channels=32,
                    kernel_size=self._cnn_layers_kernel_size[0],
                    stride=self._cnn_layers_stride[0],
                ),
                nn.ReLU(True),
                nn.Conv2d(
                    in_channels=32,
                    out_channels=64,
                    kernel_size=self._cnn_layers_kernel_size[1],
                    stride=self._cnn_layers_stride[1],
                ),
                nn.ReLU(True),
                nn.Conv2d(
                    in_channels=64,
                    out_channels=64,
                    kernel_size=self._cnn_layers_kernel_size[2],
                    stride=self._cnn_layers_stride[2],
                ),
                #  nn.ReLU(True),
                Flatten(),
                nn.Linear(64 * cnn_dims[0] * cnn_dims[1], output_size),
                nn.ReLU(True),
            )

        self.layer_init()

    def _conv_output_dim(
        self, dimension, padding, dilation, kernel_size, stride
    ):
        r"""Calculates the output height and width based on the input
        height and width to the convolution layer.

        ref: https://pytorch.org/docs/master/nn.html#torch.nn.Conv2d
        """
        assert len(dimension) == 2
        out_dimension = []
        for i in range(len(dimension)):
            out_dimension.append(
                int(
                    np.floor(
                        (
                            (
                                dimension[i]
                                + 2 * padding[i]
                                - dilation[i] * (kernel_size[i] - 1)
                                - 1
                            )
                            / stride[i]
                        )
                        + 1
                    )
                )
            )
        return tuple(out_dimension)

    def layer_init(self):
        for layer in self.cnn:
            if isinstance(layer, (nn.Conv2d, nn.Linear)):
                nn.init.kaiming_normal_(
                    layer.weight, nn.init.calculate_gain("relu")
                )
                if layer.bias is not None:
                    nn.init.constant_(layer.bias, val=0)

    @property
    def is_blind(self):
        return self._n_input_rgb + self._n_input_depth + self._n_input_semantic == 0

    @property
    def input_shape(self):
        return self._input_shape

    @property
    def output_shape(self):
        return 1, self._output_size

    @property
    def feature_dims(self):
        return self._output_size

    def forward(self, observations):
        cnn_input = []
        if self._n_input_rgb > 0:
            rgb_observations = observations["rgb"]
            # permute tensor to dimension [BATCH x CHANNEL x HEIGHT X WIDTH]
            rgb_observations = rgb_observations.permute(0, 3, 1, 2)
            rgb_observations = rgb_observations / 255.0  # normalize RGB
            cnn_input.append(rgb_observations)

        if self._n_input_depth > 0:
            depth_observations = observations["depth"]
            # permute tensor to dimension [BATCH x CHANNEL x HEIGHT X WIDTH]
            depth_observations = depth_observations.permute(0, 3, 1, 2)
            cnn_input.append(depth_observations)

        if self._n_input_semantic > 0:
            semantic_observations = convert_semantics_to_rgb(observations["semantic"]).float()
            semantic_object_observations = observations["semantic_object"].float()
            # permute tensor to dimension [BATCH x CHANNEL x HEIGHT X WIDTH]
            semantic_observations = torch.cat([semantic_observations, semantic_object_observations], dim=-1)
            semantic_observations = semantic_observations.permute(0, 3, 1, 2) / 255.0
            cnn_input.append(semantic_observations)

        cnn_input = torch.cat(cnn_input, dim=1)

        return self.cnn(cnn_input)

def convert_semantics_to_rgb(semantics):
    r"""Converts semantic IDs to RGB images.
    """
    semantics = semantics.long() % 40
    mapping_rgb = torch.from_numpy(d3_40_colors_rgb).to(semantics.device)
    semantics_r = torch.take(mapping_rgb[:, 0], semantics)
    semantics_g = torch.take(mapping_rgb[:, 1], semantics)
    semantics_b = torch.take(mapping_rgb[:, 2], semantics)
    semantics_rgb = torch.stack([semantics_r, semantics_g, semantics_b], -1)

    return semantics_rgb

max_stars_repo_name: tynguyen/sound-spaces

max_issues_repo_name: tynguyen/sound-spaces

max_forks_repo_name: tynguyen/sound-spaces


text:
import logging

import numpy
import sdl2
from OpenGL.GL import *
from sdl2 import video

from mgl2d.math.matrix4 import Matrix4
from mgl2d.math.rect import Rect

logger = logging.getLogger(__name__)

class Screen(object):
    def __init__(self, width, height, title='Window', alpha_blending=True, full_screen=False, gl_major=4, gl_minor=1):
        self._width = width
        self._height = height
        self._aspect_ratio = float(width) / float(height)
        self._viewport = Rect(0, 0, width, height)
        self._full_screen = full_screen

        # Create the window
        if sdl2.SDL_Init(sdl2.SDL_INIT_VIDEO) != 0:
            print(sdl2.SDL_GetError())

        self._window = sdl2.SDL_CreateWindow(str.encode(title),
                                             sdl2.SDL_WINDOWPOS_UNDEFINED,
                                             sdl2.SDL_WINDOWPOS_UNDEFINED, width, height,
                                             sdl2.SDL_WINDOW_OPENGL)

        if not self._window:
            print(sdl2.SDL_GetError())
            return

        # Set up OpenGL
        video.SDL_GL_SetAttribute(video.SDL_GL_CONTEXT_MAJOR_VERSION, gl_major)
        video.SDL_GL_SetAttribute(video.SDL_GL_CONTEXT_MINOR_VERSION, gl_minor)
        video.SDL_GL_SetAttribute(video.SDL_GL_CONTEXT_PROFILE_MASK, video.SDL_GL_CONTEXT_PROFILE_CORE)
        self._context = sdl2.SDL_GL_CreateContext(self._window)
        self._projection_matrix = self._ortho_projection()

        if alpha_blending:
            glEnable(GL_BLEND)
            glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)

        if full_screen:
            self.full_screen = True

        # Post processing steps
        self._pp_steps = []

    @property
    def width(self):
        return self._width

    @property
    def height(self):
        return self._height

    @property
    def viewport(self):
        return self._viewport

    @property
    def is_opened(self):
        return self._window is not None

    @property
    def projection_matrix(self):
        return self._projection_matrix

    @property
    def full_screen(self):
        return self._full_screen

    @full_screen.setter
    def full_screen(self, value):
        sdl2.SDL_SetWindowFullscreen(self._window, value)

    def _ortho_projection(self):
        z_far = 1.0
        z_near = -1.0

        m = numpy.zeros((4, 4), dtype=numpy.float32)
        m[0, 0] = + 2.0 / self._width
        m[3, 0] = - 1.0
        m[1, 1] = - 2.0 / self._height
        m[3, 1] = + 1.0
        m[2, 2] = - 2.0 / (z_far - z_near)
        m[3, 2] = (z_far + z_near) / (z_near - z_far)
        m[3, 3] = + 1.0

        return Matrix4(m)

    def close(self):
        sdl2.SDL_GL_DeleteContext(self._context)
        self._context = None
        sdl2.SDL_DestroyWindow(self._window)
        self._window = None

    def begin_update(self):
        if len(self._pp_steps) > 0:
            self._pp_steps[0].fbo.bind()

        glClearColor(0.0, 0.0, 0.0, 1.0)
        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)

    def end_update(self):
        if len(self._pp_steps) > 0:
            if len(self._pp_steps) > 1:
                for x in range(1, len(self._pp_steps)):
                    self._pp_steps[x].fbo.bind()
                    self._pp_steps[x - 1].draw(self)

            self._pp_steps[-1].fbo.unbind()
            self._pp_steps[-1].draw(self)

        sdl2.SDL_GL_SwapWindow(self._window)

    def add_postprocessing_step(self, step):
        self._pp_steps.append(step)

    def print_info(self):
        logger.info('Resolution: %dx%d ratio: %.2f' % (self._width, self._height, self._aspect_ratio))
        logger.info('Rendeer: %s (%s)' % (glGetString(GL_RENDERER), glGetString(GL_VENDOR)))
        logger.info('OpenGL: %s GLSL: %s' % (glGetString(GL_VERSION), glGetString(GL_SHADING_LANGUAGE_VERSION)))


text:
import pandas as pd
import numpy as np
import datetime
from modeling import target_dtype, parse_dates
from modeling import get_target_cols, get_id_cols
from modeling import lgbm_params, _lgbm_params
from modeling import num_boost_round, _num_boost_round
from modeling.metrics import calc_mrr
import lightgbm as lgb

class Validation(object):
    @classmethod
    def get_pred_df(cls, X_TR, scope="--check"):
        target_cols = get_target_cols()
        id_cols = get_id_cols()

        # datetime of splitting into train and test
        split_dt = datetime.datetime.strptime("2018-11-05 09:00:00", '%Y-%m-%d %H:%M:%S')

        # split train and test
        X_train = X_TR[X_TR.timestamp_dt < split_dt]
        X_test = X_TR[X_TR.timestamp_dt >= split_dt]

        # set groups for rank learning
        Q_train = X_train[['session_id', 'gid']].groupby('session_id').count().reset_index()
        Q_train = Q_train.rename(columns={'gid': 'query'})
        Q_test = X_test[['session_id', 'gid']].groupby('session_id').count().reset_index()
        Q_test = Q_test.rename(columns={'gid': 'query'})

        # shrink by target_cols
        y_train = X_train[["clicked"]]
        X_train = X_train[target_cols + id_cols]
        y_test = X_test[["clicked"]]
        X_test = X_test[target_cols + id_cols]

        # modeling
        lgb_train = lgb.Dataset(X_train[target_cols], y_train, group=Q_train['query'])
        lgb_eval = lgb.Dataset(X_test[target_cols], y_test, group=Q_test['query'], reference=lgb_train)
        if (scope=="--check"):
            model = lgb.train(_lgbm_params
                              , lgb_train
                              , num_boost_round=_num_boost_round
                              , valid_sets=lgb_eval)
        else:
            model = lgb.train(lgbm_params
                              , lgb_train
                              , num_boost_round=num_boost_round
                              , valid_sets=lgb_eval)

        # calc mrr
        y_pred = model.predict(X_test[target_cols], num_iteration=model.best_iteration)
        y_pred_df = pd.DataFrame({"gid": X_test["gid"].values
                                     , "impression": X_test["impression"].values
                                     , "clicked": y_test["clicked"].values
                                     , "prob": y_pred})
        if (scope!="--check"):
            # print feature importance
            fti = model.feature_importance()
            fti_df = pd.DataFrame({"feature": target_cols, "importance": fti})
            print(fti_df.sort_values("importance", ascending=False))

        # display(y_pred_df.head())
        return (y_pred_df, np.mean(y_pred_df.groupby("gid").apply(lambda x: calc_mrr(x))), model)


text:
"""
A program to solve the 3D Nonlinear Schrodinger equation using a
second order splitting method

More information on visualization can be found on the Mayavi
website, in particular:
http://github.enthought.com/mayavi/mayavi/mlab.html
which was last checked on 6 April 2012

"""

import math
import numpy
from mayavi import mlab
import matplotlib.pyplot as plt
import time

# Grid
Lx=4.0 	    # Period 2*pi*Lx
Ly=4.0 	    # Period 2*pi*Ly
Lz=4.0 	    # Period 2*pi*Lz
Nx=64 	    # Number of harmonics
Ny=64 	    # Number of harmonics
Nz=64 	    # Number of harmonics
Nt=100 	    # Number of time slices
tmax=1.0    # Maximum time
dt=tmax/Nt  # time step
plotgap=10  # time steps between plots
Es= 1.0     # focusing (+1) or defocusing (-1) parameter
numplots=Nt/plotgap  # number of plots to make

x = [i*2.0*math.pi*(Lx/Nx) for i in xrange(-Nx/2,1+Nx/2)]
y = [i*2.0*math.pi*(Ly/Ny) for i in xrange(-Ny/2,1+Ny/2)]
z = [i*2.0*math.pi*(Lz/Nz) for i in xrange(-Nz/2,1+Nz/2)]
k_x = (1.0/Lx)*numpy.array([complex(0,1)*n for n in range(0,Nx/2) \
+ [0] + range(-Nx/2+1,0)])
k_y = (1.0/Ly)*numpy.array([complex(0,1)*n for n in range(0,Ny/2) \
+ [0] + range(-Ny/2+1,0)])
k_z = (1.0/Lz)*numpy.array([complex(0,1)*n for n in range(0,Nz/2) \
+ [0] + range(-Nz/2+1,0)])

k2xm=numpy.zeros((Nx,Ny,Nz), dtype=float)
k2ym=numpy.zeros((Nx,Ny,Nz), dtype=float)
k2zm=numpy.zeros((Nx,Ny,Nz), dtype=float)
xx=numpy.zeros((Nx,Ny,Nz), dtype=float)
yy=numpy.zeros((Nx,Ny,Nz), dtype=float)
zz=numpy.zeros((Nx,Ny,Nz), dtype=float)

for i in xrange(Nx):
    for j in xrange(Ny):
        for k in xrange(Nz):
            k2xm[i,j,k] = numpy.real(k_x[i]**2)
            k2ym[i,j,k] = numpy.real(k_y[j]**2)
            k2zm[i,j,k] = numpy.real(k_z[k]**2)
            xx[i,j,k]=x[i]
            yy[i,j,k]=y[j]
            zz[i,j,k]=z[k]

# allocate arrays
usquared=numpy.zeros((Nx,Ny,Nz), dtype=float)
pot=numpy.zeros((Nx,Ny,Nz), dtype=float)
u=numpy.zeros((Nx,Ny,Nz), dtype=complex)
una=numpy.zeros((Nx,Ny,Nz), dtype=complex)
unb=numpy.zeros((Nx,Ny,Nz), dtype=complex)
v=numpy.zeros((Nx,Ny,Nz), dtype=complex)
vna=numpy.zeros((Nx,Ny,Nz), dtype=complex)
vnb=numpy.zeros((Nx,Ny,Nz), dtype=complex)
mass=numpy.zeros((Nx,Ny,Nz), dtype=complex)
test=numpy.zeros((numplots-1),dtype=float)
tdata=numpy.zeros((numplots-1), dtype=float)

u=numpy.exp(-(xx**2 + yy**2 + zz**2))
v=numpy.fft.fftn(u)
usquared=abs(u)**2
src = mlab.pipeline.scalar_field(xx,yy,zz,usquared,colormap='YlGnBu')
mlab.pipeline.iso_surface(src, contours=[usquared.min()+0.1*usquared.ptp(), ],
   colormap='YlGnBu',opacity=0.85)
mlab.pipeline.iso_surface(src, contours=[usquared.max()-0.1*usquared.ptp(), ],
   colormap='YlGnBu',opacity=1.0)
mlab.pipeline.image_plane_widget(src,plane_orientation='z_axes',
                            slice_index=Nz/2,colormap='YlGnBu',
                            opacity=0.01)
mlab.pipeline.image_plane_widget(src,plane_orientation='y_axes',
                            slice_index=Ny/2,colormap='YlGnBu',
                            opacity=0.01)
mlab.pipeline.image_plane_widget(src,plane_orientation='x_axes',
                            slice_index=Nx/2,colormap='YlGnBu',
                            opacity=0.01)
mlab.scalarbar()
mlab.xlabel('x',object=src)
mlab.ylabel('y',object=src)
mlab.zlabel('z',object=src)

# initial mass
usquared=abs(u)**2
mass=numpy.fft.fftn(usquared)
ma=numpy.real(mass[0,0,0])
print(ma)
maO=ma
t=0.0
tdata[0]=t
plotnum=0
#solve pde and plot results
for nt in xrange(numplots-1):
    for n in xrange(plotgap):
        vna=v*numpy.exp(complex(0,0.5)*dt*(k2xm+k2ym+k2zm))
        una=numpy.fft.ifftn(vna)
        usquared=abs(una)**2
        pot=Es*usquared
        unb=una*numpy.exp(complex(0,-1)*dt*pot)
        vnb=numpy.fft.fftn(unb)
        v=vnb*numpy.exp(complex(0,0.5)*dt*(k2xm+k2ym+k2zm) )
        u=numpy.fft.ifftn(v)
        t+=dt
    plotnum+=1
    usquared=abs(u)**2
    src.mlab_source.scalars = usquared
    mass=numpy.fft.fftn(usquared)
    ma=numpy.real(mass[0,0,0])
    test[plotnum-1]=numpy.log(abs(1-ma/maO))
    print(test[plotnum-1])
    tdata[plotnum-1]=t

plt.figure()
plt.plot(tdata,test,'r-')
plt.title('Time Dependence of Change in Mass')
plt.show()

max_stars_repo_name: bcloutier/PSNM

max_issues_repo_name: bcloutier/PSNM

max_forks_repo_name: bcloutier/PSNM


text:
# -*- coding: utf-8 -*-
"""
Convert 3D DC/IP Data to 2D Lines
=================================

3D DC/IP surveys are frequently comprised of a set of 2D survey lines.
In this case, the 3D survey can be parsed into a list of
2D surveys; which can be imaged or inverted independently.
In this tutorial, we focus on the following:

    - Loading and plotting the distribution of 3D DC/IP data using a 3D pseudo-section
    - Parsing the 3D survey geometry and associated data to a set a 2D surveys
    - Plotting data for each 2D survey on a 2D pseudo-section
    - Including survey topography when plotting pseudo-sections

In this case, the survey consists of dipole-dipole data for three East-West lines
and two North-South lines.

"""

#########################################################################
# Import modules
# --------------
#

from discretize.utils import mkvc

from SimPEG import utils
from SimPEG.utils.io_utils.io_utils_electromagnetics import read_dcip_xyz
from SimPEG.electromagnetics.static import resistivity as dc
from SimPEG.electromagnetics.static.utils.static_utils import (
    apparent_resistivity_from_voltage,
    convert_survey_3d_to_2d_lines,
    plot_pseudosection,
)

import os
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import tarfile

mpl.rcParams.update({"font.size": 16})

try:
    import plotly
    from SimPEG.electromagnetics.static.utils.static_utils import plot_3d_pseudosection

    has_plotly = True
except:
    has_plotly = False
    pass

# sphinx_gallery_thumbnail_number = 3

##########################################################
# Download Assets
# ---------------
#
# Here we provide the file paths to assets we need to run the inversion. The
# path to the true model conductivity and chargeability models are also
# provided for comparison with the inversion results. These files are stored as a
# tar-file on our google cloud bucket:
# "https://storage.googleapis.com/simpeg/doc-assets/dcr3d.tar.gz"
#
#

# storage bucket where we have the data
data_source = "https://storage.googleapis.com/simpeg/doc-assets/dcr3d.tar.gz"

# download the data
downloaded_data = utils.download(data_source, overwrite=True)

# unzip the tarfile
tar = tarfile.open(downloaded_data, "r")
tar.extractall()
tar.close()

# path to the directory containing our data
dir_path = downloaded_data.split(".")[0] + os.path.sep

# files to work with
topo_filename = dir_path + "topo_xyz.txt"
data_filename = dir_path + "dc_data.xyz"

#############################################
# Load the Data
# -------------
#
# Here we load the file needed to run the tutorial.
# In this case, we load the surface topography and an XYZ formatted data file
# containing 3D DC resistivity data.
#

# Load 3D topography
topo_xyz = np.loadtxt(str(topo_filename))

# Load 3D data. Here, the data are loaded from an XYZ formatted data file.
# The user must supply the proper headers for the function to identify the
# correct column. Using the 'dict_headers' keyword argument, we can load and
# organize additional columns in the data file as a dictionary.
data_3d, out_dict = read_dcip_xyz(
    data_filename,
    "volt",
    a_headers=["XA", "YA", "ZA"],
    b_headers=["XB", "YB", "ZB"],
    m_headers=["XM", "YM", "ZM"],
    n_headers=["XN", "YN", "ZN"],
    data_header="V/A",
    uncertainties_header="UNCERT",
    dict_headers=["LINEID"],
)

#######################################################################
# Plot 3D Pseudosection
# ---------------------
#
# Here we demonstrate how 3D DC resistivity data can be represented on a 3D
# pseudosection plot. To use this utility, you must have Python's *plotly*
# package. Here, we represent the data as apparent conductivities.
#

# Extract 3D survey and observed data
survey_3d = data_3d.survey
dobs_3d = data_3d.dobs

# Convert predicted data to apparent conductivities
apparent_conductivity_3d = 1 / apparent_resistivity_from_voltage(
    survey_3d, dobs_3d, space_type="half space"
)

if has_plotly:

    fig = plot_3d_pseudosection(
        survey_3d, apparent_conductivity_3d, scale="log", units="S/m",
    )

    fig.update_layout(
        title_text="Apparent Conductivity",
        title_x=0.5,
        title_font_size=24,
        width=650,
        height=500,
        scene_camera=dict(
            center=dict(x=0, y=0, z=-0.4), eye=dict(x=1.6, y=-1.6, z=1.8)
        ),
    )

    plotly.io.show(fig)

else:
    print("INSTALL 'PLOTLY' TO VISUALIZE 3D PSEUDOSECTIONS")

######################################################################
# Convert From 3D to 2D
# ---------------------
#
# Here, we convert the 3D survey into a list of 2D surveys. A vector containing
# a line ID for each datum is required. By setting 'output_indexing' to True,
# we output a list containing the indices to extract the data for each 2D survey
# from vectors associated with the 3D survey.
#

# Extract line ID from dictionary
lineID = out_dict["LINEID"]

# Convert 3D survey to a list of 3D surveys
survey_2d_list, index_list = convert_survey_3d_to_2d_lines(
    survey_3d, lineID, data_type="volt", output_indexing=True
)

# Create list of 2D apparent conductivities. Note that if you converted observed
# data then computed apparent conductivities, you would be doing so assuming 2D
# survey geometry and the values would not match those on the 3D pseudosection plot.
dobs_2d_list = []
apparent_conductivities_2d = []
for ind in index_list:
    dobs_2d_list.append(dobs_3d[ind])
    apparent_conductivities_2d.append(apparent_conductivity_3d[ind])

#######################################################################
# Plot 2D Pseudosections
# ----------------------
#

title_str = [
    "East-West Line at Northing = 0 m",
    "North-South Line at Easting = -350 m",
    "North-South Line at Easting = -350 m",
]

# Plot apparent conductivity pseudo-section
for ii in range(len(survey_2d_list)):

    vlim = [apparent_conductivity_3d.min(), apparent_conductivity_3d.max()]

    fig = plt.figure(figsize=(12, 5))
    ax1 = fig.add_axes([0.1, 0.15, 0.75, 0.78])
    plot_pseudosection(
        survey_2d_list[ii],
        dobs=apparent_conductivities_2d[ii],
        plot_type="contourf",
        ax=ax1,
        vlim=vlim,
        scale="log",
        cbar_label="Apparent Conducitivty [S/m]",
        mask_topography=True,
        contourf_opts={"levels": 30, "cmap": mpl.cm.viridis},
    )
    ax1.set_title(title_str[ii])
    plt.show()

max_stars_repo_name: Prithwijit-Chak/simpeg

max_issues_repo_name: thast/simpeg

max_forks_repo_name: thast/simpeg


text:
import numpy as np

class Game:
    def __init__(self, bc, www, wwl, wpw, wpl, pww, pwl):
        # Payoff schemes:
        self.both_coorporate_utility = bc
        self.WW_winner_utility = www
        self.WW_looser_utility = -wwl
        self.WP_winner_utility = wpw
        self.WP_looser_utility = wpl
        self.PW_winner_utility = pww
        self.PW_looser_utility = pwl

    def evaluate_strategy(self, p1, p2, p1_utility, p2_utility):
        """
        Stochastic evaluation of strategies a and b, where both strategies
        have equal probability (50%) of winning. Where:
        0: defect
        1: coorporate

        :param p1: [0,1] strategy player 1
        :param p2: [0,1] strategy player 2
        :return:
        """

        if(p1_utility or p2_utility):
            prob = p1_utility / (p1_utility + p2_utility)
        else:
            prob = 0.5

        if (p1 == 1 and p2 == 1):  # both coorporate
            return (self.both_coorporate_utility, self.both_coorporate_utility, 1)

        elif (p1 == 1 and p2 == 0):  # p1 coorporate, p2 defect
            if (np.random.rand() < prob):
                return (self.PW_winner_utility, self.PW_looser_utility, 2)
            else:
                return (self.WP_looser_utility, self.WP_winner_utility, 3)

        elif (p1 == 0 and p2 == 1):  # p1 defect, p2 coorporate
            if (np.random.rand() < prob):
                return (self.WP_winner_utility, self.WP_looser_utility, 4)
            else:
                return (self.PW_looser_utility, self.PW_winner_utility, 5)

        elif (p1 == 0 and p2 == 0):  # both defect
            if (np.random.rand() < prob):
                return (self.WW_winner_utility, self.WW_looser_utility, 6)
            else:
                return (self.WW_looser_utility, self.WW_winner_utility, 7)

    def simulate_game(self, p1_strategy, p2_strategy):
        """
        Simulates a two round stochastic game

        :param p1_strategy: [int,int,int,int,int,int,int,int] binary, strategy for player1
        :param p2_strategy: [int,int,int,int,int,int,int,int] binary, strategy for player2

        :return: [int, int] where (player1_payoff, player2_payoff). +1 is winning, -1 is losing, 0 is draw
        """
        p1_utility = 10
        p2_utility = 10

        # first round
        p1_first_strategy = p1_strategy[0]
        p2_first_strategy = p2_strategy[0]
        [p1_reward, p2_reward, state] = self.evaluate_strategy(p1_first_strategy, p2_first_strategy, p1_utility, p2_utility)
        p1_utility += p1_reward
        p2_utility += p2_reward

        # second round
        p1_second_strategy = p1_strategy[state]
        p2_second_strategy = p2_strategy[state]
        [p1_reward, p2_reward, state] = self.evaluate_strategy(p1_second_strategy, p2_second_strategy, p1_utility, p2_utility)
        p1_utility += p1_reward
        p2_utility += p2_reward

        # if(p1_utility > p2_utility):
        #     return [1,0]
        # elif(p1_utility < p2_utility):
        #     return [0,1]
        # else:
        #     return [0,0]
        #
        return(p1_utility-10, p2_utility-10)


text:
import os
import time

import numpy as np
import tensorflow_hub as thub
import tensorflow as tf

from object_detection.utils import label_map_util
from object_detection.utils import ops as utils_ops
from object_detection.utils import visualization_utils as viz_utils

from effects.base_effect import Effect
from logger import logger

os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"

EFFECT_MODEL = {
    "obj_detection": {
        "path": "models/ssd_mobilenet_v2_2",
        "img": {"crop": (1280, 856)},
    }
}

COCO17_HUMAN_POSE_KEYPOINTS = [
    (0, 1),
    (0, 2),
    (1, 3),
    (2, 4),
    (0, 5),
    (0, 6),
    (5, 7),
    (7, 9),
    (6, 8),
    (8, 10),
    (5, 6),
    (5, 11),
    (6, 12),
    (11, 12),
    (11, 13),
    (13, 15),
    (12, 14),
    (14, 16),
]

CATEGORY_MAP = {
    1: {"id": 1, "name": "person"},
    2: {"id": 2, "name": "bicycle"},
    3: {"id": 3, "name": "car"},
    4: {"id": 4, "name": "motorcycle"},
    5: {"id": 5, "name": "airplane"},
    6: {"id": 6, "name": "bus"},
    7: {"id": 7, "name": "train"},
    8: {"id": 8, "name": "truck"},
    9: {"id": 9, "name": "boat"},
    10: {"id": 10, "name": "traffic light"},
    11: {"id": 11, "name": "fire hydrant"},
    13: {"id": 13, "name": "stop sign"},
    14: {"id": 14, "name": "parking meter"},
    15: {"id": 15, "name": "bench"},
    16: {"id": 16, "name": "bird"},
    17: {"id": 17, "name": "cat"},
    18: {"id": 18, "name": "dog"},
    19: {"id": 19, "name": "horse"},
    20: {"id": 20, "name": "sheep"},
    21: {"id": 21, "name": "cow"},
    22: {"id": 22, "name": "elephant"},
    23: {"id": 23, "name": "bear"},
    24: {"id": 24, "name": "zebra"},
    25: {"id": 25, "name": "giraffe"},
    27: {"id": 27, "name": "backpack"},
    28: {"id": 28, "name": "umbrella"},
    31: {"id": 31, "name": "handbag"},
    32: {"id": 32, "name": "tie"},
    33: {"id": 33, "name": "suitcase"},
    34: {"id": 34, "name": "frisbee"},
    35: {"id": 35, "name": "skis"},
    36: {"id": 36, "name": "snowboard"},
    37: {"id": 37, "name": "sports ball"},
    38: {"id": 38, "name": "kite"},
    39: {"id": 39, "name": "baseball bat"},
    40: {"id": 40, "name": "baseball glove"},
    41: {"id": 41, "name": "skateboard"},
    42: {"id": 42, "name": "surfboard"},
    43: {"id": 43, "name": "tennis racket"},
    44: {"id": 44, "name": "bottle"},
    46: {"id": 46, "name": "wine glass"},
    47: {"id": 47, "name": "cup"},
    48: {"id": 48, "name": "fork"},
    49: {"id": 49, "name": "knife"},
    50: {"id": 50, "name": "spoon"},
    51: {"id": 51, "name": "bowl"},
    52: {"id": 52, "name": "banana"},
    53: {"id": 53, "name": "apple"},
    54: {"id": 54, "name": "sandwich"},
    55: {"id": 55, "name": "orange"},
    56: {"id": 56, "name": "broccoli"},
    57: {"id": 57, "name": "carrot"},
    58: {"id": 58, "name": "hot dog"},
    59: {"id": 59, "name": "pizza"},
    60: {"id": 60, "name": "donut"},
    61: {"id": 61, "name": "cake"},
    62: {"id": 62, "name": "chair"},
    63: {"id": 63, "name": "couch"},
    64: {"id": 64, "name": "potted plant"},
    65: {"id": 65, "name": "bed"},
    67: {"id": 67, "name": "dining table"},
    70: {"id": 70, "name": "toilet"},
    72: {"id": 72, "name": "tv"},
    73: {"id": 73, "name": "laptop"},
    74: {"id": 74, "name": "mouse"},
    75: {"id": 75, "name": "remote"},
    76: {"id": 76, "name": "keyboard"},
    77: {"id": 77, "name": "cell phone"},
    78: {"id": 78, "name": "microwave"},
    79: {"id": 79, "name": "oven"},
    80: {"id": 80, "name": "toaster"},
    81: {"id": 81, "name": "sink"},
    82: {"id": 82, "name": "refrigerator"},
    84: {"id": 84, "name": "book"},
    85: {"id": 85, "name": "clock"},
    86: {"id": 86, "name": "vase"},
    87: {"id": 87, "name": "scissors"},
    88: {"id": 88, "name": "teddy bear"},
    89: {"id": 89, "name": "hair drier"},
    90: {"id": 90, "name": "toothbrush"},
}

class ObjectDetection(Effect):
    """apply object detection effect."""

    def __init__(self):
        self.model: dict = EFFECT_MODEL.get("obj_detection")
        self.detector = None
        self.img_array = None
        self.modified_img_array = None

    def load(self):
        """load model from path bound to local from the container."""
        l_time = time.time()
        self.detector = thub.load(self.model["path"])
        self.detector(np.random.random((1, 256, 256, 3)))
        logger.debug("## >> load-time {:.2f} s".format(time.time() - l_time))

    def preprocess(self, img_array):
        """preprocess the input image."""
        if len(img_array.shape) == 3:
            self.img_array = np.expand_dims(img_array, 0)
        else:
            self.img_array = img_array

    def apply(self, img_array):
        """apply effect to an image array.
        preprocess the image
        call the detector, pass the image,
        postprocess image, add effects.
        """
        e_time = time.time()
        self.preprocess(img_array)
        i_time = time.perf_counter()
        effect: dict = self.detector(self.img_array)
        logger.debug("## >> i-time {:.2f} s".format(time.perf_counter() - i_time))
        self.effect: dict = {key: value.numpy() for key, value in effect.items()}
        self.postprocess()
        logger.debug("## >> e-time {:.2f} s".format(time.time() - e_time))
        return self.modified_img_array

    def postprocess(self):
        """process input image with the effect

        params
        img_array: numpy array  of internal image
        effect: dictionary of effects from the model.
        """
        effect = self.effect
        effects_img = self.img_array.copy()
        keypoints = effect.get("detection_keypoints")
        keypoint_scores = effect.get("detection_keypoint_scores")
        if keypoints:
            keypoint_scores = effect.get("detection_keypoint_scores")[0]
        viz_utils.visualize_boxes_and_labels_on_image_array(
            effects_img[0],
            effect["detection_boxes"][0],
            effect["detection_classes"][0].astype(int),
            effect["detection_scores"][0],
            CATEGORY_MAP,
            use_normalized_coordinates=True,
            max_boxes_to_draw=10,
            min_score_thresh=0.60,
            agnostic_mode=False,
            keypoints=keypoints,
            keypoint_scores=keypoint_scores,
            keypoint_edges=COCO17_HUMAN_POSE_KEYPOINTS,
        )
        self.modified_img_array = effects_img[0]


text:
#!/usr/bin/env python3

import os.path
import time

import numpy as np
import numpy.linalg as la
import scipy.io as sio

import matplotlib.pyplot as plt
from neml import models, elasticity, parse

import sys
sys.path.append('../../..')
from srlife import receiver, structural

# Moose comparison stuff
# MOOSE test stuff
ramp = lambda x: np.piecewise(x,
    [x < 1, x>=1],
    [lambda xx: xx, lambda xx: 1.0+xx*0.0])
unit_temperature = lambda x, y, z: (np.sqrt(x**2.0+y**2.0)-9)/(10-9) * np.cos(np.arctan2(y,x))*(z/10.0 + 1)
temperature = lambda t, x, y, z: np.array([100*ramp(ti)*unit_temperature(x,y,z) for ti in t])
pressure = lambda t: 1.0 * ramp(t)

times = np.array([0, 0.5, 1, 101, 201, 301, 401, 501, 601, 701, 801, 901, 1001])
mat = parse.parse_xml(os.path.join(os.path.dirname(__file__),
  'moose-verification', 'model.xml'), 'creeping')

ri = 9.0
ro = 10.0
h = 10.0
nr = 11
nt = 20
nz = 6

moose_base = os.path.join(os.path.dirname(__file__), 'moose-verification')
moose_ver = [os.path.join(moose_base, f) for f in ['1d_out.e', '2d_out.e', '3d_out.e']]

def run_reference_simulation(d, solver):
  tube = receiver.Tube(ro, ro - ri, h, nr, nt, nz)
  tube.set_times(times)

  if d == 1:
    tube.make_1D(0, 0)
  elif d == 2:
    tube.make_2D(0)

  R, T, Z = tube.mesh
  X = R * np.cos(T)
  Y = R * np.sin(T)

  Ts = temperature(times, X, Y, Z).reshape((len(times),) + tube.dim[:tube.ndim])

  tube.add_results("temperature", Ts)
  tube.set_pressure_bc(receiver.PressureBC(times, pressure(times)))

  solver.setup_tube(tube)

  state_n = solver.init_state(tube, mat)

  for i in range(1,len(tube.times)):
    state_np1 = solver.solve(tube, i, state_n, 0.0)
    solver.dump_state(tube, i, state_np1)
    state_n = state_np1

  return tube

if __name__ == "__main__":
  if len(sys.argv) != 2:
    raise ValueError("Need to supply one argument, problem dimension!")

  d = int(sys.argv[1])

  if d not in [1,2,3]:
    raise ValueError("Invalid problem dimension %i!" % d)

  solver = structural.PythonTubeSolver()

  print("Running %iD problem..." % d)

  t1 = time.time()
  tube = run_reference_simulation(d, solver)
  t2 = time.time()

  print("Walltime: %f s" % (t2-t1))

max_stars_repo_name: arfontalvoANU/srlife

max_issues_repo_name: arfontalvoANU/srlife

max_forks_repo_name: arfontalvoANU/srlife

reason: import numpy,import scipy


text:
from pathlib import Path
import logging
import json
import numpy as np
import ast

from pydantic import BaseModel
from tqdm import tqdm
from pyserini import search

from .args import ArgumentParserBuilder, opt
from pygaggle.qa.cbqa import ClosedBookQA
from pygaggle.model.evaluate import ReaderEvaluator

class ClosedBookQuestionAnsweringEvaluationOptions(BaseModel):
    data: str
    model_name: str
    device: str

def main():
    apb = ArgumentParserBuilder()
    apb.add_opts(
        opt('--data',
            type=str,
            help='Dataset to use, either nq (natural questions) or tqa (trivia QA)',
            default='nq',
            choices=['nq', 'tqa']),
        opt('--model-name',
            type=str,
            default='google/t5-large-ssm-nq',
            help='Pretrained model for closed book question answering'),
        opt('--output-file',
            type=Path,
            default=None,
            help='File to output predictions for each example; if not specified, this output will be discarded'),
        opt('--device',
            type=str,
            default='cuda:0',
            help='Device for model computations'),
    )
    args = apb.parser.parse_args()
    options = ClosedBookQuestionAnsweringEvaluationOptions(**vars(args))

    if options.data == 'nq':
        data = search.get_topics('nq-test')
    elif options.data == 'tqa':
        data = search.get_topics('dpr-trivia-test')

    logging.info('Loading CBQA Model and Tokenizer')

    cbqa = ClosedBookQA(options.model_name, options.device)
    results = []
    scores = []
    for _, item in tqdm(data.items()):
        answers = ast.literal_eval(item['answers'])
        question = item['title']
        prediction = cbqa.predict(question)
        if args.output_file is not None:
            results.append({"question": question,
                            "answers": answers,
                            'prediction': prediction})
        scores.append(max([ReaderEvaluator.exact_match_score(prediction, ga) for ga in answers]))

    logging.info('CBQA prediction completed')
    em = np.mean(np.array(scores)) * 100.
    logging.info(f'Exact Match Accuracy: {em}')

    if args.output_file is not None:
        with open(args.output_file, 'w', encoding='utf-8', newline='\n') as f:
            json.dump(results, f, indent=4)

if __name__ == "__main__":
    main()

max_stars_repo_name: Elfsong/pygaggle

max_issues_repo_name: Elfsong/pygaggle

max_forks_repo_name: Elfsong/pygaggle


text:
import argparse
import lm_dataformat as lmd
import gensim
import itertools
import pandas as pd
import seaborn as sns
import numpy as np
import json
import networkx as nx
import matplotlib.pyplot as plt
import spacy
import fasttext

from math import pow
from multiprocessing import Pool
from collections import defaultdict
from gensim.corpora.dictionary import Dictionary
from spacy.lang.en import English
from gensim.models import LdaModel, LdaMulticore
from sklearn.manifold import TSNE
from best_download import download_file

component_list = {
    'StackExchange',
    'PubMed Central',
    'HackerNews',
    'OpenWebText2',
    'Enron Emails',
    'Ubuntu IRC',
    'BookCorpus',
    'USPTO',
    'Gutenberg (PG-19)',
    'Github',
    'CommonCrawl',
    'PubMed Abstracts',
    'FreeLaw',
    'Wikipedia (en)',
    'ArXiv',
    'DM Mathematics',
    'PhilPapers',
    'EuroParl',
    'Bibliotik',
    'OpenSubtitles',
    'YoutubeSubtitles',
    'NIH ExPorter',
}

component_options = component_list | { 'all' }

parser = argparse.ArgumentParser(description='Train topic models on a component.')

parser.add_argument('--input_path', required=True, help='Path to Pile split to use for dictionary (validation, ideally)')
parser.add_argument('--models_path', required=True, help='Path to saved LDA models')
parser.add_argument('--dictionary_file', required=True, help='File name to load the dictionary from')
parser.add_argument('--chunk_size', type=int, default=2**12,
                    help='Number of documents in a chunk')
parser.add_argument('--tokenizer_processes', type=int, default=1, help='Number of processes for input tokenization')
parser.add_argument('--lda_processes', type=int, default=1, help='Number of processes for LDA training')

args = parser.parse_args()

input_path = args.input_path
models_path = args.models_path
dictionary_file = args.dictionary_file
CHUNK_SIZE = args.chunk_size
TOK_PROCESSES = args.tokenizer_processes
LDA_PROCESSES = args.lda_processes

components = component_list

download_file('https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin', 'lid.176.bin', '7e69ec5451bc261cc7844e49e4792a85d7f09c06789ec800fc4a44aec362764e')
langdet = fasttext.load_model("lid.176.bin")
nlp = English()
tokenizer = nlp.Defaults.create_tokenizer(nlp)

def language(doc):
    details = langdet.predict(doc.replace('\n', ' '), k=1)

    return details[0][0].replace('__label__', '')

dictionary = Dictionary.load(dictionary_file)

rdr = lmd.Reader(input_path)
stream = rdr.stream_data(get_meta=True)

def baggify(item):
    text, meta = item
    component = meta['pile_set_name']
    if component in components:
        if language(text) == 'en':
            doc = tokenizer(text)
            tokens = [tok.lower_ for tok in doc if not tok.is_stop and tok.is_alpha]
            bow_or_none = dictionary.doc2bow(tokens)
        else:
            bow_or_none = None
    else:
        bow_or_none = None
    return (bow_or_none, component)

def raw_baggify(item):
    text, meta = item
    component = meta['pile_set_name']
    if component in components:
        if language(text) == 'en':
            doc = tokenizer(text)
            tokens_or_none = [tok.lower_ for tok in doc if not tok.is_stop and tok.is_alpha]
        else:
            tokens_or_none = None
    else:
        tokens_or_none = None
    return (tokens_or_none, component)

docs = defaultdict(list)
models = defaultdict()

print("Loading models...")
for component in components:
    models[component] = LdaMulticore.load(models_path + component + '.model.topic')

# Compute cross-corpus topic similarities
diffs = defaultdict(defaultdict)

for (train, test) in itertools.product(*(components, components)):
    print(f'Computing topic diff for {train} on {test}...')
    diff, annotation = models[train].diff(models[test])
    diffs[train][test] = diff.tolist()

    fig, ax = plt.subplots(1,1)

    img = plt.imshow(diff, cmap='gray')

    ax.set_title("Topic Model Difference Matrix")
    ax.set_ylabel(train)
    ax.set_xlabel(test)

    plt.savefig(f'topic_diffs_{train}_{test}.png')

with open('topic_model_diffs.json', 'w') as fp:
    json.dump(diffs, fp)
    print("Saving topic modeling cross-corpus diffs...")

# Compute cross-corpus perplexities
perplexities = defaultdict(defaultdict)

with Pool(TOK_PROCESSES) as p:
    print("Reading documents...")
    doc_iter = p.imap_unordered(baggify, stream, chunksize=CHUNK_SIZE)
    for (i, (bow, component)) in enumerate(doc_iter):
        if bow:
            docs[component].append(bow)

for (train, test) in itertools.product(*(components, components)):
    print(f'Computing perplexities for model trained on {train} tested on {test}...')
    perplexities[train][test] = pow(2, -models[train].log_perplexity(docs[test]))

with open('topic_model_perplexities.json', 'w') as fp:
    json.dump(perplexities, fp)
    print("Saving topic modeling cross-corpus perplexities...")

df = pd.DataFrame.from_dict(perplexities, orient='index')
df = df.sort_index(axis=0)
df = df.sort_index(axis=1)
log_perplexities = df.apply(np.log)

print("Mean perplexities:", log_perplexities.mean(axis=1))

cm = sns.clustermap(log_perplexities.transpose(), dendrogram_ratio=0.33, metric='euclidean', xticklabels=True, yticklabels=True, col_cluster=False, cbar_pos=(0, 0, 0.05, 0.2), tree_kws={ 'linewidths': 4})
ax = cm.ax_heatmap
ax.set_xlabel("Training Corpus")
ax.set_ylabel("Testing Corpus")

fig = ax.get_figure()
fig.savefig('topic_model_perplexities.png')

# Show topics for each component
for component in components:
    print(component, models[component].show_topics(16, 10))

reason: import numpy,import networkx


text:
import os
import pdb
import numpy as np
import torch
import torch.nn.functional as F
from torchvision import datasets, transforms
from torch.utils.data import DataLoader

from mod import ConvNet

'''testbench'''
'--- 展示数据'
# mnist_data = datasets.MNIST('dataset/mnist', download=True)
# mnist_data[0][0].show() # PIL图片可以直接show
# print(mnist_data[0][1])
'--- 将所有数据normalize所需的参数'
# mnist_data = datasets.MNIST('dataset/mnist', train=True, download=True, transform=transforms.Compose([
#     transforms.ToTensor(),
# ]))
# all_data = [d[0].data.numpy() for d in mnist_data]
# mean, std = np.mean(all_data), np.std(all_data)
# print(mean, std)    # 0.13066062 0.30810776

dir_model = 'model/ConvNet.pt'
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
batch_size = 32
lr = 0.01
momentum = 0.5
num_epochs = 1
loss_fun = F.nll_loss

def train(model, device, train_loader, optimizer, epoch):
    model.train()
    for idx, (input, target) in enumerate(train_loader):
        # print(idx, (data.shape, target.shape))  # 0 (torch.Size([32, 1, 28, 28]), torch.Size([32]))
        input, target = input.to(device), target.to(device)
        output = model(input)   # (batch_size, 10)
        loss = loss_fun(output, target)
        # SGD
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        if idx % 500 == 0:
            print(f'[TRAIN] Epoch: {epoch}, Iteration: {idx}, Loss: {loss}')

def test(model, device, test_loader):
    model.eval()
    total_loss = 0
    correct = 0
    with torch.no_grad():
        for idx, (input, target) in enumerate(test_loader):
            # print(idx, (data.shape, target.shape))  # 0 (torch.Size([32, 1, 28, 28]), torch.Size([32]))
            input, target = input.to(device), target.to(device)
            output = model(input)   # (batch_size, 10)
            total_loss += loss_fun(output, target, reduction='sum')
            pred = output.argmax(dim=1)     # (batch_size, 1)
            correct += pred.eq(target).sum().item()
    total_loss /= len(test_loader.dataset)
    acc = correct / len(test_loader.dataset) * 100
    print(f'[TEST] Loss: {total_loss}, Accuracy: {acc}%')

def main():
    train_data = datasets.MNIST('dataset/mnist', train=True, download=True, transform=transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize(
            0.13066062, 0.30810776
        )   # 预处理，对所有数据进行normalize
    ]))
    test_data = datasets.MNIST('dataset/mnist', train=False, download=True, transform=transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize(
            0.13066062, 0.30810776
        )
    ]))
    # print(len(train_data), len(test_data))  # 60000, 10000
    train_loader = DataLoader(
        train_data,
        batch_size=batch_size, shuffle=True, pin_memory=True
    )
    test_loader = DataLoader(
        test_data,
        batch_size=batch_size, shuffle=False, pin_memory=True
    )
    model = ConvNet().to(device)
    optimizer = torch.optim.SGD(
        model.parameters(),
        lr=lr, momentum=momentum
    )
    if os.path.exists(dir_model):
        model.load_state_dict(torch.load(dir_model))
    else:
        for epoch in range(num_epochs):
            train(model, device, train_loader, optimizer, epoch)
        os.makedirs('model', exist_ok=True)
        torch.save(model.state_dict(), dir_model)
    test(model, device, test_loader)

if __name__ == '__main__':
    main()

max_stars_repo_name: SecretMG/UNet-for-Image-Denoising

max_issues_repo_name: SecretMG/UNet-for-Image-Denoising

max_forks_repo_name: SecretMG/UNet-for-Image-Denoising


text:
from __future__ import annotations

from typing import List, Optional, Tuple

import numpy as np
from numpy.typing import NDArray
from qulacs import Observable
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import MinMaxScaler
from typing_extensions import Literal

from skqulacs.circuit import LearningCircuit
from skqulacs.qnn.qnnbase import QNN
from skqulacs.qnn.solver import Solver

class QNNRegressor(QNN):
    """Class to solve regression problems with quantum neural networks
    The output is taken as expectation values of pauli Z operator acting on the first qubit, i.e., output is <Z_0>.
    Examples:
        >>> from skqulacs.qnn import QNNRegressor
        >>> from skqulacs.circuit import create_qcl_ansatz
        >>> n_qubits = 4
        >>> depth = 3
        >>> evo_time = 0.5
        >>> circuit = create_qcl_ansatz(n_qubits, depth, evo_time)
        >>> model = QNNRegressor(circuit)
        >>> _, theta = model.fit(x_train, y_train, maxiter=1000)
        >>> x_list = np.arange(x_min, x_max, 0.02)
        >>> y_pred = qnn.predict(theta, x_list)
    """

    def __init__(
        self,
        circuit: LearningCircuit,
        solver: Solver,
        cost: Literal["mse"] = "mse",
        do_x_scale: bool = True,
        do_y_scale: bool = True,
        x_norm_range: float = 1.0,
        y_norm_range: float = 0.7,
    ) -> None:
        """
        :param circuit: Circuit to use in the learning.
        :param solver: Solver to use(Nelder-Mead is not recommended).
        :param cost: Cost function. MSE only for now. MSE computes squared sum after normalization.
        :param do_x_scale: Whether to scale x.
        :param do_x_scale: Whether to scale y.
        :param y_norm_range: Normalize y in [+-y_norm_range].
        :param callback: Callback function. Available only with Adam.
        Setting y_norm_range to 0.7 improves performance.
        :param tol: use n_iter_no_change
        :param n_iter_no_change: (cost reduce < tol) continues n_iter_no_change times -> stopping (Adam only)

        """
        self.n_qubit = circuit.n_qubit
        self.circuit = circuit
        self.solver = solver
        self.cost = cost
        self.do_x_scale = do_x_scale
        self.do_y_scale = do_y_scale
        self.x_norm_range = x_norm_range
        self.y_norm_range = y_norm_range
        self.observables = []

    def fit(
        self,
        x_train: NDArray[np.float_],
        y_train: NDArray[np.float_],
        maxiter: Optional[int] = None,
    ) -> Tuple[float, List[float]]:
        """
        :param x_list: List of x to fit.
        :param y_list: List of y to fit.
        :param maxiter: The number of iterations to pass scipy.optimize.minimize
        :return: Loss after learning.
        :return: Parameter theta after learning.
        """

        x_train = np.array(x_train)
        y_train = np.array(y_train)

        if x_train.ndim == 1:
            x_train = x_train.reshape((-1, 1))

        if y_train.ndim == 1:
            y_train = y_train.reshape((-1, 1))

        if self.do_x_scale:
            self.scale_x_scaler = MinMaxScaler(
                feature_range=(-self.x_norm_range, self.x_norm_range)
            )
            x_scaled = self.scale_x_scaler.fit_transform(x_train)
        else:
            x_scaled = x_train

        if self.do_y_scale:
            self.scale_y_scaler = MinMaxScaler(
                feature_range=(-self.y_norm_range, self.y_norm_range)
            )
            y_scaled = self.scale_y_scaler.fit_transform(y_train)
        else:
            y_scaled = y_train

        if y_train.ndim == 2:
            self.n_outputs = len(y_scaled[0])
        else:
            self.n_outputs = 1

        self.observables = []
        for i in range(self.n_outputs):
            observable = Observable(self.n_qubit)
            observable.add_operator(1.0, f"Z {i}")
            self.observables.append(observable)

        theta_init = self.circuit.get_parameters()
        return self.solver.run(
            self.cost_func,
            self._cost_func_grad,
            theta_init,
            x_scaled,
            y_scaled,
            maxiter,
        )

    def predict(self, x_test: NDArray[np.float_]) -> NDArray[np.int_]:
        """Predict outcome for each input data in `x_test`.

        Arguments:
            x_test: Input data whose shape is (n_samples, n_features).

        Returns:
            y_pred: Predicted outcome.
        """
        if x_test.ndim == 1:
            x_test = x_test.reshape((-1, 1))
        if self.do_x_scale:
            x_scaled = self.scale_x_scaler.transform(x_test)
        else:
            x_scaled = x_test

        if self.do_y_scale:
            y_pred = self.scale_y_scaler.inverse_transform(
                self._predict_inner(x_scaled)
            )
        else:
            y_pred = self._predict_inner(x_scaled)

        return y_pred

    def _predict_inner(self, x_scaled: NDArray[np.float_]) -> NDArray[np.float_]:
        res = []
        for x in x_scaled:
            state = self.circuit.run(x)
            r = [
                self.observables[i].get_expectation_value(state)
                for i in range(self.n_outputs)
            ]
            res.append(r)
        return np.array(res)

    def cost_func(
        self,
        theta: List[float],
        x_scaled: NDArray[np.float_],
        y_scaled: NDArray[np.int_],
    ) -> float:
        if self.cost == "mse":
            self.circuit.update_parameters(theta)
            y_pred = self._predict_inner(x_scaled)

            cost = mean_squared_error(y_pred, y_scaled)
            return cost
        else:
            raise NotImplementedError(
                f"Cost function {self.cost} is not implemented yet."
            )

    def _cost_func_grad(
        self,
        theta: List[float],
        x_scaled: NDArray[np.float_],
        y_scaled: NDArray[np.int_],
    ) -> NDArray[np.float_]:
        self.circuit.update_parameters(theta)

        mto = self._predict_inner(x_scaled).copy()

        grad = np.zeros(len(theta))

        for h in range(len(x_scaled)):
            backobs = Observable(self.n_qubit)
            if self.n_outputs >= 2:
                for i in range(self.n_outputs):
                    backobs.add_operator(
                        (-y_scaled[h][i] + mto[h][i]) / self.n_outputs, "Z {i}"
                    )
            else:
                backobs.add_operator((-y_scaled[h] + mto[h][0]) / self.n_outputs, "Z 0")
            grad += self.circuit.backprop(x_scaled[h], backobs)

        self.circuit.update_parameters(theta)
        grad /= len(x_scaled)
        return grad

reason: import numpy,from numpy


text:
# !/usr/bin/env python
# -*- coding:utf-8 -*-
# author: jeremy.zhang(szujeremy@gmail.com, Shenzhen University, China)

import numpy as np

def check_y_true_and_prob(y_true, y_prob):
    y_true = np.squeeze(np.array(y_true))
    y_prob = np.squeeze(np.array(y_prob))

    if y_true.ndim != 1 or y_prob.ndim != 1:
        raise ValueError('Expected y_true and y_prob is 1D array, but got %sD and %sD.' % (y_true.ndim, y_prob.ndim))

    if np.size(y_true) != np.size(y_prob):
        raise ValueError('Found input y_true and y_prob with inconsistent numbers of values.')

    if np.min(y_prob) < 0 or np.max(y_prob) > 1:
        raise ValueError('Expected y_prob contains values between 0 and 1.')

    y_label = np.unique(y_true)
    if np.size(y_label) != 2 or y_label[1] != 1:
        raise ValueError('Input y_true should only have 0 and 1, but got %s.' % str(y_label))

    return y_true, y_prob

def check_column_or_1d(x, y):
    x_array = np.squeeze(np.array(x))
    y_array = np.squeeze(np.array(y))

    if x_array.ndim != 1 or y_array.ndim != 1:
        raise ValueError('Expected x and y is 1D array, but got %sD and %sD.' % (x_array.ndim, y_array.ndim))

    if np.size(x_array) != np.size(y_array):
        raise ValueError('Found input x and y with inconsistent numbers of values.')

    return x_array, y_array

max_issues_repo_name: MedcAILab/scikit-radiomics

max_forks_repo_name: MedcAILab/scikit-radiomics