text:
# -*- coding: utf-8 -*-
"""Fuzzy pattern classifier with genetic algorithm based methods

The module structure is the following:

- The "MultimodalEvolutionaryClassifier" contains the classifier implementing [1].

- The "EnsembleMultimodalEvolutionaryClassifier" contains an emsemble based classifier
  extended from [1] where more than one prototype is allowed per class, see [2].

References:
[1] Stoean, Stoean, Preuss and Dumitrescu, 2005.
[2] Davidsen, Padmavathamma, 2015.
"""

import logging
import numpy as np
from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.utils.validation import check_array
from sklearn.neighbors import DistanceMetric
from sklearn.preprocessing import normalize
from sklearn.metrics import accuracy_score
from sklearn.utils import check_random_state
from fylearn.ga import GeneticAlgorithm, helper_n_generations, helper_fitness

logger = logging.getLogger("garules")

def stoean_f(X):
    return StoeanDistance(np.nanmax(X, 0) - np.nanmin(X, 0))

def distancemetric_f(name, **kwargs):
    def _distancemetric_factory(X):
        return DistanceMetric.get_metric(name)
    return _distancemetric_factory

class StoeanDistance(DistanceMetric):
    def __init__(self, d):
        self.d = d

    def pairwise(self, X, Y=None):
        if Y is None:
            Y = X
        R = np.zeros((len(X), len(Y)))
        for idx, x in enumerate(X):
            R[idx, :] = np.sum(np.abs(Y - x) / self.d, 1)
        return R

class MultimodalEvolutionaryClassifier(BaseEstimator, ClassifierMixin):

    def __init__(self, n_iterations=10, df=stoean_f):
        self.n_iterations = n_iterations
        self.df = df

    def get_params(self, deep=False):
        return {"n_iterations": self.n_iterations,
                "df": self.df}

    def set_params(self, **kwargs):
        for key, value in kwargs.items():
            setattr(self, key, value)
        return self

    def distance_sum(self, X, Y):
        return np.sum(self.distance_.pairwise(X, Y), 1)

    def build_for_class(self, X):

        distance_fitness = lambda P: self.distance_sum(P, X)

        # setup GA
        ga = GeneticAlgorithm(fitness_function=distance_fitness,
                              elitism=3,
                              n_chromosomes=100,
                              n_genes=X.shape[1],
                              p_mutation=0.3)

        ga = helper_n_generations(ga, self.n_iterations)  # advance the GA

        # return the best found parameters for this class.
        chromosomes, fitness = ga.best(1)
        return chromosomes[0]

    def fit(self, X, y):
        X = check_array(X)

        self.classes_, y_reverse = np.unique(y, return_inverse=True)

        # construct distance measure
        self.distance_ = self.df(X)

        # build models
        models = np.zeros((len(self.classes_), X.shape[1]))
        for c_idx, c_value in enumerate(self.classes_):
            models[c_idx, :] = self.build_for_class(X[y == c_value])

        self.models_ = models

        return self

    def predict_(self, X):
        X = check_array(X)
        # calculate similarity for the inputs
        return self.distance_.pairwise(X, self.models_)

    def predict(self, X):
        R = self.predict_(X)
        # reduce by taking the one with minimum distance
        return self.classes_.take(np.argmin(R, 1))

    def predict_proba(self, X):
        R = self.predict_(X)
        return 1.0 - normalize(R, 'l1')

class EnsembleMultimodalEvolutionaryClassifier(BaseEstimator, ClassifierMixin):

    def __init__(self, n_iterations=10, n_models=3, random_state=None, sample_size=10, n_iterations_weights=10):
        self.n_iterations = n_iterations
        self.n_models = n_models
        self.random_state = random_state
        self.sample_size = sample_size
        self.n_iterations_weights = n_iterations_weights

    def get_params(self, deep=False):
        return {"n_iterations": self.n_iterations,
                "n_models": self.n_models,
                "random_state": self.random_state,
                "sample_size": self.sample_size,
                "n_iterations_weights": self.n_iterations_weights}

    def set_params(self, **kwargs):
        for key, value in kwargs.items():
            self.setattr(key, value)
        return self

    def build_for_class(self, rs, X):

        def distance_fitness(c):
            return np.sum(np.abs(X - c))

        # setup GA
        ga = GeneticAlgorithm(fitness_function=helper_fitness(distance_fitness),
                              elitism=3,
                              n_chromosomes=100,
                              n_genes=X.shape[1],
                              p_mutation=0.3,
                              random_state=rs)

        ga = helper_n_generations(ga, self.n_iterations)  # advance the GA

        # return the best found parameters for this class.
        chromosomes, fitness = ga.best(1)
        return chromosomes[0]

    def fit_weights(self, rs, models, X, y):

        n_genes = self.n_models * len(self.classes_)

        def fitness_function(c):
            M = self.predict_(X, models, c)
            y_pred = np.argmin(M, 1)
            return 1.0 - accuracy_score(y, y_pred)

        ga = GeneticAlgorithm(fitness_function=helper_fitness(fitness_function),
                              elitism=3,
                              n_chromosomes=100,
                              n_genes=n_genes,
                              p_mutation=0.3,
                              random_state=rs)

        ga = helper_n_generations(ga, self.n_iterations_weights)  # advance the GA

        chromosomes, fitness = ga.best(1)

        return chromosomes[0]

    def fit(self, X, y):
        X = check_array(X)

        random_state = check_random_state(self.random_state)

        self.classes_, y_reverse = np.unique(y, return_inverse=True)

        if np.nan in self.classes_:
            raise ValueError("NaN class not supported.")

        # build models
        models = {}
        for c_idx, c_value in enumerate(self.classes_):
            X_class = X[y == c_value]
            a_sample_size = min(len(X_class), self.sample_size)
            c_models = []
            for i in range(self.n_models):
                # resample
                X_sample = X_class[random_state.choice(len(X_class), a_sample_size)]
                c_models.append(self.build_for_class(random_state, X_sample))
            models[c_value] = np.array(c_models)

        weights = self.fit_weights(random_state, models, X, y_reverse)

        self.models_ = models
        self.weights_ = weights

        return self

    def predict_(self, X, models, weights):
        X = check_array(X)

        M = np.zeros((len(X), len(self.classes_)))
        R = np.zeros((len(X), self.n_models))

        # calculate similarity for the inputs
        for c_idx, c_value in enumerate(self.classes_):
            for m_idx, model in enumerate(models[c_value]):
                R[:, m_idx] = np.sum(np.abs(X - model), 1)
            M[:, c_idx] = weights[c_idx] * np.sum(R, 1)

        return M

    def predict(self, X):

        M = self.predict_(X, self.models_, self.weights_)

        # reduce by taking the one with minimum distance
        return self.classes_.take(np.argmin(M, 1))

    def predict_proba(self, X):

        M = self.predict_(X, self.models_, self.weights_)

        return 1.0 - normalize(M, 'l1')


text:
# Copyright 1999-2021 Alibaba Group Holding Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import warnings

import numpy as np

from ... import tensor as mt
from ..utils.checks import assert_all_finite
from ..utils.multiclass import type_of_target
from ..utils.validation import check_consistent_length, column_or_1d

def auc(x, y, session=None, run_kwargs=None):
    """Compute Area Under the Curve (AUC) using the trapezoidal rule

    This is a general function, given points on a curve.  For computing the
    area under the ROC-curve, see :func:`roc_auc_score`.  For an alternative
    way to summarize a precision-recall curve, see
    :func:`average_precision_score`.

    Parameters
    ----------
    x : tensor, shape = [n]
        x coordinates. These must be either monotonic increasing or monotonic
        decreasing.
    y : tensor, shape = [n]
        y coordinates.

    Returns
    -------
    auc : tensor, with float value

    Examples
    --------
    >>> import mars.tensor as mt
    >>> from mars.learn import metrics
    >>> y = mt.array([1, 1, 2, 2])
    >>> pred = mt.array([0.1, 0.4, 0.35, 0.8])
    >>> fpr, tpr, thresholds = metrics.roc_curve(y, pred, pos_label=2)
    >>> metrics.auc(fpr, tpr)
    0.75

    See also
    --------
    roc_auc_score : Compute the area under the ROC curve
    average_precision_score : Compute average precision from prediction scores
    precision_recall_curve :
        Compute precision-recall pairs for different probability thresholds
    """
    check_consistent_length(x, y)
    x = column_or_1d(x)
    y = column_or_1d(y)

    if x.shape[0] < 2:
        raise ValueError(
            "At least 2 points are needed to compute"
            f" area under curve, but x.shape = {x.shape}"
        )

    direction = 1
    dx = mt.diff(x)
    any_dx_lt_0 = mt.any(dx < 0)
    all_dx_le_0 = mt.all(dx <= 0)
    mt.ExecutableTuple([x, any_dx_lt_0, all_dx_le_0]).execute(
        session=session, **(run_kwargs or dict())
    )
    if any_dx_lt_0.fetch(session=session):
        if all_dx_le_0.fetch(session=session):
            direction = -1
        else:
            x_data = x.fetch(session=session)
            raise ValueError(f"x is neither increasing nor decreasing : {x_data}.")

    area = direction * mt.trapz(y, x)
    return area.execute(session=session, **(run_kwargs or dict()))

def _binary_clf_curve(
    y_true, y_score, pos_label=None, sample_weight=None, session=None, run_kwargs=None
):
    """Calculate true and false positives per binary classification threshold.

    Parameters
    ----------
    y_true : tensor, shape = [n_samples]
        True targets of binary classification

    y_score : tensor, shape = [n_samples]
        Estimated probabilities or decision function

    pos_label : int or str, default=None
        The label of the positive class

    sample_weight : array-like of shape (n_samples,), default=None
        Sample weights.

    Returns
    -------
    fps : tensor, shape = [n_thresholds]
        A count of false positives, at index i being the number of negative
        samples assigned a score >= thresholds[i]. The total number of
        negative samples is equal to fps[-1] (thus true negatives are given by
        fps[-1] - fps).

    tps : tensor, shape = [n_thresholds <= len(mt.unique(y_score))]
        An increasing count of true positives, at index i being the number
        of positive samples assigned a score >= thresholds[i]. The total
        number of positive samples is equal to tps[-1] (thus false negatives
        are given by tps[-1] - tps).

    thresholds : tensor, shape = [n_thresholds]
        Decreasing score values.
    """
    y_type = type_of_target(y_true).to_numpy(session=session, **(run_kwargs or dict()))
    if not (y_type == "binary" or (y_type == "multiclass" and pos_label is not None)):
        raise ValueError(f"{y_type} format is not supported")

    check_consistent_length(y_true, y_score, sample_weight)
    y_true = column_or_1d(y_true)
    y_score = column_or_1d(y_score)
    y_true = assert_all_finite(y_true, check_only=False)
    y_score = assert_all_finite(y_score, check_only=False)

    if sample_weight is not None:
        sample_weight = column_or_1d(sample_weight)

    # ensure binary classification if pos_label is not specified
    # classes.dtype.kind in ('O', 'U', 'S') is required to avoid
    # triggering a FutureWarning by calling np.array_equal(a, b)
    # when elements in the two arrays are not comparable.
    classes = mt.unique(y_true, aggregate_size=1).to_numpy(
        session=session, **(run_kwargs or dict())
    )
    if pos_label is None and (
        classes.dtype.kind in ("O", "U", "S")
        or not (
            np.array_equal(classes, [0, 1])
            or np.array_equal(classes, [-1, 1])
            or np.array_equal(classes, [0])
            or np.array_equal(classes, [-1])
            or np.array_equal(classes, [1])
        )
    ):
        classes_repr = ", ".join(repr(c) for c in classes)
        raise ValueError(
            f"y_true takes value in {{{classes_repr}}} and "
            "pos_label is not specified: either make y_true "
            "take value in {{0, 1}} or {{-1, 1}} or "
            "pass pos_label explicitly."
        )
    elif pos_label is None:
        pos_label = 1.0

    # make y_true a boolean vector
    y_true = y_true == pos_label

    # sort scores and corresponding truth values
    desc_score_indices = mt.argsort(y_score, kind="mergesort")[::-1]
    y_score = y_score[desc_score_indices]
    y_true = y_true[desc_score_indices]
    if sample_weight is not None:
        weight = sample_weight[desc_score_indices]
    else:
        weight = 1.0

    # y_score typically has many tied values. Here we extract
    # the indices associated with the distinct values. We also
    # concatenate a value for the end of the curve.
    distinct_value_indices = mt.where(mt.diff(y_score))[0]
    threshold_idxs = mt.r_[distinct_value_indices, y_true.size - 1]

    # accumulate the true positives with decreasing threshold
    tps = (y_true * weight).cumsum()[threshold_idxs]
    if sample_weight is not None:
        # express fps as a cumsum to ensure fps is increasing even in
        # the presence of floating point errors
        fps = ((1 - y_true) * weight).cumsum()[threshold_idxs]
    else:
        fps = 1 + threshold_idxs - tps
    ret = mt.ExecutableTuple([fps, tps, y_score[threshold_idxs]])
    return ret.execute(session=session, **(run_kwargs or dict()))

def _binary_roc_auc_score(
    y_true, y_score, sample_weight=None, max_fpr=None, session=None, run_kwargs=None
):
    """Binary roc auc score."""

    from numpy import interp

    if len(mt.unique(y_true).execute()) != 2:
        raise ValueError(
            "Only one class present in y_true. ROC AUC score "
            "is not defined in that case."
        )

    fpr, tpr, _ = roc_curve(
        y_true,
        y_score,
        sample_weight=sample_weight,
        session=session,
        run_kwargs=run_kwargs,
    )
    fpr, tpr = mt.ExecutableTuple([fpr, tpr]).fetch(session=session)

    if max_fpr is None or max_fpr == 1:
        return auc(fpr, tpr, session=session, run_kwargs=run_kwargs).fetch(
            session=session
        )
    if max_fpr <= 0 or max_fpr > 1:
        raise ValueError(f"Expected max_fpr in range (0, 1], got: {max_fpr}")

    # Add a single point at max_fpr by linear interpolation
    stop = (
        mt.searchsorted(fpr, max_fpr, "right")
        .execute(session=session, **(run_kwargs or dict()))
        .fetch(session=session)
    )
    x_interp = [fpr[stop - 1], fpr[stop]]
    y_interp = [tpr[stop - 1], tpr[stop]]
    tpr = list(tpr[:stop])
    tpr.append(interp(max_fpr, x_interp, y_interp))
    fpr = list(fpr[:stop])
    fpr.append(max_fpr)
    partial_auc = auc(fpr, tpr, session=session, run_kwargs=run_kwargs)

    # McClish correction: standardize result to be 0.5 if non-discriminant
    # and 1 if maximal
    min_area = 0.5 * max_fpr ** 2
    max_area = max_fpr
    return 0.5 * (
        1 + (partial_auc.fetch(session=session) - min_area) / (max_area - min_area)
    )

def roc_curve(
    y_true,
    y_score,
    pos_label=None,
    sample_weight=None,
    drop_intermediate=True,
    session=None,
    run_kwargs=None,
):
    """Compute Receiver operating characteristic (ROC)

    Note: this implementation is restricted to the binary classification task.

    Read more in the :ref:`User Guide <roc_metrics>`.

    Parameters
    ----------

    y_true : tensor, shape = [n_samples]
        True binary labels. If labels are not either {-1, 1} or {0, 1}, then
        pos_label should be explicitly given.

    y_score : tensor, shape = [n_samples]
        Target scores, can either be probability estimates of the positive
        class, confidence values, or non-thresholded measure of decisions
        (as returned by "decision_function" on some classifiers).

    pos_label : int or str, default=None
        The label of the positive class.
        When ``pos_label=None``, if y_true is in {-1, 1} or {0, 1},
        ``pos_label`` is set to 1, otherwise an error will be raised.

    sample_weight : array-like of shape (n_samples,), default=None
        Sample weights.

    drop_intermediate : boolean, optional (default=True)
        Whether to drop some suboptimal thresholds which would not appear
        on a plotted ROC curve. This is useful in order to create lighter
        ROC curves.

        .. versionadded:: 0.17
           parameter *drop_intermediate*.

    Returns
    -------
    fpr : tensor, shape = [>2]
        Increasing false positive rates such that element i is the false
        positive rate of predictions with score >= thresholds[i].

    tpr : tensor, shape = [>2]
        Increasing true positive rates such that element i is the true
        positive rate of predictions with score >= thresholds[i].

    thresholds : tensor, shape = [n_thresholds]
        Decreasing thresholds on the decision function used to compute
        fpr and tpr. `thresholds[0]` represents no instances being predicted
        and is arbitrarily set to `max(y_score) + 1`.

    See also
    --------
    roc_auc_score : Compute the area under the ROC curve

    Notes
    -----
    Since the thresholds are sorted from low to high values, they
    are reversed upon returning them to ensure they correspond to both ``fpr``
    and ``tpr``, which are sorted in reversed order during their calculation.

    References
    ----------
    .. [1] `Wikipedia entry for the Receiver operating characteristic
            <https://en.wikipedia.org/wiki/Receiver_operating_characteristic>`_

    .. [2] Fawcett T. An introduction to ROC analysis[J]. Pattern Recognition
           Letters, 2006, 27(8):861-874.

    Examples
    --------
    >>> import mars.tensor as mt
    >>> from mars.learn import metrics
    >>> y = mt.array([1, 1, 2, 2])
    >>> scores = mt.array([0.1, 0.4, 0.35, 0.8])
    >>> fpr, tpr, thresholds = metrics.roc_curve(y, scores, pos_label=2)
    >>> fpr
    array([0. , 0. , 0.5, 0.5, 1. ])
    >>> tpr
    array([0. , 0.5, 0.5, 1. , 1. ])
    >>> thresholds
    array([1.8 , 0.8 , 0.4 , 0.35, 0.1 ])

    """
    from sklearn.exceptions import UndefinedMetricWarning

    fps, tps, thresholds = _binary_clf_curve(
        y_true,
        y_score,
        pos_label=pos_label,
        sample_weight=sample_weight,
        session=session,
        run_kwargs=run_kwargs,
    )

    # Attempt to drop thresholds corresponding to points in between and
    # collinear with other points. These are always suboptimal and do not
    # appear on a plotted ROC curve (and thus do not affect the AUC).
    # Here mt.diff(_, 2) is used as a "second derivative" to tell if there
    # is a corner at the point. Both fps and tps must be tested to handle
    # thresholds with multiple data points (which are combined in
    # _binary_clf_curve). This keeps all cases where the point should be kept,
    # but does not drop more complicated cases like fps = [1, 3, 7],
    # tps = [1, 2, 4]; there is no harm in keeping too many thresholds.
    if drop_intermediate and len(fps) > 2:
        optimal_idxs = mt.where(
            mt.r_[True, mt.logical_or(mt.diff(fps, 2), mt.diff(tps, 2)), True]
        )[0]
        fps = fps[optimal_idxs]
        tps = tps[optimal_idxs]
        thresholds = thresholds[optimal_idxs]

    # Add an extra threshold position
    # to make sure that the curve starts at (0, 0)
    tps = mt.r_[0, tps]
    fps = mt.r_[0, fps]
    thresholds = mt.r_[thresholds[0] + 1, thresholds]

    last_fps = fps[-1]
    last_tps = tps[-1]
    mt.ExecutableTuple([tps, fps, last_fps, last_tps]).execute(
        session=session, **(run_kwargs or dict())
    )
    last_fps, last_tps = mt.ExecutableTuple([last_fps, last_tps]).fetch(session=session)

    if last_fps <= 0:
        warnings.warn(
            "No negative samples in y_true, "
            "false positive value should be meaningless",
            UndefinedMetricWarning,
        )
        fpr = mt.repeat(mt.nan, fps.shape)
    else:
        fpr = fps / last_fps

    if last_tps <= 0:
        warnings.warn(
            "No positive samples in y_true, "
            "true positive value should be meaningless",
            UndefinedMetricWarning,
        )
        tpr = mt.repeat(mt.nan, tps.shape)
    else:
        tpr = tps / last_tps

    ret = mt.ExecutableTuple([fpr, tpr, thresholds]).execute(
        session=session, **(run_kwargs or dict())
    )
    return ret

ext: py

max_stars_repo_name: wjsi/mars

max_issues_repo_name: wjsi/mars

max_forks_repo_name: wjsi/mars

reason: import numpy,from numpy


text:
import numpy as np
import tensorflow as tf

from src.nn.config import NnConfig as config
from src.nn.config import NnConst as const
from src.nn.config import setup_training_environment, load_training_data_set

def create_fully_connected_layer(
        name, in_node, in_size, num_of_neurons,
        is_output_layer, dropout
):
    """
    Create fully connected layer's variables and use them to transform in-node
    into out-node with optional activation and specified dropout. There is no
    need to add activation rescaling for no-dropout case, because tensorflow
    rescales outputs with dropout automatically (scales up by 1 / keep_prob).

    :param name: name which will be given to the layer
    :param in_node: input tensorflow graph node (vector)
    :param in_size: size of the input vector
    :param num_of_neurons: number of neurons for created layer
    :param is_output_layer: states whether created layer is network's last
                            layer. In this case no activation function is
                            used
    :param dropout: probability of dropout for this layer (used in training)
    :return:
    """

    weights = tf.get_variable(
        name + "/weights",
        shape=[in_size, num_of_neurons],
        initializer=tf.contrib.layers.xavier_initializer()
    )
    biases = tf.get_variable(
        name + "/biases",
        shape=[1, num_of_neurons],
        initializer=tf.constant_initializer()
    )

    out = tf.matmul(in_node, weights) + biases

    # Add histograms.
    tf.summary.histogram(weights.op.name, weights)
    tf.summary.histogram(biases.op.name, biases)
    tf.summary.histogram(out.op.name, out)

    if is_output_layer:
        return tf.nn.dropout(out, 1 - dropout)
    else:
        return tf.nn.dropout(tf.maximum(out, out * 0.1), 1 - dropout)

def create_graph(
        in_placeholder, in_size, layer_sizes,
        labels, learning_rate_placeholder, dropout_placeholder
):
    """
    Creates graph for basic neural network both for training and teest case.

    :param in_placeholder: placeholder for input data batch
    :param in_size: size of input vector
    :param layer_sizes: list of neurons count for each layer
    :param labels: set of labels used to evaluate network
    :param learning_rate_placeholder: placeholder for network run's learning
                                      rate
    :param dropout_placeholder: placeholder for network's dropout probability
    :return: created out, loss, train and accuracy nodes
    """

    # Create network graph.
    previous_in = in_placeholder
    previous_in_size = in_size
    for layer_size, i in zip(layer_sizes, range(len(layer_sizes))):
        previous_in = create_fully_connected_layer(
            "layer_{}".format(i),
            previous_in, previous_in_size,
            layer_size,
            i == len(layer_sizes) - 1,
            dropout_placeholder
        )
        previous_in_size = layer_size

    # Create out, loss, train and accuracy nodes.
    out = tf.nn.softmax(previous_in)
    loss = tf.reduce_mean(
        tf.nn.softmax_cross_entropy_with_logits(
            labels=labels, logits=previous_in
        )
    )
    train = tf.train.AdamOptimizer(
        learning_rate_placeholder
    ).minimize(loss)
    accuracy = tf.reduce_mean(
        tf.cast(
            tf.equal(
                tf.argmax(previous_in, axis=1), tf.argmax(labels, axis=1)
            ),
            tf.float32
        )
    )

    return out, loss, train, accuracy

def perform_usage_run(run_index, data_set_path, out_path):
    """
    Perform single network's forward pass by using best model saved for given
    run index. Save results to output file on given path.

    :param run_index: index of run which will be source for used model
    :param data_set_path: path to data set file which will be processed
    :param out_path: path to file which will contain forward pass results
    :return: -
    """

    # Fill in usage model path pattern.
    usage_model_path = const.USAGE_MODEL_PATH_PATTERN.format(run_index)

    # Load data.
    data = np.load(data_set_path)

    # Create network's placeholders.
    in_placeholder = tf.placeholder(tf.float32, shape=[None, config.IN_SIZE])
    labels_placeholder = tf.placeholder(
        tf.float32, shape=[None, config.NUM_OF_LABELS]
    )
    learning_rate_placeholder = tf.placeholder(tf.float32, shape=[])
    dropout_placeholder = tf.placeholder(tf.float32, shape=[])

    # Create network's graph.
    out, _, _, _ = create_graph(
        in_placeholder, config.IN_SIZE, config.LAYER_SIZES, labels_placeholder,
        learning_rate_placeholder, dropout_placeholder
    )

    # Create session and initialize all variables.
    session = tf.Session()
    session.run(tf.global_variables_initializer())

    # Create model saver and restore best model.
    saver = tf.train.Saver()
    saver.restore(session, usage_model_path)

    # Perform one forward pass.
    calculated_out = session.run(
        [out],
        feed_dict={
            in_placeholder: data, dropout_placeholder: config.NO_DROPOUT
        }
    )

    # Save results to file and exit.
    np.save(out_path, calculated_out)

def train_network(run_index, set_index, mode, learning_rate):
    """
    Start/continue basic nn training for given run index and with data set
    described by set index.

    :param run_index:
    :param set_index:
    :param mode:
    :param learning_rate:
    :return:
    """

    # Setup training environment.
    data_set_dir, train_logs_dir, validation_logs_dir, \
    checkpoints_dir, best_model_dir = setup_training_environment(
        const, run_index, set_index, mode
    )

    train_data, train_labels, \
    validation_data, validation_labels, \
    test_data, test_labels = load_training_data_set(data_set_dir)

    # Create network's placeholders.
    in_placeholder = tf.placeholder(tf.float32, shape=[None, config.IN_SIZE])
    labels_placeholder = tf.placeholder(
        tf.float32, shape=[None, config.NUM_OF_LABELS]
    )
    learning_rate_placeholder = tf.placeholder(tf.float32, shape=[])
    dropout_placeholder = tf.placeholder(tf.float32, shape=[])

    # Create network's graph.
    out, loss, train, accuracy = create_graph(
        in_placeholder, config.IN_SIZE, config.LAYER_SIZES, labels_placeholder,
        learning_rate_placeholder, dropout_placeholder
    )

    # Create rest of summary and train, and validation logs writers.
    tf.summary.scalar("loss", loss)
    tf.summary.scalar("accuracy", accuracy)
    merged_summary = tf.summary.merge_all()
    train_logger = tf.summary.FileWriter(train_logs_dir)
    validation_logger = tf.summary.FileWriter(validation_logs_dir)

    # Create session and initialize all variables.
    session = tf.Session()
    session.run(tf.global_variables_initializer())

    lowest_loss = None

    if config.MODE == config.CONTINUE_MODE:
        # Might be not needed.
        pass

    # Create model saver which will keep all checkpoints.
    saver = tf.train.Saver(max_to_keep=config.MAX_CHECKPOINTS)

    for i in range(config.MAX_ITERATIONS):

        # Sample train data.
        batch_indices = np.random.permutation(train_data.shape[0])
        batch_indices = batch_indices[:config.TRAIN_BATCH_SIZE]
        batch_data = train_data[batch_indices]
        batch_labels = train_labels[batch_indices]

        # Perform normal train run.
        summary, _ = session.run(
            [merged_summary, train],
            feed_dict={
                in_placeholder: batch_data,
                labels_placeholder: batch_labels,
                learning_rate_placeholder: learning_rate,
                dropout_placeholder: config.DROPOUT
            }
        )

        train_logger.add_summary(summary, i)

        # Save checkpoint for model.
        if i % config.CHECKPOINT_INTERVAL == 0:
            saver.save(session, checkpoints_dir + "/model_checkpoint", i)

        # Validation run.
        if i % config.VALIDATION_INTERVAL == 0:
            summary, loss_value, accuracy_value = session.run(
                [merged_summary, loss, accuracy],
                feed_dict={
                    in_placeholder: validation_data,
                    labels_placeholder: validation_labels,
                    dropout_placeholder: config.NO_DROPOUT
                }
            )

            # If we obtained better validation loss, save the model.
            if lowest_loss is None or loss_value < lowest_loss:
                lowest_loss = loss_value
                saver.save(session, best_model_dir + "/best_model")

            validation_logger.add_summary(summary, i)

            # Also output some data to console.
            print(
                "Iteration {}:\n\tLoss: {}\n\tAccuracy: {}\n".format(
                    i, loss_value, accuracy_value
                )
            )

        # Learning rate decay.
        if i % config.DECAY_INTERVAL == 0 and i != 0:
            learning_rate *= config.LEARNING_RATE_DECAY

    # Test run.
    saver.save(session, checkpoints_dir + "/model_checkpoint_last")
    saver.restore(session, best_model_dir + "/best_model")
    loss_value, accuracy_value = session.run(
        [loss, accuracy],
        feed_dict={
            in_placeholder: test_data,
            labels_placeholder: test_labels,
            dropout_placeholder: config.NO_DROPOUT
        }
    )
    print(
        "Final Loss: {}\n\tFinal Accuracy: {}\n".format(
            loss_value, accuracy_value
        )
    )

def main(MODE):
    """
    Main function of this script.
    :param MODE: run mode
    :return: -
    """
    if MODE == const.USE_MODE:
        perform_usage_run(
            const.RUN_INDEX, const.USAGE_DATA_SET_PATH, const.USAGE_OUT_PATH
        )
    else:
        train_network(
            const.RUN_INDEX, const.SET_INDEX, const.MODE, config.LEARNING_RATE
        )

if __name__ == "__main__":
    main(const.USE_MODE)

max_stars_repo_name: puchake/market-teller

max_issues_repo_name: puchake/market-teller

max_forks_repo_name: puchake/market-teller

reason: import numpy


text:
from numpy import sum

from NGLostBarnN import NGLostBarnN

def NGLostBarnNSum(NYrs, NGPctManApp, GrazingAnimal_0, NumAnimals, AvgAnimalWt, AnimalDailyN, NGBarnNRate, Prec,
                   DaysMonth, AWMSNgPct, NgAWMSCoeffN, RunContPct, RunConCoeffN):
    # result = zeros((NYrs,))
    result = sum(
        NGLostBarnN(NYrs, NGPctManApp, GrazingAnimal_0, NumAnimals, AvgAnimalWt, AnimalDailyN, NGBarnNRate, Prec,
                    DaysMonth, AWMSNgPct, NgAWMSCoeffN, RunContPct, RunConCoeffN), axis=1)
    return result

def NGLostBarnNSum_f():
    pass

max_stars_repo_name: mudkipmaster/gwlf-e

max_issues_repo_name: mudkipmaster/gwlf-e

max_forks_repo_name: mudkipmaster/gwlf-e

reason: from numpy


text:
import numpy as np
import pandas as pd
import math
import torch
import torchvision
import matplotlib.pyplot as plt
import os
import argparse

assert(torch.__version__ <= '1.1.0')

from data_utils import *
from model_utils import *
from collections import OrderedDict
from datetime import datetime
from PIL import Image

os.environ['KMP_WARNINGS'] = 'off'

date = datetime.now().strftime('%Y-%m-%d')
time = datetime.now().strftime('%H-%M-%S')
path = 'results/'+date+'/'+time
if not(os.path.exists(path)):
    os.makedirs(path)

parser = argparse.ArgumentParser(description='Binary VGG with meta')

parser.add_argument('--lr', type = float, default = 0.01, metavar = 'LR', help='Learning rate')
parser.add_argument('--mbs', type = int, default = 32, metavar = 'MBS', help='MiniBatch size')
parser.add_argument('--channels', nargs = '+', type = int, default = [128,128,256,256,512,512], metavar = 'C', help='channels')
parser.add_argument('--epochs-per-task', type = int, default = 400, metavar = 'EPT', help='Number of epochs per tasks')
parser.add_argument('--meta', type = float, default = 0.0, metavar = 'M', help='Metaplasticity coefficient')
parser.add_argument('--weight-decay', type = float, default = 0.0, metavar = 'Wd', help='weight decay')
parser.add_argument('--init', type = str, default = 'gauss', metavar = 'INIT', help='Weight initialisation')
parser.add_argument('--init-width', type = float, default = 0.01, metavar = 'W', help='Weight initialisation width')
parser.add_argument('--save', type = bool, default = True, metavar = 'S', help='Saving the results')
parser.add_argument('--source', type = str, default = 'pytorch', metavar = 'Sc', help='Source of cifar10')
parser.add_argument('--nb-subset', type = int, default = 1, metavar = 'Sub', help='Number of sub datasets')
parser.add_argument('--device', type = int, default = 0, metavar = 'Dev', help='choice of gpu')

args = parser.parse_args()

device = torch.device("cuda:"+str(args.device) if torch.cuda.is_available() else "cpu")

createHyperparametersFile(path, args)

# Hyperparameters
lr = args.lr
epochs = args.epochs_per_task
save_result = args.save
meta = args.meta
model = ConvBNN(init = args.init, width = args.init_width, channels = args.channels).to(device)
model.apply(normal_init)

print(date+'_'+time)
print(model)
plot_parameters(model, path, save=save_result)

train_loader_list, test_loader_list, full_train_loader = create_loader_list(args)

# To see that source torchvision gives always the same partition whereas source keras can be permutted
#imgs, lbls = iter(train_loader_list[0]).next()
#img = torchvision.utils.make_grid(imgs[0:args.mbs], nrow = 8)
#npimg = img.numpy()
#plt.figure(figsize=(16,9))
#plt.imshow(np.transpose(npimg, (1,2,0)))
#plt.savefig(path+'/samples_'+args.source+'.png', format = 'png')
#plt.close()

# Data collect initialisation
data = {}
data['net'] = 'BinaryCNN'
arch = 'VGG'

data['arch'] = arch
data['lr'], data['mbs'], data['meta'], data['task_order'], data['tsk'], data['epoch'] = [], [], [], [], [], []
data['acc_tr_sub'] = []
data['acc_tr'], data['acc_test'] = [], []

name = '_'+data['net']+'_'+data['arch']+'_'+str(args.nb_subset)+'CIFAR10'

optimizer = Adam_meta(model.parameters(), lr = lr, meta = meta, weight_decay = args.weight_decay)

for task_idx, task in enumerate(train_loader_list):

    for epoch in range(1, epochs+1):

        train(model, task, task_idx, optimizer, device)

        data['task_order'].append(task_idx+1)
        data['tsk'].append('CIFAR10_'+str(task_idx+1)+'/'+str(args.nb_subset))
        data['epoch'].append(epoch)
        data['lr'].append(optimizer.param_groups[0]['lr'])
        data['mbs'].append(task.batch_size)
        data['meta'].append(meta)

        data['acc_tr_sub'].append(test(model, task, device, frac = args.nb_subset))
        data['acc_tr'].append(test(model, full_train_loader, device))

        test_accuracy = test(model, test_loader_list[0], device)
        data['acc_test'].append(test_accuracy)

        if (epoch%10==0) and save_result:
            time = datetime.now().strftime('%H-%M-%S')
            df_data = pd.DataFrame(data)
            df_data.to_csv(path +'/'+time+name+'_epoch_'+str(epoch)+'.csv', index = False)
            plot_parameters(model, path, save=save_result)

time = datetime.now().strftime('%H-%M-%S')
df_data = pd.DataFrame(data)
if save_result:
    df_data.to_csv(path +'/'+time+name+'.csv', index = False)

torch.save({'model_state_dict': model.state_dict(), 'opt': optimizer.state_dict() },  path + '/checkpoint.tar')

max_stars_repo_name: justincurl/SynapticMetaplasticityBNN

max_issues_repo_name: justincurl/SynapticMetaplasticityBNN

max_forks_repo_name: justincurl/SynapticMetaplasticityBNN


text:
# Double buffer read memory implementation
# TODO: Verification Pending
import math
import numpy as np
from tqdm import tqdm

from scalesim.memory.read_port import read_port

class read_buffer:
    def __init__(self):
        # Buffer properties: User specified
        self.total_size_bytes = 128
        self.word_size = 1                      # Bytes
        self.active_buf_frac = 0.9
        self.hit_latency = 1                    # Cycles after which a request is served if already in the buffer

        # Buffer properties: Calculated
        self.total_size_elems = math.floor(self.total_size_bytes / self.word_size)
        self.active_buf_size = int(math.ceil(self.total_size_elems * 0.9))
        self.prefetch_buf_size = self.total_size_elems - self.active_buf_size

        # Backing interface properties
        self.backing_buffer = read_port()
        self.req_gen_bandwidth = 100            # words per cycle

        # Status of the buffer
        self.hashed_buffer = dict()
        self.active_buffer_set_limits = []
        self.prefetch_buffer_set_limits = []

        # Variables to enable prefetching
        self.fetch_matrix = np.ones((1, 1))
        self.last_prefect_cycle = -1
        self.next_line_prefetch_idx = 0
        self.next_col_prefetch_idx = 0

        # Access counts
        self.num_access = 0

        # Trace matrix
        self.trace_matrix = np.ones((1, 1))

        # Flags
        self.active_buf_full_flag = False
        self.hashed_buffer_valid = False
        self.trace_valid = False

    #
    def set_params(self, backing_buf_obj,
                   total_size_bytes=1, word_size=1, active_buf_frac=0.9,
                   hit_latency=1, backing_buf_bw=1
                   ):

        self.total_size_bytes = total_size_bytes
        self.word_size = word_size

        assert 0.5 <= active_buf_frac < 1, "Valid active buf frac [0.5,1)"
        self.active_buf_frac = round(active_buf_frac, 2)
        self.hit_latency = hit_latency

        self.backing_buffer = backing_buf_obj
        self.req_gen_bandwidth = backing_buf_bw

        # Calculate these based on the values provided
        self.total_size_elems = math.floor(self.total_size_bytes / self.word_size)
        self.active_buf_size = int(math.ceil(self.total_size_elems * self.active_buf_frac))
        self.prefetch_buf_size = self.total_size_elems - self.active_buf_size

    #
    def reset(self): # TODO: check if all resets are working propoerly
        # Buffer properties: User specified
        self.total_size_bytes = 128
        self.word_size = 1  # Bytes
        self.active_buf_frac = 0.9
        self.hit_latency = 1  # Cycles after which a request is served if already in the buffer

        # Buffer properties: Calculated
        self.total_size_elems = math.floor(self.total_size_bytes / self.word_size)
        self.active_buf_size = int(math.ceil(self.total_size_elems * 0.9))
        self.prefetch_buf_size = self.total_size_elems - self.active_buf_size

        # Backing interface properties
        self.backing_buffer = read_port()
        self.req_gen_bandwidth = 100  # words per cycle

        # Status of the buffer
        self.hashed_buffer = dict()
        self.active_buffer_set_limits = []
        self.prefetch_buffer_set_limits = []

        # Variables to enable prefetching
        self.fetch_matrix = np.ones((1, 1))
        self.last_prefect_cycle = -1
        self.next_line_prefetch_idx = 0
        self.next_col_prefetch_idx = 0

        # Access counts
        self.num_access = 0

        # Trace matrix
        self.trace_matrix = np.ones((1, 1))

        # Flags
        self.active_buf_full_flag = False
        self.hashed_buffer_valid = False
        self.trace_valid = False

    #
    def set_fetch_matrix(self, fetch_matrix_np):
        # The operand matrix determines what to pre-fetch into both active and prefetch buffers
        # req_gen_bandwidth is set to 100 by default and will be used in 'calc' mode
        # In 'user' mode, this will be set in the set_params

        num_elems = fetch_matrix_np.shape[0] * fetch_matrix_np.shape[1]
        num_lines = int(math.ceil(num_elems / self.req_gen_bandwidth))
        self.fetch_matrix = np.ones((num_lines, self.req_gen_bandwidth)) * -1

        # Put stuff into the fetch matrix
        # This is done to ensure that there is no shape mismatch
        # Not sure if this is the optimal way to do it or not
        for i in range(num_elems):
            src_row = math.floor(i / fetch_matrix_np.shape[1])
            src_col = math.floor(i % fetch_matrix_np.shape[1])

            dest_row = math.floor(i / self.req_gen_bandwidth)
            dest_col = math.floor(i % self.req_gen_bandwidth)

            self.fetch_matrix[dest_row][dest_col] = fetch_matrix_np[src_row][src_col]

        # Once the fetch matrices are set, populate the data structure for fast lookups and servicing
        self.prepare_hashed_buffer()

    #
    def prepare_hashed_buffer(self):
        elems_per_set = math.ceil(self.total_size_elems / 100)

        prefetch_rows = self.fetch_matrix.shape[0]
        prefetch_cols = self.fetch_matrix.shape[1]

        line_id = 0
        elem_ctr = 0
        current_line = set()

        for r in range(prefetch_rows):
            for c in range(prefetch_cols):
                elem = self.fetch_matrix[r][c]

                if not elem == -1:
                    current_line.add(elem)
                    elem_ctr += 1

                if not elem_ctr < elems_per_set:    # ie > or =
                    self.hashed_buffer[line_id] = current_line
                    line_id += 1
                    elem_ctr = 0
                    current_line = set()        # new set

        self.hashed_buffer[line_id] = current_line
        self.hashed_buffer_valid = True

    #
    def service_reads(self, incoming_requests_arr_np,   # 2D array with the requests
                            incoming_cycles_arr):       # 1D vector with the cycles at which req arrived
        # Service the incoming read requests
        # returns a cycles array corresponding to the requests buffer
        # Logic: Always check if an addr is in active buffer.
        #        If hit, return with hit latency
        #        Else, make the contents of prefetch buffer as active and then check
        #              finish till an ongoing prefetch is done before reassiging prefetch buffer

        if not self.active_buf_full_flag:
            start_cycle = incoming_cycles_arr[0][0]
            self.prefetch_active_buffer(start_cycle=start_cycle)    # Needs to use the entire operand matrix
                                                                    # keeping in mind the tile order and everything

        out_cycles_arr = []
        offset = self.hit_latency
        #for cycle, request_line in tqdm(zip(incoming_cycles_arr, incoming_requests_arr_np)):
        for i in tqdm(range(incoming_requests_arr_np.shape[0])):
            cycle = incoming_cycles_arr[i]
            request_line = set(incoming_requests_arr_np[i]) #shaves off a few seconds

            for addr in request_line:
                if addr == -1:
                    continue

                if addr not in self.active_buffer_contents: #this is super slow!!!
                    self.new_prefetch()
                    potential_stall_cycles = self.last_prefect_cycle - (cycle + offset)
                    offset += potential_stall_cycles        # Offset increments if there were potential stalls

            out_cycles = cycle + offset
            out_cycles_arr.append(out_cycles)

        out_cycles_arr_np = np.asarray(out_cycles_arr).reshape((len(out_cycles_arr), 1))

        return out_cycles_arr_np

    #
    def prefetch_active_buffer(self, start_cycle):
        # Depending on size of the active buffer, calculate the number of lines from op mat to fetch
        # Also, calculate the cycles arr for requests

        # 1. Preparing the requests:
        num_lines = math.ceil(self.active_buf_size / self.req_gen_bandwidth)
        if not num_lines < self.fetch_matrix.shape[0]:
            num_lines = self.fetch_matrix.shape[0]

        requested_data_size = num_lines * self.req_gen_bandwidth
        self.num_access += requested_data_size

        start_idx = 0
        end_idx = num_lines

        prefetch_requests = self.fetch_matrix[start_idx:end_idx, :]
        # 1.1 See if extra requests are made, if so nullify them
        self.next_col_prefetch_idx = 0
        if requested_data_size > self.active_buf_size:
            valid_cols = int(self.active_buf_size % self.req_gen_bandwidth)
            row = end_idx - 1
            self.next_col_prefetch_idx = valid_cols
            for col in range(valid_cols, self.req_gen_bandwidth):
                prefetch_requests[row][col] = -1

        # 2. Preparing the cycles array
        #    The start_cycle variable ensures that all the requests have been made before any incoming reads came
        cycles_arr = np.zeros((num_lines, 1))
        for i in range(cycles_arr.shape[0]):
            cycles_arr[i][0] = -1 * (num_lines - start_cycle - (i - self.backing_buffer.get_latency()))

        # 3. Send the request and get the response cycles count
        response_cycles_arr = self.backing_buffer.service_reads(incoming_cycles_arr=cycles_arr,
                                                                incoming_requests_arr_np=prefetch_requests)

        # 4. Update the variables
        self.last_prefect_cycle = int(response_cycles_arr[-1][0])

        # Update the trace matrix
        self.trace_matrix = np.concatenate((response_cycles_arr, prefetch_requests), axis=1)
        self.trace_valid = True

        # Set active buffer contents
        num_elems_to_fetch = min(self.active_buf_size, requested_data_size)
        req_col_size = prefetch_requests.shape[1]
        for i in range(num_elems_to_fetch):
            r = int(i / req_col_size)
            c = int(i % req_col_size)

            addr = prefetch_requests[r][c]
            if not addr == -1:
                self.active_buffer_contents.add(addr)
        self.active_buf_full_flag = True

        # Set the line to be prefetched next
        if requested_data_size > self.active_buf_size:
            self.next_line_prefetch_idx = num_lines % self.fetch_matrix.shape[0]
        else:
            self.next_line_prefetch_idx = (num_lines + 1) % self.fetch_matrix.shape[0]

    #
    def new_prefetch(self):
        # In a new prefetch, some portion of the original data needs to be deleted to accomodate the prefetched data
        # In this case we overwrite some data in the active buffer with the prefetched data
        # And then create a new prefetch request
        # Also return when the prefetched data was made available

        # 1. Rewrite the active buffer
        assert self.active_buf_full_flag, 'Active buffer is empty'
        for i in range(len(self.prefetch_buffer_contents)):
            data = self.prefetch_buffer_contents[i]
            self.active_buffer_contents.add(data)

        self.prefetch_buffer_contents = []

        # 2. Create the request
        start_idx = self.next_line_prefetch_idx
        num_lines = math.ceil(self.prefetch_buf_size / self.req_gen_bandwidth)
        end_idx = start_idx + num_lines
        requested_data_size = num_lines * self.req_gen_bandwidth
        self.num_access += requested_data_size

        # In care we need to circle back
        if end_idx > self.fetch_matrix.shape[0]:
            last_idx = self.fetch_matrix.shape[0]
            prefetch_requests = self.fetch_matrix[start_idx:,:]

            new_end_idx = min(end_idx - last_idx, start_idx)    # In case the entire array is engulfed
            prefetch_requests = np.concatenate((prefetch_requests, self.fetch_matrix[:new_end_idx,:]))
        else:
            prefetch_requests = self.fetch_matrix[start_idx:end_idx, :]

        # Modify the prefetch request to drop unwanted addresses
        # a. Chomp the elements in the first line included in previous fetches
        for i in range(0, self.next_col_prefetch_idx):
            prefetch_requests[0][i] = -1

        # b. Chomp the excess elements in the last line
        if requested_data_size > self.active_buf_size:
            valid_cols = int(self.active_buf_size % self.req_gen_bandwidth)
            row = end_idx - 1
            for col in range(valid_cols, self.req_gen_bandwidth):
                prefetch_requests[row][col] = -1

        # 3. Create the request cycles
        cycles_arr = np.zeros((num_lines, 1))
        for i in range(cycles_arr.shape[0]):
            cycles_arr[i][0] = self.last_prefect_cycle + i

        # 4. Send the request
        response_cycles_arr = self.backing_buffer.service_reads(incoming_cycles_arr=cycles_arr,
                                                                incoming_requests_arr_np=prefetch_requests)

        # 5. Update the variables
        self.last_prefect_cycle = response_cycles_arr[-1][0]

        this_prefetch_trace = np.concatenate((response_cycles_arr, prefetch_requests), axis=1)
        self.trace_matrix = np.concatenate((self.trace_matrix, this_prefetch_trace), axis=0)

        # Set the contents of the prefetch buffer
        num_prefetch_reqs = prefetch_requests.shape[0] * prefetch_requests.shape[1]
        elems_to_prefectch = min(self.prefetch_buf_size, num_prefetch_reqs)
        req_col_size = prefetch_requests.shape[1]
        for i in range(elems_to_prefectch):
            r = int(i / req_col_size)
            c = int(i % req_col_size)

            addr = prefetch_requests[r][c]
            self.prefetch_buffer_contents.append(addr)

        # Set the line to be prefetched next
        if requested_data_size > self.active_buf_size:
            self.next_line_prefetch_idx = num_lines % self.fetch_matrix.shape[0]
        else:
            self.next_line_prefetch_idx = (num_lines + 1) % self.fetch_matrix.shape[1]

        # This does not need to return anything
        #return response_cycles_arr

    #
    def get_trace_matrix(self):
        if not self.trace_valid:
            print('No trace has been generated yet')
            return

        return self.trace_matrix

    #
    def get_hit_latency(self):
        return self.hit_latency

    #
    def get_num_accesses(self):
        assert self.trace_valid, 'Traces not ready yet'
        return self.num_access

    #
    def get_external_access_start_stop_cycles(self):
        assert self.trace_valid, 'Traces not ready yet'
        start_cycle = self.trace_matrix[0][0]
        end_cycle = self.trace_matrix[-1][0]

        return start_cycle, end_cycle

    #
    def print_trace(self, filename):
        if not self.trace_valid:
            print('No trace has been generated yet')
            return

        np.savetxt(filename, self.trace_matrix, fmt='%s', delimiter=",")

max_stars_repo_name: KangSK-KAIST/SCALE-Sim

max_issues_repo_name: KangSK-KAIST/SCALE-Sim

max_forks_repo_name: KangSK-KAIST/SCALE-Sim


text:
"""
mnist_loader
~~~~~~~~~~~~

A library to load the MNIST image data.  For details of the data
structures that are returned, see the doc strings for ``load_data``
and ``load_data_wrapper``.  In practice, ``load_data_wrapper`` is the
function usually called by our neural network code.
"""

import gzip

import numpy as np
import six.moves.cPickle as cPickle

def load_data(path='../data/mnist.pkl.gz'):
    with gzip.open(path, 'rb') as f:
        return cPickle.load(f, encoding='latin1')

max_stars_repo_name: lucasdavid/convolutional-cuda

max_issues_repo_name: lucasdavid/convolutional

max_forks_repo_name: lucasdavid/convolutional


text:
import numpy as num
import pytest
from pytch.data import Buffer, RingBuffer

def test_BufferIndex():

    b = Buffer(sampling_rate=10, buffer_length_seconds=10)

    for x in range(5):
        b.append(num.array([x * 2, x * 2 + 1]))

    for t in range(10):
        assert b.index_at_time(t) == t * 10

def test_Buffer():

    b = Buffer(sampling_rate=1, buffer_length_seconds=10)

    for x in range(5):
        b.append(num.array([x * 2, x * 2 + 1]))

    num.testing.assert_array_almost_equal(b.ydata, num.arange(10))
    # num.testing.assert_array_almost_equal(b.xdata, num.arange(10))

def test_benchmark_fill():
    iall = 100
    sampling_rate = 44100
    blength = 3 * 60.0
    chunk_length = 0.025 * sampling_rate
    b = Buffer(sampling_rate=sampling_rate, buffer_length_seconds=blength)

    for i in range(iall):
        b.append(num.arange(i * chunk_length, (i + 1) * chunk_length))
        (
            x,
            y,
        ) = b.latest_frame(5)

    # num.testing.assert_array_almost_equal(b.xdata,
    #                                      num.arange(iall*chunk_length))

    # num.testing.assert_array_almost_equal(b.ydata/sampling_rate,
    #                                      num.tile(num.arange(chunk_length),
    # Jiall))

def test_get_latest_frame():
    sampling_rate = 10.0
    dt = 1.0 / sampling_rate
    b = Buffer(sampling_rate=sampling_rate, buffer_length_seconds=60 * 3)
    b.append(num.arange(20))
    x, y = b.latest_frame(2)
    num.testing.assert_array_almost_equal(y, num.arange(20))
    num.testing.assert_array_almost_equal(
        x, num.arange(2 * sampling_rate, dtype=num.float) * dt
    )

    b.append(num.arange(20))
    x, y = b.latest_frame(1)
    num.testing.assert_array_almost_equal(y, num.arange(10, 20))
    num.testing.assert_array_almost_equal(
        x, num.arange(3 * sampling_rate, 4 * sampling_rate, dtype=num.float) * dt
    )

@pytest.mark.skip(reason="This test needs to be fixed")
def test_ringbuffer_array():
    r = RingBuffer(1, 10)
    d = num.arange(4)
    num.testing.assert_equal(len(r.latest_frame_data(5)), 5)
    r.append(d)
    num.testing.assert_array_equal(
        r.latest_frame_data(5), num.array([9.0, 0.0, 1.0, 2.0, 3.0])
    )

    r.append(d)
    num.testing.assert_array_equal(
        r.latest_frame_data(5), num.array([3.0, 0.0, 1.0, 2.0, 3.0])
    )

    r.append(d)
    num.testing.assert_array_equal(
        r.latest_frame_data(5), num.array([3.0, 0.0, 1.0, 2.0, 3.0])
    )

    num.testing.assert_array_equal(
        r.data, num.array([2.0, 3.0, 2.0, 3.0, 0.0, 1.0, 2.0, 3.0, 0.0, 1.0])
    )

def test_ringbuffer_value():
    r = RingBuffer(1, 3)
    for i in range(4):
        r.append_value(i)
    num.testing.assert_array_equal(r.data, num.array([3.0, 1.0, 2.0]))
    num.testing.assert_array_equal(r.latest_frame_data(2), num.array([2.0, 3.0]))

def test_ringbuffer_array_retrieve_by_time():
    sampling_rate = 10  # Hertz
    buffer_length_seconds = 10
    r = RingBuffer(
        sampling_rate=sampling_rate, buffer_length_seconds=buffer_length_seconds
    )

    d = num.arange(20)  # 2 seconds data at 10 Hz
    r.append(d)

    # check last 5 samples
    x, y = r.latest_frame(3, clip_min=True)
    assert len(x) == len(y)
    assert min(x) >= 0.0

    # check last 5 samples
    y = r.latest_frame_data(5)
    num.testing.assert_equal(y, d[-5:])

    x, y = r.latest_frame(2)
    num.testing.assert_equal(y[1:], d)
    num.testing.assert_array_almost_equal(
        num.asarray(x[:-1], num.float64), d / float(sampling_rate)
    )

    d = num.arange(100)  # 5 seconds data at 10 Hz
    r.append(d)
    x, y = r.latest_frame(3, clip_min=True)
    num.testing.assert_array_almost_equal(
        num.asarray(x[:-1], num.float64), num.arange(90, 120) / sampling_rate
    )

max_stars_repo_name: pytchtracking/pytch

max_issues_repo_name: pytchtracking/pytch

max_forks_repo_name: pytchtracking/pytch


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import argparse
import training.facenet
import os
import sys
import math
import pickle
import training.detect_face
import numpy as np
import cv2
import collections
from sklearn.svm import SVC
import detect_face_detection
import time

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--path', help = 'Path of the video you want to test on.', default = 0)
    args = parser.parse_args()

    MINSIZE = 20
    THRESHOLD = [0.6, 0.7, 0.7]
    FACTOR = 0.709
    IMAGE_SIZE = 182
    INPUT_IMAGE_SIZE = 160
    CLASSIFIER_PATH = 'training/classifier/face_classifier.pkl'
    VIDEO_PATH = args.path
    FACENET_MODEL_PATH = 'training/models/20170512-110547/20170512-110547.pb'
    fps = 0
    frame_num = 0

    # Load The Custom Classifier
    with open(CLASSIFIER_PATH, 'rb') as file:
        model, class_names = pickle.load(file)
    print("Custom Classifier, Successfully loaded")

    with tf.Graph().as_default():

        gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.7)
        sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False))

        with sess.as_default():

            # Load the model
            print('Loading feature extraction model')

            training.facenet.load_model(FACENET_MODEL_PATH)

            # Get input and output tensors
            images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0")
            embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0")
            phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0")
            embedding_size = embeddings.get_shape()[1]

            pnet, rnet, onet = detect_face_detection.create_mtcnn(sess, os.path.join(os.getcwd()))

            people_detected = set()
            person_detected = collections.Counter()

            cap = cv2.VideoCapture(VIDEO_PATH)

            while(cap.isOpened()):
                start_time = time.time()
                ret, frame = cap.read()

                bounding_boxes, _ = detect_face_detection.detect_face(frame, MINSIZE, pnet, rnet, onet, THRESHOLD, FACTOR)

                faces_found = bounding_boxes.shape[0]

                try:
                    if faces_found > 0:
                        det = bounding_boxes[:, 0:4]
                        # print(det)
                        bb = np.zeros((faces_found, 4), dtype=np.int32)
                        for i in range(faces_found):
                            bb[i][0] = det[i][0]
                            bb[i][1] = det[i][1]
                            bb[i][2] = det[i][2]
                            bb[i][3] = det[i][3]

                            cropped = frame[bb[i][1]:bb[i][3], bb[i][0]:bb[i][2], :]
                            scaled = cv2.resize(cropped, (INPUT_IMAGE_SIZE, INPUT_IMAGE_SIZE), interpolation=cv2.INTER_CUBIC)
                            scaled = training.facenet.prewhiten(scaled)
                            scaled_reshape = scaled.reshape(-1, INPUT_IMAGE_SIZE, INPUT_IMAGE_SIZE, 3)
                            feed_dict = {images_placeholder: scaled_reshape, phase_train_placeholder: False}
                            emb_array = sess.run(embeddings, feed_dict=feed_dict)
                            predictions = model.predict_proba(emb_array)
                            best_class_indices = np.argmax(predictions, axis=1)
                            best_class_probabilities = predictions[np.arange(len(best_class_indices)), best_class_indices]
                            best_name = class_names[best_class_indices[0]]
                            # print("Name: {}, Probability: {}".format(best_name, best_class_probabilities))

                            cv2.rectangle(frame, (bb[i][0], bb[i][1]), (bb[i][2], bb[i][3]), (255, 255, 255), 1)
                            # cv2.rectangle(frame, (bb[i][0], bb[i][1]), (bb[i][2], bb[i][3] + 20), (255, 255, 255), cv2.FILLED)
                            text_x = bb[i][0]
                            text_y = bb[i][3] + 20

                            if best_class_probabilities > 0.92:
                                name = class_names[best_class_indices[0]]
                            else:
                                name = "Unauthorized"
                            cv2.putText(frame, name, (text_x, text_y), cv2.FONT_HERSHEY_SIMPLEX,
                                            0.50, (255, 255, 255), thickness=1)

                            cv2.putText(frame, str(round(best_class_probabilities[0], 3)), (text_x, text_y+17), cv2.FONT_HERSHEY_SIMPLEX,
                                            0.50, (0, 0, 0), thickness=1)
                            person_detected[best_name] += 1
                except:
                    pass

                end_time = time.time()
                fps = fps * 0.9 + 1/(end_time - start_time) * 0.1
                start_time = end_time
                frame_info = 'Frame: {0}, FPS: {1:.2f}'.format(frame_num, fps)
                cv2.putText(frame, frame_info, (10, frame.shape[0]-10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1)
                cv2.imshow('Face Recognition',frame)
                if cv2.waitKey(1) & 0xFF == ord('q'):
                    break

            cap.release()
            cv2.destroyAllWindows()
main()

max_stars_repo_licenses: Naumen Condor-1.1 MS-PL

max_issues_repo_licenses: Naumen Condor-1.1 MS-PL

max_forks_repo_licenses: Naumen Condor-1.1 MS-PL


text:
# Copyright (c) 2021, NVIDIA CORPORATION.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import pytest

from cuml.cluster import HDBSCAN
from cuml.cluster import condense_hierarchy
from sklearn.datasets import make_blobs

from cuml.metrics import adjusted_rand_score
from cuml.test.utils import get_pattern

import numpy as np

from cuml.common import logger

import hdbscan
from hdbscan.plots import CondensedTree

from sklearn import datasets

import cupy as cp

test_datasets = {
 "digits": datasets.load_digits(),
 "boston": datasets.load_boston(),
 "diabetes": datasets.load_diabetes(),
 "cancer": datasets.load_breast_cancer(),
}

dataset_names = ['noisy_circles', 'noisy_moons', 'varied']

def assert_cluster_counts(sk_agg, cuml_agg, digits=25):
    sk_unique, sk_counts = np.unique(sk_agg.labels_, return_counts=True)
    sk_counts = np.sort(sk_counts)

    cu_unique, cu_counts = cp.unique(cuml_agg.labels_, return_counts=True)
    cu_counts = cp.sort(cu_counts).get()

    np.testing.assert_almost_equal(sk_counts, cu_counts, decimal=-1 * digits)

def get_children(roots, parents, arr):
    """
    Simple helper function to return the children of the condensed tree
    given an array of parents.
    """
    ret = []
    for root in roots:
        level = np.where(parents == root)
        ret.extend(arr[level])

    return np.array(ret).ravel()

def get_bfs_level(n, roots, parents, children, arr):
    """
    Simple helper function to perform a bfs through n
    levels of a condensed tree.
    """
    level = roots
    for i in range(n-1):
        level = get_children(level, parents, children)

    return get_children(level, parents, arr)

def assert_condensed_trees(sk_agg, min_cluster_size):
    """
    Because of differences in the renumbering and sort ordering,
    the condensed tree arrays from cuml and scikit-learn cannot
    be compared directly. This function performs a BFS through
    the condensed trees, comparing the cluster sizes and lambda
    values at each level of the trees.
    """

    slt = sk_agg.single_linkage_tree_._linkage

    condensed_tree = condense_hierarchy(slt, min_cluster_size)

    cu_parents = condensed_tree._raw_tree["parent"]
    sk_parents = sk_agg.condensed_tree_._raw_tree["parent"]

    cu_children = condensed_tree._raw_tree["child"]
    sk_children = sk_agg.condensed_tree_._raw_tree["child"]

    cu_lambda = condensed_tree._raw_tree["lambda_val"]
    sk_lambda = sk_agg.condensed_tree_._raw_tree["lambda_val"]

    cu_child_size = condensed_tree._raw_tree["child_size"]
    sk_child_size = sk_agg.condensed_tree_._raw_tree["child_size"]

    # Start at the root, perform bfs

    l2_cu = [1000]
    l2_sk = [1000]

    lev = 1
    while len(l2_cu) != 0 or len(l2_sk) != 0:
        l2_cu = get_bfs_level(lev, [1000], cu_parents, cu_children, cu_lambda)
        l2_sk = get_bfs_level(lev, [1000], sk_parents, sk_children, sk_lambda)

        s2_cu = get_bfs_level(lev, [1000], cu_parents, cu_children,
                              cu_child_size)
        s2_sk = get_bfs_level(lev, [1000], sk_parents, sk_children,
                              sk_child_size)

        s2_cu.sort()
        s2_sk.sort()
        l2_cu.sort()
        l2_sk.sort()

        lev += 1

        assert np.allclose(l2_cu, l2_sk, atol=1e-5, rtol=1e-6)
        assert np.allclose(s2_cu, s2_sk, atol=1e-5, rtol=1e-6)
    assert lev > 1

@pytest.mark.parametrize('nrows', [500])
@pytest.mark.parametrize('ncols', [25])
@pytest.mark.parametrize('nclusters', [2, 5])
@pytest.mark.parametrize('min_samples', [25, 60])
@pytest.mark.parametrize('allow_single_cluster', [True, False])
@pytest.mark.parametrize('min_cluster_size', [30, 50])
@pytest.mark.parametrize('cluster_selection_epsilon', [0.0])
@pytest.mark.parametrize('max_cluster_size', [0])
@pytest.mark.parametrize('cluster_selection_method', ['eom', 'leaf'])
@pytest.mark.parametrize('connectivity', ['knn'])
def test_hdbscan_blobs(nrows, ncols, nclusters,
                       connectivity,
                       cluster_selection_epsilon,
                       cluster_selection_method,
                       allow_single_cluster,
                       min_cluster_size,
                       max_cluster_size,
                       min_samples):

    X, y = make_blobs(n_samples=int(nrows),
                      n_features=ncols,
                      centers=nclusters,
                      cluster_std=0.7,
                      shuffle=False,
                      random_state=42)

    cuml_agg = HDBSCAN(verbose=logger.level_info,
                       allow_single_cluster=allow_single_cluster,
                       min_samples=min_samples,
                       max_cluster_size=max_cluster_size,
                       min_cluster_size=min_cluster_size,
                       cluster_selection_epsilon=cluster_selection_epsilon,
                       cluster_selection_method=cluster_selection_method)

    cuml_agg.fit(X)
    sk_agg = hdbscan.HDBSCAN(
        allow_single_cluster=allow_single_cluster,
        approx_min_span_tree=False,
        gen_min_span_tree=True,
        min_samples=min_samples,
        min_cluster_size=min_cluster_size,
        cluster_selection_epsilon=cluster_selection_epsilon,
        cluster_selection_method=cluster_selection_method,
        algorithm="generic")

    sk_agg.fit(cp.asnumpy(X))

    assert_condensed_trees(sk_agg, min_cluster_size)
    assert_cluster_counts(sk_agg, cuml_agg)

    assert(adjusted_rand_score(cuml_agg.labels_, sk_agg.labels_) >= 0.95)
    assert(len(np.unique(sk_agg.labels_)) == len(cp.unique(cuml_agg.labels_)))

    assert np.allclose(np.sort(sk_agg.cluster_persistence_),
           np.sort(cuml_agg.cluster_persistence_), rtol=0.01, atol=0.01)

@pytest.mark.parametrize('dataset', test_datasets.values())
@pytest.mark.parametrize('cluster_selection_epsilon', [0.0, 50.0, 150.0])
@pytest.mark.parametrize('min_samples_cluster_size_bounds', [(150, 150, 0),
                                                             (50, 25, 0)])
@pytest.mark.parametrize('allow_single_cluster', [True, False])
@pytest.mark.parametrize('cluster_selection_method', ['eom', 'leaf'])
@pytest.mark.parametrize('connectivity', ['knn'])
def test_hdbscan_sklearn_datasets(dataset,
                                  connectivity,
                                  cluster_selection_epsilon,
                                  cluster_selection_method,
                                  min_samples_cluster_size_bounds,
                                  allow_single_cluster):

    min_samples, min_cluster_size, max_cluster_size = \
        min_samples_cluster_size_bounds

    X = dataset.data

    cuml_agg = HDBSCAN(verbose=logger.level_info,
                       allow_single_cluster=allow_single_cluster,
                       gen_min_span_tree=True,
                       min_samples=min_samples,
                       max_cluster_size=max_cluster_size,
                       min_cluster_size=min_cluster_size,
                       cluster_selection_epsilon=cluster_selection_epsilon,
                       cluster_selection_method=cluster_selection_method)

    cuml_agg.fit(X)

    sk_agg = hdbscan.HDBSCAN(
        allow_single_cluster=allow_single_cluster,
        approx_min_span_tree=False,
        gen_min_span_tree=True,
        min_samples=min_samples,
        min_cluster_size=min_cluster_size,
        cluster_selection_epsilon=cluster_selection_epsilon,
        cluster_selection_method=cluster_selection_method,
        algorithm="generic")

    sk_agg.fit(cp.asnumpy(X))

    assert_condensed_trees(sk_agg, min_cluster_size)
    assert_cluster_counts(sk_agg, cuml_agg)

    assert(len(np.unique(sk_agg.labels_)) == len(cp.unique(cuml_agg.labels_)))
    assert(adjusted_rand_score(cuml_agg.labels_, sk_agg.labels_) > 0.85)

    assert np.allclose(np.sort(sk_agg.cluster_persistence_),
           np.sort(cuml_agg.cluster_persistence_), rtol=0.1, atol=0.1)

@pytest.mark.parametrize('dataset', test_datasets.values())
@pytest.mark.parametrize('cluster_selection_epsilon', [0.0, 50.0, 150.0])
@pytest.mark.parametrize('min_samples', [150, 50, 5, 400])
@pytest.mark.parametrize('min_cluster_size', [150, 25, 5, 250])
@pytest.mark.parametrize('max_cluster_size', [0])
@pytest.mark.parametrize('allow_single_cluster', [True, False])
@pytest.mark.parametrize('cluster_selection_method', ['eom', 'leaf'])
@pytest.mark.parametrize('connectivity', ['knn'])
def test_hdbscan_sklearn_extract_clusters(dataset,
                                          connectivity,
                                          cluster_selection_epsilon,
                                          cluster_selection_method,
                                          min_samples,
                                          min_cluster_size,
                                          max_cluster_size,
                                          allow_single_cluster):

    X = dataset.data

    cuml_agg = HDBSCAN(verbose=logger.level_info,
                       allow_single_cluster=allow_single_cluster,
                       gen_min_span_tree=True,
                       min_samples=min_samples,
                       max_cluster_size=max_cluster_size,
                       min_cluster_size=min_cluster_size,
                       cluster_selection_epsilon=cluster_selection_epsilon,
                       cluster_selection_method=cluster_selection_method)

    sk_agg = hdbscan.HDBSCAN(
        allow_single_cluster=allow_single_cluster,
        approx_min_span_tree=False,
        gen_min_span_tree=True,
        min_samples=min_samples,
        min_cluster_size=min_cluster_size,
        cluster_selection_epsilon=cluster_selection_epsilon,
        cluster_selection_method=cluster_selection_method,
        algorithm="generic")

    sk_agg.fit(cp.asnumpy(X))

    cuml_agg._extract_clusters(sk_agg.condensed_tree_)

    assert adjusted_rand_score(cuml_agg.labels_test, sk_agg.labels_) == 1.0
    assert np.allclose(cp.asnumpy(cuml_agg.probabilities_test),
                       sk_agg.probabilities_)

@pytest.mark.parametrize('nrows', [1000])
@pytest.mark.parametrize('dataset', dataset_names)
@pytest.mark.parametrize('min_samples', [15])
@pytest.mark.parametrize('cluster_selection_epsilon', [0.0])
@pytest.mark.parametrize('min_cluster_size', [25])
@pytest.mark.parametrize('allow_single_cluster', [True, False])
@pytest.mark.parametrize('max_cluster_size', [0])
@pytest.mark.parametrize('cluster_selection_method', ['eom'])
@pytest.mark.parametrize('connectivity', ['knn'])
def test_hdbscan_cluster_patterns(dataset, nrows,
                                  connectivity,
                                  cluster_selection_epsilon,
                                  cluster_selection_method,
                                  min_cluster_size,
                                  allow_single_cluster,
                                  max_cluster_size,
                                  min_samples):

    # This also tests duplicate data points
    X, y = get_pattern(dataset, nrows)[0]

    cuml_agg = HDBSCAN(verbose=logger.level_info,
                       allow_single_cluster=allow_single_cluster,
                       min_samples=min_samples,
                       max_cluster_size=max_cluster_size,
                       min_cluster_size=min_cluster_size,
                       cluster_selection_epsilon=cluster_selection_epsilon,
                       cluster_selection_method=cluster_selection_method)

    cuml_agg.fit(X)

    sk_agg = hdbscan.HDBSCAN(
        allow_single_cluster=allow_single_cluster,
        approx_min_span_tree=False,
        gen_min_span_tree=True,
        min_samples=min_samples,
        min_cluster_size=min_cluster_size,
        cluster_selection_epsilon=cluster_selection_epsilon,
        cluster_selection_method=cluster_selection_method,
        algorithm="generic")

    sk_agg.fit(cp.asnumpy(X))

    assert_condensed_trees(sk_agg, min_cluster_size)
    assert_cluster_counts(sk_agg, cuml_agg)

    assert(len(np.unique(sk_agg.labels_)) == len(cp.unique(cuml_agg.labels_)))
    assert(adjusted_rand_score(cuml_agg.labels_, sk_agg.labels_) > 0.95)

    assert np.allclose(np.sort(sk_agg.cluster_persistence_),
           np.sort(cuml_agg.cluster_persistence_), rtol=0.1, atol=0.1)

@pytest.mark.parametrize('nrows', [1000])
@pytest.mark.parametrize('dataset', dataset_names)
@pytest.mark.parametrize('min_samples', [5, 50, 400, 800])
@pytest.mark.parametrize('cluster_selection_epsilon', [0.0, 50.0, 150.0])
@pytest.mark.parametrize('min_cluster_size', [10, 25, 100, 350])
@pytest.mark.parametrize('allow_single_cluster', [True, False])
@pytest.mark.parametrize('max_cluster_size', [0])
@pytest.mark.parametrize('cluster_selection_method', ['eom', 'leaf'])
@pytest.mark.parametrize('connectivity', ['knn'])
def test_hdbscan_cluster_patterns_extract_clusters(dataset, nrows,
                                                   connectivity,
                                                   cluster_selection_epsilon,
                                                   cluster_selection_method,
                                                   min_cluster_size,
                                                   allow_single_cluster,
                                                   max_cluster_size,
                                                   min_samples):

    # This also tests duplicate data points
    X, y = get_pattern(dataset, nrows)[0]

    cuml_agg = HDBSCAN(verbose=logger.level_info,
                       allow_single_cluster=allow_single_cluster,
                       min_samples=min_samples,
                       max_cluster_size=max_cluster_size,
                       min_cluster_size=min_cluster_size,
                       cluster_selection_epsilon=cluster_selection_epsilon,
                       cluster_selection_method=cluster_selection_method)

    sk_agg = hdbscan.HDBSCAN(
        allow_single_cluster=allow_single_cluster,
        approx_min_span_tree=False,
        gen_min_span_tree=True,
        min_samples=min_samples,
        min_cluster_size=min_cluster_size,
        cluster_selection_epsilon=cluster_selection_epsilon,
        cluster_selection_method=cluster_selection_method,
        algorithm="generic")

    sk_agg.fit(cp.asnumpy(X))

    cuml_agg._extract_clusters(sk_agg.condensed_tree_)

    assert adjusted_rand_score(cuml_agg.labels_test, sk_agg.labels_) == 1.0
    assert np.allclose(cp.asnumpy(cuml_agg.probabilities_test),
                       sk_agg.probabilities_)

def test_hdbscan_core_dists_bug_4054():
    """
    This test explicitly verifies that the MRE from
    https://github.com/rapidsai/cuml/issues/4054
    matches the reference impl
    """

    X, y = datasets.make_moons(n_samples=10000, noise=0.12, random_state=0)

    cu_labels_ = HDBSCAN(min_samples=25, min_cluster_size=25).fit_predict(X)
    sk_labels_ = hdbscan.HDBSCAN(min_samples=25,
                                 min_cluster_size=25,
                                 approx_min_span_tree=False).fit_predict(X)

    assert adjusted_rand_score(cu_labels_, sk_labels_) > 0.99

def test_hdbscan_empty_cluster_tree():

    raw_tree = np.recarray(shape=(5,),
                           formats=[np.intp, np.intp, float, np.intp],
                           names=('parent', 'child', 'lambda_val',
                                  'child_size'))

    raw_tree['parent'] = np.asarray([5, 5, 5, 5, 5])
    raw_tree['child'] = [0, 1, 2, 3, 4]
    raw_tree['lambda_val'] = [1.0, 1.0, 1.0, 1.0, 1.0]
    raw_tree['child_size'] = [1, 1, 1, 1, 1]

    condensed_tree = CondensedTree(raw_tree, 0.0, True)

    cuml_agg = HDBSCAN(allow_single_cluster=True,
                       cluster_selection_method="eom")
    cuml_agg._extract_clusters(condensed_tree)

    # We just care that all points are assigned to the root cluster
    assert np.sum(cuml_agg.labels_test.to_output("numpy")) == 0

def test_hdbscan_plots():

    X, y = make_blobs(n_samples=int(100),
                      n_features=100,
                      centers=10,
                      cluster_std=0.7,
                      shuffle=False,
                      random_state=42)

    cuml_agg = HDBSCAN(gen_min_span_tree=True)
    cuml_agg.fit(X)

    assert cuml_agg.condensed_tree_ is not None
    assert cuml_agg.minimum_spanning_tree_ is not None
    assert cuml_agg.single_linkage_tree_ is not None

    cuml_agg = HDBSCAN(gen_min_span_tree=False)
    cuml_agg.fit(X)

    assert cuml_agg.minimum_spanning_tree_ is None

max_stars_repo_name: siddheshmhatre/cuml

max_issues_repo_name: siddheshmhatre/cuml

max_forks_repo_name: siddheshmhatre/cuml

reason: import numpy,import cupy


text:
__all__ = ["make_test", "df_test_input"]

import numpy as np
import pandas as pd
import grama as gr

## Define a test dataset
df_test_input = pd.DataFrame(data={"x0": [0], "x1": [0], "x2": [0]})

## Define a test model
def fun(x):
    x1, x2, x3 = x
    return x1 + x2 + x3

def make_test():
    md = gr.Model() >> \
         gr.cp_function(fun=fun, var=3, out=1) >> \
         gr.cp_bounds(x0=(-1,+1), x1=(-1,+1), x2=(-1,+1)) >> \
         gr.cp_marginals(
             x0={"dist": "uniform", "loc": -1, "scale": 2},
             x1={"dist": "uniform", "loc": -1, "scale": 2}
         ) >> \
         gr.cp_copula_independence()

    return md


text:
# Copyright 2021 The Commplax Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import jax
from jax import jit, vmap, numpy as jnp, device_put
from jax.core import Value
import numpy as np
from commplax import xop, comm, experimental as exp
from functools import partial

def getpower(x, axis=0, real=False):
    ''' get signal power '''
    return jnp.mean(x.real**2, axis=axis) + jnp.array(1j) * jnp.mean(x.imag**2, axis=axis) \
        if real else jnp.mean(jnp.abs(x)**2, axis=axis)

def normpower(x, axis=0, real=False):
    ''' normalize signal power '''
    if real:
        p = getpower(x, axis=axis, real=True)
        return x.real / jnp.sqrt(p.real) + jnp.array(1j) * x.imag / jnp.sqrt(p.imag)
    else:
        return x / jnp.sqrt(getpower(x, axis=axis))

def qamscale(modformat):
    M = comm.parseqamorder(modformat)
    return jnp.sqrt((M-1) * 2 / 3)

def dbp_params(
    sample_rate,                                      # sample rate of target signal [Hz]
    span_length,                                      # length of each fiber span [m]
    spans,                                            # number of fiber spans
    freqs,                                            # resulting size of linear operator
    launch_power=30,                                  # launch power [dBm]
    steps_per_span=1,                                 # steps per span
    virtual_spans=None,                               # number of virtual spans
    carrier_frequency=299792458/1550E-9,              # carrier frequency [Hz]
    fiber_dispersion=16.5E-6,                         # [s/m^2]
    fiber_dispersion_slope=0.08e3,                    # [s/m^3]
    fiber_loss=.2E-3,                                 # loss of fiber [dB]
    fiber_core_area=80E-12,                           # effective area of fiber [m^2]
    fiber_nonlinear_index=2.6E-20,                    # nonlinear index [m^2/W]
    fiber_reference_frequency=299792458/1550E-9,      # fiber reference frequency [Hz]
    ignore_beta3=False,
    polmux=True):

    # short names
    pi  = jnp.pi
    log = jnp.log
    exp = jnp.exp
    ifft = jnp.fft.ifft

    # virtual span is used in cases where we do not use physical span settings
    if virtual_spans is None:
        virtual_spans = spans

    C       = 299792458. # speed of light [m/s]
    lambda_ = C / fiber_reference_frequency
    B_2     = -fiber_dispersion * lambda_**2 / (2 * pi * C)
    B_3     = 0. if ignore_beta3 else \
        (fiber_dispersion_slope * lambda_**2 + 2 * fiber_dispersion * lambda_) * (lambda_ / (2 * pi * C))**2
    gamma   = 2 * pi * fiber_nonlinear_index / lambda_ / fiber_core_area
    LP      = 10.**(launch_power / 10 - 3)
    alpha   = fiber_loss / (10. / log(10.))
    L_eff   = lambda h: (1 - exp(-alpha * h)) / alpha
    NIter   = virtual_spans * steps_per_span
    delay   = (freqs - 1) // 2
    dw      = 2 * pi * (carrier_frequency - fiber_reference_frequency)
    w_res   = 2 * pi * sample_rate / freqs
    k       = jnp.arange(freqs)
    w       = jnp.where(k > delay, k - freqs, k) * w_res # ifftshifted

    H   = exp(-1j * (-B_2 / 2 * (w + dw)**2 + B_3 / 6 * (w + dw)**3) * \
                  span_length * spans / virtual_spans / steps_per_span)
    H_casual = H * exp(-1j * w * delay / sample_rate)
    h_casual = ifft(H_casual)

    phi = spans / virtual_spans * gamma * L_eff(span_length / steps_per_span) * LP * \
        exp(-alpha * span_length * (steps_per_span - jnp.arange(0, NIter) % steps_per_span-1) / steps_per_span)

    dims = 2 if polmux else 1

    H = jnp.tile(H[None, :, None], (NIter, 1, dims))
    h_casual = jnp.tile(h_casual[None, :, None], (NIter, 1, dims))
    phi = jnp.tile(phi[:, None, None], (1, dims, dims))

    return H, h_casual, phi

def mimoconv(y, h, mode='same', conv=xop.fftconvolve):
    dims = y.shape[-1]
    y = jnp.tile(y, (1, dims))
    h = jnp.reshape(h, (h.shape[0], dims * dims))
    zflat = jax.vmap(lambda a, b: conv(a, b, mode=mode), in_axes=-1, out_axes=-1)(y, h)
    z = zflat.T.reshape((dims, dims, -1)).sum(axis=1).T
    return z

def dbp_timedomain(y, h, c, mode='SAME', homosteps=True, scansteps=True, conv=xop.fftconvolve):

    y = device_put(y)
    h = device_put(h)
    c = device_put(c)

    dims = y.shape[-1]

    optpowscale = jnp.sqrt(dims)
    y /= optpowscale

    md = 'SAME' if homosteps else mode

    D = jit(vmap(lambda y, h: conv(y, h, mode=md), in_axes=1, out_axes=1))
    N = jit(lambda y, c: y * jnp.exp(1j * (abs(y)**2 @ c)))

    T = h.shape[1] - 1
    K = h.shape[0]

    if homosteps and scansteps: # homogeneous steps is faster on first jitted run
    # scan not working on 'SAME' mode due to carry shape change
        y = xop.scan(lambda x, p: (N(D(x, p[0]), p[1]), 0.), y, (h, c))[0]
    else:
        steps = c.shape[0]
        for i in range(steps):
            y = D(y, h[i])
            y = N(y, c[i])

    if homosteps and mode.lower() == 'valid':
       y = y[K * T // 2: -K * T // 2]

    return y * optpowscale

def dbp_direct(y, H, c):
    y = device_put(y)
    H = device_put(H)
    c = device_put(c)

    steps = c.shape[0]

    fft = lambda x: jnp.fft.fft(x, axis=0)
    ifft = lambda x: jnp.fft.ifft(x, axis=0)

    D = jit(lambda y,H: ifft(fft(y) * H))
    N = jit(lambda y,c: y * jnp.exp(1j * (abs(y)**2 @ c)))

    for i in range(steps):
        y = D(y, H[i])
        y = N(y, c[i])

    return y

def foe_mpowfftmax(x, M=4, sr=2*jnp.pi):
    X4 = jnp.fft.fft(x**M, axis=0)
    h  = jnp.abs(X4)**2
    f  = jnp.argmax(h, axis=0)
    N  = len(h)
    f = jnp.where(f >= N / 2, f - N, f)
    fo_hat = f / N * sr / 4
    return fo_hat, h

def foe_daxcorr(y, x, L=100):
    N = y.shape[0]
    if N < L:
        raise TypeError('signal length %d is less then xcorr length %d' % (N, L))
    s = y * x.conj() # remove modulated data phase
    sf = xop.frame(s, L, L)
    sf2 = sf[:-1]
    sf1 = sf[1:]
    return jnp.mean(jnp.angle(sf1 * sf2.conj())) / L

def measure_cd(x, sr, start=-0.25, end=0.25, bins=2000, wavlen=1550e-9):
    '''
    References:
        Zhou, H., Li, B., Tang, et. al, 2016. Fractional fourier transformation-based
        blind chromatic dispersion estimation for coherent optical communications.
        Journal of Lightwave Technology, 34(10), pp.2371-2380.
    '''
    c = 299792458.
    p = jnp.linspace(start, end, bins)
    N = x.shape[0]
    K = p.shape[0]

    L = jnp.zeros(K, dtype=jnp.float32)

    def f(_, pi):
        return None, jnp.sum(jnp.abs(xop.frft(jnp.abs(xop.frft(x, pi))**2, -1))**2)

    # Use `scan` instead of `vmap` here to avoid potential large memory allocation.
    # Despite the speed of `scan` scales surprisingly well to large bins,
    # the speed has a lowerbound e.g 600ms at bins=1, possiblely related to the blind
    # migration of `frft` from Github :) (could frft be jitted in theory?).
    # TODO review `frft`
    _, L = xop.scan(f, None, p)

    B2z = jnp.tan(jnp.pi/2 - (p - 1) / 2 * jnp.pi)/(sr * 2 * jnp.pi / N * sr)
    Dz_set  = -B2z / wavlen**2 * 2 * jnp.pi * c # the swept set of CD metrics
    Dz_hat = Dz_set[jnp.argmin(L)] # estimated accumulated CD

    return Dz_hat, L, Dz_set

def dimsdelay(x):
    dims = x.shape[-1]
    if dims <= 1:
        raise ValueError('input dimension must be at least 2 but given %d' % dims)
    x = device_put(x)
    return jax.vmap(xop.finddelay, in_axes=(None, -1), out_axes=0)(x[:, 0], x[:, 1:])[0]

def repalign(y, x, skipfirst=0):
    N = y.shape[0]
    M = x.shape[0]
    offsets = -jax.vmap(xop.finddelay, in_axes=-1)(y[skipfirst:], x)[0] + skipfirst
    rep = -(-N // M)
    xrep = jnp.tile(x, [rep, 1])
    z = jax.vmap(jnp.roll, in_axes=-1, out_axes=-1)(xrep, offsets)[:N, :]
    return z

def alignphase(y, x, testing_phases=4):

    y = device_put(y)
    x = device_put(x)

    TPS = jnp.exp(1j * 2 * jnp.pi / testing_phases * jnp.arange(testing_phases))

    def searchphase(y, x):
        y_t = jnp.outer(y, TPS)
        x_t = jnp.tile(x[:,None], (1, testing_phases))
        snr_t = 10. * jnp.log10(getpower(y_t) / getpower(y_t - x_t))
        return TPS[jnp.argmax(snr_t)]

    return y * vmap(searchphase, in_axes=-1)(y, x)

def localfoe(signal, frame_size=16384, frame_step=5000, sps=1, fitkind=None, degree=2,
             method=lambda x: foe_mpowfftmax(x)[0]):
    '''
    resolution = samplerate / N / 4 / sps (linear interp.)
    '''
    cpus = jax.devices('cpu')

    # [BUG]: polyfit is buggy in GPU
    y = device_put(signal, cpus[0])
    dims = y.shape[-1]
    fo_local = xop.framescaninterp(y, method, frame_size, frame_step, sps)
    if fitkind is not None:
        if fitkind.lower() == 'poly':
            fo_T = jnp.tile(jnp.arange(fo_local.shape[0])[:, None], (1, dims))
            fo_local = exp.polyfitval(fo_T, fo_local, degree)
        else:
            raise ValueError('invlaid fitting method')
    return fo_local

def localpower(signal, frame_size=5000, frame_step=1000):
    y = device_put(signal)
    poweval = lambda y: jnp.mean(jnp.abs(y)**2, axis=0)
    return xop.framescaninterp(y, poweval, frame_size, frame_step)

def localdc(signal, frame_size=5000, frame_step=1000):
    y = device_put(signal)
    dceval = lambda y: jnp.mean(y, axis=0)
    return xop.framescaninterp(y, dceval, frame_size, frame_step)

max_stars_repo_name: remifan/commplax

max_issues_repo_name: remifan/commplax

max_forks_repo_name: remifan/commplax

reason: import numpy,import jax,from jax


text:
import pandas as pd
import numpy as np

from yafs.metrics import Metrics

class Stats:

    def __init__(self,defaultPath="result"):
        self.df_link = pd.read_csv(defaultPath + "_link.csv")
        self.df = pd.read_csv(defaultPath + ".csv")

    def bytes_transmitted(self):
        return self.df_link["size"].sum()

    def count_messages(self):
        return len(self.df_link)

    def utilization(self,id_entity, total_time, from_time=0.0):
        if "time_service" not in self.df.columns: #cached
            self.df["time_service"] = self.df.time_out - self.df.time_in
        values = self.df.groupby("DES.dst").time_service.agg("sum")
        return values[id_entity] / total_time

    def compute_times_df(self):
        self.df["time_latency"] = self.df["time_reception"] - self.df["time_emit"]
        self.df["time_wait"] = self.df["time_in"] - self.df["time_reception"]  #
        self.df["time_service"] = self.df["time_out"] - self.df["time_in"]
        self.df["time_response"] = self.df["time_out"] - self.df["time_reception"]
        self.df["time_total_response"] = self.df["time_response"] + self.df["time_latency"]

    def times(self,time,value="mean"):
        if "time_response" not in self.df.columns:
            self.compute_times_df()
        return self.df.groupby("message").agg({time:value})

    def average_loop_response(self,time_loops):
        """
        No hay chequeo de la existencia del loop: user responsability
        """
        if "time_response" not in self.df.columns:
            self.compute_times_df()

        resp_msg = self.df.groupby("message").agg({"time_total_response": ["mean","count"]}) #Its not necessary to have "count"
        resp_msg.columns = ['_'.join(col).strip() for col in resp_msg.columns.values]
        results = []

        for loop in time_loops:
            total = 0.0
            for msg in loop:
                try:
                    total += resp_msg[resp_msg.index == msg].time_total_response_mean[0]
                except IndexError:
                    total +=0

            results.append(total)

        return results

    def get_watt(self,totaltime,topology,by=Metrics.WATT_SERVICE):
        results = {}
        nodeInfo = topology.get_info()
        if by == Metrics.WATT_SERVICE:
            # Tiempo de actividad / runeo
            if "time_response" not in self.df.columns:  # cached
                self.compute_times_df()

            nodes = self.df.groupby("TOPO.dst").agg({"time_service": "sum"})
            for id_node in nodes.index:
                results[id_node] = {"model": nodeInfo[id_node]["model"], "type": nodeInfo[id_node]["type"],
                                 "watt": nodes.loc[id_node].time_service * nodeInfo[id_node]["WATT"]}
        else:
            for node_key in nodeInfo:
                if not nodeInfo[node_key]["uptime"][1]:
                    end = totaltime
                start = nodeInfo[node_key]["uptime"][0]
                uptime = end-start
                results[node_key] = {"model":nodeInfo[node_key]["model"],"type":nodeInfo[node_key]["type"],"watt":uptime*nodeInfo[node_key]["WATT"],"uptime":uptime}

        return results

    # def get_cost_cloud(self, topology):
    #     cost = 0.0
    #     nodeInfo = topology.get_info()
    #     results = {}
    #     # Tiempo de actividad / runeo
    #     if "time_response" not in self.df.columns:  # cached
    #         self.__compute_times_df()
    #
    #     nodes = self.df.groupby("TOPO.dst").agg({"time_service": "sum"})
    #
    #     for id_node in nodes.index:
    #         if nodeInfo[id_node]["type"] == Entity.ENTITY_CLOUD:
    #             results[id_node] = {"model": nodeInfo[id_node]["model"], "type": nodeInfo[id_node]["type"],
    #                                 "watt": nodes.loc[id_node].time_service * nodeInfo[id_node]["WATT"]}
    #             cost += nodes.loc[id_node].time_service * nodeInfo[id_node]["COST"]
    #     return cost,results

    def showLoops(self,time_loops):
        results = self.average_loop_response(time_loops)
        for i, loop in enumerate(time_loops):
            print ("\t\t%i - %s :\t %f" % (i, str(loop), results[i]))
        return results

    def showResults(self, total_time, topology, time_loops=None):
        print ("\tSimulation Time: %0.2f" % total_time)

        if time_loops is not None:
            print ("\tApplication loops delays:")
            results = self.average_loop_response(time_loops)
            for i, loop in enumerate(time_loops):
                print ("\t\t%i - %s :\t %f" % (i, str(loop), results[i]))

        print ("\tEnergy Consumed (WATTS by UpTime):")
        values = self.get_watt(total_time, topology, Metrics.WATT_UPTIME)
        for node in values:
            print ("\t\t%i - %s :\t %.2f" % (node, values[node]["model"], values[node]["watt"]))

        print ("\tEnergy Consumed by Service (WATTS by Service Time):")
        values = self.get_watt(total_time, topology, Metrics.WATT_SERVICE)
        for node in values:
            print ("\t\t%i - %s :\t %.2f" % (node, values[node]["model"], values[node]["watt"]))

        print ("\tCost of execution in cloud:")
        total, values = self.get_cost_cloud(topology)
        print ("\t\t%.8f" % total)

        print ("\tNetwork bytes transmitted:")
        print ("\t\t%.1f" % self.bytes_transmitted())

    def showResults2(self, total_time, time_loops=None):
        print ("\tSimulation Time: %0.2f" % total_time)

        if time_loops is not None:
            print ("\tApplication loops delays:")
            results = self.average_loop_response(time_loops)
            for i, loop in enumerate(time_loops):
                print ("\t\t%i - %s :\t %f" % (i, str(loop), results[i]))

        print ("\tNetwork bytes transmitted:")
        print ("\t\t%.1f" % self.bytes_transmitted())

    """ONLYE THE FIRST ONE : DEBUG"""
    def valueLoop(self, total_time, time_loops=None):
            if time_loops is not None:
                results = self.average_loop_response(time_loops)
                for i, loop in enumerate(time_loops):
                    return results[i]

    def average_messages_not_transmitted(self):
        return np.mean(self.df_link.buffer)

    def peak_messages_not_transmitted(self):
        return np.max(self.df_link.buffer)

    def messages_not_transmitted(self):
        return self.df_link.buffer[-1:]

    def get_df_modules(self):
        g = self.df.groupby(["module", "DES.dst"]).agg({"service": ['mean', 'sum', 'count']})
        return g.reset_index()

    def get_df_service_utilization(self,service,time):
        """
        Returns the utilization(%) of a specific module
        """
        g = self.df.groupby(["module", "DES.dst"]).agg({"service": ['mean', 'sum', 'count']})
        g.reset_index(inplace=True)
        h = pd.DataFrame()
        h["module"] = g[g.module == service].module
        h["utilization"] = g[g.module == service]["service"]["sum"]*100 / time
        return h

max_stars_repo_name: MarkoRimac/YAFS

max_issues_repo_name: MarkoRimac/YAFS

max_forks_repo_name: MarkoRimac/YAFS