text:
import math
import collections
import random
import numpy as np
import tensorflow as tf
import itertools
import time

def sum_rows(x):
    """Returns a vector summing up each row of the matrix x."""
    cols = tf.shape(x)[1]
    ones_shape = tf.stack([cols, 1])
    ones = tf.ones(ones_shape, x.dtype)
    return tf.reshape(tf.matmul(x, ones), [-1])

def compute_sampled_logits(weights,
                           biases,
                           labels,
                           inputs,
                           num_sampled,
                           num_classes,
                           num_true=1):
    if not isinstance(weights, list):
        weights = [weights]
    if labels.dtype != tf.int64:
        labels = tf.cast(labels, tf.int64)
    labels_flat = tf.reshape(labels, [-1])
    sampled_ids, true_expected_count, sampled_expected_count = tf.nn.log_uniform_candidate_sampler(
        true_classes=labels,
        num_true=num_true,
        num_sampled=num_sampled,
        unique=True,
        range_max=num_classes)

    true_w = tf.nn.embedding_lookup(weights, labels_flat)
    true_b = tf.nn.embedding_lookup(biases, labels_flat)
    sampled_w = tf.nn.embedding_lookup(weights, sampled_ids)
    sampled_b = tf.nn.embedding_lookup(biases, sampled_ids)
    dim = tf.shape(true_w)[1:2]
    new_true_w_shape = tf.concat([[-1, num_true], dim], 0)
    row_wise_dots = tf.multiply(tf.expand_dims(inputs, 1), tf.reshape(true_w, new_true_w_shape))
    dots_as_matrix = tf.reshape(row_wise_dots, tf.concat([[-1], dim], 0))
    true_logits = tf.reshape(sum_rows(dots_as_matrix), [-1, num_true])
    true_b = tf.reshape(true_b, [-1, num_true])
    true_logits += true_b
    sampled_b_vec = tf.reshape(sampled_b, [num_sampled])
    sampled_logits = tf.matmul(inputs, sampled_w, transpose_b=True) + sampled_b_vec

    return true_logits, sampled_logits

def nce_loss(weights,
             biases,
             labels,
             inputs,
             num_sampled,
             num_classes,
             num_true=1,
             v=None):
    batch_size = int(labels.get_shape()[0])
    if v is None:
        v = tf.ones([batch_size, 1])

    true_logits, sampled_logits = compute_sampled_logits(
        weights=weights,
        biases=biases,
        labels=labels,
        inputs=inputs,
        num_sampled=num_sampled,
        num_classes=num_classes,
        num_true=num_true)
    true_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=tf.ones_like(true_logits), logits=true_logits)
    sampled_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=tf.zeros_like(sampled_logits), logits=sampled_logits)

    true_loss = tf.multiply(true_loss, v)
    return tf.div(tf.reduce_sum(true_loss) + tf.reduce_sum(sampled_loss), tf.constant(batch_size, dtype=tf.float32))

max_issues_repo_name: codeinging/JAPE

max_forks_repo_name: codeinging/JAPE


text:
r"""
======================
Static Modelling steps
======================

Steps for modelling static or quasi static situations:
should include:
specified global interference
specified global loading
specified surface loading
specified surface displacement
closure plot generator/ adhesive pull of tester (the same but backwards)

No models should have to do wear or other time varying stuff.

All should return a current state dict
"""

import numpy as np

from ._model_utils import get_gap_from_model
from ._step_utils import HeightOptimisationFunction
from .steps import _ModelStep

__all__ = ['StaticStep']

class StaticStep(_ModelStep):
    """
    Static loading between two bodies

    Parameters
    ----------
    step_name: str
        An identifying name for the step used for errors and outputs
    time_period: float, optional (1.0)
        The total time period of this model step, used for solving sub-models and writing outputs
    off_set_x, off_set_y: float
        The off set between the surfaces origins, in the same units as the grid spacings of the surfaces.
    normal_load, interference, mean_gap: float
        The total compressive load and the interference between the two surfaces (measured from the point of first
        contact). Exactly one of these must be set. See notes for valid methods for each parameter.
    relative_loading: bool, optional (False)
        If True the load or displacement will be applied relative to the value at the start of the step,
        otherwise the absolute value will be used. eg, if the previous step ended with a load of 10N and this step ramps
        from 0 to 10N setting relative_loading to True will ramp the total load form 10 to 20N over this step.
    adhesion: bool, optional (True)
        If True the adhesion model set for the contact model will be used, If set to false this step will ignore the
        adhesion model (typically used for loading steps)
    unloading: bool, optional (False)
        If True the contact nodes will be constrained to be a sub set of those found in the previous time step.
    profile_interpolation_mode: {'nearest', 'linear'}, optional ('nearest')
        Used to generate the grid points for the second surface at the location of the grid points for the first
        surface, nearest ensures compatibility with sub models which change the profile, if the grid spacings of the
        surfaces match
    periodic_geometry: bool, optional (False)
        If True the surface profile will warp when applying the off set between the surfaces
    periodic_axes: tuple, optional ((False, False))
        For each True value the corresponding axis will be solved by circular convolution, meaning the result is
        periodic in that direction
    method: {'auto', 'pk', 'double', 'rey'}, optional ('auto')
        The method by which the normal contact is solved, only used for load controlled contact.
        'pk' uses the Polonsky and Keer algorithm linear contact.
        'double' uses a double iteration procedure, suitable for elastic contact with a maximum pressure.
        'rey' uses the Rey algorithm for adhesive contact.
        'auto' automatically selects 'pk' if there is no maximum pressure and 'double' if there is.
    max_it: int, optional (1000)
        The maximum number of iterations used in the main loop
    tolerance: float, optional (1e-8)
        The relative tolerance used for convergnece of the main loop
    max_it_outer: int, optional (100)
        Only used for the double iteration method
    tolerance_outer: float, optional (1e-4)
        The norm of the residual used to declare convergence of the bccg iterations

    Notes
    -----
    Not all methods can be used for all set parameters and periodic combinations:
    rey: Both axes must be periodic, mean_gap or normal_load can be set, adhesion allowed, linear materials
    pk: Any combination of periodic axes, normal load or interference set (interference requires spartially
        defined influence matrix), no adhesion, linear materials
    double: Any combination of periodic axes, normal load or interference set (interference requires spartially
        defined influence matrix), no adhesion, maximum load allowed for materials.

    Setting the method to 'auto' will choose a method automatically or raise an error if no method can be used. If
    multiple methods can be used the pk solver is used.

    Examples
    --------
    In this example we will recreate the hertz solution using a numerical solver.

    >>> import slippy.surface as s
    >>> import slippy.contact as c
    >>> # make surface geometry
    >>> flat_surface = s.FlatSurface(shift=(0,0))
    >>> round_surface = s.RoundSurface((1,1,1), extent = (0.006, 0.006),
    >>>                                shape = (255, 255), generate = True)
    >>> # make and set materials
    >>> steel = c.Elastic('Steel', {'E': 200e9, 'v':0.3})
    >>> aluminum = c.Elastic('Aluminum', {'E': 70e9, 'v':0.33})
    >>> flat_surface.material = aluminum
    >>> round_surface.material = steel
    >>> # make contact model
    >>> my_model = c.ContactModel('model-1', round_surface, flat_surface)
    >>> # make and add step
    >>> total_load = 100
    >>> my_step = c.StaticStep('contact', normal_load=total_load, rtol_interference=1e-2)
    >>> my_model.add_step(my_step)
    >>> # solve the model
    >>> result = my_model.solve()
    """

    def __init__(self, step_name: str, time_period: float = 1.0,
                 off_set_x: float = 0.0, off_set_y: float = 0.0,
                 normal_load: float = None, interference: float = None,
                 mean_gap: float = None,
                 relative_loading: bool = False, adhesion: bool = True,
                 unloading: bool = False, profile_interpolation_mode: str = 'nearest',
                 periodic_geometry: bool = False, periodic_axes: tuple = (False, False),
                 method: str = 'auto',
                 max_it: int = 1000, tolerance=1e-8,
                 max_it_outer: int = 100, tolerance_outer=1e-8):

        self._off_set = (off_set_x, off_set_y)
        self._relative_loading = bool(relative_loading)
        self.profile_interpolation_mode = profile_interpolation_mode
        self._periodic_profile = periodic_geometry
        self._periodic_axes = periodic_axes
        self._max_it = max_it
        self._rtol = tolerance
        self._max_it_outer = max_it_outer
        self._rtol_outer = tolerance_outer
        self._height_optimisation_func = None
        self._adhesion = adhesion
        self._unloading = unloading

        if method not in {'auto', 'pk', 'double', 'rey'}:
            raise ValueError(f"Unrecognised method for step {step_name}: {method}")
        self._opt_func = None
        sum_param = (normal_load is not None) + (interference is not None) + (mean_gap is not None)
        if sum_param > 1 or sum_param == 0:
            raise ValueError(f"Exactly one of normal_load, interference and mean_gap must be set, {sum_param} were set")
        if mean_gap is not None:
            if method not in {'auto', 'rey'}:
                raise ValueError("pk and double methods don't support mean gap")
            else:
                method = 'rey'
        if interference is not None and method == 'rey':
            raise ValueError("Rey method doesn't support interference")

        self._method = method
        # noinspection PyTypeChecker
        self.interference = interference
        # noinspection PyTypeChecker
        self.normal_load = normal_load
        self.mean_gap = mean_gap

        self.load_controlled = normal_load is not None

        provides = {'off_set', 'loads_z', 'surface_1_displacement_z', 'surface_2_displacement_z',
                    'total_displacement_z',
                    'interference', 'just_touching_gap', 'surface_1_points', 'contact_nodes', 'total_normal_load',
                    'surface_2_points', 'time', 'time_step', 'new_step', 'converged', 'gap'}

        super().__init__(step_name, time_period, provides)

    def solve(self, previous_state: dict, output_file) -> dict:
        """
        Solve this model step

        Parameters
        ----------
        previous_state: dict
            The previous state of the model
        output_file: file buffer
            The file in which the outputs will be written

        Returns
        -------
        current_state: dict
            The current state of the model

        """
        # just in case the displacement finder in a scipy optimise block should be a continuous function, no special
        # treatment required
        for s in self.sub_models:
            s.no_time = False

        if self._unloading and 'contact_nodes' in previous_state:
            initial_contact_nodes = previous_state['contact_nodes']
        else:
            initial_contact_nodes = None

        # noinspection PyTypeChecker
        just_touching_gap, surface_1_points, surface_2_points = get_gap_from_model(self.model, interference=0,
                                                                                off_set=self._off_set,
                                                                                mode=self.profile_interpolation_mode,
                                                                                periodic=self._periodic_profile)

        current_state = {'just_touching_gap': just_touching_gap, 'surface_1_points': surface_1_points,
                         'surface_2_points': surface_2_points, 'time': previous_state['time'] + self.max_time,
                         'time_step': self.max_time, 'new_step': True, 'off_set': self._off_set}

        adhesion_model = self.model.adhesion if self._adhesion else None
        # for some reason the type checker messes up here, types are actually correct
        # noinspection PyTypeChecker
        max_load = self.normal_load if self.load_controlled else 1.0
        opt_func = HeightOptimisationFunction(just_touching_gap=just_touching_gap, model=self.model,
                                              adhesion_model=adhesion_model,
                                              initial_contact_nodes=initial_contact_nodes,
                                              max_it=self._max_it, rtol=self._rtol,
                                              max_set_load=max_load,
                                              rtol_outer=self._rtol_outer, max_it_outer=self._max_it_outer,
                                              material_options=None,
                                              periodic_axes=self._periodic_axes, )
        self._opt_func = opt_func

        if self._method == 'auto':
            if not np.isinf(opt_func.max_pressure):
                if adhesion_model:
                    raise ValueError("Adhesion and maximum load not allowed")
                self._method = 'double'
            elif adhesion_model is not None:
                self._method = 'rey'
            else:
                self._method = 'pk'

        if self._unloading and 'contact_nodes' in previous_state:
            contact_nodes = previous_state['contact_nodes']
        else:
            contact_nodes = None

        converged = None
        if self.load_controlled:
            if self._relative_loading:
                load = previous_state['total_normal_load'] + self.normal_load
            else:
                load = self.normal_load

            if self._method == 'pk':
                print('Solving contact by PK method')
                opt_func.contact_nodes = None
                opt_func.p_and_k(load)
            elif self._method == 'rey':
                print('Solving contact by Rey method')
                opt_func.contact_nodes = None
                opt_func.rey(target_load=load)
            else:
                upper = 3 * max(just_touching_gap.flatten())
                print(f'upper bound set at: {upper}')
                print(f'Interference tolerance set to {self._rtol_outer} Relative')
                opt_func.change_load(load, contact_nodes)
                opt_func.brent(0, upper)
        else:
            if self._relative_loading:
                interference = previous_state['interference'] + self.interference
            else:
                interference = self.interference
            if self._method == 'rey':
                # safe because of error checking in init
                opt_func.rey(target_mean_gap=self.mean_gap)
            else:
                opt_func.change_load(1, contact_nodes)
                _ = opt_func(interference, current_state)
                converged = not opt_func.last_call_failed

        current_state.update(opt_func.results)
        if converged is not None:
            current_state['converged'] = converged
        self.solve_sub_models(current_state)
        self.save_outputs(current_state, output_file)

        return current_state

    def __repr__(self):
        string = (f'StaticStep({self.name}, time_period={self.max_time},'
                  f'off_set_x={self._off_set[0]}, off_set_y={self._off_set[1]},'
                  f'normal_load={self.normal_load}, interference={self.interference}'
                  f'relative_loading:={self._relative_loading}, adhesion={self._adhesion},'
                  f'unloading={self._unloading}, profile_interpolation_mode={self.profile_interpolation_mode},'
                  f'periodic_geometry={self._periodic_profile}, periodic_axes{self._periodic_axes},'
                  f'max_it_interference={self._max_it}, rtol_interference:{self._rtol},'
                  f'max_it_displacement={self._max_it_outer}, rtol_displacement={self._rtol_outer})')
        return string

ext: py

max_stars_repo_name: FrictionTribologyEnigma/SlipPY

max_issues_repo_name: FrictionTribologyEnigma/SlipPY

max_forks_repo_name: FrictionTribologyEnigma/SlipPY

reason: import numpy


text:
import numpy as np
from task.Schema import Schema

# import pdb

class StimSampler():
    '''
    a sampler of sequences
    '''

    def __init__(
            self,
            n_param,
            n_branch,
            pad_len=0,
            max_pad_len=None,
            def_path=None,
            def_prob=None,
            def_tps=None,
            key_rep_type='time',
            rm_kv=False,
            context_onehot=True,
            context_dim=1,
            context_drift=False,
            n_rm_fixed=False,
            sampling_mode='enumerative'
    ):
        self.n_param = n_param
        self.n_branch = n_branch
        self.pad_len = pad_len
        if max_pad_len is None:
            self.max_pad_len = np.max([n_param // 3 - 1, 0])
        #
        self.def_path = def_path
        self.def_prob = def_prob
        self.def_tps = def_tps
        #
        self.context_onehot = context_onehot
        self.context_dim = context_dim
        self.context_drift = context_drift
        #
        self.key_rep_type = key_rep_type
        self.sampling_mode = sampling_mode
        #
        self.rm_kv = rm_kv
        self.n_rm_fixed = n_rm_fixed
        #
        self.reset_schema()

    def reset_schema(self):
        """re-initialize the schema
        """
        self.schema = Schema(
            n_param=self.n_param,
            n_branch=self.n_branch,
            def_path=self.def_path,
            def_prob=self.def_prob,
            def_tps=self.def_tps,
            context_onehot=self.context_onehot,
            context_dim=self.context_dim,
            context_drift=self.context_drift,
            key_rep_type=self.key_rep_type,
            sampling_mode=self.sampling_mode,
        )
        self.k_dim = self.schema.k_dim
        self.v_dim = self.schema.v_dim
        self.c_dim = self.schema.c_dim

    def _sample(self):
        """sample an event sequence, one-hot vector representation

        Returns
        -------
        2d np array, 2d np array; T x (T x B), T x B
            sequence of keys / parameter values over time

        """
        # sample keys and parameter values, integer representation
        keys, vals = self.schema.sample()
        # translate to vector representation
        keys_vec = np.vstack([self.schema.key_rep[k_t, :] for k_t in keys])
        vals_vec = np.vstack([self.schema.val_rep[v_t, :] for v_t in vals])
        # ctxs_vec = np.vstack([self.schema.ctx_rep[v_t, :] for v_t in vals])
        ctxs_vec = np.vstack([self.schema.ctx_rep])
        misc = [keys, vals]
        return keys_vec, vals_vec, ctxs_vec, misc

    def sample(
            self,
            n_parts=2, p_rm_ob_enc=0, p_rm_ob_rcl=0,
            permute_observations=True, permute_queries=False,
    ):
        """sample a multi-part "movie", with repetition structure

        Parameters
        ----------
        n_parts : int
            the number of parts in this event sequence
        format: string
            the output data format
            - 'okv-qkv': human-readble form
            - 'xy': nn-readable form

        Returns
        -------
        3d np array, 3d np array; nP x T x (T x B), nP x T x B
            sequence of keys / parameter values over time
            different parts are consistent

        """
        # sample the state-param associtations
        keys_vec_, vals_vec_, ctxs_vec_, misc = self._sample()
        # sample for the observation phase
        o_keys_vec, o_vals_vec = self._sample_permutations_sup(
            keys_vec_, vals_vec_, n_parts, permute_observations)
        q_keys_vec, q_vals_vec = self._sample_permutations_sup(
            keys_vec_, vals_vec_, n_parts, permute_queries)
        # if permute_observations:
        #     o_keys_vec, o_vals_vec = self._sample_permutations(
        #         keys_vec_, vals_vec_, n_parts)
        # else:
        #     o_keys_vec = np.stack([keys_vec_ for _ in range(n_parts)])
        #     o_vals_vec = np.stack([vals_vec_ for _ in range(n_parts)])
        # # sample for the query phase
        # if permute_queries:
        #     q_keys_vec, q_vals_vec = self._sample_permutations(
        #         keys_vec_, vals_vec_, n_parts)
        # else:
        #     q_keys_vec = np.stack([keys_vec_ for _ in range(n_parts)])
        #     q_vals_vec = np.stack([vals_vec_ for _ in range(n_parts)])
        # corrupt input during encoding
        o_keys_vec, o_vals_vec = self._corrupt_observations(
            o_keys_vec, o_vals_vec, p_rm_ob_enc, p_rm_ob_rcl)
        # context are assumed to repeat across the two phases
        o_ctxs_vec = q_ctxs_vec = ctxs_vec_
        # pack sample
        o_sample_ = [o_keys_vec, o_vals_vec, o_ctxs_vec]
        q_sample_ = [q_keys_vec, q_vals_vec, q_ctxs_vec]
        # padding, if there is a delay
        [o_sample_, q_sample_] = self._delay_pred_demand(o_sample_, q_sample_)
        # pack sample
        sample_ = [o_sample_, q_sample_]
        return sample_, misc

    def _sample_permutations_sup(
        self, keys_vec_raw, vals_vec_raw, n_parts, permute
    ):
        if permute:
            s_keys_vec, s_vals_vec = self._sample_permutations(
                keys_vec_raw, vals_vec_raw, n_parts)
        else:
            s_keys_vec = np.stack([keys_vec_raw for _ in range(n_parts)])
            s_vals_vec = np.stack([vals_vec_raw for _ in range(n_parts)])
        return s_keys_vec, s_vals_vec

    def _sample_permutations(self, keys_vec_raw, vals_vec_raw, n_perms):
        """given some raw key-val pairs, generate temporal permutation sets
        """
        T = self.n_param
        keys_vec = np.zeros((n_perms, T, self.k_dim))
        vals_vec = np.zeros((n_perms, T, self.v_dim))
        # ctxs_vec = np.zeros((n_perms, T, self.c_dim))
        for ip in range(n_perms):
            # unique permutation for each movie part
            perm_op = np.random.permutation(T)
            keys_vec[ip] = keys_vec_raw[perm_op, :]
            vals_vec[ip] = vals_vec_raw[perm_op, :]
            # ctxs_vec[ip] = ctxs_vec_raw[perm_op, :]
        return keys_vec, vals_vec

    def _corrupt_observations(
        self,
        o_keys_vec, o_vals_vec,
        p_rm_ob_enc, p_rm_ob_rcl,
    ):
        """corrupt observations
        currently I only implemented zero-ing out random rows, but this function
        can be more general than this

        Parameters
        ----------
        o_keys_vec : 3d np array, nP x T x (T x B)
            keys, or states
        o_vals_vec : 3d np array, nP x T x B
            values, or actions
        p_rm_ob_enc : float
            p(zero-ing out observation at time t) during encoding
        p_rm_ob_rcl : float
            p(zero-ing out observation at time t) during recall

        Returns
        -------
        3d np array, 3d np array; nP x T x (T x B), nP x T x B
            keys,values after corruption

        """
        # the 1st part is the encoding phase
        # all remaining parts are query phase
        n_parts = len(o_keys_vec)
        # get a list of p_rm, only the 1st phase is the encoding phase
        # the rest of phases are considered as recall phases
        p_rms = [p_rm_ob_enc] * (n_parts - 1) + [p_rm_ob_rcl]
        # zero out random rows (time steps)
        for ip in range(n_parts):
            # zero out both key and values
            if self.rm_kv:
                [o_keys_vec[ip], o_vals_vec[ip]] = _zero_out_random_rows(
                    [o_keys_vec[ip], o_vals_vec[ip]], p_rms[ip],
                    n_rm_fixed=self.n_rm_fixed
                )
            # zero out values only
            # in this case the agent know which state is unknown
            else:
                [o_vals_vec[ip]] = _zero_out_random_rows(
                    [o_vals_vec[ip]], p_rms[ip],
                    n_rm_fixed=self.n_rm_fixed
                )
        return o_keys_vec, o_vals_vec

    def _delay_pred_demand(self, o_sample_, q_sample_):
        """apply delay to the queries, and zero pad the end of observations

        Parameters
        ----------
        o_sample_ : list
            observations
        q_sample_ : list
            queries

        Returns
        -------
        list, list
            padded observations and queries

        """
        if self.pad_len == 0 or self.max_pad_len == 0:
            return o_sample_, q_sample_

        # uniformly sample a padding length
        if self.pad_len == 'random':
            # high is exclusive so need to add 1
            pad_len = np.random.randint(low=0, high=self.max_pad_len + 1)
        # fixed padding length
        elif self.pad_len > 0:
            pad_len = self.pad_len
        else:
            raise ValueError(f'Invalid delay length: {self.pad_len}')

        # padd the data
        o_sample_ = _zero_pad_kvc(o_sample_, pad_len, side='bot')
        q_sample_ = _zero_pad_kvc(q_sample_, pad_len, side='top')
        return o_sample_, q_sample_

def _zero_out_random_rows(matrices, p_rm, n_rm_fixed=True):
    """zero out the same set of (randomly selected) rows for all input matrices

    Parameters
    ----------
    matrices : list
        a list of 2d arrays
    p_rm : float
        probability for set a row of zero

    Returns
    -------
    list
        a list of 2d arrays
    """
    assert 0 <= p_rm <= 1
    n_rows, _ = np.shape(matrices[0])
    for matrix in matrices:
        assert np.shape(matrix)[0] == n_rows
    # select # row(s) to zero out
    if n_rm_fixed:
        n_rows_to0 = np.ceil(p_rm * n_rows)
    else:
        # in this case, p_rm == E[rows_to_remove]
        max_rows_to_remove = p_rm * n_rows
        n_rows_to0 = np.round(np.random.uniform(high=max_rows_to_remove))
    # select some rows to zero out
    rows_to0 = np.random.choice(
        range(n_rows), size=int(n_rows_to0), replace=False
    )
    # zero out the same rows for all input matrices
    for i in range(len(matrices)):
        matrices[i][rows_to0, :] = 0
    return matrices

def _zero_pad_kvc(kvc: list, pad_len: int, side: str):
    """delay the prediction demand by shifting the query value to later time
    points

    Parameters
    ----------
    kvc : list
        Description of parameter `kvc`.
    pad_len : int
        Description of parameter `pad_len`.

    Returns
    -------
    type
        Description of returned object.

    """
    # unpack data
    keys_vec, vals_vec, ctxs_vec = kvc
    n_parts, n_params, k_dim = np.shape(keys_vec)
    _, _, v_dim = np.shape(vals_vec)
    _, c_dim = np.shape(ctxs_vec)
    # pad to delay prediction time
    keys_vec = [_vpad(k_mat, pad_len, side=side) for k_mat in keys_vec]
    vals_vec = [_vpad(v_mat, pad_len, side=side) for v_mat in vals_vec]
    # TODO here i assumed context is always in sync with the queries
    # but probably want to generate additional context for the padding period
    ctxs_vec = _vpad(ctxs_vec, pad_len, side='top')
    # pack the data
    kvc_ = [keys_vec, vals_vec, ctxs_vec]
    return kvc_

def _vpad(matrix, pad_len: int, side: str):
    '''vertically pad zeros from the top or bot'''
    #
    n_rows, n_cols = np.shape(matrix)
    zero_padding = np.zeros((pad_len, n_cols))
    if side == 'top':
        padded_matrix = np.vstack([zero_padding, matrix])
    elif side == 'bot':
        padded_matrix = np.vstack([matrix, zero_padding])
    else:
        raise ValueError('Unrecognizable padding side')
    return padded_matrix

'''test'''
if __name__ == "__main__":
    import matplotlib.pyplot as plt
    # from task.utils import sample_rand_path,sample_def_tps
    # init a graph
    n_param, n_branch = 10, 2
    n_parts = 2
    pad_len = 0
    # def_tps = np.array([1, 0] * 5)
    # def_tps = np.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
    # def_tps = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
    def_tps = np.array([1, 0] * (n_param // 2))
    def_tps = np.array([0, 1] * (n_param // 2))
    # def_path = np.tile(np.array([[1, 0], [0, 1]]), (1, 5)).T
    # def_path = np.tile(np.array([[1, 0], [0, 1]]), (1, n_param // 2)).T
    def_path = np.vstack([[1, 0] for i in range(10)])
    def_prob = .9
    # pad_len = 'random'
    p_rm_ob_enc, p_rm_ob_rcl = .5, .5
    p_rm_ob_enc, p_rm_ob_rcl = 0, 0
    key_rep_type = 'time'
    permute_queries = False
    permute_observations = False
    # key_rep_type = 'gaussian'
    sampler = StimSampler(
        n_param, n_branch,
        pad_len=pad_len,
        key_rep_type=key_rep_type,
        def_tps=def_tps, def_path=def_path, def_prob=def_prob
    )
    sample_, misc = sampler.sample(
        n_parts, p_rm_ob_enc=p_rm_ob_enc, p_rm_ob_rcl=p_rm_ob_rcl,
        # permute_queries=permute_queries, permute_observations=permute_observations
    )
    observations, queries = sample_
    [o_keys_vec, o_vals_vec, o_ctxs_vec] = observations
    [q_keys_vec, q_vals_vec, q_ctxs_vec] = queries

    # plot
    cmap = 'bone'
    n_timesteps = n_param
    width_ratios = [sampler.k_dim, sampler.v_dim] * 2 + [sampler.c_dim]
    f, axes = plt.subplots(
        n_parts, 5, figsize=(8, 5), sharey=True,
        gridspec_kw={'width_ratios': width_ratios}
    )
    for ip in range(n_parts):
        axes[ip, 0].imshow(o_keys_vec[ip], cmap=cmap)
        axes[ip, 1].imshow(o_vals_vec[ip], cmap=cmap)
        axes[ip, 2].imshow(q_keys_vec[ip], cmap=cmap)
        axes[ip, 3].imshow(q_vals_vec[ip], cmap=cmap)

    axes[0, 4].imshow(o_ctxs_vec, cmap=cmap)
    axes[1, 4].imshow(q_ctxs_vec, cmap=cmap)

    # label
    axes[0, 0].set_title('Observation')
    axes[0, 2].set_title('Queries')
    axes[-1, 0].set_xlabel('Keys/States')
    axes[-1, 1].set_xlabel('Values/Action')
    axes[-1, 2].set_xlabel('Keys/States')
    axes[-1, 3].set_xlabel('Values/Action')
    axes[0, 4].set_title('o, Context')
    axes[1, 4].set_title('q, Context')
    # modify y ticks/labels
    for ip in range(n_parts):
        axes[ip, 0].set_yticks(range(n_timesteps))
        axes[ip, 0].set_yticklabels(range(n_timesteps))
        axes[ip, 0].set_ylabel(f'Time, part {ip+1}')
    f.tight_layout()

max_stars_repo_name: jacksonlli/learn-hippo

max_issues_repo_name: jacksonlli/learn-hippo

max_forks_repo_name: jacksonlli/learn-hippo


text:
'''
getYahooData.py
Date Created: 4/14/20
Date Last Modified: 05/02/20
By: Nick Piacente

Functions for generating processed dataframes and plots
For future plotting in a GUI environment
important columns:

'Potential Gain' - this is the option contract price * 100
'POP' - this is the probability of profit. The

Currently filtering options by budget = strike price * 100
It takes a few seconds per stock to fetch data from the internet on my machine (NP)
'''

from yahoo_fin import options
from yahoo_fin import stock_info
import pandas as pd
import numpy as np
from scipy import stats
from datetime import datetime

# user inputs
personalRiskTolerance = .9
budget = 10000

def getOptionsData(personalRiskTolerance, budget, printOutput = 'True'):
    '''
    get the up to date stock options pareto.
    Currently hardcoded for the DOW stocks
    inputs:
        personalRiskTolerance - risk level - number between 0 and 0.9
        budget - Number - larger than 2000
        printOutput - bool to determine whether print statements show
    outputs:
       stockPareto DataFrame (all options within the budget)
       bestPick DataFrame (highest value return within risk tolerance)
    hardcoded:
       options date - 5/15/2020
       stocks to pick from: All DOW stocks
    '''

    # date selection
    today = datetime.today()
    optionsDate = datetime(2020,5,15)
    t = ((optionsDate - today).days + ((optionsDate - today).seconds/86400))/365
    daysLeft = (optionsDate - today).days
    hoursLeft = int((optionsDate - today).seconds/3600)
    if printOutput:
        print('\nOptions will expire in approximately {} days and {} hours'.format(daysLeft,hoursLeft))
    stockPareto = pd.DataFrame()

    if printOutput:
        print('Pulling stock data...\n')

    # initial data capture and processing
    for stock in stock_info.tickers_dow():

        # checks if there is an option during the particular week.
        try:
            individualOptionsData = options.get_puts(stock,date=optionsDate)
            # capture stock, current price for each stock
            individualOptionsData['Stock Name'] = stock
            individualOptionsData['Current Price'] = stock_info.get_live_price(stock)
            # IV formatting to a number
            individualOptionsData['IV']= individualOptionsData['Implied Volatility'].str.slice_replace(-1,repl='').astype(float)/100

        # if not, skip the ticker
        except:
            if printOutput:
                print('No data from {}'.format(stock))
            continue

        # if successful at obtaining options data, fill DataFrame stockPareto with the values
        stockPareto = stockPareto.append(individualOptionsData)
        if printOutput:
            print('Data from {} collected'.format(stock))

    if printOutput:
        print('Data Capture Complete!')

    # bool logic to make sure options are worth investing in
    beingTraded = stockPareto['Volume'] != '-'
    asksExist = stockPareto['Ask'] != '-'
    bidsExist = stockPareto['Bid'] != '-'

    # filtering by bool logic
    stockPareto = stockPareto[beingTraded & asksExist & bidsExist]

    ####################################################
    ### formatting of key variables in the DataFrame ###
    ####################################################

    # POP derived from Black-Scholes model
    stockPareto['POP'] = 100*stats.norm.cdf((np.log(stockPareto['Current Price'] / stockPareto['Strike'] ) +
                                         ( (stockPareto['IV']**2) /2)*t) / (stockPareto['IV']*np.sqrt(t)))
    stockPareto['Strike'] = stockPareto['Strike'].astype(float)
    stockPareto['Bid'] = stockPareto['Bid'].astype(float)
    stockPareto['Ask'] = stockPareto['Ask'].astype(float)

    # determine number of each contract that is in budget
    # potential gain is the average contract selling price ~ (bid / ask) * 100
    # potential gain multiple contracts - potential gain * number of contracts that can be sold within budget
    stockPareto['contractsInBudget'] = np.floor(budget/(stockPareto['Strike']*100))
    stockPareto['Potential Gain'] = ((stockPareto['Ask'] + stockPareto['Bid'])/2) * 100
    stockPareto['Potential Gain Multiple Contracts'] = stockPareto['Potential Gain'] * stockPareto['contractsInBudget']
    ###########################
    ### plotting the pareto ###
    ###########################

    inBudget = stockPareto['contractsInBudget'] > 0
    isInteresting = stockPareto['Bid'] != 0
    stockPareto = stockPareto[inBudget & isInteresting]
    stockPareto = stockPareto.set_index('Contract Name')

    # this is the field which will show in the interactive plot
    stockPareto['printString'] = ('Stock: ' + stockPareto['Stock Name']+'\n'
                                  'Strike Price = $'+ round(stockPareto['Strike'],2).astype(str)+'\n'+
                                  'Current Price = $' + round(stockPareto['Current Price'],2).astype(str) + '\n' +
                                  'Potential Gain = $'+ round(stockPareto['Potential Gain Multiple Contracts'],2).astype(str)+'\n' +
                                  'ROI = ' + round(((stockPareto['Potential Gain Multiple Contracts']/budget) * 100),2).astype(str) + '%\n' +
                                  'Probability of Profit = ' + round(stockPareto['POP'],2).astype(str))

    ####################
    ## Best Fit Logic ##
    ####################

    notBelowRisk = stockPareto['POP'] > (personalRiskTolerance*100)
    bestPick = stockPareto[notBelowRisk].sort_values(by='Potential Gain Multiple Contracts',ascending = False)
    bestPick = bestPick.loc[bestPick['Potential Gain Multiple Contracts'].idxmax()]

    if printOutput:
        print('The best OPTION is:')
        print(bestPick)

    return stockPareto, bestPick

def getDetailedQuote(stock, ax1 = None):

    '''
    # get the detailded bid/ask quote data for charting
    # input :
        stock - stock ticker of the option of interest
        ax1 - matplotlib axis , if desired to specify
    # for use with bestPick frame:
    #   bestPick['Stock'] would be the input
    # output is a matplotlib AxesSubplot object
    '''

    # find the current price of the stock in question
    currentPrice=stock_info.get_live_price(stock)
    stockOptionVals=options.get_puts(stock, date='05/15/20')
    asksExist = stockOptionVals['Ask'] != '-'
    bidsExist = stockOptionVals['Bid'] != '-'

    stockOptionVals = stockOptionVals[bidsExist & asksExist]
    stockOptions = stockOptionVals.plot(x='Strike',y=['Bid','Ask'],
                                                xlim=[.25*currentPrice,1.75*currentPrice],
                                                title='Bid and Ask Prices for {} Options Contracts'.format(stock), ax = ax1)
    # plot the current stock price as a vertical line in green
    stockOptions.axvline(currentPrice, color='green', ls='--')
    stockOptions.set_xlabel('Strike Price ($)')
    stockOptions.set_ylabel('Contract Price ($)')

    return stockOptions

if __name__ == '__main__':
    #stockPareto, bestPick = getOptionsData(personalRiskTolerance, budget, printOutput = True)
    chart = getDetailedQuote('DOW')

reason: import numpy,from scipy


text:
import numpy as np
import matplotlib.pyplot as plt
from pyrsistent import b
from scipy.stats import multivariate_normal
from mpl_toolkits.mplot3d import Axes3D
from scipy.optimize import minimize
np.set_printoptions(threshold=np.inf)
plt.rcParams['figure.figsize'] = [9,9]

N_features = 2            # Number of features
N_labels = 2              # Number of classes
N_mixtures = 3          # Number of gaussian distributions

#Seed to obtain same results for random numbers
np.random.seed(10)

# Class Priors
priors = np.array([0.65, 0.35])

# Mean vectors
mean_matrix = np.zeros(shape=[N_mixtures, N_features])
mean_matrix [0, :] = [3, 0]
mean_matrix [1, :] = [0, 3]
mean_matrix [2, :] = [2, 2]

# Covariance matrices
covariance_matrix = np.zeros(shape=[N_mixtures, N_features, N_features])
covariance_matrix [0, :, :] = [[2, 0],[0, 1]]
covariance_matrix [1, :, :] = [[1, 0],[0, 2]]
covariance_matrix [2, :, :] = [[1, 0],[0, 1]]

# Prior weights for gaussian components of class 2 and assigning labels
weights = [0.5,0.5]
cumsum = np.cumsum(priors)

#Function to generate data samples
def generateData(N_Samples):
    label = (np.random.rand(N_Samples) >= priors[1]).astype(int)

    # Generate gaussian distribution for 10000 samples using mean and covariance matrices for each label
    X = np.zeros(shape = [N_Samples, N_features])
    for i in range(N_Samples):
        if (label[i] == 0):
            # Split samples based on mixture weights
            if (np.random.rand(1,1) >= weights[1]):
                X[i, :] = np.random.multivariate_normal(mean_matrix[0, :], covariance_matrix[0, :, :])
            else:
                X[i, :] = np.random.multivariate_normal(mean_matrix[1, :], covariance_matrix[1, :, :])

        elif (label[i] == 1):
            X[i, :] = np.random.multivariate_normal(mean_matrix[2, :], covariance_matrix[2, :, :])

    return X, label

# Plot the data distribution to visualize the samples
def plotDistribution(X, label, w, type):
    fig = plt.figure()
    ax = Axes3D(fig)
    ax.scatter(X[np.where(label==0),0],X[np.where(label==0),1],color = 'blue')
    ax.scatter(X[np.where(label==1),0],X[np.where(label==1),1],color = 'red')
    plt.xlabel('X1')
    plt.ylabel('X2')
    plt.title('Data Distribution')
    plot_decision_boundary(X, w, type)
    plt.show()

# Function to plot classified data along with decision boundary
def plot_classified_data(X, label, decision, w, type):
    plt.plot(X[np.where((label == 1) & (decision == 1)),0], X[np.where((label == 1) & (decision == 1)),1], color ='green', marker = '^', markersize = 1)
    plt.plot(X[np.where((label == 1) & (decision == 0)),0], X[np.where((label == 1) & (decision == 0)),1], color ='red', marker = '^', markersize = 1)
    plt.plot(X[np.where((label == 0) & (decision == 1)),0], X[np.where((label == 0) & (decision == 1)),1], color ='red', marker = 's', markersize = 1)
    plt.plot(X[np.where((label == 0) & (decision == 0)),0], X[np.where((label == 0) & (decision == 0)),1], color ='green', marker = 's', markersize = 1)
    plt.xlabel("Feature x1")
    plt.ylabel("Feature x2")
    plt.title("Plot showing decision boundary and classified data")
    plot_decision_boundary(X, w, type)

# Function to decide decision boundary based on classifier type
def plot_decision_boundary(X, w, type):
    hgrid = np.linspace(min(X[:,0]), max(X[:,0]), 100)
    vgrid = np.linspace(min(X[:,1]), max(X[:,1]), 100)
    x1, x2 = np.meshgrid(hgrid,vgrid)
    b = np.zeros((100,100))
    loggamma = np.log(priors[0] / priors[1])
    for i in range(len(x1)):
        for j in range(len(x2)):
            if (type == 'T'):
                GaussPDF0 = np.log((w[0] * (multivariate_normal.pdf(np.array([x1[i][j], x2[i][j]]), mean = mean_matrix[0, :], cov = covariance_matrix[0, :,:]))) + (w[1] * (multivariate_normal.pdf(np.array(x1[i][j], x2[i][j]),mean = mean_matrix[1, :], cov = covariance_matrix[1, :,:]))))
                GaussPDF1 = np.log(multivariate_normal.pdf(np.array([x1[i][j], x2[i][j]]), mean = mean_matrix[2, :], cov = covariance_matrix[2, :,:]))
                b[i][j] = GaussPDF1 - GaussPDF0 - loggamma
            elif (type == 'L'):
                z = np.c_[1, x1[i][j], x2[i][j]].T
                b[i][j] = np.sum(np.dot(w.T, z))
            elif (type == 'Q'):
                z = np.c_[1, x1[i][j], x2[i][j], x1[i][j]**2, x1[i][j] * x2[i][j], x2[i][j] ** 2].T
                b[i][j] = np.sum(np.dot(w.T, z))

    plt.contour(x1, x2, b, levels = [0])
    plt.show()

# Generate Validation dataset
X_validate, Label_validate = generateData(10000)

## Part A - Theoretical Bayesian Classifier

# Compute discriminant score using class conditional PDF
GaussPDF0 = np.log(weights[0] * (multivariate_normal.pdf(X_validate,mean = mean_matrix[0, :], cov = covariance_matrix[0, :,:])) + weights[1] * (multivariate_normal.pdf(X_validate,mean = mean_matrix[1, :], cov = covariance_matrix[1, :,:])))
GaussPDF1 = np.log(multivariate_normal.pdf(X_validate,mean = mean_matrix[2, :], cov = covariance_matrix[2, :,:]))
discrim_score = GaussPDF1 - GaussPDF0

# Sort tau values to navigate from minimum to maximum value
sorted_tau = np.sort(discrim_score)
tau_sweep = []

# Calculate mid-points which will be used as threshold values
for i in range(0,9999):
        tau_sweep.append((sorted_tau[i] + sorted_tau[i+1])/2.0)

# Array initialization for results
decision = []
TP = [None] * len(tau_sweep)
FP = [None] * len(tau_sweep)
minPerror = [None] * len(tau_sweep)

# Classify for each threshold and compute error and evaluation metrics
for (index, tau) in enumerate(tau_sweep):
        decision = (discrim_score >= tau)
        TP[index] = (np.size(np.where((decision == 1) & (Label_validate == 1))))/np.size(np.where(Label_validate == 1))
        FP[index] = (np.size(np.where((decision == 1) & (Label_validate == 0))))/np.size(np.where(Label_validate == 0))
        minPerror[index] = (priors[0] * FP[index]) + (priors[1] * (1 - TP[index]))

# Theoretical classification based on class priors
loggamma_ideal = np.log(priors[0] / priors[1])
ideal_decision = (discrim_score >= loggamma_ideal)
TP_ideal = (np.size(np.where((ideal_decision == 1) & (Label_validate == 1))))/np.size(np.where(Label_validate == 1))
FP_ideal = (np.size(np.where((ideal_decision == 1) & (Label_validate == 0))))/np.size(np.where(Label_validate == 0))
minPerror_ideal = (priors[0] * FP_ideal) + (priors[1] * (1 - TP_ideal))
print("Gamma Ideal - %f and corresponding minimum error %f" %(np.exp(loggamma_ideal), minPerror_ideal))

plot_classified_data(X_validate, Label_validate, ideal_decision, weights, 'T')

# Plot ROC curve
plt.plot(FP, TP, color = 'red')
plt.xlabel('False Positive')
plt.ylabel('True Positive')
plt.title('ROC Curve')
plt.plot(FP[np.argmin(minPerror)], TP[np.argmin(minPerror)],'o',color = 'black')
plt.show()

print("Gamma Practical - %f and corresponding minimum error %f" %(np.exp(tau_sweep[np.argmin(minPerror)]), np.min(minPerror)))

# Loss Function for Logistic Regression
def cost_function(w, z, N, Label_train):
    h = 1 / (1 + np.exp(-(np.dot(w.T, z))))
    cost = (-1 / N) * ((np.sum(Label_train * np.log(h))) + (np.sum((1 - Label_train) * np.log(1 - h))))
    return cost

## Part B- Logistic Based Linear and Quadratic Classifiers
N_train = [20, 200, 2000]                # Number of training samples for each model
for N in N_train:
    # Generate Training Data
    X_train, Label_train = generateData(N)
    # Create basis vector
    z_L = np.column_stack((np.ones(X_train.shape[0]), X_train)).T
    z_Q = np.column_stack((np.ones(X_train.shape[0]), X_train[:, 0], X_train[:, 1], np.square(X_train[:, 0]), np.multiply(X_train[:, 0], X_train[:, 1]), np.square(X_train[:, 1]))).T
    #Initialize weight vector
    w_L = np.zeros(shape = (3, 1))
    w_Q = np.zeros(shape = (6, 1))
    #Perform Optimization using cost function defined
    res_L = minimize(cost_function, w_L, args = (z_L, N, Label_train), method = 'Nelder-Mead')
    res_Q = minimize(cost_function, w_Q, args = (z_Q, N, Label_train), method = 'Nelder-Mead')
    wL_final = res_L.x
    wQ_final = res_Q.x

    # Visualize the training data with decision boundary
    plotDistribution(X_train, Label_train, wL_final, 'L')
    plotDistribution(X_train, Label_train, wQ_final, 'Q')

    # Create Validation dataset
    X_validate, Label_validate = generateData(10000)
    ztest_L = np.column_stack((np.ones(X_validate.shape[0]), X_validate)).T
    # Make decisions to test the models
    decision_L = []
    decision_L = (1 / (1+ np.exp(-(np.dot(wL_final.T, ztest_L)))) >= 0.5).astype(int)

    ztest_Q = np.column_stack((np.ones(X_validate.shape[0]), X_validate[:, 0], X_validate[:, 1], np.square(X_validate[:, 0]), np.multiply(X_validate[:, 0], X_validate[:, 1]), np.square(X_validate[:, 1]))).T
    decision_Q = []
    decision_Q = (1 / (1+ np.exp(-(np.dot(wQ_final.T, ztest_Q)))) >= 0.5).astype(int)

    # Evaluate the model and compute error
    TP_L = (np.size(np.where((decision_L == 1) & (Label_validate == 1))))
    TN_L = (np.size(np.where((decision_L == 0) & (Label_validate == 0))))
    print("Error for Logistic Linear Model with ", N, " samples is ", (10000 - (TP_L + TN_L))/100)

    TP_Q = (np.size(np.where((decision_Q == 1) & (Label_validate == 1))))
    TN_Q = (np.size(np.where((decision_Q == 0) & (Label_validate == 0))))
    print("Error for Logistic Quadratic Model with ", N, " samples is ", (10000 - (TP_Q + TN_Q))/100)

    # Plot Validation dataset with decision boundary
    plot_classified_data(X_validate, Label_validate, decision_L, wL_final, 'L')
    plot_classified_data(X_validate, Label_validate, decision_Q, wQ_final, 'Q')


text:
from abc import ABC, abstractmethod

import numpy as np

from infiniteremixer.data.aggregation.aggregator import Aggregator

class BatchAggregator(ABC):
    """BatchAggregator is an abstract class that provides an interface
    to apply multiple statistical aggregation on 2d numpy arrays.
    """

    def __init__(self) -> None:
        self.aggregators = []

    def add_aggregator(self, aggregator: Aggregator) -> None:
        """Add an aggregator function to the aggregators.

        :param aggregator: (Aggregator) Concrete Aggregator
        """
        self.aggregators.append(aggregator)

    @abstractmethod
    def aggregate(self, array: np.ndarray) -> np.ndarray:
        pass

max_stars_repo_name: nroldanf/infiniteremixer

max_issues_repo_name: nroldanf/infiniteremixer

max_forks_repo_name: nroldanf/infiniteremixer


text:
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import ast
import json
import os
import random
import statistics
from abc import ABC
from collections import defaultdict
from copy import deepcopy
from typing import List, Dict

import numpy as np
import torch
from sklearn.metrics import f1_score, precision_score, recall_score
from transformers.data.metrics import simple_accuracy

import logg
from classification.utils_classif import InputExample, exact_match, save_logits, save_predictions, softmax, LogitsList, set_seed, eq_div
from classification.wrapper_classif import TransformerModelWrapper, SEQUENCE_CLASSIFIER_WRAPPER, WrapperConfig

logger = logg.get_logger('root')

class PetConfig(ABC):
    """Abstract class for a PET configuration that can be saved to and loaded from a json file."""

    def __repr__(self):
        return repr(self.__dict__)

    def save(self, path: str):
        """Save this config to a file."""
        with open(path, 'w', encoding='utf8') as fh:
            json.dump(self.__dict__, fh)

    @classmethod
    def load(cls, path: str):
        """Load a config from a file."""
        cfg = cls.__new__(cls)
        with open(path, 'r', encoding='utf8') as fh:
            cfg.__dict__ = json.load(fh)
        return cfg

class TrainConfig(PetConfig):
    """Configuration for training a model."""

    def __init__(self, device: str = None, per_gpu_train_batch_size: int = 8, per_gpu_unlabeled_batch_size: int = 8,
                 n_gpu: int = 1, num_train_epochs: int = 3, max_steps: int = -1, gradient_accumulation_steps: int = 1,
                 weight_decay: float = 0.0, learning_rate: float = 5e-5, adam_epsilon: float = 1e-8,
                 warmup_steps: int = 0, max_grad_norm: float = 1, lm_training: bool = False, use_logits: bool = False,
                 alpha: float = 0.9999, temperature: float = 1):
        """
        Create a new training config.

        :param device: the device to use ('cpu' or 'gpu')
        :param per_gpu_train_batch_size: the number of labeled training examples per batch and gpu
        :param per_gpu_unlabeled_batch_size: the number of unlabeled examples per batch and gpu
        :param n_gpu: the number of gpus to use
        :param num_train_epochs: the number of epochs to train for
        :param max_steps: the maximum number of steps to train for (overrides ``num_train_epochs``)
        :param gradient_accumulation_steps: the number of steps to accumulate gradients for before performing an update
        :param weight_decay: the weight decay to use
        :param learning_rate: the maximum learning rate to use
        :param adam_epsilon: the epsilon value for Adam
        :param warmup_steps: the number of warmup steps to perform before reaching the maximum learning rate
        :param max_grad_norm: the maximum norm for the gradient
        :param lm_training: whether to perform auxiliary language modeling (only for MLMs)
        :param use_logits: whether to use each training example's logits instead of its label (used for distillation)
        :param alpha: the alpha parameter for auxiliary language modeling
        :param temperature: the temperature for distillation
        """
        self.device = device
        self.per_gpu_train_batch_size = per_gpu_train_batch_size
        self.per_gpu_unlabeled_batch_size = per_gpu_unlabeled_batch_size
        self.n_gpu = n_gpu
        self.num_train_epochs = num_train_epochs
        self.max_steps = max_steps
        self.gradient_accumulation_steps = gradient_accumulation_steps
        self.weight_decay = weight_decay
        self.learning_rate = learning_rate
        self.adam_epsilon = adam_epsilon
        self.warmup_steps = warmup_steps
        self.max_grad_norm = max_grad_norm
        self.lm_training = lm_training
        self.use_logits = use_logits
        self.alpha = alpha
        self.temperature = temperature

class EvalConfig(PetConfig):
    """Configuration for evaluating a model."""

    def __init__(self, device: str = None, n_gpu: int = 1, per_gpu_eval_batch_size: int = 8,
                 metrics: List[str] = None, decoding_strategy: str = 'default', priming: bool = False):
        """
        Create a new evaluation config.

        :param device: the device to use ('cpu' or 'gpu')
        :param n_gpu: the number of gpus to use
        :param per_gpu_eval_batch_size: the number of evaluation examples per batch and gpu
        :param metrics: the evaluation metrics to use (default: accuracy only)
        :param decoding_strategy: the decoding strategy for PET with multiple masks ('default', 'ltr', or 'parallel')
        :param priming: whether to use priming
        """
        self.device = device
        self.n_gpu = n_gpu
        self.per_gpu_eval_batch_size = per_gpu_eval_batch_size
        self.metrics = metrics
        self.decoding_strategy = decoding_strategy
        self.priming = priming

def init_model(config: WrapperConfig) -> TransformerModelWrapper:
    """Initialize a new model from the given config."""
    assert config.pattern_id is not None, 'A pattern_id must be set for initializing a new PET model'
    model = TransformerModelWrapper(config)
    return model

def train_classifier(model_config: WrapperConfig, train_config: TrainConfig, eval_config: EvalConfig, output_dir: str,
                     repetitions: int = 3, train_data: List[InputExample] = None, val_data: List[InputExample] = None,
                     unlabeled_data: List[InputExample] = None, eval_data: List[InputExample] = None,
                     do_train: bool = True, do_eval: bool = True, seed: int = 42):
    """
    Train and evaluate a sequence classification model.

    :param model_config: the model configuration to use
    :param train_config: the training configuration to use
    :param eval_config: the evaluation configuration to use
    :param output_dir: the output directory
    :param repetitions: the number of training repetitions
    :param train_data: the training examples to use
    :param unlabeled_data: the unlabeled examples to use
    :param eval_data: the evaluation examples to use
    :param do_train: whether to perform training
    :param do_eval: whether to perform evaluation
    :param seed: the random seed to use
    """

    logger.info("\n--- CLASSIFICATION ---")

    # Step 2: Merge the annotations created by each individual model
    logger.info("\n--- Load Logits ---")
    logits_file = os.path.join(output_dir, 'unlabeled_logits.txt')
    logits = LogitsList.load(logits_file).logits
    assert len(logits) == len(unlabeled_data)
    logger.info("Got {} logits from file {}".format(len(logits), logits_file))
    for example, example_logits in zip(unlabeled_data, logits):
        example.logits = example_logits

    # Step 3: Train the final sequence classifier model
    model_config.wrapper_type = SEQUENCE_CLASSIFIER_WRAPPER
    model_config.use_logits = True

    train_pet_ensemble(model_config, train_config, eval_config, pattern_ids=[0], output_dir=output_dir,
                       repetitions=repetitions, train_data=train_data, val_data=val_data, unlabeled_data=unlabeled_data,
                       eval_data=eval_data, do_train=do_train, do_eval=do_eval, seed=seed, apply_classification=True)

def train_pet_ensemble(model_config: WrapperConfig, train_config: TrainConfig, eval_config: EvalConfig,
                       pattern_ids: List[int], output_dir: str, ipet_data_dir: str = None, repetitions: int = 3,
                       train_data: List[InputExample] = None, val_data: List[InputExample] = None,
                       unlabeled_data: List[InputExample] = None,
                       eval_data: List[InputExample] = None, do_train: bool = True, do_eval: bool = True,
                       save_unlabeled_logits: bool = False, seed: int = 42, apply_classification: bool = False):
    """
    Train and evaluate an ensemble of PET models without knowledge distillation.

    :param model_config: the model configuration to use
    :param train_config: the training configuration to use
    :param eval_config: the evaluation configuration to use
    :param pattern_ids: the ids of all PVPs to use
    :param output_dir: the output directory
    :param ipet_data_dir: optional directory containing additional training data for iPET
    :param repetitions: the number of training repetitions
    :param train_data: the training examples to use
    :param unlabeled_data: the unlabeled examples to use
    :param eval_data: the evaluation examples to use
    :param do_train: whether to perform training
    :param do_eval: whether to perform evaluation
    :param save_unlabeled_logits: whether logits for unlabeled examples should be saved in a file ``logits.txt``. This
           is required for both iPET and knowledge distillation.
    :param seed: the random seed to use
    :param apply_classification: apply a Classification model
    """

    results = defaultdict(lambda: defaultdict(list))
    set_seed(seed)

    logger.info("\n--- VERBALIZER : {} ---".format(model_config.verbalizer))

    for pattern_id in pattern_ids:

        if not apply_classification:
            logger.info("\n--- PATTERN {} : {} ---".format(pattern_id, model_config.pattern[pattern_id]))

        for iteration in range(repetitions):

            logger.info("Iteration : {}/{}".format(iteration, repetitions))

            model_config.pattern_id = pattern_id
            results_dict = {}

            if apply_classification:
                pattern_iter_output_dir = "{}/final_model".format(output_dir)
            else:
                pattern_iter_output_dir = "{}/p{}-i{}".format(output_dir, pattern_id, iteration)

            if os.path.exists(pattern_iter_output_dir):
                logger.warning(f"Path {pattern_iter_output_dir} already exists, skipping it...")
                continue

            if not os.path.exists(pattern_iter_output_dir):
                os.makedirs(pattern_iter_output_dir)

            wrapper = init_model(model_config)

            # Training
            if do_train:
                if ipet_data_dir:
                    p = os.path.join(ipet_data_dir, 'p{}-i{}-train.bin'.format(pattern_id, iteration))
                    ipet_train_data = InputExample.load_examples(p)
                    for example in ipet_train_data:
                        example.logits = None
                else:
                    ipet_train_data = None

                results_dict.update(train_single_model(wrapper, train_data, train_config, eval_config,
                                                       val_data=val_data,
                                                       ipet_train_data=ipet_train_data,
                                                       unlabeled_data=unlabeled_data))

                with open(os.path.join(pattern_iter_output_dir, 'results.txt'), 'w') as fh:
                    fh.write(str(results_dict))

                logger.info("Saving trained model at {}...".format(pattern_iter_output_dir))
                wrapper.save(pattern_iter_output_dir)
                train_config.save(os.path.join(pattern_iter_output_dir, 'train_config.json'))
                eval_config.save(os.path.join(pattern_iter_output_dir, 'eval_config.json'))
                logger.info("Saving complete")

                if save_unlabeled_logits:
                    logits = evaluate(wrapper, unlabeled_data, eval_config, type_dataset="unlabeled")['logits']
                    save_logits(os.path.join(pattern_iter_output_dir, 'logits.txt'), logits)

                if not do_eval:
                    wrapper.model = None
                    wrapper = None
                    torch.cuda.empty_cache()

            # Evaluation
            if do_eval:
                logger.info("Starting evaluation on eval set...")
                if not wrapper:
                    wrapper = TransformerModelWrapper.from_pretrained(pattern_iter_output_dir)

                eval_result = evaluate(wrapper, eval_data, eval_config, type_dataset="eval_set", priming_data=train_data)

                save_predictions(os.path.join(pattern_iter_output_dir, 'predictions.jsonl'), wrapper, eval_result)
                save_logits(os.path.join(pattern_iter_output_dir, 'eval_logits.txt'), eval_result['logits'])

                scores = eval_result['scores']
                if apply_classification:
                    logger.info("--- RESULT (iteration={}) ---".format(iteration))
                else:
                    logger.info("--- RESULT (pattern_id={}, iteration={}) ---".format(pattern_id, iteration))
                logger.info(scores)

                results_dict['test_set_after_training'] = scores
                with open(os.path.join(pattern_iter_output_dir, 'results.json'), 'w') as fh:
                    json.dump(results_dict, fh)

                for metric, value in scores.items():
                    results[metric][pattern_id].append(value)

                wrapper.model = None
                wrapper = None
                torch.cuda.empty_cache()

    if do_eval:
        logger.info("=== OVERALL RESULTS ===")
        _write_results(os.path.join(output_dir, 'result_test.txt'), results)
    else:
        logger.info("=== ENSEMBLE TRAINING COMPLETE ===")

def train_single_model(model: TransformerModelWrapper, train_data: List[InputExample], config: TrainConfig,
                       eval_config: EvalConfig = None, val_data: List[InputExample] = None,
                       ipet_train_data: List[InputExample] = None,
                       unlabeled_data: List[InputExample] = None, return_train_set_results: bool = True):
    """
    Train a single model.

    :param model: the model to train
    :param train_data: the training examples to use
    :param config: the training config
    :param eval_config: the evaluation config
    :param ipet_train_data: an optional list of iPET training examples to use
    :param unlabeled_data: an optional list of unlabeled examples to use
    :param return_train_set_results: whether results on the train set before and after training should be computed and
           returned
    :return: a dictionary containing the global step, average loss and (optionally) results on the train set
    """

    device = torch.device(config.device if config.device else "cuda" if torch.cuda.is_available() else "cpu")

    results_dict = {}

    model.model.to(device)

    if train_data and return_train_set_results:
        logger.info("Evaluation on Train set before training")
        scores = evaluate(model, train_data, eval_config, type_dataset="train")['scores']
        results_dict['train_set_before_training'] = scores['acc']
        logger.info("Scores :")
        logger.info(scores)

    all_train_data = train_data + ipet_train_data

    if not all_train_data and not config.use_logits:
        logger.warning('Training method was called without training examples')
    else:
        global_step, tr_loss = model.train(
            all_train_data, device,
            val_data=val_data,
            per_gpu_train_batch_size=config.per_gpu_train_batch_size,
            per_gpu_unlabeled_batch_size=config.per_gpu_unlabeled_batch_size,
            n_gpu=config.n_gpu,
            num_train_epochs=config.num_train_epochs,
            max_steps=config.max_steps,
            gradient_accumulation_steps=config.gradient_accumulation_steps,
            weight_decay=config.weight_decay,
            learning_rate=config.learning_rate,
            adam_epsilon=config.adam_epsilon,
            warmup_steps=config.warmup_steps,
            max_grad_norm=config.max_grad_norm,
            unlabeled_data=unlabeled_data if config.lm_training or config.use_logits else None,
            lm_training=config.lm_training,
            use_logits=config.use_logits,
            alpha=config.alpha,
            temperature=config.temperature
        )
        results_dict['global_step'] = global_step
        results_dict['average_loss'] = tr_loss

    # if train_data and return_train_set_results:
    #    logger.info("Evaluation on Train set after training")
    #    results_dict['train_set_after_training'] = evaluate(model, train_data, eval_config, type_dataset="train")['scores']['acc']

    return results_dict

def evaluate(model: TransformerModelWrapper, eval_data: List[InputExample], config: EvalConfig,
             type_dataset: str = 'unlabeled', priming_data: List[InputExample] = None) -> Dict:
    """
    Evaluate a model.

    :param model: the model to evaluate
    :param eval_data: the examples for evaluation
    :param config: the evaluation config
    :param priming_data: an optional list of priming data to use
    :param type_dataset: 'train', 'dev', 'unlabeled', 'test'
    :return: a dictionary containing the model's logits, predictions and (if any metrics are given) scores
    """

    if config.priming:
        for example in eval_data:
            example.meta['priming_data'] = priming_data

    metrics = config.metrics if config.metrics else ['acc']
    device = torch.device(config.device if config.device else "cuda" if torch.cuda.is_available() else "cpu")

    model.model.to(device)
    results = model.eval(eval_data, device, per_gpu_eval_batch_size=config.per_gpu_eval_batch_size,
                         n_gpu=config.n_gpu, decoding_strategy=config.decoding_strategy,
                         type_dataset=type_dataset, priming=config.priming)

    predictions = np.argmax(results['logits'], axis=1)
    scores = {}

    for metric in metrics:
        if metric == 'acc':
            scores[metric] = simple_accuracy(predictions, results['labels'])
        elif metric == 'f1':
            scores[metric] = f1_score(results['labels'], predictions)
        elif metric == 'f1-macro':
            scores[metric] = f1_score(results['labels'], predictions, average='macro')
        elif metric == 'f1-weighted':
            scores[metric] = f1_score(results['labels'], predictions, average='weighted')
        elif metric == 'recall':
            scores[metric] = recall_score(results['labels'], predictions)
        elif metric == 'recall-macro':
            scores[metric] = recall_score(results['labels'], predictions, average='macro')
        elif metric == 'recall-weighted':
            scores[metric] = recall_score(results['labels'], predictions, average='weighted')
        elif metric == 'precision':
            scores[metric] = precision_score(results['labels'], predictions)
        elif metric == 'precision-macro':
            scores[metric] = precision_score(results['labels'], predictions, average='macro')
        elif metric == 'precision-weighted':
            scores[metric] = precision_score(results['labels'], predictions, average='weighted')
        elif metric == 'em':
            scores[metric] = exact_match(predictions, results['labels'], results['question_ids'])
        else:
            raise ValueError(f"Metric '{metric}' not implemented")

    results['scores'] = scores
    results['predictions'] = predictions
    return results

def test(output_dir: str, eval_data: List[InputExample], config: EvalConfig, label_map: dict,
         type_dataset: str = 'unlabeled', priming_data: List[InputExample] = None) -> Dict:
    """
    Test a model.

    :param model: the model to evaluate
    :param eval_data: the examples for evaluation
    :param config: the evaluation config
    :param priming_data: an optional list of priming data to use
    :param type_dataset: 'train', 'dev', 'unlabeled', 'test'
    :return: a dictionary containing the model's logits, predictions and (if any metrics are given) scores
    """

    TransformerModelWrapper_output_dir = "{}/final_model".format(output_dir)
    model = TransformerModelWrapper.from_pretrained(TransformerModelWrapper_output_dir)

    if config.priming:
        for example in eval_data:
            example.meta['priming_data'] = priming_data

    device = torch.device(config.device if config.device else "cuda" if torch.cuda.is_available() else "cpu")

    model.model.to(device)
    results = model.eval(eval_data, device, per_gpu_eval_batch_size=config.per_gpu_eval_batch_size,
                         n_gpu=config.n_gpu, decoding_strategy=config.decoding_strategy,
                         type_dataset=type_dataset, priming=config.priming)

    predictions = results['logits']

    logits_dict = {}
    for i in label_map.keys():
        logits_dict[label_map[i]] = predictions[:, int(i)]

    return logits_dict

def _write_results(path: str, results: Dict):
    with open(path, 'w') as fh:
        for metric in results.keys():
            for pattern_id, values in results[metric].items():
                mean = statistics.mean(values)
                stdev = statistics.stdev(values) if len(values) > 1 else 0
                result_str = "{}-p{}: {} +- {}".format(metric, pattern_id, mean, stdev)
                logger.info(result_str)
                fh.write(result_str + '\n')

        for metric in results.keys():
            all_results = [result for pattern_results in results[metric].values() for result in pattern_results]
            all_mean = statistics.mean(all_results)
            all_stdev = statistics.stdev(all_results) if len(all_results) > 1 else 0
            result_str = "{}-all-p: {} +- {}".format(metric, all_mean, all_stdev)
            logger.info(result_str)
            fh.write(result_str + '\n')

max_stars_repo_name: Tarandro/pet

max_issues_repo_name: Tarandro/pet

max_forks_repo_name: Tarandro/pet


text:
# code to be added to mine.py to display live paper/real trading
#
#

import matplotlib.pyplot as plt
import numpy as np

X = np.linspace(0, 2, 1000)
Y = X**2 + np.random.random(X.shape)
go = True

plt.ion()
graph = plt.plot(X, Y)[0]

while go:
    Y = X**2 + np.random.random(X.shape)
    graph.set_ydata(Y)
    plt.draw()
    plt.pause(3)


text:
import os
import glob
import picamera
import cv2
import numpy as np
import importlib.util
from datetime import datetime
import videorecorder as vr
import time
from collections import Counter

# If using TPU, need to load a different library
# from tensorflow.lite.python.interpreter import Interpreter

def take_picture(path):
    if path is None:
        path = "/home/pi/Pictures"
    camera = picamera.PiCamera()
    try:
        camera.capture(os.path.join(path, "image_{0}.jpg".format(datetime.now().strftime('%m%d%Y%H%M%S'))))
    finally:
        print('Picture taken')
        camera.close()

def record_video(path=None, cone_color='green', duration=5, runid=0):
    if path is None:
        path="/home/pi/Videos"
    path = os.path.join(path,cone_color)
    try:
        recorder = vr.VideoRecorder(path,runid)
        print('Loaded Video Recorder')
        recorder.start_recording()
        time.sleep(duration)
        recorder.stop_recording()
    except:
        print('Video Recording failed')
    finally:
        print('Video recorded')

class ObjectClassificationModel:

    def __init__(self, model_dir, image_dir, graph_name='detect.tflite', min_conf_threshold=0.5, use_TPU=False):

        self.model_dir = model_dir
        self.image_dir = image_dir
        self.min_conf_threshold = float(min_conf_threshold)
        self.use_TPU = use_TPU
        self._load_model(model_dir=model_dir, graph_name=graph_name)

    def _load_model(self, model_dir, graph_name):
        CWD_PATH = os.getcwd()

        # Load model labels
        PATH_TO_LABELS = os.path.join(CWD_PATH, model_dir, 'labelmap.txt')
        with open(PATH_TO_LABELS, 'r') as f:
            labels = [line.strip() for line in f.readlines()]
        if labels[0] == '???':
            del(labels[0])
        self.labels = labels

        pkg = importlib.util.find_spec('tensorflow')
        if pkg is None:
            from tflite_runtime.interpreter import Interpreter
            if self.use_TPU:
                print('Loading tflite interpreter')
                from tflite_runtime.interpreter import load_delegate
        else:
            from tensorflow.lite.python.interpreter import Interpreter
            if self.use_TPU:
                print('Loading tflite interpreter')
                from tflite_runtime.interpreter import load_delegate

        # If using Edge TPU, assign filename for Edge TPU model
        if self.use_TPU:
            # If user has specified the name of the .tflite file, use that name, otherwise use default 'edgetpu.tflite'
            if (graph_name == 'detect.tflite'):
                graph_name = 'edgetpu.tflite'

        PATH_TO_CKPT = os.path.join(CWD_PATH, model_dir, graph_name)

        # Load the Tensorflow Lite model.
        # If using Edge TPU, use special load_delegate argument
        if self.use_TPU:
            self.interpreter = Interpreter(model_path=PATH_TO_CKPT,
                                    experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
            print(PATH_TO_CKPT)
        else:
            self.interpreter = Interpreter(model_path=PATH_TO_CKPT)
        self.interpreter.allocate_tensors()

        # Get model details
        self.input_details = self.interpreter.get_input_details()
        self.output_details = self.interpreter.get_output_details()
        self.height = self.input_details[0]['shape'][1]
        self.width = self.input_details[0]['shape'][2]
        self.floating_model = (self.input_details[0]['dtype'] == np.float32)
        self.input_mean = 127.5
        self.input_std = 127.5

    def classify(self, image_dir):
        images = glob.glob(image_dir + '/*')
        classes_list = []
        scores_list = []
        for image_path in images:
            print('Classifying: {}'.format(image_path))

            # Load image and resize to expected shape [1xHxWx3]
            image = cv2.imread(image_path)
            image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
            imH, imW, _ = image.shape
            image_resized = cv2.resize(image_rgb, (self.width, self.height))
            input_data = np.expand_dims(image_resized, axis=0)

            # Normalize pixel values if using a floating model (i.e. if model is non-quantized)
            if self.floating_model:
                input_data = (np.float32(input_data) - self.input_mean) / self.input_std

            # Perform the actual detection by running the model with the image as input
            self.interpreter.set_tensor(self.input_details[0]['index'],input_data)
            self.interpreter.invoke()

            # Retrieve detection results
            # We are not using the boxes right now since we do not need to know
            # where picture the object is, only that it is there.
            # boxes = interpreter.get_tensor(output_details[0]['index'])[0] # Bounding box coordinates of detected objects

            classes = self.interpreter.get_tensor(self.output_details[1]['index'])[0] # Class index of detected objects
            scores = self.interpreter.get_tensor(self.output_details[2]['index'])[0] # Confidence of detected objects
            classes_list.append(classes[scores > self.min_conf_threshold])
            scores_list.append(scores[scores > self.min_conf_threshold])

        objects_detected = {}
        for classes in classes_list:
            objects = set([self.labels[int(c)] for c in classes])
            for obj in objects:
                if obj in objects_detected.keys():
                    objects_detected[obj] += 1
                else:
                    objects_detected[obj] = 1

        return classes_list, scores_list, objects_detected

    def classify_video(self, video_dir):
        """Function to detect objects in video file"""
        #1. Get the list of all video files from the directory passed in
        videos = glob.glob(video_dir + '/*')

        #2. Check the number of *.avi files in the folder
        num_videos = len(videos)
        #3. Do not run classification if number of videos in the folder are more than 10 and alert
        if num_videos > 10:
            print('Found more than 10 videos in the directory: {}'.format(video_dir))
            return
        #4. For each video file
        for video_file in videos:
            video_name=os.path.basename(video_file)
            print('Processing video: {}'.format(video_name))
            #4.1 Open the video file
            video = cv2.VideoCapture(video_file)
            imW = video.get(cv2.CAP_PROP_FRAME_WIDTH)
            imH = video.get(cv2.CAP_PROP_FRAME_HEIGHT)
            collect_labels = []
            #4 .1.1 pass frame by frame to the  detection model
            index = 0
            print('Min Threshold',self.min_conf_threshold)
            while(video.isOpened()):
                # Acquire frame and resize to expected shape [1xHxWx3]
                ret, frame = video.read()
                if frame is None:
                    break
                frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
                frame_resized = cv2.resize(frame_rgb, (self.width, self.height))
                input_data = np.expand_dims(frame_resized, axis=0)

                # Normalize pixel values if using a floating model (i.e. if model is non-quantized)
                if self.floating_model:
                    input_data = (np.float32(input_data) - self.input_mean) / self.input_std

                # Perform the actual detection by running the model with the image as input
                self.interpreter.set_tensor(self.input_details[0]['index'],input_data)
                self.interpreter.invoke()

                # Retrieve detection results
                boxes = self.interpreter.get_tensor(self.output_details[0]['index'])[0] # Bounding box coordinates of detected objects
                classes = self.interpreter.get_tensor(self.output_details[1]['index'])[0] # Class index of detected objects
                scores = self.interpreter.get_tensor(self.output_details[2]['index'])[0] # Confidence of detected objects

                # Loop over all detections and draw detection box if confidence is above minimum threshold
                for i in range(len(scores)):
                    if ((scores[i] > self.min_conf_threshold) and (scores[i] <= 1.0)):

                        # Get bounding box coordinates and draw box
                        # Interpreter can return coordinates that are outside of image dimensions, need to force them to be within image using max() and min()
                        ymin = int(max(1,(boxes[i][0] * imH)))
                        xmin = int(max(1,(boxes[i][1] * imW)))
                        ymax = int(min(imH,(boxes[i][2] * imH)))
                        xmax = int(min(imW,(boxes[i][3] * imW)))

                        cv2.rectangle(frame, (xmin,ymin), (xmax,ymax), (10, 255, 0), 4)

                        # Draw label
                        object_name = self.labels[int(classes[i])] # Look up object name from "labels" array using class index
                        label = '%s: %d%%' % (object_name, int(scores[i]*100)) # Example: 'person: 72%'
                        labelSize, baseLine = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2) # Get font size
                        label_ymin = max(ymin, labelSize[1] + 10) # Make sure not to draw label too close to top of window
                        cv2.rectangle(frame, (xmin, label_ymin-labelSize[1]-10), (xmin+labelSize[0], label_ymin+baseLine-10), (255, 255, 255), cv2.FILLED) # Draw white box to put label text in
                        cv2.putText(frame, label, (xmin, label_ymin-7), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2) # Draw label text
                        collect_labels.append(object_name)
                index += 1
                # All the results have been drawn on the frame, so it's time to display it.
                cv2.imshow('Object detector', frame)
            video.release()
            #cv2.destroyAllWindows()
            if len(collect_labels)>0:
                most_common,num_most_common = Counter(collect_labels).most_common(1)[0]
                max_object = most_common
            else:
                max_object = 'Nothing'
            print('Maximum detected object :{}'.format(max_object))
            print(Counter(collect_labels))
        return max_object

class ConeClassificationModel:
    def __init__(self, model_dir, image_dir, graph_name='cone_detect.tflite', min_conf_threshold=0.5, use_TPU=False):

        self.model_dir = model_dir
        self.image_dir = image_dir
        self.min_conf_threshold = float(min_conf_threshold)
        self.use_TPU = use_TPU
        self._load_model(model_dir=model_dir, graph_name=graph_name)

    def _load_model(self, model_dir, graph_name):
        CWD_PATH = os.getcwd()

        # Load model labels
        PATH_TO_LABELS = os.path.join(CWD_PATH, model_dir, 'labelmap.txt')
        with open(PATH_TO_LABELS, 'r') as f:
            labels = [line.strip() for line in f.readlines()]
        if labels[0] == '???':
            del(labels[0])
        self.labels = labels

        pkg = importlib.util.find_spec('tensorflow')
        if pkg is None:
            from tflite_runtime.interpreter import Interpreter
            if self.use_TPU:
                print('Loading tflite interpreter')
                from tflite_runtime.interpreter import load_delegate
        else:
            from tensorflow.lite.python.interpreter import Interpreter
            if self.use_TPU:
                print('Loading tflite interpreter')
                from tflite_runtime.interpreter import load_delegate

        # If using Edge TPU, assign filename for Edge TPU model
        if self.use_TPU:
            # If user has specified the name of the .tflite file, use that name, otherwise use default 'edgetpu.tflite'
            if (graph_name == 'cone_detect.tflite'):
                graph_name = 'cone_edgetpu.tflite'
        # Load the model
        PATH_TO_CKPT = os.path.join(CWD_PATH, model_dir, graph_name)
        #self.interpreter = Interpreter(model_path=PATH_TO_CKPT)
        # Load the Tensorflow Lite model.
        # If using Edge TPU, use special load_delegate argument
        if self.use_TPU:
            self.interpreter = Interpreter(model_path=PATH_TO_CKPT,
                                    experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
            print(PATH_TO_CKPT)
        else:
            self.interpreter = Interpreter(model_path=PATH_TO_CKPT)
        self.interpreter.allocate_tensors()

        # Get model details
        self.input_details = self.interpreter.get_input_details()
        self.output_details = self.interpreter.get_output_details()
        self.height = self.input_details[0]['shape'][1]
        self.width = self.input_details[0]['shape'][2]
        self.floating_model = (self.input_details[0]['dtype'] == np.float32)
        self.input_mean = 127.5
        self.input_std = 127.5

    def classify(self, image_dir):
        images = glob.glob(image_dir + '/*.jpg')
        classes_list = []
        scores_list = []
        boxes_list =[]
        for image_path in images:
            print('Classifying: {}'.format(image_path))
            # Load image and resize to expected shape [1xHxWx3]
            image = cv2.imread(image_path)
            image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
            imH, imW, _ = image.shape
            image_resized = cv2.resize(image_rgb, (self.width, self.height))
            input_data = np.expand_dims(image_resized, axis=0)

            # Normalize pixel values if using a floating model (i.e. if model is non-quantized)
            if self.floating_model:
                input_data = (np.float32(input_data) - self.input_mean) / self.input_std

            # Perform the actual detection by running the model with the image as input
            self.interpreter.set_tensor(self.input_details[0]['index'],input_data)
            self.interpreter.invoke()

            boxes = self.interpreter.get_tensor(self.output_details[0]['index'])[0] # Bounding box coordinates of detected objects
            classes = self.interpreter.get_tensor(self.output_details[1]['index'])[0] # Class index of detected objects
            scores = self.interpreter.get_tensor(self.output_details[2]['index'])[0] # Confidence of detected objects
            boxes_list.append(boxes[scores > self.min_conf_threshold])
            classes_list.append(classes[scores > self.min_conf_threshold])
            scores_list.append(scores[scores > self.min_conf_threshold])

            # Get bounding box coordinates and draw box
            # Interpreter can return coordinates that are outside of image dimensions, need to force them to be within image using max() and min()
            objects_dict ={}
            for i in range(len(scores)):
                if ((scores[i] > self.min_conf_threshold) and (scores[i] <= 1.0)):

                    ymin = int(max(1,(boxes[i][0] * imH)))
                    xmin = int(max(1,(boxes[i][1] * imW)))
                    ymax = int(min(imH,(boxes[i][2] * imH)))
                    xmax = int(min(imW,(boxes[i][3] * imW)))
                    print((xmin,ymin), (xmax,ymax),(imH,imW))
                    cv2.rectangle(image, (xmin,ymin), (xmax,ymax), (10, 255, 0), 2)

                    # Draw label
                    object_name = self.labels[int(classes[i])] # Look up object name from "labels" array using class index
                    objects_dict[object_name] = [(xmin,ymin), (xmax,ymax),(imH,imW)]
                    label = '%s: %d%%' % (object_name, int(scores[i]*100)) # Example: 'person: 72%'
                    labelSize, baseLine = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2) # Get font size
                    label_ymin = max(ymin, labelSize[1] + 10) # Make sure not to draw label too close to top of window
                    cv2.rectangle(image, (xmin, label_ymin-labelSize[1]-10), (xmin+labelSize[0], label_ymin+baseLine-10), (255, 255, 255), cv2.FILLED) # Draw white box to put label text in
                    cv2.putText(image, label, (xmin, label_ymin-7), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 0), 2) # Draw label text
            classes_list.append(classes[scores > self.min_conf_threshold])
            scores_list.append(scores[scores > self.min_conf_threshold])
        objects_detected = {}
        for classes in classes_list:
            objects = set([self.labels[int(c)] for c in classes])
            for obj in objects:
                if obj in objects_detected.keys():
                    objects_detected[obj] += 1
                else:
                    objects_detected[obj] = 1
        return boxes_list, classes_list, scores_list, objects_detected, objects_dict

if __name__ == '__main__':

    model = ObjectClassificationModel('Sample_TFLite_model', '/home/pi/Pictures/scav_hunt')
    classes, scores, objects = model.classify(os.path.join(model.image_dir, 'archive/orange'))

max_stars_repo_name: scottshepard/MScA-Robotics-Capstone

max_issues_repo_name: scottshepard/MScA-Robotics-Capstone


text:
# -*- coding: utf-8 -*-
"""
Created on Sun Jul 24 19:36:30 2016

@author: hochthom
"""

import time
import numpy as np
from scipy.interpolate import interp1d
from keras.models import Sequential
from keras.layers import Dense, Activation
from keras import initializations
from PIL import Image

def my_init(shape, name=None):
    return initializations.normal(shape, scale=1.2, name=name)

def create_grid(x_dim=32, y_dim=32, scale=1.0):
    '''
    returns a vector of x and y coordinates and th ecorresponding radius from
    the centre of the image.
    '''
    N = 0.5*(x_dim + y_dim)
    x1 = np.linspace(-1.0*x_dim/N*scale, 1.0*x_dim/N*scale, x_dim)
    y1 = np.linspace(-1.0*y_dim/N*scale, 1.0*y_dim/N*scale, y_dim)
    X, Y = np.meshgrid(x1, y1)
    x1 = np.ravel(X).reshape(-1, 1)
    y1 = np.ravel(Y).reshape(-1, 1)
    r1 = np.sqrt(x1**2 + y1**2)
    return x1, y1, r1

def interpolate_z(z, n_frames=25, mode=None):
    '''
    Interpolate movement through latent space with spline approximation.
    '''
    x_max = float(z.shape[0])
    if mode is not None:
        x_max += 1
        if 'smooth' in mode:
            x_max += 2

    xx = np.arange(0, x_max)
    zt = []
    for k in range(z.shape[1]):
        yy = list(z[:,k])
        if mode is not None:
            yy.append(z[0,k])
            if 'smooth' in mode:
                yy = [z[-1,k]] + yy + [z[1,k]]
        fz = interp1d(xx, yy, kind='cubic')
        if 'smooth' in mode:
            x_new = np.linspace(1, x_max-2, num=n_frames, endpoint=False)
        else:
            x_new = np.linspace(0, x_max-1, num=n_frames, endpoint=False)
        zt.append(fz(x_new))

    return np.column_stack(zt)

def create_image(model, x, y, r, z):
    '''
    create an image for the given latent vector z
    '''
    # create input vector
    Z = np.repeat(z, x.shape[0]).reshape((-1,x.shape[0]))
    X = np.concatenate([x, y, r, Z.T], axis=1)

    pred = model.predict(X)

    img = []
    for k in range(pred.shape[1]):
        yp = pred[:, k]
#        if k == pred.shape[1]-1:
#            yp = np.sin(yp)
        yp = (yp - yp.min()) / (yp.max()-yp.min())
        img.append(yp.reshape(y_dim, x_dim))

    img = np.dstack(img)
    if img.shape[-1] == 3:
        from skimage.color import hsv2rgb
        img = hsv2rgb(img)

    return (img*255).astype(np.uint8)

def create_image_seq(model, x, y, r, z, n_frames=25, mode=None):
    '''
    create a list of images with n_frames between the given latent vectors in z
    '''
    # create all z values
    zt = interpolate_z(z, n_frames, mode)

    images = []
    for k in range(zt.shape[0]):
        print 'Creating image %3i/%i ...' % (k+1, zt.shape[0])
        images.append(create_image(model,x,y,r,zt[k,:]))

    return images

SINGLE_IMAGE = False
COLORING = True

n_z = 16
x_dim = 720
y_dim = 720
x, y, r = create_grid(x_dim=x_dim, y_dim=y_dim, scale=5.0)
# create latent space
z = np.random.normal(0, 1, (3,n_z))

# create neural network
n_l = 32
model = Sequential([
    Dense(n_l, init=my_init, input_dim=n_z+3),
    Activation('tanh'),
    Dense(n_l, init=my_init),
    Activation('tanh'),
    Dense(n_l, init=my_init),
    Activation('tanh'),
    Dense(3 if COLORING else 1),
    #Activation('sigmoid'),
    Activation('linear'),
])

model.compile(optimizer='rmsprop', loss='mse')

# create images
t_start = time.time()

if SINGLE_IMAGE:
    img = create_image(model, x, y, r, z[0,:])
    im = Image.fromarray(img)
    im.save('test.png')
else:
    img_seq = create_image_seq(model, x, y, r, z, n_frames=100, mode='smooth')
    for k, img in enumerate(img_seq):
        im = Image.fromarray(img)
        im.save('seq/img_%03i.png' % k)

    if True:
        from moviepy.editor import *
        if COLORING:
            tmp = img_seq
        else:
            tmp = [np.dstack(3*[img]).astype("uint8") for img in img_seq]
        clip = ImageSequenceClip(tmp, fps=25)
        clip.write_videofile("video.avi", fps=25, codec='libx264')

t_stop = time.time()
print 'done in %.1f seconds.' % (t_stop-t_start)

max_stars_repo_name: hochthom/cppn-keras

max_issues_repo_name: hochthom/cppn-keras

max_forks_repo_name: hochthom/cppn-keras


text:
import re
import torch
import torch.nn as nn
import torchvision.transforms as transforms
from argparse import ArgumentParser
import pytorch_lightning as pl
from .lsegmentation_module import LSegmentationModule
from .models.lseg_net import LSegNet
from encoding.models.sseg.base import up_kwargs

import os
import clip
import numpy as np

from scipy import signal
import glob

from PIL import Image
import matplotlib.pyplot as plt
import pandas as pd

class LSegModule(LSegmentationModule):
    def __init__(self, data_path, dataset, batch_size, base_lr, max_epochs, **kwargs):
        super(LSegModule, self).__init__(
            data_path, dataset, batch_size, base_lr, max_epochs, **kwargs
        )

        if dataset == "citys":
            self.base_size = 2048
            self.crop_size = 768
        else:
            self.base_size = 520
            self.crop_size = 480

        use_pretrained = True
        norm_mean= [0.5, 0.5, 0.5]
        norm_std = [0.5, 0.5, 0.5]

        print('** Use norm {}, {} as the mean and std **'.format(norm_mean, norm_std))

        train_transform = [
            transforms.ToTensor(),
            transforms.Normalize(norm_mean, norm_std),
        ]

        val_transform = [
            transforms.ToTensor(),
            transforms.Normalize(norm_mean, norm_std),
        ]

        self.train_transform = transforms.Compose(train_transform)
        self.val_transform = transforms.Compose(val_transform)

        self.trainset = self.get_trainset(
            dataset,
            augment=kwargs["augment"],
            base_size=self.base_size,
            crop_size=self.crop_size,
        )

        self.valset = self.get_valset(
            dataset,
            augment=kwargs["augment"],
            base_size=self.base_size,
            crop_size=self.crop_size,
        )

        use_batchnorm = (
            (not kwargs["no_batchnorm"]) if "no_batchnorm" in kwargs else True
        )
        # print(kwargs)

        labels = self.get_labels('ade20k')

        self.net = LSegNet(
            labels=labels,
            backbone=kwargs["backbone"],
            features=kwargs["num_features"],
            crop_size=self.crop_size,
            arch_option=kwargs["arch_option"],
            block_depth=kwargs["block_depth"],
            activation=kwargs["activation"],
        )

        self.net.pretrained.model.patch_embed.img_size = (
            self.crop_size,
            self.crop_size,
        )

        self._up_kwargs = up_kwargs
        self.mean = norm_mean
        self.std = norm_std

        self.criterion = self.get_criterion(**kwargs)

    def get_labels(self, dataset):
        labels = []
        path = 'label_files/{}_objectInfo150.txt'.format(dataset)
        assert os.path.exists(path), '*** Error : {} not exist !!!'.format(path)
        f = open(path, 'r')
        lines = f.readlines()
        for line in lines:
            label = line.strip().split(',')[-1].split(';')[0]
            labels.append(label)
        f.close()
        if dataset in ['ade20k']:
            labels = labels[1:]
        return labels

    @staticmethod
    def add_model_specific_args(parent_parser):
        parser = LSegmentationModule.add_model_specific_args(parent_parser)
        parser = ArgumentParser(parents=[parser])

        parser.add_argument(
            "--backbone",
            type=str,
            default="clip_vitl16_384",
            help="backbone network",
        )

        parser.add_argument(
            "--num_features",
            type=int,
            default=256,
            help="number of featurs that go from encoder to decoder",
        )

        parser.add_argument("--dropout", type=float, default=0.1, help="dropout rate")

        parser.add_argument(
            "--finetune_weights", type=str, help="load weights to finetune from"
        )

        parser.add_argument(
            "--no-scaleinv",
            default=True,
            action="store_false",
            help="turn off scaleinv layers",
        )

        parser.add_argument(
            "--no-batchnorm",
            default=False,
            action="store_true",
            help="turn off batchnorm",
        )

        parser.add_argument(
            "--widehead", default=False, action="store_true", help="wider output head"
        )

        parser.add_argument(
            "--widehead_hr",
            default=False,
            action="store_true",
            help="wider output head",
        )

        parser.add_argument(
            "--arch_option",
            type=int,
            default=0,
            help="which kind of architecture to be used",
        )

        parser.add_argument(
            "--block_depth",
            type=int,
            default=0,
            help="how many blocks should be used",
        )

        parser.add_argument(
            "--activation",
            choices=['lrelu', 'tanh'],
            default="lrelu",
            help="use which activation to activate the block",
        )

        return parser


text:
import pandas as pd
import numpy as np

df = pd.read_csv(r'C:\Users\Gabriel Paladini\Desktop\projetos\Breast cancer\data.csv')
del df['id']
del df['Unnamed: 32']
x = df.drop('diagnosis', axis=1)
y = df['diagnosis']
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.4)
from sklearn.ensemble import RandomForestClassifier
rfc = RandomForestClassifier(n_estimators=1000)
rfc.fit(x_train, y_train)
rfc_prd = rfc.predict(x_test)

#Interactive way

a1 = int(input('If you want to predict digit 1, if you want to see some metrcis digit 2 '))

if a1 == 1:
    print('Say more about the data: ')
    rm = int(input('radius mean: 	'))
    tm = int(input('texture mean:   '))
    pm = int(input('perimeter mean: '))
    am = int(input('area mean:  '))
    sm = int(input('smoothness mean:    '))
    cm = int(input('compactness mean:   '))
    cm1 = int(input('concavity mean: '))
    cm2 = int(input('concave points mean: '))
    tw = int(input('texture worst:  '))
    pw = int(input('perimeter worst: '))
    aw = int(input('area worst: '))
    sm2 = int(input('smoothness worst:	'))
    cw = int(input('compactness worst: '))
    cw2 = int(input('concavity worst:   '))
    cpw = int(input('concave points worst:	'))
    syw = int(input('symmetry worst: '))
    fdw = int(input('fractal dimension worst: '))

    p1 = rfc.predict([[rm, tm, pm, am, sm, cm, cm1, cm2, tw, pw, aw, sm2, cw, cw2, cpw, syw, fdw]])
    print('The diagnosis is {}'.format(p1))
elif a1 == 2:
    from sklearn.metrics import classification_report, confusion_matrix

    print('Some metrics: \n')
    print(classification_report(y_test, rfc_prd))
    print('\n')
    print(confusion_matrix(y_test, rfc_prd))

max_stars_repo_name: gabrielrpaladini/BreastCancerDiagnosis

max_issues_repo_name: gabrielrpaladini/BreastCancerDiagnosis

max_forks_repo_name: gabrielrpaladini/BreastCancerDiagnosis