text:
# Copied from https://gist.github.com/karpathy/d4dee566867f8291f086, by pavelkomarov on 17 Sep 2016
# Ref: The Unreasonable Effectiveness of Recurrent Neural Networks; http://karpathy.github.io/2015/05/21/rnn-effectiveness/
# Ref: https://blog.csdn.net/u014380165/article/details/77284921
# Ref: https://www.cnblogs.com/pinard/p/6509630.html

#
# Author: Rock Zhuang
# Date  : Dec 20, 2018
#

#implemented as I read Andrej Karpathy's post on RNNs.
import numpy as np
import matplotlib.pyplot as plt

class RNN(object):

    def __init__(self, insize, outsize, hidsize, learning_rate):
        self.insize = insize
        self.hidsize = hidsize

        self.h = np.zeros((self.hidsize , 1))#a [h x 1] hidden state stored from last batch of inputs

        #parameters
        self.W_xh = np.random.randn(hidsize, insize)*0.01#[h x x]
        self.W_hh = np.random.randn(hidsize, hidsize)*0.01#[h x h]
        self.W_hy = np.random.randn(outsize, hidsize)*0.01#[y x h]
        self.b_h = np.zeros((hidsize, 1))#biases
        self.b_y = np.zeros((outsize, 1))

        #the Adagrad gradient update relies upon having a memory of the sum of squares of dparams
        self.adaW_hh = np.zeros((hidsize, hidsize))
        self.adaW_xh = np.zeros((hidsize, insize))
        self.adaW_hy = np.zeros((outsize, hidsize))
        self.adab_h = np.zeros((hidsize, 1))
        self.adab_y = np.zeros((outsize, 1))

        self.learning_rate = learning_rate

    def reset_h(self):
        self.h = np.zeros((self.hidsize , 1))#a [h x 1] hidden state stored from last batch of inputs

    #give the RNN a sequence of inputs and outputs (seq_length long), and use
    #them to adjust the internal state
    def train(self, x, y):
        #=====initialize=====
        xhat = {}#holds 1-of-k representations of x
        yhat = {}#holds 1-of-k representations of predicted y (unnormalized log probs)
        p = {}#the normalized probabilities of each output through time
        h = {}#holds state vectors through time
        h[-1] = np.copy(self.h)#we will need to access the previous state to calculate the current state

        dW_xh = np.zeros_like(self.W_xh)
        dW_hh = np.zeros_like(self.W_hh)
        dW_hy = np.zeros_like(self.W_hy)
        db_h = np.zeros_like(self.b_h)
        db_y = np.zeros_like(self.b_y)
        dh_next = np.zeros_like(self.h)

        # print 'train length of x is %d' % (len(x))

        #=====forward pass=====
        loss = 0
        for t in range(len(x)):
            xhat[t] = np.zeros((self.insize, 1))
            xhat[t][x[t]] = 1#xhat[t] = 1-of-k representation of x[t]

            h[t] = np.tanh(np.dot(self.W_xh, xhat[t]) + np.dot(self.W_hh, h[t-1]) + self.b_h)#find new hidden state
            # print '------------------------------------------x'
            # print (xhat[t])
            # print '------------------------------------------self.W_xh'
            # print (self.W_xh)
            # print '------------------------------------------self.W_hh'
            # print (self.W_hh)
            # print '------------------------------------------self.b_h'
            # print (self.b_h)
            # print '------------------------------------------h_pref'
            # print (h[t - 1])
            # print '------------------------------------------h1'
            # print (np.dot(self.W_xh, xhat[t]) + np.dot(self.W_hh, h[t-1]) + self.b_h)
            # print '------------------------------------------h'
            # print (h[t])
            yhat[t] = np.dot(self.W_hy, h[t]) + self.b_y#find unnormalized log probabilities for next chars

            p[t] = np.exp(yhat[t]) / np.sum(np.exp(yhat[t]))#find probabilities for next chars

            # print '------------------------------------------yhat'
            # print (yhat[t])
            # print '------------------------------------------p'
            # print p[t]

            loss += -np.log(p[t][y[t],0])#softmax (cross-entropy loss)

            # print '------------------------------------------py'
            # print p[t][y[t],0]
            # print '------------------------------------------loss'
            # print loss

        #=====backward pass: compute gradients going backwards=====
        for t in reversed(range(len(x))):
            #backprop into y. see http://cs231n.github.io/neural-networks-case-study/#grad if confused here
            dy = np.copy(p[t])
            dy[y[t]] -= 1

            #find updates for y
            dW_hy += np.dot(dy, h[t].T)
            db_y += dy

            #backprop into h and through tanh nonlinearity
            dh = np.dot(self.W_hy.T, dy) + dh_next
            dh_raw = (1 - h[t]**2) * dh

            #find updates for h
            dW_xh += np.dot(dh_raw, xhat[t].T)
            dW_hh += np.dot(dh_raw, h[t-1].T)
            db_h += dh_raw

            # print '------------------------------------------xhat[t]'
            # print (xhat[t])
            # print '------------------------------------------dh_raw'
            # print (dh_raw)
            # print '------------------------------------------dW_xh'
            # print (dW_xh)

            #save dh_next for subsequent iteration
            dh_next = np.dot(self.W_hh.T, dh_raw)

            # print '------------------------------------------dy'
            # print dy
            # print '------------------------------------------dh_raw'
            # print dh_raw
            # print '------------------------------------------dh_next'
            # print dh_next

        for dparam in [dW_xh, dW_hh, dW_hy, db_h, db_y]:
            np.clip(dparam, -5, 5, out=dparam)#clip to mitigate exploding gradients

        #update RNN parameters according to Adagrad
        for param, dparam, adaparam in zip([self.W_hh, self.W_xh, self.W_hy, self.b_h, self.b_y], \
                                [dW_hh, dW_xh, dW_hy, db_h, db_y], \
                                [self.adaW_hh, self.adaW_xh, self.adaW_hy, self.adab_h, self.adab_y]):
            adaparam += dparam*dparam
            param += -self.learning_rate*dparam/np.sqrt(adaparam+1e-8)

        # print '------------------------------------------self.W_xh'
        # print (self.W_xh)

        self.h = h[len(x)-1]

        return loss

    #let the RNN generate text
    def sample(self, seed, n):
        ndxs = []
        h = self.h

        xhat = np.zeros((self.insize, 1))
        xhat[seed] = 1#transform to 1-of-k

        for t in range(n):
            h = np.tanh(np.dot(self.W_xh, xhat) + np.dot(self.W_hh, h) + self.b_h)#update the state
            y = np.dot(self.W_hy, h) + self.b_y
            p = np.exp(y) / np.sum(np.exp(y))
            ndx = np.random.choice(range(self.insize), p=p.ravel())

            xhat = np.zeros((self.insize, 1))
            xhat[ndx] = 1

            ndxs.append(ndx)

        return ndxs

def test2():
    a = [[1, 2, 3], [6, 5, 4]]
    b = [[1, 2], [4, 5], [5, 6]]
    c = np.dot(a, b)
    print 'test begin-------------------------------'
    print c

    h = np.zeros((2, 1))
    h[0][0] = 2
    h[1][0] = 3
    # h[0][1] = 3
    # h[1][1] = 4
    h2 = h**2
    print 'test h2-------------------------------'
    print h2

    h3 = 1 - h2
    print 'test h3-------------------------------'
    print h3

    dy = np.zeros((2, 1))
    dy[0][0] = 3
    dy[1][0] = 4
    h4 = h3 * dy
    print 'test h4-------------------------------'
    print h4

    #open a text file
    data = open('data/input.txt', 'r').read() # should be simple plain text file
    for i in range(len(data)/10):
        print i

    print 'test end-------------------------------'

def test():
    #open a text file
    data = open('data/input.txt', 'r').read() # should be simple plain text file
    chars = list(set(data))
    data_size, vocab_size = len(data), len(chars)
    print 'data has %d characters, %d unique.' % (data_size, vocab_size)

    #make some dictionaries for encoding and decoding from 1-of-k
    char_to_ix = { ch:i for i,ch in enumerate(chars) }
    ix_to_char = { i:ch for i,ch in enumerate(chars) }

    # print '---------------------------ix_to_char'
    # print ix_to_char

    #insize and outsize are len(chars). hidsize is 15. seq_length is 10. learning_rate is 0.1.
    rnn = RNN(vocab_size, vocab_size, 15, 0.1)

    #iterate over batches of input and target output
    seq_length = 10
    losses = []
    smooth_loss = -np.log(1.0/len(chars))*seq_length#loss at iteration 0
    losses.append(smooth_loss)

    g_i = 0

    while g_i < 10000:
        rnn.reset_h()

        for i in range(len(data)/seq_length):
            x = [char_to_ix[c] for c in data[i*seq_length:(i+1)*seq_length]]#inputs to the RNN
            y = [char_to_ix[c] for c in data[i*seq_length+1:(i+1)*seq_length+1]]#the targets it should be outputting
            # print '---------------------------x'
            # print data[i*seq_length:(i+1)*seq_length]
            # print '---------------------------y'
            # print data[i*seq_length+1:(i+1)*seq_length+1]

            if g_i%100==0:
                sample_ix = rnn.sample(x[0], 200)
                txt = ''.join([ix_to_char[n] for n in sample_ix])
                print txt

            loss = rnn.train(x, y)
            smooth_loss = smooth_loss*0.999 + loss*0.001

            if g_i%100==0:
                print 'iteration %d, smooth_loss = %f, loss = %f' % (g_i, smooth_loss, loss)
                losses.append(smooth_loss)

            g_i += 1

    plt.plot(range(len(losses)), losses, 'b', label='smooth loss')
    plt.xlabel('time in thousands of iterations')
    plt.ylabel('loss')
    plt.legend()
    plt.show()

if __name__ == "__main__":
    test2()
    test()


text:
from evaluation.generativity_metrics.shared_api import mean_fn, sigma_fn, bootstrapping_additive, slerp
import numpy as np
import time
epsilon=1e-4
from tensorflow.keras.applications import VGG16
import dask.delayed as delayed

def batch_pairwise_distances(U, V):
    """ Compute pairwise distances between two batches of feature vectors."""
    # Squared norms of each row in U and V.
    norm_u = np.sum(np.square(U), 1)
    norm_v = np.sum(np.square(V), 1)

    # norm_u as a row and norm_v as a column vectors.
    norm_u = np.reshape(norm_u, [-1, 1])
    norm_v = np.reshape(norm_v, [1, -1])

    # Pairwise squared Euclidean distances.
    D = np.maximum(norm_u - 2*np.matmul(U, V.numpy().T) + norm_v, 0.0)

    return D

class ManifoldEstimator():
    """Finds an estimate for the manifold of given feature vectors."""
    def __init__(self, features, row_batch_size, col_batch_size, nhood_sizes, clamp_to_percentile=None):
        """Find an estimate of the manifold of given feature vectors."""
        num_images = features.shape[0]
        self.nhood_sizes = nhood_sizes
        self.num_nhoods = len(nhood_sizes)
        self.row_batch_size = row_batch_size
        self.col_batch_size = col_batch_size
        self._ref_features = features

        # Estimate manifold of features by calculating distances to kth nearest neighbor of each sample.
        self.D = np.zeros([num_images, self.num_nhoods], dtype=np.float16)
        distance_batch = np.zeros([row_batch_size, num_images], dtype=np.float16)
        seq = np.arange(max(self.nhood_sizes) + 1, dtype=np.int32)

        for begin1 in range(0, num_images, row_batch_size):
            end1 = min(begin1 + row_batch_size, num_images)
            row_batch = features[begin1:end1]

            for begin2 in range(0, num_images, col_batch_size):
                end2 = min(begin2 + col_batch_size, num_images)
                col_batch = features[begin2:end2]

                # Compute distances between batches.
                distance_batch[0:end1-begin1, begin2:end2] = batch_pairwise_distances(row_batch, col_batch)

            # Find the kth nearest neighbor from the current batch.
            self.D[begin1:end1, :] = np.partition(distance_batch[0:end1-begin1, :], seq, axis=1)[:, self.nhood_sizes]

        if clamp_to_percentile is not None:
            max_distances = np.percentile(self.D, clamp_to_percentile, axis=0)
            self.D[self.D > max_distances] = 0  #max_distances  # 0

    def evaluate(self, eval_features, return_realism=False, return_neighbors=False):
        """Evaluate if new feature vectors are in the estimated manifold."""
        num_eval_images = eval_features.shape[0]
        num_ref_images = self.D.shape[0]
        distance_batch = np.zeros([self.row_batch_size, num_ref_images], dtype=np.float16)
        batch_predictions = np.zeros([num_eval_images, self.num_nhoods], dtype=np.int32)

        realism_score = np.zeros([num_eval_images,], dtype=np.float32)
        nearest_indices = np.zeros([num_eval_images,], dtype=np.int32)

        for begin1 in range(0, num_eval_images, self.row_batch_size):
            end1 = min(begin1 + self.row_batch_size, num_eval_images)
            feature_batch = eval_features[begin1:end1]

            for begin2 in range(0, num_ref_images, self.col_batch_size):
                end2 = min(begin2 + self.col_batch_size, num_ref_images)
                ref_batch = self._ref_features[begin2:end2]

                distance_batch[0:end1-begin1, begin2:end2] = batch_pairwise_distances(feature_batch, ref_batch)

            # From the minibatch of new feature vectors, determine if they are in the estimated manifold.
            # If a feature vector is inside a hypersphere of some reference sample, then the new sample lies on the estimated manifold.
            # The radii of the hyperspheres are determined from distances of neighborhood size k.
            samples_in_manifold = distance_batch[0:end1-begin1, :, None] <= self.D
            batch_predictions[begin1:end1] = np.any(samples_in_manifold, axis=1).astype(np.int32)

            nearest_indices[begin1:end1] = np.argmin(distance_batch[0:end1-begin1, :], axis=1)
            realism_score[begin1:end1] = self.D[nearest_indices[begin1:end1], 0] / np.min(distance_batch[0:end1-begin1, :], axis=1)

        if return_realism and return_neighbors:
            return batch_predictions, realism_score, nearest_indices
        elif return_realism:
            return batch_predictions, realism_score
        elif return_neighbors:
            return batch_predictions, nearest_indices

        return batch_predictions

def knn_precision_recall_features(ref_features, eval_features, feature_net, nhood_sizes, row_batch_size, col_batch_size):
    """Calculates k-NN precision and recall for two sets of feature vectors."""
    state = dict()
    num_images = ref_features.shape[0]
    num_features = feature_net.output_shape[1]
    state['ref_features'] = ref_features
    state['eval_features'] = eval_features

    # Initialize DistanceBlock and ManifoldEstimators.
    state['ref_manifold'] = ManifoldEstimator(state['ref_features'], row_batch_size, col_batch_size, nhood_sizes)
    state['eval_manifold'] = ManifoldEstimator(state['eval_features'], row_batch_size, col_batch_size, nhood_sizes)

    # Evaluate precision and recall using k-nearest neighbors.
    print('Evaluating k-NN precision and recall with %i samples...' % num_images)
    start = time.time()

    # Precision: How many points from eval_features are in ref_features manifold.
    state['precision'], state['realism_scores'], state['nearest_neighbors'] = state['ref_manifold'].evaluate(state['eval_features'], return_realism=True, return_neighbors=True)
    state['knn_precision'] = state['precision'].mean(axis=0)

    # Recall: How many points from ref_features are in eval_features manifold.
    state['recall'] = state['eval_manifold'].evaluate(state['ref_features'])
    state['knn_recall'] = state['recall'].mean(axis=0)

    elapsed_time = time.time() - start
    print('Done evaluation in: %gs' % elapsed_time)

    return state

def precision_score(model, data_generator, nhood_size=3, row_batch_size=10000, col_batch_size=10000,
                    tolerance_threshold=1e-6, max_iteration=200):
    # prepare the inception v3 model
    VGG16_model = VGG16(include_top=False, pooling='avg', input_shape=model.get_inputs_shape())
    VGG16_featues_fn = lambda x: VGG16_model(x)

    # prepare the ae model random_images_generator
    def model_random_images_generator():
        while True:
            data = next(data_generator)
            ref_features = VGG16_featues_fn(data)

            # Generate latents from the data
            latents_real = model.encode(data)

            # Generate random latents and interpolation t-values.
            latents_t = np.random.normal(size=latents_real.shape)
            lerp_t = np.random.uniform(size=1)[0]

            latents_e = slerp(lerp_t+epsilon, latents_real, latents_t)
            images = model.decode(latents_e).numpy()
            images = (images * 255).astype(np.float32)

            eval_features = VGG16_featues_fn(images)

            # Calculate precision and recall.
            state = knn_precision_recall_features(ref_features=ref_features, eval_features=eval_features,
                                                  feature_net=VGG16_model,
                                                  nhood_sizes=[nhood_size], row_batch_size=row_batch_size,
                                                  col_batch_size=col_batch_size)

            knn_precision = state['knn_precision'][0]
            knn_recall = state['knn_recall'][0]
            yield knn_precision

    knn_precision_mean = bootstrapping_additive(
        data_generator=model_random_images_generator(), func=delayed(lambda x: x), \
        stopping_func=mean_fn, tolerance_threshold=tolerance_threshold, max_iteration=max_iteration
    )

    return knn_precision_mean.compute()

def recall_score(model, data_generator, nhood_size=3, row_batch_size=10000, col_batch_size=10000,
                    tolerance_threshold=1e-6, max_iteration=200):
    # prepare the inception v3 model
    VGG16_model = VGG16(include_top=False, pooling='avg', input_shape=model.get_inputs_shape())
    VGG16_featues_fn = lambda x: VGG16_model(x)

    # prepare the ae model random_images_generator
    def model_random_images_generator():
        while True:
            data = next(data_generator)
            ref_features = VGG16_featues_fn(data)

            # Generate latents from the data
            latents_real = model.encode(data)

            # Generate random latents and interpolation t-values.
            latents_t = np.random.normal(size=latents_real.shape)
            lerp_t = np.random.uniform(size=1)[0]

            latents_e = slerp(lerp_t+epsilon, latents_real, latents_t)
            images = model.decode(latents_e).numpy()
            images = (images * 255).astype(np.float32)

            eval_features = VGG16_featues_fn(images)

            # Calculate precision and recall.
            state = knn_precision_recall_features(ref_features=ref_features, eval_features=eval_features,
                                                  feature_net=VGG16_model,
                                                  nhood_sizes=[nhood_size], row_batch_size=row_batch_size,
                                                  col_batch_size=col_batch_size)

            knn_precision = state['knn_precision'][0]
            knn_recall = state['knn_recall'][0]
            yield knn_recall

    knn_recall_mean = bootstrapping_additive(
        data_generator=model_random_images_generator(), func=delayed(lambda x: x), \
        stopping_func=mean_fn, tolerance_threshold=tolerance_threshold, max_iteration=max_iteration
    )

    return knn_recall_mean.compute()

ext: py

reason: import numpy


text:
"""
Here, we ensure that contrastive loss
"""

import numpy as np
import torch
import torch.nn as nn

def nc2(n):
    return int(n * (n - 1) / 2)

def get_similarity_matrix(noise_levels):
    """
    noise_levels: N*1
    Returns:
        N*N boolean matrix X. X[i,j] is true iff noise_levels[i] == noise_levels[j]. For digonal elements, it has 0
    """
    batch_size = len(noise_levels)
    noise_levels = noise_levels.view(batch_size, -1)
    b = noise_levels[:, :, None].repeat(1, 1, batch_size)
    c = torch.swapaxes(b, 0, 2)
    sim_matrix = b == c
    assert sim_matrix.shape == (batch_size, 1, batch_size)
    sim_matrix = sim_matrix[:, 0, :]
    sim_matrix[np.arange(batch_size), np.arange(batch_size)] = 0
    return sim_matrix

def get_contrastive_loss_vectorized(representation, noise_levels, tau_pos=0.1, tau_neg=0.1):

    # compute pairwise MSE between all representations.
    batch_size = len(representation)
    representation = representation.view(batch_size, -1)
    b = representation[:, :, None].repeat(1, 1, batch_size)
    c = torch.swapaxes(b, 0, 2)
    pairwise_mse = torch.sum((b - c)**2, dim=1) / b.shape[1]

    # clamp mse values
    pairwise_sim = get_similarity_matrix(noise_levels)
    pairwise_mse[pairwise_sim] = torch.clamp(pairwise_mse[pairwise_sim] - tau_pos, min=0)
    pairwise_mse[~pairwise_sim] = torch.clamp(tau_neg - pairwise_mse[~pairwise_sim], min=0)
    pairwise_mse[np.arange(batch_size), np.arange(batch_size)] = 0

    # scale mse values
    c_pos = torch.sum(pairwise_sim) / 2
    c_neg = nc2(batch_size) - c_pos
    pairwise_mse[pairwise_sim] = pairwise_mse[pairwise_sim] * 1 / c_pos
    pairwise_mse[~pairwise_sim] = pairwise_mse[~pairwise_sim] * 1 / c_neg
    return pairwise_mse.sum() / 2

class ContrastiveLearninglossOnLatent(nn.Module):
    def __init__(self, latent_size_dict) -> None:
        super().__init__()
        self._lsizes = latent_size_dict

        self._tau_pos = 0.1
        self._tau_neg = 0.1

    def get_loss(self, latent_activations, noise_levels):
        latent_loss_dict = {}
        seen_loss_sz = []
        for one_layer_activation in latent_activations:
            sz = one_layer_activation.shape[-1]

            assert sz not in seen_loss_sz
            seen_loss_sz.append(sz)

            if self._lsizes[sz] == 0:
                continue
            N = self._lsizes[sz]
            noise_z = one_layer_activation[:, :N]
            latent_loss_dict[sz] = get_contrastive_loss_vectorized(noise_z,
                                                                   noise_levels,
                                                                   tau_pos=0,
                                                                   tau_neg=self._tau_neg)

        return latent_loss_dict

    def forward(self, latent_activations, noise_levels):
        return self.get_loss(latent_activations, noise_levels)

class DisentanglementModule(nn.Module):
    def __init__(self, latent_size_dict, image_model, cl_latent_weight=None) -> None:
        super().__init__()
        self._lsizes = latent_size_dict
        self._image_model = image_model
        for param in self._image_model.parameters():
            param.requires_grad = False

        self._tau_pos = 0.1
        self._tau_neg = 0.1
        assert cl_latent_weight is not None and cl_latent_weight <= 1 and cl_latent_weight >= 0
        self._w = cl_latent_weight

    def get_loss(self, activations, pred_image, noise_levels):
        latent_loss = 0
        cnt = 0
        for key in activations:
            if self._lsizes[key] == 0:
                continue
            N = self._lsizes[key]
            cnt += 1
            noise_z = activations[key][:, :N]
            latent_loss += get_contrastive_loss_vectorized(noise_z, noise_levels, tau_pos=0, tau_neg=self._tau_neg)

        latent_loss = latent_loss / cnt

        if self._w < 1:
            representation = self._image_model(pred_image)
            output_contrastive_loss = get_contrastive_loss_vectorized(representation,
                                                                      noise_levels,
                                                                      tau_pos=self._tau_pos,
                                                                      tau_neg=self._tau_neg)

            return self._w * latent_loss + (1 - self._w) * output_contrastive_loss
        return latent_loss

    def forward(self, activations, pred_image, noise_levels):
        return self.get_loss(activations, pred_image, noise_levels)

if __name__ == '__main__':
    rep = torch.Tensor(np.arange(40).reshape(8, 5))
    noise_level_list = torch.Tensor([10, 20, 10, 40, 30, 40, 20, 30])
    loss = get_contrastive_loss_vectorized(rep, noise_level_list).item()
    assert abs(loss - 262.4) < 1e-4, f'{loss - 262.4}'


text:
# encoding: utf-8
"""
@author:  liaoxingyu
@contact: liaoxingyu2@jd.com
"""

import math
import random

from skimage.draw import polygon
import numpy as np
import torch

class RandomErasing(object):
    """ Randomly selects a rectangle region in an image and erases its pixels.
        'Random Erasing Data Augmentation' by Zhong et al.
        See https://arxiv.org/pdf/1708.04896.pdf
    Args:
         probability: The probability that the Random Erasing operation will be performed.
         sl: Minimum proportion of erased area against input image.
         sh: Maximum proportion of erased area against input image.
         r1: Minimum aspect ratio of erased area.
         mean: Erasing value.
    """

    def __init__(self, probability=0.5, sl=0.02, sh=0.4, r1=0.3, mean=(0.4914, 0.4822, 0.4465),std =  (0.229, 0.224, 0.225)):
        self.probability = probability
        self.mean = mean
        self.std = std
        self.sl = sl
        self.sh = sh
        self.r1 = r1

    def __call__(self, img):

        if random.uniform(0, 1) >= self.probability:
            return img

        for attempt in range(100):
            area = img.size()[1] * img.size()[2]

            target_area = random.uniform(self.sl, self.sh) * area
            aspect_ratio = random.uniform(self.r1, 1 / self.r1)

            h = int(round(math.sqrt(target_area * aspect_ratio)))
            w = int(round(math.sqrt(target_area / aspect_ratio)))

            if w < img.size()[2] and h < img.size()[1]:
                x1 = random.randint(0, img.size()[1] - h)
                y1 = random.randint(0, img.size()[2] - w)
                if img.size()[0] == 3:
                    img[0, x1:x1 + h, y1:y1 + w] = self.mean[0]#(1.0 - self.mean[0])/self.std[0]#
                    img[1, x1:x1 + h, y1:y1 + w] = self.mean[1]#(1.0 - self.mean[1])/self.std[1]#
                    img[2, x1:x1 + h, y1:y1 + w] = self.mean[2]#(1.0 - self.mean[2])/self.std[2]#
                else:
                    img[0, x1:x1 + h, y1:y1 + w] = self.mean[0]#(1.0 - self.mean[0])/self.std[0]#
                return img

        return img

class RandomErasingCorner(object):
    """ Randomly selects a triangle region in an image's corner and erases its pixels.

        paste four corners
    Args:
         probability: The probability that the Random Erasing operation will be performed.
         sl: Minimum proportion of erased area against input image. e.g. 0.02x0.02
         sh: Maximum proportion of erased area against input image. e.g. 0.25x0.25
         r1: Minimum aspect ratio of erased area. e.g. 0.5   (0.25/sqrt(2)x  0.25*sqrt(2))

         # drl: Different ratio for four corners, e.g. (drl,drh)*s
         #drh:
         mean: Erasing value.
    """

    def __init__(self, probability=0.5, sl=0.02, sh=0.25, r1=0.5,
                 mean=(0.4914, 0.4822, 0.4465),std =  (0.229, 0.224, 0.225)):#drl=0.8, drh = 1.0,
        self.probability = probability
        self.mean = mean
        self.std = std
        self.sl = sl
        self.sh = sh
        self.r1 = r1
        #self.drl = drl
        #self.drh = drh

    def __call__(self, img):

        if random.uniform(0, 1) >= self.probability:
            return img

        ss  = np.random.uniform(self.sl, self.sh,(4,))
        asr = np.random.uniform(self.r1, 1 / self.r1,(4,))
        hh, ww = img.size()[1], img.size()[2]

        crp_h = hh*ss*np.sqrt(asr)
        crp_w = ww*ss*np.sqrt(1.0 / asr)

        #drs = np.random.uniform(self.drl,self.drh,(4,))

#        crp_h_lu,crp_w_lu =  int(round(drs[0] * crp_h)), int(round(drs[0] * crp_w))
#        crp_h_ru,crp_w_ru =  int(round(drs[1] * crp_h)), int(round(drs[1] * crp_w))
#        crp_h_ld,crp_w_ld =  int(round(drs[2] * crp_h)), int(round(drs[2] * crp_w))
#        crp_h_rd,crp_w_rd =  int(round(drs[3] * crp_h)), int(round(drs[3] * crp_w))
#
        crp_h  = np.round(crp_h).astype('int')
        crp_w  = np.round(crp_w).astype('int')

        crp_h_lu,crp_w_lu =  crp_h[0],crp_w[0]
        crp_h_ru,crp_w_ru =  crp_h[1],crp_w[1]
        crp_h_ld,crp_w_ld =  crp_h[2],crp_w[2]
        crp_h_rd,crp_w_rd =  crp_h[3],crp_w[3]

        r_lu,c_lu = np.array([-1,-1, crp_h_lu]), np.array([-1, crp_w_lu,-1])
        r_ru,c_ru = np.array([-1,-1, crp_h_ru]), np.array([ww-crp_w_ru-1, ww, ww])
        r_ld,c_ld = np.array([hh-crp_h_ld-1, hh, hh]), np.array([-1, crp_w_ld,-1])
        r_rd,c_rd = np.array([hh-crp_h_rd-1, hh, hh]), np.array([ ww, ww,ww-crp_w_rd-1])

        rr_lu, cc_lu = polygon(r_lu, c_lu)
        rr_ru, cc_ru = polygon(r_ru, c_ru)
        rr_ld, cc_ld = polygon(r_ld, c_ld)
        rr_rd, cc_rd = polygon(r_rd, c_rd)

        r_all = torch.from_numpy(np.hstack((rr_lu,rr_ru,rr_ld,rr_rd)))
        c_all = torch.from_numpy(np.hstack((cc_lu,cc_ru,cc_ld,cc_rd)))

        if img.size()[0] == 3:
            img[0, r_all, c_all] = (1.0 - self.mean[0])/self.std[0]#self.mean[0]#1.0
            img[1, r_all, c_all] = (1.0 - self.mean[1])/self.std[1]#self.mean[1]#1.0
            img[2, r_all, c_all] = (1.0 - self.mean[2])/self.std[2]#self.mean[2]#1.0
        else:
            img[0, r_all,c_all] = (1.0 - self.mean[0])/self.std[0]# self.mean[0]#

        return img

if __name__ == '__main__':
    import cv2
    img = cv2.imread('../data/fingerprint/lens_176176_newlens/ori/2/wet/L1/V2/0000_p_v2.bmp')
    img = np.transpose(img,(2,0,1))
    rec = RandomErasingCorner()
    num_aug = 20
    for n in range(num_aug):
        imgt = torch.from_numpy(img.copy()).float()/255.0
        img_aug = (rec(imgt).numpy()*255).astype('uint8')
        img_aug = np.transpose(img_aug,(1,2,0))
        cv2.imwrite(f'{n}.png', img_aug)


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
CS231n Convolutional Neural Networks for Visual Recognition
http://cs231n.github.io/

Python Numpy Tutorial
http://cs231n.github.io/python-numpy-tutorial/

matplotlib
https://matplotlib.org/

￣
We will highlight some parts of SciPy that you might find useful for this class.

python_numpy_tutorial-matplotlib-images.py
2019-07-03 (Wed)
"""
# Python Numpy Tutorial > Matplotlib > Images
#   Matplotlib is a plotting library similar to that of MATLAB.
#
# Read more about:
#     matplotlib.pyplot.imshow
#     https://matplotlib.org/3.1.0/api/_as_gen/matplotlib.pyplot.imshow.html

import numpy as np
from imageio import imread
import matplotlib.pyplot as plt

img        = imread('assets/cat.jpg')
img_tinted = img * [1,0.95,0.9]

# Show the original image
plt.subplot(1,2,1)
plt.imshow(img)

plt.subplot(1,2,2)
# Note nothing will show without np.uint8( ... )
plt.imshow( np.uint8( img_tinted) )
#plt.imshow( img_tinted )
plt.show()

max_stars_repo_name: aimldl/coding

max_issues_repo_name: aimldl/coding

max_forks_repo_name: aimldl/coding


text:
from statistics import mean
from collections import OrderedDict
import collections
import numpy as np
import csv
#input_file_name=('C://source.csv')
#output_file_name=('D://1.csv')

def calculate_averages(input_file_name, output_file_name):
    mydict = OrderedDict()
    rows=[]
    with open(input_file_name, mode='r') as input_file_name1:
        reader = csv.reader(input_file_name1)
        mydict = {rows[0]: rows[1:] for rows in reader}

    #print(mydict.values())
    mydict = dict((k, [int(s) for s in v]) for k,v in mydict.items())
    #print(mydict)
    with open(output_file_name, 'w', newline='') as output_file_name1:
        writer = csv.writer(output_file_name1,dialect='excel',delimiter =' ')
        for st,vals in mydict.items():
            row=("{},{}".format(st,mean(vals)))
            #writer.writerows([row])
            print(row)
            writer.writerows([i.strip().split(' ') for i in [row]])

#calculate_averages('C://source.csv','D://1.csv')

def calculate_sorted_averages(input_file_name, output_file_name):
    mydict = OrderedDict()
    rows=[]
    with open(input_file_name, mode='r') as input_file_name11:
        reader = csv.reader(input_file_name11)
        mydict = {rows[0]: rows[1:] for rows in reader}
    #print(mydict.values())
    mydict = dict((k, [int(s) for s in v]) for k,v in mydict.items())
    #print(mydict)
    with open(output_file_name, 'w', newline='') as output_file_name11:
        writer = csv.writer(output_file_name11,dialect='excel',delimiter =' ')
        od = collections.OrderedDict(sorted(mydict.items()))
        student_averages = ((sum(scores) / len(scores), s) for s, scores in od.items())
        for average, student in sorted(student_averages, reverse=False):
            row=("{},{}".format(student,average))
            writer.writerows([i.strip().split(' ') for i in [row]])

#calculate_sorted_averages('C://source.csv','D://1.csv')

def calculate_three_best(input_file_name, output_file_name):
    mydict = OrderedDict()
    rows=[]
    with open(input_file_name, mode='r') as input_file_name111:
        reader = csv.reader(input_file_name111)
        mydict = {rows[0]: rows[1:] for rows in reader}
    #print(mydict.values())
    mydict = dict((k, [int(s) for s in v]) for k,v in mydict.items())
    #print(mydict)
    with open(output_file_name, 'w', newline='') as output_file_name111:
        writer = csv.writer(output_file_name111,dialect='excel',delimiter =' ')
        od2 = collections.OrderedDict(sorted(mydict.items()))
        student_averages = ((sum(scores) / len(scores), s) for s, scores in od2.items())
        student_averages=sorted(student_averages, reverse=True)
        student_averages=student_averages[:3]
        for average, student in student_averages:
            row=("{},{}".format(student,average))
            writer.writerows([i.strip().split(' ') for i in [row]])

#calculate_three_best('C://source.csv','D://1.csv')

def calculate_three_worst(input_file_name, output_file_name):
    mydict = OrderedDict()
    rows=[]
    with open(input_file_name, mode='r') as input_file_name1111:
        reader = csv.reader(input_file_name1111)
        mydict = {rows[0]: rows[1:] for rows in reader}
    #print(mydict.values())
    mydict = dict((k, [int(s) for s in v]) for k,v in mydict.items())
    #print(mydict)
    with open(output_file_name, 'w', newline='') as output_file_name1111:
        writer = csv.writer(output_file_name1111,dialect='excel',delimiter =' ')
        student_averages = ((sum(scores) / len(scores), s) for s, scores in mydict.items())
        student_averages=sorted(student_averages, reverse=False)
        student_averages=student_averages[:3]
        for average, student in student_averages:
            row=("{}".format(average))
            writer.writerows([i.strip().split(' ') for i in [row]])

#calculate_three_worst('C://source.csv','D://1.csv')

def calculate_average_of_averages(input_file_name, output_file_name):
    mydict = OrderedDict()
    rows=[]
    with open(input_file_name, mode='r') as input_file_name11111:
        reader = csv.reader(input_file_name11111)
        mydict = {rows[0]: rows[1:] for rows in reader}
    #print(mydict.values())
    mydict = dict((k, [int(s) for s in v]) for k,v in mydict.items())
    #print(mydict)
    with open(output_file_name, 'w', newline='') as output_file_name11111:
        writer = csv.writer(output_file_name11111,dialect='excel',delimiter =' ')
        student_averages = ((sum(scores) / len(scores), s) for s, scores in mydict.items())
        student_averages=sorted(student_averages, reverse=False)
        mydict2=OrderedDict()
        for i,j in student_averages:
            mydict2={j:i for i,j in student_averages}
        test=list(mydict2.values())
        row=(sum(test) / len(test) )
        writer.writerow([row])

#calculate_average_of_averages('C://source.csv','D://1.csv')

max_stars_repo_name: techmod/maktabkhooneh-python

max_issues_repo_name: techmod/maktabkhooneh-python

max_forks_repo_name: techmod/maktabkhooneh-python


text:
import os
import numpy as np
import matlab.engine
import matlab
import datetime
import tqdm
import glob

class Get_Faces():
    def __init__(self):
        pwd = os.getcwd()
        os.chdir('tools')
        from get_faces import __init__, face_from_file, imshow
        net_face = __init__()
        os.chdir(pwd)
        self.net = net_face
        self.face_from_file = face_from_file
        self.imshow = imshow

    def from_file(self, img_file):
        return self.face_from_file(self.net, img_file)

def display_time(start, end):
    hours, rem = divmod(end - start, 3600)
    minutes, seconds = divmod(rem, 60)
    string_time = "{:0>2}:{:0>2}:{:05.2f}".format(
        int(hours), int(minutes), seconds)
    print("Time elapsed: " + string_time)
    print(datetime.datetime.now())
    print(" ")

def get_resize(org_file, resized_file, img_size):
    import skimage.transform
    import imageio
    if isinstance(img_size, int):
        img_size = [img_size, img_size]
    elif isinstance(img_size, list) and len(img_size) == 1:
        img_size = [img_size[0], img_size[0]]
    folder = os.path.dirname(resized_file)
    if not os.path.isdir(folder):
        os.makedirs(folder)
    imageio.imwrite(resized_file, (skimage.transform.resize(
        imageio.imread(org_file),
        (img_size[0], img_size[1])) * 255).astype(np.uint8))

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser(
        description='Txt file with path_to_image and 12 different AUs to LMDB')
    parser.add_argument(
        '--mode',
        type=str,
        default='train',
        help='Mode: Training/Test (default: Training)')
    parser.add_argument(
        '--img_size',
        type=int,
        default=256,
        help='size of the image to resize')
    parser.add_argument('--gpu', type=str, default='2', help='GPU device')
    parser.add_argument(
        '--fold', type=str, default='all', help='fold crossvalidation')
    parser.add_argument('--aligned', action='store_true', default=False)
    parser.add_argument('--google', action='store_true', default=False)
    parser.add_argument(
        '--folder_to_align',
        type=str,
        default='data/Google/Org',
        help='Folder where to perform alignment, optional.')
    # folder_to_align='data/CelebA/Org'
    args = parser.parse_args()
    os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu

    pwd = os.getcwd()
    os.chdir('/home/afromero/Codes/Face_Alignment/\
            MTCNN_face_detection_alignment/code/codes/MTCNNv2')
    future = matlab.engine.connect_matlab(True)
    eng = future.result()
    eng = matlab.engine.start_matlab()
    os.chdir(pwd)

    if not args.google:

        if args.fold == 'all':
            folds = [0, 1, 2]
        else:
            folds = [args.fold]

        for fold in folds:
            fold = int(fold)

            txt_file = 'data/MultiLabelAU/aligned/fold_{}/{}.txt'.format(
                fold, args.mode)

            org_files = [
                line.split()[0] for line in open(txt_file).readlines()
            ]

            resized_files = []
            count = 0
            for file_ in tqdm.tqdm(
                    org_files,
                    total=len(org_files),
                    desc='Resizing - fold %d' % (fold),
                    ncols=80,
                    leave=True):
                org_file = file_.replace('BP4D_256', 'BP4D')
                org_file = org_file.replace('Faces_aligned', 'Faces')
                resized_file = org_file.replace('BP4D',
                                                'BP4D_' + str(args.img_size))

                if not os.path.isfile(resized_file):
                    get_resize(org_file, resized_file, args.img_size)
                resized_files.append(resized_file)

            if args.aligned:
                print(' [*] Performing alignment...')
                _f = os.getcwd() + '/temp_txt'
                f = open(_f, 'w')
                for rs_file in resized_files:
                    f.writelines(rs_file + '\n')
                f.close()
                _ = eng.face_alignment(_f)
                os.remove(_f)
                print(' [°] Alignment done')

    else:
        import imageio
        folder_root = os.path.abspath(os.path.dirname(args.folder_to_align))
        face_root = os.path.abspath(os.path.join(folder_root, 'Faces'))
        if not os.path.isdir(face_root):
            os.makedirs(face_root)
        algined_root = os.path.abspath(
            os.path.join(folder_root, 'Faces_aligned'))
        if not os.path.isdir(algined_root):
            os.makedirs(algined_root)
        algined_size_root = os.path.abspath(
            os.path.join(folder_root,
                         'Faces_aligned_{}'.format(args.img_size)))
        if not os.path.isdir(algined_root):
            os.makedirs(algined_size_root)

        img_files = glob.glob(os.path.join(args.folder_to_align, '*.jpg'))
        txt_file = os.path.join(folder_root, 'data.txt')
        f = open(txt_file, 'w')
        for im in img_files:
            f.writelines(os.path.abspath(im) + '\n')
        f.close()

        org_files = [line.strip() for line in open(txt_file).readlines()]
        Faces = Get_Faces()
        face_files = []
        count = 0
        for org_file in tqdm.tqdm(
                org_files,
                total=len(org_files),
                desc='Extracting Faces',
                ncols=80,
                leave=True):
            file_name = os.path.basename(org_file)
            # folder_name = os.path.dirname(org_file)
            # face_name = file_name.split('.')[0]+\
            #   '_Faces.'+file_name.split('.')[1]
            face_file = os.path.join(face_root, file_name)

            if os.path.isfile(face_file):
                face_files.append(face_file)
                continue
            img_face = Faces.from_file(org_file)
            # Faces.imshow(img_face)
            try:
                imageio.imwrite(face_file, img_face[:, :, ::-1])
            except BaseException:
                continue
            face_files.append(face_file)

        if args.aligned:
            print(' [*] Performing alignment...')
            _f = os.path.abspath(txt_file.replace('.txt', '_faces.txt'))
            f = open(_f, 'w')
            for rs_file in face_files:
                # if os.path.isfile(rs_file.replace('Faces', 'Faces_aligned')):
                #   os.remove(rs_file.replace('Faces', 'Faces_aligned'))
                f.writelines(rs_file + '\n')
            f.close()
            _ = eng.face_alignment(_f)
            # os.remove(_f)
            print(' [°] Alignment done')

            _f = os.path.abspath(
                txt_file.replace('.txt', '_faces_aligned_{}.txt'.format(
                    args.img_size)))
            f = open(_f, 'w')
            for rs_file in tqdm.tqdm(
                    face_files,
                    total=len(face_files),
                    desc='Resizing Facial Alignment to %d' % (args.img_size),
                    ncols=80,
                    leave=True):
                file_aligned = rs_file.replace('Faces', 'Faces_aligned')
                if not os.path.isfile(file_aligned):
                    continue
                file_name = os.path.basename(rs_file)
                resized_file = os.path.join(algined_size_root, file_name)
                get_resize(file_aligned, resized_file, args.img_size)
                f.writelines(resized_file + '\n')
            f.close()

max_stars_repo_name: affromero/AUNets

max_issues_repo_name: affromero/AUNets

max_forks_repo_name: affromero/AUNets


text:
'''
Created on Apr 19, 2016

@author: franzefn
'''
from pysgpp.extensions.datadriven.uq.operations import checkPositivity, \
    insertHierarchicalAncestors, insertPoint, copyGrid, \
    dehierarchize, hierarchize, hasChildren, hasAllChildren
from pysgpp import HashGridPoint, createOperationEval, DataVector, IndexList
from pysgpp.extensions.datadriven.uq.plot.plot2d import plotSG2d
import matplotlib.pyplot as plt
from pysgpp.extensions.datadriven.uq.operations.sparse_grid import getHierarchicalAncestors, \
    insertTruncatedBorder, getBoundsOfSupport, evalSGFunctionMulti, \
    evalSGFunction
import numpy as np
from matplotlib.patches import Rectangle
from pysgpp.extensions.datadriven.uq.transformation import LinearTransformation, \
    JointTransformation
from pysgpp.extensions.datadriven.uq.operations.forcePositivity.fullGridSearch import FullGridCandidates
from pysgpp.extensions.datadriven.uq.operations.forcePositivity.localFullGridSearch import LocalFullGridCandidates
from pysgpp.extensions.datadriven.uq.operations.forcePositivity.findIntersections import IntersectionCandidates

class OperationMakePositiveFast(object):

    def __init__(self, grid,
                 candidateSearchAlgorithm=None,
                 interpolationAlgorithm=None,):
        self.grid = grid
        self.numDims = grid.getStorage().getDimension()
        self.maxLevel = grid.getStorage().getMaxLevel()
        self.interpolationAlgorithm = interpolationAlgorithm
        self.candidateSearchAlgorithm = candidateSearchAlgorithm
        if self.candidateSearchAlgorithm is None:
            self.candidateSearchAlgorithm = IntersectionCandidates()

        self.maxNewGridPoints = 10
        self.addAllGridPointsOnNextLevel = True

        self.lastMinimumCandidateLevelSum = None

        self.verbose = True

    def plotDebugIntersections(self, newGrid, overlappingGridPoints):
        # -----------------------------------------------------------------
        # plot result
        gs = newGrid.getStorage()
        for n, ((i, j), (gpi, gpj)) in enumerate(overlappingGridPoints.items()):
            fig = plt.figure()
            for k in xrange(gs.getSize()):
                gp = gs.getPoint(k)
                x, y = gp.getStandardCoordinate(0), gp.getStandardCoordinate(1)
                if alpha[k] < 0.0:
                    plt.plot(x, y, "v ", color="red")
                else:
                    plt.plot(x, y, "^ ", color="white")

            # annotate the
            plt.annotate(str(i), (gpi.getStandardCoordinate(0), gpi.getStandardCoordinate(1)))
            plt.annotate(str(j), (gpj.getStandardCoordinate(0), gpj.getStandardCoordinate(1)))

            # draw support
            # get level index
            for gp, col in ((gpi, "b"), (gpj, "r")):
                l0, i0, l1, i1 = gp.getLevel(0), gp.getIndex(0), gp.getLevel(1), gp.getIndex(1)
                xlim0, xlim1 = getBoundsOfSupport(l0, i0), getBoundsOfSupport(l1, i1)
                diff0, diff1 = xlim0[1] - xlim0[0], xlim1[1] - xlim1[0]
                currentAxis = plt.gca()
                currentAxis.add_patch(Rectangle((xlim0[0], xlim1[0]),
                                                diff0, diff1, facecolor=col,
                                                alpha=0.5))

            xlim = self.computeBoundsOfOverlappingPatch(gpi, gpj)
            currentAxis = plt.gca()
            currentAxis.add_patch(Rectangle((xlim[0, 0], xlim[1, 0]),
                                            np.diff(xlim[0, :2]), np.diff(xlim[1, :2]),
                                            facecolor="gray", alpha=0.9))

            gp = self.findIntersection(gpi, gpj)
            plt.plot(gp.getStandardCoordinate(0), gp.getStandardCoordinate(1), "o ", color="black")

            plt.title("%i/%i" % (n + 1, len(overlappingGridPoints)))
            plt.xlim(0, 1)
            plt.ylim(0, 1)
            plt.show()
            plt.close(fig)

    def plotDebug(self, grid, alpha, addedGridPoints, candidates):
        # -----------------------------------------------------------------
        # plot result
        fig = plt.figure()
        plotSG2d(grid, alpha, show_negative=True, show_grid_points=True)
        plt.title("iteration = %i" % self.candidateSearchAlgorithm.iteration)

        p = DataVector(grid.getStorage().getDimension())
        for gp in candidates:
            gp.getStandardCoordinates(p)
            plt.plot(p[0], p[1], "o ", color="green")

        for gp in addedGridPoints:
            gp.getStandardCoordinates(p)
            plt.plot(p[0], p[1], "o ", color="yellow")

        fig.show()

    def setInterpolationAlgorithm(self, algorithm):
        self.interpolationAlgorithm = algorithm

    def setCandidateSetSearchAlgorithm(self, algorithm):
        self.candidateSearchAlgorithm = algorithm

    def makeAddedNodalValuesPositive(self, grid, alpha, addedGridPoints, tol=-1e-14):
        neg = []
        gs = grid.getStorage()
        x = DataVector(gs.getDimension())
        for gp in addedGridPoints:
            gp.getStandardCoordinates(x)
            yi = evalSGFunction(grid, alpha, x.array())
            if yi < tol:
                i = gs.getSequenceNumber(gp)
                alpha[i] -= yi
                assert alpha[i] > -1e-14
                assert evalSGFunction(grid, alpha, x.array()) < 1e-14
        return alpha

    def makeCurrentNodalValuesPositive(self, grid, alpha, tol=-1e-14):
        nodalValues = dehierarchize(grid, alpha)
        neg = []
        for i, yi in enumerate(nodalValues):
            if yi < tol:
                nodalValues[i] = 0
                neg.append(i)
        if len(neg) > 0:
            alpha = hierarchize(grid, nodalValues)

        return alpha

    def sortCandidatesByLevelSum(self, candidates):
        gps = {}
        for gp in candidates:
            levelSum = gp.getLevelSum()
            if levelSum not in gps:
                gps[levelSum] = [gp]
            else:
                gps[levelSum].append(gp)
        return gps

    def addFullGridPoints(self, grid, alpha, candidates, tol=-1e-14):
        """
        Add all those full grid points with |accLevel|_1 <= n, where n is the
        maximun level of the sparse grid
        @param grid: Grid sparse grid to be discretized
        @param candidates:
        @param tol:
        """
        # remove all the already existing candidates
        gs = grid.getStorage()
        nonExistingCandidates = [gp for gp in candidates if not gs.isContaining(gp)]

        # sort the non existing grid points by level
        finalCandidates = self.sortCandidatesByLevelSum(nonExistingCandidates)

        levelSums = sorted(finalCandidates.keys())
        if self.lastMinimumCandidateLevelSum is None:
            ix = 0
        else:
            ixs = np.where(np.array(levelSums) == self.lastMinimumCandidateLevelSum)[0]
            if len(ixs) == 0:
                ix = 0
            elif len(ixs) == 1:
                ix = ixs[0] + 1
            else:
                raise AttributeError("this should never happen")

        done = False
        addedGridPoints = []
        minLevelSum = -1
        nextLevelCosts = 0

        while not done and ix < len(levelSums):
            minLevelSum = levelSums[ix]
            self.lastMinimumCandidateLevelSum = minLevelSum

            currentCandidates = finalCandidates[minLevelSum]
            if self.verbose:
                print "# check candidates    : %i/%i (at |l|_1 = %i <= %i)" % (len(currentCandidates),
                                                                               len(candidates),
                                                                               minLevelSum,
                                                                               np.max(levelSums)),

            # evaluate the remaining candidates
            samples = np.ndarray((len(currentCandidates), self.numDims))
            p = DataVector(self.numDims)
            for j, gp in enumerate(currentCandidates):
                gp.getStandardCoordinates(p)
                samples[j, :] = p.array()
            eval = evalSGFunctionMulti(grid, alpha, samples)

            negativeNonExistingCandidates = [gp for j, gp in enumerate(currentCandidates) if eval[j] < tol]

            if self.verbose:
                print "-> %i : considered candidates" % len(negativeNonExistingCandidates)

            for gp in negativeNonExistingCandidates:
                addedGridPoints += insertPoint(grid, gp)
                addedGridPoints += insertHierarchicalAncestors(grid, gp)

                if not self.addAllGridPointsOnNextLevel and len(addedGridPoints) > self.maxNewGridPoints:
                    done = True
                    break

            if self.addAllGridPointsOnNextLevel and len(addedGridPoints) > 0:
                nextLevelCosts = len(finalCandidates[minLevelSum])
                done = True

            ix += 1

        # recompute the leaf property and return the result
        grid.getStorage().recalcLeafProperty()

        return addedGridPoints, minLevelSum, nextLevelCosts

    def coarsening(self, grid, alpha, newGridPoints, tol=1e-14):
        """
        Removes all unnecessary grid points. A grid point is defined as
        unnecessary if it is a leaf node and its hierarchical coefficient is
        negative. This is applied just for the newly added points.
        @param grid: Grid
        @param alpha: numpy array hierarchical coefficients
        """
        notAffectedGridPoints = []
        # remove all entirely negative weighted sub-branches in the tree
        # just consider the newly added ones by discretization
        iteration = 1
        while True:
            gs = grid.getStorage()

            notAffectedGridPoints = []
            toBeRemoved = IndexList()
            for gp in newGridPoints:
                # if the grid point is a leaf and has negative weight
                # we dont need it to make the function positive
                if gs.isContaining(gp):
                    ix = gs.getSequenceNumber(gp)
                    if gs.getPoint(ix).isLeaf() and np.abs(alpha[ix]) < tol:
                        toBeRemoved.append(ix)

            # remove the identified grid points
            if toBeRemoved.size() > 0:
                newGrid = copyGrid(grid)
                newGs = newGrid.getStorage()
                # delete the unneccessary grid points from the new grid
                newGs.deletePoints(toBeRemoved)
                newGs.recalcLeafProperty()
                # copy the remaining alpha values
                newAlpha = np.ndarray(newGs.getSize())
                for i in xrange(newGs.getSize()):
                    newAlpha[i] = alpha[gs.getSequenceNumber(newGs.getPoint(i))]

                grid, alpha = newGrid, newAlpha
                iteration += 1
            else:
                break

        return grid, alpha

    def makePositive(self, alpha):
        """
        insert recursively all grid points such that the function is positive
        defined. Interpolate the function values for the new grid points using
        the registered algorithm.
        @param alpha: numpy array hierarchical coefficients
        """
        # make sure that the function is positive at every existing grid point
        newGrid = copyGrid(self.grid)
        newAlpha = alpha

        if self.verbose:
            print

        iteration = 0
        newAlpha = self.makeCurrentNodalValuesPositive(newGrid, newAlpha)

        newGs = newGrid.getStorage()
        numDims = newGs.getDimension()
        newGridPoints = []
        addedGridPoints = {}

        numFullGridPoints = (2 ** self.maxLevel - 1) ** numDims
        minLevelSum = self.maxLevel + self.numDims - 1
        maxLevelSum = self.numDims * self.maxLevel
        totalCosts = 0
        currentCosts = 0
        candidates = []
        while minLevelSum < maxLevelSum and self.candidateSearchAlgorithm.hasMoreCandidates(newGrid, newAlpha, addedGridPoints):
            candidates, costs = self.candidateSearchAlgorithm.nextCandidateSet()
            totalCosts += costs
            oldGridSize = newGs.getSize()

            if self.verbose:
                print "iteration             : %i" % self.candidateSearchAlgorithm.iteration
                print "# found candidates    : %i/%i (costs = %i)" % (len(candidates), numFullGridPoints, costs)

            if len(candidates) > 0:
                addedGridPoints, minLevelSum, nextLevelCosts = self.addFullGridPoints(newGrid, newAlpha, candidates)
                currentCosts += nextLevelCosts
                assert oldGridSize + len(addedGridPoints) == newGs.getSize()
                if self.verbose:
                    print "# new grid points     : %i -> %i -> %i" % (oldGridSize,
                                                                      len(addedGridPoints),
                                                                      newGrid.getSize())
                    print "  current total costs : %i <= %i <= %i" % (currentCosts, totalCosts, numFullGridPoints)
                    print "-" * 80

                newAlpha = np.append(newAlpha, np.zeros(len(addedGridPoints)))

                if self.interpolationAlgorithm is not None:
                    # compute now the hierarchical coefficients for the newly added points
                    newAlpha = self.interpolationAlgorithm.computeHierarchicalCoefficients(newGrid,
                                                                                newAlpha,
                                                                                addedGridPoints)
                else:
                    newAlpha = self.makeAddedNodalValuesPositive(newGrid, newAlpha, addedGridPoints)

                # update list of new grid points
                newGridPoints += addedGridPoints

#                 if newGs.getDimension() == 2:
#                     self.plotDebug(newGrid, newAlpha, addedGridPoints, candidates)
            else:
                break

#         # coarsening: remove all new grid points with zero surplus
#         coarsedGrid, coarsedAlpha = self.coarsening(newGrid, newAlpha, newGridPoints)
#         if self.verbose:
#             print "                        old | coarsed | new | max (cand) | full"
#             print "# final grid          : %i <= %i <= %i <= %i (%i) <= %i" % (self.grid.getSize(),
#                                                                                coarsedGrid.getSize(),
#                                                                                newGrid.getSize(),
#                                                                                self.grid.getSize() + len(candidates),
#                                                                                len(candidates),
#                                                                                (2 ** self.maxLevel - 1) ** self.numDims)

        return newGrid, newAlpha

max_stars_repo_name: valentjn/thesis

max_issues_repo_name: valentjn/thesis

max_forks_repo_name: valentjn/thesis


text:
import numpy as np
import matplotlib.pyplot as plt
import math

TIME_SLEEP = 0.000000001

def train_sgd(X, y, alpha, w=None):
    """Trains a linear regression model using stochastic gradient descent.

    Parameters
    ----------
    X : numpy.ndarray
        Numpy array of data
    y : numpy.ndarray
        Numpy array of outputs. Dimensions are n * 1, where n is the number of rows
        in `X`.
    alpha : float
        Describes the learning rate.
    w : numpy.ndarray, optional
        The initial w vector (the default is zero).

    Returns
    -------
    w : numpy.ndarray
        Trained vector with dimensions (m + 1) * 1, where m is the number of
        columns in `X`.

    """

    X_b = np.hstack((np.ones((X.shape[0], 1)), X))

    previous_error = -1
    error = -1

    stop = False
    num_iters = 0

    if w is None:
        w = np.zeros((x.shape[1] + 1, 1))

    while not stop:
        for i in range(0, len(X)):
            w = w - alpha / len(X) * (np.dot(np.transpose(w),
                                      X_b[i].reshape(X_b.shape[1], 1)) -
                                      y[i]) * X_b[i].reshape(X_b.shape[1], 1)

            error = evaluate_error(X, y, w)
            if previous_error == -1:
                previous_error = error
            elif (math.fabs(error - previous_error) < 0.01 * previous_error and
                  num_iters > 10000):
                stop = True
                break

            previous_error = error
            num_iters += 1

    return w

def train(X, y):
    """Trains a linear regression model using linear algebra.

    Parameters
    ----------
    X : numpy.ndarray
        Numpy array of data
    y : numpy.ndarray
        Numpy array of outputs. Dimensions are n * 1, where n is the number of rows
        in `X`.

    Returns
    -------
    w : numpy.ndarray
        Trained vector with dimensions (m + 1) * 1, where m is the number of
        columns in `X`.

    """

    # Add bias term
    X_b = np.hstack((np.ones((X.shape[0], 1)), X))

    # Compute pseudo-inverse
    X_inverse = (np.linalg.inv(np.transpose(X_b).dot(X_b)).dot(
                 np.transpose(X_b)))

    # Compute w
    w = X_inverse.dot(y)

    return w

# Plot data
def plot(X, y, w):
    """Plot X data, the actual y output, and the prediction line.

    Parameters
    ----------
    X : numpy.ndarray
        Numpy array of data with 1 column.
    y : numpy.ndarray
        Numpy array of outputs. Dimensions are n * 1, where n is the number of
        rows in `X`.
    w : numpy.ndarray
        Numpy array with dimensions 2 * 1.

    """

    X_b = np.hstack((np.ones((X.shape[0], 1)), X))

    y_predict = X_b.dot(w)

    plt.clf()
    plt.plot(X[:, 0], y_predict, 'r-', X[:, 0], y, 'o')
    plt.pause(TIME_SLEEP)

def init_plot(figsize=(15, 8)):
    """Initializes the plot.

    Parameters
    ----------
    figsize : tuple, optional
        A tuple containing the width and height of the plot (the default is
        (15, 8)).

    """
    plt.ion()
    f = plt.figure(figsize=figsize)
    plt.show()

def evaluate_error(X, y, w):
    """Returns the mean squared error.

    X : numpy.ndarray
        Numpy array of data.
    y : numpy.ndarray
        Numpy array of outputs. Dimensions are n * 1, where n is the number of
        rows in `X`.
    w : numpy.ndarray
        Numpy array with dimensions (m + 1) * 1, where m is the number of
        columns in `X`.

    Returns
    -------
    float
        The mean squared error

    """

    X_b = np.hstack((np.ones((X.shape[0], 1)), X))
    y_predict = X_b.dot(w)
    dist = (y - y_predict) ** 2

    return float(np.sum(dist)) / X.shape[0]

def predict(X, w):
    """Returns the prediction for one data point.

    Parameters
    ----------
    X : numpy.ndarray
        Numpy array of data
    w : numpy.ndarray
        Numpy array with dimensions (m + 1) * 1, where m is the number of
        columns in `X`.

    Returns
    -------
    float
        The mean squared error

    """
    X_b = np.hstack((np.ones((X.shape[0], 1)), X))
    return X_b.dot(w)

max_stars_repo_name: AnomalyInc/StreamPy

max_issues_repo_name: StreamPy/StreamPy

max_forks_repo_name: StreamPy/StreamPy


text:
import numpy as np
from learning_to_adapt.utils.serializable import Serializable
from learning_to_adapt.envs.mujoco_env import MujocoEnv
from learning_to_adapt.logger import logger
import os

class HalfCheetahHFieldEnv(MujocoEnv, Serializable):
    def __init__(self, task='hfield', reset_every_episode=False, reward=True, *args, **kwargs):
        Serializable.quick_init(self, locals())

        self.cripple_mask = None
        self.reset_every_episode = reset_every_episode
        self.first = True
        MujocoEnv.__init__(self, os.path.join(os.path.abspath(os.path.dirname(__file__)),
                                              "assets", "half_cheetah_hfield.xml"))

        task = None if task == 'None' else task

        self._init_geom_rgba = self.model.geom_rgba.copy()
        self._init_geom_contype = self.model.geom_contype.copy()
        self._init_geom_size = self.model.geom_size.copy()
        self._init_geom_pos = self.model.geom_pos.copy()
        self.dt = self.model.opt.timestep

        assert task in [None, 'hfield', 'hill', 'basin', 'steep', 'gentle']

        self.task = task
        self.x_walls = np.array([250, 260, 261, 270, 280, 285])
        self.height_walls = np.array([0.2, 0.2, 0.2, 0.2, 0.2, 0.2])
        self.height = 0.8
        self.width = 15

    def get_current_obs(self):
        return np.concatenate([
            self.model.data.qpos.flatten()[1:],
            self.model.data.qvel.flat,
            self.get_body_com("torso").flat,
        ])

    def get_body_xmat(self, body_name):
        idx = self.model.body_names.index(body_name)
        return self.model.data.xmat[idx].reshape((3, 3))

    def get_body_com(self, body_name):
        idx = self.model.body_names.index(body_name)
        return self.model.data.com_subtree[idx]

    def step(self, action):
        self.forward_dynamics(action)
        next_obs = self.get_current_obs()
        ctrl_cost = 1e-1 * 0.5 * np.sum(np.square(action))
        forward_reward = self.get_body_comvel("torso")[0]
        reward = forward_reward - ctrl_cost
        done = False
        info = {}
        return next_obs, reward, done, info

    def reward(self, obs, action, next_obs):
        assert obs.ndim == 2
        assert obs.shape == next_obs.shape
        assert obs.shape[0] == action.shape[0]
        ctrl_cost = 1e-1 * 0.5 * np.sum(np.square(action), axis=1)
        forward_reward = (next_obs[:, -3] - obs[:, -3])/self.dt
        reward = forward_reward - ctrl_cost
        return reward

    def reset_mujoco(self, init_state=None):
        super(HalfCheetahHFieldEnv, self).reset_mujoco(init_state=init_state)
        if self.reset_every_episode and not self.first:
            self.reset_task()

        if self.first:
            self.first = False

    def reset_task(self, value=None):
        if self.task == 'hfield':
            height = np.random.uniform(0.2, 1)
            width = 10
            n_walls = 6
            self.model.hfield_size = np.array([50, 5, height, 0.1])
            x_walls = np.random.choice(np.arange(255, 310, width), replace=False, size=n_walls)
            x_walls.sort()
            sign = np.random.choice([1, -1], size=n_walls)
            sign[:2] = 1
            height_walls = np.random.uniform(0.2, 0.6, n_walls) * sign
            row = np.zeros((500,))
            for i, x in enumerate(x_walls):
                terrain = np.cumsum([height_walls[i]] * width)
                row[x:x + width] += terrain
                row[x+width:] = row[x+width - 1]
            row = (row - np.min(row))/(np.max(row) - np.min(row))
            hfield = np.tile(row.reshape(-1, 1), (1, 528)).T.reshape(-1, 1)
            self.model.hfield_data = hfield

        elif self.task == 'basin':
            self.height_walls = np.array([-1, 1, 0., 0., 0., 0.])
            self.height = 0.55
            height = self.height
            width = self.width
            self.x_walls = np.array([255, 270, 285, 300, 315, 330]) - 5
            self.model.hfield_size = np.array([50, 5, height, 0.1])
            row = np.zeros((500,))
            for i, x in enumerate(self.x_walls):
                terrain = np.cumsum([self.height_walls[i]] * width)
                row[x:x + width] += terrain
                row[x + width:] = row[x + width - 1]
            row = (row - np.min(row)) / (np.max(row) - np.min(row))
            hfield = np.tile(row.reshape(-1, 1), (1, 528)).T.reshape(-1, 1)
            self.model.hfield_data = hfield

        elif self.task == 'hill':
            self.height_walls = np.array([1, -1, 0, 0., 0, 0])
            self.height = 0.6
            height = self.height
            width = self.width
            self.x_walls = np.array([255, 270, 285, 300, 315, 330]) - 5
            self.model.hfield_size = np.array([50, 5, height, 0.1])
            row = np.zeros((500,))
            for i, x in enumerate(self.x_walls):
                terrain = np.cumsum([self.height_walls[i]] * width)
                row[x:x + width] += terrain
                row[x + width:] = row[x + width - 1]
            row = (row - np.min(row)) / (np.max(row) - np.min(row))
            hfield = np.tile(row.reshape(-1, 1), (1, 528)).T.reshape(-1, 1)
            self.model.hfield_data = hfield

        elif self.task == 'gentle':
            self.height_walls = np.array([1, 1, 1, 1, 1, 1])
            self.height = 1
            height = self.height
            width = self.width
            self.x_walls = np.array([255, 270, 285, 300, 315, 330]) - 5
            self.model.hfield_size = np.array([50, 5, height, 0.1])
            row = np.zeros((500,))
            for i, x in enumerate(self.x_walls):
                terrain = np.cumsum([self.height_walls[i]] * width)
                row[x:x + width] += terrain
                row[x + width:] = row[x + width - 1]
            row = (row - np.min(row)) / (np.max(row) - np.min(row))
            hfield = np.tile(row.reshape(-1, 1), (1, 528)).T.reshape(-1, 1)
            self.model.hfield_data = hfield

        elif self.task == 'steep':
            self.height_walls = np.array([1, 1, 1, 1, 1, 1])
            self.height = 4
            height = self.height
            width = self.width
            self.x_walls = np.array([255, 270, 285, 300, 315, 330]) - 5
            self.model.hfield_size = np.array([50, 5, height, 0.1])
            row = np.zeros((500,))
            for i, x in enumerate(self.x_walls):
                terrain = np.cumsum([self.height_walls[i]] * width)
                row[x:x + width] += terrain
                row[x + width:] = row[x + width - 1]
            row = (row - np.min(row)) / (np.max(row) - np.min(row))
            hfield = np.tile(row.reshape(-1, 1), (1, 528)).T.reshape(-1, 1)
            self.model.hfield_data = hfield

        elif self.task is None:
            pass

        else:
            raise NotImplementedError

        self.model.forward()

    def log_diagnostics(self, paths, prefix):
        progs = [
            path["observations"][-1][-3] - path["observations"][0][-3]
            for path in paths
            ]
        logger.logkv(prefix + 'AverageForwardProgress', np.mean(progs))
        logger.logkv(prefix + 'MaxForwardProgress', np.max(progs))
        logger.logkv(prefix + 'MinForwardProgress', np.min(progs))
        logger.logkv(prefix + 'StdForwardProgress', np.std(progs))

if __name__ == '__main__':
    env = HalfCheetahHFieldEnv(task='hfield')
    while True:
        env.reset()
        env.reset_task()
        for _ in range(100):
            env.render()
            env.step(env.action_space.sample())
        env.stop_viewer()

max_stars_repo_name: jeonggwanlee/varibad

max_issues_repo_name: jeonggwanlee/varibad

max_forks_repo_name: jeonggwanlee/varibad


text:
import os
import sys

import h5py
import numpy as np

from pisa import ureg, Q_
from pisa.core.binning import OneDimBinning, MultiDimBinning
from pisa.core.map import Map, MapSet
from pisa.core.stage import Stage
from pisa.utils.log import logging
from pisa.utils.comparisons import normQuant
from pisa.utils.resources import find_resource

__author__ = 'P. Eller'

__license__ = '''Copyright (c) 2014-2017, The IceCube Collaboration

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.'''

class data(Stage):
    """Data loader stage

    Paramaters
    ----------

    params : ParamSet
        data_file : string
            path pointing to the hdf5 file containing the events
        sim_ver: string
            indicateing the sim version, wither 4digit, 5digit or dima
        bdt_cut : float
            futher cut apllied to events for the atm. muon rejections BDT

    Notes
    -----

    The curent versio of this code is a port from pisa v2 nutau branch.
    It clearly needs to be cleand up properly at some point.

    """

    def __init__(self, params, output_binning, disk_cache=None,
                memcache_deepcopy=True, error_method=None,
                outputs_cache_depth=20, debug_mode=None):

        expected_params = (
            'data_file',
            'sim_ver',
            'bdt_cut'
        )

        output_names = ('total')

        super().__init__(
            use_transforms=False,
            params=params,
            expected_params=expected_params,
            output_names=output_names,
            error_method=error_method,
            disk_cache=disk_cache,
            memcache_deepcopy=memcache_deepcopy,
            outputs_cache_depth=outputs_cache_depth,
            output_binning=output_binning,
            debug_mode=debug_mode
        )

    def _compute_nominal_outputs(self):
        """load the evnts from file, perform sanity checks and histogram them
        (into final MapSet)

        """
        # get params
        data_file_name = self.params.data_file.value
        sim_version = self.params.sim_ver.value
        bdt_cut = self.params.bdt_cut.value.m_as('dimensionless')

        self.bin_names = self.output_binning.names

        # TODO: convert units using e.g. `comp_units` in stages/reco/hist.py
        self.bin_edges = []
        for name in self.bin_names:
            if 'energy' in  name:
                bin_edges = self.output_binning[name].bin_edges.to('GeV').magnitude
            else:
                bin_edges = self.output_binning[name].bin_edges.magnitude
            self.bin_edges.append(bin_edges)

        # the rest of this function is PISA v2 legacy code...
        # right now only use burn sample with sim_version = '4digit'
        #print "sim_version == ", sim_version
        if sim_version == "4digit":
            Reco_Neutrino_Name = 'IC86_Dunkman_L6_MultiNest8D_PDG_Neutrino'
            Reco_Track_Name = 'IC86_Dunkman_L6_MultiNest8D_PDG_Track'
        elif sim_version == "5digit" or sim_version=="dima":
            Reco_Neutrino_Name = 'IC86_Dunkman_L6_PegLeg_MultiNest8D_NumuCC'
            Reco_Track_Name = 'IC86_Dunkman_L6_PegLeg_MultiNest8D_Track'
        else:
            raise ValueError(
                'only allow 4digit, 5digit(H2 model for hole ice) or'
                ' dima (dima p1 and p2 for hole ice)!'
            )

        data_file = h5py.File(find_resource(data_file_name), 'r')
        L6_result = np.array(data_file['IC86_Dunkman_L6']['result'])
        dLLH = np.array(data_file['IC86_Dunkman_L6']['delta_LLH'])
        reco_energy_all = np.array(data_file[Reco_Neutrino_Name]['energy'])
        reco_coszen_all = np.array(np.cos(
            data_file[Reco_Neutrino_Name]['zenith']
        ))
        reco_trck_len_all = np.array(data_file[Reco_Track_Name]['length'])
        #print "before L6 cut, no. of burn sample = ", len(reco_coszen_all)

        # sanity check
        santa_doms = data_file['IC86_Dunkman_L6_SANTA_DirectDOMs']['value']
        l3 = data_file['IC86_Dunkman_L3']['value']
        l4 = data_file['IC86_Dunkman_L4']['result']
        l5 = data_file['IC86_Dunkman_L5']['bdt_score']
        assert(np.all(santa_doms>=3) and np.all(l3 == 1) and np.all(l5 >= 0.1))

        # l4==1 was not applied when i3 files were written to hdf5 files, so do
        # it here
        dLLH = dLLH[l4==1]
        reco_energy_all = reco_energy_all[l4==1]
        reco_coszen_all = reco_coszen_all[l4==1]
        l5 = l5[l4==1]
        L6_result = L6_result[l4==1]
        data_file.close()

        dLLH_L6 = dLLH[L6_result==1]
        l5 = l5[L6_result==1]
        reco_energy_L6 = reco_energy_all[L6_result==1]
        reco_coszen_L6 = reco_coszen_all[L6_result==1]
        #print "after L6 cut, no. of burn sample = ", len(reco_coszen_L6)

        # Cut: Only keep bdt score >= 0.2 (from MSU latest result, make data/MC
        # agree much better); if use no such further cut, use bdt_cut = 0.1
        logging.info(
            "Cut2, removing events with bdt_score < %s i.e. only keep bdt > %s"
            %(bdt_cut, bdt_cut)
        )
        cut_events = {}
        cut = l5>=bdt_cut
        cut_events['reco_energy'] = reco_energy_L6[cut]
        cut_events['reco_coszen'] = reco_coszen_L6[cut]
        cut_events['pid'] = dLLH_L6[cut]

        hist, _ = np.histogramdd(sample = np.array(
            [cut_events[bin_name] for bin_name in self.bin_names]
        ).T, bins=self.bin_edges)

        maps = [Map(name=self.output_names[0], hist=hist,
                    binning=self.output_binning)]
        self.template = MapSet(maps, name='data')

    def _compute_outputs(self, inputs=None):
        """return the precomputed MpSets, since this is data, the distributions
        don't change

        """
        return self.template

max_stars_repo_name: austinschneider/pisa

max_issues_repo_name: austinschneider/pisa

max_forks_repo_name: austinschneider/pisa


text:
import json
import os

import numpy as np
import requests
import imageio

data_path = os.environ.get('DATA_PATH') or '/tmp/data/sk_mnist/'
mnist_images = np.load(os.path.join(data_path, "train_data.npy"))

# url = 'http://localhost:8080/mnist-all-sklearn-axon-2979892518-983389825/predict'
# url = 'http://localhost:8080/predict'
url = 'http://localhost:5000/predict'

image = mnist_images[19:20, :]

jsondata = json.dumps({"data": {"ndarray": image.tolist()}})
payload = {'json': jsondata}
session = requests.session()
response = session.post(url, data=payload, headers={'User-Agent': 'test'})

imageio.imwrite('outfile.jpg', image.reshape([28, 28]))

print(payload)
session = requests.session()
response = session.post(url, data=payload, headers={'User-Agent': 'test'})
print(response.status_code)
print(response.text)
json_data = json.loads(response.text)
prediction = json_data["data"]["ndarray"]

print("predicted class: %s" % np.argmax(np.array(prediction)))

max_stars_repo_name: khsibr/examples

max_issues_repo_name: khsibr/examples

max_forks_repo_name: khsibr/examples


text:
import numpy as np
import sys
import cv2
import os

def get_masks(img, old):

	b = img[:, :, 0]
	g = img[:, :, 1]
	r = img[:, :, 2]

	# the background mask of the original labels
	if old:
		background_mask = (r == 0) & (g == 0) & (b == 255)
	else:
		background_mask = (r == 0) & (g == 0) & (b == 0)

	skin_mask       = (r == 0)   & (g == 255) & (b == 0)
	hair_mask       = (r == 255) & (g == 0)   & (b == 0)

	return background_mask, skin_mask, hair_mask

base_path = "/media/jaume.gibert/Data/elfw"
images           = os.path.join(base_path, "elfw_01_basic/faces")
labels_1         = os.path.join(base_path, "parts_lfw_funneled_gt_images")
labels_2         = os.path.join(base_path, "elfw_01_basic/labels")
output           = os.path.join(base_path, "comparisons")
out_improvements = os.path.join(output, "relabelled")
out_new          = os.path.join(output, "new_labels")

if not os.path.exists(output):
	os.mkdir(output)
if not os.path.exists(out_improvements):
	os.mkdir(out_improvements)
if not os.path.exists(out_new):
	os.mkdir(out_new)

for image in os.listdir(images):

	base_name = os.path.splitext(image)[0]
	print(base_name)

	# Face image
	img_file   = os.path.join(images, image)
	lab_1_file = os.path.join(labels_1, base_name + '.ppm')
	lab_2_file = os.path.join(labels_2, base_name + '.png')

	# If all three exist, it means that we either didn't touch the labels or we improved it
	# Save into improvements
	# TODO: check equal old and new labels and only save the improved ones
	if os.path.exists(img_file) and os.path.exists(lab_1_file) and os.path.exists(lab_2_file):
		img   = cv2.imread(img_file)
		lab_1 = cv2.imread(lab_1_file)
		lab_2 = cv2.imread(lab_2_file)

		# I take the original labels (lab_1), convert the blue (background into black)
		# and check if it's the same as in the new label image (lab_2)
		# if so, I don't want it, I want the other cases.
		b1, s1, h1 = get_masks(lab_1, old=True)
		b2, s2, h2 = get_masks(lab_2, old=False)

		if not (np.array_equal(b1, b2) and np.array_equal(s1, s2) and np.array_equal(h1, h2)):
			result = np.vstack((img, np.vstack((lab_2, lab_1))))
			cv2.imwrite(os.path.join(out_improvements, base_name+".png"), result)

	# if only the image and our labels exist, it means the image-label is new and didn't exist in LFW
	elif os.path.exists(img_file) and not os.path.exists(lab_1_file) and os.path.exists(lab_2_file):

		img   = cv2.imread(img_file)
		lab_2 = cv2.imread(lab_2_file)
		result = np.vstack((img, lab_2))
		cv2.imwrite(os.path.join(out_new, base_name+".png"), result)

	else:
		continue

max_stars_repo_name: multimedia-eurecat/ELFW

max_issues_repo_name: multimedia-eurecat/ELFW

max_forks_repo_name: multimedia-eurecat/ELFW


text:
# -*- coding: utf-8 -*-
"""

Author
------
Bo Zhang

Email
-----
bozhang@nao.cas.cn

Created on
----------
- Mon Nov 28 15:00:00 2016

Modifications
-------------
-

Aims
----
- wavelength calibration module
    all operations are based on extracted ThAr (1d)

"""

import itertools

import numpy as np
from astropy.io import fits
from joblib import Parallel, delayed
from scipy.interpolate import splrep, splev
from scipy.optimize import curve_fit, leastsq

# ############################## #
#      fix thar spectra
# ############################## #

def thar1d_fix(thar1d, conv_len=20, sat_count=50000):
    """ fix thar image, particularly for negative & saturated pixels

    Parameters
    ----------
    conv_len:
        convolved length
    sat_count:
        saturated count / max count

    Return
    ------
    thar1d_fixed:
        fixed THAR image

    """
    # ind_sat_conv
    ind_sat = thar1d >= sat_count
    ind_sat_conv = np.zeros_like(ind_sat)
    x_sat, y_sat = np.where(ind_sat)
    for i in range(len(x_sat)):
        ind_sat_conv[x_sat[i], y_sat[i] - conv_len:y_sat[i] + conv_len] = 1
    # ind_neg
    ind_neg = thar1d < 0.
    # combine to ind_bad
    ind_bad = np.logical_or(ind_neg, ind_sat_conv)
    thar1d_fixed = np.where(ind_bad, 0, thar1d)

    return thar1d_fixed

# ############################## #
#  2d correlation for 1d thar
# ############################## #

def thar1d_corr2d(thar1d_fixed, thar_temp, x_shiftmax=20, y_shiftmax=5,
                  verbose=False):
    """ determine the shift of *thar1d_fixed* relative to *thar_temp*

    Parameters
    ----------
    thar1d_fixed:
        image whose shift is to be determined
    thar_temp:
        reference image
    xtrim: tuple
        (xmin, xmax)
    ytrim: tuple
        (ymin, ymax)
    x_shiftmax: int
        the max y shift for correlation
    y_shiftmax: int
        the max y shift for correlation

    Returns
    -------
    (x_shift, y_shift), corr2d

    """

    # default trim region is the center 1/4 area
    xtrim = (thar1d_fixed.shape[1] * np.array([.2, .8])).astype(int)
    ytrim = (thar1d_fixed.shape[0] * np.array([.2, .8])).astype(int)

    # in case that the input data are int
    thar1d_fixed = np.array(thar1d_fixed).astype(float)
    thar_temp = np.array(thar_temp).astype(float)

    if verbose:
        print("@Cham: computing 2D cross-correlation...")

    # initialize result
    corr2d = np.zeros((1 + 2 * y_shiftmax, 1 + 2 * x_shiftmax))
    # make slice
    xslice_temp = slice(*xtrim)
    yslice_temp = slice(*ytrim)
    # 2D correlation
    for xofst in range(-x_shiftmax, x_shiftmax + 1, 1):
        for yofst in range(-y_shiftmax, y_shiftmax + 1, 1):
            xslice = slice(xtrim[0] + xofst, xtrim[1] + xofst, 1)
            yslice = slice(ytrim[0] + yofst, ytrim[1] + yofst, 1)
            corr2d[yofst + y_shiftmax, xofst + x_shiftmax] = \
                np.mean(thar1d_fixed[yslice, xslice] *
                        thar_temp[yslice_temp, xslice_temp])
    # select maximum value
    y_, x_ = np.where(corr2d == np.max(corr2d))
    x_shift = np.int(x_shiftmax - x_)  # reverse sign
    y_shift = np.int(y_shiftmax - y_)

    return (x_shift, y_shift), corr2d

# ############################## #
#      shift wave & order
# ############################## #

def interpolate_wavelength(w, shift, thar_temp, thar1d_fixed):
    """ interpolate given a regerence wavelength solution and shift """
    xshift, yshift = shift

    xcoord = np.arange(w.shape[1])
    ycoord = np.arange(w.shape[0])

    # for X, no difference, for Y, orders are different
    xcoord_xshift = xcoord + xshift
    ycoord_yshift = np.arange(
        w.shape[0] + thar1d_fixed.shape[0] - thar_temp.shape[0]) + yshift
    # shfit X
    w_x = np.zeros(w.shape)
    for i in range(w.shape[0]):
        s = splrep(xcoord, w[i], k=3, s=0)
        w_x[i] = splev(xcoord_xshift, s)

    # shift Y
    w_x_y = np.zeros((thar1d_fixed.shape[0], w.shape[1]))
    for j in range(w.shape[1]):
        s = splrep(ycoord, w_x[:, j], k=3, s=0)
        w_x_y[:, j] = splev(ycoord_yshift, s)

    return w_x_y

def interpolate_order(order_temp, shift, thar1d_fixed):
    xshift, yshift = shift

    # for X, no difference, for Y, orders are different
    ycoord_yshift = np.arange(thar1d_fixed.shape[0]) + \
                    order_temp[0, 0] + yshift
    order_interp = np.repeat(ycoord_yshift.reshape(-1, 1),
                             thar1d_fixed.shape[1], axis=1)

    return order_interp

# ############################## #
#      load temp thar spectra
# ############################## #

def load_thar_temp(thar_temp_path):
    hl = fits.open(thar_temp_path)
    wave = hl['wave'].data
    thar = hl['thar'].data
    order = hl['order'].data
    return wave, thar, order

# ############################## #
#      refine thar positions
# ############################## #

def refine_thar_positions(wave_init, order_init, thar1d_fixed, thar_list,
                          fit_width=5., lc_tol=5., k=3, n_jobs=10, verbose=10):
    """ refine ThAr positions """
    print("@TWODSPEC: refine ThAr positions ...")

    # refine thar positions for each order
    r = Parallel(n_jobs=n_jobs, verbose=verbose, batch_size=1)(
        delayed(refine_thar_positions_order)(
            wave_init[i_order],
            np.arange(wave_init.shape[1]),
            thar1d_fixed[i_order],
            thar_list[(thar_list > np.min(wave_init[i_order]) + 1.) * (
                thar_list < np.max(wave_init[i_order]) - 1.)],
            order_init[i_order, 0],
            fit_width=fit_width, lc_tol=lc_tol, k=k
        ) for i_order in range(wave_init.shape[0]))

    # remove all null values
    null_value = None
    for i in range(r.count(null_value)):
        r.remove(null_value)
        # print(len(r))

    # collect data
    lc_coord = np.array(np.hstack([_[0] for _ in r]))
    lc_order = np.array(np.hstack([_[1] for _ in r]))
    lc_thar = np.array(np.hstack([_[2] for _ in r]))
    popt = np.array(np.vstack([_[3] for _ in r]))
    pcov = np.array(np.vstack([_[4] for _ in r]))

    return lc_coord, lc_order, lc_thar, popt, pcov

def refine_thar_positions_order(this_wave_init, this_xcoord, this_thar,
                                this_thar_list, this_order, fit_width=5.,
                                lc_tol=5., k=3):
    if len(this_thar_list) == 0:
        return None

    popt_list = []
    pcov_list = []

    # refine all thar positions in this order
    for i_thar_line, each_thar_line in enumerate(this_thar_list):

        # cut local spectrum
        ind_local = (this_wave_init > each_thar_line - fit_width) * (
            this_wave_init < each_thar_line + fit_width)

        # set bounds
        p0 = (0., 1E5, each_thar_line, 0.1)
        bounds = ((-1., 0., each_thar_line - lc_tol, 0.01),
                  (+np.inf, np.inf, each_thar_line + lc_tol, 2.))

        try:
            popt, pcov = curve_fit(gauss_poly0, this_wave_init[ind_local],
                                   this_thar[ind_local], p0=p0, bounds=bounds)
            pcov = np.diagonal(pcov)
        except RuntimeError:
            popt = np.ones_like(p0) * np.nan
            pcov = np.ones((len(p0),)) * np.nan

        popt_list.append(popt)
        pcov_list.append(pcov)

    # interpolation for X corrdinates
    if np.all(np.diff(this_wave_init) >= 0):
        tck = splrep(this_wave_init, this_xcoord, k=k)
        # print(this_wave_init, this_xcoord, k)
    elif np.all(np.diff(this_wave_init) <= 0):
        tck = splrep(this_wave_init[::-1], this_xcoord[::-1], k=k)
        print("@SONG: wavelength inversed!")
    else:
        raise (ValueError("@Cham: error occurs in interpolation!"))

    # lccov_list = np.array(lccov_list)
    popt_list = np.array(popt_list)
    pcov_list = np.array(pcov_list)

    lc_coord = splev(popt_list[:, 2], tck)
    lc_order = np.ones_like(lc_coord) * this_order

    return lc_coord, lc_order, this_thar_list, popt_list, pcov_list

# ############################## #
#      2D surface fit
# ############################## #

def polyval2d(x, y, coefs, orders=None):
    if orders is None:
        orderx, ordery = coefs.shape
    else:
        orderx, ordery = orders

    ij = itertools.product(range(orderx + 1), range(ordery + 1))
    z = np.zeros_like(x)
    for a, (i, j) in zip(coefs.flatten(), ij):
        if i + j < np.max((orderx, ordery)):
            #        print a,i,j
            z += a * x ** i * y ** j
    return z

def gauss_poly1(x, p0, p1, a, b, c):
    return p0 + p1 * x + a/np.sqrt(2.*np.pi)/c * np.exp(-0.5*((x - b) / c) ** 2.)

def gauss_poly0(x, p0, a, b, c):
    return p0 + a/np.sqrt(2.*np.pi)/c * np.exp(-0.5 * ((x - b) / c) ** 2.)

def gauss(x, a, b, c):
    return a/np.sqrt(2.*np.pi)/c * np.exp(-0.5 * ((x - b) / c) ** 2.)

def residual_chi2(coefs, x, y, z, w, poly_order):
    return np.nansum(residual(coefs, x, y, z, w, poly_order) ** 2.)

def residual_lar(coefs, x, y, z, w, poly_order):
    fitted = polyval2d(x, y, coefs, poly_order)
    return np.sqrt(np.abs((fitted - z) * w))

def residual(coefs, x, y, z, w, poly_order):
    fitted = polyval2d(x, y, coefs, poly_order)
    return (fitted - z) * w

# ############################## #
#      standardization
# ############################## #

def standardize_inverse(x, xmean, xstd):
    return (x * xstd) + xmean

def standardize(x):
    return (x - np.nanmean(x)) / np.nanstd(x), np.nanmean(x), np.nanstd(x)

# ############################## #
#      fit grating equation
# ############################## #

def fit_grating_equation(lc_coord, lc_order, lc_thar, popt, pcov, ind_good_thar0=None,
                         poly_order=(3, 5), max_dev_threshold=100, n_iter=400, lar=False,
                         nl_eachorder=10):
    # pick good thar lines
    if ind_good_thar0 is None:
        ind_good_thar0 = np.ones_like(lc_coord, dtype=bool)
    try:
        ind_good_thar = ind_good_thar0 * np.isfinite(lc_coord) * \
                        (popt[:, 3] < 1.0) * (pcov[:, 2] < 1.0)
    except:
        ind_good_thar = np.ones_like(lc_coord, dtype=bool)

    lc_coord = lc_coord[ind_good_thar]
    lc_order = lc_order[ind_good_thar]
    lc_thar = lc_thar[ind_good_thar]

    # standardization
    lc_coord_s, lc_coord_mean, lc_coord_std = standardize(lc_coord)
    lc_order_s, lc_order_mean, lc_order_std = standardize(lc_order)
    ml_s, ml_mean, ml_std = standardize(lc_thar * lc_order)
    # scaler
    scaler_coord = lc_coord_mean, lc_coord_std
    scaler_order = lc_order_mean, lc_order_std
    scaler_ml = ml_mean, ml_std

    # weight
    weight = popt[ind_good_thar, 1] / popt[ind_good_thar, 0]
    weight = weight > 0

    # fit surface
    x0 = np.zeros(poly_order)
    #print(lc_coord_s, lc_order_s, ml_s, weight, poly_order)
    x0, ier = leastsq(residual, x0,
                      args=(lc_coord_s, lc_order_s, ml_s, weight, poly_order))
    #print(x0, ier)

    # iter
    if n_iter > 0:
        n_loop = 0
        while n_loop < n_iter:
            n_loop += 1
            if lar:
                x_mini_lsq, ier = leastsq(residual_lar, x0, args=(
                    lc_coord_s, lc_order_s, ml_s, weight, poly_order))
            else:
                x_mini_lsq, ier = leastsq(residual, x0, args=(
                    lc_coord_s, lc_order_s, ml_s, weight, poly_order))
            fitted = polyval2d(lc_coord_s, lc_order_s, x_mini_lsq, poly_order)
            fitted_wave = standardize_inverse(fitted, ml_mean, ml_std) / lc_order
            fitted_wave_diff = fitted_wave - lc_thar

            wave_dev = np.abs(fitted_wave_diff - np.nanmedian(fitted_wave_diff))
            lc_order_s_unique, ind_inverse = np.unique(lc_order_s, return_inverse=True)
            lc_order_s_unique_left = np.zeros_like(lc_order_s_unique, dtype=int)
            for i, _ in enumerate(lc_order_s_unique):
                lc_order_s_unique_left[i] = np.sum(weight[lc_order_s == _] > 0)
            lc_order_s_left = lc_order_s_unique_left[ind_inverse]
            # print(lc_order_s_left)
            # print(wave_dev)
            # print(lc_order_s_left)

            possible_outlier = wave_dev * (wave_dev > max_dev_threshold) * (lc_order_s_left > nl_eachorder)
            if np.any(possible_outlier > 0):
                ind_max_dev = np.nanargmax(possible_outlier)
                weight[ind_max_dev] = 0.
                ml_s[ind_max_dev] = np.nan
                lc_thar[ind_max_dev] = np.nan
                print("@Cham: [n_loop = %s] max_dev = %s" % (
                    n_loop, fitted_wave_diff[ind_max_dev]))
            else:
                if n_loop == 0:
                    print("@Cham: no points cut in iterarions ...")
                break
    else:
        if lar:
            x_mini_lsq, ier = leastsq(residual_lar, x0, args=(
                lc_coord_s, lc_order_s, ml_s, weight, poly_order))
        else:
            x_mini_lsq, ier = leastsq(residual, x0, args=(
                lc_coord_s, lc_order_s, ml_s, weight, poly_order))
        fitted = polyval2d(lc_coord_s, lc_order_s, x_mini_lsq, poly_order)
        fitted_wave = standardize_inverse(fitted, ml_mean, ml_std) / lc_order
        fitted_wave_diff = fitted_wave - lc_thar

        ind_kick = np.abs(fitted_wave_diff - np.nanmedian(fitted_wave_diff)) > max_dev_threshold
        weight[ind_kick] = 0.
        ml_s[ind_kick] = np.nan
        lc_thar[ind_kick] = np.nan
        x_mini_lsq, ier = leastsq(residual_lar, x0, args=(
            lc_coord_s, lc_order_s, ml_s, weight, poly_order))

    ind_good_thar = np.where(ind_good_thar)[0][np.isfinite(lc_thar)]
    print("@SONG: RMS = {0} | n_points = {1} ".format(
        np.sqrt(np.nanmean(np.square(fitted_wave - lc_thar))),
        np.sum(np.isfinite(lc_thar)))
    )

    return x_mini_lsq, ind_good_thar, scaler_coord, scaler_order, scaler_ml

def grating_equation_predict(grid_coord, grid_order, x_mini_lsq, poly_order,
                             scaler_coord, scaler_order, scaler_ml):
    sgrid_coord = (grid_coord - scaler_coord[0]) / scaler_coord[1]
    sgrid_order = (grid_order - scaler_order[0]) / scaler_order[1]

    sgrid_fitted = polyval2d(sgrid_coord.flatten(), sgrid_order.flatten(),
                             x_mini_lsq, poly_order)
    sgrid_fitted_wave = (sgrid_fitted * scaler_ml[1] + scaler_ml[0]) / \
                        grid_order.flatten()
    sgrid_fitted_wave = sgrid_fitted_wave.reshape(grid_coord.shape)
    return sgrid_fitted_wave

def polyfit_reject(x, y, deg=1, w=None, epsilon=0.002, n_reserve=-1):

    x, y = np.array(x), np.array(y)
    ind_good = np.ones_like(x, bool)
    x_, y_ = x[ind_good], y[ind_good]

    if w is None:
        w = np.ones_like(x, float)
    w_ = np.array(w)[ind_good]

    n_good = np.sum(ind_good)
    if isinstance(n_reserve, float):
        # fraction
        assert 0 < n_reserve < 1
        n_reserve = np.int(n_good * n_reserve)

    # set up the iteration
    ind_reserved = np.ones_like(x_, bool)
    while np.sum(ind_reserved) > n_reserve:
        # p0 = np.polyfit(x_[ind_reserved], y_[ind_reserved], deg, w=w_[ind_reserved])
        p0, ier = leastsq(polyfit_costfun_lar, np.zeros((deg+1,), float), args=(x_[ind_reserved], y_[ind_reserved], w_[ind_reserved]))

        y_res = y_ - np.polyval(p0, x_)
        y_res = np.where(w_ > 0, y_res, np.nan)
        # y_res_std = np.nanstd(y_res)
        # y_res_med = np.nanmedian(y_res)
        # y_res_scaled = (y_res>0) * (y_res/y_res_std/sigma_reject[0]) + (y_res<=0) * (y_res/y_res_std/sigma_reject[1])
        ind_bad = np.abs(y_res) > epsilon

        if np.any(ind_bad[ind_reserved]):
            # print(np.sum(ind_bad[ind_reserved]))
            i_rejected = np.nanargmax(np.abs(y_res))
            # print(i_rejected, np.abs(y_res)[i_rejected])
            ind_reserved[i_rejected] = False
            w_[i_rejected] = 0
            # print("@SONG: {0} points left".format(np.sum(ind_reserved)))
        else:
            break
    p = np.polyfit(x_[ind_reserved], y_[ind_reserved], deg, w=w_[ind_reserved])
    return p, w_>0

def polyfit_costfun_lar(p, x, y, w):
    return np.sqrt(np.abs(np.polyval(p, x) - y)) * w

def polyfit_costfun(p, x, y, w):
    return (np.polyval(p, x) - y) * w

def clean_thar_polyfit1d_reject(lc_coord, lc_order, lc_thar, popt,
                                ind_good0=None, deg=1, w=None,
                                epsilon=0.004, n_reserve=3):
    if ind_good0 is not None:
        lc_coord[~ind_good0] = np.nan
    print("@TWODSPEC: number of finite line centers: {}".format(
        np.sum(np.isfinite(lc_coord))))

    sub_good = np.zeros((0,), int)
    for i_order in np.unique(lc_order)[:]:
        this_ind = lc_order == i_order
        this_sub = np.where(this_ind)[0]
        this_lc_coord = lc_coord[this_ind]
        # this_lc_order = lc_order[this_ind]
        this_lc_thar = lc_thar[this_ind]

        x, y, z = this_lc_coord, popt[this_ind, 2] - this_lc_thar, this_lc_thar
        ind_use = np.isfinite(x) * np.isfinite(y)
        x, y, z = x[ind_use], y[ind_use], z[ind_use]

        p, ind_reserved = polyfit_reject(x, y, deg, w, epsilon=epsilon, n_reserve=n_reserve)
        sub_good = np.hstack((sub_good, this_sub[ind_use][ind_reserved]))
        # print(len(this_sub[ind_use][ind_reserved]))

    ind_good = np.zeros_like(lc_coord, bool)
    ind_good[sub_good] = True
    return ind_good

max_stars_repo_name: hypergravity/songcn

max_issues_repo_name: hypergravity/songcn

max_forks_repo_name: hypergravity/songcn

reason: import numpy,from scipy,from astropy