text:
from torch.utils.data import DataLoader
import numpy as np
from sklearn.cluster import DBSCAN
import torch
from torch.nn import functional as F
from itertools import chain

from torch.nn import DataParallel
from torch.nn.parallel.scatter_gather import scatter, gather
from torch.nn.parallel.replicate import replicate
from torch.nn.parallel.parallel_apply import parallel_apply
from .dataset import imgdataset, imgdataset_cam

class CamDataParallel(DataParallel):
    def forward(self, *inputs, **kwargs):
        if not self.device_ids:
            return self.module(*inputs, **kwargs)

        for t in chain(self.module.parameters(), self.module.buffers()):
            if t.device != self.src_device_obj:
                raise RuntimeError("module must have its parameters and buffers "
                                   "on device {} (device_ids[0]) but found one of "
                                   "them on device: {}".format(self.src_device_obj, t.device))

        all_inputs = inputs[0]
        all_kwargs = kwargs
        all_outputs = []

        while len(all_inputs) > 0:
            num_required_gpu = min(len(all_inputs), len(self.device_ids))
            actual_inputs = [all_inputs.pop(0) for _ in range(num_required_gpu)]
            inputs, kwargs = self.scatter(actual_inputs, all_kwargs, self.device_ids[:num_required_gpu])
            replicas = self.replicate(self.module, self.device_ids[:num_required_gpu])
            all_outputs.extend(self.parallel_apply(replicas, inputs, kwargs))

        return self.gather(all_outputs, self.output_device)

    def replicate(self, module, device_ids):
        return replicate(module, device_ids)

    def scatter(self, input_list, kwargs, device_ids):
        inputs = []
        for input, gpu in zip(input_list, device_ids):
            inputs.extend(scatter(input, [gpu], dim=0))
        kwargs = scatter(kwargs, device_ids, dim=0) if kwargs else []
        if len(inputs) < len(kwargs):
            inputs.extend([() for _ in range(len(kwargs) - len(inputs))])
        elif len(kwargs) < len(inputs):
            kwargs.extend([{} for _ in range(len(inputs) - len(kwargs))])
        inputs = tuple(inputs)
        kwargs = tuple(kwargs)
        return inputs, kwargs

    def parallel_apply(self, replicas, inputs, kwargs):
        return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])

    def gather(self, outputs, output_device):
        return gather(outputs, output_device, dim=self.dim)

def organize_data(input_data,camids,input_labels):
    unique_camids = torch.unique(camids).cpu().numpy()
    reorg_data = []
    reorg_labels = []
    for cid in unique_camids:
        current_camid = (camids == cid).nonzero().view(-1)
        if current_camid.size(0) > 1:
            data = torch.index_select(input_data, index=current_camid, dim=0)
            labels = torch.index_select(input_labels, index=current_camid, dim=0)
            reorg_data.append(data)
            reorg_labels.append(labels)
    return reorg_data,torch.cat(reorg_labels,dim=0)

def l2_dist(fea_query, fea_gallery):
    dist = np.zeros((fea_query.shape[0], fea_gallery.shape[0]), dtype = np.float64)
    for i in range(fea_query.shape[0]):
        dist[i, :] = np.sum((fea_gallery-fea_query[i,:])**2, axis=1)
    return dist

def cluster(dist, rho=1.6e-3):

    tri_mat = np.triu(dist, 1)
    tri_mat = tri_mat[np.nonzero(tri_mat)]
    tri_mat = np.sort(tri_mat,axis=None)
    top_num = np.round(rho*tri_mat.size).astype(int)
    eps = tri_mat[:top_num].mean()
    #low eps for training without source domain
    #eps = eps*0.9
    #print('eps in cluster: {:.3f}'.format(eps))
    cluster = DBSCAN(eps=eps,min_samples=1,metric='precomputed', n_jobs=8)
    labels = cluster.fit_predict(dist)

    return labels

def get_info(file_path):
    with open(file_path) as f:
        lines = f.readlines()
        #self.img_list = [os.path.join(dataset_dir, i.split()[0]) for i in lines]
        labels = [int(i.split()[1]) for i in lines]
        cam_ids = [int(i.split()[2]) for i in lines]
        frames = [int(i.split()[3]) for i in lines]

    return labels, cam_ids, frames

def extract_fea_camtrans(model, loader):
    feas = []
    for i, data in enumerate(loader, 1):
        #break
        with torch.no_grad():
            image = data[0].cuda()

            batch_size = image.size(0)
            K = image.size(1)

            image = image.view(image.size(0)*image.size(1), image.size(2), image.size(3), image.size(4))
            #image = Variable(image).cuda()
            out = model(image)
            fea = out[2]
            fea = fea.view(batch_size, K, -1)
            fea = fea.mean(dim=1)
            fea = F.normalize(fea)
            feas.append(fea)

    feas = torch.cat(feas)
    #print('duke_train_feas', feas.size())
    return feas.cpu().numpy()

def extract_fea_test_cbn(model,img_dir, ann_file_path):
    # divide dataset according to cam id
    test_dataset = imgdataset(dataset_dir=img_dir, txt_path=ann_file_path, transformer='test')
    cams = test_dataset.cams
    fea, pid, cid, fid, is_q = [],[],[],[],[]
    for c in cams:
        test_dataset_cam = imgdataset_cam(dataset_dir=img_dir,txt_path = ann_file_path, camid=c, transformer='test')
        test_loader = DataLoader(dataset=test_dataset_cam, batch_size=64, shuffle=True, num_workers=4)
        collect_sim_bn_info(model,test_loader)
        test_loader = DataLoader(dataset=test_dataset_cam, batch_size=64, shuffle=False, num_workers=0)
        for i ,data in enumerate(test_loader):
            inputs, label, cam, frame, is_query = data
            inputs = inputs.cuda()
            feature = None
            for i in range(2):
                if i == 1:
                    inputs = flip_tensor_lr(inputs)
                with torch.no_grad():
                    global_f = model(inputs)[1]

                if feature is None:
                    feature = global_f
                else:
                    feature += global_f
            feature=feature/2
            fea.append(feature)
            pid.extend(label)
            cid.extend(cam)
            fid.extend(frame)
            is_q.extend(is_query)
    fea = torch.cat(fea,0).cpu().numpy()
    pid = np.asarray(pid)
    cid = np.asarray(cid)
    fid = np.asarray(fid)
    is_q= np.asarray(is_q)
    return fea,pid,cid,fid,is_q

def extract_fea_test(model, loader):
    feas = []
    for i, data in enumerate(loader, 1):
        #break
        with torch.no_grad():
            image = data[0].cuda()
            out = model(image)
            fea = out[1]
            feas.append(fea)

    feas = torch.cat(feas)
    #print('duke_train_feas', feas.size())
    return feas.cpu().numpy()

def flip_tensor_lr(img):
    inv_idx = torch.arange(img.size(3) - 1, -1, -1).long().cuda()
    img_flip = img.index_select(3, inv_idx)
    return img_flip

def collect_sim_bn_info(model, dataloader, num=10):
    network_bns = [x for x in list(model.modules()) if
                       isinstance(x, torch.nn.BatchNorm2d) or isinstance(x, torch.nn.BatchNorm1d)]
    for bn in network_bns:
        bn.running_mean = torch.zeros(bn.running_mean.size()).float().cuda()
        bn.running_var = torch.ones(bn.running_var.size()).float().cuda()
        bn.num_batches_tracked = torch.tensor(0).cuda().long()

    model.train()
    for batch_idx, inputs in enumerate(dataloader):
        if batch_idx==num:
            break
        # each camera should has at least 2 images for estimating BN statistics
        assert len(inputs[0].size()) == 4 and inputs[0].size(
            0) > 1, 'Cannot estimate BN statistics. Each camera should have at least 2 images'
        image = inputs[0].cuda()
        for i in range(2):
            if i == 1:
                image = flip_tensor_lr(image)
            with torch.no_grad():
                model(image)
    model.eval()

max_stars_repo_name: FlyingRoastDuck/UnrealPerson

max_issues_repo_name: FlyingRoastDuck/UnrealPerson

max_forks_repo_name: FlyingRoastDuck/UnrealPerson


text:
from functools import reduce
from operator import mul

import numpy as np

import visualization

image_shape = (8, 100, 100)
truth_shape = (6, 80, 80)
pred_shape = (4, 40, 40)
random_state = np.random.RandomState(seed=0)

dataset = dict(
    data=random_state.randint(0, 256, reduce(mul, image_shape)).reshape(image_shape),
    label=random_state.randint(0, 2, 3 * reduce(mul, truth_shape)).reshape((3,) + truth_shape),
    components=random_state.randint(0, 5, reduce(mul, truth_shape)).reshape(truth_shape),
    pred=random_state.uniform(0, 1, 3 * reduce(mul, pred_shape)).reshape((3,) + pred_shape),
    predseg=random_state.randint(0, 5, reduce(mul, pred_shape)).reshape(pred_shape),
)

z = 2

f = visualization.showme(dataset, z)

f.savefig('test.png')

ext: py

max_stars_repo_name: Rekrau/PyGreentea

max_issues_repo_name: Rekrau/PyGreentea

max_forks_repo_name: Rekrau/PyGreentea

reason: import numpy


text:
import pytest
import numpy as np
import astropy.units as u

from dust_extinction.parameter_averages import CCM89, O94, F99, F04, M14, F19

# fmt: off
models_corvals = {
    CCM89: {
        "Rv": 3.1,
        "x_values": [2.78, 2.27, 1.82, 1.43, 1.11, 0.80, 0.63, 0.46],
        "y_values": [1.569, 1.322, 1.000, 0.751, 0.479, 0.282, 0.190, 0.114],
        "atol": 1e-2,
        # values from Table 3 of Cardelli et al. (1989)
        #   ignoring the last value at L band as it is outside the
        #   valid range for the relationship
        #  updated for correction for incorrect value in the table for B band
        #    correction from Geoff Clayton via email
    },
    O94: {
        "Rv": [3.1, 3.1, 2.0, 3.0, 4.0, 5.0, 6.0],
        "x_values": [[2.939, 2.863, 2.778, 2.642, 2.476, 2.385, 2.275, 2.224, 2.124, 2.000,
                      1.921, 1.849, 1.785, 1.718, 1.637, 1.563, 1.497, 1.408, 1.332, 1.270],
                     [10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.6, 4.0, 0.8, 0.63, 0.46],
                     [10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.6, 4.0, 0.8, 0.63, 0.46],
                     [10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.6, 4.0, 0.8, 0.63, 0.46],
                     [10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.6, 4.0, 0.8, 0.63, 0.46],
                     [10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.6, 4.0, 0.8, 0.63, 0.46],
                     [10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.6, 4.0, 0.8, 0.63, 0.46]],
        # convert the table values of E(l-1.5)/E(2.2-1.5) to A(x)/A(V)
        "y_values": [[(np.array([1.725, 1.651, 1.559, 1.431, 1.292, 1.206, 1.100, 1.027, 0.907, 0.738,
                                 0.606, 0.491, 0.383, 0.301, 0.190, 0.098, -0.004, -0.128, -0.236, -0.327])
                       * (1.27992402 - 0.79848375)) + 0.79848375],
                     [5.23835484, 4.13406452, 3.33685933, 2.77962453, 2.52195399, 2.84252644,
                      3.18598916, 2.31531711, 0.28206957, 0.19200814, 0.11572348],
                     [9.407, 7.3065, 5.76223881, 4.60825807, 4.01559036, 4.43845534,
                      4.93952892, 3.39275574, 0.21678862, 0.14757062, 0.08894094],
                     [5.491, 4.32633333, 3.48385202, 2.8904508, 2.6124774, 2.9392494,
                      3.2922643, 2.38061642, 0.27811315, 0.18931496, 0.11410029],
                     [3.533, 2.83625, 2.34465863, 2.03154717, 1.91092092, 2.18964643,
                      2.46863199, 1.87454675, 0.30877542, 0.21018713, 0.12667997],
                     [2.3582, 1.9422, 1.66114259, 1.51620499, 1.48998704, 1.73988465,
                      1.97445261, 1.57090496, 0.32717278, 0.22271044, 0.13422778],
                     [1.575, 1.34616667, 1.20546523, 1.17264354, 1.20936444, 1.44004346,
                      1.64499968, 1.36847709, 0.33943769, 0.23105931, 0.13925965]],
        "atol": [6e-2, 1e-6, 1e-6, 1e-6, 1e-6, 1e-6, 1e-6],
        # 1st set of values from Bastiaansen (1992) Table 6 (testing versus that average)
    },
    F99: {
        "Rv": 3.1,
        "x_values": [0.377, 0.820, 1.667, 1.828, 2.141, 2.433, 3.704, 3.846],
        "y_values": np.array([0.265, 0.829, 2.688, 3.055, 3.806, 4.315, 6.265, 6.591]) / 3.1,
        "atol": 1e-3,
        # from Fitzpatrick (1999) Table 3
    },
    F04: {
        "Rv": 3.1,
        "x_values": [0.377, 0.820, 1.667, 1.828, 2.141, 2.433, 3.704, 3.846],
        "y_values": np.array([0.185, 0.772, 2.688, 3.055, 3.805, 4.315, 6.456, 6.781]) / 3.1,
        "atol": 1e-3,
        # from Fitzpatrick (1999) Table 3
        # keep optical from Fitzpatrick (1999),
        # replce NIR with Fitzpatrick (2004) function for Rv=3.1:
        # (0.63*3.1 - 0.84)*x**1.84
    },
    M14: {
        # using R5495 = 3.1
        "Rv": 3.1,
        "x_values": [0.5, 2.0],
        "y_values": [0.1323, 1.141],
        "atol": 1e-3,
    },
    F19: {
        "Rv": 3.1,
        "x_values": [1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0],
        # convert from E(x-V)/E(B-V) to A(x)/A(V)
        "y_values": (np.array([-1.757, -0.629, 0.438, 2.090, 4.139, 5.704, 4.904, 5.684, 7.150]) + 3.1) / 3.1,
        "atol": 1e-3,
        # use x values from Fitzpatrick et al. (2000) Table 3
    },
}
# fmt: on

@pytest.mark.parametrize(
    ("model_class", "test_parameters"),
    sorted(models_corvals.items(), key=lambda x: str(x[0])),
)
def test_corvals(model_class, test_parameters):
    Rv_vals = np.atleast_1d(test_parameters["Rv"])
    for k, Rv in enumerate(Rv_vals):
        tol = np.atleast_1d(test_parameters["atol"])[k]
        if len(Rv_vals) > 1:
            x_vals = test_parameters["x_values"][k]
            y_vals = test_parameters["y_values"][k]
        else:
            x_vals = test_parameters["x_values"]
            y_vals = test_parameters["y_values"]
        x_vals = np.array(x_vals).flatten() / u.micron
        y_vals = np.array(y_vals).flatten()

        # instantiate extinction model
        tmodel = model_class(Rv=Rv)

        # test array evaluation
        np.testing.assert_allclose(tmodel(x_vals), y_vals, atol=tol)

        # test single value evalutation
        for x, y in zip(x_vals, y_vals):
            np.testing.assert_allclose(tmodel(x), y, atol=tol)
            np.testing.assert_allclose(tmodel.evaluate(x, Rv), y, atol=tol)

reason: import numpy,import astropy


text:
from sklearn.datasets import make_classification, make_regression
from sklearn.metrics import log_loss
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import StandardScaler
import numpy as np
from ..mlp import MLP

class TestMLP(object):

    def __init__(self):
        self.X_cl, self.y_cl = make_classification(100)
        self.X_mcl, self.y_mcl = make_classification(
            1000, n_informative=5, n_classes=3)
        self.X_re, self.y_re = make_regression(100)

    def test_if_fit_classification(self):
        model = MLP()
        model.fit(self.X_cl, self.y_cl)
        assert(model.type_of_target_ == 'binary')

    def test_if_fit_regression(self):
        model = MLP()
        model.fit(self.X_re, self.y_re)
        assert(model.type_of_target_ == 'continuous')

    def test_sigmoid(self):
        model = MLP(hidden_layer_type='sigm')
        model.fit(self.X_cl, self.y_cl)

    def test_dropout(self):
        model = MLP(max_iter=100, dropout='True')
        model.fit(self.X_re, self.y_re)
        linear = LinearRegression()
        linear.fit(model.predict(self.X_re).reshape(-1, 1),
                   self.y_re.reshape(-1, 1))
        assert(model.type_of_target_ == 'continuous')
        assert (np.abs(1 - linear.coef_[0]) < 0.05)
        assert (np.abs(linear.intercept_) < 0.05)

    def test_accuracy(self):
        model = MLP()
        model.fit(self.X_cl[:50], self.y_cl[:50])
        y_pred = model.predict_proba(self.X_cl[50:])
        ll = log_loss(self.y_cl[50:], y_pred)
        assert(ll < .05)

    def test_multiclass(self):
        model = MLP()
        model.fit(self.X_mcl, self.y_mcl)

max_stars_repo_name: JakeMick/graymatter

max_issues_repo_name: JakeMick/graymatter

max_forks_repo_name: JakeMick/graymatter


text:
#-*-coding:utf-8-*-
import numpy as np
import pandas as pd
import scipy as sp
import scipy.special as special
import random

#平滑，这个快一点；
# hyper=HyperParam(1, 1);
# hyper.update_from_data_by_moment(show, click)
class HyperParam(object):
    def __init__(self, alpha, beta):
        self.alpha = alpha
        self.beta = beta

    def sample_from_beta(self, alpha, beta, num, imp_upperbound):
        sample = numpy.random.beta(alpha, beta, num)
        I = []
        C = []
        for click_ratio in sample:
            imp = random.random() * imp_upperbound
            #imp = imp_upperbound
            click = imp * click_ratio
            I.append(imp)
            C.append(click)
        return I, C

    def update_from_data_by_FPI(self, tries, success, iter_num, epsilon):
        '''estimate alpha, beta using fixed point iteration'''
        for i in range(iter_num):
            new_alpha, new_beta = self.__fixed_point_iteration(tries, success, self.alpha, self.beta)
            if abs(new_alpha-self.alpha)<epsilon and abs(new_beta-self.beta)<epsilon:
                break
            self.alpha = new_alpha
            self.beta = new_beta

    def __fixed_point_iteration(self, tries, success, alpha, beta):
        '''fixed point iteration'''
        sumfenzialpha = 0.0
        sumfenzibeta = 0.0
        sumfenmu = 0.0
        for i in range(len(tries)):
            sumfenzialpha += (special.digamma(success[i]+alpha) - special.digamma(alpha))
            sumfenzibeta += (special.digamma(tries[i]-success[i]+beta) - special.digamma(beta))
            sumfenmu += (special.digamma(tries[i]+alpha+beta) - special.digamma(alpha+beta))

        return alpha*(sumfenzialpha/sumfenmu), beta*(sumfenzibeta/sumfenmu)

    def update_from_data_by_moment(self, tries, success):
        '''estimate alpha, beta using moment estimation'''
        mean, var = self.__compute_moment(tries, success)
        self.alpha = (mean+0.000001) * ((mean+0.000001) * (1.000001 - mean) / (var+0.000001) - 1)

        self.beta = (1.000001 - mean) * ((mean+0.000001) * (1.000001 - mean) / (var+0.000001) - 1)

    def __compute_moment(self, tries, success):
        '''moment estimation'''
        ctr_list = []
        var = 0.0

        for i in range(len(tries)):

            ctr_list.append(float(success[i])/float(tries[i]+0.0000001))
        mean = sum(ctr_list)/len(ctr_list)
        for ctr in ctr_list:
            var += pow(ctr-mean, 2)

        return mean, var/(len(ctr_list)-1)

class BayesianSmoothing(object):
    def __init__(self, alpha, beta):
        self.alpha = alpha
        self.beta = beta

    def sample(self, alpha, beta, num, imp_upperbound):
        sample = np.random.beta(alpha, beta, num)
        I = []
        C = []
        for clk_rt in sample:
            imp = random.random() * imp_upperbound
            imp = imp_upperbound
            clk = imp * clk_rt
            I.append(imp)
            C.append(clk)
        return I, C

    def update(self, imps, clks, iter_num, epsilon):
        for i in range(iter_num):
            print('~~~~~~~~~~~~~~%d~~~~~~~~~~~~~~~~' %(i))
            new_alpha, new_beta = self.__fixed_point_iteration(imps, clks, self.alpha, self.beta)
            if abs(new_alpha - self.alpha) < epsilon and abs(new_beta - self.beta) < epsilon:
                break
            self.alpha = new_alpha
            self.beta = new_beta

    def __fixed_point_iteration(self, imps, clks, alpha, beta):
        numerator_alpha = 0.0
        numerator_beta = 0.0
        denominator = 0.0

        for i in range(len(imps)):
            numerator_alpha += (special.digamma(clks[i] + alpha) - special.digamma(alpha))
            numerator_beta += (special.digamma(imps[i] - clks[i] + beta) - special.digamma(beta))
            denominator += (special.digamma(imps[i] + alpha + beta) - special.digamma(alpha + beta))

        return alpha * (numerator_alpha / denominator), beta * (numerator_beta / denominator)

def bs_utilize(count_all_column, count_1_column):

    hyper = HyperParam(1, 1);
    hyper.update_from_data_by_moment(count_all_column.values, count_1_column.values)

    smoothed = (count_1_column + hyper.alpha) / (count_all_column + hyper.alpha + hyper.beta)

    return smoothed

if __name__ == '__main__':
    pass

reason: import numpy,import scipy


text:
"""Reads the text data stored as sparse matrix."""
import numpy as np
import scipy.sparse as sp
from sklearn.model_selection import train_test_split

def removeFirstColumn(data):
    new_data = []
    for i in range(len(data)):
        new_data.append(data[i][1:])
    new_data = np.array(new_data)
    return new_data

def LoadData(inputfile):
    print("read")
    print("loaded")
    data = np.load(inputfile, mmap_mode='r')
    print("loaded")
    print(type(data))
    print(data.shape)
    return data

def DivideTrainTestFiles():

    print("Reading Sentiment Training")
    orig_folder = "city_search_bert_run"
    orig_path = "./data_text_embedding/"+orig_folder+"/sentiment_features_train_new.npy"
    print("orig_path ",orig_path)
    sent_train = LoadData(orig_path)
    #print(sent_train.shape)
    folder = "Asp_Sent_joint_city_search_bert"
    text_unlabled = "./data/text/unlabelled/"
    sent_unlabled = "./data/image/unlabelled/"

    '''
    print("Reading Text Train")
    text_train = LoadData('./data_text_embedding/'+orig_folder+'/text_features_train_new.npy')
    print(text_train.shape)
    print("Writing text unlabled")
    file_count = 0
    file_name = "text_unlabelled-" + str(file_count) + ".npy"
    start = 0
    for i in range(len(text_train)):
        if i % 5000 == 0 and i != 0:
            file_path = text_unlabled + file_name
            np.save(file_path, text_train[start:i])
            start = i
            file_count += 1
            file_name = "text_unlabelled-" + str(file_count) + ".npy"

    file_count += 1
    file_name = "text_unlabelled-" + str(file_count) + ".npy"
    file_path = text_unlabled + file_name
    np.save(file_path, text_train[start:len(text_train) - 1])
    '''
    #'''
    print("Writing sentiment unlabled")
    file_count = 0
    file_name = "sent_unlabelled-" + str(file_count) + ".npy"
    start = 0
    for i in range(len(sent_train)):
        if i ==150000:
            break
        if i % 5000 == 0 and i != 0:
            file_path = sent_unlabled + file_name
            np.save(file_path, sent_train[start:i])
            start = i
            file_count += 1
            file_name = "sent_unlabelled-" + str(file_count) + ".npy"

    file_count += 1
    file_name = "sent_unlabelled-" + str(file_count) + ".npy"
    file_path = sent_unlabled + file_name
    np.save(file_path, sent_train[start:len(sent_train) - 1])
    #'''

if __name__ == '__main__':
    DivideTrainTestFiles()

max_stars_repo_name: yassienshaalan/DTOPS

max_issues_repo_name: yassienshaalan/DTOPS

max_forks_repo_name: yassienshaalan/DTOPS


text:
from jax import grad, vmap, jit

import jax.scipy.stats as jsps
import jax.numpy as jnp
import jax.scipy as jsp
import jax.scipy.stats as jsps

import matplotlib.animation as animation
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns

import numpy as np

from gmm import sample_gmm, gmm_gld, gmm_pdf_vectorized
from svgd import SVGD

def make_video(frames):
    fig = plt.figure()

    ani = animation.ArtistAnimation(fig, frames, interval=0.5)
    plt.show()
    ani.save('movie.mp4')
    plt.close()

def get_plotfunc(true_samples):
    def plotfunc(l, particles):
        fig, ax = plt.subplots(1,1)

        kde1 = sns.kdeplot(true_samples.flatten(), ax=ax, label='Sampler')
        kde2 = sns.kdeplot(particles.flatten(), ax=ax, label='SVGD')

        ax.legend()

        plt.savefig('./plots/toy/toy_{}.png'.format(l))
        plt.close()

    return plotfunc

if __name__ == '__main__':
    num_iterations = 5000
    num_samples = 100

    weights = [1./3, 2./3]
    mus = [np.array([-2]),
           np.array([2])]
    sigmas = [np.eye(1),
              np.eye(1)]

    toy_init_particles = sample_gmm(num_samples,
                                    [1.],
                                    [np.array([-10])],
                                    [np.eye(1)])

    true_samples = sample_gmm(num_samples, weights, mus, sigmas)

    plotfunc = get_plotfunc(true_samples)

    toy_score = gmm_gld(weights, mus, sigmas)

    svgd = SVGD(gld=toy_score)

    particles= svgd.do_svgd_iterations_optimized(init_particles=toy_init_particles,
                                         num_iterations=num_iterations,
                                                  learning_rate=0.1,
                                                  plotfunc=plotfunc)

    x = np.linspace(-15, 10, 1000)
    fig, ax = plt.subplots(1,1)

    sns.kdeplot(true_samples.flatten(), ax=ax, label='Sampler')
    sns.kdeplot(particles.flatten(), ax=ax, label='SVGD')

    ax.legend()

    #plt.show()
    plt.close()

reason: import numpy,import jax,from jax


text:
import numpy as np
import os
import argparse
import sys
caffe_path = '/home/yz/caffe3/'
sys.path.append(caffe_path+'python')
import caffe
import build_net as b
import os
os.environ["GLOG_minloglevel"] = "2"

def net7(n):
    n.add_image(batch_size = 512, test_batch_size = 32,
        transformer_dict = dict(mirror = False,
        mean_value=128, scale=0.00390625,
        contrast_jitter_range=0.3
        ),
        label_scale=1,
        root_folder='', is_color=True, shuffle=True, test_shuffle=True,
        source_train='train_caffe.txt', source_test='val_caffe.txt',
        height = 192, width = 256)
    bins = [0.01, 0.03, 0.05, 0.075, 0.1, 0.15,
            0.2, 0.3, 0.4, 0.5, 0.7,
            1, 2, 3, 4, 5, 6]
    reversebins = list(bins)
    reversebins.reverse()
    bins = [-x for x in reversebins]+bins
    print('bins:',bins)
    n.add_conv(32, kernel_size=7, stride=2, pad=3, bias_term=True)
    for numoutput in [32, 64, 128, 256, 512]:
        n.add_conv(numoutput, bias_term=True)
        n.add_conv(numoutput, stride=2, bias_term=True)
    n.add_fc(len(bins)+1)
    n.add_softmax_decay(bins, decay_rate = 2)
    n.add_solver_sdg(test_interval = 100, test_iter = 16,
                max_iter = 4e4, base_lr = 0.001, momentum = 0.9,
                weight_decay = 1e-5, gamma = 0.1, stepsize = 1e4,
                display = 10, snapshot = 5e3, iter_size = 1)

net = b.BuildNet(net7, name = 'net7', caffe_path = caffe_path)
net.save()

max_stars_repo_name: yao-zhao/udacity-steering-prediction

max_issues_repo_name: yao-zhao/udacity-steering-prediction

max_forks_repo_name: yao-zhao/udacity-steering-prediction


text:
from torch.optim import Optimizer
import math
import torch
import numpy as np
import matplotlib
import matplotlib.cm as cm
import matplotlib.pyplot as plt

class Optim(Optimizer):

    def __init__(self, params, defaults):
        super(Optim, self).__init__(params, defaults)

    def step(self, closure=None):
        """Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        """
        loss = None
        if closure is not None:
            loss = closure()

        for group in self.param_groups:
            for p in group['params']:
                if p.grad is None:
                    continue
                self.my_step(p, self.state[p], group)
        return loss

    def my_step(self, p, state, group):
      raise NotImplementedError

plt.ioff() #(interactive mode)

A = torch.tensor([[1.0, 0.0,], [0.0, 5.0]])
b = torch.tensor([0.0, 0.0])

def objective(x, y):
    xy = torch.tensor([x, y])
    return (0.5 * xy @ (A @ xy) + b @ xy).item()

delta = 0.025
x = np.arange(-0.5, 1.5, delta)
y = np.arange(-8, 1.8, delta)
X, Y = np.meshgrid(x, y)
Z = np.copy(X)
for i in range(X.shape[0]):
    for j in range(X.shape[1]):
        Z[i, j] = objective(X[i,j], Y[i,j])

w = torch.tensor([1.0, 1.0])
w.requires_grad_()

def output(opt, nsteps = 10, noise=0.0, fname="example.png"):
    torch.manual_seed(42)
    w.data[0] = 1.0
    w.data[1] = 1.0

    xopt = []
    yopt = []

    for i in range(nsteps):
        xopt.append(w[0].item())
        yopt.append(w[1].item())

        def obj():
            if noise > 0.0:
                bnoise = torch.normal(b, noise)
            else:
                bnoise = b
            loss = 0.5 * w @ (A @ w) + bnoise @ w
            opt.zero_grad()
            loss.backward()
            return loss

        opt.step(closure=obj)

    fig, ax = plt.subplots(figsize=(3,6))
    plt.axis('equal')
    CS = ax.contour(X, Y, Z, levels=[0.01 , 0.05, 0.1, 0.5, 1, 2, 3, 4, 5, 6])
    ax.clabel(CS, fontsize=7)

    ax.plot(0, 0, '.', color='blue')
    ax.plot(xopt, yopt, '-', color='red')
    ax.plot(xopt, yopt, '.', color='black')
    plt.ylim([-1.5, 1.5])
    plt.xlim([-0.6, 1.1])

    # fig.savefig(fname, bbox_inches='tight', pad_inches=0)
    fig.canvas.draw()
    plt.show()


text:
from functools import wraps

import numpy as np

from aesara import Constant
from aesara import scalar as scal
from aesara import tensor
from aesara.gof import local_optimizer
from aesara.gof.op import Op
from aesara.gof.opt import copy_stack_trace, inherit_stack_trace
from aesara.gpuarray.basic_ops import (
    GpuAllocEmpty,
    GpuFromHost,
    GpuReshape,
    HostFromGpu,
    host_from_gpu,
)
from aesara.gpuarray.elemwise import GpuDimShuffle, GpuElemwise
from aesara.gpuarray.type import GpuArrayType, get_context, move_to_gpu
from aesara.tensor import DimShuffle, NotScalarConstantError, get_scalar_constant_value

# Define a few operations to use in optimizations,
# in order to avoid introducin new CPU Ops, or useless ones.
def safe_to_gpu(x, ctx_name):
    if isinstance(x.type, tensor.TensorType):
        return GpuFromHost(ctx_name)(x)
    else:
        return x

def safe_to_cpu(x):
    if isinstance(x.type, GpuArrayType):
        return x.transfer("cpu")
    else:
        return x

def grab_cpu_scalar(v, nd):
    """
    Get a scalar variable value from the tree at `v`.

    This function will dig through transfers and dimshuffles to get
    the constant value. If no such constant is found, it returns None.

    Parameters
    ----------
    v
        Aesara variable to extract the constant value from.
    nd : int
        Expected number of dimensions for the variable (for
        broadcasted constants).

    """
    if v.owner is not None:
        n = v.owner
        if (
            isinstance(n.op, (GpuDimShuffle, DimShuffle))
            and n.op.new_order == ("x",) * nd
        ):
            return grab_cpu_scalar(n.inputs[0], n.inputs[0].ndim)
        elif isinstance(n.op, (GpuFromHost, HostFromGpu)):
            return grab_cpu_scalar(n.inputs[0], nd)
        else:
            return None
    else:
        if isinstance(v, Constant) and v.broadcastable == (True,) * nd:
            return v.dimshuffle(())

def find_node(v, cls, ignore_clients=False):
    """
    Find the node that has an op of of type `cls` in `v`.

    This digs through possibly redundant transfers to for the node
    that has the type `cls`. If `ignore_clients` is False (the
    default) it will only dig through nodes that have a single client
    to avoid duplicating computations.

    Parameters
    ----------
    v
        The variable to dig through
    cls : Op class
        The type of the node we are looking for
    ignore_clients : bool, optional
        Whether to ignore multiple clients or not.

    """
    if v.owner is not None and (ignore_clients or len(v.clients) == 1):
        if isinstance(v.owner.op, cls):
            return v.owner
        elif (
            isinstance(v.owner.op, GpuFromHost)
            and v.owner.inputs[0].owner is not None
            and (ignore_clients or len(v.owner.inputs[0].clients) == 1)
            and isinstance(v.owner.inputs[0].owner.op, HostFromGpu)
        ):
            return find_node(v.owner.inputs[0].owner.inputs[0], cls)
        else:
            return None

def is_equal(var, val):
    """
    Returns True if `var` is always equal to `val`.

    This will only return True if the variable will always be equal to
    the value.  If it might not be true in some cases then it returns False.

    Parameters
    ----------
    var
        Variable to compare
    val
        Python value

    """
    try:
        v = get_scalar_constant_value(var)
        return v == val
    except NotScalarConstantError:
        return False

def alpha_merge(cls, alpha_in, beta_in):
    """
    Decorator to merge multiplication by a scalar on the output.

    This will find a pattern of `scal * <yourop>(some, params, alpha,
    beta)` and update it so that the scalar multiplication happens as
    part of your op.

    The op needs to accept an alpha and a beta scalar which act this way::

       out = Op() * alpha + out_like * beta

    Where out_like is a buffer that has the same size as the output
    and gets added to the "real" output of the operation.  An example
    of an operation that respects this pattern is GEMM from blas.

    The decorated function must have this signature::

        maker(node, *inputs)

    The `node` argument you receive is the original apply node that
    contains your op.  You should use it to grab relevant properties
    for your op so that the new version performs the same computation.
    The `*inputs` parameters contains the new inputs for your op.  You
    MUST use those inputs instead of the ones on `node`.  Note that
    this function can be as simple as::

        def maker(node, *inputs):
            return node.op(*inputs)

    Parameters
    ----------
    cls : op class
        The class of the op you want to merge
    alpha_in : int
        The input index for the alpha scalar for your op (in node.inputs).
    beta_in : int
        The input index for the beta scalar for your op (in node.inputs).

    Returns
    -------
    local optimizer
        an unregistered local optimizer that has the same name as the
        decorated function.

    Notes
    -----
    This was factored out since the code to deal with intervening
    transfers and correctness in the presence of different values of
    alpha and beta scaling factors is not trivial.

    """

    def wrapper(maker):
        @local_optimizer([GpuElemwise])
        @wraps(maker)
        def opt(node):
            if (
                isinstance(node.op, GpuElemwise)
                and node.op.scalar_op == scal.mul
                and node.nin == 2
            ):
                targ = find_node(node.inputs[0], cls)
                if targ is None:
                    targ = find_node(node.inputs[1], cls)
                    if targ is None:
                        return
                    lr = grab_cpu_scalar(node.inputs[0], nd=targ.outputs[0].ndim)
                else:
                    lr = grab_cpu_scalar(node.inputs[1], nd=targ.outputs[0].ndim)
                if lr is None or lr.dtype != targ.outputs[0].dtype:
                    return None
                inputs = list(targ.inputs)
                try:
                    c = get_scalar_constant_value(lr)
                    if c == 0:
                        inputs[alpha_in] = lr
                        inputs[beta_in] = lr
                    elif c == 1:
                        inputs[alpha_in] = targ.inputs[alpha_in]
                        inputs[beta_in] = targ.inputs[beta_in]
                    else:
                        inputs[alpha_in] = lr * targ.inputs[alpha_in]
                        inputs[beta_in] = lr * targ.inputs[beta_in]
                except NotScalarConstantError:
                    inputs[alpha_in] = lr * targ.inputs[alpha_in]
                    inputs[beta_in] = lr * targ.inputs[beta_in]
                with inherit_stack_trace(node.outputs):
                    return maker(targ, *inputs)

        return opt

    return wrapper

def output_merge(cls, alpha_in, beta_in, out_in):
    """
    Decorator to merge addition by a value on the output.

    This will find a pattern of `val * <yourop>(some, params, alpha,
    beta, out_like)` and update it so that the addtition happens as
    part of your op.

    The op needs to accept an alpha and a beta scalar which act this way::

       out = Op() * alpha + out_like * beta

    Where out_like is a buffer that has the same size as the output
    and gets added to the "real" output of the operation.  An example
    of an operation that respects this pattern is GEMM from blas.

    The decorated function must have this signature::

        maker(node, *inputs)

    The `node` argument you receive is the original apply node that
    contains your op.  You should use it to grab relevant properties
    for your op so that the new version performs the same computation.
    The `*inputs` parameters contains the new inputs for your op.  You
    MUST use those inputs instead of the ones on `node`.  Note that
    this function can be as simple as::

        def maker(node, *inputs):
            return node.op(*inputs)

    Parameters
    ----------
    cls : op class
        The class of the op you want to merge
    alpha_in : int
        The input index for the alpha scalar for your op (in node.inputs).
    beta_in : int
        The input index for the beta scalar for your op (in node.inputs).
    out_in : int
        The input index for the out_like input for your op (in node.inputs).

    Returns
    -------
    local optimizer
        an unregistered local optimizer that has the same name as the
        decorated function.

    Notes
    -----
    This was factored out since the code to deal with intervening
    transfers and correctness in the presence of different values of
    alpha and beta scaling factors is not trivial.

    This also correctly handles the case where the added value is
    broadcasted (by not performing the replacement).

    """

    def wrapper(maker):
        @local_optimizer([GpuElemwise])
        @wraps(maker)
        def opt(node):
            if (
                isinstance(node.op, GpuElemwise)
                and node.op.scalar_op == scal.add
                and node.nin == 2
            ):
                targ = find_node(node.inputs[0], cls)
                W = node.inputs[1]
                if targ is None:
                    targ = find_node(node.inputs[1], cls)
                    W = node.inputs[0]
                if targ is None:
                    return None
                if W.dtype != targ.outputs[0].dtype:
                    return None
                if not is_equal(targ.inputs[beta_in], 0.0):
                    # other cases are too complex for now
                    return None
                if W.broadcastable != targ.inputs[out_in].broadcastable:
                    # Would need to explicitly tile the output to fill
                    # the full shape here.  Disable for now.
                    return None
                inputs = list(targ.inputs)
                inputs[out_in] = W
                dtype = inputs[beta_in].dtype
                one = scal.constant(np.asarray(1.0, dtype=dtype))
                inputs[beta_in] = one
                with inherit_stack_trace(node.outputs):
                    return maker(targ, *inputs)

        return opt

    return wrapper

def inplace_allocempty(op, idx):
    """
    Wrapper to make an inplace optimization that deals with AllocEmpty

    This will duplicate the alloc input if it has more than one client
    to allow the op to work on it inplace.

    The decorated function must have this signature::

        maker(node, inputs)

    The `node` argument you receive is the original apply node that
    contains your op.  You should use it to grab relevant properties
    for your op so that the new version performs the same computation.
    You should also switch the op to work inplace.  The `*inputs`
    parameters contains the new inputs for your op.  You MUST use
    those inputs instead of the ones on `node`.  Note that this
    function can be as simple as::

        def maker(node, inputs):
            return [node.op.__class__(inplace=True)(*inputs)]

    Parameters
    ----------
    op : op class
        The op class to look for to make inplace
    idx : int
        The index of the (possibly) AllocEmpty input (in node.inputs).

    Returns
    -------
    local optimizer
        an unregistered inplace local optimizer that has the same name
        as the decorated function.

    """

    def wrapper(maker):
        @local_optimizer([op], inplace=True)
        @wraps(maker)
        def opt(node):
            if type(node.op) != op or node.op.inplace:
                return
            inputs = list(node.inputs)
            alloc = inputs[idx]
            if (
                alloc.owner
                and isinstance(alloc.owner.op, GpuAllocEmpty)
                and len(alloc.clients) > 1
            ):
                alloc_op = GpuAllocEmpty(
                    alloc.owner.op.dtype, alloc.owner.op.context_name
                )
                inputs[idx] = alloc_op(*alloc.owner.inputs)
            with inherit_stack_trace(node.outputs):
                return maker(node, inputs)

        return opt

    return wrapper

def pad_dims(input, leftdims, rightdims):
    """Reshapes the input to a (leftdims + rightdims) tensor

    This helper function is used to convert pooling inputs with arbitrary
    non-pooling dimensions to the correct number of dimensions for the
    GPU pooling ops.

    This reduces or expands the number of dimensions of the input to
    exactly `leftdims`, by adding extra dimensions on the left or by
    combining some existing dimensions on the left of the input.

    Use `unpad_dims` to reshape back to the original dimensions.

    Examples
    --------
    Given input of shape (3, 5, 7), ``pad_dims(input, 2, 2)``
    adds a singleton dimension and reshapes to (1, 3, 5, 7).
    Given that output from pad_dims, ``unpad_dims(output, input, 2, 2)``
    reshapes back to (3, 5, 7).

    Given input of shape (3, 5, 7, 9), ``pad_dims(input, 2, 2)``
    does not reshape and returns output with shape (3, 5, 7, 9).

    Given input of shape (3, 5, 7, 9, 11), ``pad_dims(input, 2, 2)``
    combines the first two dimensions and reshapes to (15, 7, 9, 11).

    Given input of shape (3, 5, 7, 9), ``pad_dims(input, 2, 3)``
    adds a singleton dimension and reshapes to (1, 3, 5, 7, 9).
    """
    assert input.ndim >= rightdims

    if input.ndim == (leftdims + rightdims):
        return input

    # extract image dimensions
    img_shape = input.shape[-rightdims:]

    non_pool_ndim = input.ndim - rightdims
    if non_pool_ndim < leftdims:
        # too few dimensions, pad on the left
        dummy_dims = tensor.as_tensor([1] * (leftdims - non_pool_ndim))
        new_shape = tensor.join(0, dummy_dims, input.shape[:non_pool_ndim], img_shape)
    else:
        # too many dimensions, combine the leading dimensions
        batched_ndim = non_pool_ndim - leftdims + 1
        batch_size = tensor.prod(input.shape[:batched_ndim])
        # convert to a vector for tensor.join
        batch_size = tensor.shape_padright(batch_size, 1)
        new_shape = tensor.join(
            0, batch_size, input.shape[batched_ndim:non_pool_ndim], img_shape
        )

    # store in the required shape
    new_shape = tensor.cast(new_shape, "int64")
    input_ND = GpuReshape(leftdims + rightdims)(input, new_shape)
    return input_ND

def unpad_dims(output, input, leftdims, rightdims):
    """Reshapes the output after pad_dims.

    This reverts the padding by `pad_dims`.
    """
    if output.ndim == input.ndim:
        return output

    # restore the output to the original shape
    outshp = tensor.join(0, input.shape[:-rightdims], output.shape[-rightdims:])
    return GpuReshape(input.ndim)(output, outshp)

def op_lifter(OP, cuda_only=False):
    """
    OP(..., host_from_gpu(), ...) -> host_from_gpu(GpuOP(...))

    gpu_from_host(OP(inp0, ...)) -> GpuOP(inp0, ...)

    """

    def f(maker):
        def local_opt(node):
            if type(node.op) in OP:
                # Either one of our inputs is on the gpu or
                # all of our clients are on the gpu
                replace = False
                # TODO: Maybe set context_name with infer_context_name()?
                context_name = None
                # We replace if any input is a host_from_gpu
                for i in node.inputs:
                    if i.owner and i.owner.op == host_from_gpu and move_to_gpu(i):
                        context_name = i.owner.inputs[0].type.context_name
                        replace = True
                        break

                if not replace:
                    # We replace if *all* clients are on the GPU
                    clients = [c for o in node.outputs for c in o.clients]
                    replace = len(clients) != 0
                    for c, idx in clients:
                        if c == "output" or not isinstance(c.op, GpuFromHost):
                            replace = False
                    # TODO: check that the clients want the same context?
                    if replace:
                        # All clients are GpuFromHost and we have at least one
                        context_name = clients[0][0].op.context_name

                # Check if we should replace
                if (
                    not replace
                    or (cuda_only and get_context(context_name).kind != b"cuda")
                    or any(["complex" in getattr(i, "dtype", "") for i in node.inputs])
                ):
                    return False

                # tag the inputs with the context in case
                # the context was derived from the outputs
                for i in node.inputs:
                    i.tag.context_name = context_name

                new_op = maker(node.op, context_name, node.inputs, node.outputs)

                # This is needed as sometimes new_op inherits from OP.
                if new_op and new_op != node.op:
                    if isinstance(new_op, Op):
                        new_outputs = new_op(*node.inputs, return_list=True)
                        to_cpu_fn = safe_to_cpu
                    elif isinstance(new_op, (tuple, list)):
                        new_outputs = new_op
                        to_cpu_fn = safe_to_cpu
                    else:  # suppose it is a variable on the GPU
                        new_outputs = [new_op]

                        def to_cpu_fn(x):
                            return x.transfer("cpu")

                    # copy stack traces onto gpu outputs
                    # also copy the stack traces onto HostFromGpu outputs
                    on_cpu = []
                    for old_output, new_output in zip(node.outputs, new_outputs):
                        copy_stack_trace(old_output, new_output)
                        cpu = to_cpu_fn(new_output)
                        on_cpu.append(cpu)
                        copy_stack_trace(old_output, cpu)
                    return on_cpu
            return False

        local_opt.__name__ = maker.__name__
        return local_optimizer(OP)(local_opt)

    return f

max_stars_repo_name: abdalazizrashid/Theano-PyMC

max_issues_repo_name: abdalazizrashid/Theano-PyMC

max_forks_repo_name: abdalazizrashid/Theano-PyMC


text:
from itertools import chain

import cupy
import chainer
import chainer.functions as F
import chainer.links as L
import chainer_nn.functions as nn_F
import chainer_nn.links as nn_L
import numpy as np
from models import modeling

import ipdb

class Encoder(chainer.Chain):

    def __init__(self,
                 word_embeddings,
                 postag_embeddings=None,
                 char_embeddings=None,
                 contextualized_embeddings=None,
                 char_feature_size=50,
                 char_pad_id=1,
                 char_window_size=5,
                 char_dropout=0.0,
                 n_lstm_layers=2,
                 lstm_hidden_size=200,
                 embeddings_dropout=0.0,
                 lstm_dropout=0.0,
                 recurrent_dropout=0.0,
                 bert_model=0,
                 bert_dir=''):
        super().__init__()
        with self.init_scope():
            embeddings = {'word_embed': word_embeddings,
                          'postag_embed': postag_embeddings}
            lstm_in_size = 0
            for name, weights in embeddings.items():
                if weights is not None:
                    assert weights.ndim == 2
                    s = weights.shape
                    self.__setattr__(name, L.EmbedID(s[0], s[1], weights))
                    lstm_in_size += s[1]
                else:
                    self.__setattr__(name, None)
            if char_embeddings is not None:
                self.char_cnn = nn_L.CharCNN(
                    char_embeddings, char_pad_id, char_feature_size,
                    char_window_size, char_dropout)
                lstm_in_size += char_feature_size
            else:
                self.char_cnn = None
            if contextualized_embeddings is not None:
                self.cont_embed = contextualized_embeddings
                lstm_in_size += contextualized_embeddings.out_size
            else:
                self.cont_embed = None

            self.bert_model = bert_model
            if self.bert_model:
                bert_config = modeling.BertConfig.from_json_file(bert_dir+'/bert_config.json')
                bert = modeling.BertModel(config=bert_config)
                model = modeling.BertClassifier(bert, num_labels=len([1,2,3]))
                chainer.serializers.load_npz(bert_dir+'/bert_model.ckpt.npz', model, ignore_names=['output/W', 'output/b'])
                self.model = model
                self._out_size = model.bert.pooler.in_size
            else:
                self.model = nn_L.NStepBiLSTM(
                    n_lstm_layers, lstm_in_size, lstm_hidden_size,
                    lstm_dropout, recurrent_dropout)
                self._out_size = lstm_hidden_size*2
        self.embeddings_dropout = embeddings_dropout
        self.lstm_dropout = lstm_dropout
        self._lstm_in_size = lstm_in_size
        self._hidden_size = lstm_hidden_size

    def forward(self, words, postags=None, chars=None, cont_embeds=None):
        if self.bert_model:
            max_length = max(np.array([embed.shape[1] for embed in cont_embeds], np.int32))
            # xp = chainer.cuda.get_array_module(cont_embeds[0])
            batch_tokens, batch_word_indices = [], []
            for i in range(len(cont_embeds)):
                tokens = cont_embeds[i][0,:,0]
                pad_width = max_length-len(tokens)
                padded_tokens = F.pad(tokens, pad_width, mode='constant')[pad_width:]
                batch_tokens.append(F.expand_dims(padded_tokens, axis=0))
                batch_word_indices.append(set(cont_embeds[i][1,:,0].tolist()))

            batch_tokens = F.concat(batch_tokens, axis=0)
            input_ids = self.model.bert.xp.array(batch_tokens.data, dtype=int)
            model_embed = self.model.bert(input_ids, get_sequence_output=True)

            batch_size, num_tokens, dim = model_embed.shape
            hs = []
            for i in range(batch_size):
                hs_i = []
                for j in range(1, num_tokens):
                    if j in batch_word_indices[i]:
                        hs_i.append(F.expand_dims(model_embed[i][j], axis=0))
                hs.append(F.concat(hs_i, axis=0))
            lengths = np.array([len(word_indices)-1 for word_indices in batch_word_indices])
        else:
            lengths = np.array([seq.size for seq in words], np.int32)
            xs = [self._forward_embed(embed, x) for embed, x
                  in ((self.word_embed, words),
                      (self.postag_embed, postags))
                  if embed is not None]
            if self.char_cnn is not None:
                xs.append(self.char_cnn(list(chain.from_iterable(chars))))
            if self.cont_embed is not None:
                xp = chainer.cuda.get_array_module(cont_embeds[0])
                xs.append(self.cont_embed._forward_top(
                    xp.concatenate(cont_embeds, axis=1)))
            xs = F.concat(xs) if len(xs) > 1 else xs[0]
            xs = nn_F.dropout(xs, self.embeddings_dropout)
            xs = F.split_axis(xs, lengths[:-1].cumsum(), axis=0)
            hs = self.model(hx=None, cx=None, xs=xs)[-1]
            # NOTE(chantera): Disable to reproduce [Teranishi et al., 2019].
            # hs = nn_F.dropout(F.pad_sequence(hs), self.lstm_dropout)
        return hs, lengths

    @staticmethod
    def _forward_embed(embed, x):
        xp = chainer.cuda.get_array_module(x[0])
        return embed(embed.xp.asarray(xp.concatenate(x, axis=0)))

    @property
    def out_size(self):
        return self._out_size

max_stars_repo_name: SaiKeshav/coordparser

max_issues_repo_name: SaiKeshav/coordparser

max_forks_repo_name: SaiKeshav/coordparser

reason: import numpy,import cupy


text:
import pathlib
import os

import numpy as np

from nn_analysis import utils
from nn_analysis import exceptions
from nn_analysis.constants import ENV_CONFIG_PATH

env_config = utils.load_config(ENV_CONFIG_PATH)

def _get_data_path(model_name, epoch, layer_name, metric_name, version):
    if epoch is not None:
        path = f"{env_config['save_results_path']}/{metric_name}/{version:02d}/{model_name}/{epoch:04d}/{layer_name}.pkl"
    else:
        path = f"{env_config['save_results_path']}/{metric_name}/{version:02d}/{model_name}/{layer_name}.pkl"

    return pathlib.Path(path)

def data_exists(*args, **kwargs):
    return _get_data_path(*args, **kwargs).is_file()

def save_data(model_name, epoch, layer_name, metric_name, version, data_dict, overwrite=False):
    if epoch is not None:
        path = f"{env_config['save_results_path']}/{metric_name}/{version:02d}/{model_name}/{epoch:04d}/{layer_name}.pkl"
    else:
        path = f"{env_config['save_results_path']}/{metric_name}/{version:02d}/{model_name}/{layer_name}.pkl"
    utils.save_data(path, data_dict, overwrite=overwrite)

def load_data(*args, **kwargs):
    return utils.load_data(_get_data_path(*args, **kwargs))

def load_neural_data(base_dataset_name, region, standardize=True):
    data = np.load(
        os.path.join(env_config['neural_data_path'], base_dataset_name, f'{region}_data.npy'),
        allow_pickle=True,
        encoding='latin1',
    )

    if region == 'V4' and base_dataset_name == 'hvm':
        data = data[:,:,:17]

    if base_dataset_name == 'rust':
        data = np.transpose(data, axes=(1,0,2))
#     print(data.shape)
    if standardize and data.ndim == 3:
        # Standardize s.t. trial-averaged firing rate has zero mean and unit variance across images.
        data = data - data.mean(axis=(0, 2), keepdims=True)
        data = data / data.mean(axis=2, keepdims=True).std(axis=0, keepdims=True)
    elif data.ndim == 2:
        # Standardize s.t. voxel values have zero mean and unit variance across images.
        data = data - data.mean(axis=0, keepdims=True)
        data = data / data.std(axis=0, keepdims=True)
    else:
        raise RuntimeError("data should have either 2 or 3 dimensions.")
    return data


text:
import numpy as np
from util import *
import matplotlib.pyplot as plt
import matplotlib
matplotlib.rc('text', usetex=True)
matplotlib.rcParams['text.latex.preamble']=[r"\usepackage{amsmath}"]
matplotlib.rcParams['contour.negative_linestyle'] = 'solid'

f, axs = plt.subplots(4, 4, figsize=(10,14),
                      gridspec_kw={'width_ratios':[1,1,1,0.05], 'hspace':0.1, 'wspace':0.25})

polar = 0
w = 2.0
out_path = 'compare-otf/'
n_px = 2**9 + 1
pad = 2**10
[X, Y] = np.meshgrid(np.linspace(-w, w, n_px),
                     np.linspace(-w, w, n_px))
R = np.sqrt(X**2 + Y**2)
Phi = np.nan_to_num(np.arctan(Y/X))

otf_funcs = [H00, H20, H22] # XXX
otf_files = ['H00an', 'H20an', 'H22an']
psf_funcs = [h00, h20, h22] # XXX
psf_files = ['hh00an', 'hh20an', 'hh22an']
ylabels = ['Image', 'OTF Profile', 'PSF Profile', 'PSF Diff']
titles = [r"${H}_0^{0(p)}(\boldsymbol{\nu})$", r"${H}_2^{0(p)}(\boldsymbol{\nu})$", r"${H}_2^{2(p)}(\boldsymbol{\nu})$"]

clabels = [[(n_px/2,0), (n_px/2,100), (n_px/2,150)],
           [(n_px/2,0), (3*n_px/4, n_px/3), (n_px/4, n_px/3), (3*n_px/4, n_px/4), (n_px/4, n_px/4)],
           [(n_px/2,0), (n_px/2,100), (n_px/2,450)]]
for i, (otf_func, otf_file) in enumerate(zip(otf_funcs, otf_files)):
    # Calculations
    otf_arr = (otf_func(R, phi=Phi, phi_p=polar)).astype('float32')
    psf_arr = (psf_funcs[i](R, phi=Phi, phi_p=polar)).astype('float32')
    xF, psf_arr_ft = myfft(otf_arr, pad=pad)

    if i==0:
        ft_norm = np.max(psf_arr_ft)
    # elif i==2: # XXX
    #     ft_norm = np.max(psf_arr_ft)

    # Save images
    # save_tiff(otf_arr, out_path+otf_file+'.tiff')
    # save_tiff(psf_arr, out_path+psf_files[i]+'.tiff')
    # save_tiff(psf_arr_ft, out_path+psf_files[i]+'ft.tiff')

    # Plot 2D OTF
    im = axs[0,i].imshow(otf_arr, cmap="bwr", vmin=-1, vmax=1)
    levels = [-0.2, -0.1, -1e-5, 1e-5, 0.1, 0.2]
    ct = axs[0,i].contour(otf_arr, levels, colors='k',linewidths=0.5)
    # plt.clabel(ct, levels, inline=1, inline_spacing=15, fmt='%1.1f', fontsize=8, manual=clabels[i])

    # Plot OTF
    axs[0,i].set_axis_off()
    x = np.linspace(-w, w, n_px)
    xd = np.linspace(-w*np.sqrt(2), w*np.sqrt(2), n_px)
    axs[1,i].plot(x, cs(otf_arr), '--', c='k', lw=1)
    axs[1,i].plot(x, cs(otf_arr, row=True), '-', c='k', lw=1)
    axs[1,i].plot(xd, ds(otf_arr), ':', c='k', lw=1)
    # axs[1,i].plot(x, cs(H20(R, phi=Phi, phi_p=None)), '-', c='g', lw=1)

    axs[1,i].set_xlim([-2.1,2.1])
    axs[1,i].set_ylim([-1,1])
    axs[1,i].set_xlabel(r"$\nu\, \lambda/\textrm{NA}$")
    axs[1,i].get_yaxis().set_ticks([-1, -0.5, 0.0, 0.5, 1])

    # Plot PSF
    axs[2,i].plot(x, np.abs(cs(psf_arr)), '--', c='k', lw=1)
    axs[2,i].plot(x, np.abs(cs(psf_arr, row=True)), '-', c='k', lw=1)
    # axs[2,i].plot(xd, ds(psf_arr), ':', c='k', lw=1)

    axs[2,i].plot(xF, cs(psf_arr_ft)/ft_norm, '.', c='r', lw=0.5, markersize=1)
    axs[2,i].plot(xF, cs(psf_arr_ft, row=True)/ft_norm, '.', c='r', lw=0.5, markersize=1)

    axs[2,i].set_xlim([-2.1,2.1])
    axs[2,i].set_ylim([-1,1])
    axs[2,i].set_xlabel(r"$r'_o \textrm{NA}/\lambda$")
    axs[2,i].get_yaxis().set_ticks([-1, -0.5, 0.0, 0.5, 1])

    # Calculate diff
    [XF, YF] = np.meshgrid(xF, xF)
    RF = np.sqrt(XF**2 + YF**2)
    PhiF = np.nan_to_num(np.arctan(YF/XF))
    psf_arrF = (psf_funcs[i](RF, phi=PhiF, phi_p=polar)).astype('float32')

    axs[3,i].plot(xF, cs(psf_arr_ft)/ft_norm - np.abs(cs(psf_arrF)), '--', c='k', lw=0.5)
    axs[3,i].plot(xF, cs(psf_arr_ft, row=True)/ft_norm - np.abs(cs(psf_arrF, row=True)), '-', c='k', lw=0.5)

    axs[3,i].set_xlim([-2.1,2.1])
    axs[3,i].set_ylim([-0.1,0.1])
    axs[3,i].set_xlabel(r"$r'_o \textrm{NA}/\lambda$")
    axs[3,i].get_yaxis().set_ticks([-0.1, -0.05, 0.0, 0.05, 0.1])

    axs[i,0].annotate(ylabels[i], xy=(0,0), xytext=(-0.2, 0.5), textcoords='axes fraction', ha='center', va='center', rotation=90)
    axs[0,i].set_title(titles[i])

# Final plotting settings
cbar = f.colorbar(im, cax=axs[0,3], orientation='vertical', ticks=[-1,-0.5, 0, 0.5, 1])
axs[0,3].get_xaxis().set_ticks([])
axs[0,3].yaxis.tick_right()
axs[1,3].set_axis_off()
axs[2,3].set_axis_off()
axs[3,3].set_axis_off()

# Save
plt.savefig('otf-compare.pdf', bbox_inches='tight')

max_stars_repo_name: talonchandler/polharmonic

max_issues_repo_name: talonchandler/polharmonic

max_forks_repo_name: talonchandler/polharmonic


text:
from numpy.testing import assert_array_almost_equal
from pytest import raises

from wonambi import Dataset

from .paths import ktlx_file

def test_xltek_data():
    d = Dataset(ktlx_file)
    data = d.read_data(begsam=1000, endsam=1001, chan=('FZ', ))
    assert_array_almost_equal(data.data[0][0, 0], -90.119315)

def test_xltek_marker():
    d = Dataset(ktlx_file)
    markers = d.read_markers()
    assert markers[0]['name'] == 'Start Recording (varkha)'
    assert int(markers[-1]['end']) == 1314

def test_xltek_videos():
    d = Dataset(ktlx_file)

    with raises(IndexError):
        d.read_videos(0, 2)

    videos, v_beg, v_end = d.read_videos(100, 200)
    assert len(videos) == 2
    assert v_beg == 58.410209
    assert v_end == 37.177808

max_stars_repo_name: wonambi-python/wonambi

max_issues_repo_name: wonambi-python/wonambi

max_forks_repo_name: wonambi-python/wonambi

reason: from numpy


text:
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 24 09:28:02 2018

@author: fc48286
"""

from imageio import imread, imwrite
import matplotlib.pyplot as plt
import numpy as np
import scipy.ndimage
import scipy.misc
import scipy.stats

Img = imread('Z:/DRPI/imagens/marilyn.tif')
dim = Img.shape
col = dim[1]
lin = dim[0]

miu = 0
var = 0.1
U = np.random.uniform(-1, 1, [lin, col])
G = np.sqrt(var)*255*U+miu
Rg =(Img.astype(float)+G.astype(float)).clip(0, 255)

densidade = 2.0/100
sp = np.random.random_sample([lin, col])
px_escuros = sp<(densidade/2)
px_claros = (sp>=(densidade/2)) & (sp<densidade)
rsp = Img.astype(float)+(~px_escuros).astype(float)
rsp = 255*(px_claros.astype(float))+rsp*(~px_claros).astype(float)

gauss_denoised = ndimage.filters.gaussian_filter(Img, 1)

median_denoised = ndimage.filters.gaussian_filter(Img, 3)

plt.figure(figsize=(10, 5))
plt.title('###### Questão 1.1 c) ######')
plt.imshow(gauss_denoised)
plt.axis('off')

m = 3
mat = np.ones((m,m))

max_stars_repo_name: Tjccs/DRPI

max_issues_repo_name: Tjccs/DRPI

max_forks_repo_name: Tjccs/DRPI


text:
import sys
import os
import numpy as np

import imageio
from myssim import compare_ssim as ssim

#SCALE = 8
SCALE = 1

def output_psnr_mse(img_orig, img_out):
    squared_error = np.square(img_orig - img_out)
    mse = np.mean(squared_error)
    psnr = 10 * np.log10(1.0 / mse)
    return psnr

def _open_img(img_p):
    F = np.array(imageio.imread(img_p)).astype(float)/255.0
    h, w, c = F.shape
    F = F[:h-h%SCALE, :w-w%SCALE, :]
    boundarypixels = SCALE
    F = F[boundarypixels:-boundarypixels,boundarypixels:-boundarypixels,:]
    return F

def _open_img_ssim(img_p):
    F = np.array(imageio.imread(img_p))
    h, w, c = F.shape
    F = F[:h-h%SCALE, :w-w%SCALE, :]
    boundarypixels = SCALE
    F = F[boundarypixels:-boundarypixels,boundarypixels:-boundarypixels,:]
    return F

def compute_psnr(ref_im, res_im):
    return output_psnr_mse(_open_img(ref_im), _open_img(res_im))

def compute_mssim(ref_im, res_im):
    ref_img = _open_img_ssim(ref_im)
    res_img = _open_img_ssim(res_im)
    channels = []
    for i in range(3):
        channels.append(ssim(ref_img[:,:,i],res_img[:,:,i],
        gaussian_weights=True, use_sample_covariance=False))
    return np.mean(channels)


text:
# coding: utf-8

# Author: Christian Michelsen, NBI, 2018

import numpy as np

def plot_hist_of_exp_data(tau=1 / np.e):
    # General input:
    Nbins = 100
    xmin, xmax = 0, 10
    binwidth = (xmax - xmin) / Nbins
    Nbkg = 5000  # Number of random Exponential points
    x_bkg = r.exponential(tau, Nbkg)
    x_all = x_bkg
    # Create just a single figure and axes, and a (classic) histogram:
    fig, ax = plt.subplots(figsize=(16, 6))  # figsize is in inches
    hist = ax.hist(x_all, bins=Nbins, range=(xmin, xmax), histtype='step', linewidth=2, color='red',
                   label='Data, normal histogram')

    # Find the x, y and error on y (sy) given the histogram:
    counts, bin_edges = np.histogram(x_all, bins=Nbins, range=(xmin, xmax))
    x = (bin_edges[1:] + bin_edges[:-1]) / 2
    y = counts
    sy = np.sqrt(counts)  # NOTE: We (naturally) assume that the bin count is Poisson distributed.
    # This is an approximation, since there is a low count in the last bins.

    # Did you make sure, that all bins were non-zero???
    # x = (bin_edges[1:][counts>0] + bin_edges[:-1][counts>0])/2
    # y = counts[counts>0]
    # sy = np.sqrt(counts[counts>0])   # NOTE: We (naturally) assume that the bin count is Poisson distributed.

    # Now create a histogram with uncertainties (better, I would argue):
    ax.errorbar(x, y, yerr=sy, xerr=0.0, label='Data, with Poisson errors', fmt='.k', ecolor='k', elinewidth=1,
                capsize=1, capthick=1)

    # Set the figure texts; xlabel, ylabel and title.
    ax.set(xlabel="Random numbers",  # the label of the y axis
           ylabel="Frequency / 10",  # the label of the y axis
           title="Distribution of Gaussian and exponential numbers")  # the title of the plot
    ax.legend(loc='best', fontsize=20);  # could also be # loc = 'upper right' e.g.

def format_value(value, decimals):
    """
    Checks the type of a variable and formats it accordingly.
    Floats has 'decimals' number of decimals.
    """

    if isinstance(value, (float, np.float)):
        return f'{value:.{decimals}f}'
    elif isinstance(value, (int, np.integer)):
        return f'{value:d}'
    else:
        return f'{value}'

def values_to_string(values, decimals):
    """
    Loops over all elements of 'values' and returns list of strings
    with proper formating according to the function 'format_value'.
    """

    res = []
    for value in values:
        if isinstance(value, list):
            tmp = [format_value(val, decimals) for val in value]
            res.append(f'{tmp[0]} +/- {tmp[1]}')
        else:
            res.append(format_value(value, decimals))
    return res

def len_of_longest_string(s):
    """ Returns the length of the longest string in a list of strings """
    return len(max(s, key=len))

def nice_string_output(d, extra_spacing=5, decimals=3):
    """
    Takes a dictionary d consisting of names and values to be properly formatted.
    Makes sure that the distance between the names and the values in the printed
    output has a minimum distance of 'extra_spacing'. One can change the number
    of decimals using the 'decimals' keyword.
    """

    names = d.keys()
    max_names = len_of_longest_string(names)

    values = values_to_string(d.values(), decimals=decimals)
    max_values = len_of_longest_string(values)

    string = ""
    for name, value in zip(names, values):
        spacing = extra_spacing + max_values + max_names - len(name) - 1
        string += "{name:s} {value:>{spacing}} \n".format(name=name, value=value, spacing=spacing)
    return string[:-2]

def add_text_to_ax(x_coord, y_coord, string, ax, fontsize=12, color='k'):
    """ Shortcut to add text to an ax with proper font. Relative coords."""
    ax.text(x_coord, y_coord, string, family='monospace', fontsize=fontsize,
            transform=ax.transAxes, verticalalignment='top', color=color)
    return None

# =============================================================================
#  Probfit replacement
# =============================================================================

from iminuit.util import make_func_code
from iminuit import describe  # , Minuit,

def set_var_if_None(var, x):
    if var is not None:
        return np.array(var)
    else:
        return np.ones_like(x)

def compute_f(f, x, *par):
    try:
        return f(x, *par)
    except ValueError:
        return np.array([f(xi, *par) for xi in x])

class Chi2Regression:  # override the class with a better one

    def __init__(self, f, x, y, sy=None, weights=None, bound=None):
        if bound is not None:
            x = np.array(x)
            y = np.array(y)
            sy = np.array(sy)
            mask = (x >= bound[0]) & (x <= bound[1])
            x = x[mask]
            y = y[mask]
            sy = sy[mask]

        self.f = f  # model predicts y for given x
        self.x = np.array(x)
        self.y = np.array(y)

        self.sy = set_var_if_None(sy, self.x)
        self.weights = set_var_if_None(weights, self.x)
        self.func_code = make_func_code(describe(self.f)[1:])

    def __call__(self, *par):  # par are a variable number of model parameters

        # compute the function value
        f = compute_f(self.f, self.x, *par)

        # compute the chi2-value
        chi2 = np.sum(self.weights * (self.y - f) ** 2 / self.sy ** 2)

        return chi2

def simpson38(f, edges, bw, *arg):
    yedges = f(edges, *arg)
    left38 = f((2. * edges[1:] + edges[:-1]) / 3., *arg)
    right38 = f((edges[1:] + 2. * edges[:-1]) / 3., *arg)

    return bw / 8. * (np.sum(yedges) * 2. + np.sum(left38 + right38) * 3. - (yedges[0] + yedges[-1]))  # simpson3/8

def integrate1d(f, bound, nint, *arg):
    """
    compute 1d integral
    """
    edges = np.linspace(bound[0], bound[1], nint + 1)
    bw = edges[1] - edges[0]

    return simpson38(f, edges, bw, *arg)

class UnbinnedLH:  # override the class with a better one

    def __init__(self, f, data, weights=None, bound=None, badvalue=-100000, extended=False, extended_bound=None,
                 extended_nint=100):

        if bound is not None:
            data = np.array(data)
            mask = (data >= bound[0]) & (data <= bound[1])
            data = data[mask]
            if (weights is not None):
                weights = weights[mask]

        self.f = f  # model predicts PDF for given x
        self.data = np.array(data)
        self.weights = set_var_if_None(weights, self.data)
        self.bad_value = badvalue

        self.extended = extended
        self.extended_bound = extended_bound
        self.extended_nint = extended_nint
        if extended and extended_bound is None:
            self.extended_bound = (np.min(data), np.max(data))

        self.func_code = make_func_code(describe(self.f)[1:])

    def __call__(self, *par):  # par are a variable number of model parameters

        logf = np.zeros_like(self.data)

        # compute the function value
        f = compute_f(self.f, self.data, *par)

        # find where the PDF is 0 or negative (unphysical)
        mask_f_positive = (f > 0)

        # calculate the log of f everyhere where f is positive
        logf[mask_f_positive] = np.log(f[mask_f_positive]) * self.weights[mask_f_positive]

        # set everywhere else to badvalue
        logf[~mask_f_positive] = self.bad_value

        # compute the sum of the log values: the LLH
        llh = -np.sum(logf)

        if self.extended:
            extended_term = integrate1d(self.f, self.extended_bound, self.extended_nint, *par)
            llh += extended_term

        return llh

    def default_errordef(self):
        return 0.5

class BinnedLH:  # override the class with a better one

    def __init__(self, f, data, bins=40, weights=None, weighterrors=None, bound=None, badvalue=1000000, extended=False,
                 use_w2=False, nint_subdiv=1):

        if bound is not None:
            data = np.array(data)
            mask = (data >= bound[0]) & (data <= bound[1])
            data = data[mask]
            if (weights is not None):
                weights = weights[mask]
            if (weighterrors is not None):
                weighterrors = weighterrors[mask]

        self.weights = set_var_if_None(weights, data)

        self.f = f
        self.use_w2 = use_w2
        self.extended = extended

        if bound is None:
            bound = (np.min(data), np.max(data))

        self.mymin, self.mymax = bound

        h, self.edges = np.histogram(data, bins, range=bound, weights=weights)

        self.bins = bins
        self.h = h
        self.N = np.sum(self.h)

        if weights is not None:
            if weighterrors is None:
                self.w2, _ = np.histogram(data, bins, range=bound, weights=weights ** 2)
            else:
                self.w2, _ = np.histogram(data, bins, range=bound, weights=weighterrors ** 2)
        else:
            self.w2, _ = np.histogram(data, bins, range=bound, weights=None)

        self.badvalue = badvalue
        self.nint_subdiv = nint_subdiv

        self.func_code = make_func_code(describe(self.f)[1:])
        self.ndof = np.sum(self.h > 0) - (self.func_code.co_argcount - 1)

    def __call__(self, *par):  # par are a variable number of model parameters

        # ret = compute_bin_lh_f(self.f, self.edges, self.h, self.w2, self.extended, self.use_w2, self.badvalue, *par)
        ret = compute_bin_lh_f2(self.f, self.edges, self.h, self.w2, self.extended, self.use_w2, self.nint_subdiv, *par)

        return ret

    def default_errordef(self):
        return 0.5

import warnings

def xlogyx(x, y):
    # compute x*log(y/x) to a good precision especially when y~x

    if x < 1e-100:
        warnings.warn('x is really small return 0')
        return 0.

    if x < y:
        return x * np.log1p((y - x) / x)
    else:
        return -x * np.log1p((x - y) / y)

# compute w*log(y/x) where w < x and goes to zero faster than x
def wlogyx(w, y, x):
    if x < 1e-100:
        warnings.warn('x is really small return 0')
        return 0.
    if x < y:
        return w * np.log1p((y - x) / x)
    else:
        return -w * np.log1p((x - y) / y)

def compute_bin_lh_f2(f, edges, h, w2, extended, use_sumw2, nint_subdiv, *par):
    N = np.sum(h)
    n = len(edges)

    ret = 0.

    for i in range(n - 1):
        th = h[i]
        tm = integrate1d(f, (edges[i], edges[i + 1]), nint_subdiv, *par)

        if not extended:
            if not use_sumw2:
                ret -= xlogyx(th, tm * N) + (th - tm * N)

            else:
                if w2[i] < 1e-200:
                    continue
                tw = w2[i]
                factor = th / tw
                ret -= factor * (wlogyx(th, tm * N, th) + (th - tm * N))
        else:
            if not use_sumw2:
                ret -= xlogyx(th, tm) + (th - tm)
            else:
                if w2[i] < 1e-200:
                    continue
                tw = w2[i]
                factor = th / tw
                ret -= factor * (wlogyx(th, tm, th) + (th - tm))

    return ret

def compute_bin_lh_f(f, edges, h, w2, extended, use_sumw2, badvalue, *par):
    mask_positive = (h > 0)

    N = np.sum(h)
    midpoints = (edges[:-1] + edges[1:]) / 2
    b = np.diff(edges)

    midpoints_pos = midpoints[mask_positive]
    b_pos = b[mask_positive]
    h_pos = h[mask_positive]

    if use_sumw2:
        warnings.warn('use_sumw2 = True: is not yet implemented, assume False ')
        s = np.ones_like(midpoints_pos)
        pass
    else:
        s = np.ones_like(midpoints_pos)

    E_pos = f(midpoints_pos, *par) * b_pos
    if not extended:
        E_pos = E_pos * N

    E_pos[E_pos < 0] = badvalue

    ans = -np.sum(s * (h_pos * np.log(E_pos / h_pos) + (h_pos - E_pos)))

    return ans

max_stars_repo_name: paul-cvp/delay-deadline-miner

max_issues_repo_name: paul-cvp/delay-deadline-miner

max_forks_repo_name: paul-cvp/delay-deadline-miner


text:
import porespy as ps
import pytrax as pt
import scipy as sp
import os
import matplotlib.pyplot as plt

class SimulationTest():
    def setup_class(self):
        self.l = 100
        self.im = ps.generators.overlapping_spheres(shape=[self.l, self.l],
                                                    radius=5,
                                                    porosity=0.55)
        self.blobs = ps.generators.blobs([self.l, self.l, self.l])
        self.rw = pt.RandomWalk(self.blobs)
        self.blobs_2d = ps.generators.blobs([self.l, self.l]).astype(int)
        self.rw_2d = pt.RandomWalk(self.blobs_2d, seed=True)

    def test_random_walk(self):
        self.rw.run(nt=1000, nw=100, stride=1)
        assert sp.shape(self.rw.real_coords) == (1000, 100, 3)

    def test_plot_msd(self):
        self.rw.plot_msd()
        pass

    def test_random_walk_2d(self):
        self.rw_2d.run(nt=1000, nw=100, same_start=True, stride=100)
        assert sp.shape(self.rw_2d.real_coords) == (10, 100, 2)

    def test_plot_walk_2d(self):
        self.rw_2d.plot_walk_2d(data='w')
        assert hasattr(self.rw_2d, 'im_big')
        assert sp.sum(self.rw_2d.im_big > self.rw_2d.nw) == 0
        plt.close('all')

    def test_export(self):
        cwd = os.getcwd()
        self.rw_2d.export_walk(sub='temp', image=self.rw_2d.im, sample=10)
        subdir = os.path.join(cwd, 'temp')
        assert os.path.exists(subdir)
        file_list = os.listdir(subdir)
        # 10 coordinate files based on the stride and number of steps + image
        assert len(file_list) == 11
        # Delete all files and folder
        for file in file_list:
            fp = os.path.join(subdir, file)
            os.remove(fp)
        os.rmdir(subdir)

    def test_seed(self):
        # rw_2d was initialized with seed = True, this should mean running it
        # repeatedly produces the same movements
        self.rw_2d.run(nt=1000, nw=100, same_start=True)
        temp_coords = self.rw_2d.real_coords.copy()
        self.rw_2d.run(nt=1000, nw=100, same_start=True)
        assert sp.allclose(self.rw_2d.real_coords, temp_coords)

    def test_axial_density_plot(self):
        self.rw.axial_density_plot(time=0, axis=0)
        plt.close('all')

    def test_rw_analytics(self):
        w_list = [100]
        t_list = [100]
        self.rw_2d.run_analytics(w_list, t_list, fname='test.csv', num_proc=1)
        cwd = os.getcwd()
        fpath = os.path.join(cwd, 'test.csv')
        assert os.path.exists(fpath)
        os.remove(fpath)
        plt.close('all')

if __name__ == '__main__':
    t = SimulationTest()
    t.setup_class()
    t.test_random_walk()
    t.test_plot_msd()
    t.test_random_walk_2d()
    t.test_plot_walk_2d()
    t.test_export()
    t.test_seed()
    t.test_axial_density_plot()
    t.test_rw_analytics()

max_stars_repo_name: denfromufa/pytrax

max_issues_repo_name: denfromufa/pytrax

max_forks_repo_name: denfromufa/pytrax

reason: import scipy


text:
import os
import shutil
import time
from collections import OrderedDict
from copy import deepcopy

import matplotlib.patches as patches
import matplotlib.pyplot as plt
import numpy as np
from skimage import io

from FfmpegWrapper.ffmpeg_wrapper import FfmpegWrapper
from Tracker.tracks_data import VideoMotAssignment
from Utils import multiprocess_worker as mp
from Animator.utils import eprint

class DemoCompiler:
    def __init__(self, sampled_fps, out_fps, video_root, output_dir):
        """
        demo video generator
        :param sampled_fps: the frame sample rate
        :param out_fps: the output video frame rate
        :param video_root:
        :param output_dir:
        """
        self.sampled_fps = sampled_fps
        self.out_fps = out_fps
        self.num_distinct_colors = 16
        self.colors = np.random.rand(self.num_distinct_colors, 3)  # RGB
        self.frame_name_format = '%06d.jpg'
        self.visualization_video_name = 'demo.mp4'
        containing_folder, vid_name = os.path.split(video_root)
        self.original_video_name = vid_name.split(".")[0]
        demo_file_name = f'{self.original_video_name}_demo.mp4'
        self.demo_path = os.path.join(output_dir, demo_file_name)

    def generate_demo(self, shot_to_frames_to_detections: VideoMotAssignment, shot_to_offline_mot: OrderedDict,
                      working_dir: str) -> None:
        """
        compile a demo video based on the tracking output
        :param working_dir: the visualization directory in which all frames will be rendered and saved
        :param shot_to_frames_to_detections: mapping a shot to its frames and a frame to its detections
        :param shot_to_offline_mot: shot name to tracks
        :return: None (void)
        """
        start_time = time.time()
        plt.ioff()

        # clear the visualization folder
        if os.path.isdir(working_dir):
            shutil.rmtree(working_dir)
        os.mkdir(working_dir)

        video_level_frame_number = 0
        frame_index_to_kwargs = []
        for shot_name, frame_to_detections in shot_to_frames_to_detections.shot_mot.items():
            for frame_name, frame_path_and_detections in frame_to_detections.frame_mot.items():
                # making sure no resources are shared between processes!
                curr_frame_detections = deepcopy(frame_path_and_detections.detections)
                current_frame_path = frame_path_and_detections.frame_path
                current_shot_mot = OrderedDict({shot_name: deepcopy(shot_to_offline_mot[shot_name])})
                curr_frame_args = dict(frame_str=str(video_level_frame_number),
                                       frame_path=current_frame_path, shot_name=shot_name,
                                       detections=curr_frame_detections,
                                       shot_to_offline_mot=current_shot_mot, visualization_dir=working_dir)
                frame_index_to_kwargs.append(curr_frame_args)
                video_level_frame_number += 1

        # process visualization in parallel
        semaphore = mp.Semaphore(n_processes=10)
        semaphore.parallelize(frame_index_to_kwargs, self.demo_single_frame)
        plt.close('all')

        # stats
        duration = int(time.time()-start_time)
        print(f'done with demo frames at {video_level_frame_number/duration:.3f} FPS in total time of {duration}')

        # compile frames to video with ffmpeg
        try:
            FfmpegWrapper().compile_video(working_dir, self.sampled_fps, self.out_fps, self.demo_path)
            print(f'Saved the demo video into: {self.demo_path}')
            time.sleep(2)
            shutil.rmtree(working_dir)
        except PermissionError as e:
            eprint(f'Generating a demo video raised a PermissionError exception: {e}', e)
            eprint(f'Continue execution without deleting the visualization dir: {working_dir}')
        except Exception as e:
            eprint('Generating a demo video raised an exception.', e)
            raise e

    def demo_single_frame(self, frame_str: str, frame_path: str, shot_name: str, detections: list,
                          shot_to_offline_mot: OrderedDict, visualization_dir: str):
        """
        draw a single frame as part of the demo video
        :param frame_str: the string representation of the frame's index
        :param frame_path: the path to the frame
        :param shot_name: the shot str
        :param detections: the list of CharacterDetections from the json
        :param shot_to_offline_mot: the shot's MOT track ids
        :param visualization_dir: the output visualization repo
        :return:
        """
        fig = plt.figure(frame_str)
        ax = fig.add_subplot(111, aspect='equal')
        im = io.imread(frame_path)
        ax.imshow(im)
        ax.set_title(f"{shot_name}, {frame_str}")
        for box in detections:
            if shot_name not in shot_to_offline_mot or box.Id not in shot_to_offline_mot[shot_name]:
                continue
            track_id = shot_to_offline_mot[shot_name][box.Id]

            # draw a rectangular bbox
            rect = patches.Rectangle((box.Rect.X, box.Rect.Y), box.Rect.Width, box.Rect.Height, fill=False,
                                     lw=2.5, ec=self.colors[track_id % self.num_distinct_colors, :])
            ax.add_patch(rect)
            box_title = f'{track_id}, C:{box.Confidence:.2f}'
            ax.text(box.Rect.X, box.Rect.Y, box_title, horizontalalignment='left', verticalalignment='top', fontsize=12)
        fig.tight_layout()
        fig.savefig(os.path.join(visualization_dir, self.frame_name_format % int(frame_str)))
        fig.canvas.flush_events()
        ax.cla()
        plt.close(frame_str)

max_stars_repo_name: oronnir/CAST

max_issues_repo_name: oronnir/CAST

max_forks_repo_name: oronnir/CAST


text:
from math import exp, sqrt
import numpy as np

from utils.histogram.cl_run_iif import CL_IIF, CL_IIF_BINID
from utils.histogram.lsh import num_bins

def illumination_invariant_features_hybrid(image, histogram, k=0.1, debug_ax=None):
    width, height, nbins = np.shape(histogram)
    fast_binid = CL_IIF_BINID()
    fast_binid.load_program()
    bp_mtx = fast_binid.compute(image, nbins)

    b_mtx = np.ndarray((1,1, nbins))
    b_mtx[0, 0,:] = np.arange(0, nbins)
    b_mtx = np.tile(b_mtx, (width, height, 1))

    # contruct pixel intensity matrix
    i_mtx = np.repeat(image[:, :, np.newaxis], nbins, axis=2)
    i_mtx *= 255
    i_mtx = k * i_mtx
    i_mtx[i_mtx < k] = k

    # compute illumination invariant features
    X = -((b_mtx - bp_mtx) ** 2) / (2 * (i_mtx ** 2))
    e_mtx = np.exp(X)
    Ip_mtx = e_mtx * histogram
    feature_img = np.sum(Ip_mtx, 2)
    return feature_img

fast_iif = CL_IIF(num_bins=num_bins)
fast_iif.load_program()

def illumination_invariant_features_cl(image, histogram, k=0.1, debug_ax=None):
    return fast_iif.compute(image, histogram, k)


text:
# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.

# Standard library imports
import numpy as np

# Third-party imports
import pytest
import mxnet as mx
from mxnet import autograd

# First-party imports
from gluonts.mx.activation import get_activation, get_activation_deriv

@pytest.mark.parametrize(
    "activation, kwargs",
    [
        ("tanh", dict()),
        ("softrelu", dict()),
        ("elu", dict()),
        ("swish", dict(beta=100)),
        ("swish", dict(beta=0.5)),
        ("lipswish", dict(beta_initializer=mx.init.Constant(1.0))),
        ("lipswish", dict(beta_initializer=mx.init.Constant(10.0))),
    ],
)
def test_activation_deriv(activation, kwargs):
    def get_deriv_autograd(x, act):
        x.attach_grad()
        with autograd.record():
            output = act(x)
        return autograd.grad(output, [x], create_graph=True)[0]

    x = mx.nd.random.randn(500, 20)
    act = get_activation(activation, **kwargs)
    act.initialize()
    correct_deriv = get_deriv_autograd(x, act)
    act_deriv = get_activation_deriv(act)
    output_deriv = act_deriv(mx.ndarray, x)

    assert np.allclose(
        output_deriv.asnumpy(), correct_deriv.asnumpy(), atol=5e-7
    )

max_stars_repo_name: Xiaoxiong-Liu/gluon-ts

max_issues_repo_name: Xiaoxiong-Liu/gluon-ts

max_forks_repo_name: Xiaoxiong-Liu/gluon-ts


text:
import numpy as np
import pandas as pd
from .SEIR import SEIR

class CModel(SEIR):
    '''
    Not_susceptible Susceptible-Infected-Recovered-Susceptible with Interventions (NSEIRS_I) model
    '''

    def __init__(self, name, population, init_s, init_e, init_i, init_r, init_n, beta, incubation_duration, recovery_duration, mortality_rate, mortality_duration, interventions={}, dist_params={}):
        '''
        Initializes an SEIRS_I model
        -----
        INPUTS:
        name: Model name
        population: Number of individuals in the simulation
        init_s: Initial proportion of the population in state S (Susceptible)
        init_e: Initial proportion of the population in state E (Exposed)
        init_i: Initial proportion of the population in state I (Infected)
        init_r: Initial proportion of the population in state R (Recovered)
        init_n: Initial proportion of the population in state N (Not Susceptible)
        beta: Average number of susceptible individuals infected by one infected individual in one time step
        incubation_rate: Average number of exposed individuals who become infected by one infected per time unit
        recovery_rate: Average number of infected individuals recovered in one time step
        mortality_rate: Average number of infected individuals who die in one time step
        interventions: D
        dist_params: Dictonary with parameter names, distribution names, and distribution parameters (using numpy.random.* distribution parameter names) (e.g. {param_name : {prob_dist_name : {dist_params: dist_param_values }}})
        -----
        '''

        super(CModel, self).__init__(name, population, init_s, init_e, init_i, init_r, beta, incubation_duration, mortality_rate, recovery_duration, mortality_duration, dist_params={})
        self.states['N'] = np.round(init_n*population)
        self.interventions = interventions

        # Dataframe with state values over time
        self.state_timeline = pd.DataFrame([[i for i in [1,0,self.states['S'], self.states['E'], self.states['I'], self.states['R'], self.states['N'], self.states['D']]]], columns=['Run','Time','Susceptible', 'Exposed','Infected', 'Recovered', 'Not Susceptible','Dead'],
                                   dtype='int')
        self.state_timeline.set_index(['Run','Time'],inplace=True)
        self.time = 1
        self.run_num = 1

    def reset(self):
        '''
        Resets model states to initial conditions
        '''
        self.state_timeline = self.state_timeline.iloc[[0]]
        self.states['S'], self.states['E'], self.states['I'], self.states['R'], self.states['N'], self.states['D'] = self.state_timeline.iloc[0].tolist()
        self.time = 1
        self.run_num = 1

    def calculate_deltas(self):
        # calculate changes in state counts

        intervention_time = self.interventions.get("intervention_time", None)
        if intervention_time is not None:
            if self.time == intervention_time:
                effect = self.interventions.get("intervention_effect", 0)
                self.params['beta'] = self.params['beta']*(1.0-effect)

        im = 0.0
        vaccination_time = self.interventions.get("vaccination_time", None)
        if vaccination_time is not None:
            if self.time >= vaccination_time:
                im = self.interventions.get("vaccination_rate", 0)

        delta_S = - min(self.states['S'] * self.params['beta'] * self.states['I'] / self.params['population'], self.states['S']) - min(im, self.states['S'])
        delta_E = min(self.states['S'] * self.params['beta'] * self.states['I'] / self.params['population'], self.states['S']) -  self.states['E'] / self.params['incubation_duration']

        delta_I = self.states['E'] / self.params['incubation_duration'] - self.states['I']/self.params['duration_recovery'] - self.states['I']/self.params['mortality_duration']
        delta_R = (1-self.params['mortality_rate']) * self.states['I'] / self.params['duration_recovery']
        delta_D = self.params['mortality_rate'] * self.states['I'] / self.params['mortality_duration']
        delta_N = min(im,self.states['S'])

        return delta_S, delta_E, delta_I, delta_R, delta_D, delta_N

    def step(self):
        '''
        Updates model states by one time unit
        '''

        delta_S, delta_E, delta_I, delta_R, delta_D, delta_N = self.calculate_deltas()

        # update states
        self.states['S'] += delta_S
        self.states['E'] += delta_E
        self.states['I'] += delta_I
        self.states['R'] += delta_R
        self.states['D'] += delta_D
        self.states['N'] += delta_N

        # Update state dataframe

        self.state_timeline.loc[(self.run_num,self.time),:] = np.round([i for i in [self.states['S'], self.states['E'],self.states['I'], self.states['R'], self.states['N'], self.states['D']]],0)

        self.time += 1

    def run(self, T, runs=1):
        '''
        Run the simulation until time T
        '''
        for run in range(1,runs+1):
            self.time = 1
            for time in range(1,T+1):
                self.step()
            self.run_num += 1

max_stars_repo_name: jamescodella/EpyModels

max_issues_repo_name: jamescodella/EpyModels

max_forks_repo_name: jamescodella/EpyModels