text:
import torch
import numpy as np
import pickle

def centernet2ModelToCustomerModel(num_class=10, model_path='models/CenterNet2_R50_1x.pth', model_save_dir="models"):
    # 将centernet2的模型转换为自己的训练模型
    pretrained_weights = torch.load(model_path)
    pretrained_weights['iteration'] = 0

    pretrained_weights['model']["roi_heads.box_predictor.0.cls_score.weight"].resize_(num_class + 1, 1024)
    pretrained_weights['model']["roi_heads.box_predictor.0.cls_score.bias"].resize_(num_class + 1)
    pretrained_weights['model']["roi_heads.box_predictor.1.cls_score.weight"].resize_(num_class + 1, 1024)
    pretrained_weights['model']["roi_heads.box_predictor.1.cls_score.bias"].resize_(num_class + 1)
    pretrained_weights['model']["roi_heads.box_predictor.2.cls_score.weight"].resize_(num_class + 1, 1024)
    pretrained_weights['model']["roi_heads.box_predictor.2.cls_score.bias"].resize_(num_class + 1)

    torch.save(pretrained_weights, "{}/CenterNet2_{}.pth".format(model_save_dir, num_class))

if __name__ == '__main__':
    centernet2ModelToCustomerModel(model_path="/home/zengxh/workspace/CenterNet2/projects/CenterNet2/models/CenterNet2_R50_1x.pth", model_save_dir="/home/zengxh/workspace/CenterNet2/projects/CenterNet2/models")


text:
import os
import argparse
import time
import numpy as np

import torch
import torch.nn as nn
import torch.optim as optim

parser = argparse.ArgumentParser('ODE demo')
parser.add_argument('--method', type=str, choices=['rk4','dopri5', 'adams'], default='rk4')
parser.add_argument('--data_size', type=int, default=201) #RCL modified default
parser.add_argument('--batch_time', type=int, default=20) #RCL modified default
parser.add_argument('--batch_size', type=int, default=10) #RCL modified default
parser.add_argument('--niters', type=int, default=1000) #RCL modified default
parser.add_argument('--test_freq', type=int, default=10) #RCL modified default
parser.add_argument('--test_freq2', type=int, default=10) #RCL Added new option
parser.add_argument('--viz', action='store_true')
parser.add_argument('--gpu', type=int, default=0)
parser.add_argument('--adjoint', action='store_true')
parser.add_argument('--nhidden', type=int, default=50) #RCL Added new option
args = parser.parse_args()

if args.adjoint:
    from torchdiffeq import odeint_adjoint as odeint
else:
    from torchdiffeq import odeint

device = torch.device('cuda:' + str(args.gpu) if torch.cuda.is_available() else 'cpu')

#RCL
true_y0 = torch.tensor([[1., 0.5]])
t = torch.linspace(0., 10., args.data_size)
t_extend = torch.linspace(0., 20., 2*args.data_size)
feed = {'alpha': 1.0, 'beta': 1.0, 'gamma': 2.0, 'delta': 1.0}

class Lambda(nn.Module):

    def forward(self, t, y):
        #RCL Lotka-Volterra modification
        x=y.t()
        dx = torch.zeros_like(x)
        dx[0] = feed['alpha']*x[0] - feed['beta']*x[0]*x[1]
        dx[1] = -feed['gamma']*x[1] + feed['delta']*x[0]*x[1]
        return dx.t()

with torch.no_grad():
    true_y = odeint(Lambda(), true_y0, t, method=args.method)
    true_y_extend = odeint(Lambda(), true_y0, t_extend, method=args.method)

#RCL Set an explicit seed for noise for reproducibility
torch.manual_seed(0)
#Add noise to observations
noise_dist = torch.distributions.normal.Normal(0, 0.2)
noise_shape = torch.Size([true_y.size()[0], true_y.size()[1], true_y.size()[2]])
noise_samples = noise_dist.sample(sample_shape=noise_shape)
#true_y = true_y + noise_samples

np.random.seed(0) #RCL
def get_batch():
    s = torch.from_numpy(np.random.choice(np.arange(args.data_size - args.batch_time), args.batch_size, replace=False))
    batch_y0 = true_y[s]  # (M, D)
    batch_t = t[:args.batch_time]  # (T)
    batch_y = torch.stack([true_y[s + i] for i in range(args.batch_time)], dim=0)  # (T, M, D)
    return batch_y0, batch_t, batch_y

def makedirs(dirname):
    if not os.path.exists(dirname):
        os.makedirs(dirname)

if args.viz:
    makedirs('png')
    import matplotlib.pyplot as plt
    fig = plt.figure(figsize=(12, 4), facecolor='white')
    ax_traj = fig.add_subplot(131, frameon=False)
    ax_phase = fig.add_subplot(132, frameon=False)
    ax_vecfield = fig.add_subplot(133, frameon=False)
    plt.show(block=False)

#Original
def visualize(true_y, pred_y, odefunc, itr, viz=args.viz, filename=None):

    if viz:

        # makedirs('png')
        # import matplotlib.pyplot as plt
        # fig = plt.figure(figsize=(12, 4), facecolor='white')
        # ax_traj = fig.add_subplot(131, frameon=False)
        # ax_phase = fig.add_subplot(132, frameon=False)
        # ax_vecfield = fig.add_subplot(133, frameon=False)
        # plt.show(block=False)

        ax_traj.cla()
        ax_traj.set_title('Trajectories')
        ax_traj.set_xlabel('t')
        ax_traj.set_ylabel('x,y')
        ax_traj.plot(t.numpy(), true_y.numpy()[:, 0, 0], t.numpy(), true_y.numpy()[:, 0, 1], 'g-')
        ax_traj.plot(t.numpy(), pred_y.numpy()[:, 0, 0], '--', t.numpy(), pred_y.numpy()[:, 0, 1], 'b--')
        ax_traj.set_xlim(t.min(), t.max())
        ax_traj.set_ylim(0, 4)
        #ax_traj.legend()

        ax_phase.cla()
        ax_phase.set_title('Phase Portrait')
        ax_phase.set_xlabel('x')
        ax_phase.set_ylabel('y')
        ax_phase.plot(true_y.numpy()[:, 0, 0], true_y.numpy()[:, 0, 1], 'g-')
        ax_phase.plot(pred_y.numpy()[:, 0, 0], pred_y.numpy()[:, 0, 1], 'b--')
        ax_phase.set_xlim(0, 4)
        ax_phase.set_ylim(0, 4)

        ax_vecfield.cla()
        ax_vecfield.set_title('Learned Vector Field')
        ax_vecfield.set_xlabel('x')
        ax_vecfield.set_ylabel('y')

        y, x = np.mgrid[0:4:21j, 0:4:21j]
        dydt = odefunc(0, torch.Tensor(np.stack([x, y], -1).reshape(21 * 21, 2))).cpu().detach().numpy()
        mag = np.sqrt(dydt[:, 0]**2 + dydt[:, 1]**2).reshape(-1, 1)
        dydt = (dydt / mag)
        dydt = dydt.reshape(21, 21, 2)

        ax_vecfield.streamplot(x, y, dydt[:, :, 0], dydt[:, :, 1], color="black")
        ax_vecfield.set_xlim(0, 4)
        ax_vecfield.set_ylim(0, 4)

        fig.tight_layout()
        if filename is None:
            plt.savefig('png/{:03d}'.format(itr))
        else:
            plt.savefig('png/{}'.format(filename))

        plt.draw()
        plt.pause(0.001)

class ODEFunc(nn.Module):

    def __init__(self):
        super(ODEFunc, self).__init__()

        print('Number of nodes in a hidden layer:',args.nhidden)
        self.net = nn.Sequential(
            nn.Linear(2, args.nhidden),
            #nn.Tanh(),
            nn.ReLU(), #RCL One layer nhidden 50,40,30,20,10 works. To a lesser extent 8,6,4 is close or getting there. 2 is not.
            #nn.Linear(args.nhidden, args.nhidden),
            #nn.Tanh(),
            nn.Linear(args.nhidden, 2),
        )

        torch.manual_seed(0) #RCL
        for m in self.net.modules():
            if isinstance(m, nn.Linear):
                nn.init.normal_(m.weight, mean=0, std=0.1)
                nn.init.constant_(m.bias, val=0)

    def forward(self, t, y):
        return self.net(y) #RCL modified

class RunningAverageMeter(object):
    """Computes and stores the average and current value"""

    def __init__(self, momentum=0.99):
        self.momentum = momentum
        self.reset()

    def reset(self):
        self.val = None
        self.avg = 0

    def update(self, val):
        if self.val is None:
            self.avg = val
        else:
            self.avg = self.avg * self.momentum + val * (1 - self.momentum)
        self.val = val

if __name__ == '__main__':

    ii = 0

    func = ODEFunc()
    SavedNNModel = ODEFunc()
    #optimizer = optim.RMSprop(func.parameters(), lr=1e-3)
    #optimizer = optim.RMSprop(func.parameters(), lr=1e-2)
    optimizer = optim.Adam(func.parameters(), lr=1e-2)
    end = time.time()

    time_meter = RunningAverageMeter(0.97)
    loss_meter = RunningAverageMeter(0.97)

    lossfunc = torch.nn.MSELoss()

    min_totalloss=1e10

    for itr in range(1, args.niters + 1):
        optimizer.zero_grad()
        batch_y0, batch_t, batch_y = get_batch()
        pred_y = odeint(func, batch_y0, batch_t, method=args.method)
        #loss = torch.mean(torch.abs(pred_y - batch_y))
        loss = lossfunc(pred_y, batch_y)
        loss.backward()
        optimizer.step()

        time_meter.update(time.time() - end)
        loss_meter.update(loss.item())

        if itr % args.test_freq == 0:
            with torch.no_grad():
                pred_y = odeint(func, true_y0, t, method=args.method)
                #loss = torch.mean(torch.abs(pred_y - true_y))
                loss = lossfunc(pred_y, true_y)
                print('Iter {:04d} | Total Loss {:.6f}'.format(itr, loss.item()))
                #visualize(true_y, pred_y, func, ii, viz=False)
                ii += 1
                if loss.item() < min_totalloss:
                    min_totalloss=loss.item()
                    min_iter=itr
                    SavedNNModel.load_state_dict(func.state_dict())
                    visualize(true_y, pred_y, func, ii, viz=True, filename='LV_Best_NNfit.png')
                if itr==1:
                    visualize(true_y, pred_y, func, ii, viz=True, filename='LV_Start_NNfit.png')

        end = time.time()

#Part two of fitting
print('Begin fitting f_TrueModel to f_NN')

class TrueODEFunc(nn.Module):

    def __init__(self):
        super(TrueODEFunc, self).__init__()

        #RCL Initial parameter values set to half the ground truth (from global feed)
        self.feed = {'alpha': torch.nn.Parameter(0.0*torch.tensor(feed['alpha'])),
                     'beta': (torch.tensor(feed['beta'])),
                     'gamma': torch.nn.Parameter(0.0*torch.tensor(feed['gamma'])),
                     'delta': (torch.tensor(feed['delta']))}

        #https://discuss.pytorch.org/t/nn-parameters-vs-nn-module-register-parameter/18641/2
        self.register_parameter('alpha',self.feed['alpha'])
        self.register_parameter('gamma',self.feed['gamma'])

    def forward(self, t, y):
        #RCL Lotka-Volterra modification
        #y may contain multiple batches, take the transpose
        x=y.t()
        dx = torch.zeros_like(x)
        dx[0] = self.feed['alpha']*x[0] - self.feed['beta']*x[0]*x[1]
        dx[1] = -self.feed['gamma']*x[1] + self.feed['delta']*x[0]*x[1]
        return dx.t()

#https://matplotlib.org/mpl_toolkits/mplot3d/tutorial.html
def plot_surface_3d_demo(model1,model2):
    '''
    ======================
    3D surface (color map)
    ======================

    Demonstrates plotting a 3D surface colored with the coolwarm color map.
    The surface is made opaque by using antialiased=False.

    Also demonstrates using the LinearLocator and custom formatting for the
    z axis tick labels.
    '''

    from mpl_toolkits.mplot3d import Axes3D
    import matplotlib.pyplot as plt
    from matplotlib import cm
    from matplotlib.ticker import LinearLocator, FormatStrFormatter
    import numpy as np

    fig = plt.figure()
    ax = fig.gca(projection='3d')

    # Make data.
    X = np.arange(0, 4, 0.25)
    Y = np.arange(0, 4, 0.25)
    X, Y = np.meshgrid(X, Y)

    #R = np.sqrt(X ** 2 + Y ** 2)
    #Z = np.sin(R.numpy())
    Z1 = np.zeros_like(X)
    Z2 = np.zeros_like(X)
    with torch.no_grad():
        for i in range(X.shape[0]):
            for j in range(X.shape[1]):
                Z1[i,j]=model1.forward(0,torch.tensor([[X[i,j],Y[i,j]]],dtype=torch.float))[0,0].numpy()
                Z2[i,j]=model2.forward(0,torch.tensor([[X[i,j],Y[i,j]]],dtype=torch.float))[0,0].numpy()
                #Z1[i,j]=model1.forward(0,torch.tensor([[X[i,j],Y[i,j]]],dtype=torch.float))[0,1].numpy()
                #Z2[i,j]=model2.forward(0,torch.tensor([[X[i,j],Y[i,j]]],dtype=torch.float))[0,1].numpy()

    # Plot the surface.
    surf1 = ax.plot_surface(X, Y, Z1, color='red', #cmap=cm.coolwarm,
                           linewidth=0, antialiased=False)
    surf2 = ax.plot_surface(X, Y, Z2, color='blue', #cmap=cm.coolwarm,
                           linewidth=0, antialiased=False)
    # Customize the z axis.
    ax.set_zlim(Z1.min(), Z1.max())
    ax.zaxis.set_major_locator(LinearLocator(10))
    ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))

    # Add a color bar which maps values to colors.
    #fig.colorbar(surf1, shrink=0.5, aspect=5)
    #fig.colorbar(surf2, shrink=0.5, aspect=5)

    plt.show()

if __name__ == '__main__':

    ii = 0

    TrueModel = TrueODEFunc()
    #RCL Lotka-Volterra modification
    #optimizer = optim.RMSprop(func.parameters(), lr=1e-2)
    #optimizer = optim.RMSprop([func.feed['alpha'],func.feed['gamma']], lr=1e-2)
    optimizer = optim.Adam(TrueModel.parameters(), lr=1e-1) #This one works too
    end = time.time()

    #RCL Use MSE instead of abs
    #lossfunc = torch.nn.MSELoss(reduction='elementwise_mean') #pytorch version 0.4.1, elementwise_mean is the default
    lossfunc = torch.nn.MSELoss()

    for itr in range(1, args.niters + 1):
        optimizer.zero_grad()
        NNf=torch.zeros([t.size()[0],2])
        Truef=torch.zeros([t.size()[0],2])
        for i in range(t.size()[0]):
            #print(NNf[i].shape,func.forward(0,true_y[i,0].view(1,-1)).shape,TrueModel.forward(0,true_y[i,0].view(1,-1)).shape)
            NNf[i]=SavedNNModel.forward(0,true_y[i,0].view(1,-1))
            Truef[i]=TrueModel.forward(0, true_y[i,0].view(1,-1))
        loss = lossfunc(Truef,NNf)
        loss.backward()
        optimizer.step()

        if itr % args.test_freq2 == 0:
            with torch.no_grad():
                pred_y1 = odeint(SavedNNModel, true_y0, t, method=args.method)
                pred_y2 = odeint(TrueModel, true_y0, t, method=args.method)
                # loss = torch.mean(torch.abs(pred_y - true_y))
                # RCL Use MSE instead of abs
                loss = lossfunc(pred_y1, pred_y2)
                print('Iter {:04d} | Total Loss {:.6f}'.format(itr, loss.item()))
                if itr==1:
                    visualize(pred_y1, pred_y2, TrueModel, ii, viz=True, filename='LV_Start_TrueModelFit.png')
                    print(TrueModel.feed)
                    #plot_surface_3d_demo(SavedNNModel, TrueModel)
                elif itr==args.niters:
                    visualize(pred_y1, pred_y2, TrueModel, ii, viz=True, filename='LV_Final_TrueModelFit.png')
                    print(TrueModel.feed)
                    plot_surface_3d_demo(SavedNNModel, TrueModel)
                ii += 1

        end = time.time()

ext: py

max_stars_repo_name: rlua/torchdiffeq

max_issues_repo_name: rlua/torchdiffeq

max_forks_repo_name: rlua/torchdiffeq

reason: import numpy


text:
import unittest
import numpy as np

from squidward.kernels import distance

# useful for debugging
np.set_printoptions(suppress=True)

class DistanceTestCase(unittest.TestCase):
    """Class for distance function tests."""

    def setUp(self):
        """
        Set Up
        Set up shared environment or variables for tests
        """

        self.alpha = np.array([0.70908325, 0.97063918, 0.2901618, 0.20842557, 0.12981446,
                               0.86155565, 0.71940193, 0.34456898, 0.65642592, 0.70362763])

        self.beta = np.array([0.89592768, 0.83254226, 0.67318587, 0.06753976, 0.4851254,
                              0.60731089, 0.29373965, 0.59003959, 0.40602443, 0.34742655])

class RBFTestCase(DistanceTestCase):
    """Tests for radial basis function."""

    def test_1(self):
        """
        Params Assertions
        Test case for invalid kernel parameter arguments.
        """

        # pass an invalid lengthscale
        with self.assertRaises(Exception) as context:
            distance.RBF(0.0,14.0**2)
        self.assertTrue('Lengthscale parameter must be greater than zero' in str(context.exception))

        # pass an invalid kernel variance
        with self.assertRaises(Exception) as context:
            distance.RBF(12.0,0.0**2)
        print( str(context.exception) )
        self.assertTrue('Kernel variance parameter must be greater than zero' in str(context.exception))

    def test_2(self):
        """
        Normal Input
        Test case for expected inputs.
        """
        d = distance.RBF(12.0, 14.0 ** 2)
        a = self.alpha
        b = self.beta

        true = 195.42750477495602
        output = d(a, b)
        self.assertEqual(output, true)

        a = a.reshape(-1, 1)
        b = b.reshape(-1, 1)

        true = 195.42750477495602
        output = d(a, b)
        self.assertEqual(output, true)

        a = np.random.rand(10, 2)
        with self.assertRaises(Exception) as context:
            output = d(a, a)
        self.assertTrue('Not appropriate input shape.' in str(context.exception))

if __name__ == '__main__':
    unittest.main()


text:
from functools import partial
from pathlib import Path

import pytest
from graphviz import ExecutableNotFound
from numpy.random import binomial
from synbn.dag import generate_dag

def test_generated_n_nodes():
    """Example test with parametrization."""
    distribution = partial(binomial, n=4, p=0.5)
    dag = generate_dag(20, distribution)
    assert len(dag.vs) == 20

def test_generated_plot():
    """Example test with parametrization."""
    plot_path = Path(__file__).parent / "plot.png"
    distribution = partial(binomial, n=5, p=0.5)
    dag = generate_dag(20, distribution)
    try:
        dag.plot(plot_path)
        assert plot_path.is_file()
        plot_path.unlink()
    except ExecutableNotFound:
        pass

max_stars_repo_name: Bradley-Butcher/pysynbn

max_issues_repo_name: Bradley-Butcher/pysynbn

max_forks_repo_name: Bradley-Butcher/pysynbn

reason: from numpy


text:
# Importing some modules and setting up a few properties
import sys
import os
from matplotlib import pyplot as plt
import numpy as np
import csv
from collections import Counter
sys.path.append(os.getcwd())
os.environ["DJANGO_SETTINGS_MODULE"] = "tsdb_app.settings"

import django
django.setup()
from core import backup

def main():
    backup.main()
    fund_source_location()
    fund_source_funding_possibility()
    organization_location()
    organization_category()
    organization_status()
    social_post_platform()
    product_price()
    consumer_location()
    consumer_age()

def height_calc(num):
    if num <= 2:
        return 2

    elif num <= 5:
        return 4

    elif num <= 10:
        return 7

    elif num <= 20:
        return 10

    else:
        return 15

def fund_source_location():
    with open("backupfiles/FundSourceBUF.csv", "r") as rfile:
        my_reader = csv.reader(rfile)
        location_dict = Counter()

        next(my_reader)
        for line in my_reader:
            location_dict.update([line[1]])

        temp_tuple_list = sorted(dict(location_dict).items(), key=lambda x:x[1])
        x_value = []
        y_value = []
        for tup in temp_tuple_list:
            x_value.append(tup[0])
            y_value.append(tup[1])
        height_num = len(x_value)

        plt.style.use("seaborn")
        plt.figure(figsize=(10,height_calc(height_num)))
        plt.barh(x_value, y_value , color="#0AB210")

        plt.title("FUND SOURCES' LOCATION")
        plt.xlabel("Number Of Fund Sources")
        plt.ylabel("Location")

        plt.tight_layout()
        plt.savefig("static/images/plots/plot1.png")

def fund_source_funding_possibility():
    with open("backupfiles/FundSourceBUF.csv", "r") as rfile:
        my_reader = csv.reader(rfile)
        possibility_dict = Counter()

        next(my_reader)
        for line in my_reader:
            possibility_dict.update([line[4]])

        temp_tuple_list = sorted(dict(possibility_dict).items(), key=lambda x:x[1])
        x_value = []
        y_value = []
        for tup in temp_tuple_list:
            x_value.append(tup[0])
            y_value.append(tup[1])

        colors = ["#0e8f13", "#0AB210", "#0ffc17"]
        plt.style.use("seaborn")
        plt.figure(figsize=(10,4))
        _, _, autotexts = plt.pie(y_value, labels=x_value , colors=colors, wedgeprops={"edgecolor":"white"}, autopct="%1.1f%%")
        for autotext in autotexts:
            autotext.set_color('white')

        plt.title("FUND SOURCES' FUNDING POSSIBILITY")
        plt.tight_layout()
        plt.savefig("static/images/plots/plot2.png")

def organization_location():
    with open("backupfiles/OrganizationBUF.csv", "r") as rfile:
        my_reader = csv.reader(rfile)
        location_dict = Counter()

        next(my_reader)
        for line in my_reader:
            location_dict.update([line[1]])

        temp_tuple_list = sorted(dict(location_dict).items(), key=lambda x:x[1])
        x_value = []
        y_value = []
        for tup in temp_tuple_list:
            x_value.append(tup[0])
            y_value.append(tup[1])
        height_num = len(x_value)

        plt.style.use("seaborn")
        plt.figure(figsize=(10,height_calc(height_num)))
        plt.barh(x_value, y_value , color="#0AB210")

        plt.title("ORGANIZATIONS' LOCATION")
        plt.xlabel("Number Of Organizations")
        plt.ylabel("Location")

        plt.tight_layout()
        plt.savefig("static/images/plots/plot3.png")

def organization_category():
    with open("backupfiles/OrganizationBUF.csv", "r") as rfile:
        my_reader = csv.reader(rfile)
        category_dict = Counter()

        next(my_reader)
        for line in my_reader:
            category_dict.update([line[2]])

        temp_tuple_list = sorted(dict(category_dict).items(), key=lambda x:x[1])
        x_value = []
        y_value = []
        for tup in temp_tuple_list:
            x_value.append(tup[0])
            y_value.append(tup[1])

        colors = ["#0e8f13", "#0AB210", "#0ffc17", "#9dff7d"]
        plt.style.use("seaborn")
        plt.figure(figsize=(10,4))
        _, _, autotexts = plt.pie(y_value, labels=x_value , colors=colors, wedgeprops={"edgecolor":"white"}, autopct="%1.1f%%")
        for autotext in autotexts:
            autotext.set_color('white')

        plt.title("ORGANIZATIONS' CATEGORY")
        plt.tight_layout()
        plt.savefig("static/images/plots/plot4.png")

def organization_status():
    with open("backupfiles/OrganizationBUF.csv", "r") as rfile:
        my_reader = csv.reader(rfile)
        status_dict = Counter()

        next(my_reader)
        for line in my_reader:
            status_dict.update([line[5]])

        temp_tuple_list = sorted(dict(status_dict).items(), key=lambda x:x[1])
        x_value = []
        y_value = []
        for tup in temp_tuple_list:
            x_value.append(tup[0])
            y_value.append(tup[1])

        colors = ["#0e8f13", "#0AB210"]
        plt.style.use("seaborn")
        plt.figure(figsize=(10,4))
        _, _, autotexts = plt.pie(y_value, labels=x_value , colors=colors, wedgeprops={"edgecolor":"white"}, autopct="%1.1f%%")
        for autotext in autotexts:
            autotext.set_color('white')

        plt.title("ORGANIZATIONS' STATUS")
        plt.tight_layout()
        plt.savefig("static/images/plots/plot5.png")

def social_post_platform():
    with open("backupfiles/SocialPostBUF.csv", "r") as rfile:
        my_reader = csv.reader(rfile)
        platform_dict = Counter()

        next(my_reader)
        for line in my_reader:
            platform_dict.update([line[0]])

        temp_tuple_list = sorted(dict(platform_dict).items(), key=lambda x:x[1])
        x_value = []
        y_value = []
        for tup in temp_tuple_list:
            x_value.append(tup[0])
            y_value.append(tup[1])

        colors = ["#191a18", "#0e8f13", "#0AB210", "#0ffc17", "#9dff7d"]
        plt.style.use("seaborn")
        plt.figure(figsize=(10,4))
        _, _, autotexts = plt.pie(y_value, labels=x_value , colors=colors, wedgeprops={"edgecolor":"white"}, autopct="%1.1f%%")
        for autotext in autotexts:
            autotext.set_color('white')

        plt.title("SOCIALPOSTS' PLATFORM")
        plt.tight_layout()
        plt.savefig("static/images/plots/plot6.png")

def product_price():
    with open("backupfiles/ProductBUF.csv", "r") as rfile:
        my_reader = csv.reader(rfile)
        price_list = []

        next(my_reader)
        for line in my_reader:
            price = int(line[1])
            price_list.append(price)

        lowest = min(price_list)
        highest = max(price_list)
        bins = [0]

        current = lowest
        while current <= highest:
            bins.append(current)
            current += 500
        bins.append(current)

        plt.style.use("seaborn")
        plt.figure(figsize=(10, 6))
        plt.hist(price_list, bins=bins, color="#0AB210", edgecolor="#ffffff")

        plt.title("PRODUCTS' PRICE")
        plt.xlabel("Price")
        plt.ylabel("Number of Products")

        plt.tight_layout()
        plt.savefig("static/images/plots/plot7.png")

def consumer_location():
    with open("backupfiles/ConsumerBUF.csv", "r") as rfile:
        my_reader = csv.reader(rfile)
        location_dict = Counter()

        next(my_reader)
        for line in my_reader:
            location_dict.update([line[3]])

        temp_tuple_list = sorted(dict(location_dict).items(), key=lambda x:x[1])
        x_value = []
        y_value = []
        for tup in temp_tuple_list:
            x_value.append(tup[0])
            y_value.append(tup[1])
        height_num = len(x_value)

        plt.style.use("seaborn")
        plt.figure(figsize=(10,height_calc(height_num)))
        plt.barh(x_value, y_value , color="#0AB210")

        plt.title("CONSUMERS' LOCATION")
        plt.xlabel("Number Of Consumers")
        plt.ylabel("Location")

        plt.tight_layout()
        plt.savefig("static/images/plots/plot8.png")

def consumer_age():
    with open("backupfiles/ConsumerBUF.csv", "r") as rfile:
        my_reader = csv.reader(rfile)
        age_list = []

        next(my_reader)
        for line in my_reader:
            age = int(line[2])
            age_list.append(age)

        lowest = min(age_list)
        highest = max(age_list)
        bins = [0]

        current = lowest
        while current <= highest:
            bins.append(current)
            current += 5
        bins.append(current)

        plt.style.use("seaborn")
        plt.figure(figsize=(10, 6))
        plt.hist(age_list, bins=bins, color="#0AB210", edgecolor="#ffffff")

        plt.title("CONSUMERS' AGE")
        plt.xlabel("Age")
        plt.ylabel("Number of Consumers")

        plt.tight_layout()
        plt.savefig("static/images/plots/plot9.png")


text:
import random
import numpy as np

def get_n_first_elements(py_list, n_elements):
    """Get the first `n` elements of a list.
    Args:
        py_list (list): A list of elements.
        n_elements (int): The number of elements.
    Returns:
        sub_list (list): A list with the first `n` elements of `py_list`.
    Examples:
        >>> py_list = [1,2,3,4,5,6,7,8]
        >>> py_list_first = get_n_first_elements(py_list, 2)
        >>> py_list_first
        [1, 2]

    """
    return py_list[:n_elements]

def get_n_last_elements(py_list, n_elements):
    """Get the last `n` elements of a list.
    Args:
        py_list (list): A list of elements.
        n_elements (int): The number of elements.
    Returns:
        sub_list (list): A list with the last `n` elements of `py_list`.
    Examples:
        >>> py_list = [1,2,3,4,5,6,7,8]
        >>> py_list_first = get_n_last_elements(py_list, 2)
        >>> py_list_first
        [7, 8]

    """
    return py_list[-n_elements:]

def find_item_index(py_list, item):
    """Find the index of an item in a list.
    Args:
        py_list (list): A list of elements.
        item (int or str): The element in the list.
        n (int): The upper limit of the range to generate, from 0 to `n` - 1.
    Returns:
        index (int): the index of the element.
    Examples:
        >>> py_list = ["foo", "bar", "baz"]
        >>> item = "bar"
        >>> idx = find_item_index(py_list, item)
        >>> print("Index: %d" % idx)
        Index: 1

    """
    return py_list.index(item)

def generate_random_integers(number_values, min_val, max_val):
    """Generate a list with `n` random integers between a `minimum` and a `maximum`.
    Args:
        number_values (int): Number of values to generate.
        min_val (int): Minimum value.
        max_val (int): Maximum value.
    Returns:
        result_list (list): A list with random values.
    Examples:
        >>> result_list = generate_random_integers(number_values=5, min_val=0, max_val=10)
        >>> len(result_list)
        5
        >>> min(result_list)>=0 and max(result_list)<=10
        True

    """
    l = random.sample(range(min_val, max_val), number_values)
    return l

def shuffle_list(py_list, inplace=False):
    """Shuffle a list
    If `inplace=True` the input list is modified (faster & less memory), if `inplace=False` a new list is
    generated (slower & more memory).
    Args:
        py_list (list): A list of elements.
        inplace (bool): If `True` the input list is modified, if `False` a new list is generated.
    Returns:
        result_list (list): A list with the values shuffled.
    Examples:
        >>> py_list = [1,2,3,4,5]
        >>> l = shuffle_list(py_list, False)
        >>> all(elem in l  for elem in py_list)
        True

    """
    if inplace:
        random.shuffle(py_list)
    else:
        # fastest way to shallow copy a list https://stackoverflow.com/a/2612990
        l = py_list[:]
        random.shuffle(l)
        return l

def reverse_list(py_list, inplace=True):
    """Reverse a list.
    If `inplace=True` the input list is modified (faster & less memory), if
    `inplace=False` a new list is generated (slower & more memory).
    Args:
        py_list (list): A list of elements.
        inplace (bool): If `True` the input list is modified, if `False` a new
        list is generated.
    Returns:
        result_list (list): A list with the values reversed.
    Examples:
        >>> py_list = [1,2,3,4,5]
        >>> reverse_list(py_list, False)
        [5, 4, 3, 2, 1]
        >>> reverse_list(py_list, True)
        [5, 4, 3, 2, 1]

    """
    if inplace:
        py_list.reverse()
        # py_list[::-1] #don't use, much slower than reverse()
        return py_list
    else:
        result_list = list(reversed(py_list))
        return result_list

def create_consecutive_numbers(initial_num, final_num):
    """Create a list of consecutive numbers from `initial_num` to `final_num`.
    source: http://interactivepython.org/runestone/static/pythonds/AlgorithmAnalysis/Lists.html
    Args:
        initial_num (int): The first number of the series.
        final_num (int): The last number of the series.
    Returns:
        result_list (list): A list with values from `initial_num` to
                            `final_num`.
    Examples:
        >>> create_consecutive_numbers(1,5)
        [1, 2, 3, 4, 5]

    """
    # Method 1: append
    # Do not use, slowest of all methods
    # l = []
    # for i in range(initial_num, final_num+1):
    #    l.append(i)

    # Method 2: list comprehension
    # Twice as fast as append
    # l = [i for i in range(initial_num, final_num+1)]

    # Method 3: list constructor
    # Twice as fast as list comprehension
    l = list(range(initial_num, final_num + 1))
    return l

def flatten_list_of_lists(list_of_lists):
    """Convert a list os lists to a single list
    Args:
         list_of_lists (list of lists): A list of lists.
    Returns:
        result_list (list): A list
    Examples:
        >>> list_of_lists = [[1,2,3],[0,3,9]]
        >>> flatten_list_of_lists(list_of_lists)
        [1, 2, 3, 0, 3, 9]

    """
    result_list = [item for sublist in list_of_lists for item in sublist]
    return result_list

def split_list(py_list, perc_size=[0.8, 0.2], shuffle=False):
    """Split a list in weighted chunks
    Args:
        py_list (list): A list of elements.
        perc_size (list): The percentual size of each chunk size.
        shuffle (bool): Shuffle the list or not
    Returns:
        result_list (list of list): A list of lists with the chunks.
    Examples:
        >>> split_list(list(range(7)),[0.47,0.33,0.2])
        [[0, 1, 2], [3, 4, 5], [6]]

    """
    assert sum(perc_size) == 1, "Percentage sizes do not sum to 1"
    l = py_list[:]
    if shuffle:
        random.shuffle(l)
    # Turn percentages into values between 0 and 1
    splits = np.cumsum(perc_size)

    # Split doesn't need last percent, it will just take what is left
    splits = splits[:-1]

    # Turn values into indices
    splits *= len(l)

    # Turn double indices into integers.
    splits = splits.round().astype(np.int)

    return [list(chunks) for chunks in np.split(l, splits)]

def intersection(list1, list2):
    """Intersection of two lists, returns the common elements in both lists.
    Args:
        list1 (list): A list of elements.
        list2 (list): A list of elements.
    Returns:
        result_list (list): A list with the common elements.
    Examples:
        >>> intersection([1,2,3], [2,3,4])
        [2, 3]

    """
    return list(set(list1) & set(list2))

def union(list1, list2):
    """Union of two lists, returns the elements that appear in one list OR the
    other.
    Args:
        list1 (list): A list of elements.
        list2 (list): A list of elements.
    Returns:
        result_list (list): A list with the union elements.
    Examples:
        >>> union([1,2,3], [2,3,4])
        [1, 2, 3, 4]

    """
    return list(set(list1) | set(list2))

def unique(list1, list2):
    """Get the unique items that are in the first list but not in the second
    list.
    NOTE: unique(l1,l2) is not always equal to unique(l2,l1)

    Args:
        list1 (list): A list of elements.
        list2 (list): A list of elements.

    Returns:
        list: A list with the unique elements.

    Examples:
        >>> unique([1,2,3], [2,3,4])
        [1]
        >>> unique([2,3,4], [1,2,3])
        [4]
    """
    return list(set(list1) - set(list2))

max_stars_repo_name: miguelgfierro/pybase

max_issues_repo_name: miguelgfierro/pybase

max_forks_repo_name: miguelgfierro/pybase


text:
import json
import numpy as np
import os

from photogrammetry_importer.types.camera import Camera
from photogrammetry_importer.types.point import Point
from photogrammetry_importer.utility.blender_camera_utility import check_radial_distortion
from photogrammetry_importer.utility.blender_logging_utility import log_report

def get_element(data_list, id_string, query_id, op):
    result = None
    for ele in data_list:
        if int(ele[id_string]) == query_id:
            result = ele
            break
    assert result is not None
    return result

class MeshroomFileHandler:

    # Note: *.SfM files are actually just *.JSON files.

    @staticmethod
    def _parse_cameras_from_json_data(json_data, image_dp, image_fp_type, suppress_distortion_warnings, op):

        cams = []
        image_index_to_camera_index = {}

        is_valid_file = 'views' in json_data and 'intrinsics' in json_data and 'poses' in json_data

        if not is_valid_file:
            op.report(
                {'ERROR'},
                'FILE FORMAT ERROR: Incorrect SfM/JSON file. Must contain the SfM reconstruction results: ' +
                'view, intrinsics and poses.')
            return cams, image_index_to_camera_index

        views = json_data['views']              # is a list of dicts (view)
        intrinsics = json_data['intrinsics']    # is a list of dicts (intrinsic)
        extrinsics = json_data['poses']         # is a list of dicts (extrinsic)

        # IMPORTANT:
        # Views contain the number of input images
        # Extrinsics may contain only a subset of views!
        # (Not all views are necessarily contained in the reconstruction)

        for rec_index, extrinsic in enumerate(extrinsics):

            camera = Camera()
            view_index = int(extrinsic['poseId'])
            image_index_to_camera_index[view_index] = rec_index

            corresponding_view = get_element(
                views, "poseId", view_index, op)

            camera.image_fp_type = image_fp_type
            camera.image_dp = image_dp
            camera._absolute_fp = str(corresponding_view['path'])
            camera._relative_fp = os.path.basename(str(corresponding_view['path']))
            camera._undistorted_relative_fp = str(extrinsic['poseId']) + '.exr'
            if image_dp is None:
                camera._undistorted_absolute_fp = None
            else:
                camera._undistorted_absolute_fp = os.path.join(
                    image_dp, camera._undistorted_relative_fp)

            camera.width = int(corresponding_view['width'])
            camera.height = int(corresponding_view['height'])
            id_intrinsic = int(corresponding_view['intrinsicId'])

            intrinsic_params = get_element(
                intrinsics, "intrinsicId", id_intrinsic, op)

            focal_length = float(intrinsic_params['pxFocalLength'])
            cx = float(intrinsic_params['principalPoint'][0])
            cy = float(intrinsic_params['principalPoint'][1])

            if 'distortionParams' in intrinsic_params and len(intrinsic_params['distortionParams']) > 0:
                # TODO proper handling of distortion parameters
                radial_distortion = float(intrinsic_params['distortionParams'][0])
            else:
                radial_distortion = 0.0

            if not suppress_distortion_warnings:
                check_radial_distortion(radial_distortion, camera._relative_fp, op)

            camera_calibration_matrix = np.array([
                [focal_length, 0, cx],
                [0, focal_length, cy],
                [0, 0, 1]])

            camera.set_calibration(
                camera_calibration_matrix,
                radial_distortion)
            extrinsic_params = extrinsic['pose']['transform']

            cam_rotation_list = extrinsic_params['rotation']
            camera.set_rotation_mat(
                np.array(cam_rotation_list, dtype=float).reshape(3,3).T)
            camera.set_camera_center_after_rotation(
                np.array(extrinsic_params['center'], dtype=float))
            camera.view_index = view_index

            cams.append(camera)
        return cams, image_index_to_camera_index

    @staticmethod
    def _parse_points_from_json_data(json_data, image_index_to_camera_index, op):

        points = []
        is_valid_file = 'structure' in json_data

        if not is_valid_file:
            op.report(
                {'ERROR'},
                'FILE FORMAT ERROR: Incorrect SfM/JSON file. Must contain the SfM reconstruction results: structure.')
            return points

        structure = json_data['structure']
        for json_point in structure:
            custom_point = Point(
                coord=np.array(json_point['X'], dtype=float),
                color=np.array(json_point['color'], dtype=int),
                id=int(json_point['landmarkId']),
                scalars=[])
            points.append(custom_point)
        return points

    @staticmethod
    def parse_sfm_file(sfm_sfm_fp, image_dp, image_fp_type, suppress_distortion_warnings, op):
        """
        Parses cameras.sfm/sfm.sfm/sfm.json files created by the StructureFromMotion / ConvertSfMFormat node in Meshroom
        :param sfm_sfm_fp:
        :return:
        """
        op.report({'INFO'}, 'parse_sfm_sfm_file: ...')
        op.report({'INFO'},'sfm_sfm_fp: ' + sfm_sfm_fp)
        input_file = open(sfm_sfm_fp, 'r')
        json_data = json.load(input_file)

        cams, image_index_to_camera_index = MeshroomFileHandler._parse_cameras_from_json_data(
            json_data, image_dp, image_fp_type, suppress_distortion_warnings, op)
        if 'structure' in json_data:
            points = MeshroomFileHandler._parse_points_from_json_data(
                json_data, image_index_to_camera_index, op)
        else:
            points = []
        op.report({'INFO'},'parse_sfm_sfm_file: Done')
        return cams, points

    @staticmethod
    def get_data_fp_of_node(cache_dp, data_node, fn_or_fn_list):
        if isinstance(fn_or_fn_list, str):
            fn_list = [fn_or_fn_list]
        else:
            fn_list = fn_or_fn_list
        data_fp = None
        if data_node is not None:
            node_type = data_node['nodeType']
            uid_0 = data_node['uids']['0']
            for fn in fn_list:
                possible_data_fp = os.path.join(cache_dp, node_type, uid_0, fn)
                if os.path.isfile(possible_data_fp):
                    data_fp = possible_data_fp
                    break
        return data_fp

    @staticmethod
    def get_latest_node(json_graph, node_type):
        i = 0
        while node_type + "_" + str(i + 1) in json_graph:
            i = i + 1
        if i == 0:
            return None
        else:
            return json_graph[node_type + "_" + str(i)]

    @staticmethod
    def get_node(json_graph, node_type, node_number, op):
        if node_number == -1:
            return MeshroomFileHandler.get_latest_node(json_graph, node_type)
        else:
            node_key = node_type + "_" + str(node_number)
            if node_key in json_graph:
                return json_graph[node_key]
            else:
               log_report(
                   'ERROR', 'Invalid combination of node type (i.e. ' + node_type + ') ' +
                   'and node number (i.e. ' + str(node_number) + ') provided', op)
               assert False

    @staticmethod
    def get_node_data_fp(cache_dp, json_graph, node_type, node_number, fn_or_fn_list, op):
        data_node = MeshroomFileHandler.get_node(json_graph, node_type, node_number, op)
        data_fp = MeshroomFileHandler.get_data_fp_of_node(cache_dp, data_node, fn_or_fn_list)
        return data_fp

    @staticmethod
    def parse_meshrom_mg_file(mg_fp, sfm_node_type, sfm_node_number, mesh_node_type, mesh_node_number, op):
        cache_dp = os.path.join(os.path.dirname(mg_fp), 'MeshroomCache')
        json_data = json.load(open(mg_fp, 'r'))
        json_graph = json_data['graph']

        if sfm_node_type == 'ConvertSfMFormatNode':
            sfm_fp = MeshroomFileHandler.get_node_data_fp(
                cache_dp, json_graph, 'ConvertSfMFormat', sfm_node_number, ['sfm.sfm', 'sfm.json'], op)
        elif sfm_node_type == 'StructureFromMotionNode':
            sfm_fp = MeshroomFileHandler.get_node_data_fp(
                cache_dp, json_graph, 'StructureFromMotion', sfm_node_number, 'cameras.sfm', op)
        elif sfm_node_type == 'AUTOMATIC':
            sfm_fp = MeshroomFileHandler.get_node_data_fp(
                cache_dp, json_graph, 'ConvertSfMFormat', sfm_node_number, ['sfm.sfm', 'sfm.json'], op)
            if sfm_fp is None:
                sfm_fp = MeshroomFileHandler.get_node_data_fp(
                    cache_dp, json_graph, 'StructureFromMotion', sfm_node_number, 'cameras.sfm', op)
        else:
            log_report('ERROR', 'Select SfM node is not supported', op)
            assert False

        if mesh_node_type == 'Texturing':
            mesh_fp = MeshroomFileHandler.get_node_data_fp(
                cache_dp, json_graph, 'Texturing', mesh_node_number, 'texturedMesh.obj', op)
        elif mesh_node_type == 'MeshFiltering':
            mesh_fp = MeshroomFileHandler.get_node_data_fp(
                cache_dp, json_graph, 'MeshFiltering', mesh_node_number, 'mesh.obj', op)
        elif mesh_node_type == 'Meshing':
            mesh_fp = MeshroomFileHandler.get_node_data_fp(
                cache_dp, json_graph, 'Meshing', mesh_node_number, 'mesh.obj', op)
        elif mesh_node_type == 'AUTOMATIC':
            mesh_fp = MeshroomFileHandler.get_node_data_fp(
                cache_dp, json_graph, 'Texturing', mesh_node_number, 'texturedMesh.obj', op)
            if mesh_fp is None:
                mesh_fp = MeshroomFileHandler.get_node_data_fp(
                    cache_dp, json_graph, 'MeshFiltering', mesh_node_number, 'mesh.obj', op)
            if mesh_fp is None:
                mesh_fp = MeshroomFileHandler.get_node_data_fp(
                    cache_dp, json_graph, 'Meshing', mesh_node_number, 'mesh.obj', op)
        else:
            log_report('ERROR', 'Select Mesh node is not supported', op)
            assert False

        if sfm_fp is not None:
            log_report('INFO', 'Found the following sfm file: ' + sfm_fp, op)
        else:
            log_report('INFO', 'Request target SfM result does not exist in this meshroom project', op)

        if mesh_fp is not None:
            log_report('INFO', 'Found the following mesh file: ' + mesh_fp, op)
        else:
            log_report('INFO', 'Request target mesh does not exist in this meshroom project', op)

        return sfm_fp, mesh_fp

    @staticmethod
    def parse_meshroom_file(meshroom_ifp, image_dp, image_fp_type, suppress_distortion_warnings,
                            sfm_node_type, sfm_node_number, mesh_node_type, mesh_node_number, op):
        """
        :param meshroom_ifp:
        :return:
        """
        op.report({'INFO'}, 'parse_meshroom_file: ...')
        op.report({'INFO'},'meshroom_ifp: ' + meshroom_ifp)

        ext = os.path.splitext(meshroom_ifp)[1].lower()
        if ext == '.mg':
            meshroom_ifp, mesh_fp = MeshroomFileHandler.parse_meshrom_mg_file(
                meshroom_ifp, sfm_node_type, sfm_node_number, mesh_node_type, mesh_node_number, op)
        else:
            assert ext == '.json' or ext == '.sfm'
            mesh_fp = None

        if meshroom_ifp is not None:
            cams, points = MeshroomFileHandler.parse_sfm_file(
                meshroom_ifp, image_dp, image_fp_type, suppress_distortion_warnings, op)
        else:
            log_report('WARNING', 'Meshroom project does not contain cameras or points', op)
            cams = []
            points = []

        op.report({'INFO'},'parse_meshroom_file: Done')
        return cams, points, mesh_fp


text:
# Copyright 2018 The TensorFlow Probability Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Tests of the No U-Turn Sampler."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

# Dependency imports
from absl import flags
import numpy as np
import tensorflow.compat.v1 as tf
import tensorflow_probability as tfp

from tensorflow_probability.python.experimental.auto_batching import tf_backend
from tensorflow_probability.python.internal import test_util

tfd = tfp.distributions

flags.DEFINE_string('test_device', None,
                    'TensorFlow device on which to place operators under test')
FLAGS = flags.FLAGS

def run_nuts_chain(event_size, batch_size, num_steps):
  def f():
    def target_log_prob_fn(event):
      return tfd.MultivariateNormalDiag(
          tf.zeros(event_size),
          scale_identity_multiplier=1.).log_prob(event)

    state = tf.zeros([batch_size, event_size])
    chain_state, extra = tfp.mcmc.sample_chain(
        num_results=num_steps,
        num_burnin_steps=0,
        current_state=[state],
        kernel=tfp.experimental.mcmc.NoUTurnSampler(
            target_log_prob_fn,
            step_size=[0.3],
            use_auto_batching=True,
            seed=1,
            backend=tf_backend.TensorFlowBackend(
                safety_checks=False, while_parallel_iterations=1)),
        parallel_iterations=1)
    return chain_state, extra.leapfrogs_taken
  return f

class NutsXLATest(test_util.TestCase):

  def testMultivariateNormalNd(self, event_size=32, batch_size=8, num_steps=2):
    tf.set_random_seed(3)
    with tf.device(FLAGS.test_device):
      f = run_nuts_chain(event_size, batch_size, num_steps)
      f = tf.function(f, autograph=False, jit_compile=True)
      samples, leapfrogs = self.evaluate(f())

    # TODO(axch) Figure out what the right thing to test about the leapfrog
    # count really is and test it, instead of just flailing around like this
    # does.
    print(type(samples), type(leapfrogs))
    print(samples, leapfrogs)
    ev_leapfrogs = leapfrogs[0]
    self.assertGreater(len(set(ev_leapfrogs.tolist())), 1)
    self.assertTrue(all(ev_leapfrogs > 1))

  def test_xla_compile_and_correctness(self):
    tf.set_random_seed(3)
    nsamples = 1000
    nchains = 10
    nd = 5
    theta0 = np.zeros((nchains, nd))
    mu = np.arange(nd)
    stddev = np.arange(nd) + 1.
    rng = np.random.RandomState(seed=4)
    step_size = rng.rand(nchains, 1)*.1 + 1

    num_steps = nsamples
    initial_state = tf.cast(theta0, dtype=tf.float32)
    unrolled_leapfrog_steps = 2

    @tf.function(autograph=False)
    def target_log_prob_fn(event):
      with tf.name_scope('nuts_test_target_log_prob'):
        return tfd.MultivariateNormalDiag(
            loc=tf.cast(mu, dtype=tf.float32),
            scale_diag=tf.cast(stddev, dtype=tf.float32)).log_prob(event)

    @tf.function(autograph=False, jit_compile=True)
    def _run_nuts_chain():
      kernel = tfp.mcmc.NoUTurnSampler(
          target_log_prob_fn,
          step_size=[tf.cast(step_size, dtype=tf.float32)],
          unrolled_leapfrog_steps=unrolled_leapfrog_steps)
      [x], (is_accepted, leapfrogs_taken) = tfp.mcmc.sample_chain(
          num_results=num_steps,
          num_burnin_steps=0,
          current_state=[initial_state],
          kernel=kernel,
          trace_fn=lambda _, pkr: (pkr.is_accepted, pkr.leapfrogs_taken),
          seed=9)
      return (
          tf.reduce_mean(x, axis=[0, 1]),
          tf.math.reduce_std(x, axis=[0, 1]),
          is_accepted,
          leapfrogs_taken
          )

    with tf.device(FLAGS.test_device):
      [
          sample_mean, sample_stddev, is_accepted, leapfrogs_taken
      ] = self.evaluate(_run_nuts_chain())

    self.assertAllClose(mu, sample_mean, atol=0.1, rtol=0.1)
    self.assertAllClose(stddev, sample_stddev, atol=0.15, rtol=0.15)
    # Test early stopping in tree building
    self.assertTrue(np.any(np.isin(
        np.asarray([5, 9, 11, 13]) * unrolled_leapfrog_steps,
        np.unique(leapfrogs_taken[is_accepted]))))

if __name__ == '__main__':
  tf.test.main()


text:
import torch
import numpy as np
import scipy.io as sio
import random
import model.utils as utils
import model.pack as pack
import model.noCluster as noCluster
import sys
import os

if len(sys.argv) != 8:
    print(sys.argv)
    print('Usage: run.py -DATA -outputDropout(0.2) -inputDropout(0) -batchSize(20) -embLen(50) -bagWeighting(none/ave/att) -randomseed(1234) -info')
    exit(1)

SEED = int(sys.argv[6])
print('Using Random Seed: '+str(SEED))
torch.manual_seed(SEED)
np.random.seed(SEED)
random.seed(SEED)

dataset = sys.argv[1]
drop_prob = float(sys.argv[2])
repack_ratio = float(sys.argv[3])
bat_size = int(sys.argv[4])
embLen = int(sys.argv[5])
bagWeighting = sys.argv[6]
info = sys.argv[7]

train_file = './data/intermediate/' + dataset + '/rm/train.data'
dev_file = './data/intermediate/' + dataset + '/rm/dev.data'
test_file = './data/intermediate/' + dataset + '/rm/test.data'

feature_file = './data/intermediate/' + dataset + '/rm/feature.txt'
type_file = './data/intermediate/' + dataset + '/rm/type.txt'
type_file_test = './data/intermediate/' + dataset + '/rm/type_test.txt'
none_ind = utils.get_none_id(type_file)
print("None id:", none_ind)

label_distribution = utils.get_distribution(type_file)
label_distribution_test = utils.get_distribution(type_file_test)

word_size, pos_embedding_tensor = utils.initialize_embedding(feature_file, embLen)
_, type_size, _, _, _ = utils.load_corpus(train_file)

nocluster = noCluster.noCluster(embLen, word_size, type_size, drop_prob, label_distribution, label_distribution_test)
nocluster.load_state_dict(torch.load('./dumped_models/ffnn_dump_'+'_'.join(sys.argv[1:7])+'.pth'))

torch.cuda.set_device(0)
nocluster.cuda()
if_cuda = True

packer = pack.repack(repack_ratio, 20, if_cuda)

print('in the order of: train, dev, test...\n')
for file in [train_file, dev_file, test_file]:
    doc_size_test, _, feature_list_test, label_list_test, type_list_test = utils.load_corpus(file)

    fl_t, of_t = packer.repack_eva(feature_list_test)

    nocluster.eval()
    scores = nocluster(fl_t, of_t, 'test')
    ind = utils.calcInd(scores)
    entropy = utils.calcEntropy(scores)
    maxprob = utils.calcMaxProb(scores)

    golden_list = []
    predict_list = []

    # vanilla prediction

    for i in range(len(label_list_test)):
        golden_list.append(label_list_test[i][0])
        predict_list.append(ind.data[i])

    f1score, recall, precision, val_f1, pn_precision, nprecision, nrecall, accuracy = utils.noCrossValidation(ind.data, entropy.data, label_list_test, ind.data, entropy.data, label_list_test, none_ind)
    print('F1 = %.4f, recall = %.4f, precision = %.4f, pn_precision = %.4f, accuracy = %.4f' % (f1score, recall, precision, pn_precision, accuracy))
    print('NoneType precision: ', nprecision, ' recall: ', nrecall)

    if file == test_file:
        print(len(golden_list))
        filename = './case_study/' + dataset + '_case_study.txt'
        file = open(filename, 'w')
        file.write(str([predict_list[i] for i in range(len(predict_list))])+'\n')
        file.write(str([golden_list[i] for i in range(len(golden_list))])+'\n')
        file.write('F1 = %.4f, recall = %.4f, precision = %.4f, accuracy = %.4f' % (f1score, recall, precision, accuracy))
        file.close()

        intermediate_emb_bag = nocluster.men_embedding.data.tolist()
        if not os.path.exists('./case_study/%s' % info):
            os.mkdir('./case_study/%s' % info)
        filename = './case_study/%s/'%info + dataset + '_embedding_test.mat'
        #file = open(filename, 'w')
        #for i in range(len(intermediate_emb_bag)):
        #    file.write(str(list(intermediate_emb_bag[i])))
        #file.close()
        sio.savemat(filename, {'inter_emb': intermediate_emb_bag})

filename = './case_study/' + dataset + '_case_study.txt'
file = open(filename, 'a')
# max threshold
ndevF1, f1score, recall, precision, meanBestF1, pn_precision, bestThres, acc = utils.CrossValidation_New(ind.data, maxprob.data, label_list_test, ind.data, maxprob.data, label_list_test, none_ind, thres_type='max')
print('Max Thres \tF1 = %.4f, recall = %.4f, precision = %.4f, ndevF1 = %.4f, cdevF1 = %.4f, pn_precision = %.4f' % (f1score, recall, precision, ndevF1, meanBestF1, pn_precision))
print('Accuracy: %.4f' % acc)
pre_ind_cutoff = utils.eval_score_with_thres(ind.data, maxprob.data, label_list_test,none_ind,bestThres,thres_type='max')
file.write(str([pre_ind_cutoff[i] for i in range(len(pre_ind_cutoff))])+'\n')

# entropy threshold
ndevF1, f1score, recall, precision, meanBestF1, pn_precision, bestThres, acc = utils.CrossValidation_New(ind.data, entropy.data, label_list_test, ind.data, entropy.data, label_list_test, none_ind, thres_type='entropy')
print('Entropy Thres \tF1 = %.4f, recall = %.4f, precision = %.4f, ndevF1 = %.4f, cdevF1 = %.4f, pn_precision = %.4f' % (f1score, recall, precision, ndevF1, meanBestF1, pn_precision))
print('Accuracy: %.4f' % acc)
pre_ind_cutoff = utils.eval_score_with_thres(ind.data, entropy.data, label_list_test,none_ind,bestThres,thres_type='entropy')
file.write(str([pre_ind_cutoff[i] for i in range(len(pre_ind_cutoff))])+'\n')
file.close()

# bias adjustment
doc_size_test, _, feature_list_test, label_list_test, type_list_test = utils.load_corpus(test_file)
#data = [[feature_list_test[i], label_list_test[i]] for i in range(len(feature_list_test))]
#random.shuffle(data)
labels, feature = utils.shuffle_data(label_list_test, feature_list_test)
eva_n = int(0.1 * len(label_list_test))

eva_label = labels[:eva_n]
eva_label_distribution = utils.get_distribution_from_list(eva_label)
print(eva_label_distribution)
print(label_distribution_test)
#eva_feature = feature[:eva_n]

test_label = labels[eva_n:]
test_feature = feature[eva_n:]

fl_t, of_t = packer.repack_eva(test_feature)

nocluster.eval()
scores = nocluster.test_with_bias(fl_t, of_t, eva_label_distribution)
#scores = nocluster(fl_t, of_t, 'test')
ind = utils.calcInd(scores)
# golden_list = []
# predict_list = []
#
# for i in range(len(test_label)):
#     golden_list.append(test_label[i][0])
#     predict_list.append(ind.data[i])
f1score, recall, precision, val_f1, pn_precision, nprecision, nrecall, accuracy = utils.noCrossValidation(ind.data, entropy.data, test_label, ind.data, entropy.data, test_label, none_ind)
print('F1 = %.4f, recall = %.4f, precision = %.4f, pn_precision = %.4f, accuracy = %.4f' % (f1score, recall, precision, pn_precision, accuracy))
print('NoneType precision: ', nprecision, ' recall: ', nrecall)

# python3 FFNN/test_manual.py KBP 0.5 0.1 80 50 none 4 none

n = len(label_distribution)
weights = [label_distribution[i]/label_distribution_test[i] for i in range(n)]
f1score, recall, precision = utils.evalWeightedF1(ind.data, entropy.data, test_label, none_ind, weights)
print('Using Weighted Metrics:')
print('F1 = %4f, recall = %.4f, precision = %.4f' % (f1score, recall, precision))

reason: import numpy,import scipy


text:
#!/usr/bin/env python
# coding: utf-8

# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.

"""
Data widgets for numpy arrays.
"""

from contextlib import contextmanager

from ipywidgets import register
from traitlets import Unicode, Set, Undefined, Int, validate
import numpy as np
import six

from ..widgets import DataWidget
from .traits import NDArray
from .serializers import compressed_array_serialization

class NDArrayBase(DataWidget):
    """A common base class for NDArray-based widgets
    """

    @property
    def shape(self):
        return self._get_shape()

    @property
    def dtype(self):
        return self._get_dtype()

    def _get_shape(self):
        raise NotImplementedError()

    def _get_dtype(self):
        raise NotImplementedError()

@register
class NDArraySource(NDArrayBase):
    """Base class for widgets that supplies an ndarray in the front-end only.
    """
    pass

@register
class NDArrayWidget(NDArrayBase):
    """A widget representing an arbitrary array.

    This is useful when several widgets might share the
    same array. This would allow better synchronization.

    More specific subtypes can be used instead to allow for implied
    types and dimensionality checks.
    """
    _model_name = Unicode('NDArrayModel').tag(sync=True)

    _segments_to_send = Set()

    array = NDArray().tag(sync=True, **compressed_array_serialization)

    compression_level = Int(0,
        help='If above 0, compress the data with zlib during serialization. '
        'Note: It is often more efficient to turn on compression on the '
        'notebook application level than to use this option.').tag(sync=True)

    def __init__(self, array=Undefined, **kwargs):
        self._instance_validators = set()
        super(NDArrayWidget, self).__init__(array=array, **kwargs)

    def _get_shape(self):
        return self.array.shape

    def _get_dtype(self):
        return self.array.dtype

    @validate('array')
    def _validate_array(self, proposal):
        """Validate array against external validators (instance only)

        This allows others to add constraints on the array of this
        widget dynamically. Internal use is so that a constrained
        DataUnion can validate the array of a widget set to itself.
        """
        value = proposal['value']
        for validator in self._instance_validators:
            value = validator(value)
        return value

    def notify_changed(self):
        """Use this to mark that the array is changed.

        This will cause the array to be synced as it normally would
        after a change, and is useful when the array has been modified
        in-place.

        This respects hold_trait_notifications and hold_sync.
        """
        self._notify_trait('array', self.array, self.array)

    def sync_segment(self, segments):
        """Sync a segments of contiguous memory.

        By only syncing a segment of the array, a full transmission of the
        updated array is avoided. However, this does put the responsibility
        of ensuring the correct sync state on the caller.

        This respects hold_sync, so several segments can be stacked with
        multiple calls when holding the sync.

        Parameters
        ----------
        segments : iterable of two-tuples
            An iterable collection of segments represented by (start, stop) tuples.
        """
        if self._holding_sync:
            self._segments_to_send.add(*segments)
        else:
            self.send_segment(segments)

    def send_segment(self, segments):
        """Send segments to the front-end.

        Note: This does not respect hold_sync. If that is wanted, use
        sync_segment instead.

        Parameters
        ----------
        segments : iterable of two-tuples
            An iterable collection of segments represented by (start, stop) tuples.
        """
        starts = []
        buffers = []
        raveled = np.ravel(self.array, order='C')
        length = len(raveled)
        for s in segments:
            starts.append(s[0] if s[0] >= 0 else length - s[0])
            buffers.append(np.ascontiguousarray(raveled[s[0]:s[1]]))

        msg = {'method': 'update_array_segment', 'name': 'array', 'starts': starts}
        self._send(msg, buffers)

    @contextmanager
    def hold_sync(self):
        with super(NDArrayWidget, self).hold_sync():
            try:
                yield
            finally:
                if self._segments_to_send:
                    self.send_segment(self._segments_to_send)
                    self._segments_to_send.clear()

# Signature SHOULD be create_constrained_arraywidget(*validators, dtype=None),
# but this is not supported by Python 2.7. For Python 3, we try to be
# helpful by overriding the signature below.
def create_constrained_arraywidget(*validators, **kwargs):
    """Returns a subclass of NDArrayWidget with a constrained array.

    Accepts keyword argument 'dtype' in addition to any valdiators.
    """
    dtype = kwargs.pop('dtype', None)
    return type('ConstrainedNDArrayWidget', (NDArrayWidget,), {
        'array': NDArray(dtype=dtype).tag(
            sync=True,
            **compressed_array_serialization
        ).valid(*validators)
    })

if six.PY3:
    from inspect import Signature, Parameter
    create_constrained_arraywidget.__signature__ = Signature(parameters=(
        Parameter('validators', Parameter.VAR_POSITIONAL),
        Parameter('dtype', Parameter.KEYWORD_ONLY, default=None),
    ))

def ConstrainedNDArrayWidget(*validators, **kwargs):
    import warnings
    warnings.warn('ConstrainedNDArrayWidget is deprecated, '
                  'use create_constrained_arraywidget instead')
    return create_constrained_arraywidget(*validators, **kwargs)

max_stars_repo_name: consideRatio/ipydatawidgets

max_issues_repo_name: consideRatio/ipydatawidgets

max_forks_repo_name: consideRatio/ipydatawidgets


text:
"""Data parsers & utilities module for OGC-related projects."""

import copy
import functools
import logging

import cv2 as cv
import numpy as np
import tqdm

import thelper.data
import thelper.data.geo as geo
import thelper.train.utils

logger = logging.getLogger(__name__)

class TB15D104:
    """Wrapper class for OGC Testbed-15 (D104) identifiers."""

    TYPECE_RIVER = "10"
    TYPECE_LAKE = "21"

    BACKGROUND_ID = 0
    LAKE_ID = 1

class TB15D104Dataset(geo.parsers.VectorCropDataset):
    """OGC Testbed-15 dataset parser for D104 (lake/river) segmentation task."""

    def __init__(self, raster_path, vector_path, px_size=None,
                 allow_outlying_vectors=True, clip_outlying_vectors=True,
                 lake_area_min=0.0, lake_area_max=float("inf"),
                 lake_river_max_dist=float("inf"), feature_buffer=1000,
                 master_roi=None, focus_lakes=True, srs_target="3857", force_parse=False,
                 reproj_rasters=False, reproj_all_cpus=True, display_debug=False,
                 keep_rasters_open=True, parallel=False, transforms=None):
        assert isinstance(lake_river_max_dist, (float, int)) and lake_river_max_dist >= 0, "unexpected dist type"
        self.lake_river_max_dist = float(lake_river_max_dist)
        assert isinstance(focus_lakes, bool), "unexpected flag type"
        self.focus_lakes = focus_lakes
        assert px_size is None or isinstance(px_size, (float, int)), "pixel size (resolution) must be float/int"
        px_size = (1.0, 1.0) if px_size is None else (float(px_size), float(px_size))
        # note: we wrap partial static functions for caching to see when internal parameters are changing
        cleaner = functools.partial(self.lake_cleaner, area_min=lake_area_min, area_max=lake_area_max,
                                    lake_river_max_dist=lake_river_max_dist, parallel=parallel)
        if self.focus_lakes:
            cropper = functools.partial(self.lake_cropper, px_size=px_size, skew=(0.0, 0.0),
                                        feature_buffer=feature_buffer, parallel=parallel)
        else:
            # TODO: implement river-focused cropper (i.e. river-length parsing?)
            raise NotImplementedError
        super().__init__(raster_path=raster_path, vector_path=vector_path, px_size=px_size, skew=None,
                         allow_outlying_vectors=allow_outlying_vectors, clip_outlying_vectors=clip_outlying_vectors,
                         vector_area_min=lake_area_min, vector_area_max=lake_area_max, vector_target_prop=None,
                         feature_buffer=feature_buffer, master_roi=master_roi, srs_target=srs_target,
                         raster_key="lidar", mask_key="hydro", cleaner=cleaner, cropper=cropper,
                         force_parse=force_parse, reproj_rasters=reproj_rasters, reproj_all_cpus=reproj_all_cpus,
                         keep_rasters_open=keep_rasters_open, transforms=transforms)
        meta_keys = self.task.meta_keys
        if "bboxes" in meta_keys:
            del meta_keys[meta_keys.index("bboxes")]  # placed in meta list by base class constr, moved to detect target below
        self.task = thelper.tasks.Detection(class_names={"background": TB15D104.BACKGROUND_ID, "lake": TB15D104.LAKE_ID},
                                            input_key="input", bboxes_key="bboxes",
                                            meta_keys=meta_keys, background=0, color_map={"lake": [255, 0, 0]})
        # update all already-created bboxes with new task ref
        for s in self.samples:
            for b in s["bboxes"]:
                b.task = self.task
        self.display_debug = display_debug
        self.parallel = parallel

    @staticmethod
    def lake_cleaner(features, area_min, area_max, lake_river_max_dist, parallel=False):
        """Flags geometric features as 'clean' based on type and distance to nearest river."""
        # note: we use a flag here instead of removing bad features so that end-users can still use them if needed
        for f in features:
            f["clean"] = False  # flag every as 'bad' by default, clear just the ones of interest below
        rivers = [f for f in features if f["properties"]["TYPECE"] == TB15D104.TYPECE_RIVER]
        lakes = [f for f in features if f["properties"]["TYPECE"] == TB15D104.TYPECE_LAKE]
        logger.info(f"labeling and cleaning {len(lakes)} lakes...")

        def clean_lake(lake):
            if area_min <= lake["geometry"].area <= area_max:
                if lake_river_max_dist == float("inf"):
                    return True
                else:
                    for river in rivers:
                        # note: distance check below seems to be "intelligent", i.e. it will
                        # first check bbox distance, then check chull distance, and finally use
                        # the full geometries (doing these steps here explicitly makes it slower)
                        if lake["geometry"].distance(river["geometry"]) < lake_river_max_dist:
                            return True
            return False

        if parallel:
            if not isinstance(parallel, int):
                import multiprocessing
                parallel = multiprocessing.cpu_count()
            assert parallel > 0, "unexpected min core count"
            import joblib
            flags = joblib.Parallel(n_jobs=parallel)(joblib.delayed(
                clean_lake)(lake) for lake in tqdm.tqdm(lakes, desc="labeling + cleaning lakes"))
            for flag, lake in zip(flags, lakes):
                lake["clean"] = flag
        else:
            for lake in tqdm.tqdm(lakes, desc="labeling + cleaning lakes"):
                lake["clean"] = clean_lake(lake)
        return features

    @staticmethod
    def lake_cropper(features, rasters_data, coverage, srs_target, px_size, skew, feature_buffer, parallel=False):
        """Returns the ROI information for a given feature (may be modified in derived classes)."""
        srs_target_wkt = srs_target.ExportToWkt()

        def crop_feature(feature):
            assert feature["clean"]  # should not get here with bad features
            roi, roi_tl, roi_br, crop_width, crop_height = \
                geo.utils.get_feature_roi(feature["geometry"], px_size, skew, feature_buffer)
            roi_geotransform = (roi_tl[0], px_size[0], skew[0],
                                roi_tl[1], skew[1], px_size[1])
            # test all raster regions that touch the selected feature
            raster_hits = []
            for raster_idx, raster_data in enumerate(rasters_data):
                if raster_data["target_roi"].intersects(roi):
                    raster_hits.append(raster_idx)
            # make list of all other features that may be included in the roi
            roi_centroid = feature["centroid"]
            roi_radius = np.linalg.norm(np.asarray(roi_tl) - np.asarray(roi_br)) / 2
            roi_features, bboxes = [], []
            # note: the 'image id' is in fact the id of the focal feature in the crop
            image_id = int(feature["properties"]["OBJECTID"])
            for f in features:
                # note: here, f may not be 'clean', test anyway
                if f["geometry"].distance(roi_centroid) > roi_radius:
                    continue
                inters = f["geometry"].intersection(roi)
                if inters.is_empty:
                    continue
                roi_features.append(f)
                if f["properties"]["TYPECE"] == TB15D104.TYPECE_RIVER:
                    continue
                # only lakes can generate bboxes; make sure to clip them to the roi bounds
                clip = f["clipped"] or not inters.equals(f["geometry"])
                if clip:
                    assert inters.geom_type in ["Polygon", "MultiPolygon"], "unexpected inters type"
                    corners = []
                    if inters.geom_type == "Polygon":
                        bounds = inters.bounds
                        corners.append((bounds[0:2], bounds[2:4]))
                    elif inters.geom_type == "MultiPolygon":
                        for poly in inters:
                            bounds = poly.bounds
                            corners.append((bounds[0:2], bounds[2:4]))
                else:
                    corners = [(f["tl"], f["br"])]
                for c in corners:
                    feat_tl_px = geo.utils.get_pxcoord(roi_geotransform, *c[0])
                    feat_br_px = geo.utils.get_pxcoord(roi_geotransform, *c[1])
                    bbox = [max(0, feat_tl_px[0]), max(0, feat_tl_px[1]),
                            min(crop_width - 1, feat_br_px[0]),
                            min(crop_height - 1, feat_br_px[1])]
                    if bbox[2] - bbox[0] <= 1 or bbox[3] - bbox[1] <= 1:
                        continue  # skip all bboxes smaller than 1 px (c'mon...)
                    # note: lake class id is 1 by definition
                    bboxes.append(thelper.tasks.detect.BoundingBox(TB15D104.LAKE_ID,
                                                                   bbox=bbox,
                                                                   include_margin=False,
                                                                   truncated=clip,
                                                                   image_id=image_id))
            # prepare actual 'sample' for crop generation at runtime
            return {
                "features": roi_features,
                "bboxes": bboxes,
                "focal": feature,
                "id": image_id,
                "roi": roi,
                "roi_tl": roi_tl,
                "roi_br": roi_br,
                "raster_hits": raster_hits,
                "crop_width": crop_width,
                "crop_height": crop_height,
                "geotransform": np.asarray(roi_geotransform),
                "srs": srs_target_wkt,
            }

        clean_feats = [f for f in features if f["clean"]]
        if parallel:
            if not isinstance(parallel, int):
                import multiprocessing
                parallel = multiprocessing.cpu_count()
            assert parallel > 0, "unexpected min core count"
            import joblib
            samples = joblib.Parallel(n_jobs=parallel)(joblib.delayed(
                crop_feature)(feat) for feat in tqdm.tqdm(clean_feats, desc="preparing crop regions"))
        else:
            samples = []
            for feature in tqdm.tqdm(clean_feats, desc="validating crop candidates"):
                samples.append(crop_feature(feature))
        return [s for s in samples if s is not None]

    def _show_stats_plots(self, show=False, block=False):
        """Draws and returns feature stats histograms using pyplot."""
        import matplotlib.pyplot as plt
        feature_categories = {}
        for feat in self.features:
            curr_cat = feat["properties"]["TYPECE"]
            if curr_cat not in feature_categories:
                feature_categories[curr_cat] = []
            feature_categories[curr_cat].append(feat)
        fig, axes = plt.subplots(len(feature_categories))
        for idx, (cat, features) in enumerate(feature_categories.items()):
            areas = [f["geometry"].area for f in features]
            axes[idx].hist(areas, density=True, bins=30,
                           range=(max(self.area_min, min(areas)), min(self.area_max, max(areas))))
            axes[idx].set_xlabel("Surface (m^2)")
            axes[idx].set_title(f"TYPECE = {cat}")
            axes[idx].set_xlim(xmin=0)
        if show:
            fig.show()
            if block:
                plt.show(block=block)
                return fig
            plt.pause(0.5)
        return fig, axes

    def __getitem__(self, idx):
        """Returns the data sample (a dictionary) for a specific (0-based) index."""
        if isinstance(idx, slice):
            return self._getitems(idx)
        assert idx < len(self.samples), "sample index is out-of-range"
        if idx < 0:
            idx = len(self.samples) + idx
        sample = self.samples[idx]
        crop, mask = self._process_crop(sample)
        assert crop.shape[2] == 1, "unexpected lidar raster band count"
        crop = crop[:, :, 0]
        dmap = cv.distanceTransform(np.where(mask, np.uint8(0), np.uint8(255)), cv.DIST_L2, cv.DIST_MASK_PRECISE)
        dmap_inv = cv.distanceTransform(np.where(mask, np.uint8(255), np.uint8(0)), cv.DIST_L2, cv.DIST_MASK_PRECISE)
        dmap = np.where(mask, -dmap_inv, dmap)
        dmap *= self.px_size[0]  # constructor enforces same px width/height size
        # dc mask is crop.mask, but most likely lost below
        if self.display_debug:
            crop = cv.normalize(crop, dst=crop, alpha=0, beta=255, norm_type=cv.NORM_MINMAX,
                                dtype=cv.CV_8U, mask=(~crop.mask).astype(np.uint8))
            mask = cv.normalize(mask, dst=mask, alpha=0, beta=255, norm_type=cv.NORM_MINMAX, dtype=cv.CV_8U)
            dmap = cv.normalize(dmap, dst=dmap, alpha=0, beta=255, norm_type=cv.NORM_MINMAX, dtype=cv.CV_8U)
        sample = {
            "input": np.stack([crop, mask, dmap], axis=-1),
            # note: bboxes are automatically added in the "cropper" preprocessing function
            "hydro": mask,
            **sample
        }
        if self.transforms:
            sample = self.transforms(sample)
        return sample

class TB15D104TileDataset(geo.parsers.TileDataset):
    """OGC Testbed-15 dataset parser for D104 (lake/river) segmentation task."""

    def __init__(self, raster_path, vector_path, tile_size, tile_overlap,
                 px_size=None, allow_outlying_vectors=True, clip_outlying_vectors=True,
                 lake_area_min=0.0, lake_area_max=float("inf"), master_roi=None, srs_target="3857",
                 force_parse=False, reproj_rasters=False, reproj_all_cpus=True, display_debug=False,
                 keep_rasters_open=True, parallel=False, transforms=None):
        assert px_size is None or isinstance(px_size, (float, int)), "pixel size (resolution) must be float/int"
        px_size = (1.0, 1.0) if px_size is None else (float(px_size), float(px_size))
        # note: we wrap partial static functions for caching to see when internal parameters are changing
        cleaner = functools.partial(TB15D104Dataset.lake_cleaner, area_min=lake_area_min, area_max=lake_area_max,
                                    lake_river_max_dist=float("inf"), parallel=parallel)
        super().__init__(raster_path=raster_path, vector_path=vector_path, tile_size=tile_size,
                         tile_overlap=tile_overlap, skip_empty_tiles=True, skip_nodata_tiles=False,
                         px_size=px_size, allow_outlying_vectors=allow_outlying_vectors,
                         clip_outlying_vectors=clip_outlying_vectors, vector_area_min=lake_area_min,
                         vector_area_max=lake_area_max, vector_target_prop=None, master_roi=master_roi,
                         srs_target=srs_target, raster_key="lidar", mask_key="hydro", cleaner=cleaner,
                         force_parse=force_parse, reproj_rasters=reproj_rasters, reproj_all_cpus=reproj_all_cpus,
                         keep_rasters_open=keep_rasters_open, transforms=transforms)
        meta_keys = self.task.meta_keys
        self.task = thelper.tasks.Detection(class_names={"background": TB15D104.BACKGROUND_ID, "lake": TB15D104.LAKE_ID},
                                            input_key="input", bboxes_key="bboxes",
                                            meta_keys=meta_keys, background=0, color_map={"lake": [255, 0, 0]})
        self.display_debug = display_debug
        self.parallel = parallel

    def __getitem__(self, idx):
        """Returns the data sample (a dictionary) for a specific (0-based) index."""
        if isinstance(idx, slice):
            return self._getitems(idx)
        assert idx < len(self.samples), "sample index is out-of-range"
        if idx < 0:
            idx = len(self.samples) + idx
        sample = self.samples[idx]
        crop, mask = self._process_crop(sample)
        assert crop.shape[2] == 1, "unexpected lidar raster band count"
        crop = crop[:, :, 0]
        dmap = cv.distanceTransform(np.where(mask, np.uint8(0), np.uint8(255)), cv.DIST_L2, cv.DIST_MASK_PRECISE)
        dmap_inv = cv.distanceTransform(np.where(mask, np.uint8(255), np.uint8(0)), cv.DIST_L2, cv.DIST_MASK_PRECISE)
        dmap = np.where(mask, -dmap_inv, dmap)
        dmap *= self.px_size[0]  # constructor enforces same px width/height size
        # dc mask is crop.mask, but most likely lost below
        if self.display_debug:
            crop = cv.normalize(crop, dst=crop, alpha=0, beta=255, norm_type=cv.NORM_MINMAX,
                                dtype=cv.CV_8U, mask=(~crop.mask).astype(np.uint8))
            mask = cv.normalize(mask, dst=mask, alpha=0, beta=255, norm_type=cv.NORM_MINMAX, dtype=cv.CV_8U)
            dmap = cv.normalize(dmap, dst=dmap, alpha=0, beta=255, norm_type=cv.NORM_MINMAX, dtype=cv.CV_8U)
        # note1: contrarily to the 'smart' dataset parser above, we need to add bboxes here
        # note2: bboxes should only be added over areas that are not 'nodata'
        bboxes = []
        for f in sample["features"]:
            if f["properties"]["TYPECE"] == TB15D104.TYPECE_RIVER:
                continue
            # only lakes can generate bboxes; make sure to clip them to the crop bounds
            inters = f["geometry"].intersection(sample["roi"])
            clip = f["clipped"] or not inters.equals(f["geometry"])
            if clip:
                assert inters.geom_type in ["Polygon", "MultiPolygon"], "unexpected inters type"
                corners = []
                if inters.geom_type == "Polygon":
                    bounds = inters.bounds
                    corners.append((bounds[0:2], bounds[2:4]))
                elif inters.geom_type == "MultiPolygon":
                    for poly in inters:
                        bounds = poly.bounds
                        corners.append((bounds[0:2], bounds[2:4]))
            else:
                corners = [(f["tl"], f["br"])]
            for c in corners:
                feat_tl_px = geo.utils.get_pxcoord(sample["geotransform"], *c[0])
                feat_br_px = geo.utils.get_pxcoord(sample["geotransform"], *c[1])
                bbox = [max(0, feat_tl_px[0]), max(0, feat_tl_px[1]),
                        min(sample["crop_width"] - 1, feat_br_px[0]),
                        min(sample["crop_height"] - 1, feat_br_px[1])]
                if bbox[2] - bbox[0] <= 1 or bbox[3] - bbox[1] <= 1:
                    continue  # skip all bboxes smaller than 1 px (c'mon...)
                # note: lake class id is 1 by definition
                bboxes.append(thelper.tasks.detect.BoundingBox(TB15D104.LAKE_ID,
                                                               bbox=bbox,
                                                               include_margin=False,
                                                               truncated=clip,
                                                               image_id=sample["id"],
                                                               task=self.task))
        sample = {
            "input": np.stack([crop, mask, dmap], axis=-1),
            "hydro": mask,
            "bboxes": bboxes,
            **sample
        }
        if self.transforms:
            sample = self.transforms(sample)
        return sample

class TB15D104DetectLogger(thelper.train.utils.DetectLogger):

    def __init__(self, conf_threshold=0.5):
        super().__init__(conf_threshold=conf_threshold, target_name="lake",
                         log_keys=["id", "geotransform"], format="geojson")

    def report_geojson(self):
        # here, we only care about reporting predictions, we ignore the (possibly missing) gt bboxes
        import shapely
        import geojson
        batch_size = len(self.bbox[0])
        bbox_lists = [bboxes for batch in self.bbox for bboxes in batch]  # one list per crop
        if batch_size > 1:
            geotransforms = [np.asarray(geot) for batch in self.meta["geotransform"] for geot in batch]
        else:
            # special check to avoid issues when unpacking (1,6)-dim tensor
            geotransforms = [np.asarray(geot) for geot in self.meta["geotransform"]]
        crop_ids = [id for batch in self.meta["id"] for id in batch]
        output_features = []
        for bboxes, geotransform, id in zip(bbox_lists, geotransforms, crop_ids):
            for bbox in bboxes:
                if geotransform.shape[0] == 1:
                    geotransform = geotransform[0]
                bbox_tl = geo.utils.get_geocoord(geotransform, *bbox.top_left)
                bbox_br = geo.utils.get_geocoord(geotransform, *bbox.bottom_right)
                bbox_geom = shapely.geometry.Polygon([bbox_tl, (bbox_br[0], bbox_tl[1]),
                                                      bbox_br, (bbox_tl[0], bbox_br[1])])
                output_features.append(geojson.Feature(geometry=bbox_geom, properties={
                    "image_id": id, "confidence": bbox.confidence}))
        return geojson.dumps(geojson.FeatureCollection(output_features), indent=2)

def postproc_features(input_file, bboxes_srs, orig_geoms_path, output_file,
                      final_srs=None, write_shapefile_copy=False):
    """Post-processes bounding box detections produced during an evaluation session into a GeoJSON file."""
    import ogr
    import osr
    import json
    import geojson
    import shapely
    logger.debug("importing bboxes SRS...")
    assert isinstance(bboxes_srs, (str, int, osr.SpatialReference)), \
        "target EPSG SRS must be given as int/str"
    if isinstance(bboxes_srs, (str, int)):
        if isinstance(bboxes_srs, str):
            bboxes_srs = int(bboxes_srs.replace("EPSG:", ""))
        bboxes_srs_obj = osr.SpatialReference()
        bboxes_srs_obj.ImportFromEPSG(bboxes_srs)
        bboxes_srs = bboxes_srs_obj
    logger.debug("importing lake bboxes geojson...")
    with open(input_file) as bboxes_fd:
        bboxes_geoms = thelper.data.geo.utils.parse_geojson(json.load(bboxes_fd))
    logger.debug("importing hydro features geojson...")
    with open(orig_geoms_path) as hydro_fd:
        hydro_geoms = thelper.data.geo.utils.parse_geojson(json.load(hydro_fd), srs_target=bboxes_srs)
    logger.debug("computing global cascade of lake bboxes...")
    detect_roi = shapely.ops.cascaded_union([bbox["geometry"] for bbox in bboxes_geoms])
    output_features = []

    def append_poly(feat, props, srs_transform=None):
        if feat.is_empty:
            return
        elif feat.type == "Polygon":
            if srs_transform is not None:
                ogr_geometry = ogr.CreateGeometryFromWkb(feat.wkb)
                ogr_geometry.Transform(srs_transform)
                feat = shapely.wkt.loads(ogr_geometry.ExportToWkt())
            output_features.append((geojson.Feature(geometry=feat, properties=props), feat))
        elif feat.type == "MultiPolygon" or feat.type == "GeometryCollection":
            for f in feat:
                append_poly(f, props)

    srs_transform = None
    if final_srs is not None:
        import osr
        logger.debug("importing output SRS...")
        assert isinstance(final_srs, (str, int, osr.SpatialReference)), \
            "target EPSG SRS must be given as int/str"
        if isinstance(final_srs, (str, int)):
            if isinstance(final_srs, str):
                final_srs = int(final_srs.replace("EPSG:", ""))
            final_srs_obj = osr.SpatialReference()
            final_srs_obj.ImportFromEPSG(final_srs)
            final_srs = final_srs_obj
        if not bboxes_srs.IsSame(final_srs):
            srs_transform = osr.CoordinateTransformation(bboxes_srs, final_srs)
    logger.debug("running hydro feature and lake bboxes intersection loop...")
    for hydro_feat in tqdm.tqdm(hydro_geoms, desc="computing bbox intersections"):
        # find intersection and append to list of 'lakes'
        intersection = hydro_feat["geometry"].intersection(detect_roi)
        hydro_feat["properties"]["TYPECE"] = TB15D104.TYPECE_LAKE
        append_poly(intersection, copy.deepcopy(hydro_feat["properties"]), srs_transform)
        if not intersection.is_empty:
            # subtract bbox region from feature if intersection found (leftovers at end will be 'rivers')
            hydro_feat["geometry"] = hydro_feat["geometry"].difference(detect_roi)
    logger.debug("running river cleanup loop...")
    for hydro_feat in tqdm.tqdm(hydro_geoms, desc="appending leftover geometries as rivers"):
        if not hydro_feat["geometry"].is_empty:
            # remark: hydro features outside the original ROI will appear as rivers despite never being processed
            hydro_feat["properties"]["TYPECE"] = TB15D104.TYPECE_RIVER
            append_poly(hydro_feat["geometry"], copy.deepcopy(hydro_feat["properties"]), srs_transform)
    logger.debug("exporting final geojson...")
    with open(output_file, "w") as fd:
        out_srs = final_srs if final_srs is not None else bboxes_srs
        fd.write(geo.utils.export_geojson_with_crs([o[0] for o in output_features], srs_target=out_srs))
    if write_shapefile_copy:
        logger.debug("exporting final shapefile...")
        driver = ogr.GetDriverByName("ESRI Shapefile")
        data_source = driver.CreateDataSource(output_file + ".shp")
        layer = data_source.CreateLayer("lakes", final_srs if final_srs is not None else bboxes_srs, ogr.wkbPolygon)
        for feat_tuple in output_features:
            feature = ogr.Feature(layer.GetLayerDefn())
            point = ogr.CreateGeometryFromWkt(feat_tuple[1].wkt)
            feature.SetGeometry(point)
            layer.CreateFeature(feature)
            feature = None
        data_source = None

max_stars_repo_name: crim-ca/thelper

max_issues_repo_name: crim-ca/thelper

max_forks_repo_name: crim-ca/thelper


text:
import numpy as np
from openmdao.api import IndepVarComp, Group, Problem
from openmdao.core.component import Component
from openmdao.drivers.fullfactorial_driver import FullFactorialDriver
from openmdao.drivers.latinhypercube_driver import LatinHypercubeDriver, \
    OptimizedLatinHypercubeDriver
from openmdao.drivers.uniform_driver import UniformDriver

_METHOD_TO_DRIVER = {
    'FullFactorial': {
        'cls': FullFactorialDriver,
        'req_args': [
            'num_levels'
        ]
    },
    'Uniform': {
        'cls': UniformDriver,
        'req_args': [
            'num_samples'
        ]
    },
    'LatinHypercube': {
        'cls': LatinHypercubeDriver,
        'req_args': [
            'num_samples'
        ]
    },
    'OptimizedLatinHypercube': {
        'cls': OptimizedLatinHypercubeDriver,
        'req_args': [
            'num_samples'
        ]
    }
}

class _CaptureParams(Component):
    def __init__(self, num_params, doe):
        super(_CaptureParams, self).__init__()
        self._num_params = num_params
        self._doe = doe
        for i in range(num_params):
            self.add_param('v%d' % i, val=0.0)
        self.add_output('vout', val=0.0)

    def solve_nonlinear(self, params, unknowns, resids):
        d = np.array([], dtype=float)
        for i in range(self._num_params):
            vname = 'v%d' % i
            d = np.append(d, params[vname])
        self._doe.append(d)
        unknowns['vout'] = 0.0

def generate(method, params, **kwargs):
    # map method to openmdao driver
    assert method in _METHOD_TO_DRIVER.keys()
    _driver = _METHOD_TO_DRIVER[method]['cls']

    # cast and check params matix
    if not isinstance(params, np.ndarray):
        params = np.asarray(params, dtype=float)
    assert len(params.shape) == 2
    assert params.shape[1] == 2

    # check for required keyword arguments
    parsed_keys = kwargs.keys()
    for req_arg in _METHOD_TO_DRIVER[method]['req_args']:
        assert req_arg in parsed_keys

    # construct openmdao problem
    num_params = len(params)
    top = Problem()
    top.root = Group()
    top.driver = _driver(**kwargs)
    i = 0
    for vmin, vmax in params:
        pname = 'p%d' % i
        vname = 'v%d' % i
        top.root.add(pname, IndepVarComp(vname, 0.0), promotes=['*'])
        top.driver.add_desvar(vname, lower=float(vmin), upper=float(vmax))
        i += 1
    doe = []
    top.root.add('pout', _CaptureParams(num_params, doe), promotes=['*'])
    top.driver.add_objective('vout')

    # setup, run and terminate the openmdao problem
    top.setup(check=False)
    top.run()
    top.cleanup()
    del top.driver
    del top.root
    del top

    # return as a numpy array
    return np.array(doe)

max_stars_repo_name: jarrodsinclair/doegen

max_issues_repo_name: jarrodsinclair/doegen

max_forks_repo_name: jarrodsinclair/doegen


text:
from __future__ import division

from pyfmmlib import fmm_part, HelmholtzKernel
import numpy as np
import matplotlib.pyplot as pt

sources = np.random.randn(40, 2)

targets = np.mgrid[-7:7:400j, -7:7:400j]
pot_shape = targets.shape[1:]
targets = targets.reshape(2, -1)

pot, grad = fmm_part("PG", iprec=2, kernel=HelmholtzKernel(5),
        sources=sources, mop_charge=1, target=targets.T)

pot = pot.reshape(pot_shape)

pt.imshow(pot.real)
pt.show()

max_stars_repo_name: mattwala/pyfmmlib

max_stars_repo_licenses: MIT Unlicense

max_issues_repo_name: mattwala/pyfmmlib

max_issues_repo_licenses: MIT Unlicense

max_forks_repo_name: mattwala/pyfmmlib

max_forks_repo_licenses: MIT Unlicense


text:
import numpy as np

class FileData(object):
    def __init__(self, file_path, feature_cols=None, reward_col=None):
        self.file = file_path
        self.dataset = np.genfromtxt(self.file, delimiter=',', filling_values=0.0)
        self.sample_num = len(self.dataset)
        self.feature_cols = feature_cols
        self.reward_col = reward_col

    # sample a batch of data
    def sample(self, num, feature_cols=None, label_col=None):
        assert num <= self.sample_num
        rcol = self.reward_col if label_col is None else label_col
        fcols = self.feature_cols if feature_cols is None else feature_cols
        idx = np.random.choice(len(self.dataset), num) # a vector indicating columns to be selected
        sampled_data = self.dataset[idx]
        #feature = sampled_data[:, fcols]
        #reward = sampled_data[:, rcol]
        return sampled_data

    def sample_by_index(self, idx, feature_cols=None, label_col=None):
        """
        return samples of given indices
        """
        rcol = self.reward_col if label_col is None else label_col
        fcols = self.feature_cols if feature_cols is None else feature_cols
        sampled_data = self.dataset[idx]
        #print(sampled_data.shape)
        return sampled_data

    def sample_feature_tensor(self, num, pkey_col=0, skey_col=1):
        idx = np.lexsort((self.dataset[:, skey_col], self.dataset[:,pkey_col]))
        pkey_num = len(np.unique(self.dataset[:,pkey_col]))
        assert len(self.dataset)%pkey_num==0
        skey_num = len(self.dataset)/pkey_num
        smp_ind = np.random.choice(skey_num, num, replace=False)
        feature = self.dataset[:,self.feature_cols]
        feature = feature[idx]
        feature = feature.reshape(pkey_num,skey_num,-1)
        feature = feature[:,smp_ind,:]
        return feature

    def get_features(self, feature_cols=None):
        fcols = self.feature_cols if feature_cols is None else feature_cols
        return self.dataset[:, fcols]

# historical data for offline training
class HistoryData(FileData):
    def __init__(self, size, file_path, feature_cols=None, reward_col=None):
        super().__init__(file_path, feature_cols, reward_col)
        # generate historical data by rewriting dataset by a sample
        self.dataset = self.sample(size, self.feature_cols, self.reward_col)

    # append new data
    def append(self, data):
        self.dataset = np.concatenate((self.dataset, data), axis=0)

max_stars_repo_name: anon-usr/INLUCB

max_issues_repo_name: anon-usr/INLUCB

max_forks_repo_name: anon-usr/INLUCB