text:
import numpy as np
import matplotlib.pyplot as plt
from numpy.core.einsumfunc import _parse_possible_contraction

#lrs = [1e-4, 5e-4, 1e-3, 2e-3, 3e-3]

vgg_bn_grad_lst = np.load('landscape/vgg_bn_grad_lst.npy',allow_pickle=True)
vgg_bn_loss_lst = np.load('landscape/vgg_bn_loss_lst.npy',allow_pickle=True)
vgg_bn_train_lst = np.load('landscape/vgg_bn_train_lst.npy',allow_pickle=True)
vgg_bn_val_lst = np.load('landscape/vgg_bn_val_lst.npy',allow_pickle=True)
vgg_bn_wei_lst = np.load('landscape/vgg_bn_wei_lst.npy',allow_pickle=True)

vgg_grad_lst = np.load('landscape/vgg_grad_lst.npy',allow_pickle=True)
vgg_loss_lst = np.load('landscape/vgg_loss_lst.npy',allow_pickle=True)
vgg_train_lst = np.load('landscape/vgg_train_lst.npy',allow_pickle=True)
vgg_val_lst = np.load('landscape/vgg_val_lst.npy',allow_pickle=True)
vgg_wei_lst = np.load('landscape/vgg_wei_lst.npy',allow_pickle=True)

def draw_loss_landscape(savepath):
    plt.style.use('ggplot')
    plt.clf()
    interval = 10

    #画图时修改interval

    max_loss = np.max(vgg_loss_lst[:3],0)[::interval]
    min_loss = np.min(vgg_loss_lst[:3],0)[::interval]

    x = np.arange(0,len(max_loss),1)
    plt.plot(x, max_loss, c='deeppink')
    plt.plot(x, min_loss, c='deeppink')
    plt.fill_between(x, max_loss, min_loss ,facecolor='lightpink', label='without BN')

    max_bn_loss = np.max(vgg_bn_loss_lst[:3],0)[::interval]
    min_bn_loss = np.min(vgg_bn_loss_lst[:3],0)[::interval]
    plt.plot(x, max_bn_loss, c='mediumseagreen')
    plt.plot(x, min_bn_loss, c='mediumseagreen')
    plt.fill_between(x, max_bn_loss, min_bn_loss, facecolor="lightgreen", label = 'with BN')
    plt.legend()
    plt.savefig(savepath)

def draw_predgrad(savepath):
    plt.style.use('ggplot')
    plt.clf()
    interval = 10
    n_lr,n_iter,_,_ = vgg_bn_grad_lst.shape
    grad_diff = np.zeros((n_lr,n_iter-1))
    for i in range(n_lr):
        for j in range(n_iter-1):
            grad_diff[i,j] = np.linalg.norm(vgg_grad_lst[i,j+1,:] - vgg_grad_lst[i,j,:])

    max_grad_diff = np.max(grad_diff[:3],0)[::interval]
    min_grad_diff = np.min(grad_diff[:3],0)[::interval]
    x = np.arange(0,len(max_grad_diff),1)
    begin = 1
    x = x[begin:]
    max_grad_diff = max_grad_diff[begin:]
    min_grad_diff = min_grad_diff[begin:]
    plt.plot(x, max_grad_diff, c='mediumseagreen')
    plt.plot(x, min_grad_diff, c='mediumseagreen')
    plt.fill_between(x,max_grad_diff,min_grad_diff,facecolor='lightgreen', label='with BN')

    grad_diff = np.zeros((n_lr,n_iter-1))
    for i in range(n_lr):
        for j in range(n_iter-1):
            grad_diff[i,j] = np.linalg.norm(vgg_bn_grad_lst[i,j+1,:] - vgg_bn_grad_lst[i,j,:])
    max_grad_diff = np.max(grad_diff[:3],0)[::interval]
    min_grad_diff = np.min(grad_diff[:3],0)[::interval]
    x = np.arange(0,len(max_grad_diff),1)
    x = x[begin:]
    max_grad_diff = max_grad_diff[begin:]
    min_grad_diff = min_grad_diff[begin:]
    plt.plot(x, max_grad_diff, c='deeppink')
    plt.plot(x, min_grad_diff, c='deeppink')
    plt.fill_between(x,max_grad_diff,min_grad_diff,facecolor='lightpink', label='without BN')

    plt.legend()
    plt.savefig(savepath)

def draw_beta_smooth(savepath):

    plt.style.use('ggplot')
    plt.clf()
    interval = 10
    n_lr,n_iter,_,_ = vgg_bn_grad_lst.shape
    grad_diff = np.zeros((n_lr,n_iter-1))
    for i in range(n_lr):
        for j in range(n_iter-1):
            grad_diff[i,j] = np.linalg.norm(vgg_grad_lst[i,j+1,:] - vgg_grad_lst[i,j,:])

    max_grad_diff = np.max(grad_diff[:],0)[::interval]
    min_grad_diff = np.min(grad_diff[:],0)[::interval]
    x = np.arange(0,len(max_grad_diff),1)
    begin = 1
    x = x[begin:]
    max_grad_diff = max_grad_diff[begin:]
    min_grad_diff = min_grad_diff[begin:]
    plt.plot(x, max_grad_diff, c='mediumseagreen', label='with BN')
    #plt.plot(x, min_grad_diff, c='mediumseagreen')
    #plt.fill_between(x,max_grad_diff,min_grad_diff,facecolor='lightgreen', label='without BN')

    grad_diff = np.zeros((n_lr,n_iter-1))
    for i in range(n_lr):
        for j in range(n_iter-1):
            grad_diff[i,j] = np.linalg.norm(vgg_bn_grad_lst[i,j+1,:] - vgg_bn_grad_lst[i,j,:])
    max_grad_diff = np.max(grad_diff[:],0)[::interval]
    min_grad_diff = np.min(grad_diff[:],0)[::interval]
    x = np.arange(0,len(max_grad_diff),1)
    x = x[begin:]
    max_grad_diff = max_grad_diff[begin:]
    min_grad_diff = min_grad_diff[begin:]
    plt.plot(x, max_grad_diff, c='deeppink', label='without BN')
    #plt.plot(x, min_grad_diff, c='deeppink')
    #plt.fill_between(x,max_grad_diff,min_grad_diff,facecolor='lightpink', label='with BN')

    plt.legend()
    plt.savefig(savepath)

if __name__ == '__main__':
    draw_loss_landscape('landscape/loss_landscape.png')
    draw_predgrad('landscape/pred_grad.png')
    draw_beta_smooth('landscape/beta_smooth.png')

reason: import numpy,from numpy


text:
from jax import ops, lax, jacfwd, jacrev, jit, jvp, grad
import itertools
import math
from functools import partial
from tqdm import tqdm

from NewtonLAX import *

class GPMP:
    def __init__(self,kernel,h,DataPairs,DataTriples):
        self.k = kernel
        self.h = h
        self.DataPairs = DataPairs
        self.DataTriples = DataTriples
        self.dk = jacfwd(self.k)
        self.ddk = jacfwd(self.dk)
        self.dim = len(DataPairs[0]) # dimension of state space
        self.mp_d = jnp.block([[1/2*jnp.identity(self.dim), -1/self.h*jnp.identity(self.dim)],[1/2*jnp.identity(self.dim), 1/self.h*jnp.identity(self.dim)]]) # derivative of midpoint approximation
        self.Z = jnp.hstack( [ 1/2*(self.DataPairs[:,:,0]+self.DataPairs[:,:,1]) ,  1/self.h*(self.DataPairs[:,:,1]-self.DataPairs[:,:,0]) ]) # points with maximal information

    # midpoint for midpoint discretisation
    def mp(self,q0,q1):
        return jnp.hstack([1/2*(q0+q1),1/self.h*(q1-q0)])

    # train GP
    def train(self,sympl_std_vol=1):

        print('start training')
        Z = self.Z # points with maximal information

        def kZ(x):
            out = jnp.zeros(len(Z))
            out = lax.fori_loop(0,len(Z), lambda j, out: ops.index_update(out,ops.index[j], self.k(x,Z[j]) ),out)
            return out

        dkZ = jacfwd(kZ)

        d2kZ = lambda x: dkZ(x)[:,self.dim:]

        def lhs0(qqq):

            m1 = self.mp(qqq[0],qqq[1])
            m2 = self.mp(qqq[1],qqq[2])

            dkZ2 = dkZ(m2)
            dkZ1 = dkZ(m1)

            return self.mp_d[:self.dim] @ dkZ2.transpose() + self.mp_d[self.dim:] @ dkZ1.transpose()

        # non-triviality condition
        print('calculate non-triviality condition')
        verts=jnp.array(list(itertools.product([0, 1], repeat=2*self.dim)),dtype=jnp.float64)
        normalise = 1./(math.factorial(self.dim) * 2**(2*self.dim)) * jnp.sum(jnp.sum(jnp.array([ d2kZ(v) for v in verts]),0),1)

        # normalise
        z0 = jnp.zeros(2*self.dim)
        normaliseTotal = kZ(z0)

        # data consistency equations
        print('calculate data consistency equations')
        lhsDEL = jnp.zeros((self.DataTriples.shape[0],self.dim,Z.shape[0]))
        bodyfun = lambda j, lhsDEL: ops.index_update(lhsDEL,ops.index[j], lhs0(self.DataTriples[j].transpose()))
        lhsDEL = lax.fori_loop(0,len(self.DataTriples), bodyfun,lhsDEL)
        lhsDEL=lhsDEL.reshape(self.DataTriples.shape[0]*self.dim,Z.shape[0])
        lhs = jnp.vstack([lhsDEL,normalise,normaliseTotal])

        rhs = jnp.zeros(lhs.shape[0])
        rhs = ops.index_update(rhs,ops.index[-2], sympl_std_vol) # symplectic volume of unit simplex normalised to sympl_std_vol

        # solve minimal norm / least square
        print('solve linear system dimensions: '+str(lhs.shape))
        kinvL,res,rank,singVals=jnp.linalg.lstsq(lhs,rhs,rcond=None)

        self.kinvL = kinvL
        self.res = res
        self.rank = rank

        return kinvL,res,rank

    # inverse modified Lagrangian and derivatives

    #Lmi = lambda y: np.array([k(y,qq) for qq in Z]) @ kinvL
    # inverse modified Lagrangian
    def Lmi(self,y):

        kyqq = jnp.zeros(len(self.Z))
        bodyfun = lambda j,kyqq0: ops.index_update(kyqq0, ops.index[j], self.k(y,self.Z[j]))
        kyqq = lax.fori_loop(0,len(self.Z), bodyfun,kyqq)

        return kyqq @ self.kinvL

    def Lmi_d(self,z):
        return jacfwd(self.Lmi)(z) # gradient

    def Lmi_dd(self,z):
        return jacfwd(self.Lmi_d)(z) # Hessian

    def Lmi_ddd(self,z):
        return jacfwd(self.Lmi_dd)(z) # 3rd derivative

    def Lmi_dddd(self,z):
        return jacfwd(self.Lmi_ddd)(z) # 4rth derivative

    # discrete Lagrangian based on inverse modified Lagrangian and derivatives
    def Ldisc(self,q0,q1):
        return self.h*self.Lmi(self.mp(q0,q1))        # discrete Lagrangian based on inverse modified Lagrangian

    def Ldisc_d1(self,q0,q1):
        return self.mp_d[:self.dim] @ (self.h*self.Lmi_d(self.mp(q0,q1)))  # derivative w.r.t. first variable

    def Ldisc_d2(self,q0,q1):
        return self.mp_d[self.dim:] @ (self.h*self.Lmi_d(self.mp(q0,q1)))  # derivative w.r.t. second variable

    # mixed second derivative
    def Ldisc_d1d2(self,q0,q1):
        return self.mp_d[self.dim:] @ (self.h*self.Lmi_dd(self.mp(q0,q1))) @ self.mp_d[:self.dim]

    ## Hessian
    def Ldisc_dd(self,q0,q1):
        return self.mp_d.transpose() @ (self.h*self.Lmi_dd(self.mp(q0,q1))) @ self.mp_d

    # computation of momenta using discrete Lagrangian
    @partial(jit, static_argnums=(0,))
    def p0_fun(self,q0,q1):
        return -self.Ldisc_d1(q0,q1)

    def p1_fun(self,q0,q1):
        return self.Ldisc_d2(q0,q1)

    def DEL(self,q0,q1,q2):
        return self.Ldisc_d1(q1,q2) + self.Ldisc_d2(q0,q1)

    @partial(jit, static_argnums=(0,))
    def q2_solve(self,q0,q1):
        # computation q2 using DEL

        print("compiling q2_solve")

        p1 = self.p1_fun(q0,q1)

        obj = lambda q2: self.Ldisc_d1(q1,q2)+p1
        obj_prime = jacfwd(obj)

        q2 = MyNewtonLAX(obj,q1,fprime=obj_prime)

        return q2

    # recovered Lagrangian from backward error analysis, 1 dimensional state space, order 2

    @partial(jit, static_argnums=(0,2))
    def Lrec2(self,y,nargout=1):

        print('Compiling Lrec2 (or a derivative)')

        q  = y[:self.dim]
        qd = y[self.dim:]

        L_0 = self.Lmi(y)

        # compute jet
        gradL = self.Lmi_d(y)
        HessL = self.Lmi_dd(y)

        # assign variables compatible with formulas spit out by Mathematica
        L10 = gradL[0]
        L01 = gradL[1]

        L20 = HessL[0,0]
        L11 = HessL[0,1]
        L02 = HessL[1,1]

        # compute higher order correction term
        L_2 = (-(L20*qd**2) + (L10 - L11*qd)**2/L02)/24.

        # compute recovered Lagrangian truncated to order 0,2
        Lrecov2 = L_0 + self.h**2*L_2

        if nargout==1:
                return Lrecov2

        if nargout==2:
                return Lrecov2, L_0

    def Lrec2_d(self,z):
        return jacfwd(self.Lrec2)(z)

    def Lrec2_dd(self,z):
        return jacfwd(self.Lrec2_d)(z)

    def compute_motion(self,q0,q0dot,n):

        # compute q1 from q0,q0dot

        p0  = self.Lrec2_d(jnp.hstack([q0,q0dot]))[0,self.dim:]
        obj = lambda q1: self.p0_fun(q0,q1) - p0
        q1  = MyNewtonLAX(obj,q0,jacfwd(obj))

        # compute trajectory
        trj = jnp.zeros((self.dim,n+1))
        trj = ops.index_update(trj,ops.index[:,0],q0)
        trj = ops.index_update(trj,ops.index[:,1],q1)

        body_update = lambda j, trj: ops.index_update(trj,ops.index[:,j+2], self.q2_solve(trj[:,j],trj[:,j+1]) )
        trj = lax.fori_loop(0,len(trj[0])-2, body_update,trj)

        return trj

    def DELtrj(self,trj):
		# check discrete EL equation on trajectory
		# output: 1-norm of all errors

        err_q2_solver = jnp.zeros((self.dim,len(trj[0])-2))

        for j in tqdm(range(len(trj[0])-2)):
                err_q2_solver = ops.index_update(err_q2_solver,ops.index[:,j],self.DEL(trj[:,j],trj[:,j+1],trj[:,j+2]))

        return jnp.sum(jnp.abs(err_q2_solver)) # 1-norm of internal error (worst case: all errors sum up)

    # approximate recovery of qdot based on truncation of recovered L

    # truncation to fourth order
    @partial(jit, static_argnums=(0,))
    def qdotRecover2(self,q,p,qdotGuess):

        print("compiling qdotRecover2")

        obj = lambda qdot: self.Lrec2_d(jnp.hstack([q,qdot]))[0,self.dim:] - p
        obj_prime = lambda qdot: self.Lrec2_dd(jnp.block([q,qdot]))[0,self.dim:,self.dim:]

        return MyNewtonLAX(obj,qdotGuess,obj_prime)

    # computes derivatives to trajectory using the Lagrangian framework
    def motionqdot(self,trj):
        qdotRectrj = jnp.zeros((self.dim,len(trj[0])-1))
        qdotRectrj = ops.index_update(qdotRectrj,ops.index[:,0], self.qdotRecover2(trj[:,0],self.p0_fun(trj[:,0],trj[:,1]),jnp.array([0.])) )
        for j in tqdm(range(1,len(trj[0])-1)):
            qdotRectrj = ops.index_update(qdotRectrj,ops.index[:,j], self.qdotRecover2(trj[:,j],self.p0_fun(trj[:,j],trj[:,j+1]),qdotRectrj[:,j-1]))

        qdotfinal = self.qdotRecover2(trj[:,-1],self.p1_fun(trj[:,-2],trj[:,-1]),qdotRectrj[:,-1])
        qdotRectrj=jnp.block([ qdotRectrj, jnp.array([qdotfinal]).transpose()  ])

        return qdotRectrj

    # compute Hamiltonian (energy) along a trajectory using 2nd order truncation of identified L
    def energy_motion(self,trj,trjqdot):

        qqdot_trj = jnp.vstack([trj,trjqdot])

        p_trj= [self.p0_fun(trj[:,j],trj[:,j+1]) for j in range(len(trj[0])-1)]
        p_trj.append(self.p1_fun(trj[:,-2],trj[:,-1]))
        p_trj = jnp.array(p_trj)
        L_trj=jnp.array([self.Lrec2(qqdot_trj[:,j]) for j in tqdm(range(0,len(trj[0])))])
        HRec2 = jnp.sum(trjqdot*p_trj.transpose(),axis=0) - L_trj.flatten()

        return HRec2

    # compute Hamiltonian to z using 2nd order truncation of identified L
    @partial(jit, static_argnums=(0,2))
    def Hamiltonian(self,z,nargout=1):

        print("compiling Hamiltonian expression")

        if nargout==1:
                return jnp.dot(z[self.dim:],self.Lrec2_d(z)[0,self.dim:]) - self.Lrec2(z)

        else:
                L2,L0 = self.Lrec2(z,nargout=2)

                Lrec2all_d=jacfwd(lambda z :self.Lrec2(z,nargout=2))

                p2,p0=Lrec2all_d(z)

                p2 = p2[0,self.dim:]
                p0 = p0[self.dim:]

                H2 = jnp.dot(z[self.dim:],p2) - L2
                H0 = jnp.dot(z[self.dim:],p0) - L0

                return H2,H0

ext: py

reason: from jax


text:
from torch.utils.data import Dataset, DataLoader
import torch
import numpy as np
import utils as u

class splitter():
    '''
    creates 3 splits
    train
    dev
    test
    '''
    def __init__(self,args,tasker):

        if tasker.is_static: #### For static datsets
            assert args.train_proportion + args.dev_proportion < 1, \
                'there\'s no space for test samples'
            #only the training one requires special handling on start, the others are fine with the split IDX.

            random_perm=False
            indexes = tasker.data.nodes_with_label

            if random_perm:
                perm_idx = torch.randperm(indexes.size(0))
                perm_idx = indexes[perm_idx]
            else:
                print ('tasker.data.nodes',indexes.size())
                perm_idx, _ = indexes.sort()
            #print ('perm_idx',perm_idx[:10])

            self.train_idx = perm_idx[:int(args.train_proportion*perm_idx.size(0))]
            self.dev_idx = perm_idx[int(args.train_proportion*perm_idx.size(0)): int((args.train_proportion+args.dev_proportion)*perm_idx.size(0))]
            self.test_idx = perm_idx[int((args.train_proportion+args.dev_proportion)*perm_idx.size(0)):]
            # print ('train,dev,test',self.train_idx.size(), self.dev_idx.size(), self.test_idx.size())

            train = static_data_split(tasker, self.train_idx, test = False)
            train = DataLoader(train, shuffle=True,**args.data_loading_params)

            dev = static_data_split(tasker, self.dev_idx, test = True)
            dev = DataLoader(dev, shuffle=False,**args.data_loading_params)

            test = static_data_split(tasker, self.test_idx, test = True)
            test = DataLoader(test, shuffle=False,**args.data_loading_params)

            self.tasker = tasker
            self.train = train
            self.dev = dev
            self.test = test

        else: #### For datsets with time
            assert args.train_proportion + args.dev_proportion < 1, \
                'there\'s no space for test samples'
            #only the training one requires special handling on start, the others are fine with the split IDX.
            start = tasker.data.min_time + args.num_hist_steps #-1 + args.adj_mat_time_window
            end = args.train_proportion

            end = int(np.floor(tasker.data.max_time.type(torch.float) * end))
            train = data_split(tasker, start, end, test = False)
            train = DataLoader(train,**args.data_loading_params)

            start = end
            end = args.dev_proportion + args.train_proportion
            end = int(np.floor(tasker.data.max_time.type(torch.float) * end))
            if args.task == 'link_pred':
                dev = data_split(tasker, start, end, test = True, all_edges=True)
            else:
                dev = data_split(tasker, start, end, test = True)

            # dev = DataLoader(dev,num_workers=args.data_loading_params['num_workers'])
            dev = DataLoader(dev, **args.data_loading_params)

            start = end

            #the +1 is because I assume that max_time exists in the dataset
            end = int(tasker.max_time) + 1
            if args.task == 'link_pred':
                test = data_split(tasker, start, end, test = True, all_edges=True)
            else:
                test = data_split(tasker, start, end, test = True)

            # test = DataLoader(test,num_workers=args.data_loading_params['num_workers'])
            test = DataLoader(test, **args.data_loading_params)

            print ('Dataset splits sizes:  train',len(train), 'dev',len(dev), 'test',len(test))

            self.tasker = tasker
            self.train = train
            self.dev = dev
            self.test = test

class data_split(Dataset):
    def __init__(self, tasker, start, end, test, **kwargs):
        '''
        start and end are indices indicating what items belong to this split
        '''
        self.tasker = tasker
        self.start = start
        self.end = end
        self.test = test
        self.kwargs = kwargs

    def __len__(self):
        return self.end-self.start

    def __getitem__(self,idx):
        idx = self.start + idx
        t = self.tasker.get_sample(idx, test = self.test, **self.kwargs)
        return t

class static_data_split(Dataset):
    def __init__(self, tasker, indexes, test):
        '''
        start and end are indices indicating what items belong to this split
        '''
        self.tasker = tasker
        self.indexes = indexes
        self.test = test
        self.adj_matrix = tasker.adj_matrix

    def __len__(self):
        return len(self.indexes)

    def __getitem__(self,idx):
        idx = self.indexes[idx]
        return self.tasker.get_sample(idx,test = self.test)

max_stars_repo_name: junwei-h/EvolveGCN

max_issues_repo_name: junwei-h/EvolveGCN

max_forks_repo_name: junwei-h/EvolveGCN

reason: import numpy


text:
import warnings

from .dist_math import *

import numpy as np
from . import transforms

from theano.tensor.nlinalg import det, matrix_inverse, trace, eigh
from theano.tensor import dot, cast, eye, diag, eq, le, ge, gt, all
from theano.printing import Print
from pymc3.distributions.distribution import draw_values, generate_samples
import scipy.stats as st
import numpy.random as nr

__all__ = ['MvNormal', 'Dirichlet', 'Multinomial', 'Wishart', 'LKJCorr']

class MvNormal(Continuous):
    """
    Multivariate normal

    :Parameters:
        mu : vector of means
        tau : precision matrix

    .. math::
        f(x \mid \pi, T) = \frac{|T|^{1/2}}{(2\pi)^{1/2}} \exp\left\{ -\frac{1}{2} (x-\mu)^{\prime}T(x-\mu) \right\}

    :Support:
        2 array of floats
    """
    def __init__(self, mu, tau, *args, **kwargs):
        super(MvNormal, self).__init__(*args, **kwargs)
        self.mean = self.median = self.mode = self.mu = mu
        self.tau = tau

    def random(self, point=None, size=None):
        mu, tau = draw_values([self.mu, self.tau], point=point)
        samples = generate_samples(lambda mean, cov, size=None: \
                                    st.multivariate_normal.rvs(mean, cov, None if size == mean.shape else size),
                                   mean=mu, cov=tau,
                                   dist_shape=self.shape,
                                   broadcast_shape=mu.shape,
                                   size=size)
        return samples

    def logp(self, value):
        mu = self.mu
        tau = self.tau

        delta = value - mu
        k = tau.shape[0]

        result = k * log(2*pi) + log(1./det(tau))
        result += (delta.dot(tau) * delta).sum(axis=delta.ndim - 1)
        return -1/2. * result

class Dirichlet(Continuous):
    """
    Dirichlet

    This is a multivariate continuous distribution.

    .. math::
        f(\mathbf{x}) = \frac{\Gamma(\sum_{i=1}^k \theta_i)}{\prod \Gamma(\theta_i)}\prod_{i=1}^{k-1} x_i^{\theta_i - 1}
        \cdot\left(1-\sum_{i=1}^{k-1}x_i\right)^\theta_k

    :Parameters:
        a : float tensor
            a > 0
            concentration parameters
            last index is the k index

    :Support:
        x : vector
            sum(x) == 1 and x > 0

    .. note::
        Only the first `k-1` elements of `x` are expected. Can be used
        as a parent of Multinomial and Categorical nevertheless.
    """
    def __init__(self, a, transform=transforms.stick_breaking, *args, **kwargs):
        self.k = shape = a.shape[0]
        if "shape" not in kwargs.keys():
            kwargs.update({"shape": shape})
        super(Dirichlet, self).__init__(transform=transform, *args, **kwargs)
        self.a = a
        self.mean = a / sum(a)

        self.mode = switch(all(a > 1),
                           (a - 1) / sum(a - 1),
                           nan)

    def random(self, point=None, size=None):
        a = draw_values([self.a], point=point)
        samples = generate_samples(lambda a, size=None: \
                                    st.dirichlet.rvs(a, None if size == a.shape else size),
                                   a,
                                   dist_shape=self.shape,
                                   size=size)
        return samples

    def logp(self, value):
        k = self.k
        a = self.a

        # only defined for sum(value) == 1
        return bound(
            sum(logpow(value, a - 1) - gammaln(a), axis=0) + gammaln(sum(a)),
            k > 1,
            all(a > 0),
            all(value >= 0),
            all(value <= 1))

class Multinomial(Discrete):
    """
    Generalization of the binomial
    distribution, but instead of each trial resulting in "success" or
    "failure", each one results in exactly one of some fixed finite number k
    of possible outcomes over n independent trials. 'x[i]' indicates the number
    of times outcome number i was observed over the n trials.

    .. math::
        f(x \mid n, p) = \frac{n!}{\prod_{i=1}^k x_i!} \prod_{i=1}^k p_i^{x_i}

    :Parameters:
        n : int
            Number of trials.
        p : (k,)
            Probability of each one of the different outcomes.
            :math:`\sum_{i=1}^k p_i = 1)`, :math:`p_i \ge 0`.

    :Support:
        x : (ns, k) int
            Random variable indicating the number of time outcome i is
            observed. :math:`\sum_{i=1}^k x_i=n`, :math:`x_i \ge 0`.

    .. note::
        - :math:`E(X_i)=n p_i`
        - :math:`Var(X_i)=n p_i(1-p_i)`
        - :math:`Cov(X_i,X_j) = -n p_i p_j`
    """
    def __init__(self, n, p, *args, **kwargs):
        super(Multinomial, self).__init__(*args, **kwargs)
        self.n = n
        self.p = p
        self.mean = n * p
        self.mode = cast(round(n * p), 'int32')

    def _random(self, n, p, size=None):
        if size == p.shape:
            size = None
        return nr.multinomial(n, p, size=size)

    def random(self, point=None, size=None):
        n, p = draw_values([self.n, self.p], point=point)
        samples = generate_samples(self._random, n, p,
                                   dist_shape=self.shape,
                                   size=size)
        return samples

    def logp(self, x):
        n = self.n
        p = self.p
        # only defined for sum(p) == 1
        return bound(
            factln(n) + sum(x * log(p) - factln(x)),
            n >= 0,
            eq(sum(x), n),
            all(0 <= x), all(x <= n))

class Wishart(Continuous):
    """
    The Wishart distribution is the probability
    distribution of the maximum-likelihood estimator (MLE) of the precision
    matrix of a multivariate normal distribution. If V=1, the distribution
    is identical to the chi-square distribution with n degrees of freedom.

    For an alternative parameterization based on :math:`C=T{-1}` (Not yet implemented)

    .. math::
        f(X \mid n, T) = \frac{{\mid T \mid}^{n/2}{\mid X \mid}^{(n-k-1)/2}}{2^{nk/2}
        \Gamma_p(n/2)} \exp\left\{ -\frac{1}{2} Tr(TX) \right\}

    where :math:`k` is the rank of X.

    :Parameters:
      n : int
        Degrees of freedom, > 0.
      V : ndarray
        p x p positive definite matrix

    :Support:
      X : matrix
        Symmetric, positive definite.
    """
    def __init__(self, n, V, *args, **kwargs):
        super(Wishart, self).__init__(*args, **kwargs)
        warnings.warn('The Wishart distribution can currently not be used for MCMC sampling. The probability of sampling a symmetric matrix is basically zero. Instead, please use the LKJCorr prior. For more information on the issues surrounding the Wishart see here: https://github.com/pymc-devs/pymc3/issues/538.', UserWarning)
        self.n = n
        self.p = p = V.shape[0]
        self.V = V
        self.mean = n * V
        self.mode = switch(1*(n >= p + 1),
                     (n - p - 1) * V,
                      nan)

    def logp(self, X):
        n = self.n
        p = self.p
        V = self.V

        IVI = det(V)
        IXI = det(X)

        return bound(
            ((n - p - 1) * log(IXI) - trace(matrix_inverse(V).dot(X)) -
                n * p * log(2) - n * log(IVI) - 2 * multigammaln(n / 2., p)) / 2,
            gt(n, (p - 1)),
            all(gt(eigh(X)[0], 0)),
            eq(X, X.T)
        )

class LKJCorr(Continuous):
    """
    The LKJ (Lewandowski, Kurowicka and Joe) is a prior distribution for
    correlation matrices. If n = 1 this corresponds to the uniform distribution
    over correlation matrices. For n -> oo the LKJ prior approaches the identity
    matrix.

    For more details see:
    http://www.sciencedirect.com/science/article/pii/S0047259X09000876

    This implementation only returns the values of the upper triangular matrix
    excluding the diagonal. Here is a schematic for p = 5, showing the indexes
    of the elements:
        [[- 0 1 2 3]
         [- - 4 5 6]
         [- - - 7 8]
         [- - - - 9]
         [- - - - -]]

    :Parameters:
      n : float
        Shape parameter, Uniform distribution at n=1, > 0
      p : int
        Dimension of correlation matrix

    :Support:
      x : array of size p * (p - 1) / 2
        Upper triangular matrix values [-1,1].
    """
    def __init__(self, n, p, *args, **kwargs):
        self.n = n
        self.p = p
        n_elem = int(p * (p - 1) / 2)
        self.mean = np.zeros(n_elem)
        super(LKJCorr, self).__init__(shape=n_elem, *args, **kwargs)

        self.tri_index = np.zeros([p, p], dtype=int)
        self.tri_index[np.triu_indices(p, k=1)] = np.arange(n_elem)
        self.tri_index[np.triu_indices(p, k=1)[::-1]] = np.arange(n_elem)

    def _normalizing_constant(self, n, p):
        if n == 1:
            result = gammaln(2. * arange(1, int((p-1) / 2) + 1)).sum()
            if p % 2 == 1:
                result += (0.25 * (p ** 2 - 1) * log(pi)
                    - 0.25 * (p - 1) ** 2 * log(2.)
                    - (p - 1) * gammaln(int((p + 1) / 2))
                )
            else:
                result += (0.25 * p * (p - 2) * log(pi)
                    + 0.25 * (3 * p ** 2 - 4 * p) * log(2.)
                    + p * gammaln(p / 2) - (p-1) * gammaln(p)
                )
        else:
            result = -(p - 1) * gammaln(n + 0.5 * (p - 1))
            k = arange(1, p)
            result += (0.5 * k * log(pi) + gammaln(n + 0.5 * (p - 1 - k))).sum()
        return result

    def logp(self, x):
        n = self.n
        p = self.p

        X = x[self.tri_index]
        X = t.fill_diagonal(X, 1)

        result = self._normalizing_constant(n, p)
        result += (n - 1.) * log(det(X))
        return bound(result,
            n > 0,
            all(le(X, 1)),
            all(ge(X, -1)))

reason: import numpy,import scipy,from theano,from pymc3


text:
# coding=utf-8
import math
import os
import os.path as osp

import cv2
import numpy as np

SIZE_IMG = 255

#-中心旋转(x,y)坐标angle角度-----
def rotate_onepoint(x, y, angle):
    angle = math.pi * 2 - angle * math.pi / 180
    n = SIZE_IMG
    m = SIZE_IMG
    X = x * math.cos(angle) - y * math.sin(angle) - 0.5 * n * math.cos(
        angle) + 0.5 * m * math.sin(angle) + 0.5 * n
    Y = y * math.cos(angle) + x * math.sin(angle) - 0.5 * n * math.sin(
        angle) - 0.5 * m * math.cos(angle) + 0.5 * m
    return int(X), int(Y)

#-中心旋转标准飞机关键点集合p_sets中的点坐标，并存入savepath文件中-----
def rotate_anno(savepath, angle, p_sets):
    f_save = open(savepath, 'w')
    for p in p_sets:
        #-------------to do-----------------------
        x = p_sets[p][0]
        y = p_sets[p][1]
        #-------------to do-----------------------
        x, y = rotate_onepoint(x, y, angle)
        newline = str(p) + ' ' + str(x) + ' ' + str(y) + '\n'
        f_save.writelines(newline)
    f_save.close()

#-------------对path_read文件夹下的img_file图像文件，根据其对应的anno_file文件，绘制关键点并将结果存入path_save文件夹中-----
def draw_image_labels(img_file, anno_file, path_save, path_read):
    image = cv2.imread(os.path.join(path_read, img_file))
    with open(os.path.join(path_read, anno_file)) as f_anno:
        lines = f_anno.readlines()
        for line in lines:
            name_id, x, y = line.split()
            cv2.circle(image, (int(x), int(y)),
                       radius=1,
                       color=(0, 0, 255),
                       thickness=5)
            cv2.putText(image, str(name_id), (int(x) + 10, int(y) + 10),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1)
    cv2.imwrite(os.path.join(path_save, img_file), image)

#-------------对数据集文件夹root_data下的子文件夹进行遍历，对里面的样本文件利用draw_image_labels绘制其关键点，并将所有结果保存在root_save文件夹下-----
def draw_points(root_data, root_save):
    flight_name = []
    for rr, dn, files in os.walk(root_data):
        if rr == root_data:
            flight_name = dn
            for d in flight_name:
                if not osp.exists(os.path.join(root_save, d)):
                    os.makedirs(os.path.join(root_save, d))
        elif rr.split(os.path.sep)[-1] in flight_name and len(
                flight_name) != 0 and len(files) != 0:
            data_name = rr.split(os.path.sep)[-1]
            for img_file in files:
                if osp.splitext(img_file)[1] == '.jpg':
                    anno_file = img_file.replace(img_file[-4:], '.txt')
                    draw_image_labels(img_file, anno_file,
                                      os.path.join(root_save, data_name),
                                      os.path.join(root_data, data_name))

max_stars_repo_name: riciche/SimpleCVReproduction

max_issues_repo_name: riciche/SimpleCVReproduction

max_forks_repo_name: riciche/SimpleCVReproduction


text:
import time
import cv2
import os
import sys
from PIL import Image
import numpy as np
from matplotlib import pyplot as plt
from sklearn.model_selection import train_test_split
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import BatchNormalization
from tensorflow.keras.layers import Conv2D
from tensorflow.keras.layers import MaxPooling2D
from tensorflow.keras.layers import Activation
from tensorflow.keras.layers import Flatten
from tensorflow.keras.layers import Dropout
from tensorflow.keras.layers import Dense
import tensorflow.keras as tfk
from tensorflow.python.keras.callbacks import TensorBoard

EPOCHS = 15
IMG_WIDTH = 30
IMG_HEIGHT = 30
NUM_CATEGORIES = 43
TEST_SIZE = 0.4
batch_size = 16
pool_size = (2, 2)
inputShape = (IMG_WIDTH, IMG_HEIGHT, 3)
Name = "trafficsSignsModel-{}".format(int(time.time()))

def main():
	# Check command-line arguments
	if len(sys.argv) not in [1, 3]:
		sys.exit("Usage: python traffic.py data_directory [model.h5]")

	# Get image arrays and labels for all image files
	images, labels = load_data(os.path.dirname(sys.argv[0]))

	# Split data into training and testing sets
	labels = tfk.utils.to_categorical(labels)
	x_train, x_test, y_train, y_test = train_test_split(
		np.array(images), np.array(labels), test_size=TEST_SIZE)

	# Get a compiled neural network
	model = get_model()
	model.summary()
	# Fit model on training data
	tensorboard = TensorBoard(log_dir='logs/{}'.format(Name))

	history = model.fit(x_train, y_train, epochs=EPOCHS,callbacks=[tensorboard])

	# Evaluate neural network performance
	_,acc_train =model.evaluate(x_train, y_train, verbose=2)
	_,acc_test = model.evaluate(x_test,y_test,verbose=2)
	print('Train accuracy : %.3f, Test accuracy: %.3f' % (acc_train, acc_test))

	# Save model to file
	if len(sys.argv) == 1:
		folder_name = os.path.dirname(sys.argv[0])
		print(os.path.join(folder_name,"model1.h5"))
		model.save(os.path.join(folder_name,"model1.h5"))
		print(f"Model saved to {folder_name}.")
		plt.figure(0)
		plt.plot(history.history['loss'], label='training loss')
		plt.title("Loss")
		plt.xlabel("Epochs")
		plt.ylabel("loss")
		plt.legend()
		plt.show()

		plt.figure(1)
		plt.plot(history.history['accuracy'], label='training accuracy ')
		plt.title("accuracy")
		plt.xlabel("Epochs")
		plt.ylabel("accu")
		plt.legend()
		plt.show()

def load_data(data_dir):
	data = []
	labels = []
	for i in range(NUM_CATEGORIES):
		path = os.path.join(data_dir, "gtsrb", str(i))
		images = os.listdir(path)
		for j in images:
			try:
				image = cv2.imread(os.path.join(path, j))
				image_from_array = Image.fromarray(image, 'RGB')
				resized_image = image_from_array.resize((IMG_HEIGHT, IMG_WIDTH))
				data.append(np.array(resized_image))
				labels.append(i)
			except AttributeError:
				print("Error loading the image!")

	images_data = (data, labels)
	return images_data

def get_model():
	# initialize the model along with the input shape to be
	# "channels last" and the channels dimension itself
	model = Sequential()
	chDimension = -1
	# 1 set of layers (CONV , RELU , BNormalization and POOL layers)
	model.add(Conv2D(8, (5, 5), padding="same",input_shape=inputShape))
	model.add(Activation("relu"))
	model.add(BatchNormalization(axis=chDimension))
	model.add(MaxPooling2D(pool_size=(2, 2)))
	# first 2 sets of layers 1st set (CONV , RELU ,) and another one of( CONV and RELU) then POOL layer
	model.add(Conv2D(16, (3, 3), padding="same"))
	model.add(Activation("relu"))
	model.add(BatchNormalization(axis=chDimension))
	model.add(Conv2D(16, (3, 3), padding="same"))
	model.add(Activation("relu"))
	model.add(BatchNormalization(axis=chDimension))
	model.add(MaxPooling2D(pool_size=(2, 2)))
	# second 2 second two sets of (CONV , RELU ,CONV  RELU) layers and POOL layer
	model.add(Conv2D(32, (3, 3), padding="same"))
	model.add(Activation("relu"))
	model.add(BatchNormalization(axis=chDimension))
	model.add(Conv2D(32, (3, 3), padding="same"))
	model.add(Activation("relu"))
	model.add(BatchNormalization(axis=chDimension))
	model.add(MaxPooling2D(pool_size=(2, 2)))
	# first set of fully connected layers , RELU layers
	model.add(Flatten())
	model.add(Dense(128))
	model.add(Activation("relu"))
	model.add(BatchNormalization())
	model.add(Dropout(0.7))
	# second set of fully connected layers ,RELU layers
	model.add(Flatten())
	model.add(Dense(128))
	model.add(Activation("relu"))
	model.add(BatchNormalization())
	model.add(Dropout(0.7))
	# softmax classifier ,The softmax function is used as the activation function in the output layer of neural network models that predict a multinomial probability distribution.
	# That is, softmax is used as the activation function for multi-class classification problems where class membership is required on more than two class labels.
	model.add(Dense(NUM_CATEGORIES))
	model.add(Activation("softmax"))

	# compiling the model
	model.compile(loss="categorical_crossentropy", optimizer='adam',metrics=["accuracy"])

	return model

if __name__ == "__main__":
	main()


text:
"""Intra-op parallelism only e2e evaluation."""
import argparse
import time

import numpy as np
import ray

from alpa import DeviceCluster, global_config
from alpa.util import write_tsv, to_str_round, GB
from benchmark_2d_one_case import benchmark_one_case
from suite_paper_manual_gpt import gpt_specs
from suite_paper_manual_moe import moe_specs
from suite_paper_wresnet import wresnet_specs

benchmark_one_case_gpt = (lambda case, niter, num_host, num_devices_per_host:
    benchmark_one_case("gpt", case, niter, num_host, num_devices_per_host,
                       local=False, use_separate_process=True))

benchmark_one_case_moe = (lambda case, niter, num_host, num_devices_per_host:
    benchmark_one_case("moe", case, niter, num_host, num_devices_per_host,
                       local=False, use_separate_process=True))

benchmark_one_case_wresnet = (lambda case, niter, num_host, num_devices_per_host:
    benchmark_one_case("wresnet", case, niter, num_host, num_devices_per_host,
                       local=False, use_separate_process=True))

_ = None

gpt_intra_only = [
    # model,                   LD0, LD1, PD0, PD1, PP, NB,  FM,    Remat, RS,    Other, _
    (1024, *gpt_specs["350M"], 1,   1,   _,   _,   1,  64,  True,  True,  True,  _, _),
    (1024, *gpt_specs["760M"], 2,   1,   _,   _,   1,  64,  True,  True,  True,  _, _),
    (1024, *gpt_specs["1.3B"], 4,   1,   _,   _,   1,  64,  True,  True,  True,  _, _),
    (1024, *gpt_specs["2.6B"], 4,   2,   _,   _,   1,  32,  True,  True,  True,  _, _),
    (1024, *gpt_specs["6.7B"], 2,   8,   _,   _,   1,  64,  True,  True,  True,  _, _),
    (64,   *gpt_specs["15B"],  4,   8,   _,   _,   1,  8,   True,  True,  True,  _, _), # all_reduce_threshold = 1<<20
    (128,  *gpt_specs["15B"],  4,   8,   _,   _,   1,  16,  True,  True,  True,  _, _),
    (256,  *gpt_specs["15B"],  4,   8,   _,   _,   1,  32,  True,  True,  True,  _, _),
    (16,   *gpt_specs["39B"],  2,   32,  _,   _,   1,  8,   True,  True,  True,  _, _),
    (32,   *gpt_specs["39B"],  2,   32,  _,   _,   1,  16,  True,  True,  True,  _, _),
    (64,   *gpt_specs["39B"],  2,   32,  _,   _,   1,  32,  True,  True,  True,  _, _),
]

moe_intra_only = [
    # model,                   S_,   LD0, LD1, PD0, PD1, PP, NB, FM,    Remat, RS,    Other, _
    (1024, *moe_specs["380M"], 2048, 1,   1,   _,   _,   1,  32, False, True,  True,  _, _),
    (1024, *moe_specs["690M"], 2048, 2,   1,   _,   _,   1,  32, False, True,  True,  _, _),
    (1024, *moe_specs["1.3B"], 2048, 4,   1,   _,   _,   1,  16, False, True,  True,  _, _),
    (1024, *moe_specs["2.4B"], 2048, 8,   1,   _,   _,   1,  16, False, True,  True,  _, _),
    (1024, *moe_specs["10B"],  2048, 2,   8,   _,   _,   1,  32, False, True,  True,  _, _),
    (64,   *moe_specs["27B"],  2048, 1,   32,  _,   _,   1,  2,  False, True,  True,  _, _),
    (128,  *moe_specs["27B"],  2048, 1,   32,  _,   _,   1,  4,  False, True,  True,  _, _),
    (256,  *moe_specs["27B"],  2048, 1,   32,  _,   _,   1,  8,  False, True,  True,  _, _),
    (32,   *moe_specs["70B"],  2048, 1,   64,  _,   _,   1,  8,  True,  True,  True,  _, _), # allow_all_to_all = False
    (64,   *moe_specs["70B"],  2048, 1,   64,  _,   _,   1,  16, True,  True,  True,  _, _),
    (128,  *moe_specs["70B"],  2048, 1,   64,  _,   _,   1,  32, True,  True,  True,  _, _),
]

wresnet_intra_only = [
    # model,                       D0, D1, NB, FM,    RS,   Remat, other
    (1536, *wresnet_specs["250M"], 1,  1,  48, False, True, _, _),
    (1536, *wresnet_specs["500M"], 2,  1,  32, False, True, _, _),
    (1536, *wresnet_specs["1B"],   4,  1,  32, False, True, _, _),
    (1536, *wresnet_specs["2B"],   8,  1,  48, False, True, _, _),
    (1536, *wresnet_specs["4B"],   2,  8,  64, False, True, _, _),
    (64,   *wresnet_specs["6.8B"], 4,  8,  2,  False, True, _, _),
    (128,  *wresnet_specs["6.8B"], 4,  8,  4,  False, True, _, _),
    (256,  *wresnet_specs["6.8B"], 4,  8,  8,  False, True, _, _),
    (16,   *wresnet_specs["13B"],  4,  16, 2,  False, True, _, _),
    (32,   *wresnet_specs["13B"],  4,  16, 4,  False, True, _, _),
    (64,   *wresnet_specs["13B"],  4,  16, 8,  False, True, _, _),
]

suites = [
    # GPT
    ("gpt", "alpa.intra_only", gpt_intra_only, benchmark_one_case_gpt),

    # MoE
    ("moe", "alpa.intra_only", moe_intra_only, benchmark_one_case_moe),

    # Wide-ResNet
    ("wresnet", "alpa.intra_only", wresnet_intra_only, benchmark_one_case_wresnet),
]

def build_cases(wanted_model):
    instance = "p3.16"
    exp_name = "e2e"

    cases = []
    for suite in suites:
        model_name, method, args_list, benchmark_func = suite

        if wanted_model is not None and model_name != wanted_model:
            continue

        for i, args in enumerate(args_list):
            if "moe" in model_name:
                num_devices = args[8] * args[9]
            else: # GPT and W-ResNet
                num_devices = args[6] * args[7]
            num_hosts = ((num_devices + 7) // 8)
            num_devices_per_host = min(num_devices, 8)
            cases.append((exp_name, instance, num_hosts, num_devices_per_host,
                          model_name, method, benchmark_func, args))

    return cases

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--model", type=str)
    args = parser.parse_args()
    cases = build_cases(args.model)
    niter = 3

    for case in cases:
        exp_name, instance, num_hosts, num_devices_per_host, model_name,\
            method, benchmark_func, args = case

        # Benchmark case
        result = benchmark_func(args, niter, num_hosts, num_devices_per_host)
        param_count, ilp_objective, peak_mem, latencies, tflops = result
        if np.mean(latencies) < 0:
            tflops = -1

        value_dict = {
            "param_count": param_count / 1e9,
            "peak_mem": peak_mem / GB,
            "latencies": latencies,
            "tflops": tflops if tflops > 0.0 else -1.0,
        }

        # Log results
        heads = ["Exp", "Instance", "num_hosts", "num_devices_per_host", "model_name",
                 "method", "value", "tstamp"]
        values = [exp_name, instance, num_hosts, num_devices_per_host,
                  model_name, method, to_str_round(value_dict, 4),
                  int(time.time())]
        write_tsv(heads, values, f"results_intra_only.tsv")


text:
from pathlib2 import Path
import pathlib2
import os
from datetime import datetime
import numpy as np
import matplotlib.pyplot as plt

PROJECT_DIR = Path(__file__).resolve().parents[1]
DATA_DIR = PROJECT_DIR / "data"
MODEL_DIR = PROJECT_DIR / "models"
FIGURE_DIR = PROJECT_DIR / "figures"

def ensure_dir(file_path):
    """ create a safely nested folder
    """
    if type(file_path) == str:
        if "." in os.path.basename(os.path.normpath(file_path)):
            directory = os.path.dirname(file_path)
        else:
            directory = os.path.normpath(file_path)
        if not os.path.exists(directory):
            try:
                os.makedirs(directory)
            except FileExistsError as e:
                # multiprocessing can cause directory creation problems
                print(e)
    elif type(file_path) == pathlib2.PosixPath:
        # if this is a file
        if len(file_path.suffix) > 0:
            file_path.parent.mkdir(parents=True, exist_ok=True)
        else:
            file_path.mkdir(parents=True, exist_ok=True)

def most_recent_subdirectory(dataset_loc):
    """ return the subdirectory that has been generated most
    recently with the "%Y-%m-%d_%H-%M-%S" time scheme used in AVGN
    """
    subdir_list = list((dataset_loc).iterdir())
    directory_dates = [
        datetime.strptime(i.name, "%Y-%m-%d_%H-%M-%S") for i in subdir_list
    ]
    return subdir_list[np.argsort(directory_dates)[-1]]

def save_fig(
    loc,
    dpi=300,
    save_pdf=False,
    save_svg=False,
    save_png=False,
    save_jpg=True,
    pad_inches=0.0,
):
    if save_pdf:
        plt.savefig(
            str(loc) + ".pdf", dpi=dpi, bbox_inches="tight", pad_inches=pad_inches
        )
    if save_svg:
        plt.savefig(
            str(loc) + ".svg",
            dpi=dpi,
            bbox_inches="tight",
            pad_inches=pad_inches,
            transparent=True,
        )
    if save_png:
        plt.savefig(
            str(loc) + ".png",
            dpi=dpi,
            bbox_inches="tight",
            pad_inches=pad_inches,
            transparent=True,
        )
    if save_jpg:
        plt.savefig(
            str(loc) + ".jpg",
            dpi=int(dpi / 2),
            bbox_inches="tight",
            pad_inches=pad_inches,
        )


text:
import numpy as np

from uniclass_to_nf_ea_com_source.b_code.configurations.common_constants.uniclass_bclearer_constants import PARENT_CODE_COLUMN_NAME, \
    UNICLASS2015_OBJECT_TABLE_NAME

def add_parent_code_column_to_uniclass_objects_table(
        dictionary_of_dataframes: dict)\
        -> dict:
    uniclass_2015_object_table = \
        dictionary_of_dataframes[
            UNICLASS2015_OBJECT_TABLE_NAME]

    uniclass_2015_object_table[
        PARENT_CODE_COLUMN_NAME] = \
        np.NaN

    dictionary_of_dataframes[
        UNICLASS2015_OBJECT_TABLE_NAME] = \
        uniclass_2015_object_table

    return \
        dictionary_of_dataframes


text:
import numpy
from ekphrasis.classes.preprocessor import TextPreProcessor
from ekphrasis.classes.tokenizer import SocialTokenizer
from ekphrasis.dicts.emoticons import emoticons
from scipy.stats import pearsonr
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import jaccard_similarity_score, f1_score, \
    precision_score, recall_score, accuracy_score
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import Normalizer
from sklearn.svm import SVR

from modules.CustomPreProcessor import CustomPreProcessor
from modules.NBOWVectorizer import NBOWVectorizer

def eval_reg(y_hat, y):
    results = {
        "pearson": pearsonr([float(x) for x in y_hat],
                            [float(x) for x in y])[0]
    }

    return results

def eval_clf(y_test, y_p):
    results = {
        "f1": f1_score(y_test, y_p, average='macro'),
        "recall": recall_score(y_test, y_p, average='macro'),
        "precision": precision_score(y_test, y_p, average='macro'),
        "accuracy": accuracy_score(y_test, y_p)
    }

    return results

def eval_mclf(y, y_hat):
    results = {
        "jaccard": jaccard_similarity_score(numpy.array(y),
                                            numpy.array(y_hat)),
        "f1-macro": f1_score(numpy.array(y), numpy.array(y_hat),
                             average='macro'),
        "f1-micro": f1_score(numpy.array(y), numpy.array(y_hat),
                             average='micro')
    }

    return results

def bow_model(task, max_features=10000):
    if task == "clf":
        algo = LogisticRegression(C=0.6, random_state=0,
                                  class_weight='balanced')
    elif task == "reg":
        algo = SVR(kernel='linear', C=0.6)
    else:
        raise ValueError("invalid task!")

    word_features = TfidfVectorizer(ngram_range=(1, 1),
                                    tokenizer=lambda x: x,
                                    analyzer='word',
                                    min_df=5,
                                    # max_df=0.9,
                                    lowercase=False,
                                    use_idf=True,
                                    smooth_idf=True,
                                    max_features=max_features,
                                    sublinear_tf=True)
    preprocessor = TextPreProcessor(
        backoff=['url', 'email', 'percent', 'money', 'phone', 'user', 'time',
                 'url',
                 'date', 'number'],
        include_tags={"hashtag", "allcaps", "elongated", "repeated",
                      'emphasis',
                      'censored'},
        fix_html=True,
        segmenter="twitter",
        corrector="twitter",
        unpack_hashtags=True,
        unpack_contractions=True,
        spell_correct_elong=False,
        tokenizer=SocialTokenizer(lowercase=True).tokenize,
        dicts=[emoticons])

    model = Pipeline([
        ('preprocess', CustomPreProcessor(preprocessor, to_list=True)),
        ('bow-feats', word_features),
        ('normalizer', Normalizer(norm='l2')),
        ('clf', algo)
    ])

    return model

def nbow_model(task, embeddings, word2idx):
    if task == "clf":
        algo = LogisticRegression(C=0.6, random_state=0,
                                  class_weight='balanced')
    elif task == "reg":
        algo = SVR(kernel='linear', C=0.6)
    else:
        raise ValueError("invalid task!")

    embeddings_features = NBOWVectorizer(aggregation=["mean"],
                                         embeddings=embeddings,
                                         word2idx=word2idx,
                                         stopwords=False)

    preprocessor = TextPreProcessor(
        backoff=['url', 'email', 'percent', 'money', 'phone', 'user', 'time',
                 'url',
                 'date', 'number'],
        include_tags={"hashtag", "allcaps", "elongated", "repeated",
                      'emphasis',
                      'censored'},
        fix_html=True,
        segmenter="twitter",
        corrector="twitter",
        unpack_hashtags=True,
        unpack_contractions=True,
        spell_correct_elong=False,
        tokenizer=SocialTokenizer(lowercase=True).tokenize,
        dicts=[emoticons])

    model = Pipeline([
        ('preprocess', CustomPreProcessor(preprocessor, to_list=True)),
        ('embeddings-feats', embeddings_features),
        ('normalizer', Normalizer(norm='l2')),
        ('clf', algo)
    ])

    return model

reason: import numpy,from scipy


text:
import mss
import mss.tools
from window import quicktime_values

with mss.mss() as sct:

    #1334x750

    #left=x, top=y de inicio de la imagen, cuadrante 4 en plano
    monitor = {"top": 400, "left": 800, "width": 375, "height": 200}
    import time

import cv2
import mss
import numpy

with mss.mss() as sct:
    # Part of the screen to capture

    #quicktime_values = [xSize, ySize, xPos, yPos]
    quicktime_values = quicktime_values().rstrip().split(', ')
    quicktime_values = list(map(int, quicktime_values))

    monitor = {"top": quicktime_values[3],
        "left": quicktime_values[2],
        "width": quicktime_values[0],
        "height": quicktime_values[1]}

    while "Screen capturing":
        last_time = time.time()

        # Get raw pixels from the screen, save it to a Numpy array
        img = numpy.array(sct.grab(monitor))

        # Display the picture
        cv2.imshow("OpenCV/Numpy normal", cv2.resize(img, (quicktime_values[0]//2, quicktime_values[1]//2)) )

        #if img[10, 20]
        print(quicktime_values)

        # Press "q" to quit
        if cv2.waitKey(25) & 0xFF == ord("q"):
            cv2.destroyAllWindows()
            break


text:
from decision_tree import DecisionTree
import csv
import numpy as np
import ast

"""
Here,
1. X is assumed to be a matrix with n rows and d columns where n is the
number of total records and d is the number of features of each record.
2. y is assumed to be a vector of labels of length n.
3. XX is similar to X, except that XX also contains the data label for each
record.
"""
class RandomForest(object):
    num_trees = 0
    decision_trees = []

    # the bootstrapping datasets for trees
    # bootstraps_datasets is a list of lists, where each list in bootstraps_datasets is a bootstrapped dataset.
    bootstraps_datasets = []

    # the true class labels, corresponding to records in the bootstrapping datasets
    # bootstraps_labels is a list of lists, where the 'i'th list contains the labels corresponding to records in
    # the 'i'th bootstrapped dataset.
    bootstraps_labels = []

    def __init__(self, num_trees):
        # Initialization done here
        self.num_trees = num_trees
        self.decision_trees = [DecisionTree() for i in range(num_trees)]

    def _bootstrapping(self, XX, n):
        # Reference: https://en.wikipedia.org/wiki/Bootstrapping_(statistics)
        #
        # Create a sample dataset of size n by sampling with replacement
        # from the original dataset XX.
        samples = [] # sampled dataset
        labels = []  # class labels for the sampled records

        random_ind = np.random.randint(low=0, high=n, size=n)

        for index in random_ind:
            samples.append(XX[index][:-1])
            labels.append(XX[index][-1])

        return samples, labels

    def bootstrapping(self, XX):
        # Initializing the bootstap datasets for each tree
        for i in range(self.num_trees):
            data_sample, data_label = self._bootstrapping(XX, len(XX))
            self.bootstraps_datasets.append(data_sample)
            self.bootstraps_labels.append(data_label)

    def fitting(self):
        # Train `num_trees` decision trees using the bootstraps datasets
        # and labels by calling the learn function from DecisionTree class.
        for idx, tree in enumerate(self.decision_trees):
            tree.learn(self.bootstraps_datasets[idx], self.bootstraps_labels[idx])

    def voting(self, X):
        y = []

        for record in X:
            # Following steps have been performed here:
            #   1. Find the set of trees that consider the record as an
            #      out-of-bag sample.
            #   2. Predict the label using each of the above found trees.
            #   3. Use majority vote to find the final label for this recod.
            votes = []
            for i in range(len(self.bootstraps_datasets)):
                dataset = self.bootstraps_datasets[i]
                if record not in dataset:
                    OOB_tree = self.decision_trees[i]
                    effective_vote = OOB_tree.classify(record)
                    votes.append(effective_vote)

            counts = np.bincount(votes)

            if len(counts) == 0:
                #  Handle the case where the record is not an out-of-bag sample
                #  for any of the trees.
                index = self.bootstraps_datasets[0].index(record)
                y = np.append(y, self.bootstraps_labels[0][index])
            else:
                y = np.append(y, np.argmax(counts))

        return y

def main():
    X = list()
    y = list()
    XX = list()  # Contains data features and data labels
    numerical_cols = set([0,10,11,12,13,15,16,17,18,19,20]) # indices of numeric attributes (columns)

    # Loading data set
    print("reading sample_data")
    with open("sample_data.csv") as f:
        next(f, None)

        for line in csv.reader(f, delimiter=","):
            xline = []
            for i in range(len(line)):
                if i in numerical_cols:
                    xline.append(ast.literal_eval(line[i]))
                else:
                    xline.append(line[i])

            X.append(xline[:-1])
            y.append(xline[-1])
            XX.append(xline[:])

    forest_size = 10

    # Initializing a random forest.
    randomForest = RandomForest(forest_size)

    # Creating the bootstrapping datasets
    print("creating the bootstrap datasets")
    randomForest.bootstrapping(XX)

    # Building trees in the forest
    print("fitting the forest")
    randomForest.fitting()

    # Calculating an unbiased error estimation of the random forest
    # based on out-of-bag (OOB) error estimate.
    y_predicted = randomForest.voting(X)

    # Comparing predicted and true labels
    results = [prediction == truth for prediction, truth in zip(y_predicted, y)]

    # Accuracy
    accuracy = float(results.count(True)) / float(len(results))

    print("accuracy: %.4f" % accuracy)
    print("OOB estimate: %.4f" % (1-accuracy))

if __name__ == "__main__":
    main()

max_stars_repo_name: codywynn/random-forest-model

max_issues_repo_name: codywynn/random-forest-model

max_forks_repo_name: codywynn/random-forest-model


text:
import unittest
from typing import List

import numpy

from classes.ml.sgd import Sgd

class TestSgd(unittest.TestCase):

    def test_sigmoid_f(self):
        self.assertAlmostEqual(Sgd.sigmoid_f(-1.0), 0.2689414213699951)
        self.assertAlmostEqual(Sgd.sigmoid_f(0.0), 0.5)
        self.assertAlmostEqual(Sgd.sigmoid_f(1.0), 0.7310585786300049)

    def test_sigmoid(self):
        z = numpy.array([-1.0, 0.0, 1.0])
        output = Sgd.sigmoid(z).tolist()
        self.assertEqual(output, [0.2689414213699951, 0.5, 0.7310585786300049])

    def test_sigmoid_prime_f(self):
        self.assertAlmostEqual(Sgd.sigmoid_prime_f(-1.0), 0.19661193324148185)
        self.assertAlmostEqual(Sgd.sigmoid_prime_f(0.0), 0.25)
        self.assertAlmostEqual(Sgd.sigmoid_prime_f(1.0), 0.19661193324148185)

    def test_sigmoid_prime(self):
        z = numpy.array([-1.0, 0.0, 1.0])
        output = Sgd.sigmoid_prime(z).tolist()
        self.assertEqual(output, [0.19661193324148185, 0.25, 0.19661193324148185])

    def test_softmax_01(self):
        z = numpy.array([2.0, 1.0, 0.0])
        output = Sgd.softmax(z).tolist()
        self.assertEqual(output, [0.6652409557748219, 0.24472847105479764, 0.09003057317038046])
        self.assertAlmostEqual(sum(output), 1.0)

    def test_softmax_02(self):
        z = numpy.array([2.0, 1.0, 0.0, -1.0, -2.0])
        output = Sgd.softmax(z).tolist()
        self.assertEqual(output, [0.6364086465588308, 0.23412165725273662, 0.0861285444362687, 0.03168492079612427,
                                  0.011656230956039609])
        self.assertAlmostEqual(sum(output), 1.0)

    def test_learn_xnor_operations(self):
        sgd = Sgd(sizes=[2, 3, 1])

        print('Training...')
        for _ in range(500):
            sgd.update_network(input=numpy.array([[0], [0]]), expected_output=numpy.array([[1]]), mini_batch=2, eta=2.0)
            sgd.update_network(input=numpy.array([[0], [1]]), expected_output=numpy.array([[0]]), mini_batch=2, eta=2.0)
            sgd.update_network(input=numpy.array([[1], [0]]), expected_output=numpy.array([[0]]), mini_batch=2, eta=2.0)
            sgd.update_network(input=numpy.array([[1], [1]]), expected_output=numpy.array([[1]]), mini_batch=2, eta=2.0)

        print('Testing...')
        output1: List[float] = sgd.feed_forward(input=numpy.array([[0], [0]]))
        self.assertEqual(round(output1[0][0]), 1.0)
        output4: List[float] = sgd.feed_forward(input=numpy.array([[0], [1]]))
        self.assertEqual(round(output4[0][0]), 0.0)
        output3: List[float] = sgd.feed_forward(input=numpy.array([[1], [0]]))
        self.assertEqual(round(output3[0][0]), 0.0)
        output2: List[float] = sgd.feed_forward(input=numpy.array([[1], [1]]))
        self.assertEqual(round(output2[0][0]), 1.0)

max_stars_repo_name: mckev/ml

max_issues_repo_name: mckev/ml

max_forks_repo_name: mckev/ml


text:
import os

import joblib
import numpy as np
import pandas as pd

from Fuzzy_clustering.version2.common_utils.logging import create_logger

def compute_area_grid(lat, long, resolution, round_coord, levels):
    lat_range = np.arange(np.around(lat, round_coord) - 20, np.around(lat, round_coord) + 20,
                          resolution)
    lat1 = lat_range[np.abs(lat_range - lat).argmin()] - resolution / 10
    lat2 = lat_range[np.abs(lat_range - lat).argmin()] + resolution / 10

    long_range = np.arange(np.around(long, round_coord) - 20, np.around(long, round_coord) + 20,
                           resolution)
    long1 = long_range[np.abs(long_range - long).argmin()] - resolution / 10
    long2 = long_range[np.abs(long_range - long).argmin()] + resolution / 10

    return [[lat1 - resolution * levels, long1 - resolution * levels],
            [lat2 + resolution * levels, long2 + resolution * levels]]

class ProjectGroupInit:
    """
    Class responsible for managing and loading the
    power output or load.

    """

    def __init__(self, static_data):
        """

        Parameters
        ----------
        static_data: python dict
            contains all the information required to load the power measurement for specific project(s).
        """
        self.static_data = static_data  # dict containing information about project paths, model structure and training
        # params, input file, see in util_database_timos.py and config_timos.py
        self.file_data = static_data['data_file_name']  # input .csv file PROBLEM_TYPE + '_ts.csv' i.e. wind_ts.csv
        self.project_owner = static_data[
            'project_owner']  # Name of project owner or research program i.e. my_projects or CROSSBOW
        self.projects_group = static_data['projects_group']  # Name of the country
        self.area_group = static_data['area_group']  # coordinates of the country
        self.version_group = static_data['version_group']
        self.version_model = static_data['version_model']
        self.weather_in_data = static_data[
            'weather_in_data']  # True if input file contains more columns than the power output column
        self.nwp_model = static_data['NWP_model']
        self.nwp_resolution = static_data['NWP_resolution']
        self.data_variables = static_data['data_variables']  # Variable names used

        self.projects = []  # list containing all the parks, we're interested in. Each park is considered as a project.
        self.use_rated = True

        self.model_type = self.static_data['type']
        self.sys_folder = self.static_data['sys_folder']
        self.path_nwp = self.static_data['path_nwp']
        self.path_group = self.static_data['path_group']
        self.path_nwp_group = self.static_data['path_nwp_group']
        self.group_static_data = []
        self.logger = create_logger(logger_name=f'ProjectInitManager_{self.model_type}', abs_path=self.path_group,
                                    logger_path=f'log_{self.projects_group}.log', write_type='a')

    def initialize(self):
        if os.path.exists(os.path.join(os.path.dirname(self.file_data), 'coord_auto_' + self.model_type + '.csv')):
            self.file_coord = os.path.join(os.path.dirname(self.file_data), 'coord_auto_' + self.model_type + '.csv')
        else:
            self.file_coord = os.path.join(os.path.dirname(self.file_data), 'coord_' + self.model_type + '.csv')

        if not os.path.exists(self.file_coord) and not self.weather_in_data:
            raise IOError('File with coordinates does not exist')

        self.file_rated = os.path.join(os.path.dirname(self.file_data), 'rated_' + self.model_type + '.csv')
        if not os.path.exists(self.file_rated):
            if self.model_type in {'wind', 'pv'} and self.projects_group not in {'IPTO'}:
                raise ValueError('Provide rated_power for each project. The type of projects is %s', self.model_type)
            self.use_rated = False
        else:
            self.use_rated = True

        self.load_power_of_parks()  # Loads power output, coordinates and rated power.

        if len(self.projects) == 0:
            raise ImportError('No project loaded. check the input file in configuration')

        if self.check_project_names():
            for project_name in self.projects:
                path_project = self.path_group + '/' + project_name
                if not os.path.exists(path_project):
                    os.makedirs(path_project)
                path_model = path_project + '/model_ver' + str(self.version_model)
                if not os.path.exists(path_model):
                    os.makedirs(path_model)
                path_backup = self.path_group + '/backup_models/' + project_name + '/model_ver' + str(
                    self.version_model)
                if not os.path.exists(path_backup):
                    os.makedirs(path_backup)
                path_data = path_model + '/DATA'
                if not os.path.exists(path_data):
                    os.makedirs(path_data)
                path_fuzzy_models = path_model + '/fuzzy_models'
                if not os.path.exists(path_fuzzy_models):
                    os.makedirs(path_fuzzy_models)
                if self.use_rated:
                    if project_name == self.projects_group + '_' + self.model_type and project_name not in self.rated.index.to_list():
                        rated = self.rated.sum().to_list()[0]
                    else:
                        rated = self.rated.loc[project_name].to_list()[0]
                else:
                    rated = None
                if hasattr(self, 'coord'):
                    if project_name == 'APE_net' or self.model_type == 'load' or project_name == self.projects_group + '_' + self.model_type:
                        coord = dict()
                        for name, lat_long in self.coord.iterrows():
                            coord[name] = lat_long.values.tolist()
                    else:
                        coord = self.coord.loc[project_name].to_list()  # [lat, long]
                else:
                    coord = None
                area = self.create_area(coord)

                temp = {'_id': project_name,
                        'owner': self.project_owner,
                        'project_group': self.projects_group,
                        'type': self.model_type,
                        'location': coord,
                        'areas': area,
                        'rated': rated,
                        'path_project': path_project,
                        'path_model': path_model,
                        'path_group': self.path_group,
                        'version_group': self.version_group,
                        'version_model': self.version_model,
                        'path_backup': path_backup,
                        'path_data': path_data,
                        'pathnwp': self.path_nwp_group,
                        'path_fuzzy_models': path_fuzzy_models,
                        'run_on_platform': False,
                        }
                static_data = dict()
                for key, value in self.static_data.items():
                    static_data[key] = value
                for key, value in temp.items():
                    static_data[key] = value

                self.group_static_data.append({'_id': project_name, 'static_data': static_data})
                joblib.dump(static_data, os.path.join(path_model, 'static_data.pickle'))
                with open(os.path.join(path_model, 'static_data.txt'), 'w') as file:
                    for k, v in static_data.items():
                        if not isinstance(v, dict):
                            file.write(str(k) + ' >>> ' + str(v) + '\n\n')
                        else:
                            file.write(str(k) + ' >>> ' + '\n')
                            for kk, vv in v.items():
                                file.write('\t' + str(kk) + ' >>> ' + str(vv) + '\n')

            joblib.dump(self.group_static_data, os.path.join(self.path_group, 'static_data_projects.pickle'))
            self.logger.info('Static data of all projects created')

    def check_project_names(self):
        flag = True
        if self.model_type in {'wind', 'pv'}:
            for name in self.projects:
                if name not in self.coord.index.to_list() and name != self.projects_group + '_' + self.model_type and name != 'APE_net':
                    flag = False
                    self.logger.info('There is inconsistency to files data and coord for the project %s', name)
            if not flag:
                raise ValueError('Inconsistency in project names between data and coord')

        if self.use_rated:
            for name in self.projects:
                if name not in self.rated.index.to_list() and name != self.projects_group + '_' + self.model_type:
                    flag = False
                    self.logger.info('There is inconsistency to files data and rated for the project %s', name)
            if not flag:
                raise ValueError('Inconsistency in project names between data and rated')

        return flag

    def load_power_of_parks(self):
        try:
            # Data containing power output or load. Each column refers to a different wind, pv park.
            self.data = pd.read_csv(self.file_data, header=0, index_col=0, parse_dates=True, dayfirst=True)
        except Exception:
            self.logger.info(f'Cannot import timeseries from the file {self.file_data}')
            raise IOError(f'Cannot import timeseries from the file {self.file_data}')
        self.logger.info('Timeseries imported successfully from the file %s', self.file_data)

        if 'total' in self.data.columns:  # In some cases, the total output of all parks is included.
            self.data = self.data.rename(
                columns={'total': self.projects_group + '_' + self.model_type})  # e.g group = 'Greece'

        if self.static_data['Evaluation_start']:

            valid_combination = True
            time_offset = pd.DateOffset(hours=0)

            if self.model_type == 'fa':
                time_offset = pd.DateOffset(days=372)
            elif self.model_type == 'load':
                if self.data.columns[0] == 'lv_load':
                    time_offset = pd.DateOffset(days=9001)
                else:
                    if self.static_data['horizon'] == 'short-term':
                        time_offset = pd.DateOffset(hours = 350)
                    else:
                        if self.static_data['ts_resolution'] == 'hourly':
                            time_offset = pd.DateOffset(hours = 9001)
                        elif self.static_data['ts_resolution'] == '15min':
                            time_offset = pd.DateOffset(minutes = 60 * 9001)

            if valid_combination:
                try:
                    eval_date = pd.to_datetime(self.static_data['Evaluation_start'], format='%d%m%Y %H:%M')
                    self.data_eval = self.data.iloc[np.where(self.data.index > eval_date - time_offset)]
                    self.data = self.data.iloc[np.where(self.data.index <= eval_date)]
                except Exception:
                    raise ValueError('Wrong date format, use %d%m%Y %H:%M. Or the date does not exist in the dataset')

        if self.model_type == 'load':
            self.projects.append(self.data.columns[0])
        elif self.model_type == 'fa':
            if self.version_model == 0:
                self.projects.append('fa_curr_morning')
            elif self.version_model == 1:
                self.projects.append('fa_ahead_morning')
            else:
                raise ValueError(
                    'Version model should be 0 for current day and 1 for day ahead otherwise choose another group version')
        else:
            for name in self.data.columns:
                var = f'{self.projects_group}_{self.model_type}' if name == 'total' else name
                self.projects.append(var)
        if not self.weather_in_data:
            try:
                # For each of the park, load its coordinates. (lat,long) single tuple
                self.coord = pd.read_csv(self.file_coord, header=None, index_col=0)
            except Exception:
                self.logger.info('Cannot import coordinates from the file %s', self.file_coord)
                raise IOError('Cannot import coordinates from the file %s', self.file_coord)
            self.logger.info('Coordinates imported successfully from the file %s', self.file_coord)
        else:
            self.logger.info('Coordinates in the data')

        if self.use_rated:
            try:
                # For each park, read its rated (maximum) power.
                self.rated = pd.read_csv(self.file_rated, header=None, index_col=0)
            except Exception:
                self.logger.info('Cannot import Rated Power from the file %s', self.file_rated)
                raise IOError('Cannot import Rated Power from the file %s', self.file_rated)
            self.logger.info('Rated Power imported successfully from the file %s', self.file_rated)

        self.logger.info('Data loaded successfully')

    def create_area(self, coord):
        levels = 4 if self.nwp_resolution == 0.05 else 2
        round_coord = 1 if self.nwp_resolution == 0.05 else 0

        if coord is None:
            area = dict()
        elif isinstance(coord, list):
            if len(coord) == 2:
                lat, long = coord[0], coord[1]
                area = compute_area_grid(lat, long, self.nwp_resolution, round_coord, levels)
            elif len(coord) == 4:
                area = list(np.array(coord).reshape(2, 2))
            else:
                raise ValueError(
                    'Wrong coordinates. Should be point (lat, long) or area [lat1, long1, lat2, long2]')
        elif isinstance(coord, dict):
            area = dict()
            for key, value in coord.items():
                if len(value) == 2:
                    lat, long = value[0], value[1]
                    area[key] = compute_area_grid(lat, long, self.nwp_resolution, round_coord, levels)
                else:
                    area[key] = np.array(value).reshape(2, 2)
        else:
            raise ValueError('Wrong coordinates. Should be dict or list')

        self.logger.info('Areas created successfully')

        return area


text:
from flask import Blueprint, request, render_template
import string
from src.crypto import hash_password
from src.sqlDAO import sqlDAO_User
import numpy as np
from src.utils import does_string_contain_only_allowed_chars, alert_types

register_page = Blueprint('register_page', __name__, template_folder='templates')

@register_page.route('/register', methods=['GET', 'POST'])
def register():
    mess = "Wprowadź poniżej swoje dane"
    data = init_data()

    if request.method == 'POST':
        try:
            data = read_user_data()
            validate_data(data)
            register_user(data)
            mess = f"Witaj na pokładzie {data['user']}!"
            return render_template('register.html', message=mess, alert_t=alert_types[1], data=data)
        except Exception as err:
            return render_template('register.html', message=err, alert_t=alert_types[2], data=data)

    return render_template('register.html', message=mess, data=data, alert_t = alert_types[0])

def init_data():
    data = {
        'user' : 'username',
        'email' : 'email@mail.com',
        'password' : '',
        'repassword' : ''
    }
    return data

def read_user_data():
    data = {}
    data['user'] = request.form.get('nickname')
    data['email'] = request.form.get('email')
    data['password'] = request.form.get('password')
    data['repassword'] = request.form.get('repassword')
    return data

def validate_data(data):
    allowed = string.ascii_letters + string.digits + "_-"
    condition, c = does_string_contain_only_allowed_chars(data['user'], allowed)
    if not condition:
        raise Exception(f"W nazwie użytkownika znalazły się niedozwolone znaki: {c}")

    allowed = string.ascii_letters + string.digits + "@._"
    condition, c = does_string_contain_only_allowed_chars(data['email'], allowed)
    if not condition:
        raise Exception(f"W podanym emailu znalazły się niedozwolone znaki: {c}")

    allowed = string.ascii_letters + string.digits + string.punctuation
    condition, c = does_string_contain_only_allowed_chars(data['password'], allowed)
    if not condition:
        raise Exception(f"W podanym haśle znalazły się niedozwolone znaki: {c}")

    allowed = string.ascii_letters + string.digits + string.punctuation
    condition, c = does_string_contain_only_allowed_chars(data['repassword'], allowed)
    if not condition:
        raise Exception(f"W podanym haśle znalazły się niedozwolone znaki: {c}")

    if sqlDAO_User.is_exists_user(data['user']):
        raise Exception(f"Użytkownik {data['user']} już istnieje!")

    if sqlDAO_User.is_exists_email(data['email']):
        raise Exception(f"Użytkownik z takim emailem {data['email']} już istnieje!")

    username_constriants(data['user'])
    password_constraints(data['password'])

    if data['password'] != data['repassword']:
        raise Exception("Podałeś dwa różne hasła!")

def register_user(data):
    hash_pass, salt = hash_password(data['password'])
    sqlDAO_User.insert_user(data['user'], data['email'], hash_pass, salt)

def username_constriants(name):
    minLength = 6
    maxLength = 30
    if len(name) < minLength or len(name) > maxLength:
        raise Exception(f"Nazwa użytkownika powinna składać się od {minLength} do {maxLength} znaków")

def password_constraints(password):
    min_nLowercases = 1
    min_nUppercases = 1
    min_nDigits = 1
    min_nPunctuations = 1
    length = 8
    if len(password) < length:
        raise Exception(f"Hasło powinno składać się z conajmniej {length} znaków!")
    nLowercases = np.sum([c in string.ascii_lowercase for c in password])
    if nLowercases < min_nLowercases:
        raise Exception(f"Hasło powinno zawierać conajmniej {min_nLowercases} małą literę!")
    nUppercases = np.sum([c in string.ascii_uppercase for c in password])
    if nUppercases < min_nUppercases:
        raise Exception(f"Hasło powinno zawierać conajmniej {min_nUppercases} wielką literę!")
    nDigits = np.sum([c in string.digits for c in password])
    if nDigits < min_nDigits:
        raise Exception(f"Hasło powinno zawierać conajmniej {min_nDigits} cyfrę!")
    nPunctuations = np.sum([c in string.punctuation for c in password])
    if nPunctuations < min_nPunctuations:
        raise Exception(f"Hasło powinno zawierać conajmniej {min_nPunctuations} znak specjalny!")


text:
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for fedjax.algorithms.fed_avg."""
from absl.testing import absltest

from fedjax.algorithms import fed_avg
from fedjax.core import client_datasets
from fedjax.core import optimizers

import jax
import jax.numpy as jnp
import numpy.testing as npt

def grad_fn(params, batch, rng):
  del rng
  return jax.tree_util.tree_map(lambda l: l / jnp.sum(batch['x']), params)

class FedAvgTest(absltest.TestCase):

  def test_federated_averaging(self):
    client_optimizer = optimizers.sgd(learning_rate=1.0)
    server_optimizer = optimizers.sgd(learning_rate=1.0)
    client_batch_hparams = client_datasets.ShuffleRepeatBatchHParams(
        batch_size=2, num_epochs=1, seed=0)
    algorithm = fed_avg.federated_averaging(grad_fn, client_optimizer,
                                            server_optimizer,
                                            client_batch_hparams)

    with self.subTest('init'):
      state = algorithm.init({'w': jnp.array([0., 2., 4.])})
      npt.assert_array_equal(state.params['w'], [0., 2., 4.])
      self.assertLen(state.opt_state, 2)

    with self.subTest('apply'):
      clients = [
          (b'cid0',
           client_datasets.ClientDataset({'x': jnp.array([2., 4., 6.])}),
           jax.random.PRNGKey(0)),
          (b'cid1',
           client_datasets.ClientDataset({'x': jnp.array([8., 10.])}),
           jax.random.PRNGKey(1)),
      ]
      state, client_diagnostics = algorithm.apply(state, clients)
      npt.assert_allclose(state.params['w'], [0., 1.5655555, 3.131111])
      npt.assert_allclose(client_diagnostics[b'cid0']['delta_l2_norm'],
                          1.4534444262)
      npt.assert_allclose(client_diagnostics[b'cid1']['delta_l2_norm'],
                          0.2484521282)

  def test_create_train_for_each_client(self):
    client_optimizer = optimizers.sgd(learning_rate=1.0)
    train_for_each_client = fed_avg.create_train_for_each_client(
        grad_fn, client_optimizer)
    batched_clients = [
        (b'cid0',
         [{'x': jnp.array([2., 4., 6.])}, {'x': jnp.array([8., 10., 12.])}],
         jax.random.PRNGKey(0)),
        (b'cid1',
         [{'x': jnp.array([1., 3., 5.])}, {'x': jnp.array([7., 9., 11.])}],
         jax.random.PRNGKey(1)),
    ]
    server_params = {'w': jnp.array(4.0)}
    client_outputs = dict(train_for_each_client(server_params, batched_clients))
    npt.assert_allclose(client_outputs[b'cid0']['w'], 0.45555544)
    npt.assert_allclose(client_outputs[b'cid1']['w'], 0.5761316)

if __name__ == '__main__':
  absltest.main()

reason: import numpy,import jax


text:
import numpy as np
from sklearn.model_selection import KFold
from tqdm import tqdm
from sklearn.metrics import accuracy_score

from model import SimpleMLP
from loss import CrossEntropyLoss
from optimizer import SGD
from utils import export_data

def train():
    X_train, y_train = export_data()
    X_test, y_test = export_data(mode="test")
    batch_size = 32
    n_epochs = 30
    kfold_train = KFold(n_splits=y_train.shape[0] // batch_size)
    kfold_test = KFold(n_splits=y_test.shape[0] // batch_size)

    model = SimpleMLP(hidden_layers=[28 * 28, 256, 64, 10])

    optimizer = SGD(model.parameters, model.grads, lr=1e-2)
    criterion = CrossEntropyLoss()

    for epoch in range(n_epochs):
        print(f"*****************Epoch {epoch + 1}: ********************")
        train_losses = []
        train_epoch_iterator = tqdm(kfold_train.split(y_train),
                                    desc="Training (Step X) (loss=X.X)",
                                    bar_format="{l_bar}{r_bar}",
                                    dynamic_ncols=True, )
        for id, (_, index) in enumerate(train_epoch_iterator):
            X_batch_train = X_train[index].T
            y_batch_train = y_train[index]
            output = model.forward(X_batch_train)
            loss = criterion.forward(output, y_batch_train)
            train_losses.append(loss)
            train_epoch_iterator.set_description(
                "Training (Step %d / %d) (loss=%2.5f)" % (id + 1, y_train.shape[0] // batch_size, loss)
            )

            optimizer.zero_grad()
            criterion.backward()
            optimizer.step()
        print("Train loss: ", sum(train_losses) / len(train_losses))
        print("Test phase: ")
        accuracy_scores = []
        precision_scores = []
        recall_scores = []
        f1_scores = []

        for _, index in tqdm(kfold_test.split(y_test)):
            X_batch_test = X_test[index].T
            y_batch_test = y_test[index]
            predict = np.argmax(model.forward(X_batch_test)[0], axis=0)

            accuracy_scores.append(accuracy_score(predict, y_batch_test))

        print("Test result: ")
        print(f"Accuracy: {sum(accuracy_scores) / len(accuracy_scores)}, ")

if __name__ == "__main__":
    train()
    pass

max_stars_repo_name: ngocphucck/SimpleMLP

max_issues_repo_name: ngocphucck/simple-MLP

max_forks_repo_name: ngocphucck/simple-MLP


text:
# https://adventofcode.com/2019/day/11

import collections
import numpy as np

from intcode import IntcodeComputer

def paint(starting_color):
    robot.reset()
    panels = collections.defaultdict(int)
    x, y = 0, 0
    panels[(x, y)] = starting_color  # 0 = black, 1 = white
    # right, down, left, up
    dx = (1, 0, -1, 0)
    dy = (0, -1, 0, 1)
    d_i = 3
    left_or_right = (-1, 1)
    while not robot.has_halted:
        panels[(x, y)] = robot.run([panels[(x, y)]])
        if robot.has_halted:
            break
        d_i = (d_i + left_or_right[robot.run()]) % 4
        x += dx[d_i]
        y += dy[d_i]
    return panels

with open("../../input/2019-11-input.txt") as file:
    program = [int(i) for i in file.read().split(",")]
robot = IntcodeComputer(program, True, True)

# part 1:
print(len(paint(0)))  # 2428

# part 2:
image = paint(1)
x_min = min(image.keys())[0]
x_max = max(image.keys())[0]
y_min = min(image.keys(), key=lambda v: v[1])[1]
y_max = max(image.keys(), key=lambda v: v[1])[1]
registration = np.full((y_max - y_min + 1, x_max - x_min + 1), " ")
for k, v in image.items():
    x, y = k
    x -= x_min
    y -= y_min
    if v == 0:
        registration[y, x] = " "
    elif v == 1:
        registration[y, x] = "█"
registration = np.flip(registration, 0)
registration = np.apply_along_axis(
    lambda line: "".join(line),
    1,
    registration
)
registration = "\n".join(registration)
print(registration)  # RJLFBUCU


text:
import tensorflow as tf
from dltk.core.activations import leaky_relu
import numpy as np

def test_leaky_relu():
    test_alpha = tf.constant(0.1)
    test_inp_1 = tf.constant(1.)
    test_inp_2 = tf.constant(-1.)

    test_relu_1 = leaky_relu(test_inp_1, test_alpha)
    test_relu_2 = leaky_relu(test_inp_2, test_alpha)

    with tf.Session() as s:
        out_1 = s.run(test_relu_1)
        assert np.isclose(out_1, 1.), \
            'Got {} but expected {}'.format(out_1, 1.)

        out_2 = s.run(test_relu_2)
        assert np.isclose(out_2, -0.1), \
            'Got {} but expected {}'.format(out_2, -0.1)

max_stars_repo_name: themantalope/DLTK

max_issues_repo_name: themantalope/DLTK

max_forks_repo_name: themantalope/DLTK


text:
# -*- coding: utf-8 -*-
"""
Created on Thu Nov  8 11:31:10 2018

@author: Eugen Rusu
A Spyder script to visualize the sensor data.
LINK to the GitHub Repo:
https://github.com/EugenSusurrus/har_android_data
"""

import numpy as np
import pandas as pd
from scipy import integrate
#from scipy.signal import argrelextrema
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt

# Importing the data
RAW_SENSOR_DATA_500MS = pd.read_csv('har_sensor_data_500.csv', sep=',')

#Ploting the raw data
AX = plt.figure(figsize=(16, 8))
plt.plot(RAW_SENSOR_DATA_500MS['Time'], RAW_SENSOR_DATA_500MS['ACCELEROMETER X'])
plt.plot(RAW_SENSOR_DATA_500MS['Time'], RAW_SENSOR_DATA_500MS['ACCELEROMETER Y'])
plt.plot(RAW_SENSOR_DATA_500MS['Time'], RAW_SENSOR_DATA_500MS['ACCELEROMETER Z'])
plt.grid()
plt.legend()
plt.title('$ACC_X$, $ACC_Y$ and $ACC_Z$ Raw sensor data')

# Setting the working dataframe and other variables
CONSTANT_G = 9.82912

DF = pd.DataFrame({'Time': [], \
                   'ACC_X': [], \
                   'ACC_Y': [], \
                   'ACC_Z':[], \
                   'ACC_RMS': [], \
                   'Speed': [], \
                   'Distance': [], \
                   'Cumulated Distance':[]})

# Calibrating the ACC_X, ACC_Y and ACC_Z data along each accelerometer axis
DF['ACC_X'] = RAW_SENSOR_DATA_500MS['ACCELEROMETER X']
DF['ACC_Y'] = RAW_SENSOR_DATA_500MS['ACCELEROMETER Y']
DF['ACC_Z'] = RAW_SENSOR_DATA_500MS['ACCELEROMETER Z']

# Transforming the time to IS units
DF['Time'] = (RAW_SENSOR_DATA_500MS['Time'] - RAW_SENSOR_DATA_500MS['Time'].iloc[0]) / 1000 #s

# Rounds the time units
DF['Time'] = np.round(DF['Time'], decimals=2)

# The acceleration root mean square magnitude
DF['ACC_RMS'] = (DF['ACC_X']**2 + DF['ACC_Y'] **2 + DF['ACC_Z']**2) ** 0.5 - 9.81

# Speed in m/s
# Returns the cumulative integral, hence needs to be differentiated to get the instantaneous speed
DF['Speed'] = integrate.cumtrapz(DF['ACC_Z'], DF['Time'], initial=0.0)
# Differentiating to get the instantaneous speed
# df['Speed'] = df['Speed'].diff()
# Fills the first NaN value at index 0 resulted from differentiation
DF['Speed'].fillna(0, inplace=True)

# Distance in m
# Returns the cumulative integral, hence needs to be differentiated to get the
# instantaneous distance
DF['Cumulated Distance'] = integrate.cumtrapz(DF['Speed'], DF['Time'], initial=0.0)
# Diferentiating to get the instantaneous distance
# df['Distance'] = df['Cumulated Distance'].diff()
# Fills the first NaN value at the index 0 resulted from differentiation
DF['Distance'].fillna(0, inplace=True)

# Shows the resulting df
DF.head()

AX = plt.figure(figsize=(16, 8))
plt.plot(DF['Time'], DF['ACC_X'])
plt.plot(DF['Time'], DF['ACC_Y'])
plt.plot(DF['Time'], DF['ACC_Z'])
plt.plot(DF['Time'], DF['ACC_RMS'])
plt.grid()
plt.title('$ACC_X$, $ACC_Y$ and $ACC_Z$ vs $ACC_{RMS}$')
plt.legend()

# Standard deviation
AX = plt.figure(figsize=(16, 8))
plt.plot(DF['Time'], DF['ACC_RMS'])
plt.axhline(y=9.82912, color='r', linestyle=':')
plt.plot(DF['Time'], DF['ACC_RMS'].rolling(10).std())
plt.plot(DF['Time'], DF['ACC_RMS'].rolling(10).mean())
plt.legend(['ACC_RMS', 'G_TRESH', 'ACC_5S_STD', 'ACC_5S_MEAN'])
plt.grid()

AX3D = Axes3D(plt.figure(figsize=(10, 10)))

ACC_SURF = AX3D.plot_trisurf(DF['ACC_X'], DF['ACC_Y'], DF['ACC_Z'], cmap='jet', linewidth=0.1)

AX3D.scatter(DF['ACC_X'], DF['ACC_Y'], DF['ACC_Z'], color='k')

AX3D.set_xlabel('$ACC_X$')
AX3D.set_ylabel('$ACC_Y$')
AX3D.set_zlabel('$ACC_Z$')

CBAR = plt.colorbar(ACC_SURF, shrink=0.7)
CBAR.set_label('Acceleration $m/s^2$')

plt.axis('equal')
plt.title('$ACC_X$, $ACC_Y$ and $ACC_Z$ Scatter')
plt.show()

#SAcceleration RMS, Speed, Distance and Cumulated distance

FIG, AX1 = plt.subplots(figsize=(16, 8))

COLOR = 'tab:red'
AX1.set_xlabel('time (s)')
AX1.set_ylabel('ACC_Z', color=COLOR)
AX1.plot(DF['Time'], DF['ACC_Z'], color=COLOR)
AX1.tick_params(axis='y', labelcolor=COLOR)

AX2 = AX1.twinx()  # instantiate 2nd axis to plot speed
COLOR = 'tab:blue'
AX2.set_ylabel('Speed $m/s$', color=COLOR)
AX2.plot(DF['Time'], DF['Speed'], color=COLOR)
AX2.tick_params(axis='y', labelcolor=COLOR)

AX3 = AX1.twinx()  # instantiate 3rd axis to plot distance
COLOR = 'tab:green'
AX3.set_ylabel('Distance $m$', color=COLOR)
AX3.plot(DF['Time'], DF['Distance'], color=COLOR)
AX3.tick_params(axis='y', labelcolor=COLOR)

X3 = AX1.twinx()  # instantiate 3rd axis to plot distance
COLOR = 'tab:orange'
AX3.set_ylabel('Cumulated Distance $m$', color=COLOR)
AX3.plot(DF['Time'], DF['Cumulated Distance'], color=COLOR)
AX3.tick_params(axis='y', labelcolor=COLOR)

FIG.tight_layout()  # otherwise the right y-label is slightly clipped
plt.grid()
plt.show()

# Calculates the number of steps
# Steps are considered for an ACC_RMS magnitude between 0 and 5
N_ = 5 # number of points to be checked before and after
STEP_SIGNAL = pd.DataFrame()
STEP_SIGNAL = DF[(DF['ACC_RMS'] > 0) & (DF['ACC_RMS'] <= 5)]
STEP_SIGNAL_LIST = STEP_SIGNAL['ACC_RMS'].tolist()
STEP_SIGNAL_LIST = np.array(STEP_SIGNAL_LIST)

# the number of steps
NUM_STEPS = len(STEP_SIGNAL_LIST)

# Calculate the number of jumps
# Jumps are considered for an ACC_RMS magnitude greater or equal than 6
JUMP_SIGNAL = pd.DataFrame()
JUMP_SIGNAL = DF[DF['ACC_RMS'] >= 6]
JUMP_SIGNAL_LIST = JUMP_SIGNAL['ACC_RMS'].tolist()
JUMP_SIGNAL_LIST = np.array(JUMP_SIGNAL_LIST)

# the number of jumps
NUM_JUMPS = len(JUMP_SIGNAL_LIST)

print('\n\tWelcome to the HAR app using Android sensor data\n')
print('\n\t**************************************************\n')
print('\n\tThe total distance traveled is: {} meters'.format(DF['Cumulated Distance'].max()))
print('\n\tThe average registered velocity is: {} m/s'.format((DF['Speed'].mean() ** 2) ** 0.5))
print('\n\tYou have made {} steps'.format(NUM_STEPS))
print('\n\tYou have made {} jumps'.format(NUM_JUMPS))