text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Functions about using HSC catalogs."""

import os

import numpy as np

import astropy.units as u
from astropy.table import Table, Column

__all__ = ['moments_to_shape', 'abmag_to_image', 'world_to_image', 'select_clean_objects',
           'objects_to_galsim', 'mag_to_flux']

# Flux unit in HSC catalog
FLUX_UNIT_S16A = (u.erg / u.s / u.Hz / u.cm ** 2)
FLUX_UNIT_S17A = (u.erg / u.s / u.Hz / u.cm ** 2)
FLUX_UNIT_S18A = (u.erg / u.s / u.Hz / u.cm ** 2)
FLUX_UNIT_PDR1 = (u.Jansky * 1.E-9)
FLUX_UNIT_PDR2 = (u.Jansky * 1.E-9)

def abmag_to_image(abmag):
    """
    Convert AB magnitude into HSC image flux unit.
    """
    return 10.0 ** ((27.0 - abmag) / 2.5)

def mag_to_flux(catalog, mag_col, zeropoint=27.0, update=True):
    """
    Convert AB magnitude into HSC image flux unit.
    """
    flux = 10.0 ** ((zeropoint - catalog[mag_col]) / 2.5)
    if update:
        flux_col = mag_col.replace('mag', 'flux')
        if flux_col in catalog.colnames:
            catalog.remove_column(flux_col)
        catalog.add_column(data=flux, name=flux_col)
        return catalog
    return flux

def world_to_image(catalog, wcs, ra='ra', dec='dec', update=True):
    """
    Get the X, Y coordinate on the image.
    """
    xy_arr = np.asarray(
        [wcs.all_world2pix(catalog[ra][ii], catalog[dec][ii], 0)
         for ii in np.arange(len(catalog))])
    x_arr, y_arr = xy_arr[:, 0], xy_arr[:, 1]

    if update:
        if 'x' in catalog.colnames:
            catalog.remove_column('x')
        if 'y' in catalog.colnames:
            catalog.remove_column('y')
        catalog.add_column(Column(data=x_arr, name='x'))
        catalog.add_column(Column(data=y_arr, name='y'))
        return catalog
    return x_arr, y_arr

def moments_to_shape(catalog, shape_type='i_sdssshape', axis_ratio=False,
                     radian=False, update=True, to_pixel=False):
    """
    Convert the 2nd moments into elliptical shape: radius, ellipticity, position angle.
    """
    try:
        xx = catalog["{}_11".format(shape_type)]
        yy = catalog["{}_22".format(shape_type)]
        xy = catalog["{}_12".format(shape_type)]
    except KeyError:
        print("Wrong column name!")
        raise

    e1 = (xx - yy) / (xx + yy)
    e2 = (2.0 * xy / (xx + yy))
    # Get the r50 or determinant radius
    rad = np.sqrt(xx + yy)
    rad = rad / 0.168 if to_pixel else rad
    # Ellipticity or axis ratio
    ell = np.sqrt(e1 ** 2.0 + e2 ** 2.0)
    ell = 1.0 - ell if axis_ratio else ell
    # Position angle in degree or radian
    theta = (-0.5 * np.arctan2(e2, e1))
    theta = (theta * 180. / np.pi) if not radian else theta

    if update:
        rad_col = "{}_r".format(shape_type)
        theta_col = "{}_theta".format(shape_type)
        if axis_ratio:
            ell_col = "{}_ba".format(shape_type)
        else:
            ell_col = "{}_e".format(shape_type)
        if rad_col in catalog.colnames:
            catalog.remove_column(rad_col)
        catalog.add_column(Column(data=rad, name=rad_col))
        if ell_col in catalog.colnames:
            catalog.remove_column(ell_col)
        catalog.add_column(Column(data=ell, name=ell_col))
        if theta_col in catalog.colnames:
            catalog.remove_column(theta_col)
        catalog.add_column(Column(data=theta, name=theta_col))
        return catalog
    return rad, ell, theta

def select_clean_objects(catalog, check_flag='gri', check_psf='i', check_cmodel='i',
                         return_catalog=False, verbose=False):
    """
    Select the "clean" objects.
    """
    clean_mask = np.ones(len(catalog)).astype(np.bool)

    # Check data quality
    if check_flag is not None:
        for f in check_flag:
            mask_of_this_band = (
                np.isfinite(catalog['{}_extendedness'.format(f)]) &
                ~catalog['{}_flag_edge'.format(f)] &
                ~catalog['{}_flag_saturated_cen'.format(f)] &
                ~catalog['{}_flag_interpolated_cen'.format(f)]
                )
            clean_mask = clean_mask & mask_of_this_band

    # Check PSF flux/magnitude
    if check_psf is not None and check_psf in 'grizy':
        psf_mag = '{}_psf_mag'.format(check_psf)
        psf_flux = '{}_psf_flux'.format(check_psf)
        if psf_mag in catalog.colnames:
            psf_mask = (np.isfinite(catalog['{}_psf_mag'.format(check_psf)]) &
                        (catalog['{}_psf_mag'.format(check_psf)] > 0))
        elif psf_flux in catalog.colnames:
            psf_mask = (np.isfinite(catalog['{}_psf_flux'.format(check_psf)]) &
                        (catalog['{}_psf_flux'.format(check_psf)] > 0))
        else:
            raise KeyError("# PSF flux/mag not available!")
        clean_mask = clean_mask & psf_mask

    # Check PSF flux/magnitude
    if check_cmodel is not None and check_cmodel in 'grizy':
        cmodel_mag = '{}_cmodel_mag'.format(check_cmodel)
        cmodel_flux = '{}_cmodel_flux'.format(check_cmodel)
        if cmodel_mag in catalog.colnames:
            cmodel_mask = (
                np.isfinite(catalog['{}_cmodel_mag'.format(check_cmodel)]) &
                (catalog['{}_cmodel_mag'.format(check_cmodel)] > 0))
        elif cmodel_flux in catalog.colnames:
            cmodel_mask = (
                np.isfinite(catalog['{}_cmodel_flux'.format(check_cmodel)]) &
                (catalog['{}_cmodel_flux'.format(check_cmodel)] > 0))
        else:
            raise KeyError("# CModel flux/mag not available!")
        clean_mask = clean_mask & cmodel_mask

    if verbose:
        print("# {}/{} objects are clean.".format(clean_mask.sum(), len(catalog)))

    if return_catalog:
        return catalog[clean_mask], clean_mask
    return clean_mask

def objects_to_galsim(img, objects, psf_model=None, extended='i_extendedness',
                      psf_mag='psf_mag', gal_mag='cmodel_mag',
                      exp_mag='cmodel_exp_mag', dev_mag='cmodel_dev_mag'):
    """
    Convert the HSC objects into GalSim model image.
    """
    try:
        import galsim
    except ImportError:
        raise Exception("# Please install GalSim first!")

    # Shape of the image
    img_shape = img.shape

    # Empty image
    img_empty = np.zeros(img_shape)

    # This is the "canvas" of the model
    img_model = galsim.ImageF(img_shape[1], img_shape[0])

    # The "true" center of the image is allowed to be halfway between two pixels, as is the
    # case for even-sized images.  full_image.center is an integer position,
    # which would be 1/2 pixel up and to the right of the true center in this case.
    img_center = img_model.true_center

    # Pass the WCS from the header of the image to GalSim
    # img_wcs = galsim.AstropyWCS(wcs=cutout_wcs)

    # Pass the PSF image to GalSim
    if psf_model is not None:
        psf_obj = galsim.InterpolatedImage(
            galsim.image.Image(psf_model), scale=1.0)
    else:
        psf_obj = None

    # Go through the list of objects, there appears to be no other way than a for loop

    for ii, obj in enumerate(objects):
        # Position of the image
        img_position = galsim.PositionD(obj['x'], obj['y'])

        # Get the offset of the object on the canvas
        obj_offset = galsim.PositionD(
            img_position.x - img_center.x + 1.,
            img_position.y - img_center.y + 1.)

        # Seperate the type of the object
        if obj[extended] < 0.5:
            if np.isfinite(obj[psf_mag]) & (obj[psf_mag] > 0):
                # Generate a star
                psf_flux = abmag_to_image(obj[psf_mag])
                psf = psf_obj.withFlux(psf_flux)
                # Add it to the empty image
                img_empty += psf.drawImage(img_model, offset=obj_offset).array
            else:
                print("# Cannot generate star {} with magnitude {}".format(
                    ii, obj[psf_mag]))
        else:
            if (np.isfinite(obj[exp_mag]) & (obj[exp_mag] > 0) &
                    np.isfinite(obj[dev_mag]) & (obj[dev_mag] > 0) &
                    np.isfinite(obj[gal_mag]) & (obj[gal_mag] > 0)):
                try:
                    # The exponential component
                    flux_exp = abmag_to_image(obj[exp_mag])
                    shape_exp = galsim.Shear(
                        q=obj['cmodel_exp_ellipse_ba'],
                        beta=obj['cmodel_exp_ellipse_theta'] * galsim.degrees)
                    comp_exp = galsim.Exponential(
                        half_light_radius=obj['cmodel_exp_ellipse_r'], flux=flux_exp)
                    comp_exp = comp_exp.shear(shape_exp)

                    # The De Vacouleurs component
                    flux_dev = abmag_to_image(obj[dev_mag])
                    shape_dev = galsim.Shear(
                        q=obj['cmodel_dev_ellipse_ba'],
                        beta=obj['cmodel_dev_ellipse_theta'] * galsim.degrees)
                    comp_dev = galsim.DeVaucouleurs(
                        half_light_radius=obj['cmodel_dev_ellipse_r'], flux=flux_dev,
                        trunc=(6.0 * obj['cmodel_dev_ellipse_r']))
                    comp_dev = comp_dev.shear(shape_dev)

                    # Combine the two component
                    cmodel = galsim.Add([comp_exp, comp_dev])
                    if psf_obj is not None:
                        # Convolution with PSF
                        cmodel = galsim.Convolve([cmodel, psf_obj])

                    # Add it to the empty image
                    img_empty += cmodel.drawImage(
                        img_model, method='no_pixel', offset=obj_offset).array
                except Exception:
                    print("# Cannot generate galaxy {} with magnitude {}".format(
                        ii, obj[gal_mag]))
            else:
                print("# Problematic galaxy {} with magnitude {}".format(
                    ii, obj[gal_mag]))

    return img_empty

def remove_is_null(table, output=None, verbose=True, string='isnull', return_data=True):
    """
    Remove the xxx_isnull columns from the catalog.
    This is an annoying issue with FITS table from HSC database.

    Parameters
    ----------
    table : str or astropy.table object
        Name of the FITS catalog or the data itself
    cat_hdu : int, optional
        The HDU of the catalog data.
        Default: 1
    string : str, optional
        The name of the Null columns.
        Default: 'isnull'
    output : str, optional
        If output is None, will write a new FITS table with '_clean'
        suffix.
        Default: None
    return_data : bool, optional
        Whether return the cleaned data.
        Default : True
    verbose : bool, optional
        Default : True
    """
    if isinstance(table, Table):
        data = table
    else:
        if not os.path.isfile(table):
            raise Exception("# Can not find catalog: %s" % table)
        data = Table.read(table, format='fits')

    if verbose:
        print("Reading the data....")
    col_names = data.colnames
    col_remove = [col for col in col_names if string in col]
    data.remove_columns(col_remove)

    if output is None:
        if verbose:
            print("Saving data to %s ..." % output)
        data.write(table, format='fits', overwrite=True)
    else:
        data.write(output.strip(), format='fits', overwrite=True)

    if return_data:
        return data
    else:
        return None

max_stars_repo_name: Tang-SL/unagi

max_issues_repo_name: Tang-SL/unagi

max_forks_repo_name: Tang-SL/unagi

reason: import numpy,import astropy,from astropy


text:
import os
import numpy as np
import pybullet as p
import pybullet_data
import copy
import pdb
import pickle
from gym.utils import seeding

FIXED_JOINT_NUMBER = 0
JOINT_INDEX = [0, 1, 2, 3, 4, 5, 6, 7, 8]
class Panda():

    def __init__(self, basePosition=[0,0,0]):
        self.init_pos = [0.0, -np.pi/6, 0.0, -2*np.pi/4-np.pi/2, 0.0, np.pi/2+np.pi/3, np.pi/4, 0.0, 0.0, 0.03, 0.03]
        self.urdfRootPath="gym_panda\\panda_bullet\\assets"
        self.panda = p.loadURDF(os.path.join(self.urdfRootPath,"franka_panda","panda.urdf"),useFixedBase=True,basePosition=basePosition)
        self.status = "abled"

    """functions that environment should use"""

    # has two modes: joint space control (0) and ee-space control (1)
    # djoint is a 7-dimensional vector of joint velocities
    # dposition is a 3-dimensional vector of end-effector linear velocities
    # dquaternion is a 4-dimensional vector of end-effector quaternion velocities
    def step(self, mode=1, djoint=[0]*7, dposition=[0]*3, dquaternion=[0]*4, grasp_open=False):

        # velocity control
        self._velocity_control(mode=mode, djoint=djoint, dposition=dposition, dquaternion=dquaternion, grasp_open=grasp_open)
        #self. _direct_set(mode=mode, djoint=djoint, dposition=dposition, dquaternion=dquaternion, grasp_open=grasp_open)

        # update robot state measurement
        self._read_state()
        self._read_jacobian()

    def reset(self):
        init_pos = copy.deepcopy(self.init_pos)
        random_number = np.random.uniform(low=-.1, high=.1, size=len(init_pos)) * 0.5
        if(self.status == "disabled"):
            resetpos = [1,3,5]
        else:
            resetpos = [0,1,2,3,4,5]
        for i in resetpos:
            init_pos[i] += random_number[i]
        self._reset_robot(init_pos)

    def seed(self, seed=None):
        self.np_random, seed = seeding.np_random(seed)
        return [seed]

    """internal functions"""

    def _read_state(self):
        joint_position = [0]*9
        joint_velocity = [0]*9
        joint_torque = [0]*9
        joint_states = p.getJointStates(self.panda, range(9))
        for idx in range(9):
            joint_position[idx] = joint_states[idx][0]
            joint_velocity[idx] = joint_states[idx][1]
            joint_torque[idx] = joint_states[idx][3]
        ee_states = p.getLinkState(self.panda, 11)
        ee_position = list(ee_states[4])
        ee_quaternion = list(ee_states[5])
        gripper_contact = p.getContactPoints(bodyA=self.panda, linkIndexA=10)
        self.state['joint_position'] = np.asarray(joint_position) #[JOINT_INDEX]
        self.state['joint_velocity'] = np.asarray(joint_velocity) #[JOINT_INDEX]
        self.state['joint_torque'] = np.asarray(joint_torque) #[JOINT_INDEX]
        self.state['ee_position'] = np.asarray(ee_position)
        self.state['ee_quaternion'] = np.asarray(ee_quaternion)
        self.state['ee_euler'] = np.asarray(p.getEulerFromQuaternion(ee_quaternion))
        self.state['gripper_contact'] = len(gripper_contact) > 0

    def _read_jacobian(self):
        linear_jacobian, angular_jacobian = p.calculateJacobian(self.panda, 11, [0, 0, 0], list(self.state['joint_position']), [0]*9, [0]*9)
        linear_jacobian = np.asarray(linear_jacobian)[:,:7]
        angular_jacobian = np.asarray(angular_jacobian)[:,:7]
        full_jacobian = np.zeros((6,7))
        full_jacobian[0:3,:] = linear_jacobian
        full_jacobian[3:6,:] = angular_jacobian
        self.jacobian['full_jacobian'] = full_jacobian
        self.jacobian['linear_jacobian'] = linear_jacobian
        self.jacobian['angular_jacobian'] = angular_jacobian

    def _reset_robot(self, joint_position):
        self.state = {}
        self.jacobian = {}
        self.desired = {}
        for idx in range(len(joint_position)):
            p.resetJointState(self.panda, idx, joint_position[idx])
        self._read_state()
        self._read_jacobian()
        self.desired['joint_position'] = self.state['joint_position']
        self.desired['ee_position'] = self.state['ee_position']
        self.desired['ee_quaternion'] = self.state['ee_quaternion']

    def _inverse_kinematics(self, ee_position, ee_quaternion):
        return p.calculateInverseKinematics(self.panda, 11, list(ee_position), list(ee_quaternion))

    def _velocity_control(self, mode, djoint, dposition, dquaternion, grasp_open):
        if mode:
            self.desired['ee_position'] += np.asarray(dposition) / 240.0
            self.desired['ee_quaternion'] += np.asarray(dquaternion) / 240.0
            q_dot = self._inverse_kinematics(self.desired['ee_position'], self.desired['ee_quaternion']) - self.state['joint_position']
        else:
            self.desired['joint_position'] += np.asarray(djoint) / 240.0
            q_dot = self.desired['joint_position'] - self.state['joint_position']
        gripper_position = [0.0, 0.0]
        if grasp_open:
            gripper_position = [0.05, 0.05]
        p.setJointMotorControlArray(self.panda, JOINT_INDEX, p.VELOCITY_CONTROL, targetVelocities=list(q_dot))
        p.setJointMotorControlArray(self.panda, [9,10], p.POSITION_CONTROL, targetPositions=gripper_position)

    def _direct_set(self, mode, djoint, dposition, dquaternion, grasp_open):
        """ Direct set the joints."""
        if mode:
            self.desired['ee_position'] = self.state['ee_position'] + np.asarray(dposition)

            self.desired['ee_quaternion'] = self.state['ee_quaternion'] + np.asarray(dquaternion)

            joint_position = list(self._inverse_kinematics(self.desired['ee_position'], self.desired['ee_quaternion']))

            djoint = (joint_position - self.state['joint_position'])
            djoint = np.clip(djoint, -1e-3, 1e-3)
            joint_position = djoint+ self.state['joint_position']
        else:
            self.desired['joint_position'] = self.state['joint_position'] + np.asarray(djoint)
            joint_position = list(self.desired['joint_position'])
        gripper_position = [0.0, 0.0]
        if grasp_open:
            gripper_position = [0.05, 0.05]
        disabled_joint_idx = [0, 2, 4, 6]

        for idx in range(len(joint_position)):
            if idx  in disabled_joint_idx:
                continue
            p.resetJointState(self.panda, idx, joint_position[idx])

    def _set_start(self,position):
        """ Set start positions."""
        self.desired['ee_position'] = np.asarray(position)
        print("input position is", position)

        self.desired['ee_quaternion'] = self.state['ee_quaternion']

        joint_position = list(self._inverse_kinematics(self.desired['ee_position'], self.desired['ee_quaternion']))

        djoint = (np.array(joint_position) - self.state['joint_position'])
        joint_position = djoint + self.state['joint_position']

        for idx in range(len(joint_position)):
            p.resetJointState(self.panda, idx, joint_position[idx])
        self._read_state()
        self._read_jacobian()

class DisabledPanda(Panda):
    def __init__(self, basePosition=[0,0,0]):
        self.urdfRootPath = pybullet_data.getDataPath()
        current_path = os.path.dirname(os.path.abspath(__file__))
        self.panda = p.loadURDF(os.path.join(current_path, "disabled_panda/disabled_panda.urdf"),useFixedBase=True,basePosition=basePosition)
        self.init_pos = [0.0, -np.pi/6, 0.0, -2*np.pi/4-np.pi/2, 0.0, np.pi/2+np.pi/3, np.pi/4, 0.0, 0.0, 0.03, 0.03]
        self.status = "disabled"

class FeasibilityPanda(Panda):
    def __init__(self, basePosition=[0,0,0]):
        self.urdfRootPath = pybullet_data.getDataPath()
        current_path = os.path.dirname(os.path.abspath(__file__))
        self.panda = p.loadURDF(os.path.join(current_path, "disabled_panda/disabled_panda.urdf"),useFixedBase=True,basePosition=basePosition)
        self.init_pos = [0.0, -np.pi/6, 0.0, -2*np.pi/4-np.pi/2, 0.0, np.pi/2+np.pi/3, np.pi/4, 0.0, 0.0, 0.03, 0.03]
        self.status = "feasibility"

class RealPanda(Panda):
    def __init__(self, basePosition=[0,0,0]):
        self.urdfRootPath = pybullet_data.getDataPath()
        current_path = os.path.dirname(os.path.abspath(__file__))
        self.panda = p.loadURDF(os.path.join(current_path, "disabled_panda/disabled_panda.urdf"),useFixedBase=True,basePosition=basePosition)
        self.init_pos = [0.0, -np.pi/6, 0.0, -2*np.pi/4-np.pi/2, 0.0, np.pi/2+np.pi/3, 3*np.pi/4, 0.0, 0.0, 0.02, 0.02]
        self.status = "real"

ext: py

reason: import numpy


text:
import numpy as np
from scipy.spatial.distance import cdist
from scipy.cluster.vq import kmeans
from sklearn.cluster import KMeans
import pdb
def llc(X, C = None, k = None, beta = 1e-6, **kwargs):
    """
    Implements Approximate Locally Linear Coding
    Inputs:
        X: (N x d) numpy array
        C: (Default: None)
            integer: number of anchor points (kmeans used to obtain codebook)
            OR
            (c x d) array of anchor points (codebook)
        k: (Default: None) Number of nearest neighbors for sparsity. If k > c or k<1, then k is changed to c
        beta: regularization parameter (lambda in the paper)
    Outputs:
        (G,C,distortion)
            G: Gamma coefficients (N x c) numpy array
            C: Codebook (c x d)
            Y: The transformed points (N x d) Y = G*C
    """
    if type(C)==type(0):
        #C,_ = kmeans(X, C, **kwargs)
        clf=KMeans(n_clusters=C)
        clf.fit(X)
        C= clf.cluster_centers_
    #pdb.set_trace()
    X=np.array(X)
    assert X.shape[1]==C.shape[1]
    N,d = X.shape
    c,_ = C.shape
    if k is None or k < 1 or k>c:
        print "Warning: k set to ",c
        k = c
    D = cdist(X, C, 'euclidean')
    I = np.zeros((N, k),dtype=int)
    for i in range(N):
        d = D[i,:]
        idx = np.argsort(d)
        I[i, :] = idx[:k]

    II = np.eye(k)
    G = np.zeros((N, c)) #Gammas
    ones = np.ones((k,1))
    for i in range(N):
       idx = I[i,:]
       z = C[idx,:] - np.tile(X[i,:], (k, 1))  # shift ith pt to origin
       Z = np.dot(z,z.T)                       # local covariance
       Z = Z + II*beta*np.trace(Z);            # regularlization (K>D)
       w = np.linalg.solve(Z,ones)     #np.dot(np.linalg.inv(Z), ones)
       w = w/np.sum(w)                         # enforce sum(w)=1

       w=np.ravel(w)
       G[i,idx] = w.T
    Y = np.dot(G,C)
    return G, C, Y

if __name__=='__main__':
    # Let's generate some random data
    X = np.vstack((\
        np.dot(np.random.randn(300,2), [[1, -1],[-1, 0.25]])+[-2,2],\
        np.dot(np.random.randn(300,2), [[1, 1],[1, 0.25]])+[2,2],\
        np.dot(np.random.randn(300,2), [[0.25, 0],[0, 1]])+[0,-2]
        ))

    G, C, Y = llc(X, C = 8, k = 1, beta = 100)
    D = Y-X
    distortion = np.sqrt(np.sum((D)**2,axis = 1))
    dm = np.mean(distortion)
    ds = np.std(distortion)
    nd = 8+6*((distortion-dm) / ds) # size of point shows relative distortion
    print 'Mean (std) Distortion: %0.4g (%0.4g)' %(dm,ds)
    # Plotting
    import matplotlib.pyplot as plt
#    for i in range(X.shape[0]):
#        plt.annotate(
#        '', xy=X[i,:2], xycoords='data',
#        xytext=Y[i,:2], textcoords='data',
#        arrowprops={'arrowstyle': '<-'})
    plt.scatter(X[:,0],X[:,1],color = 'r', marker = '.')
#    plt.scatter(C[:,0],C[:,1],color = 'b',marker = 's')

    plt.grid()
    plt.title('Mean (std) Distortion: %0.4g (%0.4g)' %(dm,ds))
    plt.legend(['data points','anchor points'])
    plt.show()

reason: import numpy,from scipy


text:
import csv
import matplotlib.image as mpimg
import numpy as np
import cv2
# Open the csv file
lines = []
with open('../data/behavior_cloning/total_data/driving_log1.csv') as csvfile:
    reader = csv.reader(csvfile)
    for line in reader:
        lines.append(line)

# split data into training set and validation set
from sklearn.model_selection import train_test_split
train_data, valid_data = train_test_split(lines, test_size=0.2)
#print(len(train_data))#  = 23400
#print(len(valid_data))#  = 5850

from sklearn.utils import shuffle
def generator(lines, batch_size=32):
    ''' Define a python generator'''
    num_line = len(lines)
    while 1:
        shuffle(lines)
        for offset in range(0, num_line, batch_size):
            batch_data = lines[offset:offset+batch_size]
            images = []
            angles = []
            # Extract steering angle data from csv file and read the images from camera
            for line in batch_data:
                for camera in range(3):
                    source_path = line[camera]
                    file_name = source_path.split('/')[-1]
                    current_path = '../data/behavior_cloning/total_data/IMG/' + file_name
                    image = mpimg.imread(current_path)
                    images.append(image)
                    if camera == 0:
                        angle = float(line[3])
                    if camera == 1:
                        angle = float(line[3]) + 0.12
                    else:
                        angle = float(line[3]) - 0.12
                    angles.append(angle)
                    image_flipped = np.fliplr(image)
                    images.append(image_flipped)
                    angle_flipped = -angle
                    angles.append(angle_flipped)
            X_train = np.array(images)
            Y_train = np.array(angles)
            yield shuffle(X_train, Y_train)

batch_size = 32
train_generator = generator(train_data, batch_size=batch_size)
valid_generator = generator(valid_data, batch_size=batch_size)

from keras.models import Sequential
from keras.layers import Flatten, Dense, Lambda, Conv2D, Cropping2D, Dropout
import math

model = Sequential()
model.add(Lambda(lambda x: x/255.0 -0.5, input_shape=(160,320,3)))   # Normalize inputs
model.add(Cropping2D(cropping=((35,25), (0,0))))
model.add(Conv2D(24, (5,5), strides=(2,2), activation='relu'))
model.add(Conv2D(36, (5,5), strides=(2,2), activation='relu'))
model.add(Conv2D(48, (5,5), strides=(2,2), activation='relu'))
model.add(Conv2D(64, (3,3), strides=(1,1), activation='relu'))
model.add(Conv2D(80, (3,3), strides=(1,1), activation='relu'))
model.add(Flatten())
model.add(Dense(100))
model.add(Dropout(0.8))   # dropout regularization
model.add(Dense(50))
model.add(Dense(1))

model.compile(loss='mse', optimizer='Adam')
model.summary()
model.fit_generator(train_generator, steps_per_epoch=math.ceil(len(train_data) / batch_size),
                   epochs=5, verbose=1, validation_data=valid_generator,
                   validation_steps = math.ceil(len(valid_data) / batch_size))
# model.fit(X_train, Y_train, epochs=3, validation_split=0.2, shuffle=True)

model.save('new_model.h5')


text:
import scipy
import pandas as pd
import contractions, inflect
import datefinder
from scipy.io import arff
from skmultilearn.problem_transform import BinaryRelevance
from skmultilearn.problem_transform import ClassifierChain
from sklearn.naive_bayes import GaussianNB
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.datasets import make_multilabel_classification
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk import word_tokenize
from nltk.stem import WordNetLemmatizer

class Lemmatizer:
    def __init__(self):
        self.wnl = WordNetLemmatizer()
    def __call__(self, words):
        return [self.wnl.lemmatize(t) for t in words if len(t) > 1]

def preprocess(text):
    # NOTE: datefinder returns somewhat wrong indexes where the dates are located; it makes it unusable for our purpose
    # We want to replace datetime objects with a placeholder, but can't do that if the indexes are wrong
    matches = datefinder.find_dates(text, False, True)
    for match in matches:
        print(match)
        print(text)
        print(text[match[1][0]:match[1][1]])

    text = contractions.fix(text)
    words = word_tokenize(text)
    words = replace_numbers_with_placeholder(words)
    return words

def replace_numbers_with_string(words):
    #Replace all integer occurrences in list of tokenized words with its string representation
    p = inflect.engine()
    new_words = []
    for word in words:
        if word.isdigit():
            new_word = p.number_to_words(word)
            new_words.append(new_word)
        else:
            new_words.append(word)
    return new_words

def replace_numbers_with_placeholder(words):
    #Replace all integer occurrences in list of tokenized words with the placeholder <number>
    new_words = []
    for word in words:
        if word.isdigit():
            new_words.append('<number>')
        else:
            new_words.append(word)
    return new_words

df = pd.read_csv("Data/HCM.csv")
#df.head()

X = df["Utterance"]
y = df.drop("Utterance", axis=1)
#vect = CountVectorizer()
vect = TfidfVectorizer(preprocessor=preprocess, tokenizer=Lemmatizer())

# learn the vocabulary and transform it to a document-term-matrix
X_dtm = vect.fit_transform(X)

vect.get_feature_names()
# show all the features after they have been vectorized
pd.DataFrame(X_dtm.toarray(), columns=vect.get_feature_names())

# show all the labels
print(list(y))

#classifier = BinaryRelevance(MultinomialNB())
classifier = ClassifierChain(MultinomialNB())
# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)

# train
classifier.fit(X_dtm, y)

userInput = input("Text to classify: ")
simple_test = [userInput]
simple_test_dtm = vect.transform(simple_test)

# predict
predictions = classifier.predict_proba(simple_test_dtm)
print(predictions)

#accuracy_score(y_test, predictions)

max_stars_repo_name: mohyneenm/MultiLabelClassification

max_issues_repo_name: mohyneenm/MultiLabelClassification

max_forks_repo_name: mohyneenm/MultiLabelClassification

reason: import scipy,from scipy


text:
import unittest

import numpy as np
import gym
from gym_mazeexplorer.envs.mazeexplorer_env import MazeExplorerEnv
from time import sleep

class TestMazeExplorerEnv(unittest.TestCase):

    def test_env(self):

        env = MazeExplorerEnv()
        env.reset()
        for _ in range(50):

            #env.render()
            action = env.action_space.sample()
            observation, reward, done, info = env.step(action)

            self.assertTrue(type(observation) == np.ndarray)
            self.assertTrue(type(reward) == int or type(reward) == float)
            self.assertTrue(type(done) == bool)
            self.assertTrue(type(info) == dict)

            sleep(0.05)

        return True

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: JupiLogy/gym-mazeexplorer

max_issues_repo_name: JupiLogy/gym-mazeexplorer

max_forks_repo_name: JupiLogy/gym-mazeexplorer


text:
#!/usr/bin/python

# MIT License
#
# Copyright (c) 2018-2019 Stefan Chmiela
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import print_function

import argparse
import os
import sys

import numpy as np

from sgdml.utils import io, ui

parser = argparse.ArgumentParser(
    description='Converts a native dataset file to extended XYZ format.'
)
parser.add_argument(
    'dataset',
    metavar='<dataset>',
    type=lambda x: io.is_file_type(x, 'dataset'),
    help='path to dataset file',
)
parser.add_argument(
    '-o',
    '--overwrite',
    dest='overwrite',
    action='store_true',
    help='overwrite existing xyz dataset file',
)

args = parser.parse_args()
dataset_path, dataset = args.dataset

name = os.path.splitext(os.path.basename(dataset_path))[0]
dataset_file_name = name + '.xyz'

xyz_exists = os.path.isfile(dataset_file_name)
if xyz_exists and args.overwrite:
    print(ui.color_str('[INFO]', bold=True) + ' Overwriting existing xyz dataset file.')
if not xyz_exists or args.overwrite:
    print(ui.color_str('[INFO]', bold=True) + ' Writing dataset to \'{}\'...'.format(dataset_file_name))
else:
    sys.exit(
        ui.color_str('[FAIL]', fore_color=ui.RED, bold=True) + ' Dataset \'{}\' already exists.'.format(dataset_file_name)
    )

R = dataset['R']
z = dataset['z']
F = dataset['F']

lattice = dataset['lattice'] if 'lattice' in dataset else None

try:
    with open(dataset_file_name, 'w') as file:

        n = R.shape[0]
        for i, r in enumerate(R):

            e = np.squeeze(dataset['E'][i]) if 'E' in dataset else None
            f = dataset['F'][i,:,:]
            ext_xyz_str = io.generate_xyz_str(r, z, e=e, f=f, lattice=lattice) + '\n'

            file.write(ext_xyz_str)

            progr = float(i) / (n - 1)
            ui.progr_bar(i, n - 1, disp_str='Exporting %d data points...' % n)

except IOError:
    sys.exit("ERROR: Writing xyz file failed.")

print()

max_stars_repo_name: keithgroup/sGDML

max_issues_repo_name: keithgroup/sGDML

max_forks_repo_name: keithgroup/sGDML


text:
import numpy as np
import keras

import os
import shutil
import sys

#from docopt import docopt
#import envoy
from PIL import Image
#from PIL import ImageDraw

from matplotlib import pyplot as plt

from model import build_model, FRAME_W, FRAME_H
from keras.preprocessing.image import img_to_array
from vis.utils import utils
from vis.visualization import visualize_saliency, overlay, visualize_cam

# Create a temp path for images with telemetry.
tmp_dir = '/users/anoopa/mydonkey/tmp'

# Build the outdir path.
in_path = '/users/anoopa/mydonkey/sessions/Self-7-22-1'

model = keras.models.load_model("model.hdf5")

file_count = len([f for f in os.listdir(in_path) if 'lidar' not in f])

i = 0
for filename in os.listdir(in_path):
  if 'frame' not in filename:
    continue
  image = utils.load_img(os.path.join(in_path, filename))
  # Convert to BGR, create input with batch_size: 1.
  #bgr_img = utils.bgr2rgb(image)
  #img_input = np.expand_dims(img_to_array(bgr_img), axis=0)

  heatmap = visualize_cam(model, layer_idx=-1, filter_indices=0,
                          seed_input=image, grad_modifier=None)

  image = overlay(image, heatmap, alpha=0.7)
  im = Image.fromarray(image)
  im.save(os.path.join(tmp_dir, filename))
  i += 1
  if i % 100 == 0 or i == file_count:
    sys.stdout.write('\rwriting attention map.. %0.1f%%' % (100. * i / file_count))
    sys.stdout.flush()

max_stars_repo_name: purplemuskox/sparky

max_issues_repo_name: purplemuskox/sparky

max_forks_repo_name: purplemuskox/sparky


text:
"""
====================
Plot Activation Foci
====================

Plot spheroids at positions on the surface manifold
according to coordinates or vertex ids.

"""
import os
import os.path as op
from numpy import arange, linspace
from numpy.random import permutation
import nibabel as nib
from surfer import Brain
from mayavi import mlab

print(__doc__)

subject_id = "fsaverage"
subjects_dir = os.environ["SUBJECTS_DIR"]

"""
Bring up the visualization.
"""
brain = Brain(subject_id, "lh", "inflated")

"""
First we'll get a set of stereotaxic foci in the MNI
coordinate system. These might be peak activations from
a volume based analysis.
"""
coords = [[-36, 18, -3],
          [-43, 25, 24],
          [-48, 26, -2]]

"""
Now we plot the foci on the inflated surface. We will map
the foci onto the surface by finding the vertex on the "white"
mesh that is closest to the coordinate of each point we want
to display.

While this is not a perfect transformation, it can give you
some idea of where peaks from a volume-based analysis would
be located on the surface.

You can use any valid matplotlib color for the foci; the
default is white.
"""
brain.add_foci(coords, map_surface="white", color="gold")

"""
You can also plot foci with a set of surface vertex ids.
For instance, you might want to plot the peak activation
within an ROI for each of your indivdiual subjects over
the group activation map.

Here, we will just demonstrate with a set of randomly
choosen vertices from within the superior temporal sulcus.

First, we load in the Destrieux parcellation annotation file
and find 10 random vertices within the STS.
"""
annot_path = op.join(subjects_dir, subject_id, "label/lh.aparc.a2009s.annot")
ids, ctab, names = nib.freesurfer.read_annot(annot_path)
verts = arange(0, len(ids))
coords = permutation(verts[ids == 74])[:10]

"""
You can also control the size of the focus glpyhs.
We'll make these a little bit smaller than our
other foci.
"""
scale_factor = 0.7

"""
Finally, plot the foci using the coords_as_verts option to
center each sphereoid at its vertex id.
"""
brain.add_foci(coords, coords_as_verts=True,
               scale_factor=scale_factor, color="#A52A2A")

"""
Now we demonstrate plotting some data with a set of randomly
choosen vertices from within the middle temporal sulcus.
"""
verts = arange(0, len(ids))
coords = permutation(verts[ids == 26])[:10]

"""
Generate a dataset with some values in the [-1, 1] range
"""
dataset_name ='some_data'
data = linspace(start=-1, stop=1, num=10)

"""
Now we plot the foci colorcoded acording to data values, whereas foci sizes will be
coded using the absolute values of data.
Using the 'name' argument we can reference newly added foci to create a colorbar using mayavi.
"""
brain.add_foci(coords, coords_as_verts=True, name=dataset_name,
               scale_factor=1, data=data, colormap='cool')

"""
Sometimes a qualitative representation is not enough, and we need to plot the colorbar as well.
This can be done using the 'name' argument in add_foci, by referencing the newly added foci to
create a colorbar with mayavi.
"""
mlab.colorbar(brain.foci[dataset_name], nb_labels=5, label_fmt='  %.2f  ')

max_stars_repo_name: cristidonos/PySurfer

max_issues_repo_name: cristidonos/PySurfer

max_forks_repo_name: cristidonos/PySurfer

reason: from numpy


text:
# MIT License
#
# Copyright (C) IBM Corporation 2021
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
# Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
Random Forest Classifier with Differential Privacy
"""
from collections import defaultdict, namedtuple
import numbers
import warnings
from joblib import Parallel, delayed
import numpy as np

from sklearn.utils import check_array
from sklearn.utils.fixes import _joblib_parallel_args
from sklearn.ensemble._forest import ForestClassifier
from sklearn.tree import DecisionTreeClassifier as BaseDecisionTreeClassifier

from diffprivlib.accountant import BudgetAccountant
from diffprivlib.utils import PrivacyLeakWarning
from diffprivlib.mechanisms import PermuteAndFlip
from diffprivlib.validation import DiffprivlibMixin

Dataset = namedtuple('Dataset', ['X', 'y'])

class RandomForestClassifier(ForestClassifier, DiffprivlibMixin):
    r"""Random Forest Classifier with differential privacy.

    This class implements Differentially Private Random Decision Forests using Smooth Sensitivity [1].
    :math:`\epsilon`-Differential privacy is achieved by constructing decision trees via random splitting criterion and
    applying Exponential Mechanism to produce a noisy label.

    Parameters
    ----------
    n_estimators: int, default: 10
        The number of trees in the forest.

    epsilon: float, default: 1.0
        Privacy parameter :math:`\epsilon`.

    cat_feature_threshold: int, default: 10
        Threshold value used to determine categorical features. For example, value of ``10`` means
        any feature that has less than or equal to 10 unique values will be treated as a categorical feature.

    n_jobs : int, default: 1
        Number of CPU cores used when parallelising over classes. ``-1`` means
        using all processors.

    verbose : int, default: 0
        Set to any positive number for verbosity.

    accountant : BudgetAccountant, optional
        Accountant to keep track of privacy budget.

    max_depth: int, default: 15
        The maximum depth of the tree. Final depth of the tree will be calculated based on the number of continuous
        and categorical features, but it wont be more than this number.
        Note: The depth translates to an exponential increase in memory usage.

    random_state: float, optional
        Sets the numpy random seed.

    feature_domains: dict, optional
        A dictionary of domain values for all features where keys are the feature indexes in the training data and
        the values are an array of domain values for categorical features and an array of min and max values for
        continuous features. For example, if the training data is [[2, 'dog'], [5, 'cat'], [7, 'dog']], then
        the feature_domains would be {'0': [2, 7], '1': ['dog', 'cat']}. If not provided, feature domains will
        be constructed from the data, but this will result in :class:`.PrivacyLeakWarning`.

    Attributes
    ----------
    n_features_in_: int
        The number of features when fit is performed.

    n_classes_: int
        The number of classes.

    classes_: array of shape (n_classes, )
        The classes labels.

    cat_features_: array of categorical feature indexes
        Categorical feature indexes.

    max_depth_: int
        Final max depth used for constructing decision trees.

    estimators_: list of DecisionTreeClassifier
        The collection of fitted sub-estimators.

    feature_domains_: dictionary of domain values mapped to feature
        indexes in the training data

    Examples
    --------
    >>> from sklearn.datasets import make_classification
    >>> from diffprivlib.models import RandomForestClassifier
    >>> X, y = make_classification(n_samples=1000, n_features=4,
    ...                            n_informative=2, n_redundant=0,
    ...                            random_state=0, shuffle=False)
    >>> clf = RandomForestClassifier(n_estimators=100, random_state=0)
    >>> clf.fit(X, y)
    >>> print(clf.predict([[0, 0, 0, 0]]))
    [1]

    References
    ----------
    [1] Sam Fletcher, Md Zahidul Islam. "Differentially Private Random Decision Forests using Smooth Sensitivity"
    https://arxiv.org/abs/1606.03572

    """

    def __init__(self, n_estimators=10, *, epsilon=1.0, cat_feature_threshold=10, n_jobs=1, verbose=0, accountant=None,
                 max_depth=15, random_state=None, feature_domains=None, **unused_args):
        super().__init__(
            base_estimator=DecisionTreeClassifier(),
            n_estimators=n_estimators,
            estimator_params=("cat_feature_threshold", "max_depth", "epsilon", "random_state"),
            n_jobs=n_jobs,
            random_state=random_state,
            verbose=verbose)
        self.epsilon = epsilon
        self.cat_feature_threshold = cat_feature_threshold
        self.max_depth = max_depth
        self.accountant = BudgetAccountant.load_default(accountant)
        self.feature_domains = feature_domains

        if random_state is not None:
            np.random.seed(random_state)

        self._warn_unused_args(unused_args)

    def fit(self, X, y, sample_weight=None):
        """Fit the model to the given training data.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training vector, where n_samples is the number of samples and n_features is the number of features.

        y : array-like, shape (n_samples,)
            Target vector relative to X.

        sample_weight : ignored
            Ignored by diffprivlib.  Present for consistency with sklearn API.

        Returns
        -------
        self: class

        """
        if sample_weight is not None:
            self._warn_unused_args("sample_weight")

        if not isinstance(self.n_estimators, numbers.Integral) or self.n_estimators < 0:
            raise ValueError(f'Number of estimators should be a positive integer; got {self.n_estimators}')

        if not isinstance(self.cat_feature_threshold, numbers.Integral) or self.cat_feature_threshold < 0:
            raise ValueError('Categorical feature threshold should be a positive integer;'
                             f'got {self.cat_feature_threshold}')

        self.accountant.check(self.epsilon, 0)

        X, y = self._validate_data(X, y, multi_output=False)

        self.n_outputs_ = 1
        self.n_features_in_ = X.shape[1]
        self.cat_features_ = get_cat_features(X, self.cat_feature_threshold)
        self.max_depth_ = calc_tree_depth(n_cont_features=self.n_features_in_-len(self.cat_features_),
                                          n_cat_features=len(self.cat_features_), max_depth=self.max_depth)
        self.classes_ = np.unique(y)
        self.n_classes_ = len(self.classes_)
        self.feature_domains_ = self.feature_domains

        if self.feature_domains_ is None:
            warnings.warn(
                "`feature_domains` parameter hasn't been specified, "
                "so falling back to determining domains from the data.\n"
                "This may result in additional privacy leakage. To ensure differential privacy with no "
                "additional privacy loss, specify `feature_domains` according to the documentation",
                PrivacyLeakWarning)
            self.feature_domains_ = get_feature_domains(X, self.cat_features_)

        if len(self.feature_domains_) != self.n_features_in_:
            raise ValueError("Missing domains for some features in `feature_domains`")

        if self.n_estimators > len(X):
            raise ValueError('Number of estimators is more than the available samples')

        subset_size = int(len(X) / self.n_estimators)
        datasets = []
        estimators = []

        for i in range(self.n_estimators):
            estimator = DecisionTreeClassifier(max_depth=self.max_depth_,
                                               epsilon=self.epsilon,
                                               feature_domains=self.feature_domains_,
                                               cat_features=self.cat_features_,
                                               classes=self.classes_)
            estimators.append(estimator)
            datasets.append(Dataset(X=X[i*subset_size:(i+1)*subset_size], y=y[i*subset_size:(i+1)*subset_size]))

        estimators = Parallel(n_jobs=self.n_jobs, verbose=self.verbose, **_joblib_parallel_args(prefer='processes'))(
            delayed(lambda estimator, X, y: estimator.fit(X, y))(estimator, dataset.X, dataset.y)
            for estimator, dataset in zip(estimators, datasets)
        )

        self.estimators_ = estimators
        self.accountant.spend(self.epsilon, 0)
        self.fitted_ = True

        return self

class DecisionTreeClassifier(BaseDecisionTreeClassifier, DiffprivlibMixin):
    r"""Decision Tree Classifier with differential privacy.

    This class implements the base differentially private decision tree classifier
    for the Random Forest classifier algorithm. Not meant to be used separately.

    Parameters
    ----------
    epsilon: float, default: 1.0
        Privacy parameter :math:`\epsilon`.

    cat_feature_threshold: int, default: 10
        Threshold value used to determine categorical features. For example, value of ``10`` means
        any feature that has less than or equal to 10 unique values will be treated as a categorical feature.

    max_depth: int, default: 15
        The maximum depth of the tree.

    random_state: float, optional
        Sets the numpy random seed.

    cat_features: array, optional
        Array of categorical feature indexes. If not provided, will be determined from the data based on the
        cat_feature_threshold.

    classes: array of shape (n_classes_, ), optional
        Array of class labels. If not provided, will be determined from the data.

    feature_domains: dict, optional
        A dictionary of domain values for all features where keys are the feature indexes in the training data and
        the values are an array of domain values for categorical features and an array of min and max values for
        continuous features. For example, if the training data is [[2, 'dog'], [5, 'cat'], [7, 'dog']], then
        the feature_domains would be {'0': [2, 7], '1': ['dog', 'cat']}. If not provided, feature domains will
        be constructed from the data, but this will result in :class:`.PrivacyLeakWarning`.

    Attributes
    ----------
    n_features_in_: int
        The number of features when fit is performed.

    n_classes_: int
        The number of classes.

    classes_: array of shape (n_classes, )
        The class labels.

    cat_features_: array of categorical feature indexes
        Categorical feature indexes.

    feature_domains_: dictionary of domain values mapped to feature
        indexes in the training data

    """
    def __init__(self, cat_feature_threshold=10, max_depth=15, epsilon=1, random_state=None, feature_domains=None,
                 cat_features=None, classes=None):
        # TODO: Remove try...except when sklearn v1.0 is min-requirement
        try:
            super().__init__(
                criterion=None,
                splitter=None,
                max_depth=max_depth,
                min_samples_split=None,
                min_samples_leaf=None,
                min_weight_fraction_leaf=None,
                max_features=None,
                random_state=random_state,
                max_leaf_nodes=None,
                min_impurity_decrease=None,
                min_impurity_split=None
            )
        except TypeError:
            super().__init__(
                criterion=None,
                splitter=None,
                max_depth=max_depth,
                min_samples_split=None,
                min_samples_leaf=None,
                min_weight_fraction_leaf=None,
                max_features=None,
                random_state=random_state,
                max_leaf_nodes=None,
                min_impurity_decrease=None
            )
        self.feature_domains = feature_domains
        self.cat_feature_threshold = cat_feature_threshold
        self.epsilon = epsilon
        self.cat_features = cat_features
        self.classes = classes

        if random_state is not None:
            np.random.seed(random_state)

    def _build(self, features, feature_domains, current_depth=1):
        if not features or current_depth >= self.max_depth+1:
            return DecisionNode(level=current_depth, classes=self.classes_)

        split_feature = np.random.choice(features)
        node = DecisionNode(level=current_depth, classes=self.classes_, split_feature=split_feature)

        if split_feature in self.cat_features_:
            node.set_split_type(DecisionNode.CAT_SPLIT)
            for value in feature_domains[str(split_feature)]:
                child_node = self._build([f for f in features if f != split_feature], feature_domains, current_depth+1)
                node.add_cat_child(value, child_node)
        else:
            node.set_split_type(DecisionNode.CONT_SPLIT)
            split_value = np.random.uniform(feature_domains[str(split_feature)][0],
                                            feature_domains[str(split_feature)][1])
            node.set_split_value(split_value)
            left_domain = {k: v if k != str(split_feature) else [v[0], split_value]
                           for k, v in feature_domains.items()}
            right_domain = {k: v if k != str(split_feature) else [split_value, v[1]]
                            for k, v in feature_domains.items()}
            left_child = self._build(features, left_domain, current_depth+1)
            right_child = self._build(features, right_domain, current_depth+1)
            node.set_left_child(left_child)
            node.set_right_child(right_child)

        return node

    def fit(self, X, y, sample_weight=None, check_input=True, X_idx_sorted="deprecated"):
        if sample_weight is not None:
            self._warn_unused_args("sample_weight")

        if not isinstance(self.cat_feature_threshold, numbers.Integral) or self.cat_feature_threshold < 0:
            raise ValueError('Categorical feature threshold should be a positive integer;'
                             f'got {self.cat_feature_threshold}')

        if check_input:
            X, y = self._validate_data(X, y, multi_output=False)
        self.n_outputs_ = 1

        self.feature_domains_ = self.feature_domains
        self.cat_features_ = self.cat_features

        if self.cat_features_ is None:
            self.cat_features_ = get_cat_features(X, self.cat_feature_threshold)

        if self.feature_domains_ is None:
            warnings.warn(
                "feature_domains parameter hasn't been specified, "
                "so falling back to determining domains from the data.\n"
                "This may result in additional privacy leakage. To ensure differential privacy with no "
                "additional privacy loss, specify `feature_domains` according to the documentation",
                PrivacyLeakWarning)
            self.feature_domains_ = get_feature_domains(X, self.cat_features_)

        self.classes_ = self.classes

        if self.classes_ is None:
            self.classes_ = np.unique(y)

        self.n_classes_ = len(self.classes_)

        self.n_features_in_ = X.shape[1]
        features = list(range(self.n_features_in_))

        self.tree_ = self._build(features, self.feature_domains_)

        for i, _ in enumerate(X):
            node = self.tree_.classify(X[i])
            node.update_class_count(y[i].item())

        self.tree_.set_noisy_label(self.epsilon, self.classes_)

        return self

    @property
    def n_features_(self):
        return self.n_features_in_

    def _more_tags(self):
        return {}

class DecisionNode:
    """Base Decision Node
    """
    CONT_SPLIT = 0
    CAT_SPLIT = 1

    def __init__(self, level, classes, split_feature=None, split_value=None, split_type=None):
        """
        Initialize DecisionNode

        Parameters
        ----------
        level: int
            Node level in the tree

        classes: list
            List of class labels

        split_feature: int
            Split feature index

        split_value: Any
            Feature value to split at

        split_type: int
            Type of split

        """
        self._level = level
        self._classes = classes
        self._split_type = split_type
        self._split_feature = split_feature
        self._split_value = split_value
        self._left_child = None
        self._right_child = None
        self._cat_children = {}
        self._class_counts = defaultdict(int)
        self._noisy_label = None

    @property
    def noisy_label(self):
        """Get noisy label"""
        return self._noisy_label

    def set_split_value(self, split_value):
        """Set split value"""
        self._split_value = split_value

    def set_split_type(self, split_type):
        """Set split type"""
        self._split_type = split_type

    def set_left_child(self, node):
        """Set left child of the node"""
        self._left_child = node

    def set_right_child(self, node):
        """Set right child of the node"""
        self._right_child = node

    def add_cat_child(self, cat_value, node):
        """Add a categorical child node"""
        self._cat_children[str(cat_value)] = node

    def is_leaf(self):
        """Check whether the node is leaf node"""
        return not self._left_child and not self._right_child and not self._cat_children

    def update_class_count(self, class_value):
        """Update the class count for the given class"""
        self._class_counts[class_value] += 1

    def classify(self, x):
        """Classify the given data"""
        if self.is_leaf():
            return self

        child = None

        if self._split_type == self.CAT_SPLIT:
            x_val = str(x[self._split_feature])
            child = self._cat_children.get(x_val)
        else:
            x_val = x[self._split_feature]
            if x_val < self._split_value:
                child = self._left_child
            else:
                child = self._right_child

        if child is None:
            return self

        return child.classify(x)

    def set_noisy_label(self, epsilon, class_values):
        """Set the noisy label for this node"""
        if self.is_leaf():
            if not self._noisy_label:
                for val in class_values:
                    if val not in self._class_counts:
                        self._class_counts[val] = 0

                utility = list(self._class_counts.values())
                candidates = list(self._class_counts.keys())
                mech = PermuteAndFlip(epsilon=epsilon, sensitivity=1, monotonic=True, utility=utility,
                                      candidates=candidates)
                self._noisy_label = mech.randomise()
        else:
            if self._left_child:
                self._left_child.set_noisy_label(epsilon, class_values)
            if self._right_child:
                self._right_child.set_noisy_label(epsilon, class_values)
            for child_node in self._cat_children.values():
                child_node.set_noisy_label(epsilon, class_values)

    def predict(self, X):
        """Predict using this node"""
        y = []
        X = np.array(X)
        check_array(X)

        for x in X:
            node = self.classify(x)
            proba = np.zeros(len(self._classes))
            proba[np.where(self._classes == node.noisy_label)[0].item()] = 1
            y.append(proba)

        return np.array(y)

def get_feature_domains(X, cat_features):
    """Calculate feature domains from the data.

    Parameters:
        X : array-like, shape (n_samples, n_features)
            Training vector, where n_samples is the number of samples and n_features is the number of features.

        cat_features : array of integers
            List of categorical feature indexes

    Returns:
        [dict]: Dictionary with keys as feature indexes and values as feature domains.
    """
    feature_domains = {}
    X_t = np.transpose(X)
    cont_features = list(set(range(X.shape[1])) - set(cat_features))

    for i in cat_features:
        feature_domains[str(i)] = [str(x) for x in set(X_t[i])]

    for i in cont_features:
        vals = [float(x) for x in X_t[i]]
        feature_domains[str(i)] = [min(vals), max(vals)]

    return feature_domains

def get_cat_features(X, feature_threshold=2):
    """Determine categorical features

    Parameters:
        X : array-like, shape (n_samples, n_features)
            Training vector, where n_samples is the number of samples and n_features is the number of features.

        feature_threshold: int, defaults to 2.
            Threshold value used to determine categorical features. For example, value of ``10`` means
            any feature that has less than or equal to 10 unique values will be treated as a categorical feature.

    Returns:
        [list]: List of categorical feature indexes
    """
    n_features = X.shape[1]
    cat_features = []

    for i in range(n_features):
        values = set(X[:, i])
        if len(values) <= feature_threshold:
            cat_features.append(i)

    return cat_features

def calc_tree_depth(n_cont_features, n_cat_features, max_depth=15):
    """Calculate tree depth

    Args:
        n_cont_features (int): Number of continuous features
        n_cat_features ([type]): Number of categorical features
        max_depth (int, optional): Max depth tree. Defaults to 15.

    Returns:
        [int]: Final depth tree
    """
    if n_cont_features < 1:
        return min(max_depth, np.floor(n_cat_features / 2.))
    # Designed using balls-in-bins probability. See the paper for details.
    m = float(n_cont_features)
    depth = 0
    expected_empty = m   # the number of unique attributes not selected so far
    while expected_empty > m / 2.:   # repeat until we have less than half the attributes being empty
        expected_empty = m * ((m - 1.) / m) ** depth
        depth += 1
    # the above was only for half the numerical attributes. now add half the categorical attributes
    final_depth = np.floor(depth + (n_cat_features / 2.))
    return min(max_depth, final_depth)


text:
import os, sys
import torch
import numpy as np
import pickle as p

from struct import unpack

data_path = os.path.expanduser(os.path.join('~', 'code', 'spiketorch', 'data'))

if not os.path.isdir(data_path):
	os.makedirs(data_path)

def get_MNIST(train=True, data_path=data_path):
	'''
	Read input-vector (image) and target class (label, 0-9) and return it as
	a list of tuples.
	'''
	fname = 'train' if train else 'test'

	if os.path.isfile(os.path.join(data_path, '%s.p' % fname)):
		# Get pickled data from disk.
		data = p.load(open(os.path.join(data_path, '%s.p' % fname), 'rb'))
	else:
		# Open the images with gzip in read binary mode.
		if train:
			images = open(os.path.join(data_path, 'train-images-idx3-ubyte'), 'rb')
			labels = open(os.path.join(data_path, 'train-labels-idx1-ubyte'), 'rb')
		else:
			images = open(os.path.join(data_path, 't10k-images-idx3-ubyte'), 'rb')
			labels = open(os.path.join(data_path, 't10k-labels-idx1-ubyte'), 'rb')

		# Get metadata for images.
		images.read(4)
		number_of_images = unpack('>I', images.read(4))[0]
		rows = unpack('>I', images.read(4))[0]
		cols = unpack('>I', images.read(4))[0]

		# Get metadata for labels.
		labels.read(4)
		N = unpack('>I', labels.read(4))[0]

		if number_of_images != N:
			raise Exception('number of labels did not match the number of images')

		# Get the data.
		print('...Loading MNIST data from disk.')
		print('\n')

		X = np.zeros((N, rows, cols), dtype=np.uint8)
		y = np.zeros((N, 1), dtype=np.uint8)

		for i in range(N):
			if i % 1000 == 0:
				print('Progress :', i, '/', N)
			X[i] = [[unpack('>B', images.read(1))[0] for unused_col in \
							range(cols)] for unused_row in range(rows) ]
			y[i] = unpack('>B', labels.read(1))[0]

		print('Progress :', N, '/', N, '\n')

		X = X.reshape([N, 784])
		data = {'X': X, 'y': y }

		p.dump(data, open(os.path.join(data_path, '%s.p' % fname), 'wb'))

	return data

def generate_spike_train(image, intensity, time):
	'''
	Generates Poisson spike trains based on image ink intensity.
	'''
	# Multiply image by desired intensity.
	image = image * intensity

	# Get number of input neurons.
	n_input = image.shape[0]

	# Image data preprocessing (divide by 4, invert (for spike rates),
	# multiply by 1000 (conversion from milliseconds to seconds).
	image = (1 / (image / 4)) * 1000
	image[np.isinf(image)] = 0

	# Make the spike data.
	spike_times = np.random.poisson(image, [time, n_input])
	spike_times = np.cumsum(spike_times, axis=0)
	spike_times[spike_times >= time] = 0

	# Create spikes matrix from spike times.
	spikes = np.zeros([time, n_input])
	for idx in range(time):
		spikes[spike_times[idx, :], np.arange(n_input)] = 1

	# Temporary fix: The above code forces a spike from
	# every input neuron on the first time step.
	spikes[0, :] = 0

	# Return the input spike occurrence matrix.
	return spikes

def generate_2d_spike_train(image, intensity, time):
	'''
	Generates Poisson spike trains based on image ink intensity.
	'''
	# Multiply image by desired intensity.
	image = image * intensity

	# Get number of input neurons.
	n_input = image.shape[0]
	n_input_sqrt = int(np.sqrt(n_input))

	# Image data preprocessing (divide by 4, invert (for spike rates),
	# multiply by 1000 (conversion from milliseconds to seconds).
	image = (1 / (image / 4)) * 1000
	image[np.isinf(image)] = 0

	# Make the spike data.
	spike_times = np.random.poisson(image, [time, n_input])
	spike_times = np.cumsum(spike_times, axis=0)
	spike_times[spike_times >= time] = 0

	# Create spikes matrix from spike times.
	spikes = np.zeros([time, n_input])
	for idx in range(time):
		spikes[spike_times[idx, :], np.arange(n_input)] = 1

	# Temporary fix: The above code forces a spike from
	# every input neuron on the first time step.
	spikes[0, :] = 0

	# Return the input spike occurrence matrix.
	return spikes.reshape([time, 1, n_input_sqrt, n_input_sqrt])


text:
# -*- coding: utf-8 -*-
"""
Created on Wed May 13 12:39:31 2017

@author: David Zhang
"""

from sklearn.cluster import KMeans
import scipy.sparse as sps
import scipy.spatial.distance as spd
from numpy import *
from numpy.random import *
import numpy as np
#import matplotlib.pyplot as plt
#import kmedoids
from scipy.sparse.linalg import eigsh
import scipy
import xml.etree.ElementTree
import os, time
import json
from math import*
from decimal import Decimal
from indexFlann import IndexFlann
import cv2
import io

#http://dataaspirant.com/2015/04/11/five-most-popular-similarity-measures-implementation-in-python/
class Similarity():
    def euclidean_distance(self, x,y):
       return sqrt(sum(pow(a-b,2) for a, b in zip(x, y)))
    def manhattan_distance(self,x,y):
       return sum(abs(a-b) for a,b in zip(x,y))
    def nth_root(self,value, n_root):
       root_value = 1/float(n_root)
       return round (Decimal(value) ** Decimal(root_value),3)

    def minkowski_distance(self,x,y,p_value):
       return self.nth_root(sum(pow(abs(a-b),p_value) for a,b in zip(x, y)),p_value)

    def square_rooted(self,x):
       return round(sqrt(sum([a*a for a in x])),3)

    def cosine_similarity(self,x,y):
       numerator = sum(a*b for a,b in zip(x,y))
       denominator = self.square_rooted(x)* self.square_rooted(y)
       return round(numerator/float(denominator),3)

    def jaccard_similarity(self,x,y):
       intersection_cardinality = len(set.intersection(*[set(x), set(y)]))
       union_cardinality = len(set.union(*[set(x), set(y)]))
       return intersection_cardinality/float(union_cardinality)

#####################
def rowNormalize(D):
   row_norm= np.linalg.norm(D,axis=1);
   for r in range(D.shape[0]):
       D[r,:] = D[r,:]/row_norm[r]

   return D
## knee computation as described in the paper: "Finding a “Kneedle” in a Haystack: Detecting Knee Points in System Behavior"
def getClustersBasedonKnee(lam):
   x = np.asarray([i for i in range(np.prod(lam.shape))], dtype=float)
   xmax = np.max(x)
   if xmax != 0.0:
        x = x/(np.max(x))
   y = lam
   ymin = np.min(y)
   yrange = np.max(y) - ymin
   if yrange != 0.0:
        y = (y-ymin)/yrange
   delta =y-x
   knee = np.argmax(delta)

   # At least one cluster
   knee = max(knee, 1)

   #plt.figure()
   #plt.plot(x,y,'r')
   #plt.show()
   #plt.plot(x,delta, 'b')
   #plt.show()
   #plt.figure()
   #plt.plot(lam, 'r')
   return knee

def getDegreeAndInvSqrt(M):
    D  = np.zeros(M.shape[0])
    DI = np.zeros(M.shape[0])
    DISqrt = np.zeros(M.shape[0])
    r =  np.zeros(M.shape[0], dtype=int)

    rowsum =  sps.csr_matrix.sum(M,1)
    for i in range(M.shape[0]):
        r[i] = i
        if rowsum[i] >0:
            D[i] = rowsum[i]
            DI[i] = 1/rowsum[i]
        DISqrt[i] = np.sqrt(DI[i])

    D  = sps.csr_matrix((D, (r,r)), shape=(M.shape))
    DI = sps.csr_matrix((DI, (r,r)), shape=(M.shape))
    DISqrt = sps.csr_matrix((DISqrt, (r,r)), shape=(M.shape))
    return D,DI,DISqrt

def computeSymmetricLaplacian(A):
   ## compute the degree matrix, its inverse and inverse square root
   D,DI, DI_sqrt = getDegreeAndInvSqrt(A)
   ## compute laplacian
   L = D-A
   ## compute symmetric normalized Laplacian
   Lsym = DI_sqrt*L*DI_sqrt
   return Lsym, D, DI_sqrt

def computeRandomWalkLaplacian(A):
    ## compute the degree matrix, its inverse and inverse square root
   D,DI, DI_sqrt = getDegreeAndInvSqrt(A)
   ## compute Random walk Laplacian
   L = D-A
   Lrw = DI*L
   return Lrw, D, DI_sqrt

def createSparseMatrix(simMeasure, makesym=True):
   r = []  # row indexes
   c = []  # column indexes
   v = []  # Similarity scores
   nodes_in_graph = len(simMeasure)
   for n in range(nodes_in_graph):
       node_index = n
       for key, value in simMeasure[n]:
           nbr_index = key
           similarity = value
           r.append(node_index)
           c.append(nbr_index)
           v.append(similarity)

   r = np.asarray(r, dtype=int)
   c = np.asarray(c, dtype=int)
   v = np.asarray(v, dtype= 'float')
   if len(r) > 0 and len(c) > 0 and len(v) > 0:
   # exponential transform
       vmax  = np.max(v)
       if vmax != 0.0:
           v = np.exp(v/vmax)
       S = sps.csr_matrix((v, (r,c)), shape=(np.max(r)+1,np.max(r)+1))
       if makesym:
           ## make the matrix symetric
           print 'making the matrix symmetric'
           A = S + np.transpose(S)
       else:
           A = S
   else:
       A = None
   print 'done creating sparse matrix'
   #plt.figure()
   #plt.imshow(S.todense())
   #plt.show()
   return A, r,c,v

def data_read(sample_dir, json_file):
    namelist = []
    featurelist = []

    print 'start reading data'
    start_time = time.time()
    reader = open(json_file, 'r')
    data = json.load(reader)
    imagelist = data["images"]
    listsize = len(imagelist)

    # decompose image and feature list
    for i in range(listsize):
        namelist.append(imagelist[i]["name"])
        featurelist.append(imagelist[i]["feature"])
    for i in range(listsize):
        namelist[i] = namelist[i];

    end_time = time.time()
    print 'done with data read in %f seconds'%(end_time-start_time)
    print 'number of images = ', listsize
    return namelist, featurelist

def data_segment(featurelist, page_size, top_k, diff_thresh, threshMethod, clusteringType):
    print 'data segment()'
    randomSeed = None
    similarity = Similarity();
    _labels = []
    # similarity by nearest neighbor
    nsamples = len(featurelist)
    k = page_size # 40
    l = top_k #5
    threshold = diff_thresh #0.5
    myindex = IndexFlann(k, l, threshold);
    ntype=threshMethod #'byValue' #'byNumber' #

    for i in range(nsamples): #range(listsize):
        myindex.addFeatures(featurelist[i]);

    print 'sparse matrix computing'
    start_time = time.time()
    #similarity_measure = myindex.nearestneighbor(ntype);
    similarity_measure = myindex.nearestneighbor_cv(ntype);
    S, r,c,v = createSparseMatrix(similarity_measure, False)

    Lsym,D, DI_sqrt = computeSymmetricLaplacian(S);
    end_time = time.time()
    print 'done sparse matrix computing in %f seconds'%(end_time-start_time)

    ## pick the first 300 eigenvalues if it is less than the total number of items in the dataset
    #M = np.minimum(400, np.max(r))
    M = np.minimum(300, np.max(r))
    print 'computing eigenpairs'
    start_time = time.time()
    ## call ARPACK routine to compute top k eigenpairs of sparse symmetric matrix
    lam, v = eigsh(Lsym, M, which='SM' )
    end_time = time.time()
    print 'done computing eigenpairs in %f seconds'%(end_time-start_time)
    #plt.figure()
    #plt.plot(lam)
    #plt.show()
    ## find the "knee" in the the eigenvalue curve to decide on number of clusters
    NC = getClustersBasedonKnee(lam)
    ## create the data matrix fo K-means
    print 'numcluster = %d'%NC
    Data = v[:,0:NC]   # number of points x number of dimensions
    #row normalize D
    Data = rowNormalize(Data)
    clusters = {}

    if clusteringType == 'kmeans':

        kmeans = KMeans(init='k-means++', n_clusters=NC, n_init=10, random_state=randomSeed)
        print 'doing kmeans'
        start_time = time.time()
        kmeans.fit(Data)
        _labels = kmeans.predict(Data)
        cluster_centers = kmeans.cluster_centers_
        end_time = time.time()
        print 'done with k means in %f seconds'%(end_time-start_time)
        #return labels, cluster_centers, Data
    elif clusteringType == 'kmedoids':
        D = spd.cdist(Data, Data)
        print 'doing K-medoids'
        start_time = time.time()
        labels_medoid_ids, medoid_ids = kmedoids.cluster_kmedoid(D, NC)
        unique_labels = np.sort(np.unique(labels_medoid_ids))
        medoid_id_to_cluster_id = {}
        cluster_id = 0
        for ul in unique_labels:
            medoid_id_to_cluster_id[ul] = cluster_id
            cluster_id = cluster_id+1
        end_time = time.time()
        print 'done with k medoids in %f seondds'%(end_time-start_time)
        medoid_ids = np.sort(medoid_ids)
        medoids = Data[medoid_ids,:]
        _labels =[]
        for lab in labels_medoid_ids:
            _labels.append(medoid_id_to_cluster_id[lab])

        _labels = np.asarray(_labels, dtype=int)

    cluster_list = range(NC)
    clusters = dict.fromkeys(cluster_list, [])
    i = 0
    for indx in _labels:
        clusters[indx] = clusters[indx] + [i]
        i = i + 1

    _keyimage_list = {}
    std = {}
    for key, value in clusters.items():
        flist = []
        cluster_index = key
        cluster_center = cluster_centers[cluster_index]
        for index in value:
            v = Data[index]
            flist = flist + [v]
        diff = flist - cluster_center
        dist = [np.linalg.norm(v) for v in diff]
        std[cluster_index] = sqrt(np.sum(np.array(dist)*np.array(dist))/len(dist));
        print 'index= ', cluster_index, 'std= ', std[cluster_index]
        x = argmin(dist)
        _keyimage_list[cluster_index] = value[x]
    #return labels,
    return _labels, _keyimage_list

def data_hierarchy_segment(featurelist, page_size, top_k, diff_thresh, threshMethod, clusteringType, std_thresh, deviate_ratio_thresh, eigen_num=300, label_offset=0):
    print 'data_hierarchy_segment'
    randomSeed = None
    similarity = Similarity();
    _labels = []
    # similarity by nearest neighbor
    nsamples = len(featurelist)
    k = page_size # 40
    l = top_k #5
    threshold = diff_thresh #0.5
    myindex = IndexFlann(k, l, threshold);
    ntype=threshMethod #'byValue' #'byNumber' #

    for i in range(nsamples): #range(listsize):
        myindex.addFeatures(featurelist[i]);

    print 'sparse matrix computing'
    start_time = time.time()
    #similarity_measure = myindex.nearestneighbor(ntype);
    similarity_measure = myindex.nearestneighbor_cv(ntype);
    S, r,c,v = createSparseMatrix(similarity_measure, False)

    Lsym,D, DI_sqrt = computeSymmetricLaplacian(S);
    end_time = time.time()
    print 'done sparse matrix computing in %f seconds'%(end_time-start_time)

    ## pick the first 300 eigenvalues if it is less than the total number of items in the dataset
    #M = np.minimum(400, np.max(r))
    M = np.minimum(eigen_num, np.max(r))
    print 'computing eigenpairs'
    start_time = time.time()
    ## call ARPACK routine to compute top k eigenpairs of sparse symmetric matrix
    lam, v = eigsh(Lsym, M, which='SM' )
    end_time = time.time()
    print 'done computing eigenpairs in %f seconds'%(end_time-start_time)
    #plt.figure()
    #plt.plot(lam)
    #plt.show()
    ## find the "knee" in the the eigenvalue curve to decide on number of clusters
    NC = getClustersBasedonKnee(lam)
    ## create the data matrix fo K-means
    print 'numcluster = %d'%NC
    Data = v[:,0:NC]   # number of points x number of dimensions
    #row normalize D
    Data = rowNormalize(Data)
    _clusters = {}

    if clusteringType == 'kmeans':

        kmeans = KMeans(init='k-means++', n_clusters=NC, n_init=10, random_state=randomSeed)
        print 'doing kmeans'
        start_time = time.time()
        kmeans.fit(Data)
        _labels = kmeans.predict(Data)
        cluster_centers = kmeans.cluster_centers_
        end_time = time.time()
        print 'done with k means in %f seconds'%(end_time-start_time)
        #return labels, cluster_centers, Data
    elif clusteringType == 'kmedoids':
        D = spd.cdist(Data, Data)
        print 'doing K-medoids'
        start_time = time.time()
        labels_medoid_ids, medoid_ids = kmedoids.cluster_kmedoid(D, NC)
        unique_labels = np.sort(np.unique(labels_medoid_ids))
        medoid_id_to_cluster_id = {}
        cluster_id = 0
        for ul in unique_labels:
            medoid_id_to_cluster_id[ul] = cluster_id
            cluster_id = cluster_id+1
        end_time = time.time()
        print 'done with k medoids in %f seondds'%(end_time-start_time)
        medoid_ids = np.sort(medoid_ids)
        medoids = Data[medoid_ids,:]
        _labels =[]
        for lab in labels_medoid_ids:
            _labels.append(medoid_id_to_cluster_id[lab])

        _labels = np.asarray(_labels, dtype=int)

    _labels = _labels + label_offset;
    max_label = np.max(_labels);
    cluster_list = np.array(range(NC)) + label_offset;
    _clusters = dict.fromkeys(cluster_list, [])
    i = 0
    for indx in _labels:
        _clusters[indx] = _clusters[indx] + [i]
        i = i + 1

    _cluster_feature_list = {}
    _std_list = {}
    _dist_list = {}
    _keyimage_list = {}
    for key, value in _clusters.items():
        features = []
        flist = []
        cluster_index = key
        cluster_center = cluster_centers[cluster_index-label_offset]
        for index in value:
            v = Data[index]
            flist = flist + [v]
            u = featurelist[index]
            features.append(u);
        _cluster_feature_list[key] = features;
        diff = flist - cluster_center;
        dist = [np.linalg.norm(v) for v in diff]
        std = sqrt(np.sum(np.array(dist)*np.array(dist))/len(dist));
        dist = np.sort(dist);
        x = argmin(dist)
        _keyimage_list[cluster_index] = value[x]
        _std_list[key] = std;
        _dist_list[key] = dist;
        print 'index= ', cluster_index, 'size= ', len(value), 'std= ', std, 'min= ',dist[0], 'max= ', dist[len(dist)-1];

    for key, value in _clusters.items():
       features = _cluster_feature_list[key];
       std = _std_list[key]
       dist = _dist_list[key]
       if std > std_thresh or dist[len(dist)-1] > deviate_ratio_thresh*std:
           labels2,clusters2,keyimage_list2,cluster_feature_list2 = data_hierarchy_segment(features, page_size, 32, 0.8, threshMethod, clusteringType, std_thresh, deviate_ratio_thresh, 100, max_label+1);
           if len(labels2) > 0:
               # update _labels
               for i in range(len(labels2)):
                   _labels[value[i]] = labels2[i];
               # update _clusters
               _clusters.pop(key, None)
               for k, v in clusters2.items():
                   u = []
                   for i in v:
                       u = u + [value[i]];
                   clusters2[k] = u;
               _clusters = dict(_clusters.items() + clusters2.items());

               # update _cluster_feature_list
               max_label = np.max(_labels);
               _cluster_feature_list.pop(key, None)
               _cluster_feature_list = dict(_cluster_feature_list.items() + cluster_feature_list2.items())

               # update _keyimage_list
               _keyimage_list.pop(key, None)
               _keyimage_list = dict(_keyimage_list.items() + keyimage_list2.items())
    #return labels,
    return _labels, _clusters, _keyimage_list, _cluster_feature_list

# write all clusters to output folders
def outputClusters_1(_name_list, _label_list, _keyimage_list, sample_dir, output_folder):
    # number of clusters
    print 'write clustered images to ', output_folder
    k = len(_keyimage_list)
    for i in range(k):
        adir = output_folder + str(i)
        if( not os.path.exists(adir)):
            os.mkdir(adir);
    for i in range(len(_name_list)):
        alabel = _label_list[i]
        #print i, alabel
        a, b = _name_list[i].split('_')
        aframe = 'frame' + a
        aname = sample_dir + aframe + '/' + _name_list[i]
        afile = output_folder + str(alabel) + '/' + _name_list[i]
        img = cv2.imread(aname)
        cv2.imwrite(afile, img)

# write all clusters to output folders
def outputClusters_2(_name_list, _label_list, _keyimage_list, sample_dir, output_folder):
    # number of clusters
    if len(_keyimage_list) < 1:
        print 'no cluster is found'
        return

    for key, value in _keyimage_list.items():
        adir = output_folder + '/' + str(key)
        if( not os.path.exists(adir)):
            os.mkdir(adir);
    for i in range(len(_name_list)):
        alabel = _label_list[i]
        #print i, alabel
        a, b = _name_list[i].split('_')
        aframe = 'frame' + a
        aname = sample_dir + '/' + aframe + '/' + _name_list[i]
        afile = output_folder + '/' + str(alabel) + '/' + _name_list[i]
        img = cv2.imread(aname)
        cv2.imwrite(afile, img)

# write all clusters to output folders
def outputClusters_3(_name_list, _label_list, _keyimage_list, sample_dir, output_folder, output_keys):
    # number of clusters
    if len(_keyimage_list) < 1:
        print 'no cluster is found'
        return;

    for key, value in _keyimage_list.items():
        adir = output_folder + '/' + str(key)
        if( not os.path.exists(adir)):
            os.mkdir(adir);
        kdir = output_keys + '/key' + str(key)
        if( not os.path.exists(kdir)):
            os.mkdir(kdir);

    # clusters in each folder
    for i in range(len(_name_list)):
        alabel = _label_list[i]
        #print i, alabel
        a, b = _name_list[i].split('_')
        aframe = 'frame' + a
        aname = sample_dir + '/' + aframe + '/' + _name_list[i]
        afile = output_folder + '/' + str(alabel) + '/' + _name_list[i]
        img = cv2.imread(aname)
        cv2.imwrite(afile, img)

    # key images in each folder
    for key, value in _keyimage_list.items():
        a, b = _name_list[value].split('_')
        kframe = 'frame' + a
        kname = sample_dir + '/' + kframe + '/' + _name_list[value]
        afile = output_keys + '/key' + str(key) + '/' + _name_list[value]
        img = cv2.imread(kname)
        cv2.imwrite(afile, img)

def example_1():
    sample_dir_ = '/media/dzhang/CorticalProcessor/data/MBARI/active_learning/usf/00161/All/images/';
    file_dir = '/media/dzhang/CorticalProcessor/data/MBARI/active_learning/usf/00161/All/';

    output_folder_ = file_dir + 'sc/'
    json_file_ = file_dir + 'features.json'

    clusteringType = 'kmeans'; # 'kmedoids'
    threshMethod = 'byValue' #'byValue' # 'byNumber'
    page_size = 3000
    top_k = 100  #20
    diff_thresh = 0.6

    namelist_, featurelist_ = data_read(sample_dir_, json_file_)
    label_index = [x for x in range(len(namelist_))]
    label_list_, keyimage_list_ = data_segment(featurelist_, page_size, top_k, diff_thresh, threshMethod, clusteringType)

    outputClusters_1(namelist_, label_list_, keyimage_list_, sample_dir_, output_folder_)
    print  'done'

def example_2():
    video_example = 'testvideo2' #'testvideo1'

    sample_dir_ = '/data0/flaskfs/dataset/' + video_example + '/selected/All/images/';
    file_dir = '/data0/flaskfs/dataset/' + video_example + '/selected/All/';

    output_folder_ = file_dir + 'sc/'
    output_keys_ = file_dir + 'keys/'
    json_file_ = file_dir + 'features.json'

    clusteringType = 'kmeans'; # 'kmedoids'
    threshMethod = 'byValue' #'byValue' # 'byNumber'
    page_size = 3000
    top_k = 100  #20
    diff_thresh = 0.6
    eigen_num = 300
    label_offset = 0
    std_thresh = 0.55
    deviate_ratio_thresh = 2.0;

    namelist_, featurelist_ = data_read(sample_dir_, json_file_)
    label_list_, clusters_, keyimage_list_, cluster_feature_list_ = data_hierarchy_segment(featurelist_, page_size, top_k,
                                                                           diff_thresh, threshMethod, clusteringType,
                                                                           std_thresh, deviate_ratio_thresh,
                                                                           eigen_num, label_offset);
    outputClusters_3(namelist_, label_list_, keyimage_list_, sample_dir_, output_folder_, output_keys_)
    print  'done'

def example_3():
    video_example = 'testvideo2' #'testvideo1'

    sample_dir_ = '/media/dzhang/CorticalProcessor/data/MBARI/MBARI_cluster_tests/' + video_example + '/images/';
    file_dir = '/media/dzhang/CorticalProcessor/data/MBARI/MBARI_cluster_tests/' + video_example + '/';

    output_folder_ = file_dir + 'sc/'
    output_keys_ = file_dir + 'keys/'
    json_file_ = file_dir + 'features.json'

    clusteringType = 'kmeans'; # 'kmedoids'
    threshMethod = 'byValue' #'byValue' # 'byNumber'
    page_size = 3000
    top_k = 100  #20
    diff_thresh = 0.6
    eigen_num = 300
    label_offset = 0
    std_thresh = 0.55
    deviate_ratio_thresh = 2.0;

    namelist_, featurelist_ = data_read(sample_dir_, json_file_)
    label_list_, clusters_, keyimage_list_, cluster_feature_list_ = data_hierarchy_segment(featurelist_, page_size, top_k,
                                                                           diff_thresh, threshMethod, clusteringType,
                                                                           std_thresh, deviate_ratio_thresh,
                                                                           eigen_num, label_offset);
    outputClusters_3(namelist_, label_list_, keyimage_list_, sample_dir_, output_folder_, output_keys_)
    print  'done'

def example_4():
    video_example = 'testvideo2' #'testvideo1'

    sample_dir_ = '/media/dzhang/CorticalProcessor/data/MBARI/USF_dataset/USF_SD_LinStrtch/All/images/';
    file_dir = '/media/dzhang/CorticalProcessor/data/MBARI/USF_dataset/USF_SD_LinStrtch/All/';

    output_folder_ = file_dir + 'sc/'
    output_keys_ = file_dir + 'keys/'
    json_file_ = file_dir + 'features.json'

    clusteringType = 'kmeans'; # 'kmedoids'
    threshMethod = 'byValue' #'byValue' # 'byNumber'
    page_size = 3000
    top_k = 100  #20
    diff_thresh = 0.6
    eigen_num = 300
    label_offset = 0
    std_thresh = 0.55
    deviate_ratio_thresh = 2.0;

    namelist_, featurelist_ = data_read(sample_dir_, json_file_)
    label_list_, clusters_, keyimage_list_, cluster_feature_list_ = data_hierarchy_segment(featurelist_, page_size, top_k,
                                                                           diff_thresh, threshMethod, clusteringType,
                                                                           std_thresh, deviate_ratio_thresh,
                                                                           eigen_num, label_offset);
    outputClusters_3(namelist_, label_list_, keyimage_list_, sample_dir_, output_folder_, output_keys_)
    print  'done'

if __name__ == '__main__':
    example_2()

max_stars_repo_name: readicculus/VIAME

max_issues_repo_name: readicculus/VIAME

max_forks_repo_name: readicculus/VIAME

reason: import numpy,from numpy,import scipy,from scipy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import numpy as np
from scipy.optimize import minimize, line_search
import matplotlib.pyplot as plt

def rosen(x):
    """The Rosenbrock function"""
    return sum(100.0*(x[1:]-x[:-1]**2.0)**2.0 + (1-x[:-1])**2.0)

def rosen_der(x):
    xm = x[1:-1]
    xm_m1 = x[:-2]
    xm_p1 = x[2:]
    der = np.zeros_like(x)
    der[1:-1] = 200*(xm-xm_m1**2) - 400*(xm_p1 - xm**2)*xm - 2*(1-xm)
    der[0] = -400*x[0]*(x[1]-x[0]**2) - 2*(1-x[0])
    der[-1] = 200*(x[-1]-x[-2]**2)
    return der

def test_gd():
    xi = np.array([1.3, 0.7, 0.8, 1.9, 1.2], dtype=np.float64)
    # value found by one iteration of ANO (leading to local minimum)
    # xi[:] = [1.09666,0.8126,0.934773,1.07724,1.39017]
    mom = np.zeros_like(xi)
    lr = np.float64(0.0007)
    for i in range(20):
        gx = -rosen_der(xi)
        # mom = mom * 0.9 + gx
        # xi -= lr * mom
        # actually converges faster without line search
        lr, *_ = line_search(rosen, rosen_der, xi, gx)
        print(i, rosen(xi), lr)
        xi += lr * gx

def test_gx0():
    x0 = np.array([1.3, 0.7, 0.8, 1.9, 1.2])
    x0 = np.array([0.55210179903703016, 0.20098467487614619,
                   -0.086018900221027095, 0.38088062880673074,
                   0.042037353198560869])
    print('x0', x0)
    gx0 = rosen_der(x0)
    print('gx0', gx0)
    px = np.linspace(-0.002, 0.002, 100)
    py = [rosen(x0 + gx0 * i) for i in px]

    plt.plot(px, py)
    plt.show()

def plot_poly():
    coeff = np.array([
        0.0244193, 0.0451206, 23.2877, 0.760964, 0.0265465, 0, 0, 0, 0,
    ])
    def ev(x):
        return np.power(x, np.arange(coeff.size)).dot(coeff)

    # px = np.linspace(-0.051765, 0.051765, 100)
    px = np.linspace(-1e-2, 1e-2, 100)
    py = [ev(i) for i in px]
    plt.plot(px, py)
    plt.show()

if __name__ == '__main__':
    # test_gd()
    test_gx0()
    # plot_poly()

max_stars_repo_name: jia-kai/SANM

max_issues_repo_name: jia-kai/SANM

max_forks_repo_name: jia-kai/SANM


text:
import torch
import torch.utils.data as data
import numpy as np
import os
from PIL import Image
import logging
import matplotlib.pyplot as plt

logging.basicConfig(format='%(asctime)s : %(levelname)s - %(message)s',
                    datefmt='%d/%m/%Y %I:%M:%S %p',
                    level=logging.INFO)
logger = logging.getLogger('MemeDatasetLog')

class Dataset_Template(data.Dataset):

    def __init__(self,
                 filepath : str,
                 feature_dir: str = None,
                 preload_images : bool = True,
                 text_preprocess = None,
                 text_getitem = None,
                 text_padding = None,
                 text_only : bool = False,
                 return_ids : bool = False,
                 include_object_tags: bool = False,
                 object_to_text_file_path: str = '',
                 compact_batch: bool = True,
                 confidence_threshold : float = 0.0,
                 debug : bool = False):
        """
        Inputs:
            filepath - Filepath to the ".jsonl" file which stores a list of all data points.
            feature_dir - Directory containing image features.
            preload_images - If True, the dataset will load all images from the hard disk into RAM during creation of the object.
                             Saves time during training, but takes longer to create the dataset object and requires more memory.
            text_preprocess - Function to execute after loading the text. Takes as input a list of all meme text in the dataset.
                              Is expected to return a list of processed text elements.
            text_getitem - Function to apply on each text element during "__getitem__" method of the dataset.
            text_padding - Function to apply for padding the input text to match a batch tensor.
            text_only - If True, only text will be loaded from the dataset. In the __getitem__, None values will be returned for images.
            return_ids - If True, the __getitem__ function returns the ids of the data points as last element
            include_object_tags - If True, include the object tags in the text
            object_to_text_filepath - File containing mapping between object IDs and object text
            compact_batch - If True, batches with text and image will be compacted without padding between them
            confidence_threshold - Threshold of object confidence under which bounding boxes are ignored
            debug - If True, more output information is logged. For instance, it displays a tqdm loading bar when preloading images.
        """
        super().__init__()
        self.filepath = filepath
        self.name = filepath.split("/")[-1].split(".")[0]
        self.feature_dir = feature_dir
        self.preload_images = preload_images
        self.text_preprocess = text_preprocess
        self.text_getitem = text_getitem
        self.text_padding = text_padding
        self.text_only = text_only
        self.include_object_tags = include_object_tags
        self.return_ids = return_ids
        self.compact_batch = compact_batch
        self.confidence_threshold = confidence_threshold
        self.debug = debug

        self._prepare_data_list()

        if self.include_object_tags:
            self.object_to_text_filepath = object_to_text_file_path
            self._load_object2text()

    def _prepare_data_list(self):
        raise NotImplementedError

    def _load_dataset(self):
        raise NotImplementedError

    def _load_img(self, img_filepath):
        with open(img_filepath, "rb") as f:
            img = Image.open(f).convert("RGB")
        return img

    def _expand_id(self, img_id):
        return str(img_id).zfill(5)

    def _load_img_feature(self, img_id, normalize=False):
        img_id = self._expand_id(img_id)
        feature_file_name = os.path.join(self.feature_dir, '{}.npy'.format(img_id))
        feature_info_file = os.path.join(self.feature_dir, '{}_info.npy'.format(img_id))
        img_feat = torch.from_numpy(np.load(feature_file_name))
        img_feat_info = np.load(feature_info_file, allow_pickle=True).item()
        x1, y1, x2, y2 = np.split(img_feat_info['bbox'], 4, axis=1)
        img_width = img_feat_info['image_width']
        img_height = img_feat_info['image_height']
        objects = img_feat_info['objects']
        if 'objects_conf' in img_feat_info:
            objects_conf = img_feat_info['objects_conf']
        else:
            objects_conf = img_feat_info['cls_prob'].max(axis=-1)
        if normalize:
            x1 /= img_width
            x2 /= img_width
            y1 /= img_height
            y2 /= img_height
        w = x2 - x1
        h = y2 - y1
        img_pos_feat = torch.from_numpy(np.concatenate((x1, y1, x2, y2, w, h, w*h), axis=1))
        return img_feat, img_pos_feat, objects, objects_conf

    def __len__(self):
        raise NotImplementedError

    def __getitem__(self, idx):
        raise NotImplementedError

    def get_collate_fn(self):
        """
        Returns functions to use in the Data loader (collate_fn).
        Image features and position features are stacked (with padding) and returned.
        For text, the function "text_padding" takes all text elements, and is expected to return a list or stacked tensor.
        """

        def collate_fn(samples):
            raise NotImplementedError

        return collate_fn

    def get_by_id(self, data_id):
        """
        Function to access data point via their ID (given by the dataset) instead of range 0-N.
        """
        if data_id in self.data.ids:
            idx = np.where(self.data.ids == data_id)[0]
            return self.__getitem__(idx)
        else:
            logger.warning("Tried to access data id \"%s\", but is not present in the dataset." % str(data_id))
            return None

    def show_img(self, idx=None, data_id=None, img=None):
        """
        Displays an image of the dataset. At least one of the following input arguments needs to be given:
            idx - Index of the image in the dataset
            data_id - ID of the data point with the image (stated in the original .jsonl file)
            img - PIL image to display
        """
        if img is None:
            if idx is None:
                if data_id is not None:
                    idx = np.where(self.data.ids == data_id)[0]
                else:
                    logger.warning("Method show_img got all empty inputs. No image displayed.")
                    return
            img = self.data.imgs[idx]
            label = self.data.labels[idx]
            if not self.preload_images:
                img = self._load_img(img)
        else:
            label = None

        plt.imshow(img)
        plt.axis('off')
        if label is not None:
            plt.title("Label: %s (%i)" % ("Hateful" if label==1 else ("Not hateful" if label==0 else "Unknown"), label))
        plt.show()
        plt.close()