text:
from __future__ import absolute_import
from __future__ import print_function
import os
import numpy as np
import pandas as pd
from scipy import interpolate
import time

#pandas columns that are "objects", but you are 100% sure contain strings
# will through this warning.  There is no way to set them as strings.
# pandas will permanently keep a reference to the object even if you
# set it as a string.  Bottom line: make it a string and ignore the error.
# it prints to hdf5 just fine.
import warnings
from six.moves import range
from six.moves import zip
warnings.simplefilter(action='ignore', category=pd.errors.PerformanceWarning)

"""

"""

# SUMMARY OF TIMING AND MEMORY TESTING AT NERSC
# 10 threads
# on denovo, about 30 seconds each, a little over 9 minutes
# on denovo using scratchb, 20 seconds each, 7.43 minutes
# on cori, about 20 seconds each, 6.52 minutes staging all files to $SCRATCG=6.6 minutes

# 20 threads
# on denovo, about 30 seconds each, a little over 6 minutes
# on denovo scratch, 5.1 minutes
# on cori, didn't run on jupyter-dev, on cori withscratch = 5.4 minutes, 5.5 using realtime scratch, burst-buffer 5.23 minutes

# 30 threads
# on denovo, about 35 seconds each, the job never finished
# on cori, burst buffer: 4.1962 minutes repeated: 4.157 minutes
# on cori, scratch: 4.31 minutes

def setup_file_slicing_parameters(atlas,filenames,extra_time=0.1,ppm_tolerance=20,polarity='positive',project_dir=False,base_dir = '/project/projectdirs/metatlas/projects/',overwrite=True):
    """
    Make parameters that have to be setup to run the fast feature finding process.

    This function is called first when doing feature selection. It standardizes
    all necessary inputs and files so downstream functions get a consistent place
    to work.

    Args:
        atlas (pandas dataframe): with [label,mz,rt_min,rt_max,rt_peak]. optional
        parameters are fine too.

        filenames (list): full paths to hdf5 files

        extra_time (float): default=0.1 Time to get in addition to rt-min/max window (for making nice EICs)
        custom is to store metatlas hdf5 files in minutes, but always double check

        ppm_tolerance (float): default=20 Calibration is sometimes a problem. 20
        is safe.

        polarity (str): default='positive' or 'negative'

        project_dir (bool/str): default=False if user doesn't want to save their results
        or "path to output files"

        only relevant if project_dir is not False
        base_dir (str): '/project/projectdirs/metatlas/projects/'
        other paths that have been tried, but only a few percent faster than project:
            scratch_dir = os.environ['SCRATCH']
            scratch_dir = os.environ['DW_JOB_STRIPED']

    Returns:
        input_data list(dict): a list of python dictionaries with the following attributes
        for each lcmsrun to process:
            outfile (str): path to output hdf5 file of feature signals
            lcmsrun (str): lcmsrun to process
            atlas (pandas dataframe): atlas with necessary attributes for feature slicing,
            polarity (str): passthrough of input polarity string

    """

    """
    setup atlas table and define extra_time and ppm_tolerance in the atlas
    for compound atlases, label isn't stritly necessary add it if not provided
    """
    if not 'label' in atlas.columns:
        atlas['label'] = list(range(atlas.shape[0]))
    atlas['extra_time'] = extra_time
    atlas['ppm_tolerance'] = ppm_tolerance

    """
    Group together m/z values that are within ppm_tolerance.
    This gives an index to acknowledge that there are multiple features with nearly
    equal m/z.  Assigning it here speeds up the file slicing and feature selection
    down the road.
    """
    atlas['group_index'] = group_consecutive(atlas['mz'].values[:],
                                             stepsize=ppm_tolerance,
                                             do_ppm=True)

    """
    define output directory
    """
    if project_dir is not False: #user doesn't want to save their results
        output_dir = os.path.join(base_dir,project_dir)
        if not os.path.isdir(output_dir):
            os.mkdir(output_dir)

    """
    get lcmsruns to process and build fullpath to output files
    """

    """
    setup input dictionary that will be the get_data input for each file
    """
    input_data = []
    for i,f in enumerate(filenames):
        #strip off the path and extension from the filename
        file_frag = ''.join(os.path.basename(f).split('.')[:-1])
        if len(file_frag)>0:
            output_filename = '%s_features.h5'%file_frag
            if project_dir is not False: #user doesn't want to save their results
                outfile = os.path.join(output_dir,output_filename)
            else:
                outfile = None
            input_data.append({'outfile':outfile,'lcmsrun':f,'atlas':atlas,'polarity':polarity}) # 'ppm_tolerance':ppm_tolerance,'extra_time':extra_time,,'start_time':time.time() 'file_index':i,

    """
    wipe out all the files and put the atlas in each one
    """
    if overwrite==True:
        for i in input_data:
            if i['outfile'] is not None: #user doesn't want to save their results
                with pd.HDFStore(i['outfile'],mode='w',complib='zlib',complevel=9) as f:
                    f.put('atlas',atlas,data_columns=True)

    return input_data

def group_consecutive(data,stepsize=10.0,do_ppm=True):
    """
    split a numpy array where consecutive elements are greater than stepsize
    can be ppm or value
    if idx is not None then returns indices of elements otherwise returns values

    The main use case is an unsorted list of m/zs as "data" and optionally
    their numerical index as "idx". Typically a user would want to retrieve
    the group indices in the original order that they provided their list
    of m/zs.

    usage:

    """
    if type(data) is np.ndarray:
        # cool way to sort and unsort array:
        idx_sorted = data.argsort()
        sort_w_unsort = np.column_stack((np.arange(idx_sorted.size),idx_sorted))
        # sort_w_unsort[:,0] are the original indices of data
        # sort_w_unsort[:,1] are the sorted indices of data
        data_sorted = data[sort_w_unsort[:,1]]
        # np.argsort(sort_w_unsort[:,1]) returns the indices to map the sorted data back to the original
        # data_unsorted = data_sorted[np.argsort(sort_w_unsort[:,1])]

        if do_ppm:
            d = np.diff(data_sorted) / data_sorted[:-1] * 1e6
        else:
            d = np.diff(data_sorted)

        # make groups of the array
        data_groups = np.split(data_sorted, np.where(d > 2.0*stepsize)[0]+1)
        # replace each group of values with group index
        for i,data_slice in enumerate(data_groups):
            data_groups[i] = data_groups[i]*0 + i
        group_indices = np.concatenate(data_groups)
        # reorder the group indices
        group_indices = group_indices[np.argsort(sort_w_unsort[:,1])]
        return group_indices.astype(int)#
    else:
        print('not a numpy array. convert it and sort it first')

def map_mzgroups_to_data(mz_atlas,mz_group_indices,mz_data):
    """
    mz_atlas: m/z values from atlas
    mz_group_indices: integer index from "group_consecutive"
    mz_data: m/z values from raw data

    """
    from scipy import interpolate

    f = interpolate.interp1d(mz_atlas,np.arange(mz_atlas.size),kind='nearest',bounds_error=False,fill_value='extrapolate') #get indices of all mz values in the atlas
    idx = f(mz_data)   # iterpolate to find the nearest mz in the data for each mz in an atlas
    idx = idx.astype('int')
#     d = 1e6#np.abs(mz_data - mz_atlas[idx]) / mz_data * 1.0e6
#     output_mat = np.column_stack((d,))
    return mz_group_indices[idx]#output_mat

def df_container_from_metatlas_file(filename,desired_key=None):
    """

    """
#     data_df = pd.DataFrame()

    pd_h5_file  = pd.HDFStore(filename)

    keys = list(pd_h5_file.keys())
    pd_h5_file.close()
    df_container = {}
    if desired_key is not None:
        return pd.read_hdf(filename,desired_key)
    else:
        for k in keys:
            if ('ms' in k) and not ('_mz' in k):
                new_df = pd.read_hdf(filename,k)
                df_container[k[1:]] = new_df
    return df_container

def group_duplicates(df,group_col,make_string=False,precision={'i':0,'mz':4,'rt':2}):
    """
    takes in a list of grouping columns and turns the rest into arrays
    """

    all_cols = np.asarray(df.columns)
    #get the index of the grouping term as array
    idx_group = np.argwhere(all_cols == group_col).flatten()
    #get the indices of all other terms as array
    idx_list = np.argwhere(all_cols != group_col).flatten()
    cols = all_cols[idx_list]

    # create a sorted numpy array (sorted by column=group_col)
    a = df.sort_values(group_col).values.T

    #get the indices of the first instance of each unique identifier
    ukeys, index = np.unique(a[idx_group,:],return_index=True)

    #split the other rows of the array into separate arrays using the
    #unique index
    arrays = np.split(a[idx_list,:],index[1:],axis=1)

    #make a list of dicts with column headings as keys
    #if there are not multiple items then return value
    #If there are multiple items then return list

#     ucpds = [dict([(c,aa) if len(aa)>1 else (c,aa[0]) for c,aa in zip(cols,a)]) for a in arrays ]
    ucpds = [dict([(c,aa) for c,aa in zip(cols,a)]) for a in arrays ]

    #make a dataframe from the list of dicts
    df2 = pd.DataFrame(ucpds,index=ukeys)

    #make strings of array columns if you want to save it in anything useful
    if make_string==True:
        for c in cols:
#             df2[c] = df2[c].apply(lambda x: np.array2string(x, precision=5, separator=','))
            if c in list(precision.keys()):
                pre_str = '{:.%df}'%precision[c]
            else:
                pre_str = '{:.4f}'
            df2[c] = df2[c].apply(lambda x: [pre_str.format(n) for n in x.tolist()])
#             df2[c] = df2[c].apply(lambda x: str(x.tolist()))

    df2.index = df2.index.set_names(group_col)
    df2.reset_index(inplace=True)

    #return dataframe
    return df2

def get_atlas_data_from_file(filename,atlas,desired_key='ms1_pos'):#,bundle=True,make_string=False):
    msdata = df_container_from_metatlas_file(filename,desired_key=desired_key)
    if 'ms2' in desired_key:
        # throw away all the intensity duplication here to make merging faster
        # this has the expense of having to remerge it later.
        msdata = msdata[['rt','precursor_MZ']].drop_duplicates('rt')
        msdata = msdata.rename(columns={'precursor_MZ':'mz'})

    g = map_mzgroups_to_data(atlas['mz'].values[:],
                               atlas['group_index'].values[:],
                               msdata['mz'].values[:])
    msdata['group_index'] = g#[:,1]
#     msdata['group_index_ppm'] = g[:,0]
    df = pd.merge(atlas,msdata,left_on='group_index',right_on='group_index',how='outer',suffixes=('_atlas','_data'))

    #grab all datapoints including "extra"
    mz_condition = abs(df['mz_data']-df['mz_atlas'])/df['mz_atlas']*1e6<df['ppm_tolerance']
    rt_min_condition = df['rt']>=(df['rt_min']-df['extra_time'])
    rt_max_condition = df['rt']<=(df['rt_max']+df['extra_time'])
    df = df[(mz_condition) & (rt_min_condition) & (rt_max_condition)]

    #label datapoints that are within the bounds of the feature vs "extra"
    df['in_feature'] = True
    if df['extra_time'].max()>0.0:
        cond_rt = (df['rt']<df['rt_min']) | (df['rt']>df['rt_max'])
        df.loc[cond_rt,'in_feature'] = False

    #above, the df has mz_data and mz_atlas.  we don't need to differentiate anymore so:
    df = df.rename(columns={'mz_data':'mz'})
    if 'ms2' in desired_key:
        # keep in mind we don't have intensity or scan attributes
        df = df[['label','rt','in_feature']]
        # you've got to add it back in; so reload original file
        msdata = df_container_from_metatlas_file(filename,desired_key=desired_key)
        # This will merge back into the MSMS data
        # the missing intensity and scan attributes
        mcols = ['rt','i','mz','precursor_MZ','precursor_intensity','collision_energy']
        df = pd.merge(df,msdata[mcols],left_on='rt',right_on='rt',how='left')
        return df.reset_index(drop=True)
    else:
        df = df[['label','rt','mz','i','in_feature']]
        return df.reset_index(drop=True)

def calculate_ms1_summary(row):
    """
    Calculate summary properties for features from data
    """
    d = {}
    #Before doing this make sure "in_feature"==True has already occured
    d['num_datapoints'] = row['i'].count()
    d['peak_area'] = row['i'].sum()
    idx = row['i'].idxmax()
    d['peak_height'] = row.loc[idx,'i']
    d['mz_centroid'] = sum(row['i']*row['mz'])/d['peak_area']
    d['rt_peak'] = row.loc[idx,'rt']
    return pd.Series(d)

# def calculate_ms1_summary(df):
#     a = df[['label','rt','mz','i','in_feature']].values
#     labels, row_pos = np.unique(a[:, 0], return_inverse=True) #these are feature labels
#     rt, col_pos = np.unique(a[:, 1], return_inverse=True) #these are rt values

#     pivot_table = np.zeros((len(labels), len(rt),3), dtype=float)
#     pivot_table[row_pos, col_pos] = a[:, [2,3,4]]
#     eic = pd.DataFrame(index=labels,data=pivot_table[:,:,1],columns=rt)
#     emzc = pd.DataFrame(index=labels,data=pivot_table[:,:,0],columns=rt)
#     efeaturec = pd.DataFrame(index=labels,data=pivot_table[:,:,2],columns=rt)
#     in_feature = efeaturec.values.astype(int)
#     intensity = np.multiply(eic.values,in_feature)
#     mz = np.multiply(emzc.values,in_feature)
#     rt = np.asarray(eic.columns)
#     labels = eic.index.tolist()
#     idx_max = np.argmax(intensity,axis=1)
#     df = pd.DataFrame(index=labels)
#     df['num_datapoints']=in_feature.sum(axis=1)
#     df['peak_area']=intensity.sum(axis=1)
#     df['peak_height']=np.diag(intensity[:,idx_max]) #I shouldn't have to do this and must be doing numpy slicing wrong!
#     df['mz_centroid']=np.divide(np.sum(np.multiply(mz,intensity),axis=1),intensity.sum(axis=1))
#     df['rt_peak']=rt[idx_max]
#     return df

def get_data(input_data,return_data=False,save_file=True):
    """
    Required Inputs a Dict that has these attributes:
    {'file_index':i, #a numerical index that helps with bookkeeping
    'outfile':outfile, #the hdf5 container to store the results
    'lcmsrun':new_file, #the hdf5 file corresponding to an lcms run
    'atlas':atlas, #the atlas dataframe containing minimally: [mz, rt_min,rt_max,rt_peak)]
    'ppm_tolerance':ppm_tolerance, #ppm tolerance in m/z
    'extra_time':extra_time} #time to add to the collected data beyond rt_min and rt_max

    The goal is to write to a file,
    ms1_data:

    ms1_summary:

    ms2_data:

    Returns a dictionary
    """
    out_data = {} #setup a container to store any data to return to the user otherwise save it to file
    polarity_short_string = input_data['polarity'][:3]
    d = get_atlas_data_from_file(input_data['lcmsrun'],input_data['atlas'],desired_key='ms1_%s'%polarity_short_string)#,bundle=True,make_string=True)
    if return_data is True:
        out_data['atlas'] = input_data['atlas']
        out_data['ms1_data'] = d
    if save_file is True:
        with pd.HDFStore(input_data['outfile'],mode='a',complib='zlib',complevel=9) as f:
            f.put('ms1_data',d,data_columns=True)

    d = d[d['in_feature']==True].groupby('label').apply(calculate_ms1_summary).reset_index()
    if d.shape[0]==0: #there isn't any data!
        for c in ['num_datapoints','peak_area','peak_height','mz_centroid','rt_peak']:
            d[c] = 0

    if return_data is True:
        out_data['ms1_summary'] = d
    if save_file is True:
        with pd.HDFStore(input_data['outfile'],mode='a',complib='zlib',complevel=9) as f:
            f.put('ms1_summary',d,data_columns=True)

#     input_data['atlas']['extra_time'] = 0.0 # set extratime here to be zero for msms getting
    d = get_atlas_data_from_file(input_data['lcmsrun'],input_data['atlas'],desired_key='ms2_%s'%polarity_short_string)#,bundle=True,make_string=True)
    if return_data is True:
        out_data['ms2_data'] = d
    if save_file is True:
        with pd.HDFStore(input_data['outfile'],mode='a',complib='zlib',complevel=9) as f:
            f.put('ms2_data',d,data_columns=True)

    if return_data is True:
        return out_data

max_stars_repo_name: biorack/metatlas

max_issues_repo_name: biorack/metatlas

max_forks_repo_name: biorack/metatlas


text:
import argparse
import copy
import json
import os
import sys

import numpy as np
from progressbar import ProgressBar

import h5py
from src.data import import_labels
from src.processing import activity_localization, get_classification, smoothing, get_label_sequence_from_info

def process_prediction(experiment_id,
                       predictions_path,
                       output_path,
                       smoothing_k,
                       activity_threshold,
                       subset=None):
    clip_length = 16.

    if subset == None:
        subsets = ['validation', 'testing']
    else:
        subsets = [subset]

    predictions_file = os.path.join(
        predictions_path,
        'predictions_{experiment_id}.hdf5'.format(experiment_id=experiment_id))

    with open('dataset/labels.txt', 'r') as f:
        labels = import_labels(f)
    with open('dataset/videos.json', 'r') as f:
        videos_info = json.load(f)

    f_predictions = h5py.File(predictions_file, 'r')
    for subset in subsets:
        print('Generating results for {} subset...'.format(subset))
        subset_predictions = f_predictions[subset]

        progbar = ProgressBar(max_value=len(subset_predictions.keys()))
        # with open('dataset/templates/results_{}.json'.format(subset),
        #           'r') as f:
        #     results_classification = json.load(f)
        # results_detection = copy.deepcopy(results_classification)

        count = 0
        progbar.update(0)
        for video_id in subset_predictions.keys():
            prediction = subset_predictions[video_id][...]
            video_info = videos_info[video_id]
            fps = float(video_info['num_frames']) / video_info['duration']

            # Post processing to obtain frame-level classification
            get_label_sequence_from_info(video_info, labels)

            # Post processing to obtain the classification
            # labels_idx, scores = get_classification(prediction, k=5)
            # result_classification = []
            # for idx, score in zip(labels_idx, scores):
            #     label = labels[idx]
            #     if score > 0:
            #         result_classification.append({
            #             'score': score,
            #             'label': label
            #         })
            # results_classification['results'][video_id] = result_classification

            # Post Processing to obtain the detection
            # prediction_smoothed = smoothing(prediction, k=smoothing_k)
            # activities_idx, startings, endings, scores = activity_localization(
            #     prediction_smoothed, activity_threshold)
            # result_detection = []
            # for idx, s, e, score in zip(activities_idx, startings, endings,
            #                             scores):
            #     label = labels[idx]
            #     result_detection.append({
            #         'score':
            #         score,
            #         'segment': [s * clip_length / fps, e * clip_length / fps],
            #         'label':
            #         label
            #     })
            # results_detection['results'][video_id] = result_detection

            count += 1
            progbar.update(count)
        progbar.finish()

        # classification_output_file = os.path.join(
        #     output_path, 'results_classification_{}_{}.json'.format(
        #         experiment_id, subset))
        # detection_output_file = os.path.join(
        #     output_path, 'results_detection_{}_{}.json'.format(
        #         experiment_id, subset))
        # with open(classification_output_file, 'w') as f:
        #     json.dump(results_classification, f)
        # with open(detection_output_file, 'w') as f:
        #     json.dump(results_detection, f)

    f_predictions.close()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description=
        'Post-process the prediction of the RNN to obtain the classification and temporal localization of the videos activity'
    )

    parser.add_argument(
        '--id',
        dest='experiment_id',
        default=0,
        help='Experiment ID to track and not overwrite results')
    parser.add_argument(
        '-p',
        '--predictions-path',
        type=str,
        dest='predictions_path',
        default='data/dataset',
        help=
        'Path where the predictions file is stored (default: %(default)s)')
    parser.add_argument(
        '-o',
        '--output-path',
        type=str,
        dest='output_path',
        default='data/dataset',
        help=
        'Path where is desired to store the results (default: %(default)s)')

    parser.add_argument(
        '-k',
        type=int,
        dest='smoothing_k',
        default=5,
        help='Smoothing factor at post-processing (default: %(default)s)')
    parser.add_argument(
        '-t',
        type=float,
        dest='activity_threshold',
        default=.2,
        help='Activity threshold at post-processing (default: %(default)s)')

    parser.add_argument(
        '-s',
        '--subset',
        type=str,
        dest='subset',
        default=None,
        choices=['validation', 'testing'],
        help=
        'Subset you want to post-process the output (default: validation and testing)'
    )

    args = parser.parse_args()

    process_prediction(args.experiment_id, args.predictions_path,
                       args.output_path, args.smoothing_k,
                       args.activity_threshold, args.subset)

ext: py

max_stars_repo_name: aclapes/breakfast-cnn-lstm

max_issues_repo_name: aclapes/breakfast-cnn-lstm

max_forks_repo_name: aclapes/breakfast-cnn-lstm

reason: import numpy


text:
import cv2
import numpy as np
from matplotlib import pyplot as plt
import skimage
import skimage.io
from skimage import data, util, measure
import pandas as pd
import imagecodecs
import tifffile
from skimage.color import rgb2gray
import skimage.segmentation as seg
from skimage.segmentation import slic
from skimage.segmentation import mark_boundaries
from skimage.util import img_as_float
import argparse
from skimage import feature

def grayschale(img):
    """converts the image into a grayscale"""
    original = skimage.io.imread(img)
    grayscale = rgb2gray(original)

    return grayscale

def create_hotogram(img):
    """This function creates a histogram for the image given, image should be inputed in a grayscale"""
    hist, bins = skimage.exposure.histogram(img)
    plt.plot(bins, hist, linewidth=1)
    plt.xlabel('pixel value (a.u.)')
    plt.ylabel('counts')
    plt.show()

def image_thresh(img, thresh):
    """ This function when given a picture and a threshold value, returns an image after thresholding"""
    thresh_val = thresh
    thresh_im = img > thresh_val
    plt.imshow(thresh_im, cmap='gray')
    plt.show()
    return thresh_im

def image_measurements(img,genotype,N,cell_number):
    results_dict={}

    cell_genotype=genotype
    N=N
    cell_number=cell_number
    total_area=img.size
    stained_area=np.count_nonzero(img)
    percent_area=stained_area/total_area
    total_intensity=np.sum(img)
    mean_intensity=np.mean(img)
    intigrated_optical_density=mean_intensity*stained_area #check math

    results_dict.update({"Cell genotype": cell_genotype, "N": N, "Cell number": cell_number, "total area":total_area, "stained area":stained_area, "percent area": percent_area, "total_intensity":total_intensity, "mean_intensity":mean_intensity, "intigrated_optical_density": intigrated_optical_density })
    df=pd.DataFrame.from_dict(results_dict, orient='index')

    return df.T

def cell_genotype(image_name):
    """returns cell genotype from the name"""
    c_name=image_name.upper()
    if (c_name.find('E3') != -1):
        genotype="E3"
    elif (c_name.find('E4') != -1):
        genotype="E4"
    else:
        genotype="Unknown"

    return genotype

if __name__ == "__main__":
    image_name='e3 hol 1250 1500_z0_ch02.tif'
    a= grayschale(image_name)
    c= image_thresh(a, 0.28)
    genotype=cell_genotype(image_name)
    df1=image_measurements(c,genotype,1,1)
    df2=image_measurements(c,genotype,1,2)

    # append the results of the current measurment to the dataframe of the results
    df1=df1.append(df2)
    print (df1)


text:
#!/usr/bin/python

'''
Learning Machines
Taught by Patrick Hebron at NYU ITP

Restricted Boltzmann Machine (RBM) implementation.
'''

import numpy as np
import Activation
from MnistReporter import *
import time

class Rbm:
	def __init__(self, name, sizeV, sizeH, continuous = False):
		self.name	 = name
		self.is_crbm = continuous
		# Initialize weights:
		self.weights = np.array( np.random.uniform( -1.0 / sizeV, 1.0 / sizeV, ( sizeV, sizeH ) ) )
		# Initialize biases:
		self.biasH	 = np.zeros( sizeH )
		self.biasV	 = np.zeros( sizeV )

	def getErrorRate(self, samples, reconstructions):
		'''returns mean square error'''
		return np.mean( np.square( samples - reconstructions ) )

	def trainEpoch(self, training_samples, learn_rate, cd_steps, batch_size):
		error	   = 0.0
		num_rows   = training_samples.shape[ 0 ]
		# Iterate over each training batch:
		for bstart in range( 0, num_rows, batch_size ):
			# Compute batch stop index:
			bstop = min( bstart + batch_size, num_rows )
			# Compute batch size:
			bsize = bstop - bstart
			# Compute batch multiplier:
			bmult = learn_rate * ( 1.0 / float( bsize ) )
			# Slice data:
			bsamples = training_samples[ bstart:bstop, : ]
			# Get hidden activations and samples:
			aH_0, sH_0 = self.getHiddenSample( bsamples )
			# Perform each contrastive divergence step:
			for i in range( cd_steps ):
				aV_inf, sV_inf, aH_inf, sH_inf = self.getGibbsHvh( ( sH_0 if i == 0 else sH_inf ) )
			# Update weights:
			self.weights += bmult * ( np.dot( bsamples.T, aH_0 ) - np.dot( sV_inf.T, aH_inf ) )
			# Update biases:
			self.biasV	 += bmult * np.mean( bsamples - sV_inf, axis = 0 )
			self.biasH	 += bmult * np.mean( aH_0 - aH_inf, axis = 0 )
			# Scale batch error and accumulate total:
			error += self.getErrorRate( bsamples, sV_inf ) * ( float( bsize ) / float( num_rows ) )
		# Return training error:
		return error

	def train(self, training_samples, validation_samples, learn_rate, cd_steps, epochs, batch_size = 10, report_freq = 10, report_buff = 100):
		# Setup error reporter:
		error_reporter = MnistUnsupervisedReporter( self.name, report_freq, report_buff )
		# Perform each training epoch:
		for epoch in range( epochs ):
			training_error = self.trainEpoch( training_samples, learn_rate, cd_steps, batch_size )
			# Report error, if applicable:
			if ( epoch + 1 ) % report_freq == 0:
				# Compute validation error:
				val_aH, val_sH, val_aV, val_sV = self.getGibbsVhv( validation_samples )
				validation_error = self.getErrorRate( validation_samples, val_sV )
				# Update error reporter:
				error_reporter.update( epoch, training_error, validation_error, validation_samples, val_sV )
		# Save final training visualization to image:
		error_reporter.saveImage( 'report_' + str(time.time()) + '_' + self.name + '_training.png' )

	def getHiddenActivations(self, inputV):
		return Activation.sigmoid( np.dot( inputV, self.weights ) + self.biasH )

	def getVisibleActivations(self, inputH):
		return Activation.sigmoid( np.dot( inputH, self.weights.T ) + self.biasV )

	def getHiddenSample(self, inputV, force_binomial = False):
		aH = self.getHiddenActivations( inputV )
		return [ aH, self.getSample( aH ) if ( force_binomial or not self.is_crbm ) else aH ]

	def getVisibleSample(self, inputH, force_binomial = False):
		aV = self.getVisibleActivations( inputH )
		return [ aV, self.getSample( aV ) if ( force_binomial or not self.is_crbm ) else aV ]

	def getGibbsHvh(self, inputH, force_binomial = False):
		aV, sV = self.getVisibleSample( inputH, force_binomial )
		aH, sH = self.getHiddenSample( sV, force_binomial )
		return [ aV, sV, aH, sH ]

	def getGibbsVhv(self, inputV, force_binomial = False):
		aH, sH = self.getHiddenSample( inputV, force_binomial )
		aV, sV = self.getVisibleSample( sH, force_binomial )
		return [ aH, sH, aV, sV ]

	@staticmethod
	def getSample(activations):
		return np.random.binomial( 1, activations, activations.shape )


text:
# -*- coding: utf-8 -*

""" LAST MODIFIED: December 9th, 2019 """

"""
Development of a Machine Vision Based Yield Monitor for Shallot Onions and Carrot crops
Precision Agriculture and Sensor Systems (PASS) Research Group
McGill University, Department of Bioresource Engineering

preprocess_image.py --- This is a sub file from the yield monitoring program for the masters thesis of
Amanda Boatswain Jacques. This file preprocesses the given images using color thresholding and watershed
segmentation. It then draws circles around the detected vegetables classifying them by size,
and then retrieves the counts for all size categories.
"""

# import the necessary python libraries
from operator import xor
import numpy as np
import time
import cv2
import math
from skimage.feature import peak_local_max
from skimage.morphology import watershed
from scipy import ndimage

""" Define variables and Functions """
# CONSTANTS
# color boundaries
LowerLowerRed = np.array([0, 0, 180])
LowerUpperRed = np.array([10, 45, 235])
UpperLowerRed = np.array([150, 20, 205])
UpperUpperRed = np.array([170, 45, 235])

LowerBlue = np.array([119, 29, 45])
UpperBlue = np.array([130, 55, 75])

LowerTeal = np.array([100, 75, 45])
UpperTeal = np.array([119, 135, 60])

""" OLD VALUES
BrightRedLower = np.array([0, 0, 180])
BrightRedUpper = np.array([12, 60, 255])

BrightPinkLower = np.array([130, 20, 200])
BrightPinkUpper = np.array([170, 60, 255])
"""

BrightRedLower = np.array([0, 0, 190])
BrightRedUpper = np.array([12, 70, 255])

BrightPinkLower = np.array([120, 20, 175])
BrightPinkUpper = np.array([170, 60, 255])

upper_white = np.array([180, 15, 255], dtype ="uint8")
lower_white = np.array([0, 0, 220], dtype = "uint8")

# Morphological operations
# creates an elliptical structuring element for the opening/closing operations
ellipse_kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (9, 9))
kernel = np.ones((3,3),np.uint8)

# criteria for KMeans clustering
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0)
K = 5
attempts=10

# helper functions
def ellipse_perimeter(major_axis, minor_axis):
    a = major_axis/2
    b = minor_axis/2
    h = math.pow((a-b), 2)/math.pow((a+b), 2)
    perimeter = math.pi*(a+b)*(1+ 3*h/(10 + math.sqrt(4-3*h)))
    return perimeter

# solve the derived quadratic equation
def calculate_mass(diameter):
    a = 0.0222
    b = 1.1239
    c = -36.483
    mass = a*diameter*diameter + b*diameter + c
    #print("mass: ", mass)
    return mass

def auto_canny(image, sigma=0.60):
    # compute the median of the single channel pixel intensities
    v = np.median(image)
    # apply automatic Canny edge detection using the computed median
    lower = int(max(0, (1.0 - sigma) * v))
    upper = int(min(255, (1.0 + sigma) * v))
    edged = cv2.Canny(image, lower, upper)
    # return the edged image
    return edged

def resize_image(image, pixel_width = 640):
    # resize the image
    r = pixel_width/image.shape[1]
    new_dim = (pixel_width, int(image.shape[0]*r))
    image = cv2.resize(image, new_dim, interpolation = cv2.INTER_AREA)

    return image

# create a function for drawing the KMEANS_PP_CENTERS
def draw_canvas(centers):
    # draw a blank canvas that can fit 6 circles
    canvas = np.zeros((400, 800,3), dtype = "uint8")
    centers = np.uint8(centers)

    # create a list of corners
    topLeftCorners = [(0,0), (200,0), (400,0), (600, 0), (0,200), (200,200), (400,200), (600, 200)]
    bottomRightCorners = [(200,200), (400,200), (600,200),(800, 200), (200,400), (400,400), (600,400), (800, 400)]

    # draw each color in the canvas:
    for i in range(len(centers)):
        color = (int(centers[i][0]), int(centers[i][1]), int(centers[i][2]))
        cv2.rectangle(canvas, topLeftCorners[i], bottomRightCorners[i], color,-1)
        cv2.putText(canvas, str(color), (topLeftCorners[i][0]+50,topLeftCorners[i][1]+100), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,0), 2)

    return canvas

def preprocess_watershed(image, pixel_metric):
    # create a copy of the image to draw on
    img = image.copy()
    img = resize_image(img)
    clone = img.copy()
    #cv2.imshow("Original", clone)
    wts_result = np.zeros(image.shape, dtype=np.uint8)
    img = cv2.GaussianBlur(img, (3,3),0)
    img = cv2.GaussianBlur(img, (3,3),0)
    img = cv2.medianBlur(img, 3)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    #cv2.imshow("Original Image", img)

    # Apply K-Means clustering to the image
    # From analysis it looks like there are 5-6 main color clusters
    # Flatten the image
    vectorized = img.reshape((-1,3))
    vectorized = np.float32(vectorized)
    ret,label,center=cv2.kmeans(vectorized,K,None,criteria,attempts,cv2.KMEANS_PP_CENTERS)
    center = np.uint8(center)

    color_canvas = draw_canvas(center)
    cv2.imshow("Clustered Centers", color_canvas)

    #print("Detected centers:", center)
    res = center[label.flatten()]
    clustered_image = res.reshape((img.shape))
    cv2.imshow("Clustered Image", clustered_image)
    # For each different image group, determine the dominant color
    result = np.ones(clustered_image.shape[0:2], dtype="uint8")*255 # create a blank canvas
    valid_masks = []

    for color in center:
        if (((BrightRedLower[0] <= color[0] <= BrightRedUpper[0])
            and (BrightRedLower[1] <= color[1] <= BrightRedUpper[1])
            and (BrightRedLower[2] <= color[2] <= BrightRedUpper[2])) or
            ((BrightPinkLower[0] <= color[0] <= BrightPinkUpper[0])
            and (BrightPinkLower[1] <= color[1] <= BrightPinkUpper[1])
            and (BrightPinkLower[2] <= color[2] <= BrightPinkUpper[2]))):

            new_mask = np.where(clustered_image[:,:,0]==color[0], result.copy(), 0)
            # save the valid masks to the new array
            valid_masks.append(new_mask)

    #print("Number of masks found: ", len(valid_masks))

    if not valid_masks:
        # Finding sure foreground area
        #thresh = cv2.bitwise_not(result.copy())
        thresh = cv2.bitwise_not(result.copy())

    else:
        final_result = sum(valid_masks)
        #final_result = cv2.bitwise_not(final_result)
        #cv2.imshow("Final Result Mask", final_result)
        # apply some morphological operations to the mask
        eroded = cv2.erode(final_result.copy(), ellipse_kernel, iterations=2)
        #cv2.imshow("Eroded Image", eroded)
        dilated = cv2.dilate(eroded, ellipse_kernel, iterations=2)
        #cv2.imshow("Dilated Image", dilated)
        #second_eroded = cv2.erode(dilated.copy(), None, iterations=3)
        #cv2.imshow("Second Eroded Image", second_eroded)
        thresh = dilated.copy()

    cv2.imshow("Thresh", thresh)

    dist_transform = ndimage.distance_transform_edt(thresh)
    #print("dist_transform", dist_transform)

    dist = cv2.normalize(dist_transform, None, 255,0, cv2.NORM_MINMAX, cv2.CV_8UC1)
    cv2.imshow("Distance Transform", dist)

    # compute the exact Euclidean distance from every binary
    # pixel to the nearest zero pixel, then find peaks in this distance map
    localMax = peak_local_max(dist_transform, indices=False, min_distance=30,
                              labels=thresh)
    # perform a connected component analysis on the local peaks,
    # using 8-connectivity, then appy the Watershed algorithm
    markers = ndimage.label(localMax, structure=np.ones((3, 3)), output = "uint8")[0]
    labels = watershed(-dist_transform, markers, mask=thresh)
    #cv2.imshow("Labelled Regions", markers[:,:])

    # loop over the unique labels returned by the Watershed algorithm
    total_extra_small = int()
    total_small = int()
    total_medium = int()
    total_large = int()
    total_extra_large = int()

    if (len(np.unique(labels)) - 1) >= 1:
        for label in np.unique(labels):
            # if the label is zero, we are examining the 'background'
            # so simply ignore it
            if label == 0:
                continue
            # otherwise, allocate memory for the label region and draw
            # it on the mask
            mask = np.zeros(eroded.shape, dtype="uint8")
            mask[labels == label] = 255

            #cv2.imshow("Watershed Mask", mask)
            #cv2.waitKey(300)

            extra_small, small, medium, large, extra_large, wts_result = find_onion_contours(mask, clone, pixel_metric)

            clone = wts_result.copy()

            total_extra_small = total_extra_small + extra_small
            total_small = total_small + small
            total_medium = total_medium + medium
            total_large = total_large + large
            total_extra_large = total_extra_large + extra_large

    if not wts_result.any():
        # if there is no watershed result, just return the original image
        wts_result = clone.copy()

    return (total_extra_small, total_small, total_medium, total_large, total_extra_large), wts_result

def find_onion_contours(mask, original, pixel_metric = 1.0):
    # initialize lists of all the onion types present in the image
    extra_small_onions = []
    small_onions = []
    medium_onions = []
    large_onions = []
    extra_large_onions = []
    result = original.copy()

    # find the contours in the image
    try:
        contours, hierarchy = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

        for c in contours:
            moments = cv2.moments(c)
            #print(len(c))

            if moments['m00'] != 0.0:
                cx = int(moments['m10']/moments['m00'])
                cy = int(moments['m01']/moments['m00'])
                centroid = (int(cx), int(cy))

                if len(c) > 5:
                    # check to se if the contour is long enough to be approximated
                    # by a circle
                    ((x, y), radius) = cv2.minEnclosingCircle(c)
                    diameter = radius*2

                    # filter the shapes by circularity
                    perimeter = cv2.arcLength(c, True)
                    area = cv2.contourArea(c)
                    circularity = 4*np.pi*(area/(perimeter*perimeter))
                    ratio = area/(np.pi*(radius*radius))
                    #print("ratio: ", ratio)
                    #print("radius: ", radius)
                    #print("circularity :", circularity)
                    #print(width)

                    width = diameter/pixel_metric
                    #print("width in mm: ", width)
                    mass = calculate_mass(width)
                    #print("Estimated mass: ", mass)

                    #if (ratio > 0.40) and (circularity > 0.60):

                    # filter the onions by size
                    if (24 <=  width < 44.45): # extra small onion
                        color = (255, 0, 0) # Blue
                        extra_small_onions.append(c)

                    if (44.45 <= width <= 46.0375): # small onion
                        small_onions.append(c)
                        color = (255, 255, 0) # Light Blue

                    if (46.0375 < width <= 47.625): # medium Onion
                        medium_onions.append(c)
                        color = (0, 255, 0) # Green

                    if (47.625 < width <= 50.80):
                        large_onions.append(c)
                        color = (0, 128, 255) # Orange

                    if (50.80 < width <= 100): # large Onion
                        extra_large_onions.append(c)
                        color = (0, 0, 255) # Red

                    if any((extra_small_onions, small_onions, medium_onions, large_onions, extra_large_onions)):
                        if  width >= 25:
                            # draw the centroid of the circle as well as the onion border
                            cv2.circle(result, centroid, 3, (255, 255, 0), -1)
                            cv2.circle(result, (int(x), int(y)), int(radius), color, 2)
                            cv2.putText(result, format(width,".3f"), (int(x+10), int(y+10)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,0), 2)

    except Exception as e:
        # if there are no contours, do nothing
        print(e)
        pass

    # return the counts for each onion category
    return len(extra_small_onions), len(small_onions), len(medium_onions), len(large_onions), len(extra_large_onions), result

reason: import numpy,from scipy


text:
"""Routine to solve the Poisson equation."""

import numpy
import scipy.sparse as sps
from scipy.sparse.linalg.dsolve import linsolve
from scipy.sparse import spdiags, csr_matrix
from scipy.sparse import linalg as sla

def solve_stub(grid, ivar, rvar, options):
    """Stub for poisson solver.

    Arguments
    ---------
    grid : Grid object
        Grid containing data.
    ivar : string
        Name of the grid variable of the numerical solution.
    rvar : string
        Name of the grid variable of the right-hand side.
    options : dictionary

    Returns
    -------
    ites: None
        Number of iterations computed.
    residual: None
        Final residual.
    """

    return None, None

def solve_cg(grid, ivar, rvar, options):
    """Solve the Poisson system using a conjugate-gradient method.

    Arguments
    ---------
    grid : Grid object
        Grid containing data.
    ivar : string
        Name of the grid variable of the numerical solution.
    rvar : string
        Name of the grid variable of the right-hand side.

    options: dictionary

    Returns
    -------
    ites: integer
        Number of iterations computed.
    residual: float
        Final residual.
    """

    verbose = options['verbose']
    maxiter = options['maxiter']
    tol = options['tol']

    def A(p):
        return ((p[1:-1, :-2] - 2 * p[1:-1, 1:-1] + p[1:-1, 2:]) / dx**2 +
                (p[:-2, 1:-1] - 2 * p[1:-1, 1:-1] + p[2:, 1:-1]) / dy**2)

    def fill_guard_cells_neumann(x, bc_val, dx, dy):
        x[:, 0] = bc_val * dx + x[:, 1]
        x[:, -1] = bc_val * dx + x[:, -2]
        x[0, :] = bc_val * dy + x[1, :]
        x[-1, :] = bc_val * dy + x[-2, :]

    p = grid[ivar][0,0,:,:]  # initial guess
    b = grid[rvar][0,0,:,:]  # RHS of the system
    dx, dy = grid.dx, grid.dy  # cell widths

    r = b[1:-1, 1:-1] - A(p)  # initial residuals
    rk_norm = numpy.sum(r * r)  # inner product
    d = numpy.zeros_like(p)  # search direction
    d[1:-1, 1:-1] = r  # set direction to initial residual

    # Check for Neumann boundary conditions.
    bc_type, bc_val = grid.bc_type[ivar][0], grid.bc_val[ivar][0]
    if bc_type == 'neumann':
        # Apply Neumann boundary conditions to search direction.
        fill_guard_cells_neumann(d, bc_val, dx, dy)

    ites = 0  # iteration index
    res = rk_norm  # initial residual
    while ites < maxiter and res > tol:
        Ad = A(d)
        alpha = rk_norm / numpy.sum(d[1:-1, 1:-1] * Ad)  # step size
        p[1:-1, 1:-1] += alpha * d[1:-1, 1:-1]  # update solution
        r -= alpha * Ad  # update residuals
        r_norm = numpy.sum(r * r)  # inner product
        beta = r_norm / rk_norm
        rk_norm = r_norm
        d[1:-1, 1:-1] = r + beta * d[1:-1, 1:-1]  # update search direction
        if bc_type == 'neumann':
            fill_guard_cells_neumann(d, bc_val, dx, dy)
        res = r_norm
        ites += 1

    grid.fill_guard_cells(ivar)

    if verbose:
        print('CG method:')
        if ites == maxiter:
            print('Warning: maximum number of iterations reached!')
        print('- Number of iterations: {}'.format(ites))
        print('- Final residual: {}'.format(res))

    return ites, res

def solve_direct(grid, ivar, rvar, options):

    """Solve the Poisson system using a direct solver from the scipy library.

    Arguments
    ---------
    grid : Grid object
        Grid containing data.
    ivar : string
        Name of the grid variable of the numerical solution.
    rvar : string
        Name of the grid variable of the right-hand side.

    options : dictionary

    """
    verbose = options['verbose']
    matrix = options['matrix']

    dx, dy = grid.dx, grid.dy
    nx, ny = grid.nx, grid.ny

    rhs = grid[rvar][0,0,:,:]
    phi = grid[ivar][0,0,:,:]

    sol = linsolve.spsolve(matrix, rhs[1:-1, 1:-1].flatten())
    residual = numpy.linalg.norm(matrix * sol - rhs[1:-1, 1:-1].flatten())

    phi[1:-1,1:-1] = numpy.reshape(sol,(ny,nx))
    grid.fill_guard_cells(ivar)

    if verbose:
        print('Direct Solver:')
        print('- Final residual: {}'.format(residual))

    return None, residual

def solve_jacobi(grid, ivar, rvar, options):
    """Solve the Poisson system using a Jacobi method.

    Arguments
    ---------
    grid : Grid object
        Grid containing data.
    ivar : string
        Name of the grid variable of the numerical solution.
    rvar : string
        Name of the grid variable of the right-hand side.
    options : dictionary

    Returns
    -------
    ites: integer
        Number of iterations computed.
    residual: float
        Final residual.
    """

    maxiter = options['maxiter']
    tol = options['tol']
    verbose = options['verbose']

    phi = grid[ivar][0,0,:,:]
    b = grid[rvar][0,0,:,:]
    dx, dy = grid.dx, grid.dy

    ites = 0
    residual = tol + 1.0
    while ites < maxiter and residual > tol:
        phi_old = numpy.copy(phi)  # previous solution
        phi[1:-1, 1:-1] = (((phi_old[:-2, 1:-1] +
                             phi_old[2:,  1:-1]) * dy**2 +
                            (phi_old[1:-1, :-2] +
                             phi_old[1:-1,  2:]) * dx**2 -
                            b[1:-1, 1:-1] * dx**2 * dy**2) /
                           (2 * (dx**2 + dy**2)))

        grid.fill_guard_cells(ivar)

        residual = (numpy.sqrt(numpy.sum((phi - phi_old)**2) /
                    ((grid.nx + 2) * (grid.ny + 2))))
        ites += 1

    if verbose:
        print('Jacobi method:')
        if ites == maxiter:
            print('Warning: maximum number of iterations reached!')
        print('- Number of iterations: {}'.format(ites))
        print('- Final residual: {}'.format(residual))

    return ites, residual

def solve_superlu(grid, ivar, rvar, options):
    """Solve the Poisson system using a direct solver from the scipy library.

    Arguments
    ---------
    grid : Grid object
        Grid containing data.
    ivar : string
        Name of the grid variable of the numerical solution.
    rvar : string
        Name of the grid variable of the right-hand side.

    options : dictionary

    """

    verbose = options['verbose']
    matrix = options['matrix']
    lu = options['lu']

    dx, dy = grid.dx, grid.dy
    nx, ny = grid.nx, grid.ny

    rhs = grid[rvar][0,0,:,:]
    phi = grid[ivar][0,0,:,:]

    sol = lu.solve(rhs[1:-1, 1:-1].flatten())
    residual = numpy.linalg.norm(matrix * sol - rhs[1:-1, 1:-1].flatten())

    phi[1:-1,1:-1] = numpy.reshape(sol,(ny,nx))
    grid.fill_guard_cells(ivar)

    if verbose:
        print('LU Decomposition:')
        print('- Final residual: {}'.format(residual))

    return None, residual

max_stars_repo_name: akashdhruv/flowX

max_issues_repo_name: akashdhruv/flowX

max_forks_repo_name: akashdhruv/flowX

reason: import numpy,import scipy,from scipy


text:
import torch
import torch.nn as nn
import numpy as np

class Predictor(nn.Module):

    def __init__(self, input_size, output_size, params):

        super(Predictor, self).__init__()

        self.fc1 = nn.Linear(input_size, params)
        self.fc2 = nn.Linear(params, params)
        self.fc3 = nn.Linear(params, output_size)

        self.relu = nn.ReLU()

        self.init_weights()

    def forward(self, X):

        X = self.relu(self.fc1(X))
        X = self.relu(self.fc2(X))
        X = self.fc3(X)
        return X

    def init_weights(self):

        for m in self.modules():
            if isinstance(m, nn.Linear):
                torch.nn.init.normal_(m.weight, std=1e-2)
                if m.bias is not None:
                    torch.nn.init.constant_(m.bias, 0)

def DH(theta, d, r, alpha, device):

    """
     Calculates the Denavit-Hartenberg Matrix
     where
     d: offset along previous z to the common normal
     theta: angle about previous z, from old x to new x
     r: length of the common normal (aka a, but if using this notation, do not confuse with alpha).
     Assuming a revolute joint, this is the radius about previous z.
     alpha: angle about common normal, from old z axis to new z axis
    """

    T = torch.zeros([theta.shape[0], 4, 4]).to(device)
    T[:, :, :] = torch.eye(4).to(device)

    cTheta = torch.cos(theta)
    sTheta = torch.sin(theta)
    calpha = torch.cos(alpha)
    salpha = torch.sin(alpha)

    T[:, 0, 0] = cTheta
    T[:, 0, 1] = -sTheta
    T[:, 0, 2] = 0.0
    T[:, 0, 3] = r

    T[:, 1, 0] = sTheta * calpha
    T[:, 1, 1] = cTheta * calpha
    T[:, 1, 2] = -salpha
    T[:, 1, 3] = - d * salpha

    T[:, 2, 0] = sTheta * salpha
    T[:, 2, 1] = cTheta * salpha
    T[:, 2, 2] = calpha
    T[:, 2, 3] = d * calpha

    return T

def end_effector_pose(thetas, device):

    alphas = torch.Tensor([0, -np.pi/2, np.pi/2, np.pi/2, -np.pi/2, np.pi/2, np.pi/2, 0]).to(device)
    ds = torch.Tensor([0.333, 0, 0.316, 0, 0.384, 0, 0, 0.107]).to(device)
    rs = torch.Tensor([0, 0, 0, 0.0825, -0.0825, 0, 0.088, 0]).to(device)

    T = torch.zeros([thetas.shape[0], 4, 4]).to(device)
    T[:, :, :] = torch.eye(4).to(device)

    # Base link coordinates
    T[:, 0, 3] = 0.0
    T[:, 1, 3] = 0.0
    T[:, 2, 3] = 0.0

    for idx in range(0, len(alphas) - 1):
        T_i = DH(thetas[:, idx], ds[idx], rs[idx], alphas[idx], device)
        T = T.bmm(T_i)

    T_i = DH(torch.zeros(thetas.shape[0]).to(device), ds[-1], rs[-1], alphas[-1], device)
    T = T.bmm(T_i)

    return torch.stack((T[:, 0, 3], T[:, 1, 3]), 1)

max_issues_repo_name: gamleksi/AffordanceGym

max_forks_repo_name: gamleksi/AffordanceGym


text:
import json
import logging
import os
import joblib
import pytest
import prediction_service
from prediction_service.prediction import form_response, api_response
import category_encoders as ce
import scipy.stats as stat
import category_encoders as ce
import scipy.stats as stat
from src import categorical_encoding
from src import numerical_scaling

"""errors
FAILED tests/test_config.py::test_form_response_correct_range - AttributeError: 'NoneType' object has no attr...
FAILED tests/test_config.py::test_api_response_correct_range - AssertionError: assert ({'response': 'Not in c...
FAILED tests/test_config.py::test_api_response_incorrect_col - AssertionError: assert 'Values enter...xpected...

"""

input_data = {"num_incorrect_range":
        {"age":14,
        "fnlwgt":10000,
        "education-num":20,
        "capital-gain":150000,
        "capital-loss":6000,
        "hours-per-week":120},

    "num_correct_range":
    {"age":20,
    "fnlwgt":20000,
    "education-num":10,
    "capital-gain":50000,
    "capital-loss":2000,
    "hours-per-week":45},

    "cat_correct_val":
    {
    "workclass": " Private",
    "education": " HS-grad",
    "marital-status": " Never-married",
    "occupation": " Prof-specialty ",
    "relationship": " Husband",
    "race": " White",
    "sex": " Male",
    "country": " United-States"
    },

    "incorrect_col":
    {'age': 5,
    'fnlwgt': 5,
    'education num': 5,
    'capital gain': 5,
    'capital loss': 5,
    'hours-per-week': 5,
    'workclass': "some_string",
    'education': "some_string",
    'marital status': "some_string",
    'occupation': "some_string",
    'relationship': "some_string",
    'race': "some_string",
    'sex': "some_string",
    'country': "some_string"
}
}

TARGET_value = {
    "min": 0,
    "max": 1
}

# e
def test_form_response_correct_range(data=input_data["num_correct_range"].update(input_data['cat_correct_val'])):
    res = form_response(data)
    assert (res==TARGET_value["min"] or res==TARGET_value["max"])

# e
def test_api_response_correct_range(data=input_data["num_correct_range"]):
    res = api_response(data)
    assert (res==TARGET_value["min"] or res==TARGET_value["max"])

def test_form_response_incorrect_range(data=input_data["num_incorrect_range"]):
    with pytest.raises(prediction_service.prediction.NotInRange):
        res=form_response(data)

def test_api_response_incorrect_range(data=input_data["num_incorrect_range"]):
    res = api_response(data)
    assert res["response"] == prediction_service.prediction.NotInRange().message
# e
def test_api_response_incorrect_col(data=input_data["incorrect_col"]):
    res = api_response(data)
    assert res["response"] == prediction_service.prediction.NotInCols().message

reason: import scipy


text:
import torch
import cv2
import numpy as np
from torch.nn import functional as F

#
def add_prediction_to_image_base(image,regression_point,target_point):
    image = image.squeeze(0)
    image = image.permute(1,2,0)
    image = image.cpu().data.numpy()
    image = image * 255.
    image_size = 512

    regression_point = regression_point.cpu().data.numpy()
    target_point = target_point.cpu().data.numpy()
    dist = np.sqrt(np.power((regression_point[0][0] * image_size - target_point[0][0] * image_size), 2) + np.power((regression_point[0][1] * image_size - target_point[0][1] * image_size), 2))

    image = cv2.circle(image, (int(regression_point[0][0] * image_size), int(regression_point[0][1] * image_size)), 5, (255, 0, 0),-1)
    image = cv2.circle(image, (int(target_point[0][0] * image_size), int(target_point[0][1] * image_size)), 5, (0, 0, 255), -1)
    cv2.putText(image, "DIST:" + str(round(dist, 4)), (5, 25), cv2.FONT_HERSHEY_PLAIN, 2.0, (255, 255, 255), 2)
    image = torch.from_numpy(image)
    image = image.permute(2, 0, 1).unsqueeze(0)
    image = image / 255.

    return image

def add_prediction_to_image_batch(images,prediction_points,target_points):
    image_sp = torch.split(images, split_size_or_sections=1, dim=0)
    prediction_point_sp = torch.split(prediction_points, split_size_or_sections=1, dim=0)
    target_point_sp = torch.split(target_points, split_size_or_sections=1, dim=0)

    return_sp = []
    for i in range(len(image_sp)):
        return_sp.append(add_prediction_to_image_base(image_sp[i], prediction_point_sp[i], target_point_sp[i]))

    return torch.cat(return_sp, dim=0)

def GetCenterfromHeatMap(heatmap):
    width,height = heatmap.size()[2:4]
    _,pos = F.max_pool2d(heatmap,(width,height),return_indices=True)
    pos = pos.squeeze(-1).squeeze(-1).cpu().data.numpy()
    pos = np.concatenate([((pos+1)%width),((pos+1)//height+1)],axis=1)

    return pos/width

max_stars_repo_name: chao-tan/TCLNet

max_issues_repo_name: chao-tan/TCLNet

max_forks_repo_name: chao-tan/TCLNet


text:
from napari_plugin_engine import napari_hook_implementation
from magicgui import magic_factory
import numpy as np
import itk
import checkerboard.utils as utils
from itk_napari_conversion import image_from_image_layer
from itk_napari_conversion import image_layer_from_image

def on_init(widget):
    """
    Initializes widget layout.
    """
    widget.native.setStyleSheet("QWidget{font-size: 12pt;}")

@magic_factory(widget_init=on_init, layout='vertical',
               call_button="create checkerboard",
               pattern={"max": 20, "step": 1,
                        "tooltip": "Select the gridsize of the checkerboard"})
def checkerboard(image1: "napari.layers.Image",
                 image2: "napari.layers.Image",
                 pattern: int = 3) -> 'napari.layers.Image':
    """
    Takes user input images and returns a checkerboard blend of the images.
    """
    if image1 is None or image2 is None:
        print("No images selected for registration.")
        return error("No images selected for registration.")

    # Convert image layer to itk_image
    itk_image1 = image_from_image_layer(image1)
    itk_image2 = image_from_image_layer(image2)
    itk_image1 = itk_image1.astype(itk.F)
    itk_image2 = itk_image2.astype(itk.F)

    region_image1 = itk_image1.GetLargestPossibleRegion()
    region_image2 = itk_image2.GetLargestPossibleRegion()
    same_physical_space = \
        np.allclose(np.array(itk_image1.GetOrigin()),
                    np.array(itk_image2.GetOrigin())) and \
        np.allclose(np.array(itk_image1.GetSpacing()),
                    np.array(itk_image2.GetSpacing())) and \
        itk_image1.GetDirection() == itk_image2.GetDirection() and \
        np.allclose(np.array(region_image1.GetIndex()),
                    np.array(region_image2.GetIndex())) and \
        np.allclose(np.array(region_image1.GetSize()), np.array(
                             region_image2.GetSize()))
    if not same_physical_space:
        upsample_image2 = True
        if itk_image1.GetSpacing() != itk_image2.GetSpacing():
            min1 = min(itk_image1.GetSpacing())
            min2 = min(itk_image2.GetSpacing())
            if min2 < min1:
                upsample_image2 = False
        else:
            size1 = max(itk.size(itk_image1))
            size2 = max(itk.size(itk_image1))
            if size2 > size1:
                upsample_image2 = False

        if upsample_image2:
            resampler = itk.ResampleImageFilter.New(itk_image2)
            resampler.UseReferenceImageOn()
            resampler.SetReferenceImage(itk_image1)
            resampler.Update()
            input2 = resampler.GetOutput()
        else:
            resampler = itk.ResampleImageFilter.New(itk_image1)
            resampler.UseReferenceImageOn()
            resampler.SetReferenceImage(itk_image2)
            resampler.Update()
            input1 = resampler.GetOutput()
    else:
        input1 = itk_image1
        input2 = itk_image2

    checkerboard_filter = itk.CheckerBoardImageFilter.New(input1, input2)

    dimension = itk_image1.GetImageDimension()
    checker_pattern = [pattern] * dimension
    checkerboard_filter.SetCheckerPattern(checker_pattern)
    checkerboard_filter.Update()
    checkerboard = checkerboard_filter.GetOutput()

    layer = image_layer_from_image(checkerboard)
    layer.name = "checkerboard " + image1.name + " " + image2.name
    return layer

@napari_hook_implementation
def napari_experimental_provide_dock_widget():
    # you can return either a single widget, or a sequence of widgets
    return checkerboard, {'area': 'bottom'}

max_stars_repo_name: chili-chiu/napari-checkerboard

max_issues_repo_name: chili-chiu/napari-checkerboard

max_forks_repo_name: chili-chiu/napari-checkerboard


text:
# Load module and example file
import pyvista as pv
from pyvista import examples
import numpy as np

# Load example beam grid
grid = pv.UnstructuredGrid(examples.hexbeamfile)

# Create fictitious displacements as a function of Z location
d = np.zeros_like(grid.points)
d[:, 1] = grid.points[:, 2]**3/250

# Displace original grid
grid.points += d


text:
import os
import time
import sys

import h5py
import matplotlib.pyplot as plt
import numpy as np
from tqdm import tqdm

import consts
from common import PatchArray, Visualize, utils, logger

class AnomalyModelBase(object):

    def __init__(self):
        self.NAME = self.__class__.__name__.replace("AnomalyModel", "")
        self.patches = None

    def __generate_model__(self, patches, silent=False):
        """Generate a model based on the features and metadata

        Args:
            patches (PatchArray): Array of patches with features as extracted by a FeatureExtractor

        Returns:
            h5py.group that will be saved to the features file
        """
        raise NotImplementedError()

    def __mahalanobis_distance__(self, patch):
        """Calculate the Mahalanobis distance between the input and the model"""
        raise NotImplementedError()

    def classify(self, patch):
        """Classify a single feature based on the loaded model

        Args:
            patch (np.record): A single patch (with feature)

        Returns:
            A label
        """
        raise NotImplementedError()

    def __save_model_to_file__(self, h5file):
        """ Internal method that should be implemented by subclasses and save the
        necessary information to the file so that the model can be reloaded later
        """
        raise NotImplementedError()

    def __load_model_from_file__(self, h5file):
        """ Internal method that should be implemented by subclasses and load the
        necessary information from the file
        """
        raise NotImplementedError()

    ########################
    # Common functionality #
    ########################

    def load_or_generate(self, patches=consts.FEATURES_FILE,
                               load_patches=False, silent=False):
        """Load a model from file or generate it based on the features

        Args:
            patches (str, PatchArray) : HDF5 file containing features (see feature_extractor for details)
        """

        # Load patches if necessary
        if isinstance(patches, basestring):
            if patches == "" or not os.path.exists(patches) or not os.path.isfile(patches):
                raise ValueError("Specified file does not exist (%s)" % patches)

            # Try loading
            loaded = self.load_from_file(patches, load_patches=load_patches)
            if loaded:
                return True

            # Read file
            if not isinstance(self.patches, PatchArray):
                patches = PatchArray(patches)
        elif isinstance(patches, PatchArray):
            self.patches = patches
            # Try loading
            loaded = self.load_from_file(patches.filename, load_patches=load_patches)
            if loaded:
                return True
        else:
            raise ValueError("patches must be a path to a file or a PatchArray.")

        assert patches.contains_features, "patches must contain features to calculate an anomaly model."

        self.patches = patches

        model_input = self.filter_training(patches)

        start = time.time()

        # Generate model
        if self.__generate_model__(model_input, silent=silent) == False:
            logger.info("Could not generate model.")
            return False

        end = time.time()

        self.save_to_file(model_input.size, start, end)

        self.calculate_mahalanobis_distances()

        return True

    def filter_training(self, patches):
        return patches.training

    def is_in_file(self, model_file):
        """ Check if model and mahalanobis distances are already in model_file """
        with h5py.File(model_file, "r") as hf:
            g = hf.get(self.NAME)

            if g is None:
                return (False, False)
            else:
                return (True, "mahalanobis_distances" in g.keys())

    def save_to_file(self, num_features=0, start=None, end=None):
        logger.info("Writing model to: %s" % self.patches.filename)
        with h5py.File(self.patches.filename, "a") as hf:
            g = hf.get(self.NAME)

            if g is not None:
                del hf[self.NAME]

            g = hf.create_group(self.NAME)

            # Add metadata to the output file
            g.attrs["Number of features used"] = num_features

            computer_info = utils.getComputerInfo()
            for key, value in computer_info.items():
                g.attrs[key] = value

            if start is not None and end is not None:
                g.attrs["Start"] = start
                g.attrs["End"] = end
                g.attrs["Duration"] = end - start
                g.attrs["Duration (formatted)"] = utils.format_duration(end - start)

            self.__save_model_to_file__(g)
        logger.info("Successfully written model to: %s" % self.patches.filename)

    def load_from_file(self, model_file, load_patches=False):
        """ Load a model from file """
        with h5py.File(model_file, "r") as hf:
            g = hf.get(self.NAME)

            if g is None:
                return False

            logger.info("Reading model from: %s" % model_file)

            if load_patches:
                self.patches = PatchArray(model_file)

            return self.__load_model_from_file__(g)

    def calculate_mahalanobis_distances(self):
        """ Calculate all the Mahalanobis distances and save them to the file """
        with h5py.File(self.patches.filename, "r+") as hf:
            g = hf.get(self.NAME)

            if g is None:
                raise ValueError("The model needs to be saved first")

            maha = np.zeros(self.patches.shape, dtype=np.float64)

            for i in tqdm(np.ndindex(self.patches.shape), desc="Calculating mahalanobis distances", total=self.patches.size, file=sys.stderr):
                maha[i] = self.__mahalanobis_distance__(self.patches[i])

            no_anomaly = maha[self.patches.labels == 1]
            anomaly = maha[self.patches.labels == 2]

            if g.get("mahalanobis_distances") is not None: del g["mahalanobis_distances"]
            m = g.create_dataset("mahalanobis_distances", data=maha)
            m.attrs["max_no_anomaly"] = np.nanmax(no_anomaly) if no_anomaly.size > 0 else np.NaN
            m.attrs["max_anomaly"]    = np.nanmax(anomaly) if anomaly.size > 0 else np.NaN

            logger.info("Saved Mahalanobis distances to file")
            return True

    def visualize(self, **kwargs):
        """ Visualize the result of a anomaly model """

        vis = Visualize(self.patches, **kwargs)

        # if "patch_to_color" not in kwargs:
        #     def patch_to_color(patch):
        #         b = 0#100 if patch in self.normal_distribution else 0
        #         g = 0
        #         threshold = vis.get_trackbar("threshold")
        #         if vis.get_trackbar("show_thresh"):
        #             r = 100 if self.__mahalanobis_distance__(patch) > threshold else 0
        #         elif threshold == 0:
        #             r = 0
        #         else:
        #             r = min(255, int(self.__mahalanobis_distance__(patch) * (255 / threshold)))
        #         return (b, g, r)
        #     kwargs["patch_to_color"] = patch_to_color
        # vis.patch_to_color = kwargs.get("patch_to_color")

        # if "patch_to_text" not in kwargs:
        #     def patch_to_text(patch):
        #         return round(self.__mahalanobis_distance__(patch), 2)
        #     kwargs["patch_to_text"] = patch_to_text
        # vis.patch_to_text = kwargs.get("patch_to_text")

        patch_to_color = kwargs.get("patch_to_color", None)
        if patch_to_color is not None:
            vis.patch_to_color = patch_to_color

        patch_to_text = kwargs.get("patch_to_text", None)
        if patch_to_text is not None:
            vis.patch_to_text = patch_to_text

        click = kwargs.get("click", None)
        if click is not None:
            vis.click = click

        vis.show()


text:
# -*- coding: utf-8 -*-
"""
    Plot a globe mesh with Background plotter and make the mesh shrink over time

    Author: Jari Honkanen

"""

import time
import numpy as np
from threading import Thread
import pyvista as pv
import pyvistaqt as pvqt
from pyvista import examples

if __name__ == "__main__":

    globe = examples.load_globe()
    globe.point_arrays['scalars'] = np.random.rand(globe.n_points)
    globe.set_active_scalars('scalars')

    # Create background plotter
    plotter_bg = pvqt.BackgroundPlotter()
    plotter_bg.add_mesh(globe, scalars='scalars', lighting=False, show_edges=True, texture=True)
    plotter_bg.view_isometric()

    # Change globe size in the background
    def change_size():
        for i in range(50):
            globe.points *= 0.95
            globe.point_arrays['scalars'] = np.random.rand(globe.n_points)
            time.sleep(0.5)

    thread = Thread(target=change_size)
    thread.start()

    plotter_bg.app.exec_()


text:
from scipy import  misc
import numpy as np
import neuralnet as nn
import csv
from random import shuffle
import os

def prep_image(im, size=256, resize=512):
    h, w, _ = im.shape
    if h < w:
        new_sh = (resize, int(w * resize / h))
    else:
        new_sh = (int(h * resize / w), resize)
    im = misc.imresize(im, new_sh, interp='bicubic')

    # Crop random
    im = nn.utils.crop_random(im, size)
    im = im.astype('float32')
    return np.transpose(im[None], (0, 3, 1, 2)) / 255.

def prep_image_test(im, size=512, resize=550):
    im = nn.utils.crop_center(im, [size, size], resize)
    im = im.astype('float32')
    return np.transpose(im[None], (0, 3, 1, 2)) / 255.

class DataManager(nn.DataManager):
    def __init__(self, placeholders, path, bs, n_epochs, shuffle=False, **kwargs):
        super(DataManager, self).__init__(None, placeholders, path=path, batch_size=bs, n_epochs=n_epochs, shuffle=shuffle, **kwargs)
        self.load_data()

    def load_data(self):
        source = os.listdir(self.path[0])
        source = [self.path[0] + '/' + f for f in source]
        if 'val' in self.path[0] or 'test' in self.path[0]:
            source = source[:self.kwargs.get('num_val_imgs', 100)]

        file = open(os.path.join(self.kwargs.get('style_path'), self.path[1]), 'r')
        contents = csv.reader(file, delimiter=',')
        style = [self.kwargs.get('style_path') + '/' + row[0] for row in contents]

        shuffle(source)
        shuffle(style)

        self.dataset = (source, style)
        self.data_size = min(len(source), len(style))

    def generator(self):
        source, style = list(self.dataset[0]), list(self.dataset[1])

        if self.shuffle:
            shuffle(source)
            shuffle(style)

        if len(source) > len(style):
            source = source[:len(style)]
        else:
            style = style[:len(source)]

        for idx in range(0, self.data_size, self.batch_size):
            source_batch = source[idx:idx + self.batch_size]
            style_batch = style[idx:idx + self.batch_size]
            imgs, stys = [], []
            exit = False
            for sou_f, sty_f in zip(source_batch, style_batch):
                try:
                    img, sty = misc.imread(sou_f), misc.imread(sty_f)
                except FileNotFoundError:
                    exit = True
                    break

                if len(img.shape) < 3 or len(sty.shape) < 3:
                    exit = True
                    break

                imgs.append(prep_image_test(img) if 'test' in self.path[0] else prep_image(img))
                stys.append(prep_image_test(sty, 256, 512) if 'test' in self.path[0] else prep_image(sty))

            if exit:
                continue

            imgs = np.concatenate(imgs)
            stys = np.concatenate(stys)
            yield imgs, stys

def get_weights(vgg_info):
    weights = [np.random.uniform(.5, 1.5, size=s).astype('float32') for s in vgg_info]
    return weights

max_stars_repo_name: justanhduc/multar

max_issues_repo_name: justanhduc/multar

max_forks_repo_name: justanhduc/multar


text:
import numpy as np
from PIL import ImageOps
from PIL import Image
import json
import requests
import os

def process_image(image):
    # Resize the image to MNIST size 28 x 28 pixels
    image = image.resize((28, 28))

    # Make the image grayscale
    image = image.convert('L')

    # If the image looks like it's dark-on-white, invert it first
    inverted = image.getpixel((0, 0)) > 192
    if inverted:
        print('Inverting image')
        image = ImageOps.invert(image)

    # Transform the image into a NumPy array
    image_data = np.array(image)

    # Normalize the pixel values from 0-255 to 0.0-1.0
    image_data = image_data / 255.0

    return image_data.reshape((1, 28, 28)), inverted

if __name__ == '__main__':
    image, inverted = process_image(Image.open("five.jpeg"))
    #with open('mnist-input.json', 'w') as f:
    #    json.dump({"instances": image.tolist()},f)
    data = json.dumps({"signature_name": "serving_default", "instances": image.tolist()})
    headers = {"content-type": "application/json","Host":"mnist-example-s3.kserve-deployement.example.com".format(os.environ["SERVICE_HOSTNAME"])}
    json_response = requests.post("http://{}:{}/v1/models/mnist-example-s3:predict".format(os.environ["INGRESS_HOST"],os.environ["INGRESS_PORT"]), data=data, headers=headers)
    predictions = json.loads(json_response.text)['predictions']
    print(predictions)
    print(np.argmax(predictions[0]))

#SERVICE_HOSTNAME=$(kubectl get inferenceservice mnist-example-s3 -n kserve-deployement -o jsonpath='{.status.url}' | cut -d "/" -f 3)
#curl -v -H "Host: ${SERVICE_HOSTNAME}" http://${INGRESS_HOST}:${INGRESS_PORT}/v1/models/mnist-example-s3:predict -d @./mnist-input.json

max_stars_repo_name: safoinme/Mnist-MLFlow-dvc-Argo

max_issues_repo_name: safoinme/mnist-dvc

max_forks_repo_name: safoinme/mnist-dvc


text:
import argparse
import base64
import json
import math
import threading
import time
import zlib

import librosa
import numpy as np
import pyaudio
import requests
from OpenGL.GL import *
from OpenGL.GLU import *
from OpenGL.GLUT import *
from PIL import Image

textures = {}
labels = [
    "unknown",
    "yes",
    "no",
    "up",
    "down",
    "left",
    "right",
    "on",
    "off",
    "stop",
    "go",
]

def load_texture(filename, width, height):
    im = Image.open(filename)
    im.convert("RGBA")
    data = im.getdata()
    pixels = []
    for pixel in reversed(data):
        pixels.extend(pixel)
    pixels = np.array(pixels, np.uint8)
    tex_id = glGenTextures(1)

    glPixelStorei(GL_UNPACK_ALIGNMENT, 1)
    glBindTexture(GL_TEXTURE_2D, tex_id)
    glTexImage2D(
        GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, pixels
    )
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
    return tex_id

def draw_spectrogram(audio_data):
    audio_data = np.frombuffer(audio_data, np.int16).astype(np.float32) / 32768
    spectrogram = np.absolute(np.fft.fft(audio_data)[: len(audio_data) // 2])
    spectrogram = np.power(np.clip(spectrogram, 0, 200) / 200.0, 0.7)
    glColor3f(0.3, 0.3, 0.3)
    h = 7
    s = 4
    for i, energy in enumerate(spectrogram.tolist()):
        glBegin(GL_QUADS)
        glVertex2f(0, i * (h + s))
        glVertex2f(int(energy * 150), i * (h + s))
        glVertex2f(int(energy * 150), i * (h + s) + h)
        glVertex2f(0, i * (h + s) + h)
        glEnd()
        glBegin(GL_QUADS)
        glVertex2f(800, i * (h + s))
        glVertex2f(800 - int(energy * 150), i * (h + s))
        glVertex2f(800 - int(energy * 150), i * (h + s) + h)
        glVertex2f(800, i * (h + s) + h)
        glEnd()

def draw_text(text, x, y):
    m = 0.0385
    glColor3f(0.9, 0.9, 0.9)
    for i, c in enumerate(text):
        idx = ord(c) - ord("a")
        glEnable(GL_TEXTURE_2D)
        glEnable(GL_BLEND)
        glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
        glBindTexture(GL_TEXTURE_2D, textures["font"])
        glBegin(GL_QUADS)
        glTexCoord2f(m * idx, 0)
        glVertex2f(x, y)
        glTexCoord2f(m * idx, 1)
        glVertex2f(x, y + 15)
        glTexCoord2f(m * (idx + 1), 1)
        glVertex2f(x + 10, y + 15)
        glTexCoord2f(m * (idx + 1), 0)
        glVertex2f(x + 10, y)
        x += 10
        glEnd()
        glDisable(GL_TEXTURE_2D)

def create_rot_matrix(rads):
    return np.array(
        [[math.cos(rads), -math.sin(rads)], [math.sin(rads), math.cos(rads)]]
    )

def draw_vertices(vertices):
    for vertex in vertices:
        glVertex2f(*vertex)

class LerpStepper(object):
    def __init__(self, a, b, speed):
        self.last_time = time.time()
        self.val = a
        self.b = b
        self.speed = speed
        self.running = False

    def reset(self, val, b=None):
        self.val = val
        if b:
            self.b = b

    def step(self):
        if self.val < self.b:
            self.val += self.speed

class Indicator(object):
    indicators = []

    def __init__(self, text, midpoint, index, radius=250, n_slices=len(labels)):
        self.text = text
        self._state = 0.0
        self.midpoint = midpoint
        self.radius = radius
        self.index = index
        self.color = (0.4, 0.4, 0.4)
        self._color_lerp = LerpStepper(1.0, 1.0, 0.01)
        Indicator.indicators.append(self)

        slice_rads = (2 * math.pi) / n_slices
        rads = index * slice_rads
        self._rotmat1 = create_rot_matrix(rads)
        rads = (index + 0.5) * slice_rads
        self._rotmat15 = create_rot_matrix(rads)
        rads = (index + 1) * slice_rads
        self._rotmat2 = create_rot_matrix(rads)
        self._init_shape()

    def highlight(self, intensity):
        self._color_lerp.reset(1 - intensity)

    def _init_shape(self):
        bp = np.array([0, self.radius])
        bp2 = np.array([0, self.radius * 0.8])
        m = np.array(self.midpoint)
        p1 = np.matmul(self._rotmat1, bp) + m
        p2 = np.matmul(self._rotmat2, bp) + m
        self.text_pos = np.matmul(self._rotmat15, bp2) + m
        self.text_pos[0] -= len(self.text) * 5
        p1 = p1.tolist()
        p2 = p2.tolist()
        self.vertices = [self.midpoint, p1, p2, self.midpoint]

    def tick(self):
        self._color_lerp.step()

    def draw(self):
        draw_text(self.text, self.text_pos[0], self.text_pos[1])
        glEnable(GL_LINE_SMOOTH)
        glLineWidth(2)
        glBegin(GL_LINE_STRIP)
        color = [0.4 + min(1 - self._color_lerp.val, 0.6)] * 3
        glColor3f(*color)
        draw_vertices(self.vertices)
        glEnd()
        glDisable(GL_LINE_SMOOTH)

    def highlight(self, intensity):
        self._color_lerp.reset(1 - intensity)

class LabelClient(object):
    def __init__(self, server_endpoint):
        self.endpoint = server_endpoint
        self.chunk_size = 1000
        self._audio = pyaudio.PyAudio()
        self._audio.open(
            format=pyaudio.paInt16,
            channels=1,
            rate=16000,
            input=True,
            frames_per_buffer=self.chunk_size,
            stream_callback=self._on_audio,
        )
        self.last_data = np.zeros(1000)
        self._audio_buf = []

    def _on_audio(self, in_data, frame_count, time_info, status):
        data_ok = (in_data, pyaudio.paContinue)
        self.last_data = in_data
        self._audio_buf.append(in_data)
        if len(self._audio_buf) != 16:
            return data_ok
        audio_data = base64.b64encode(zlib.compress(b"".join(self._audio_buf)))
        self._audio_buf = []
        response = requests.post(
            "{}/listen".format(self.endpoint),
            json=dict(wav_data=audio_data.decode(), method="all_label"),
        )
        response = json.loads(response.content.decode())
        if not response:
            return data_ok
        max_key = max(response.items(), key=lambda x: x[1])[0]
        for key in response:
            p = response[key]
            if p < 0.5 and key != "__unknown__":
                continue
            key = key.replace("_", "")
            try:
                Indicator.indicators[labels.index(key)].highlight(1.0)
            except ValueError:
                continue
        return data_ok

class DemoApplication(object):
    def __init__(self, label_client):
        glutInit()
        self.width, self.height = 800, 600
        glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_ALPHA | GLUT_DEPTH)
        glutInitWindowSize(self.width, self.height)
        glutInitWindowPosition(100, 100)
        self.window = glutCreateWindow(b"Google Speech Dataset Demo")
        self.label_client = label_client

        glutDisplayFunc(self.draw)
        glutIdleFunc(self.draw)
        glutReshapeFunc(self._on_resize)
        glClearColor(0.12, 0.12, 0.15, 1.0)
        font_path = os.path.join(
            os.path.dirname(os.path.realpath(__file__)), "fonts.png"
        )
        textures["font"] = load_texture(font_path, 208, 15)
        self.children = [
            Indicator(labels[i], [400, 300], i) for i in range(len(labels))
        ]

    def _refresh(self):
        glViewport(0, 0, self.width, self.height)
        glMatrixMode(GL_PROJECTION)
        glLoadIdentity()
        glOrtho(0.0, self.width, 0.0, self.height, 0.0, 1.0)
        glMatrixMode(GL_MODELVIEW)
        glLoadIdentity()

    def _on_resize(self, width, height):
        glutReshapeWindow(self.width, self.height)

    def draw(self):
        self.children = sorted(self.children, key=lambda x: -x._color_lerp.val)
        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
        glLoadIdentity()
        self._refresh()
        self._draw()
        for obj in self.children:
            obj.draw()
        glutSwapBuffers()

    def _tick(self):
        pass

    def tick(self):
        self._tick()
        for obj in self.children:
            obj.tick()

    def _do_tick(self, hz=60):
        delay = 1 / hz
        while True:
            a = time.time()
            self.tick()
            dt = time.time() - a
            time.sleep(max(0, delay - dt))

    def _draw(self):
        draw_spectrogram(self.label_client.last_data)

    def run(self):
        threading.Thread(target=self._do_tick).start()
        glutMainLoop()

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--server-endpoint",
        type=str,
        default="http://127.0.0.1:16888",
        help="The endpoint to use",
    )
    flags = parser.parse_args()
    app = DemoApplication(LabelClient(flags.server_endpoint))
    app.run()

if __name__ == "__main__":
    main()

max_stars_repo_name: abeja-inc/honk

max_issues_repo_name: abeja-inc/honk

max_forks_repo_name: abeja-inc/honk


text:
import math
import numpy as np
import torch
import torch.sparse as sp
import scipy.sparse as ssp

def split_bucketed_data(bin_idcs):
    """
    Splits a given set of samples into the specified number of buckets of
    equal size. Samples are assigned to buckets based on their label. Each
    bucket is split into train, validation, and test set and the overall
    training, validation, and test sets are the concatenation of the individual
    bucket subsets.
    This ensures that train, validation, and test set all contain the same
    number of samples of all sizes.
    :param bin_idcs: Specifies for each label the bucket it belongs to
    :return: Arrays specifying the indices of samples belonging to the
    training, validation, and test set respectively.
    """
    all_train_idcs = []
    all_val_idcs = []
    all_test_idcs = []
    num_bins = torch.max(bin_idcs) + 1
    for idx in range(num_bins):
        bucket_samples, = np.where(bin_idcs == idx)
        np.random.shuffle(bucket_samples)
        split1 = int(0.7 * len(bucket_samples))
        split2 = int(0.8 * len(bucket_samples))
        train_idcs = bucket_samples[:split1]
        val_idcs = bucket_samples[split1:split2]
        test_idcs = bucket_samples[split2:]
        all_train_idcs.append(train_idcs)
        all_val_idcs.append(val_idcs)
        all_test_idcs.append(test_idcs)
    return (np.concatenate(all_train_idcs), np.concatenate(all_val_idcs),
            np.concatenate(all_test_idcs))

def bin_data(labels, num_buckets, scale="linear", base=10, bin_bounds=None):
    """
    Splits the data into specified number of buckets of equal size. Returns for
    each sample the index of the the bucket it belongs to.
    :param labels: Unscaled labels used for computing bucket boundaries and
    assigning samples to buckets.
    :param num_buckets:
    :param scale: Whether to use separate the label domain into buckets on a
    linear or logarithmic scale. Hence the two options are either "linear" or
    "logarithmic".
    :param base: Only relevant if scale="logarithmic". Specifies the base of
    the logarithm.
    :param bin_bounds: Only relevant if scale="custom".
    :return: Array of the same length as labels specifying for each sample
    which bucket it belongs to.
    """
    max_label = np.max(labels)
    if scale == "logarithmic":
        bin_bounds = []
        base_size = max_label / (base**(num_buckets-1))
        for bin_idx in range(num_buckets):
            bin_bounds.append(base**bin_idx * base_size)
        bin_bounds[-1] = bin_bounds[-1] + 1.0
    elif scale == "linear":
        bin_size = int(math.ceil(float(max_label) / float(num_buckets)))
        bin_bounds = [bin_size * idx for idx in range(1, num_buckets+1)]
    elif scale == "custom" and bin_bounds != None:
        if len(bin_bounds) != num_buckets:
            raise ValueError(f"Error: Specified number of bins {num_buckets} "
                             f"does not match specified bin_bounds "
                             f"(length {len(bin_bounds)})")
    else:
        raise ValueError(f"Unknown scale type {scale}")
    print(f"\tBin bounds: {bin_bounds}")
    bin_idcs = np.digitize(labels, bin_bounds)
    return bin_idcs

def to_sparse_tensor(mat):
    """
    Converts a SciPy sparse matrix into a torch sparse tensor.
    """
    if isinstance(mat, ssp.csr_matrix) or isinstance(mat, ssp.csc_matrix):
        mat = mat.tocoo()
    data = mat.data
    indices = np.concatenate((mat.row.reshape(1, -1), mat.col.reshape(1, -1)),
                             axis=0)
    sparse_mat = sp.FloatTensor(torch.LongTensor(indices),
                                torch.FloatTensor(data),
                                torch.Size(mat.shape))
    return sparse_mat

def normalize(mx):
    """
    Row-normalize sparse matrix. Adapted from
    https://github.com/tkipf/pygcn/blob/master/pygcn/utils.py.
    """
    rowsum = np.array(mx.sum(1))
    r_inv = np.power(rowsum, -1).flatten()
    r_inv[np.isinf(r_inv)] = 0.
    r_mat_inv = ssp.diags(r_inv)
    mx = r_mat_inv.dot(mx)
    return mx

def summarize_tensor(x, title=""):
    with torch.no_grad():
        print("-"*10, title, "-"*10, sep="")
        shape = x.shape
        print(f"Shape: {shape}")

        nans = torch.sum(torch.isnan(x))
        print(f"NaNs: {nans}")

        nnz = torch.sum(x < 1e-8)
        print(f"NNZ: {nnz}")

        mean = torch.mean(x)
        print(f"Mean: {mean}")
        std = torch.std(x)
        print(f"Std: {std}")
        median = torch.median(x)
        print(f"Median: {median}")

        min = torch.min(x)
        print(f"Min: {min}")
        max = torch.max(x)
        print(f"Max: {max}")
        print("-"*(20+len(title)))

def summarize_feature_matrix(features):
    for col_idx in range(features.shape[1]):
        values = features[:, col_idx]
        mean = np.mean(values)
        std = np.std(values)
        min_val = np.min(values)
        max_val = np.max(values)
        num_values = len(np.unique(values))
        is_integer = np.sum(np.ceil(values) - values) <= 1e-10
        print("column index:", col_idx)
        print(f"statistics: {mean:.3f} +/- {std:.3f}")
        print(f"min, max: [{min_val:.3f}, {max_val:.3f}]")
        print(f"num unique values: {num_values}")
        print(f"data type:", "integer" if is_integer else "float")
        print()

reason: import numpy,import scipy


text:
# Licensed under the terms of the BSD-3-Clause license.
# Copyright (C) 2019 Michael Blaß
# michael.blass@uni-hamburg.de

"""
grapher.py -- Plot graphs from HMMs.
"""

from matplotlib import cm
from matplotlib.patches import ArrowStyle
from matplotlib.patches import Circle
from matplotlib.patches import ConnectionStyle
from matplotlib.patches import FancyArrowPatch
import matplotlib.pyplot as plt

import numpy as np
import networkx as nx
from scipy.spatial import distance

from apollon import tools

def _prepare_fig(pos):
    """Prepare a figure with the correct size.

    Params:
        pos    (dict) with structur {node_name_i: np.array([pos_x, pos_y])}
                      as return by nx.layout methods.
    Return:
        (Figure, AxesSubplot)
    """
    pos_data = np.array(list(pos.values()))
    diameter = distance.pdist(pos_data).max()
    dd = diameter / 2 + 1

    fig = plt.figure(figsize=(7, 7), frameon=False)
    ax = fig.add_subplot(111)
    r = 1.5
    ax.axis([-(dd+r), (dd+r), -(dd+r), (dd+r)])
    ax.set_axis_off()

    return fig, ax

def _draw_nodes(G, pos, ax):
    """Draw the nodes of a (small) networkx graph.

    Params:
        G    (nx.classes.*) a networkx graph.
        pos  (dict)         returned by nx.layout methods.
        ax   (AxesSubplot)  mpl axe.

    Return:
        (dict) of Circle patches.
    """
    #degree = np.array([deg for node, deg in G.degree], dtype=float)
    #degree /= degree.sum()

    flare_kwargs = {'alpha'    : 0.2,
                    'edgecolor': (0, 0, 0, 1),
                    'facecolor': None}

    node_kwargs = {'alpha'    : 0.8,
                   'edgecolor': (0, 0, 0, 1),
                   'facecolor': None}

    nodes = {}
    node_params = zip(pos.items())

    for i, (label, xy) in enumerate(pos.items()):
        size = G.nodes[label]['size']
        fsize = G.nodes[label]['fsize']
        flare_kwargs['facecolor'] = 'C{}'.format(i)
        flare = Circle(xy, size+fsize, **flare_kwargs)

        node_kwargs['facecolor'] = 'C{}'.format(i)
        node = Circle(xy, size, **node_kwargs)

        ax.add_patch(flare)
        ax.add_patch(node)

        font_style = {'size':15, 'weight':'bold'}
        text_kwargs = {'color': (0, 0, 0, .8),
                       'verticalalignment': 'center',
                       'horizontalalignment': 'center',
                       'fontdict': font_style}
        ax.text(*xy, i+1, **text_kwargs)

        nodes[label] = node

    return nodes

def _draw_edges(G, pos, nodes, ax):
    """Draw the edges of a (small) networkx graph.

    Params:
        G       (nx.classes.*)  a networkx graph.
        pos     (dict)          returned by nx.layout methods.
        nodes   (dict)          of Circle patches.
        ax      (AxesSubplot)   mpl axe.

    Return:
        (dict) of Circle patches.
    """
    pointer = ArrowStyle.Fancy(head_width=10, head_length=15)
    curved_edge = ConnectionStyle('arc3', rad=.2)

    arrow_kwargs = {'arrowstyle': pointer,
                    'antialiased': True,
                    'connectionstyle': curved_edge,
                    'edgecolor': None,
                    'facecolor': None,
                    'linewidth': None}

    edges = {}
    for i, (a, b, attr) in enumerate(G.edges.data()):
        arrow_kwargs['edgecolor'] = attr['color']
        arrow_kwargs['facecolor'] = attr['color']
        arrow_kwargs['linewidth'] = 1.0

        edge = FancyArrowPatch(pos[a], pos[b],
                               patchA=nodes[a], patchB=nodes[b],
                               shrinkA=5, shrinkB=5,
                               **arrow_kwargs)
        ax.add_patch(edge)
        edges[(a, b)] = edge

    return edges

def _legend(G, nodes, ax):
    """Draw the legend for a (small) nx graph.

    Params:
        G       (nx.classes.*) a networkx graph.
        nodes   (list)         of Circle patches.
        ax      (AxesSubplot)  mpl axe.

    Return:
        (AxesSubplot)
    """
    legend_kwargs = {'fancybox': True,
                     'fontsize': 14,
                     'bbox_to_anchor': (1.02, 1.0)}

    labels = [r'$f_c = {:>9.3f}$ Hz'.format(k) for k in G.nodes.keys()]
    legend = ax.legend(nodes.values(), labels, **legend_kwargs, borderaxespad=0)

    return legend

def draw_network(labels, tpm, delta):
    """Draw the graph of a HMM's transition probability matrix.

    Params:
        lables  (iterable)      Labels for each state.
        tpm     (np.ndarray)    A two-dimensional (row) stochastic matrix.
        delta   (iterable)

    Return:
        (Figure, AxesSubplot)
    """
    G = nx.MultiDiGraph()
    #scaled_tpm = np.exp(tools.scale(tpm, 0, 1.5))

    for i, from_state in enumerate(labels):
        G.add_node(from_state, fsize=np.exp(delta[i]))

        for j, to_state in enumerate(labels):
            if not np.isclose(tpm[i, j], 0.0):
                G.add_edge(from_state, to_state,
                           weight=tpm[i, j],
                           color='k')

    sd = np.sum([np.exp(degree) for node, degree in G.degree()])

    for node, degree in G.degree():
        G.node[node]['size'] = .5 + np.exp(degree) / sd

    #pos = nx.layout.circular_layout(G, center=(0., 0.), scale=4)
    pos = nx.layout.spring_layout(G, center=(0.0, 0.0), scale=4)

    fig, ax = _prepare_fig(pos)
    nodes = _draw_nodes(G, pos, ax)
    edges = _draw_edges(G, pos, nodes, ax)
    legend = _legend(G, nodes, ax)

    return fig, ax, G

def draw_matrix(tpm):
    """Draw a heatmap from a transition probability matrix.

    Args:
        tpm (np.ndarray)    Two-dimensional, row-stochastic square matrix.

    Returns:
        (fig, ax, img)
    """
    img_kwargs = {'origin': 'upper',
                 'interpolation': 'nearest',
                 'aspect': 'equal',
                 'cmap': 'viridis',
                 'vmin': 0.0, 'vmax': 1.0}

    nx, ny = tpm.shape

    fig = plt.figure(figsize=(10, 10))
    ax = fig.add_subplot(1, 1, 1)
    img = ax.imshow(tpm, **img_kwargs)

    # colorbar
    cbar = ax.figure.colorbar(img, ax=ax)
    cbar.ax.set_ylabel('Probability', rotation=-90, va="bottom")

    # major ticks
    ax.set_xticks(np.arange(nx))
    ax.set_yticks(np.arange(ny))
    ax.tick_params(which='major', top=True, bottom=False,
                   labeltop=True, labelbottom=False)

    # minor ticks (for grid)
    ax.set_xticks(np.arange(nx)-.5, minor=True)
    ax.set_yticks(np.arange(ny)-.5, minor=True)
    ax.tick_params(which="minor", bottom=False, left=False)
    ax.grid(which="minor", color="w", linestyle='-', linewidth=2)

    # spines
    ax.spines['top'].set_position(('outward', 10))
    ax.spines['left'].set_position(('outward', 10))
    ax.spines['bottom'].set_visible(False)
    ax.spines['right'].set_visible(False)

    # cell labels
    font_kw = {'fontsize': 14}
    text_kw = {'ha': 'center', 'va': 'center', 'fontdict': font_kw}
    for i in range(nx):
        for j in range(ny):
            val = tpm[i, j].astype(float).round(2)
            bc = cm.viridis(val)
            tc = cm.viridis(1-val)
            if np.array_equal(tc, bc):
                tc = 'k'
            ax.text(j, i, '{:.3f}'.format(val), **text_kw, color=tc)

    return fig, ax, img

def save_hmmfig(fig, path, **kwargs):
    """Save the figure to file.

    This saves the figure and ensures that the out-of-axes legend
    is completely visible in the saved version.

    All kwargs are passed on to plt.savefig.

    Params:
        fig     (Figure)    Figure of HMM tpm.
        path    (str)       Path to save file.
    """
    fig.savefig(fname=path,
                bbox_extra_artists=(fig.axes[0].legend_,))

reason: import numpy,from scipy,import networkx


text:
"""
Takes a folder with frame ranges and action annotations for the ADL dataset and produces a split file suitable for
mtl advanced
"""

import os, argparse
import numpy as np

valid_actions = [1,2,3,4,5,6,9,10,11,12,13,14,15,17,20,22,23,24]

def make_class_mapping_generic(classes):
    classes = np.sort(classes)
    mapping_dict = {}
    for i, c in enumerate(classes):
        mapping_dict[c] = i
    return mapping_dict

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('annots_dir_path')
    parser.add_argument('locations_dir_path')
    parser.add_argument('out_file_name_with_path', type=str)
    parser.add_argument('--only_valid_actions', default=False, action='store_true')
    parser.add_argument('--valid_actions_mapped', default=False, action='store_true')
    return parser.parse_args()

def search_for_locations(locations_lines, start_a, end_a):
    loc_annots = "-"
    start_a, end_a = int(start_a), int(end_a)
    for l in locations_lines:
        start_l, end_l, loc = l.strip().split()
        start_l, end_l = int(start_l), int(end_l)
        if start_a > end_l: # no overlap action starts after location -> continue
            continue
        elif end_a < start_l: # action ends before location -> break
            break
        else:
            # start of action inside location range
            # ->location valid until end of action or location, whichever comes first
            if start_l <= start_a <= end_l:
                overlap = min(end_l, end_a) - start_a + 1
                loc_annots += "{}:{}-".format(int(loc)-1, overlap) # turn location into 0-based
            # end of action inside location range but not its start
            # ->location is valid from start of location until end of action
            elif start_l <= end_a <= end_l:
                overlap = end_a - start_l + 1
                loc_annots += "{}:{}-".format(int(loc)-1, overlap)
    return loc_annots

args = parse_args()
annots_dir = args.annots_dir_path
locations_dir = args.locations_dir_path
only_valid_actions = args.only_valid_actions
valid_actions_mapped = args.valid_actions_mapped
outfile_path = args.out_file_name_with_path
outfile_dir = os.path.dirname(outfile_path)

action_mapping = make_class_mapping_generic(valid_actions)

if not os.path.exists(outfile_dir):
    os.makedirs(outfile_dir)

annot_names = os.listdir(annots_dir)
annot_files = [os.path.join(annots_dir, x) for x in annot_names]
outfile = open(outfile_path, 'a')

uid = -1
base_dir = "data\\ADL\\ADL_frames"
for i, annot_file in enumerate(annot_files):
    video_id = str(annot_names[i].split('.')[0].split('_')[1])
    with open(annot_file) as af:
        lines = af.readlines()

    location_file = os.path.join(locations_dir, "L{}.txt".format(video_id))
    with open(location_file) as lf:
        loc_lines = lf.readlines()

    annot_name = "P_" + video_id
    for l in lines:
        data = l.strip().split()
        start_frame = data[0]
        end_frame = data[1]
        # action = int(data[2]) - 1 # turn action into 0-based
        action = int(data[2])
        if only_valid_actions:
            if action not in valid_actions:
                continue
        if action < 1: # to avoid a weirdly labeled action out of the action set
            continue
        if only_valid_actions and valid_actions_mapped:
            action = action_mapping[action]
        locations = search_for_locations(loc_lines, start_frame, end_frame)
        segment_dir = os.path.join(base_dir, annot_name)
        num_frames = int(end_frame) - int(start_frame) + 1 # assume inclusive of end frame
        uid += 1 # assign manual uids
        output_line = "{} {} {} {} {} {}\n".format(segment_dir, num_frames, uid, start_frame, action, locations)
        outfile.write(output_line)
outfile.close()