text:
"""
This module allows to manipulate the Pandas dataframes containing
cell cycle information derived from MoMA output.
 """
# Author: Guillaume Witz, Biozentrum Basel, 2019
# License: MIT License

import numpy as np
import pandas as pd
import scipy.optimize
import scipy.stats
import trackpy

from . import tools_GW as tgw
from . import spot_detection as sp

def essential_props(time_mat):
    """Add basic cell cycle properties to dataframe

    Parameters
    ----------
    time_mat : pandas dataframe
        dataframe containig cell cycle information (on row per cycle)

    Returns
    -------
    time_mat : pandas dataframe
        updated cell cycle dataframe
    """
    #add basic timings and lengths such as birth or division lengths
    time_mat['Td'] = time_mat.length.apply(lambda x: x.shape[0])
    time_mat['Ld'] = time_mat.length.apply(lambda x: x[-1])
    time_mat['Lb'] = time_mat.length.apply(lambda x: x[0])
    time_mat['Tb'] = time_mat.born.apply(lambda x: x if x>=0 else 0)

    time_mat['mother_id'] = time_mat.apply(lambda row:
                                                 int(time_mat.index[time_mat.genealogy == row.genealogy[0:-1]][0])
                                                 if len(row.genealogy[0:-1])>0 else np.nan,axis = 1)

    return time_mat

def exponential_fit(row):
    """Exponential fit for growth curve

    Parameters
    ----------
    row : row index of dataframe. To be used with .apply(lambda row:)
        length data

    Returns
    -------
    fit_series : pandas series
        series with fit information (growth rate, fitted lengths etc.
    """
    tau_fit= np.nan
    Lb_fit = np.nan
    Ld_fit = np.nan
    pearson_lin = np.nan
    pearson_log = np.nan
    if len(row['length'])>5:
        xdata = range(len(row['length']))
        tau0 = row['Td']
        ydata = row['length']
        popt, pcov = scipy.optimize.curve_fit(tgw.fun_expgrowht2, xdata, ydata, p0=[15,tau0])
        tau_fit = popt[1]
        Lb_fit = popt[0]
        Ld_fit = tgw.fun_expgrowht2(row['Td'],popt[0],popt[1])
        pearson_lin = scipy.stats.pearsonr(xdata, ydata)[0]
        pearson_log = scipy.stats.pearsonr(xdata, np.log(ydata))[0]

    fit_series =   pd.Series({'tau_fit': tau_fit, 'Lb_fit': Lb_fit,'Ld_fit': Ld_fit,'pearson_lin': pearson_lin,'pearson_log': pearson_log})
    return fit_series

def get_cell_id_time_matrix(time_mat_pd):
    """Return a matrix whose where each row corresponds to one time point (column#1)
    of a given cell cycle (defind by its index column#2)

    Parameters
    ----------
    time_mat_pd : pandas dataframe
        dataframe containig cell cycle information (on row per cycle)

    Returns
    -------
    celltime : 2d numpy array
        Each row corresponds to a time point (column 1) of a row with a certain index (column2)
    """
    series_temp = time_mat_pd.pixlim.apply(lambda x: np.arange(len(x)))+time_mat_pd.born
    series_temp2 = time_mat_pd.apply(lambda row: row.name,axis = 1)*time_mat_pd.pixlim.apply(lambda x: np.ones(len(x)))
    celltime =np.stack((np.concatenate(series_temp.values),np.concatenate(series_temp2.values))).T
    return celltime

def get_long_mat(time_mat_pd):
    """Return a matrix where each row correponds to one time point (column#1)
    of a given cell cycle (defined by its index column#3) and contains other information (other columns)

    Parameters
    ----------
    time_mat_pd : pandas dataframe
        dataframe containig cell cycle information (on row per cycle)

    Returns
    -------
    celltime : 2d numpy array
        Each row corresponds to a time point (column#1) of a row with a certain index (column#3)
        Other columns:
            column#1: absolute time
            column#2: cell time
            column#3: index
            column#4: pix_max (moma output)
            column#5: pix_min (moma output)
    """
    tracklim = time_mat_pd.iloc[0].tracklim
    series_time = time_mat_pd.pixlim.apply(lambda x: np.arange(len(x)))+time_mat_pd.born
    series_time2 = time_mat_pd.pixlim.apply(lambda x: np.arange(len(x)))#cell time scale
    series_index = time_mat_pd.apply(lambda row: row.name,axis = 1)*time_mat_pd.pixlim.apply(lambda x: np.ones(len(x)))
    celltime =np.stack((np.concatenate(series_time.values),np.concatenate(series_time2.values),np.concatenate(series_index.values),
                      np.concatenate(time_mat_pd.pix_max.values)+tracklim,
                        np.concatenate(time_mat_pd.pix_min.values)+tracklim)).T
    return celltime

def get_cell(time_mat_pd, cellid, mom, image = None):
    """Return the image correponding to a given MoMAobj state and cellid

    Parameters
    ----------
    time_mat_pd : pandas dataframe
        dataframe containig cell cycle information (on row per cycle)
    cellid : int
        dataframe numerical index
    mom : MoMAobj object
        instance of a MoMAobj
    image : 2D numpy array
        image to crop already provided

    Returns
    -------
    cropped_image : 2d numpy array
        Image cropped around cell boundaries
    """
    if image == None:
        image = mom.load_moma_im()

    im_size = image.shape
    im_middle = int((im_size[1]-1)/2)

    t= mom.time-time_mat_pd.iloc[cellid]['born']

    index1 = time_mat_pd.iloc[cellid]['pixlim'][t,0]+time_mat_pd.iloc[cellid]['tracklim']+1
    index2 = time_mat_pd.iloc[cellid]['pixlim'][t,1]+time_mat_pd.iloc[cellid]['tracklim']-1

    cropped_image = image[index1:index2,im_middle-5:im_middle+6]
    return cropped_image

def find_spots(time_mat_pd, mom, sigmaXY, maxtime = None):
    """Using a MoMAobj and the corresponding segmentation, find within each
    cell all spots.

    Parameters
    ----------
    time_mat_pd : pandas dataframe
        dataframe containig cell cycle information (on row per cycle)
    mom : MoMAobj object
        instance of a MoMAobj
    sigmaXY : float
        xy spot widht (stdv Gaussian model)
    maxtime : int
        maximimum time index to consider

    Returns
    -------
    time_mat_pd : pandas dataframe
        dataframe containig cell cycle information with added key
        spots. That key contains spot detection information in the
        form of a long numpy array. To understand its content use:

        pd.DataFrame(time_mat_pd.iloc[16].spots, columns = ['x', 'y', 'A', 'b', 'A_fit', 'x_fit', 'y_fit', 'sigma_fit', 'RSS', 'xcorr', 'x_box', 'y_box', 'time','cell_time', 'x_cell', 'y_glob'])
    """

    #set spot fitting parameters
    #sigmaXY = 1.5
    fitbox = [int(np.ceil(4*sigmaXY)),int(np.ceil(4*sigmaXY))]
    gfilt = sp.make_g_filter(modelsigma=sigmaXY)
    glogfilt = sp.make_laplacelog(modelsigma=sigmaXY)

    #get matrix with cellids and boxes at all time points
    celltime = get_long_mat(time_mat_pd)

    #create an array to store a spot list for each cell
    spots = [[] for x in time_mat_pd.index]

    if maxtime is None:
        maxtime = mom.get_max_time()

    all_fits = []
    for t in range(0,maxtime):
        #print('time: '+str(t))
        mom.col = 2
        mom.time = t
        image = mom.load_moma_im()
        im_size = image.shape
        im_middle = int((im_size[1]-1)/2)
        image = image[:,im_middle-15:im_middle+16]

        conv = sp.spot_filter_convfft(image=image,templ=gfilt,logfilt=-glogfilt)

        spot_prop = sp.get_candidates(conv,image)
        fit_res = sp.fit_candidates(image, spot_prop, sigmaXY, show_fit = False, fit_type = 'B')
        fit_pd = pd.DataFrame(np.c_[fit_res,np.ones(len(spot_prop))*fitbox[0],np.ones(len(spot_prop))*fitbox[1],np.ones(len(spot_prop))*t],
                 columns=['A_fit','x_fit','y_fit','sigma_fit','RSS','xcorr','x_box','y_box','time'])

        spot_mat = pd.concat([spot_prop,fit_pd],axis=1)

        #absolute positions of spots
        abs_pos_x = (spot_mat.x+spot_mat.x_fit-spot_mat.x_box).values
        abs_pos_y = (spot_mat.y+spot_mat.y_fit-spot_mat.y_box).values
        #cell limits
        cell_lims = celltime[celltime[:,0]==t,:]
        #find the index of the cell to which each spot belongs
        assigned = [cell_lims[[np.logical_and(p > x[4], p < x[3]) for x in cell_lims],:] for p in abs_pos_x]
        #assign spot to cell
        for ind, x in enumerate(assigned):
            if len(x)>0:
                toappend = np.append(spot_mat.iloc[ind].values,[x[0][1], 0.5*(x[0][3]+x[0][4])-abs_pos_x[ind],abs_pos_y[ind]])
                spots[int(x[0][2])].append(toappend)

        #all_fits.append(fit_res)
        '''fig,ax = plt.subplots(figsize=(20,20))
        plt.imshow(image)
        plt.plot(spot_prop.y,spot_prop.x,'ro')
        plt.show()'''
    spots = [np.stack(x) if len(x)>0 else [] for x in spots]
    time_mat_pd['spots'] = pd.Series(spots)

    return time_mat_pd

#track spots stored in time_mat_pd.spots
def track_spots(time_mat_pd):
    """Track all spots within a cell and reconstruct trajectories
    using the trackpy package.

    Parameters
    ----------
    time_mat_pd : pandas dataframe
        dataframe containig cell cycle information (on row per cycle)

    Returns
    -------
    time_mat_pd : pandas dataframe
        dataframe containig cell cycle information with
        added keys with spot tracking information. In particular it
        has a key 'num_spots' with the number of spots belonging to
        full tracks per time from which the initiation is extracted
        by detecting doubling.
    """

    time_mat_pd['spots_num'] = np.nan
    time_mat_pd['spots_num'] = time_mat_pd['spots_num'].astype(object)
    time_mat_pd['spots_tracks'] = np.nan
    time_mat_pd['spots_tracks'] = time_mat_pd['spots_tracks'].astype(object)
    time_mat_pd['full_tracks'] = np.nan
    time_mat_pd['full_tracks'] = time_mat_pd['full_tracks'].astype(object)

    for index in time_mat_pd.index:#range(14,17):#
        if len(time_mat_pd.loc[0].spots)==0:
            continue
        unique, counts = np.unique(time_mat_pd.loc[0].spots[:,-2], return_counts=True)
        maxnumspots = np.max(counts)

        if (time_mat_pd.iloc[index].full_cellcycle)&(maxnumspots<10):
            mother_id = time_mat_pd.iloc[index].mother_id
            if mother_id>0:

                #recover mother daugher (current cell) and grand-daughter info
                mother_id = int(mother_id)
                mother_time = time_mat_pd.iloc[mother_id]['Td']
                mother_Ld = time_mat_pd.iloc[mother_id].length[-1]

                daughter_Ld = time_mat_pd.iloc[index].length[-1]
                daughter_time = time_mat_pd.iloc[index]['Td']

                gdaughter1_id = time_mat_pd.index[time_mat_pd.genealogy == time_mat_pd.iloc[index].genealogy+'B']
                gdaughter1_id = gdaughter1_id[0] if len(gdaughter1_id)>0 else None
                gdaughter2_id = time_mat_pd.index[time_mat_pd.genealogy == time_mat_pd.iloc[index].genealogy+'T']
                gdaughter2_id = gdaughter2_id[0] if len(gdaughter2_id)>0 else None

                if (gdaughter1_id is None) or (gdaughter2_id is None):
                    continue
                gdaughter1_time = time_mat_pd.iloc[gdaughter1_id]['Td']
                gdaughter2_time = time_mat_pd.iloc[gdaughter2_id]['Td']

                len_mother_spots = len(time_mat_pd.iloc[mother_id]['spots'])
                len_gdaughter_spots = len(time_mat_pd.iloc[gdaughter1_id]['spots'])
                len_gdaughter_spots2 = len(time_mat_pd.iloc[gdaughter2_id]['spots'])

                #proceed if mother cell is ok
                if (not np.isnan(mother_Ld))&(mother_time>10)&(gdaughter1_time>5)&(gdaughter2_time>5)&(len_mother_spots>0)&(len_gdaughter_spots>0)&(len_gdaughter_spots2>0):
                    #find out if daughter is top/bottom
                    if time_mat_pd.iloc[index].genealogy[-1]=='B':
                        sign = -1
                        displace = -1/4
                    else:
                        sign = 1
                        displace = 1/4

                    combined = np.empty((0,4))

                    #keep time, x, y, A and limit the time points in mothe and grand-daughters
                    mother_spots = time_mat_pd.iloc[mother_id]['spots'][:,[13,14,15,4]]
                    mother_spots=mother_spots[mother_spots[:,0]>=mother_time-10]
                    daughter_spots = time_mat_pd.iloc[index]['spots'][:,[13,14,15,4]]
                    gdaughter_spots = time_mat_pd.iloc[gdaughter1_id]['spots'][:,[13,14,15,4]]
                    gdaughter_spots=gdaughter_spots[gdaughter_spots[:,0]<5]
                    gdaughter_spots2 = time_mat_pd.iloc[gdaughter2_id]['spots'][:,[13,14,15,4]]
                    gdaughter_spots2=gdaughter_spots2[gdaughter_spots2[:,0]<5]

                    if ((len(mother_spots)==0)or(len(gdaughter_spots)==0)or(len(gdaughter_spots2)==0)):
                        continue

                    #create a composite spot ensemble made of mother-daugher-grand-daughters
                    #correct for positions (1/4 positions, 1/2 position etc.)
                    mother_spots = mother_spots[np.sign(mother_spots[:,1])==sign]
                    mother_spots[:,0] = mother_spots[:,0]-mother_time
                    combined = np.concatenate((combined,mother_spots))

                    daughter_spots[:,1] = daughter_spots[:,1]+displace*mother_Ld
                    combined = np.concatenate((combined,daughter_spots))

                    gdaughter_spots[:,1] = gdaughter_spots[:,1]+(displace*mother_Ld)-1/4*daughter_Ld
                    gdaughter_spots[:,0] = gdaughter_spots[:,0]+daughter_time
                    combined = np.concatenate((combined,gdaughter_spots))

                    gdaughter_spots2[:,1] = gdaughter_spots2[:,1]+(displace*mother_Ld)+1/4*daughter_Ld
                    gdaughter_spots2[:,0] = gdaughter_spots2[:,0]+daughter_time
                    combined = np.concatenate((combined,gdaughter_spots2))

                    #do the tracking
                    combined = combined[:,0:3]
                    spot_frame = pd.DataFrame(combined,columns=('frame','x','y'))
                    #spot_frame = pd.DataFrame(combined,columns=('frame','x','z','y'))
                    spot_tracked = trackpy.link_df(spot_frame,20,memory=5,link_strategy='nonrecursive',pos_columns=['x', 'y'])
                    spot_tracked = trackpy.filter_stubs(spot_tracked,5)

                    tracks = []
                    for tr in spot_tracked.particle.unique():
                        curr_track = spot_tracked[spot_tracked.particle==tr]
                        #interpolate missing values
                        tracks.append(np.c_[[np.interp(np.arange(curr_track.frame.min(),curr_track.frame.max()+1),curr_track.frame,curr_track[key]) for key in ['frame','x','y']]].transpose())

                    time_mat_pd.at[index,'full_tracks'] = tracks

                    #limit tracks to daughter segment
                    cut_tracks = [x[(x[:,0]<daughter_time),:] for x in tracks]
                    cut_tracks = [x for x in cut_tracks if len(x)>0]
                    #calculate number of spots as a f. of time
                    num_spots = np.zeros(daughter_time+10)
                    for x in range(len(cut_tracks)):
                        num_spots[int(cut_tracks[x][0,0])+10:int(cut_tracks[x][-1,0])+11]+=1
                    time_mat_pd.at[index,'spots_num'] = num_spots
                    time_mat_pd.at[index,'spots_tracks'] = cut_tracks

                    if (num_spots[0]==1)&(len(np.argwhere(num_spots==2))>0):
                        time_mat_pd.at[index,'Ti'] = np.argwhere(num_spots==2)[0][0]-10

    return time_mat_pd

def length_fit_at_T(row, time_var):
    """To be used with an .apply(lambda row:) call. Calculate cell length
    at as specific time given by time_var key.

    Parameters
    ----------
    row : int
        dataframe numerical index
    time_var : str
        key to use to recover time

    Returns
    -------
    len_fit : float
        length at time given by time_var key
    """
    len_fit = np.nan
    if (~np.isnan(row['tau_fit']))&(~np.isnan(row[time_var])):
        len_fit = tgw.fun_expgrowht2(row[time_var],row['Lb_fit'],row['tau_fit'])
    return len_fit

def inter_initiations(row, time_mat_pd,t_var,l_var):
    """To be used with an .apply(lambda row:) call. Given an
    initiation time (t_var key) and a length variable (l_var)
    calculate the difference between mother and daughter length
    adjusting for number of origins (this is only used for slow
    growth so we make the approximation that if initiaion time
    was in the mother, the cell was born with two origins.)

    Parameters
    ----------
    row : int
        dataframe numerical index
    time_mat_pd : pandas dataframe
        dataframe containig cell cycle information (on row per cycle)
    t_var : str
        key to use to recover time
    l_var : str
        key to use to recover length

    Returns
    -------
    () : tuple
        tuple containing per origin inter-initiation length and mother initiation length
    """

    if row['mother_id']>0:

        if (row[t_var]<0) and (time_mat_pd.loc[row['mother_id']][t_var]>0):
            L_inter_init = row[l_var]-0.5*time_mat_pd.loc[row['mother_id']][l_var]
            L_i_old = 0.5*time_mat_pd.loc[row['mother_id']][l_var]
        elif time_mat_pd.loc[row['mother_id']][t_var]>0:
            L_inter_init = row[l_var]-row['Lb_fit']+0.5*(time_mat_pd.loc[row['mother_id']].Ld_fit-time_mat_pd.loc[row['mother_id']][l_var])
            L_i_old = 0.5*time_mat_pd.loc[row['mother_id']][l_var]
        else:
            L_inter_init = np.nan
            L_i_old = np.nan
        return (L_inter_init, L_i_old)
    else:
        return (np.nan,np.nan)

def mother_var(row, time_mat_pd,var):
    """To be used with an .apply(lambda row:) call.
    Return specific key of mother cell

    Parameters
    ----------
    row : int
        dataframe numerical index
    time_mat_pd : pandas dataframe
        dataframe containig cell cycle information (on row per cycle)
    var : str
        key of variable

    Returns
    -------
      : float
        value of key for mother cell
    """
    if row['mother_id']>0:
        if row['mother_id'] in time_mat_pd.index:
            return time_mat_pd.loc[row['mother_id']][var]
        else:
            return np.nan
    else:
        return np.nan

max_stars_repo_name: guiwitz/DoubleAdderArticle

max_issues_repo_name: guiwitz/DoubleAdderArticle

max_forks_repo_name: guiwitz/DoubleAdderArticle

reason: import numpy,import scipy


text:
# coding: utf-8

# In[28]:

import torch
import torchvision
import torchvision.transforms as transforms
import torch.nn as nn
import torch.nn.functional as F
import torchvision.transforms.functional as TF
from skimage import io, transform

import matplotlib.pyplot as plt
import numpy as np
import cv2

# In[36]:

transform = transforms.Compose(
    [
     transforms.RandomAffine(0, translate=(0.2,0.2), scale=None, shear=None, resample=False, fillcolor=0),# randomly translate:-img_width * 0.2 < dx < img_width * 0.2
     transforms.ToTensor(), # convert Python Image Library (PIL) format to PyTorch tensors.
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) # convert the data from [0,1] to [-1,1]
    ])

testset = torchvision.datasets.CIFAR10(root='./data',
                                       train=False,
                                       download=True,
                                       transform= transform
                                       )

testloader = torch.utils.data.DataLoader(testset,
                                         batch_size=5,
                                         shuffle=False)

# In[37]:

# unnormalize the images
def convert_to_imshow_format(image):
    # first convert back to [0,1] range from [-1,1] range
    image = image / 2 + 0.5
    image = image.numpy()
    # convert from CHW to HWC
    # from 3x32x32 to 32x32x3
    return image.transpose(1,2,0)

# In[38]:

# def shift_h(im, shift_amount):
#     """Shift the image horizontally by shift_amount pixels,
#     use positive numbers to shift the image to the right,
#     use negative numbers to shift the image to the left"""
#     if shift_amount == 0:
#         return im
#     else:
#         if len(im.shape) == 3: # for a single image
#             new_image = torch.zeros_like(im)
#             new_image[:, :, :shift_amount] = im[:,:,-shift_amount:]
#             new_image[:,:,shift_amount:] = im[:,:,:-shift_amount]
#             return new_image
#         elif len(im.shape) == 4: # for batches of images
#             new_image = torch.zeros_like(im)
#             new_image[:, :, :, :shift_amount] = im[:, :, :, -shift_amount:]
#             new_image[:, :, :, shift_amount:] = im[:, :, :, :-shift_amount]
#             return new_image

dataiter = iter(testloader)
images, labels = dataiter.next()

fig, axes = plt.subplots(1, len(images), figsize=(12,2.5))
for idx, image in enumerate(images):
    image = convert_to_imshow_format(image)
#     image = shift_h(image,5)
    axes[idx].imshow(image)
#     axes[idx].set_title(classes[labels[idx]])

ext: py

max_stars_repo_path: transform testsets.py

max_stars_repo_name: TheTechZone/shift-invariant-cnns

max_issues_repo_path: transform testsets.py

max_issues_repo_name: TheTechZone/shift-invariant-cnns

max_forks_repo_path: transform testsets.py

max_forks_repo_name: TheTechZone/shift-invariant-cnns

reason: import numpy


text:
"""Kernels used in Gaussian process

"""

import jax.numpy as jnp
def gpkernel_RBF(t,tau,a,err):
    """RBF kernel with diagnoal error

    Args:
       t: variable vector (N)
       tau: scale parameter (scalar)
       a: amplitude (scalar)
       err: diagnonal error vector (N)

    Returns:
       kernel

    """

    Dt = t - jnp.array([t]).T
    K=a*jnp.exp(-(Dt)**2/2/(tau**2))+jnp.diag(err**2)
    return K

max_stars_repo_name: dcmvdbekerom/exojax

max_issues_repo_name: dcmvdbekerom/exojax

max_forks_repo_name: dcmvdbekerom/exojax

reason: import jax


text:
import itertools as itt
import math
import numbers
import random
from abc import abstractmethod
from typing import Iterable
from typing import List
from typing import Tuple

import numpy as np
import pandas as pd
from mizarlabs.transformers.targets.labeling import LABEL
from scipy.stats import rv_continuous
from sklearn.model_selection._split import _BaseKFold

class BaseTimeSeriesCrossValidator(_BaseKFold):
    """
    Abstract class for time series cross-validation.

    Time series cross-validation requires each sample has a prediction
    time pred_time, at which the features are used to predict the response,
    and an evaluation time eval_time, at which the response is known and the
    error can be computed.
    Importantly, it means that unlike in standard sklearn cross-validation,
    the samples X, response y, pred_times and eval_times must all be pandas
    dataframe/series having the same index. It is also assumed that the
    samples are time-ordered with respect to the prediction time
    (i.e. pred_times is non-decreasing).

    Parameters:
    ----------
    n_splits : int, default=10
        Number of folds. Must be at least 2.

    """

    def __init__(
        self, n_splits=10, pred_times: pd.Series = None, eval_times: pd.Series = None
    ):

        super().__init__(n_splits, shuffle=False, random_state=None)

        if not isinstance(pred_times, pd.Series):
            raise ValueError("pred_times should be a pandas Series.")
        if not isinstance(eval_times, pd.Series):
            raise ValueError("eval_times should be a pandas Series.")

        self.pred_times = pred_times
        self.eval_times = eval_times
        self.indices = None

    @abstractmethod
    def split(self, X: pd.DataFrame, y: pd.Series = None, groups=None):
        """
        Yield the indices of the train and test sets.

        :param X: pd.DataFrame, shape (n_samples, n_features), required
        :param y: pd.Series
        :param groups:  not used, inherited from _BaseKFold
        :return:
        """
        pass

class CombPurgedKFoldCV(BaseTimeSeriesCrossValidator):
    """
    Purged and embargoed combinatorial cross-validation.

    As described in Advances in financial machine learning,
    Marcos Lopez de Prado, 2018.

    The samples are decomposed into n_groups folds containing equal numbers of
    samples, without shuffling. In each cross validation round, n_test_splits
    folds are used as the test set, while the other folds are used as the train
    set. There are as many rounds as n_test_splits folds among the n_groups
    folds.Each sample should be tagged with a prediction time pred_time and an
    evaluation time eval_time. The split is such that the intervals
    [pred_times, eval_times] associated to samples in the train and test set do
    not overlap. (The overlapping samples are dropped.) In addition,
    an "embargo" period is defined, giving the minimal time between an
    evaluation time in the test set and a prediction time in the training set.
    This is to avoid, in the presence of temporal correlation, a contamination
    of the test set by the train set.

    Parameters:
    ----------
    n_groups : int, default=10
        Number of folds. Must be at least 2.
    n_test_splits : int, default=2
        Number of folds used in the test set. Must be at least 1.
    pred_times : pd.Series, shape (n_samples,), required
        Times at which predictions are made. pred_times.index has to
        coincide with X.index.
    eval_times : pd.Series, shape (n_samples,), required
        Times at which the response becomes available and the error can
        be computed. eval_times.index has to
        coincide with X.index.
    embargo_td : pd.Timedelta, default=0
        Embargo period (see explanations above).
    """

    default_embargo_td = pd.Timedelta(minutes=0)

    def __init__(
        self,
        n_groups=10,
        n_test_splits=2,
        pred_times: pd.Series = None,
        eval_times: pd.Series = None,
        embargo_td: pd.Timedelta = default_embargo_td,
    ):

        self.n_groups = n_groups

        n_splits = int(
            math.factorial(n_groups)
            / (math.factorial(n_test_splits) * math.factorial(n_groups - n_test_splits))
        )

        super().__init__(n_splits, pred_times, eval_times)
        if not isinstance(n_test_splits, numbers.Integral):
            raise ValueError(
                f"The number of test folds must be of "
                f"Integral type. {n_test_splits} of type "
                f"{type(n_test_splits)} was passed."
            )
        n_test_splits = int(n_test_splits)
        if n_test_splits <= 0 or n_test_splits > self.n_groups - 1:
            raise ValueError(
                f"K-fold cross-validation requires at least one "
                f"train/test split by setting n_test_splits"
                f" between 1 and n_groups - 1, "
                f"got n_test_splits = {n_test_splits}."
            )
        self.n_test_splits = n_test_splits
        if not isinstance(embargo_td, pd.Timedelta):
            raise ValueError(
                f"The embargo time should be of type Pandas "
                f"Timedelta. {embargo_td} of type "
                f"{type(embargo_td)} was passed."
            )
        if embargo_td < pd.Timedelta(minutes=0):
            raise ValueError(
                f"The embargo time should be positive, got embargo = {embargo_td}."
            )
        self.embargo_td = embargo_td

    def _test_fold_bounds(self, X: pd.DataFrame, y: pd.Series = None) -> List[Tuple]:
        """
        Calculate the bounds per each fold of the test set.

        :param X: pd.DataFrame, shape (n_samples, n_features), required
        :param y: pd.Series
        :return:
        """
        self.indices = np.arange(X.shape[0])

        # Fold boundaries
        fold_bounds = [
            (fold[0], fold[-1] + 1)
            for fold in np.array_split(self.indices, self.n_groups)
        ]
        # List of all combinations of n_test_splits folds selected to become
        # test sets
        selected_fold_bounds = list(itt.combinations(fold_bounds, self.n_test_splits))
        # In order for the first round to have its whole test set at the end
        # of the dataset
        selected_fold_bounds.reverse()
        return selected_fold_bounds

    def split(
        self, X: pd.DataFrame, y: pd.Series = None, groups=None
    ) -> Iterable[Tuple[np.ndarray, np.ndarray]]:
        """
        Yield the indices of the train and test sets.

        Although the samples are passed in the form of a pandas dataframe,
        the indices returned are position indices, not labels.

        :param X: pd.DataFrame, shape (n_samples, n_features), required
        :param y: pd.Series
        :param groups: not used, inherited from _BaseKFold
        :return:
        """
        selected_fold_bounds = self._test_fold_bounds(X, y)

        for fold_bound_list in selected_fold_bounds:
            # Computes the bounds of the test set, and the corresponding
            # indices
            test_fold_bounds, test_indices = self._compute_test_set(fold_bound_list)
            # Computes the train set indices
            train_indices = self._compute_train_set(test_fold_bounds, test_indices)

            yield train_indices, test_indices

    def _compute_train_set(
        self, test_fold_bounds: List[Tuple[int, int]], test_indices: np.ndarray
    ) -> np.ndarray:
        """
        Compute the position indices of samples in the train set.

        :param test_fold_bounds: List of tuples of position indices
            Each tuple records the bounds of a block of indices in the test
            set.
        :param test_indices: np.ndarray
            A numpy array containing all the indices in the test set.
        :return: np.ndarray
            A numpy array containing all the indices in the train set.
        """
        # As a first approximation, the train set is the complement of the
        # test set
        train_indices = np.setdiff1d(self.indices, test_indices)
        # But we now have to purge and embargo
        for test_fold_start, test_fold_end in test_fold_bounds:
            # Purge
            train_indices = purge(self, train_indices, test_fold_start, test_fold_end)
            # Embargo
            train_indices = embargo(self, train_indices, test_indices, test_fold_end)
        return train_indices

    def _compute_test_set(
        self, fold_bound_list: List[Tuple[int, int]]
    ) -> Tuple[List[Tuple[int, int]], np.ndarray]:
        """
        Compute the indices of the samples in the test set.

        :param fold_bound_list: List of tuples of position indices
            Each tuple records the bounds of the folds belonging to the
            test set.
        :return:
            test_fold_bounds: List of tuples of position indices
                Like fold_bound_list, but with the neighboring folds in the
                test set merged.
            test_indices: np.ndarray
                A numpy array containing the test indices.
        """
        test_indices = np.empty(0)
        test_fold_bounds = []
        for fold_start, fold_end in fold_bound_list:
            # Records the boundaries of the current test split
            if not test_fold_bounds or fold_start != test_fold_bounds[-1][-1]:
                test_fold_bounds.append((fold_start, fold_end))
            # If the current test split is contiguous to the previous one,
            # simply updates the endpoint
            elif fold_start == test_fold_bounds[-1][-1]:
                test_fold_bounds[-1] = (test_fold_bounds[-1][0], fold_end)
            test_indices = np.union1d(
                test_indices, self.indices[fold_start:fold_end]
            ).astype(int)
        return test_fold_bounds, test_indices

def embargo(
    cv: BaseTimeSeriesCrossValidator,
    train_indices: np.ndarray,
    test_indices: np.ndarray,
    test_fold_end: int,
) -> np.ndarray:
    """
    Apply the embargo procedure to part of the train set.

    This amounts to dropping the train set samples whose prediction time occurs
    within self.embargo_dt of the test set sample evaluation times.
    This method applies the embargo only to the part of the training set
    immediately following the end of the test set determined by test_fold_end.

    :param cv: Cross-validation class
        Needs to have the attributes cv.pred_times, cv.eval_times,
        cv.embargo_dt and cv.indices.
    :param train_indices: np.ndarray
        A numpy array containing all the indices of the samples currently
        included in the train set.
    :param test_indices: np.ndarray
        A numpy array containing all the indices of the samples in the test
        set.
    :param test_fold_end: int
        Index corresponding to the end of a test set block.
    :return:
        train_indices: np.ndarray
        The same array, with the indices subject to embargo removed.
    """
    if not hasattr(cv, "embargo_td"):
        raise ValueError(
            "The passed cross-validation object should have a "
            "member cv.embargo_td defining the embargo time."
        )
    last_test_eval_time = cv.eval_times.iloc[test_indices[:test_fold_end]].max()
    min_train_index = len(
        cv.pred_times[cv.pred_times <= last_test_eval_time + cv.embargo_td]
    )
    if min_train_index < cv.indices.shape[0]:
        allowed_indices = np.concatenate(
            (cv.indices[:test_fold_end], cv.indices[min_train_index:])
        )
        train_indices = np.intersect1d(train_indices, allowed_indices)
    return train_indices

def purge(
    cv: BaseTimeSeriesCrossValidator,
    train_indices: np.ndarray,
    test_fold_start: int,
    test_fold_end: int,
) -> np.ndarray:
    """
    Purge part of the train set.

    Given a left boundary index test_fold_start of the test set,
    this method removes from the train set all the
    samples whose evaluation time is posterior to the prediction time of the
    first test sample after the boundary.

    :param cv: Cross-validation class,
        Needs to have the attributes cv.pred_times, cv.eval_times and
        cv.indices.
    :param train_indices: np.ndarray, A numpy array containing all the indices
        of the samples currently included in the train set.
    :param test_fold_start: int, Index corresponding to the start of a test
        set block.
    :param test_fold_end: int, Index corresponding to the end of the same test
        set block.
    :return: train_indices: np.ndarray
        A numpy array containing the train indices purged at test_fold_start.
    """
    time_test_fold_start = cv.pred_times.iloc[test_fold_start]
    # The train indices before the start of the test fold, purged.
    train_indices_1 = np.intersect1d(
        train_indices, cv.indices[cv.eval_times < time_test_fold_start]
    )
    # The train indices after the end of the test fold.
    train_indices_2 = np.intersect1d(train_indices, cv.indices[test_fold_end:])
    return np.concatenate((train_indices_1, train_indices_2))

class LogUniformGen(rv_continuous):
    def _cdf(self, x):
        return np.log(x / self.a) / np.log(self.b / self.a)

def log_uniform(a=1, b=None):
    if b is None:
        b = np.exp(1)
    return LogUniformGen(a=a, b=b, name="logUniform")

def compute_back_test_paths(n_splits: int, n_test_splits: int) -> int:
    """
    Compute the number of backtest paths for the combinatorial crossvalidation.

    As explained in pg. 164 of De Prado book this function calculates
    the number of paths that can be used given the total number of splits
    (`n_splits`) and the test splits (`n_test_splits`).

    :param n_splits: the total number of splits
    :type n_splits: int
    :param n_test_splits: the number of splits used in the test set
    :type n_test_splits: int
    :return: number of the backtest paths
    :rtype: int
    """
    den = 1

    for i in range(1, n_test_splits):
        den = den * (n_splits - i)

    return int(den / math.factorial(n_test_splits - 1))

# TODO: adapt to signal pipeline
def combinatorial_cross_validation_paths(
    X: pd.DataFrame,
    y: pd.DataFrame,
    cv: CombPurgedKFoldCV,
    signal_pipeline,
    signal_pipeline_fit_params,
    label_column_name: str = LABEL,
) -> List[pd.DataFrame]:
    """
    Return the paths for the combinatorial cross validation analysis.

    :param X: Dataframe containing the features
    :type X: pd.DataFrame
    :param y: DataFrame containing the target and target info
    :type y: pd.DataFrame
    :param signal_pipeline: The signal pipeline we want to use for creating features
    :type signal_pipeline:
    :param signal_pipeline_fit_params: Fit params for the signal pipeline
    :type signal_pipeline_fit_params:
    :param cv: Combinatorial purged cv object
    :type cv: CombPurgedKFoldCV

    :return:
    """
    # computing the test bounds per each fold
    test_bounds = cv._test_fold_bounds(X, y[label_column_name])
    bound_pred_dict = {}

    for idx, train_test in enumerate(cv.split(X, y[label_column_name])):
        train_idx = train_test[0]
        test_idx = train_test[1]
        current_test_bounds = test_bounds[idx]

        signal_pipeline.fit(
            X.iloc[train_idx].values,
            y.iloc[train_idx][label_column_name].values,
            **signal_pipeline_fit_params,
        )

        # predict each sub test set
        for test_bound in current_test_bounds:

            # select the indeces of the current sub test set
            part_test = test_idx[
                np.where((test_idx >= test_bound[0]) & (test_idx < test_bound[1]))
            ]

            # predictions on the sub test set
            tmp_df = pd.DataFrame(index=X.index[part_test])
            predictions = signal_pipeline.predict(X.iloc[part_test].values)
            tmp_df["predictions"] = predictions
            proba = signal_pipeline.predict_proba(X.iloc[part_test].values)

            max_proba = np.max(proba, axis=1)
            tmp_df["max_proba"] = max_proba

            tmp_df = pd.merge(tmp_df, y, left_index=True, right_index=True, how="left")

            # adding the sub test set to the dictionary where each key is the
            # index start of the sub test set
            if test_bound[0] not in bound_pred_dict.keys():
                bound_pred_dict[test_bound[0]] = []

            bound_pred_dict[test_bound[0]].append(tmp_df)

    n_paths = compute_back_test_paths(cv.n_splits, cv.n_test_splits)
    paths = []

    # from the sub test computed create paths randomly picking one sub test
    # per start index.
    for _ in range(n_paths):
        path_df = pd.DataFrame()
        for k in bound_pred_dict.keys():
            random.shuffle(bound_pred_dict[k])
            tmp_df = bound_pred_dict[k].pop()
            path_df = path_df.append(tmp_df)

        paths.append(path_df.copy())

    yield paths

max_stars_repo_name: MizarAI/mizar-labs

max_issues_repo_name: MizarAI/mizar-labs

max_forks_repo_name: MizarAI/mizar-labs

reason: import numpy,from scipy


text:
# coding: utf-8
import numpy as np
import math

def cosOfTwoAcceleraters(one, two):
	cross = one[0]*two[0] + one[1]*two[1] + one[2]*two[2]
	valueX = math.sqrt(one[0]*one[0] + one[1]*one[1] + one[2]*one[2])
	valueY = math.sqrt(two[0]*two[0] + two[1]*two[1] + two[2]*two[2])
	base = (valueX * valueY)
	if base <= 0:
		return 0
	else:
		return cross / base

def cosOfAcceleratersArray(data):
	arrResult = np.zeros(data.shape[0]-1)
	for i in range(data.shape[0]-1):
		cosValue = cosOfTwoAcceleraters(data[i, :], data[i+1, :])
		arrResult[i] = cosValue
	return arrResult

def unitOfAcceleratersArray(data):
	return np.sqrt(np.sum(data * data, axis=1))

max_stars_repo_name: vvvvcp/NeverMore

max_issues_repo_name: vvvvcp/NeverMore

max_forks_repo_name: vvvvcp/NeverMore


text:
"""
Class for reading data from CED (Cambridge Electronic Design)
http://ced.co.uk/

This read *.smrx (and *.smr) from spike2 and signal software.

Note Spike2RawIO/Spike2IO is the old implementation in neo.
It still works without any dependency and should be faster.
Spike2IO only works for smr (32 bit) and not for smrx (64 bit) files.

This implementation depends on the SONPY package:
https://pypi.org/project/sonpy/

Please note that the SONPY package:
  * is NOT open source
  * internally uses a list instead of numpy.ndarray, potentially causing slow data reading
  *  is maintained by CED

Author : Samuel Garcia
"""

from .baserawio import (BaseRawIO, _signal_channel_dtype, _signal_stream_dtype,
                _spike_channel_dtype, _event_channel_dtype)

import numpy as np
from copy import deepcopy

try:
    import sonpy
    HAVE_SONPY = True
except ImportError:
    HAVE_SONPY = False

class CedRawIO(BaseRawIO):
    """
    Class for reading data from CED (Cambridge Electronic Design) spike2.
    This internally uses the sonpy package which is closed source.

    This IO reads smr and smrx files
    """
    extensions = ['smr', 'smrx']
    rawmode = 'one-file'

    def __init__(self, filename='', take_ideal_sampling_rate=False, ):
        BaseRawIO.__init__(self)
        self.filename = filename

        self.take_ideal_sampling_rate = take_ideal_sampling_rate

    def _source_name(self):
        return self.filename

    def _parse_header(self):
        assert HAVE_SONPY, 'sonpy must be installed'

        self.smrx_file = sonpy.lib.SonFile(sName=str(self.filename), bReadOnly=True)
        smrx = self.smrx_file

        self._time_base = smrx.GetTimeBase()

        channel_infos = []
        signal_channels = []
        spike_channels = []
        self._all_spike_ticks = {}

        for chan_ind in range(smrx.MaxChannels()):
            chan_type = smrx.ChannelType(chan_ind)
            chan_id = str(chan_ind)
            if chan_type == sonpy.lib.DataType.Adc:
                physical_chan = smrx.PhysicalChannel(chan_ind)
                divide = smrx.ChannelDivide(chan_ind)
                if self.take_ideal_sampling_rate:
                    sr = smrx.GetIdealRate(chan_ind)
                else:
                    sr = 1. / (smrx.GetTimeBase() * divide)
                max_time = smrx.ChannelMaxTime(chan_ind)
                first_time = smrx.FirstTime(chan_ind, 0, max_time)
                # max_times is included so +1
                time_size = (max_time - first_time) / divide + 1
                channel_infos.append((first_time, max_time, divide, time_size, sr))
                gain = smrx.GetChannelScale(chan_ind) / 6553.6
                offset = smrx.GetChannelOffset(chan_ind)
                units = smrx.GetChannelUnits(chan_ind)
                ch_name = smrx.GetChannelTitle(chan_ind)

                dtype = 'int16'
                # set later after grouping
                stream_id = '0'
                signal_channels.append((ch_name, chan_id, sr, dtype,
                                        units, gain, offset, stream_id))

            elif chan_type == sonpy.lib.DataType.AdcMark:
                # spike and waveforms : only spike times is used here
                ch_name = smrx.GetChannelTitle(chan_ind)
                first_time = smrx.FirstTime(chan_ind, 0, max_time)
                max_time = smrx.ChannelMaxTime(chan_ind)
                divide = smrx.ChannelDivide(chan_ind)
                # here we don't use filter (sonpy.lib.MarkerFilter()) so we get all marker
                wave_marks = smrx.ReadWaveMarks(chan_ind, int(max_time / divide), 0, max_time)

                # here we load in memory all spike once because the access is really slow
                # with the ReadWaveMarks
                spike_ticks = np.array([t.Tick for t in wave_marks])
                spike_codes = np.array([t.Code1 for t in wave_marks])

                unit_ids = np.unique(spike_codes)
                for unit_id in unit_ids:
                    name = f'{ch_name}#{unit_id}'
                    spike_chan_id = f'ch{chan_id}#{unit_id}'
                    spike_channels.append((name, spike_chan_id, '', 1, 0, 0, 0))
                    mask = spike_codes == unit_id
                    self._all_spike_ticks[spike_chan_id] = spike_ticks[mask]

        signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype)

        # channels are grouped into stream if they have a common start, stop, size, divide and sampling_rate
        channel_infos = np.array(channel_infos,
                    dtype=[('first_time', 'i8'), ('max_time', 'i8'),
                           ('divide', 'i8'), ('size', 'i8'), ('sampling_rate', 'f8')])
        unique_info = np.unique(channel_infos)
        self.stream_info = unique_info
        signal_streams = []
        for i, info in enumerate(unique_info):
            stream_id = str(i)
            mask = channel_infos == info
            signal_channels['stream_id'][mask] = stream_id
            num_chans = np.sum(mask)
            stream_name = f'{stream_id} {num_chans}chans'
            signal_streams.append((stream_name, stream_id))
        signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype)

        # spike channels not handled
        spike_channels = np.array(spike_channels, dtype=_spike_channel_dtype)

        # event channels not handled
        event_channels = []
        event_channels = np.array(event_channels, dtype=_event_channel_dtype)

        self._seg_t_start = np.inf
        self._seg_t_stop = -np.inf
        for info in self.stream_info:
            self._seg_t_start = min(self._seg_t_start,
                                    info['first_time'] * self._time_base)

            self._seg_t_stop = max(self._seg_t_stop,
                                   info['max_time'] * self._time_base)

        self.header = {}
        self.header['nb_block'] = 1
        self.header['nb_segment'] = [1]
        self.header['signal_streams'] = signal_streams
        self.header['signal_channels'] = signal_channels
        self.header['spike_channels'] = spike_channels
        self.header['event_channels'] = event_channels

        self._generate_minimal_annotations()

    def _segment_t_start(self, block_index, seg_index):
        return self._seg_t_start

    def _segment_t_stop(self, block_index, seg_index):
        return self._seg_t_stop

    def _get_signal_size(self, block_index, seg_index, stream_index):
        size = self.stream_info[stream_index]['size']
        return size

    def _get_signal_t_start(self, block_index, seg_index, stream_index):
        info = self.stream_info[stream_index]
        t_start = info['first_time'] * self._time_base
        return t_start

    def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop,
                                stream_index, channel_indexes):

        if i_start is None:
            i_start = 0
        if i_stop is None:
            i_stop = self.stream_info[stream_index]['size']

        stream_id = self.header['signal_streams']['id'][stream_index]
        signal_channels = self.header['signal_channels']
        mask = signal_channels['stream_id'] == stream_id
        signal_channels = signal_channels[mask]
        if channel_indexes is not None:
            signal_channels = signal_channels[channel_indexes]

        num_chans = len(signal_channels)

        size = i_stop - i_start
        sigs = np.zeros((size, num_chans), dtype='int16')

        info = self.stream_info[stream_index]
        t_from = info['first_time'] + info['divide'] * i_start
        t_upto = info['first_time'] + info['divide'] * i_stop

        for i, chan_id in enumerate(signal_channels['id']):
            chan_ind = int(chan_id)
            sig = self.smrx_file.ReadInts(chan=chan_ind,
                    nMax=size, tFrom=t_from, tUpto=t_upto)
            sigs[:, i] = sig

        return sigs

    def _spike_count(self, block_index, seg_index, unit_index):
        unit_id = self.header['spike_channels'][unit_index]['id']
        spike_ticks = self._all_spike_ticks[unit_id]
        return spike_ticks.size

    def _get_spike_timestamps(self, block_index, seg_index, unit_index, t_start, t_stop):
        unit_id = self.header['spike_channels'][unit_index]['id']
        spike_ticks = self._all_spike_ticks[unit_id]
        if t_start is not None:
            tick_start = int(t_start / self._time_base)
            spike_ticks = spike_ticks[spike_ticks >= tick_start]
        if t_stop is not None:
            tick_stop = int(t_stop / self._time_base)
            spike_ticks = spike_ticks[spike_ticks <= tick_stop]
        return spike_ticks

    def _rescale_spike_timestamp(self, spike_timestamps, dtype):
        spike_times = spike_timestamps.astype(dtype)
        spike_times *= self._time_base
        return spike_times

    def _get_spike_raw_waveforms(self, block_index, seg_index,
                                 spike_channel_index, t_start, t_stop):
        return None

max_stars_repo_name: Mario-Kart-Felix/python-neo

max_issues_repo_name: Mario-Kart-Felix/python-neo

max_forks_repo_name: Mario-Kart-Felix/python-neo


text:
"""
* By M.K
* Licensed under the MIT License.
* https://github.com/MI-K253
* https://github.com/MI-K253/Python-Plays-Game
"""

import KeyboardControl
import GetFrame
import time
import cv2
from tensorflow import keras
import numpy as np
import MotionDetection
import threading
import keyboard

ImageWeight = 224
ImageHeight = 224

Classes = {0: 'Forward', 1: 'Left', 2: 'Right'}         # , 3: 'Reverse', 4: 'FL', 5: 'FR', 6: 'RL', 7: 'RF', 8: 'Stop'}

KeyboardDelay = 0.1
MaxSpeed = 40
ReverseDelay = 1                                          # For motion detection & high speed
MotionThreshold = 1000

CountdownNumber = 5
CountdownDelay = 0.5

Model = keras.models.load_model(r"Trained-Models/Trained-Alexnet-64.h5")

def keyboard_handler(result, delay):

    if result == 0:
        KeyboardControl.left(delay*1.2)

    elif result == 1:
        KeyboardControl.straight(delay)

    elif result == 2:
        KeyboardControl.right(delay*1.2)

def countdown(num, delay):
    for i in list(range(num))[::-1]:
        if i == 0:
            print("Start driving!")
            continue
        print(i)
        time.sleep(delay)

def main():
    countdown(CountdownNumber, CountdownDelay)
    motion_detected = True

    while True:
        if not motion_detected:
            KeyboardControl.reverse(3)

        start_time = time.time()

        frame = GetFrame.get()

        motion_detected = True

        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        frame = cv2.resize(frame, (ImageWeight, ImageHeight))

        frame = np.array(frame)
        frame = frame.reshape(1, ImageHeight, ImageWeight, 1)
        frame = frame/255

        prediction = Model.predict(frame)
        prediction = np.argmax(prediction)
        keyboard_handler(prediction, KeyboardDelay)

        print(f"Chice= {prediction}\tProcess took {str(time.time() - start_time)[0: 4]} seconds.")

if __name__ == '__main__':
    main()

# END


text:
# -*- coding: utf-8 -*-
"""
Created on Mon Jun 22 18:29:17 2020

@author: Ivan Nemov
"""

import cv2
import time
import numpy as np
from PyQt5 import QtCore, QtGui
from threading import Thread
from picamera.array import PiRGBArray
from picamera import PiCamera

class detection_message(QtCore.QObject):
    detected_coordinates_signal = QtCore.pyqtSignal(list)                 #update coordinates of detedcted person
    video_frame_signal = QtCore.pyqtSignal(QtGui.QImage)

class detection(Thread):
    def __init__(self, _signal_message):
        Thread.__init__(self)
        self._signal_message = _signal_message
        self.destroyed = False
        self.preview_enabled = True

    def destroy(self):
        self.destroyed = True

    def change_preview(self, enabled):
        self.preview_enabled = enabled

    def run(self):

        camera = PiCamera()
        w = 640
        h = 480
        camera.resolution = (w, h)
        camera.framerate = 32
        camera.vflip = True

        cap = PiRGBArray(camera, size=(w, h))
        time.sleep(0.1)

        detector = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')

        detect_object = False

        for frame in camera.capture_continuous(cap, format='bgr', use_video_port = True):
            # sample rate
            time.sleep(0.005)

            # Capture frame-by-frame
            image = frame.array

            if detect_object == False:
                try:
                    # detect people in the image
                    rectangles_w_h, levels, weights = detector.detectMultiScale3(image, scaleFactor=1.05, minNeighbors=9, minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE, outputRejectLevels=True)

                    # distance to centers
                    centers = np.array([pow((pow((x+0.5*(width-w)),2)+pow((y+0.5*(hight-h)),2)),0.5) for (x, y, width, hight) in rectangles_w_h], dtype=np.int16)

                    # pick people closest to image center
                    if centers.size != 0:
                        if weights[np.argmin(centers)] > 6:
                            roi = tuple(rectangles_w_h[np.argmin(centers),:])
                        else:
                            roi = ()
                    else:
                        roi = ()

                    # initialize tracking
                    if roi != ():
                        tracker = None
                        tracker = cv2.TrackerCSRT_create()
                        successTracking = tracker.init(image, roi)
                        if successTracking:
                            detect_object = True
                    else:
                        successTracking = False
                except:
                    roi = ()
                    successTracking = False
                    detect_object = False

                # set timer to re-run detection
                time_of_detection = time.time()

            # track detected object
            if detect_object and successTracking:
                successTracking, roi = tracker.update(image)
                roi=tuple(roi)
                if successTracking == False:
                    detect_object = False
                    roi = ()
            else:
                detect_object = False
                roi = ()

            # re-run detection on timeout
            if (time.time() - time_of_detection) > 30:
                detect_object = False

            # estimate center of detection and mark-up iage
            if roi != ():
                x_avg_np = int(roi[0]+0.5*roi[2])
                y_avg_np = int(roi[1]+0.5*roi[3])
                if self.preview_enabled:
                    cv2.circle(image, (x_avg_np, y_avg_np), 80, (0, 255, 0), 1)
            else:
                x_avg_np = -888
                y_avg_np = -888

            # clear the stream in preparation for the next frame
            cap.truncate(0)

            # send coordinates
            self._signal_message.detected_coordinates_signal.emit([x_avg_np, y_avg_np, 0.5*w, 0.5*h])

            # Send the resulting frame to main thread
            if self.preview_enabled:
                Q_Image = QtGui.QImage(image.data, image.shape[1], image.shape[0], QtGui.QImage.Format_RGB888).rgbSwapped()
                self._signal_message.video_frame_signal.emit(Q_Image)

            # exit loop when destroyed by main programm
            if self.destroyed:
                break

        # When everything done, release the camera and clear objects
        camera.close()
        detector = None
        tracker = None


text:
import tensorflow as tf
# tf.disable_v2_behavior()

# def check_tf_version():
#   version = tf.__version__
#   print("==tf version==", version)
#   if int(version.split(".")[0]) >= 2 or int(version.split(".")[1]) >= 15:
#     return True
#   else:
#     return False
# if check_tf_version():
#   import tensorflow.compat.v1 as tf
#   tf.disable_v2_behavior()

def check_tf_version():
  version = tf.__version__
  print("==tf version==", version)
  if int(version.split(".")[0]) >= 2 or int(version.split(".")[1]) >= 15:
    return True
  else:
    return False
if check_tf_version():
  tf.disable_v2_behavior()

# from model.vqvae_utils import tfidf_utils
import collections
import copy
import json
import math
import re

import numpy as np
import six
import tensorflow as tf

class ETMConfig(object):
  def __init__(self,
               vocab_size,
               topic_size=768,
               hidden_size=768,
               embedding_size=768,
               num_hidden_layers=12,
               hidden_act="gelu",
               hidden_dropout_prob=0.1,
               initializer_range=0.02,
               apply_bn_vae_mean=True,
               apply_bn_vae_var=True):

    self.vocab_size = vocab_size
    self.topic_size = topic_size
    self.hidden_size = hidden_size
    self.embedding_size = embedding_size
    self.num_hidden_layers = num_hidden_layers
    self.hidden_act = hidden_act
    self.hidden_dropout_prob = hidden_dropout_prob
    self.initializer_range = initializer_range
    self.apply_bn_vae_mean = apply_bn_vae_mean
    self.apply_bn_vae_var = apply_bn_vae_var

  @classmethod
  def from_dict(cls, json_object):
    """Constructs a `BertConfig` from a Python dictionary of parameters."""
    config = ETMConfig(vocab_size=None)
    for (key, value) in six.iteritems(json_object):
      config.__dict__[key] = value
    return config

  @classmethod
  def from_json_file(cls, json_file):
    """Constructs a `BertConfig` from a json file of parameters."""
    with tf.gfile.GFile(json_file, "r") as reader:
      text = reader.read()
    return cls.from_dict(json.loads(text))

  def to_dict(self):
    """Serializes this instance to a Python dictionary."""
    output = copy.deepcopy(self.__dict__)
    return output

  def to_json_string(self):
    """Serializes this instance to a JSON string."""
    return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n"

class ETM(object):
  def __init__(self,
                etm_config,
               input_term_count,
               input_term_binary,
               input_term_freq,
               is_training=False,
               embedding_matrix=None,
               hidden_vector=None,
               scope=None,
               input_type="term_count"):
    """
    https://github.com/linkstrife/NVDM-GSM/blob/master/GSM.py
    https://github.com/adjidieng/ETM/blob/master/etm.py
    compared to NVDM, GSM or ETM add topic-word-matrix-alignment
    """
    etm_config = copy.deepcopy(etm_config)
    if not is_training:
      etm_config.hidden_dropout_prob = 0.0

    if input_type == 'term_count':
      tf.logging.info("*** model_input term_count ***")
      model_input = tf.identity(input_term_count)
    elif input_type == 'term_binary':
      tf.logging.info("*** model_input term_binary ***")
      model_input = tf.identity(input_term_binary)
    elif input_type == 'term_freq':
      tf.logging.info("*** model_input term_freq ***")
      model_input = tf.identity(input_term_freq)
    else:
      tf.logging.info("*** model_input term_freq ***")
      model_input = tf.identity(input_term_freq)

    with tf.variable_scope("etm", scope):
      with tf.variable_scope("encoder"):
        # [batch_size, hidden_size]
        self.q_theta = mlp(input_tensor=model_input,
                          num_hidden_layers=etm_config.num_hidden_layers,
                          hidden_size=etm_config.hidden_size,
                          is_training=is_training,
                          dropout_prob=etm_config.hidden_dropout_prob,
                          intermediate_act_fn=get_activation(etm_config.hidden_act),
                          initializer_range=etm_config.initializer_range,
                          scope="bow_mlp")

    if hidden_vector is not None:
      with tf.variable_scope("etm", scope):
        if hidden_vector.shape[-1] != etm_config.hidden_size:
          self.hidden_vector = tf.layers.dense(
              hidden_vector, etm_config.hidden_size,
              name="hidden_vector_project")
      self.q_theta = tf.concat([self.q_theta, self.hidden_vector], axis=-1)

    with tf.variable_scope("etm", scope):
      with tf.variable_scope("bridge"):
        # [batch_size, hidden_size]
        # use bias is false since we will apply bn
        self.mu_q_theta = mlp(
                              input_tensor=self.q_theta,
                              num_hidden_layers=1,
                              hidden_size=etm_config.topic_size,
                              is_training=is_training,
                              dropout_prob=etm_config.hidden_dropout_prob,
                              intermediate_act_fn=None,
                              initializer_range=etm_config.initializer_range,
                              scope="mu_theta_mlp",
                              matrix_start_zero=False,
                              use_bias=True if not etm_config.apply_bn_vae_mean else False
                              )
        tf.logging.info("*** mu_q_theta ***")
        tf.logging.info(self.mu_q_theta)

        if etm_config.apply_bn_vae_mean:
          with tf.variable_scope("vae_mu_bn"):
            self.mu_q_theta = tf.layers.batch_normalization(
                      self.mu_q_theta,
                      training=is_training,
                      scale=False,
                      center=False,
                      epsilon=1e-8,
                      axis=-1
              )
            self.mu_q_theta = scalar_layer(self.mu_q_theta, tau=0.5,
              mode='positive', initializer_range=0.02)

            tf.logging.info("*** after bn mu_q_theta ***")
            tf.logging.info(self.mu_q_theta)

        # zero logsigma and simga is set 1
        self.sigma_std_q_theta = mlp(
                              input_tensor=self.q_theta,
                              num_hidden_layers=1,
                              hidden_size=etm_config.topic_size,
                              is_training=is_training,
                              dropout_prob=etm_config.hidden_dropout_prob,
                              intermediate_act_fn=None,
                              initializer_range=etm_config.initializer_range,
                              scope="sigma_std_mlp",
                              matrix_start_zero=False,
                              use_bias=True if not etm_config.apply_bn_vae_var else False
                              )

        tf.logging.info("*** sigma_std_q_theta ***")
        tf.logging.info(self.sigma_std_q_theta)

        if etm_config.apply_bn_vae_var:
          with tf.variable_scope("vae_sigma_std_bn"):
            self.sigma_std_q_theta = tf.layers.batch_normalization(
                      self.sigma_std_q_theta,
                      training=is_training,
                      scale=False,
                      center=False,
                      epsilon=1e-8,
                      axis=-1
              )
            self.sigma_std_q_theta = scalar_layer(self.sigma_std_q_theta, tau=0.5,
              mode='negative', initializer_range=0.02)

            tf.logging.info("*** after bn sigma_std_q_theta ***")
            tf.logging.info(self.sigma_std_q_theta)

    with tf.variable_scope("etm", scope):
      with tf.variable_scope("reparameterize"):
        self.z = reparameterize(
                        mu_q_theta=self.mu_q_theta,
                        sigma_std_q_theta=self.sigma_std_q_theta,
                        is_training=is_training)

        tf.logging.info("*** reparameterize z ***")
        tf.logging.info(self.z)

      with tf.variable_scope("gsm"):
        # [batch_size, hidden_size]
        self.z_gsm = mlp(input_tensor=self.z,
                          num_hidden_layers=etm_config.num_hidden_layers,
                          hidden_size=etm_config.topic_size,
                          is_training=is_training,
                          dropout_prob=etm_config.hidden_dropout_prob,
                          intermediate_act_fn=None,
                          initializer_range=etm_config.initializer_range,
                          scope="decoder")

        tf.logging.info("*** z_gsm ***")
        tf.logging.info(self.z_gsm)

        # [batch_size, topic_size]
        self.theta = tf.nn.softmax(self.z_gsm, dim=-1)

        tf.logging.info("*** theta ***")
        tf.logging.info(self.theta)

      with tf.variable_scope("embeddings"):
        if embedding_matrix is None:
          self.embedding_table = tf.get_variable(
                name="vocab_word_embeddings",
                shape=[etm_config.vocab_size, etm_config.embedding_size],
                initializer=create_initializer(etm_config.initializer_range))
        else:
          self.embedding_table = tf.get_variable(
                name="vocab_word_embeddings",
                shape=[etm_config.vocab_size, etm_config.embedding_size],
                initializer=tf.constant_initializer(embedding_matrix, dtype=tf.float32),
                trainable=False)

        tf.logging.info("*** vocab_word_embeddings ***")
        tf.logging.info(self.embedding_table)

      with tf.variable_scope("embeddings"):
        self.topic_embedding_table = tf.get_variable(
                name="topic_word_embeddings",
                shape=[etm_config.topic_size, etm_config.embedding_size],
                initializer=create_initializer(etm_config.initializer_range))

        tf.logging.info("*** topic_word_embeddings ***")
        tf.logging.info(self.topic_embedding_table)

      # topic_embedding_table: [topic_size, embedding_size]
      # embedding_table: [vocab_size, embedding_size]
      with tf.variable_scope("decoder"):
        self.topic_word_align = tf.matmul(self.topic_embedding_table,
                                        self.embedding_table,
                                        transpose_b=True)

        # self.topic_word_align = tf.multiply(self.topic_word_align,
        #                          1.0 / math.sqrt(float(etm_config.embedding_size)))

        tf.logging.info("*** topic_word_align ***")
        tf.logging.info(self.topic_word_align)

        # [topic_size, vocab_size]
        self.beta = tf.nn.softmax(self.topic_word_align, axis=-1)
        tf.logging.info("*** beta ***")
        tf.logging.info(self.beta)

        # theta: [batch_size, topic_size]
        # beta : [topic_size, vocab_size]
        # preds: [batch_size, vocab_size]
        # preds needs to be log-softmax that normalized on vocab-size dims
        self.preds = tf.log(tf.matmul(self.theta, self.beta)+1e-10)

        tf.logging.info("*** preds ***")
        tf.logging.info(self.preds)

      self.per_example_recon_loss = -tf.reduce_sum(self.preds * tf.stop_gradient(model_input), axis=-1)
      self.recon_loss = tf.reduce_mean(self.per_example_recon_loss)

      self.sigma_q_theta = tf.pow(self.sigma_std_q_theta, 2.0)
      self.logsigma_q_theta = tf.log(self.sigma_q_theta+1e-10)
      self.per_example_kl_theta_loss = -0.5 * tf.reduce_sum(1 + self.logsigma_q_theta - tf.pow(self.mu_q_theta, 2) - self.sigma_q_theta, axis=-1)
      self.kl_theta_loss = tf.reduce_mean(self.per_example_kl_theta_loss)

  def get_hidden_vector(self):
    return self.z

  def get_vocab_word_embeddings(self):
    return self.embedding_table

  def get_topic_embedding_table(self):
    return self.topic_embedding_table

  def get_recon_loss(self):
    return self.recon_loss

  def get_kl_loss(self):
    return self.kl_theta_loss

def scalar_layer(input_tensor, tau=0.5,
              mode='positive', initializer_range=0.02):
  with tf.variable_scope("vae_bn_scale"):
    scale = tf.get_variable(
                  name="scale",
                  shape=[input_tensor.shape[-1]],
                  initializer=create_initializer(initializer_range))
  if mode == 'positive':
      scale = tau + (1 - tau) * tf.nn.sigmoid(scale)
  else:
      scale = (1 - tau) * tf.nn.sigmoid(-scale)
  return input_tensor * tf.sqrt(scale)

def gelu(input_tensor):
  """Gaussian Error Linear Unit.

  This is a smoother version of the RELU.
  Original paper: https://arxiv.org/abs/1606.08415

  Args:
    input_tensor: float Tensor to perform activation.

  Returns:
    `input_tensor` with the GELU activation applied.
  """
  cdf = 0.5 * (1.0 + tf.erf(input_tensor / tf.sqrt(2.0)))
  return input_tensor * cdf

def create_initializer(initializer_range=0.02):
  """Creates a `truncated_normal_initializer` with the given range."""
  return tf.truncated_normal_initializer(stddev=initializer_range)

def create_zero_initializer(initializer_range=0.02):
  return tf.zeros_initializer()

def get_activation(activation_string):
  """Maps a string to a Python function, e.g., "relu" => `tf.nn.relu`.

  Args:
    activation_string: String name of the activation function.

  Returns:
    A Python function corresponding to the activation function. If
    `activation_string` is None, empty, or "linear", this will return None.
    If `activation_string` is not a string, it will return `activation_string`.

  Raises:
    ValueError: The `activation_string` does not correspond to a known
      activation.
  """

  # We assume that anything that"s not a string is already an activation
  # function, so we just return it.
  if not isinstance(activation_string, six.string_types):
    return activation_string

  if not activation_string:
    return None

  act = activation_string.lower()
  if act == "linear":
    return None
  elif act == "relu":
    return tf.nn.relu
  elif act == "gelu":
    return gelu
  elif act == "tanh":
    return tf.tanh
  else:
    raise ValueError("Unsupported activation: %s" % act)

def tokenid2bow(input_ids, vocab_size):
  [term_count,
  term_binary,
  term_freq] = tfidf_utils.tokenid2tf(input_ids, vocab_size)

  return term_count, term_binary, term_freq

def mlp(input_tensor,
        num_hidden_layers,
        hidden_size,
        is_training,
        dropout_prob,
        intermediate_act_fn,
        initializer_range,
        scope=None,
        use_bias=True,
        matrix_start_zero=False
        ):
  prev_output = input_tensor
  if matrix_start_zero:
    kernel_initializer = create_zero_initializer(initializer_range)
    tf.logging.info("** apply zero initializer **")
  else:
    kernel_initializer = create_initializer(initializer_range)
    tf.logging.info("** apply truncated normal initializer **")

  with tf.variable_scope(scope, default_name="mlp"):
    for layer_idx in range(num_hidden_layers):
      with tf.variable_scope("layer_%d" % layer_idx):
        layer_input = prev_output
        layer_output = tf.layers.dense(
                layer_input,
                hidden_size,
                kernel_initializer=kernel_initializer,
                bias_initializer=tf.zeros_initializer(),
                activation=intermediate_act_fn,
                use_bias=use_bias)
        prev_output = layer_output
    final_outputs = prev_output
    return final_outputs

def reparameterize(mu_q_theta, sigma_std_q_theta, is_training):
  if is_training:
    sigma_q_theta = sigma_std_q_theta
    eps = tf.random.normal(get_shape_list(sigma_q_theta),
                            mean=0.0, stddev=1.0, dtype=tf.float32)
    return eps*sigma_q_theta+mu_q_theta
  else:
    return mu_q_theta

def get_shape_list(tensor, expected_rank=None, name=None):
  """Returns a list of the shape of tensor, preferring static dimensions.

  Args:
    tensor: A tf.Tensor object to find the shape of.
    expected_rank: (optional) int. The expected rank of `tensor`. If this is
      specified and the `tensor` has a different rank, and exception will be
      thrown.
    name: Optional name of the tensor for the error message.

  Returns:
    A list of dimensions of the shape of tensor. All static dimensions will
    be returned as python integers, and dynamic dimensions will be returned
    as tf.Tensor scalars.
  """
  if isinstance(tensor, np.ndarray) or isinstance(tensor, list):
    shape = np.array(tensor).shape
    if isinstance(expected_rank, six.integer_types):
      assert len(shape) == expected_rank
    elif expected_rank is not None:
      assert len(shape) in expected_rank
    return shape

  if name is None:
    name = tensor.name

  if expected_rank is not None:
    assert_rank(tensor, expected_rank, name)

  shape = tensor.shape.as_list()

  non_static_indexes = []
  for (index, dim) in enumerate(shape):
    if dim is None:
      non_static_indexes.append(index)

  if not non_static_indexes:
    return shape

  dyn_shape = tf.shape(tensor)
  for index in non_static_indexes:
    shape[index] = dyn_shape[index]
  return shape

def reshape_to_matrix(input_tensor):
  """Reshapes a >= rank 2 tensor to a rank 2 tensor (i.e., a matrix)."""
  ndims = input_tensor.shape.ndims
  if ndims < 2:
    raise ValueError("Input tensor must have at least rank 2. Shape = %s" %
                     (input_tensor.shape))
  if ndims == 2:
    return input_tensor

  width = input_tensor.shape[-1]
  output_tensor = tf.reshape(input_tensor, [-1, width])
  return output_tensor

def reshape_from_matrix(output_tensor, orig_shape_list):
  """Reshapes a rank 2 tensor back to its original rank >= 2 tensor."""
  if len(orig_shape_list) == 2:
    return output_tensor

  output_shape = get_shape_list(output_tensor)

  orig_dims = orig_shape_list[0:-1]
  width = output_shape[-1]

  return tf.reshape(output_tensor, orig_dims + [width])

def assert_rank(tensor, expected_rank, name=None):
  """Raises an exception if the tensor rank is not of the expected rank.

  Args:
    tensor: A tf.Tensor to check the rank of.
    expected_rank: Python integer or list of integers, expected rank.
    name: Optional name of the tensor for the error message.

  Raises:
    ValueError: If the expected shape doesn't match the actual shape.
  """
  if name is None:
    name = tensor.name

  expected_rank_dict = {}
  if isinstance(expected_rank, six.integer_types):
    expected_rank_dict[expected_rank] = True
  else:
    for x in expected_rank:
      expected_rank_dict[x] = True

  actual_rank = tensor.shape.ndims
  if actual_rank not in expected_rank_dict:
    scope_name = tf.get_variable_scope().name
    raise ValueError(
        "For the tensor `%s` in scope `%s`, the actual rank "
        "`%d` (shape = %s) is not equal to the expected rank `%s`" %
        (name, scope_name, actual_rank, str(tensor.shape), str(expected_rank)))

def get_assigment_map_from_checkpoint(tvars, init_checkpoint):
  """Compute the union of the current variables and checkpoint variables."""
  assignment_map = {}
  initialized_variable_names = {}

  name_to_variable = collections.OrderedDict()
  for var in tvars:
    name = var.name
    m = re.match("^(.*):\\d+$", name)
    if m is not None:
      name = m.group(1)
    name_to_variable[name] = var

  init_vars = tf.train.list_variables(init_checkpoint)

  assignment_map = collections.OrderedDict()
  for x in init_vars:
    (name, var) = (x[0], x[1])
    if name not in name_to_variable:
      continue
    if var != name_to_variable[name].shape.as_list():
      continue
    assignment_map[name] = name
    initialized_variable_names[name] = 1
    initialized_variable_names[name + ":0"] = 1

  return (assignment_map, initialized_variable_names)


text:
# Estimate entropy
import numpy as np
from sklearn.neighbors import KDTree
from scipy.special import digamma

def get_nearest_neighbor_distances(X, k=3):
    """For each point in X, get the distance to its k-th nearest neighbor"""
    tree = KDTree(X, p=np.inf)
    dist, ind = tree.query(X, k=k+1)
    return dist[:,-1]

def estimate_entropy(X, k=3):
    """Estimate the differential entropy, using the distribution of k-nearest-neighbor
    distances

    Reference:
        https://github.com/gregversteeg/NPEET/blob/master/entropy_estimators.py"""
    if len(X.shape) == 1:
        X = X.reshape((len(X), 1))
    n, dimension = X.shape
    const = digamma(n) - digamma(k) + dimension * np.log(2)
    nearest_neighbor_distances = get_nearest_neighbor_distances(X, k)
    eps = np.random.rand(n) * 1e-10
    return (const + dimension * np.mean(np.log(nearest_neighbor_distances + eps)))

def estimate_marginal_entropies(X, k=3):
    """Estimate the entropy of each marginal of X."""
    marginal_entropies = np.zeros(X.shape[1])
    for i in range(X.shape[1]):
        marginal_entropies[i] = estimate_entropy(X[:,i], k)
    return marginal_entropies

def get_nearest_neighbor_distances_by_sorting(x):
    """1D is easy because we can sort:
    sort the array, then for each point, get its distance to the points immediately before
    or after it. Take the minimum of these two...
    """

    sorted_x = np.array(sorted(x))

    # distance from x_i to x_{i+1} for i in [1, N-1]
    distance_to_next = np.abs(sorted_x[2:] - sorted_x[1:-1])

    # distance from x_i to x_{i-1} for i in [1, N-1]
    distance_to_previous = np.abs(sorted_x[1:-1] - sorted_x[:-2])

    return np.minimum(distance_to_next, distance_to_previous)

max_stars_repo_name: choderalab/integrator-benchmark

max_issues_repo_name: choderalab/integrator-benchmark

max_forks_repo_name: choderalab/integrator-benchmark


text:
import os
import time
import numpy as np
import redis
from IPython.display import clear_output
from PIL import Image
from io import BytesIO
import base64
import json
import matplotlib.pyplot as plt
from face_detection import get_face
from utils import img_to_txt, decode_img, log_error

##########################
#
#   Global Variables
#
#
##########################

# Selecting Server
server = os.environ['redis_server'] if 'os.environ' in os.environ and len(os.environ['redis_server']) > 1 else 'localhost'
# connect with redis server as Bob
r = redis.Redis(host=server, port=6379)
# Publish and suscribe redis
req_p = r.pubsub()
# subscribe to request Channel
req_p.subscribe('new_request')

def process_request(request ):
    '''
    Do you request processing here
    '''
    im =  decode_img(request['image'])
    face = get_face(im)
    plt.imshow(face)
    plt.show()

def listen_stream():
    '''
    Listening to the stream.

    IF got any request from the stream then process it at the same time.
    '''
    count = 0
    requests =[]
    while 1:

        try:
            try:
                # Listening To the stream
                request = str(req_p.get_message()['data'].decode())
                if request is not None :requests.append(request)
            except TypeError as e: log_error(e)

            # If got any request from stream then process the function
            if len(requests) > 0:
                req_id = requests.pop(0)
                process_request(json.loads(request) )
                count += 1

            print(count)

        except Exception as e: log_error(e)

listen_stream()

max_stars_repo_name: SohaibAnwaar/Skin-care

max_issues_repo_name: SohaibAnwaar/Skin-care

max_forks_repo_name: SohaibAnwaar/Skin-care


text:
import torch
import numpy as np
import pickle

import torchvision
import h5py

class FeatureLoader(torch.utils.data.Dataset):

    def __init__(self, path_features, get_item_id=False, split='train'):
        self.path_features = path_features

        self.get_item_id = get_item_id
        self.split=split

        self.data = h5py.File(self.path_features, 'r')[self.split]

        self.unique_target = []
        for elem in self.data['target']:
            if elem not in self.unique_target:
                self.unique_target.append(elem)

    def __getitem__(self, index):
        """
        [!] FPS jittering doesn't work with AV dataloader as of now
        """

        item = self.data['data'][index]
        target = self.unique_target.index(self.data['target'][index])
        ind = self.data['video_id'][index]

        # format data to torch
        if self.get_item_id:
            return (item, target, ind)
        else:
            return (item, target)

    def __len__(self):
        return len(self.data['target'])

if __name__ == '__main__':

    loader = FeatureLoader(path_features="/workspace1/jahurtado/20BN-SOMETHING-SOMETHING-V2-1/s2s_feats_10percent.h5")

    import time
    from tqdm import tqdm

    batch_loader = torch.utils.data.DataLoader(
        loader,
        batch_size=128, shuffle=False,
        num_workers=0, pin_memory=True)

    start = time.time()
    for i, a in enumerate(batch_loader):
        print(a[0].shape)
        print(a[1].shape)
        #if i > 10:
        #    break
        pass
    print("Size --> {}".format(a[0].size()))
    print(time.time() - start)


text:
# -*- coding: utf-8 -*-
# @Time    : 10/19/20 5:15 AM
# @Author  : Yuan Gong
# @Affiliation  : Massachusetts Institute of Technology
# @Email   : yuangong@mit.edu
# @File    : prep_esc50.py

import numpy as np
import json
import os
import zipfile
import wget

# label = np.loadtxt('/data/sls/scratch/yuangong/aed-pc/src/utilities/esc50_label.csv', delimiter=',', dtype='str')
# f = open("/data/sls/scratch/yuangong/aed-pc/src/utilities/esc_class_labels_indices.csv", "w")
# f.write("index,mid,display_name\n")
#
# label_set = []
# idx = 0
# for j in range(0, 5):
#     for i in range(0, 10):
#         cur_label = label[i][j]
#         cur_label = cur_label.split(' ')
#         cur_label = "_".join(cur_label)
#         cur_label = cur_label.lower()
#         label_set.append(cur_label)
#         f.write(str(idx)+',/m/07rwj'+str(idx).zfill(2)+',\"'+cur_label+'\"\n')
#         idx += 1
# f.close()
#

def get_immediate_subdirectories(a_dir):
    return [name for name in os.listdir(a_dir) if os.path.isdir(os.path.join(a_dir, name))]

def get_immediate_files(a_dir):
    return [name for name in os.listdir(a_dir) if os.path.isfile(os.path.join(a_dir, name))]

# downlooad esc50
# dataset provided in https://github.com/karolpiczak/ESC-50
if False:#os.path.exists('./data/ESC-50-master') == False:
    esc50_url = 'https://github.com/karoldvl/ESC-50/archive/master.zip'
    wget.download(esc50_url, out='./data/')
    with zipfile.ZipFile('./data/ESC-50-master.zip', 'r') as zip_ref:
        zip_ref.extractall('./data/')
    os.remove('./data/ESC-50-master.zip')

    # convert the audio to 16kHz
    base_dir = './data/ESC-50-master/'
    os.mkdir('./data/ESC-50-master/audio_16k/')
    audio_list = get_immediate_files('./data/ESC-50-master/audio')
    for audio in audio_list:
        print('sox ' + base_dir + '/audio/' + audio + ' -r 16000 ' + base_dir + '/audio_16k/' + audio)
        os.system('sox ' + base_dir + '/audio/' + audio + ' -r 16000 ' + base_dir + '/audio_16k/' + audio)

label_set = np.loadtxt('./data/esc_class_labels_indices.csv', delimiter=',', dtype='str')
label_map = {}
for i in range(1, len(label_set)):
    label_map[eval(label_set[i][2])] = label_set[i][0]
print(label_map)

# fix bug: generate an empty directory to save json files
if os.path.exists('./data/datafiles') == False:
    os.mkdir('./data/datafiles')

for fold in [1,2,3,4,5]:
    base_path = "./data/ESC-50-master/audio_16k/"
    meta = np.loadtxt('./data/ESC-50-master/meta/esc50.csv', delimiter=',', dtype='str', skiprows=1)
    train_wav_list = []
    eval_wav_list = []
    for i in range(0, len(meta)):
        cur_label = label_map[meta[i][3]]
        cur_path = meta[i][0]
        cur_fold = int(meta[i][1])
        # /m/07rwj is just a dummy prefix
        cur_dict = {"wav": base_path + cur_path, "labels": '/m/07rwj'+cur_label.zfill(2)}
        if cur_fold == fold:
            eval_wav_list.append(cur_dict)
        else:
            train_wav_list.append(cur_dict)

    #print('fold {:d}: {:d} training samples, {:d} test samples'.format(fold, len(train_wav_list), len(eval_wav_list)))

    #print(
    with open('./data/datafiles/esc_train_data_'+ str(fold) +'.json', 'w') as f:
        json.dump({'data': train_wav_list}, f, indent=1)

    with open('./data/datafiles/esc_eval_data_'+ str(fold) +'.json', 'w') as f:
        json.dump({'data': eval_wav_list}, f, indent=1)

print('Finished ESC-50 Preparation')

max_stars_repo_name: tejasvaidhyadev/IA-for-AI

max_issues_repo_name: tejasvaidhyadev/IA-for-AI

max_forks_repo_name: tejasvaidhyadev/IA-for-AI


text:
import json
import re
from collections import OrderedDict
import numpy as np

def missed_assignments(tasks, agents=None):
    return [t.missed_assignments(agents) for t in tasks]

def max_task_pressures(tasks, agents=None):
    return [max(t.relative_affinities(agents)) for t in tasks]

def task_pressures(tasks, agents=None):
    return [t.pressure(agents) for t in tasks]

def affinity_pressure(tasks, agents=None):
    return max(task_pressures(tasks, agents))

def affinity_pressure_mean(tasks, agents=None):
    return np.mean(task_pressures(tasks, agents))

def affinity_pressure_percentile(tasks, agents=None, perc=95):
    return np.percentile(task_pressures(tasks, agents), q=perc)

def load_instance(instance):
    tasks = {}
    agents = {}
    task_avail = []
    agent_avail = []
    profits = []
    assignments = {}

    task_regex = re.compile(
        r"task\((?P<name>\d+),(?P<weights>\[[\d,\s]+\]),(?P<profits>\[[\d,\s]+\]),(?P<poss_agents>\[[\d,\s]*\])\)")
    agent_regex = re.compile(r"agent\((?P<name>\d+),(?P<capacity>\d+)\)")
    assign_regex = re.compile(r"assignment\((?P<name>\d+),(?P<assigned_tasks>\[[\d,\s]*\])\)")
    avail_regex = re.compile(r"(?P<type>(task|agent)avail)\((?P<cycle>\d+),(?P<availabilities>\[[\d,\s]*\])\)")
    profit_regex = re.compile(r"profit\((?P<cycle>\d+),(?P<profits>[\d\s,\[\]]*)\).")

    for line in open(instance, 'r'):
        line = line.replace(' ', '')

        m = task_regex.match(line)
        if m:
            name = int(m.group('name'))
            weights = json.loads(m.group('weights'))
            poss_agents = json.loads(m.group('poss_agents'))
            prios = json.loads(m.group('profits'))

            assert (len(weights) == len(poss_agents))
            assert (len(prios) == len(poss_agents))
            assert (name not in tasks)

            t = Task(name, weights, prios, poss_agents)
            tasks[name] = t

        m = agent_regex.match(line)

        if m:
            name = int(m.group('name'))
            capacity = int(m.group('capacity'))

            assert (capacity > 0)
            assert (name not in agents)

            a = Agent(name, capacity)
            agents[name] = a

        m = avail_regex.match(line)

        if m and m.group('type') == 'taskavail':
            cycle = int(m.group('cycle'))
            availabilities = json.loads(m.group('availabilities'))
            task_avail.insert(cycle - 1, availabilities)
        elif m and m.group('type') == 'agentavail':
            cycle = int(m.group('cycle'))
            availabilities = json.loads(m.group('availabilities'))
            agent_avail.insert(cycle - 1, availabilities)

        m = assign_regex.match(line)

        if m:
            agent = int(m.group('name'))
            assigned_tasks = json.loads(m.group('assigned_tasks'))
            assignments[agent] = assigned_tasks

        m = profit_regex.match(line)

        if m:
            cycle = int(m.group('cycle'))
            prof = json.loads(m.group('profits'))
            profits.append((cycle, prof))

    if len(profits) > 0:
        assert (len(profits) == len(task_avail) - 1)

        for _, prof in sorted(profits, key=lambda x: x[0]):
            for t, p in zip(tasks, prof):
                tasks[t].future_profits.append(p)

    assert (len(task_avail) == len(agent_avail))
    assert (all(len(x) <= len(tasks) for x in task_avail))
    assert (all(len(x) <= len(agents) for x in agent_avail))

    return tasks, agents, task_avail, agent_avail, assignments

def all_assignable(tasks, agents):
    agent_names = [a.name for a in agents]

    for t in tasks:
        if not any((k for k in t.profits.keys() if k in agent_names)):
            return False

    return True

class Task(object):
    def __init__(self, name, weights, profits, poss_agents, future_profits=[]):
        self.name = name
        self.weights = OrderedDict(zip(poss_agents, weights))
        self.profits = OrderedDict(zip(poss_agents, profits))
        self.affinities = OrderedDict([(name, 1) for name in poss_agents])
        self.future_profits = future_profits
        self.history = []
        self.backup_profits = None

    def relative_affinities(self, agents=None):
        affs = self._filtered_affinities(agents)
        nb_affs = len(affs)
        return [float(x) / nb_affs for x in affs]

    def pressure(self, agents=None):
        affs = self._filtered_affinities(agents)
        C = len(affs)
        actual = self.affinity_sum(agents)
        ideal = self.ideal_affinity_sum(agents)
        return (actual - ideal) / C

    def affinity_sum(self, agents=None):
        return self._filtered_affinities(agents).sum()

    def ideal_affinity_sum(self, agents=None):
        C = len(self._filtered_affinities(agents))
        return C * (C + 1) / 2

    def _filtered_affinities(self, agents=None):
        if agents:
            agent_names = [a.name for a in agents]
            affs = np.array([v for k, v in self.affinities.items() if k in agent_names])
        else:
            affs = np.array(list(self.affinities.values()))

        # Scale affinities by their min., to give higher importance to completely unassigned tasks
        # Skipped, because experiments showed no benefit from this
        # Left here for potential later further evaluation & inspection
        #affs *= np.min(affs)

        return affs

    def missed_assignments(self, agents=None):
        rel_aff = np.floor(self.relative_affinities(agents))
        return np.sum(rel_aff)

    def update(self, assigned_agent, agents):
        if isinstance(agents, dict):
            agents = agents.values()

        for ag in agents:
            if ag.name in self.affinities:
                self.affinities[ag.name] += 1

        if assigned_agent:
            self.affinities[assigned_agent.name] = 1
            self.history.append(assigned_agent.name)

    def update_profit(self):
        """ Emulates test case prioritization """
        if self.backup_profits is not None:
            self.profits = OrderedDict(self.backup_profits)
            self.backup_profits = None

        if len(self.future_profits) == 0:
            return

        poss_agents = self.profits.keys()
        next_prio = self.future_profits.pop()
        prio_dict = [(pa, next_prio) for pa in poss_agents]
        self.profits = OrderedDict(prio_dict)

    def restrict_agent(self, agent_name):
        if self.backup_profits is None:
            self.backup_profits = OrderedDict(self.profits)

        del self.profits[agent_name]

    @property
    def poss_agents(self):
        return self.profits.keys()

    def __str__(self):
        weights = ",".join(map(str, self.weights.values()))
        prios = ",".join(map(str, self.profits.values()))
        poss_agents = ",".join(map(str, self.profits.keys()))
        stringrep = "task(%d,[%s],[%s],[%s])." % (self.name, weights, prios,
                                                  poss_agents)
        return stringrep

class Agent(object):
    def __init__(self, name, capacity):
        self.name = name
        self.capacity = capacity

    def __str__(self):
        return "agent(%d,%d)." % (self.name, self.capacity)

    def __hash__(self):
        return self.name

def matrizes(agents, tasks, pad_dummy_agent=False):
    prof_matrix = []
    affinity_matrix = []
    weight_matrix = []

    for t in tasks:
        full_profits = [0] if pad_dummy_agent else []
        full_affinities = [0] if pad_dummy_agent else []
        full_weights = [0] if pad_dummy_agent else []

        for i, a in enumerate(agents, start=1):
            if a.name in t.poss_agents:
                full_profits.append(t.profits[a.name])
                full_affinities.append(t.affinities[a.name])
                full_weights.append(t.weights[a.name])
            else:
                full_profits.append(0)
                full_affinities.append(0)
                full_weights.append(0)

        prof_matrix.append(full_profits)
        affinity_matrix.append(full_affinities)
        weight_matrix.append(full_weights)

    prof_mat = np.array(prof_matrix, dtype=int)
    aff_mat = np.array(affinity_matrix, dtype=int)
    weight_mat = np.array(weight_matrix,dtype=int)

    assert (prof_mat.shape == (len(tasks), len(agents) + pad_dummy_agent))
    assert (aff_mat.shape == (len(tasks), len(agents) + pad_dummy_agent))
    assert (weight_mat.shape == (len(tasks), len(agents) + pad_dummy_agent))

    return prof_mat, aff_mat, weight_mat


text:
import os
import pandas as pd
import numpy as np
import pandas as pd
import pymongo
import json
from flask_pymongo import PyMongo
from flask import Flask, jsonify, render_template, request
import pickle
import xgboost
model = pickle.load(open('./mode/model.pkl','rb'))

app = Flask(__name__)
MONGO_URL = os.environ.get('MONGO_URI')
if not MONGO_URL:
    MONGO_URL = "mongodb://localhost:27017/gene_db"

app.config["MONGO_URI"] = MONGO_URL
mongo = PyMongo(app)

@app.route("/")
def index():
    return render_template("index.html")

@app.route("/gene_variants")
def plot_data():
    data = mongo.db.gene_variants
    output = []
    for x in data.find():
        output.append({'Class' : x['Class'],'Gene' : x['Gene'], 'Variation':x['Variation']})
    return jsonify(output)

@app.route("/dashboard")
def dashboard():
    return render_template("dashboard.html")

@app.route("/biocircos")
def biocircos():
    return render_template("bio.html")

@app.route("/api/variation", methods=['GET'])
def get_api():
    # return render_template("api.html")
    api = mongo.db.gene_variants

    output = []
    for x in api.find():
        output.append({'Gene' : x['Gene'], 'Variation' : x['Variation']})

    return jsonify({'result': output})

@app.route("/api/class", methods=['GET'])
def get_class():
    # return render_template("api.html")
    api = mongo.db.text

    output = []
    for x in api.find():
        output.append({'Gene' : x['Gene'], 'Variation': x['Variation'],'Class' : x['Class'], 'Text': x['Text']})

    return jsonify({'result': output})

@app.route("/api/geneData", methods=['GET'])
def get_geneData():
    # return render_template("api.html")
    api = mongo.db.geneData

    output = []
    for x in api.find():
        output.append({'Gene Symbol' : x['Gene Symbol'], 'Name' : x['Name'], 'Tumour Types(Somatic)' : x['Tumour Types(Somatic)'],
        'Tissue Type' : x['Tissue Type'], 'Molecular Genetics':x['Molecular Genetics'], 'Role in Cancer': x['Role in Cancer']})

    return jsonify({'result': output})

@app.route("/api/geneData/<gene>")
def get_gene(gene):
    results = mongo.db.geneData.find({'Gene Symbol': gene})
    output = []
    for x in results:
        output.append({'Gene Symbol' : x['Gene Symbol'], 'Name' : x['Name'], 'Tumour Types(Somatic)' : x['Tumour Types(Somatic)'],
        'Tissue Type' : x['Tissue Type'], 'Molecular Genetics':x['Molecular Genetics'], 'Role in Cancer': x['Role in Cancer']})

    return jsonify({'result': output})

@app.route("/model")
def test():
    return render_template("model.html")

@app.route('/result',methods = ['POST'])
def result():
    if request.method == 'POST':
        variation = request.form['variation']
        gene = request.form['gene']
        text = request.form['text']
        data = text +" "+ gene +" "+ variation
        # print(data)
        data = [data]
        prediction = model.predict(data)
        prediction = prediction[0]

        return render_template("result.html",prediction=prediction)
if __name__ == "__main__":
    app.run(debug=True)

max_stars_repo_name: reemaratnani/cancer-genetic-mutation

max_issues_repo_name: reemaratnani/cancer-genetic-mutation

max_forks_repo_name: reemaratnani/cancer-genetic-mutation


text:
import matplotlib

matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
import os, sys

# Analyse the distribution of utterance number in dataset

data_path = sys.argv[1]

utterance_number = [np.load(os.path.join(data_path, f), mmap_mode='r').shape[0] for f in os.listdir(data_path)]

bins = [0, 5, 10, 20, 50, 100, 500]
hist, bin_edges = np.histogram(utterance_number, bins)
fig, ax = plt.subplots()
# Plot the histogram heights against integers on the x axis
ax.bar(range(len(hist)), hist, width=1)

# Set the ticks to the middle of the bars
ax.set_xticks([0.5 + i for i, j in enumerate(hist)])

# Set the xticklabels to a string that tells us what the bin edges were
ax.set_xticklabels(['{} - {}'.format(bins[i], bins[i + 1]) for i, j in enumerate(hist)])

# plt.hist(utterance_number, bins=[0,5,10,20,50,100,500])
# plt.ylabel('Number of speakers')
# plt.xlabel('Number of utterance')
plt.savefig('Distribution.png')

max_stars_repo_name: math-geec/uis-rnn-sml

max_issues_repo_name: math-geec/uis-rnn-sml

max_forks_repo_name: math-geec/uis-rnn-sml


text:
import numpy as np

class LinearRegressor(object):
    def __init__(self, X, Y):
        self._Xtr = X
        self._Ytr = Y
        self._w = np.random.randn(X.shape[1])

    def get_number_samples(self):
        return self._Xtr.shape[0]

    def set_weights(self, w):
        self._w = w

    def get_weights(self):
        return self._w

    def load_data(self, Xtr, Ytr=None):
        self._Xtr = Xtr
        self._Ytr = Ytr

    def calculate_weights(self):
        dim = self._Xtr.shape[1]
        self._w = np.dot(np.linalg.pinv(np.dot(self._Xtr.T, self._Xtr)
                                        + self._lambda * np.eye(dim)), np.dot(self._Xtr.T, self._Ytr))
        return self._w

    def predict(self, X):
        return np.dot(X, self._w)

    def test_loss(self, w, X, Y):
        w_old = self.get_weights()
        self.set_weights(w)
        error = self.predict(X) - Y

        self.set_weights(w_old)
        return np.dot(error.T, error)

    def loss(self, w, indexes):
        self.set_weights(w)
        error = self.predict(self._Xtr[indexes, :]) - self._Ytr[indexes]
        return np.dot(error.T, error) / indexes.size  # + 1./2 * self._lambda * np.dot(self._w, self._w)

    def gradient(self, w, indexes):
        self.set_weights(w)
        error = self.predict(self._Xtr[indexes, :]) - self._Ytr[indexes]
        return np.dot(self._Xtr[indexes, :].T, error) / indexes.size  # + self._lambda * w


text:
__doc__ = """Image chunk class"""

import numpy as np

from chunkflow.chunk import Chunk
from .adjust_grey import normalize_section_shang
from .convnet.inferencer import Inferencer

class Image(Chunk):
    """
    a chunk of image volume.
    """
    def __init__(self, array: np.ndarray, global_offset=None):
        super().__init__(array, global_offset=global_offset)

    def inference(self, inferencer: Inferencer):
        """run convolutional net inference for this image chunk"""
        return inferencer(self)

    def normalize_section_shang(self, nominalmin, nominalmax, clipvalues):
        return normalize_section_shang(self.array, nominalmin, nominalmax,
                                       clipvalues)

max_stars_repo_name: fcollman/chunkflow

max_issues_repo_name: fcollman/chunkflow

max_forks_repo_name: fcollman/chunkflow