text:
# generator functions to simplify and streamline signal injection and recovery of filterbanks
import setigen as stg
import numpy as np
import pandas as pd
import os
import astropy.units as u
from turbo_seti.find_doppler.find_doppler import FindDoppler
from turbo_seti.find_event.find_event import read_dat
from blimpy import Waterfall
import math
import matplotlib.pyplot as plt

from setificient import get_test_files, get_data_files, generate_test_file

import logging
logger_name = 'multigen'
logger = logging.getLogger(logger_name)

import multigen_analysis

def blueprint():
    s = """#waterfalls = stg.split_waterfall_generator()
        # opt for one small portion of a waterfall file to make this faster!!
        parameters = get_parameters(f_range, d_range, s_range, w_range) DONE
        # waterfalls = (slice_waterfall(w, params) for params in parameters)
        frames = (inject_frame(p) for p in parameters) DONE
        finddoppler_results = (find_doppler_frame(frame) for frame in frames) DONE
        #results = (get_results(fd) for fd in finddopplers)
        final_results = pd.concat(finddoppler_results)
        """
    print(s)

#def uniform_dist(v0, v1, n):
#    """
#    Returns generator of a uniform distribution
#    """
#    return (v for v in np.linspace(v0, v1, n))
def all_dist(
        nsamples,
        f_min,
        f_max,
        d_min=-10.0,
        d_max=10.0,
        s_min=30.0,
        s_max=30.0,
        w_min=4.0,
        w_max=4.0,
        nsignals=2):
    """
    Generates values by stepping through the
    input parameter ranges.  Ensures the entire
    range is spanned.
    """
    counter = 0
    while counter < nsamples*nsignals:
        f = np.linspace(f_min, f_max, nsamples*(nsignals))[counter:counter+nsignals]
        d = np.linspace(d_min, d_max, nsamples*(nsignals))[counter:counter+nsignals]
        s = np.linspace(s_min, s_max, nsamples*(nsignals))[counter:counter+nsignals]
        w = np.linspace(w_min, w_max, nsamples*(nsignals))[counter:counter+nsignals]
        #f = np.linspace(f_min, f_max, nsamples*nsignals)[nsamples:nsamples-nsignals]
        #d = np.linspace(d_min, d_max, nsamples*nsignals)[nsamples:nsamples-nsignals]
        #s = np.linspace(s_min, s_max, nsamples*nsignals)[nsamples:nsamples-nsignals]
        #w = np.linspace(w_min, w_max, nsamples*nsignals)[nsamples:nsamples-nsignals]
        yield (f, d, s, w)
        #nsamples -= nsignals
        counter += nsignals

def parameters_generator(
        nsamples,
        f_min,
        f_max,
        d_min=-10.0,
        d_max=10.0,
        s_min=20.0,
        s_max=40.0,
        w_min=1.0,
        w_max=40.0,
        f_dist='uniform',
        d_dist='uniform',
        s_dist='uniform',
        w_dist='uniform',
        nsignals=2,
        dt=0.0,
        fchans=1024,
        tchans=16):
    """
    Generator of parameter values for signal injection

    Parameters
    ----------
    nsamples : int
        Number of times to sample the parameter distributions
    f_min : float
        Minimum injection frequency
    f_max : float
        Maximum injection frequency
    d_min : float, optional
        Minimum injection drift
    d_max : float, optional
        Maximum injection drift
    s_min : float, optional
        Minimum injection SNR
    s_max : float, optional
        Maximum injection SNR
    w_min : float, optional
        Minimum injection width
    w_max : float, optional
        Maximum injection width
    f_dist : str, optional
        Injection frequency distribution ('uniform', 'gaussian', 'random')
    d_dist : str, optional
        Injection drift distribution ('uniform', 'gaussian', 'random')
    s_dist : str, optional
        Injection SNR distribution ('uniform', 'gaussian', 'random')
    w_dist : str, optional
        Injection width distribution ('uniform', 'gaussian', 'random')
    nsignals : int, optional
        Number of signals to inject

    Notes
    -----
    You can replace any distribution given in numpy.random
    for the frequency, drift rate, snr, and/or width distributions.
    """
    #random.uniform(-1, 1)*(drift*frame.dt*frame.tchans
    #separation =
    f_dist, d_dist, s_dist, w_dist = [eval("np.random."+dist) for dist in [f_dist, d_dist, s_dist, w_dist]]
    while nsamples > 0:
        f_vals = f_dist(low=f_min, high=f_max, size=nsignals)
        d_vals = d_dist(low=d_min, high=d_max, size=nsignals)
        s_vals = s_dist(low=s_min, high=s_max, size=nsignals)
        w_vals = w_dist(low=w_min, high=w_max, size=nsignals)
        if nsignals > 1:
            f_vals[1] = f_vals[0] + 5000 + np.random.uniform(-2,2)*1000#(np.random.uniform(-1,1)*(d_vals[0]*dt*tchans))
            d_vals[1] = d_vals[0] + (np.random.uniform(-1,1)*5.0)
        yield (
            f_vals,
            d_vals,
            s_vals,
            w_vals
            )
        nsamples -= 1

def Dtest_generator(
        nsamples_per_drift,
        f_min,
        f_max,
        d_min=-10.0,
        d_max=10.0,
        s_min=30.0,
        s_max=30.0,
        w_min=4.0,
        w_max=4.0,
        f_dist='uniform',
        d_dist='uniform',
        s_dist='uniform',
        w_dist='uniform',
        nsignals=1):
    """
    Generates parameters for drift rate tests
    """
    f = float(np.mean([f_min, f_max]))
    s = float(np.mean([s_min, s_max]))
    w = float(np.mean([w_min, w_max]))
    d_vals = np.array([np.full(nsamples_per_drift, i) for i in np.arange(d_min, d_max, 0.5)]).flatten()
    nsamples = len(d_vals)
    counter=0
    while counter < nsamples:
        yield (f, d_vals[counter], s, w)
        counter += 1
def Stest_generator(
        nsamples_per_drift,
        f_min,
        f_max,
        d_min=0.0,
        d_max=0.0,
        s_min=0.5,
        s_max=100.0,
        w_min=4.0,
        w_max=4.0,
        f_dist='uniform',
        d_dist='uniform',
        s_dist='uniform',
        w_dist='uniform',
        nsignals=1):
    """
    Generates parameters for SNR tests
    """
    f = float(np.mean([f_min, f_max]))
    d = float(np.mean([d_min, d_max]))
    w = float(np.mean([w_min, w_max]))
    #d_vals = np.array([np.full(nsamples_per_drift, i) for i in np.arange(d_min, d_max, 0.5)]).flatten()
    s_vals = np.array([np.full(nsamples_per_drift, i) for i in np.arange(s_min, s_max, 0.5)]).flatten()
    nsamples = len(s_vals)
    counter=0
    while counter < nsamples:
        yield (f, d, s_vals[counter], w) #d_vals[counter], s, w)
        counter += 1

def inject_frame(
        filename,
        parameters,
        nsignals):
    """
    Returns a frame object with injected signals and
    a tuple of injected injected signal parameters

    Parameters
    ----------
    filename : str
        Path to filterbank file
    parameters : tuple
        Tuple of injection parameter values
    nsignals : int, optional
        Number of signals to inject into data

    Notes
    -----
    For each signal the SNR is reduced
    """
    #while niterations > 0:
    frame = stg.Frame(filename)
    frame.add_metadata({'signals':[]})

    frequency, drift_rate, snr, width = parameters
    #print(parameters)
    #print(np.shape(parameters))
    #if len(list(np.shape(parameters))) > 1:
    if nsignals > 1:
        for i in range(nsignals):
        #for i in range(np.shape(parameters)[-1]):
        #if len(list(np.shape(parameters)))==1:
        #else:
            f = float(frequency[i])
            d = float(drift_rate[i])
            s = float(snr[i])
            w = float(width[i])

        #logger.debug("f = ",f)
        #logger.debug("d = ",d)
        #logger.debug("s = ",s)
        #logger.debug("w = ",w)
            #print(len(list(np.shape(parameters))))
        #if i > 0 and len(list(np.shape(parameters)))>1:
            s *= .5
            parameters[2][1] = s
            fexs = f - np.copysign(1, d)*w
            fex = (f + (d * frame.dt * frame.tchans)) + np.copysign(1, d)*w

        #logger.debug("fexs = ",fexs)
        #logger.debug("fex = ",fex)

            frame.add_signal(
            stg.constant_path(
                f_start=f,
                drift_rate=d*u.Hz/u.s),
            stg.constant_t_profile(level=frame.get_intensity(snr=s)),
            stg.gaussian_f_profile(width=w*u.Hz),
            stg.constant_bp_profile(level=1),
            bounding_f_range=(min(fexs, fex), max(fexs, fex))
            )
            frame.metadata['signals'].append([f, d, s, w])
    else:
        f = float(frequency)
        d = float(drift_rate)
        s = float(snr)
        w = float(width)
        fexs = f - np.copysign(1, d)*w
        fex = (f + (d * frame.dt * frame.tchans)) + np.copysign(1, d)*w
        frame.add_signal(
            stg.constant_path(
                f_start=f,
                drift_rate=d*u.Hz/u.s),
            stg.constant_t_profile(level=frame.get_intensity(snr=s)),
            stg.gaussian_f_profile(width=w*u.Hz),
            stg.constant_bp_profile(level=1),
            bounding_f_range=(min(fexs, fex), max(fexs, fex))
            )
        frame.metadata['signals'].append([f, d, s, w])
    return frame, parameters

def apply_find_doppler(frame, parameters, nsignals):
    """
    Returns results of FindDopplering a frame object

    Parameters
    ----------
    frame : stg.Frame
        Frame with injected signals
    parameters : tuple
        Parameters of injected signals
    nsignals : int
        Number of injected signals per frame

    """
    #DATA
    #0. num_recovered
    #1. num_inserted
    #2. injected frequency
    #3. injected drift rate
    #4. injected snr
    #5. injected width
    #6. [find_doppler data]
    #7. [2nd sig freq, 2nd sig drift, 2nd sig, 2nd sig snr, 2nd sig, width]

    #RESULTS
    #ratio captured = (0) / (1)
    #injected frequency = (2)
    #detected (captured) frequency = (6)(0)(0)(1)
    control = read_dat('testing_spliced_blc00010203040506o7o0111213141516o021222324252627_guppi_58806_43185_TIC458478250_0127.gpuspec.0000.dat')
    def compare(df_1, df_2, col_):
        """
        Compares DataFrame column values
        """
        def vcompare(df1, df2, col):
            return df1[col] != df2[col]
        #comp = np.vectorize(vcompare)
        #return comp(df_1, df_2, col_)
        return vcompare(df_1, df_2, col_)
    frame.save_hdf5('frame.h5')
    FindDoppler('frame.h5', max_drift=11.0, snr=10).search()
    hits = read_dat('frame.dat')
    os.remove('frame.h5')
    os.remove('frame.dat')
    os.remove('frame.log')

    # remove hits from known RFI
    RFI = read_dat('_fchans_4096_testing_spliced_blc00010203040506o7o0111213141516o021222324252627_guppi_58806_43864_TIC154089169_0129.gpuspec.0000.dat')
    print(hits.columns)
    print(RFI.columns)
    #try:
    #    hits = hits.loc[(hits['DriftRate'].eq(RFI['DriftRate'])) & (hits['SNR'].eq(RFI['SNR']))]
    #except:
    #    pass
    #hits = hits.loc[compare(hits, control, 'Freq') & compare(hits, control, 'DriftRate') & compare(hits, control, 'SNR')]
    results = {}
    #try:
    results['nInjected'] = [len(frame.metadata['signals'])]
    results['nDetected'] = [len(hits)]
    results['RatioCaptured']=[len(hits) / len(frame.metadata['signals'])]#[len(hits) / np.shape(parameters)[-1]]
    #except:
    #    results['RatioCaptured']=[len(hits) / nsignals]
    if len(hits)==0:
        #hits = {}
        hits['Freq']=[0.0]
        hits['DriftRate']=[0.0]
        hits['SNR']=[0.0]

    #if len(list(np.shape(parameters))) > 1:
    if nsignals > 1:
        results['injFreq'] = [parameters[0][0]/1.0e6]
    else:
        results['injFreq'] = [parameters[0] / 1.0e6]
    #THREW AN ERROR WHEN MORE THAN ONE SIGNAL WAS DETECTED
    #results['detFreq'] = [float(hits['Freq'])]
    results['detFreq'] = [float(hits['Freq'][0])]
    #if len(list(np.shape(parameters))) > 1:
    if nsignals > 1:
        results['injDrift'] = [parameters[1][0]]
    else:
        results['injDrift'] = [parameters[1]]
    results['detDrift'] = [float(hits['DriftRate'][0])]#.tolist()
    if nsignals > 1:#len(list(np.shape(parameters))) > 1:
        results['injSNR'] = [parameters[2][0]]
    else:
        results['injSNR'] = [parameters[2]]
    results['detSNR'] = [float(hits['SNR'][0])]
    if nsignals > 1:#len(list(np.shape(parameters))) > 1:
        results['injWidth'] = [parameters[3][0]]
    else:
        results['injWidth'] = [parameters[3]]
    #results['Separation'] = [(float(np.diff(parameters[0]))*u.MHz).to_value('kHz')]
    if nsignals > 1:#len(list(np.shape(parameters))) > 1:
        results['Separation'] = [float(np.diff(parameters[0]))]#/1.0e3]
        results['secFreq'] = [parameters[0][1]/1.0e6]
        results['secDrift'] = [parameters[1][1]]
        results['secSNR'] = [parameters[2][1]]
        results['secWidth'] = [parameters[3][1]]
        if len(hits)>1:
            results['detsecFreq'] = [float(hits['Freq'][1])]
            results['detsecDrift'] = [float(hits['DriftRate'][1])]
            results['detsecSNR'] = [float(hits['SNR'][1])]

    results['diffFreq'] = float(results['detFreq'][0]) - float(results['injFreq'][0])#.to_numpy()
    results['absdiffFreq'] = abs(results['diffFreq'])
    results['diffDrift'] = results['detDrift'][0]-results['injDrift'][0]
    results['absdiffDrift'] = abs(results['diffDrift'])
    results['diffSNR'] = results['detSNR'][0] - results['injSNR'][0]
    results['absdiffSNR'] = abs(results['diffSNR'])
    results_df = pd.DataFrame.from_dict(results)
    #results_df['injFreq'] = [float(f[1:-1]) for f in results_df['injFreq']]
    #results_df['injDrift'] = [float(d[1:-1]) for d in result_df['injDrift']]
    #results_df['injSNR'] = [float(s[1:-1]) for s in df['injSNR']]
    #results_df['injWidth'] = [float(w[1:-1]) for w in df['injWidth']]

    #results_df = pd.concat([results_df, hits], axis=1)
    #for col in ['Hit_ID','status','in_n_ons','RFI_in_range']:
    #    del results_df[col]
    return results_df

def efficiency_pipeline(
        filename,
        niterations,
        nsignals=2,
        d_min=0.0,
        d_max=0.0,
        s_min=0.0,
        s_max=100.0,
        w_min=4.0,
        w_max=4.0,
        f_dist='uniform',
        d_dist='uniform',
        s_dist='uniform',
        w_dist='uniform',
        loglevel=logging.INFO,
        dev=False):
    """
    Top-level efficiency test routine

    Parameters
    ----------
    filename : str
        Path to filterbank file
    nsignals : int, optional
        Number of signals to inject per frame
    niterations : int, optional
        Number of iterations to perform efficiency test

    """
    logger.setLevel(loglevel)
    tchans = stg.Frame(Waterfall(filename, max_load=20)).tchans
    dt = stg.Frame(Waterfall(filename, max_load=20)).dt
    fchans = stg.Frame(Waterfall(filename, max_load=20)).fchans
    f_min, f_max = stg.Frame(Waterfall(filename, max_load=20)).fmin, stg.Frame(Waterfall(filename, max_load=20)).fmax
    if dev:
        frames = (inject_frame(filename, p, nsignals) for p in Stest_generator(
            niterations,
            f_min,
            f_max,
            d_min=d_min,
            d_max=d_max,
            s_min=s_min,
            s_max=s_max,
            w_min=w_min,
            w_max=w_max,
            nsignals=1))
        #frames = (inject_frame(filename, p) for p in all_dist(
        #    niterations,
        #    f_min,
        #    f_max,
        #    d_min=d_min,
        #    d_max=d_max,
        #    s_min=s_min,
        #    s_max=s_max,
        #    w_min=w_min,
        #    w_max=w_max,
        #    nsignals=nsignals,
        #    dt=dt,
        #    fchans=fchans,
        #    tchans=tchans))
    else:
        frames = (inject_frame(filename, p, nsignals) for p in parameters_generator(
        niterations,
        f_min,
        f_max,
        d_min=d_min,
        d_max=d_max,
        s_min=s_min,
        s_max=s_max,
        w_min=w_min,
        w_max=w_max,
        f_dist=f_dist,
        d_dist=d_dist,
        s_dist=s_dist,
        w_dist=w_dist,
        nsignals=nsignals,
        dt=dt,
        fchans=fchans,
        tchans=tchans))
    results = (apply_find_doppler(fr[0], fr[-1], nsignals) for fr in frames)
    results_df = pd.concat(results, ignore_index=True)
    return results_df

def analyze_results(df, twodim=False, dev=True):
    """
    Analyzes results

    Parameters
    ----------
    df : pandas.core.frame.DataFrame
        DataFrame of results

    Notes
    -----
    Quantifies percent recovery as a function of
    1) signal separation (for 2 signals)
    2) drift rate (histogram)
    """
    # percrec as a function of drift rate
    if not twodim:
        if dev:

            n, bins, patches = plt.hist(df['injDrift'], histtype='step', density=True, bins=np.arange(-10,10,1))#df.plot.hist('injDriftRate')
            #x=df['injDrift'],
            #bins=np.arange(-10.0, 10.0, 1),
            #histtype='step',
            #facecolor='b',
            #alpha=0.7,
            #rwidth=0.85,
            #density=True)

        else:
            n, bins, patches = plt.hist(
            x=df['injDrift'],
            bins=np.arange(-10.0, 10.0, 1),
            histtype='step',
            facecolor='b',
            alpha=0.7,
            rwidth=0.85)
            plt.ylabel('Counts')
    else:
        plt.hist2d(
                x=df['injDrift'],
                y=df['RatioCaptured'],
                bins=np.arange(-10.0,10.0,1),
                #histtype='step',
                #facecolor='b',
                alpha=0.7#,
                #rwidth=0.85
                )
        plt.ylabel("Percent Captured")
    #plt.grid(axis='y', alpha=0.75)
    plt.xlabel('Drift Rate [Hz/s]')
    plt.title('Recovered vs Drift Rate')

    #if not twodim:
        #maxcount = n.max()
        # set a clean upper y-axis limit
        #plt.ylim(ymax=np.ceil(maxcount/3)*3 if maxcount %3 else maxcount + 3)
    plt.savefig('Counts_vs_DriftRate.png',bbox_inches='tight')
    print('Saved plot: /home/raffytraas14/seti_efficiency/Counts_vs_DriftRate.png')

def test():
    """
    Function to run various ad hoc tests
    """
    filename = get_test_files('x')[0]

    nsamples = 10
    nsignals = 2

    f_min, f_max = stg.Frame(Waterfall(filename, max_load=20)).fmin, stg.Frame(Waterfall(filename, max_load=20)).fmax
    d_min, d_max = -10.0, 10.0
    s_min, s_max = 30.0, 30.0
    w_min, w_max = 4.0, 4.0

    parameters_generator_list = list(parameters_generator(
        nsamples,
        f_min,
        f_max,
        d_min=d_min,
        d_max=d_max,
        s_min=s_min,
        s_max=s_max,
        w_min=w_min,
        w_max=w_max,
        nsignals=nsignals))
    all_dist_list = list(all_dist(
        nsamples,
        f_min,
        f_max,
        d_min=d_min,
        d_max=d_max,
        s_min=s_min,
        s_max=s_max,
        w_min=w_min,
        w_max=w_max,
        nsignals=nsignals))

    #print("parameters_generator() == all_dist_list: ", parameters_generator_list == all_dist_list)
    print("parameters_generator():\n", parameters_generator_list)
    print("all_dist_list():\n",all_dist_list)
    print("len(parameters_generator()) == len(all_dist_list()): ", len(parameters_generator_list) == len(all_dist_list))
    print("len(parameters_generator()): ", len(parameters_generator_list))
    print("len(all_dist()): ", len(all_dist_list))

if __name__=='__main__':
    #blueprint()
    #for i in parameters_generator(5, 1.1e10, 7.8e9):
    #    print(i)
    #waterfall_fn = [f for f in get_data_files('x') if 'TIC' in f][2]
    #generate_test_file(waterfall_fn, 'x', power_of_2=8, tchans=None, out_dir='test_files/')

    #results = efficiency_pipeline(get_test_files('x')[0], 100, dev=True)

    #results = efficiency_pipeline(get_test_files('x')[0], 1000000, nsignals=1)#, dev=True)#, loglevel=logging.DEBUG)
    results = efficiency_pipeline(get_test_files('x')[0], 10000, nsignals=1)#, dev=True)#, loglevel=logging.DEBUG)
    print(results)
    results.to_csv("10K_snr_no_rfi_filter_signal_efficiency_test.csv")
    #results.to_csv("1M_signal_efficiency_test.csv")

    #analyze_results(results)

    #results.to_csv("100_signals_per_drift_rate_signal_efficiency_test.csv")
    #multigen_analysis.linregress(results,'injDrift', 'absdiffDrift')
    ####print(results.dropna())
    ####test()

max_stars_repo_name: rtraas/setificiency

max_issues_repo_name: rtraas/setificiency

max_forks_repo_name: rtraas/setificiency

reason: import numpy,import astropy


text:
import numpy as np

iris_data = np.loadtxt('iris_data.csv', delimiter=',')
print("file content:" + str(iris_data))

# Sort
iris_data.sort()
print("Sorting Result: \n", iris_data)

# len
print("Length of data: ", len(iris_data))

# remove duplicate
after_rm_dup = np.unique(iris_data)
print("After remove duplicate: \n", after_rm_dup)

# sum array
print("Sum: ", np.sum(iris_data))

# cumsum
print("accumulative sum: \n", np.cumsum(iris_data))

# avg
print("average value: ", np.mean(iris_data))

# var
print("variance value: ", np.var(iris_data))

# min
print("min value: ", np.min(iris_data))

# max
print("max value: ", np.max(iris_data))

ext: py

reason: import numpy


text:
import numpy as np
import pandas as pd
import pytest

from functools import partial

from polling_simulator import sampling
from polling_simulator import Variable, Demographic
from polling_simulator.distributions import truncated_gaussian_distribution

class TestPredefinedSample:
    def test_fails_when_asked_for_too_many_people(self):
        data = pd.DataFrame({
            "response_likelihood": np.ones(10)
        })
        with pytest.raises(ValueError):
            sampling.predefined_sample(1, False)(20, data)

    def test_fails_when_response_likelihood_is_too_low(self):
        data = pd.DataFrame({
            "response_likelihood": np.ones(1000) * 0.000001
        })
        with pytest.raises(ValueError):
            sampling.predefined_sample(1, False)(20, data)

    def test_fails_when_turnout_likelihood_is_too_low(self):
        data = pd.DataFrame({
            "response_likelihood": np.ones(1000),
            "turnout_likelihood": np.ones(1000) * 0.000001
        })
        sampling.predefined_sample(1, False)(500, data)
        with pytest.raises(ValueError):
            sampling.predefined_sample(1, True)(500, data)

    def test_returns_fewer_than_desired_when_response_rates_are_low(self):
        np.random.seed(123)
        data = pd.DataFrame({
            "response_likelihood": np.ones(20) * 0.5
        })
        poll_responders, poll_non_responders = sampling.predefined_sample(1, False)(10, data)
        assert len(poll_responders) < 10
        assert len(poll_responders) + len(poll_non_responders) == 10

    def test_reaches_more_people_when_makes_multiple_attempts(self):
        np.random.seed(123)
        data = pd.DataFrame({
            "response_likelihood": np.ones(40) * 0.1
        })
        single_call_responders, _ = sampling.predefined_sample(1, False)(20, data)
        multi_call_responders, non_responders = sampling.predefined_sample(5, False)(20, data)
        assert len(multi_call_responders) > len(single_call_responders)
        assert len(multi_call_responders) < 20
        assert len(multi_call_responders) + len(non_responders) == 20

    def test_applies_likely_voter_screen_correctly(self):
        data = pd.DataFrame({
            "response_likelihood": np.ones(1000),
            "turnout_likelihood": np.ones(1000) * 0.1
        })
        responders, nonresponders = sampling.predefined_sample(1, True)(50, data)
        assert len(responders) + len(nonresponders) < 50

class TestGuaranteedSample:
    def test_fails_when_asked_for_too_many_people(self):
        np.random.seed(123)
        data = pd.DataFrame({
            "response_likelihood": np.ones(10) * 0.05
        })
        with pytest.raises(ValueError):
            sampling.guaranteed_sample(1, False)(5, data)

    @pytest.mark.parametrize("num_people", [10, 50, 100, 500])
    def test_always_returns_the_asked_for_number_of_people(self, num_people):
        np.random.seed(123)
        data = pd.DataFrame({
            "response_likelihood": np.ones(10000) * 0.1
        })
        responders, non_responders = sampling.guaranteed_sample(1, False)(num_people, data)
        assert len(responders) == num_people
        assert len(non_responders) > 0

    def test_works_with_multiple_attempts(self):
        np.random.seed(123)
        data = pd.DataFrame({
            "response_likelihood": np.ones(10000) * 0.1
        })
        single_attempt_responders, single_attempt_non_responders = sampling.guaranteed_sample(1, False)(100, data)
        multiple_attempt_responders, multiple_attempt_non_responders = sampling.guaranteed_sample(5, False)(100, data)

        assert len(single_attempt_responders) == len(multiple_attempt_responders)
        assert len(multiple_attempt_non_responders) < len(single_attempt_non_responders)

    def test_works_with_likely_voter_screen(self):
        np.random.seed(123)
        data = pd.DataFrame({
            "response_likelihood": np.ones(10000) * 0.5,
            "turnout_likelihood": np.ones(10000) * 0.1
        })
        _, no_screen_non_responders = sampling.guaranteed_sample(1, False)(200, data)
        responders, screened_non_responders = sampling.guaranteed_sample(1, True)(200, data)
        assert len(responders) == 200
        assert len(screened_non_responders) > 5 * len(no_screen_non_responders)

class TestPreStratifiedSample:
    def test_freezes_assumed_demographics(self):
        age = Variable("age", truncated_gaussian_distribution(25, 25, 18, 110))
        young_people = Demographic(0.5, 0.1, {"a": 1}, age < 40)
        old_people = Demographic(0.5, 0.1, {"b": 1}, age >= 40)
        demographics = [
            young_people, old_people
        ]
        sampler = sampling.stratified_sample(demographics, sampling.guaranteed_sample(1, False))
        demographics.pop(0)
        assert len(demographics) == 1
        for item in sampler.__closure__:
            if type(item.cell_contents) == list:
                assert len(item.cell_contents) == 2

    def test_errors_when_demographics_are_not_mece(self):
        age = Variable("age", truncated_gaussian_distribution(25, 25, 18, 110))
        young_people = Demographic(0.5, 0.1, {"a": 1}, age < 40)
        old_people = Demographic(0.5, 0.1, {"b": 1}, age >= 60)
        sampler = sampling.stratified_sample([young_people, old_people], sampling.guaranteed_sample(1, False))
        electorate = pd.DataFrame({
            "age": np.arange(100)
        })
        with pytest.raises(ValueError) as e:
            sampler(100, electorate)
        assert "Demographics are not mutually exclusive" in e.value.args[0]

    def test_stratification_works_as_expected(self):
        np.random.seed(123)
        gender = Variable("gender", partial(
            np.random.choice, np.array(["M", "F"]), replace=True, p=np.array([0.5, 0.5])
        ))

        men = Demographic(0.5, 0.1, {"a": 1}, (gender == "M"))
        women = Demographic(0.5, 0.2, {"b": 1}, (gender == "F"))
        demographics = [men, women]
        sampler = sampling.stratified_sample(demographics, sampling.guaranteed_sample(1, False))

        male_electorate = pd.DataFrame({
            "turnout_likelihood": np.ones(50000) * men.turnout_likelihood,
            "response_likelihood": men.response_likelihood,
            "candidate_preference": "a",
            "gender": "M"
        })
        female_electorate = pd.DataFrame({
            "turnout_likelihood": np.ones(50000) * women.turnout_likelihood,
            "response_likelihood": women.response_likelihood,
            "candidate_preference": "b",
            "gender": "F"
        })
        shuffled_electorate = pd.concat([male_electorate, female_electorate]).sample(frac=1)

        responders, non_responders = sampler(1000, shuffled_electorate)
        assert np.sum(responders["gender"] == "F") == 500
        assert np.sum(responders["gender"] == "M") == 500
        assert len(non_responders) > 3000
        assert np.sum(non_responders["gender"] == "M") > np.sum(non_responders["gender"] == "F")

class TestInternalGetResponses:
    def test_fails_when_passed_zero_attempts(self):
        with pytest.raises(ValueError):
            sampling._get_responses(None, 0)

    def test_works_with_single_attempt(self):
        np.random.seed(123)
        response_likelihoods = np.ones(40) * 0.5
        did_respond, num_attempts_required = sampling._get_responses(response_likelihoods, 1)
        assert np.sum(did_respond) < len(response_likelihoods)
        np.testing.assert_allclose(np.unique(num_attempts_required), np.array([-1, 1]))
        assert np.min(num_attempts_required[did_respond]) == 1

    def test_works_with_pandas_series(self):
        np.random.seed(123)
        response_likelihoods = pd.Series(np.ones(40) * 0.5)
        did_respond, num_attempts_required = sampling._get_responses(response_likelihoods, 1)
        assert np.sum(did_respond) < len(response_likelihoods)
        np.testing.assert_allclose(np.unique(num_attempts_required), np.array([-1, 1]))
        assert np.min(num_attempts_required[did_respond]) == 1

    def test_works_with_multiple_attempts(self):
        np.random.seed(123)
        response_likelihoods = np.ones(40) * 0.25
        did_respond_single, _ = sampling._get_responses(response_likelihoods, 1)
        did_respond_multi, num_attempts_required = sampling._get_responses(response_likelihoods, 3)
        assert np.sum(did_respond_multi) > np.sum(did_respond_single)
        np.testing.assert_allclose(np.unique(num_attempts_required), np.array([-1, 1, 2, 3]))
        assert np.min(num_attempts_required[did_respond_multi]) == 1


text:
# bca4abm
# See full license in LICENSE.txt.

import logging

import os
import pandas as pd
import numpy as np

from bca4abm import bca4abm as bca
from ...util.misc import add_aggregate_results

from activitysim.core import config
from activitysim.core import inject
from activitysim.core import tracing
from activitysim.core import assign
from activitysim.core import pipeline

logger = logging.getLogger(__name__)

"""
Aggregate demographics processor

each row in the data table to solve is an origin zone and this processor
calculates communities of concern (COC) / market segments based on mf.cval.csv
"""

@inject.injectable()
def aggregate_demographics_spec():
    return bca.read_assignment_spec('aggregate_demographics.csv')

@inject.step()
def aggregate_demographics_processor(
        zone_hhs,
        aggregate_demographics_spec,
        settings, trace_od):
    """

    Parameters
    ----------
    zone_hhs : orca table
        input zone demographics

    """

    trace_label = 'aggregate_demographics'
    model_settings = config.read_model_settings('aggregate_demographics.yaml')

    zone_hhs_df = zone_hhs.to_frame()

    logger.info("Running %s with %d zones" % (trace_label, len(zone_hhs_df), ))

    if trace_od:
        trace_orig, trace_dest = trace_od
        trace_od_rows = (zone_hhs_df.index == trace_orig) | (zone_hhs_df.index == trace_dest)
    else:
        trace_od_rows = None

    # locals whose values will be accessible to the execution context
    # when the expressions in spec are applied to choosers
    locals_dict = config.get_model_constants(model_settings)
    locals_dict.update(config.setting('globals'))

    trace_rows = None

    # eval_variables evaluates each of the expressions in spec
    # in the context of each row in of the choosers dataframe
    results, trace_results, trace_assigned_locals = \
        assign.assign_variables(aggregate_demographics_spec,
                                zone_hhs_df,
                                locals_dict,
                                df_alias='hhs',
                                trace_rows=trace_od_rows)

    pipeline.replace_table("zone_demographics", results)

    # expression file can use silos column to designate result targets (e.g. count of households)
    add_aggregate_results(results, aggregate_demographics_spec, source=trace_label)

    if trace_results is not None:

        tracing.write_csv(trace_results,
                          file_name="aggregate_demographics",
                          index_label='zone',
                          column_labels=['label', 'zone'])

        if trace_assigned_locals:
            tracing.write_csv(trace_assigned_locals, file_name="aggregate_demographics_locals")


text:
"""Spectral plotting with gammapy.spectrum.flux_point
"""
import numpy as np
from astropy.table import Table
from gammapy.spectrum.flux_point import (compute_differential_flux_points,
                                         _energy_lafferty_power_law,
                                         _x_lafferty, _integrate)
from gammapy.spectrum.powerlaw import power_law_eval, power_law_integral_flux
import matplotlib.pyplot as plt

SPECTRAL_INDEX = 4

def my_spectrum(x):
    E_1 = 1
    E_2 = 10
    E_3 = 100
    E_4 = 1000
    E_5 = 10000
    g1 = -1
    g2 = -2
    g3 = -3
    g4 = -4
    g5 = -5
    im = np.select([x <= E_1, x <= E_2, x <= E_3, x <= E_4, x <= E_5, x > E_5],
                   [(x / E_1) ** g1, 1e-2 * (x / E_2) ** g2,
                    1e-5 * (x / E_3) ** g3, 1e-9 * (x / E_4) ** g4,
                    1e-14 * (x / E_5) ** g5, 0])
    return im

def get_flux_tables(table, y_method, function, spectral_index):
    table1 = table.copy()
    lafferty_flux = compute_differential_flux_points('lafferty', y_method, table1,
                                                     function, spectral_index)
    table2 = table1.copy()
    log_flux = compute_differential_flux_points('log_center', y_method, table2,
                                                     function, spectral_index)
    return lafferty_flux, log_flux

def plot_flux_points(table, x, y, function, energy_min, energy_max, y_method):
    f, axarr = plt.subplots(2, sharex=True)
    lafferty_flux, log_flux = get_flux_tables(table, y_method, function,
                                              SPECTRAL_INDEX)
    axarr[0].loglog(x, (x ** 2) * y)
    axarr[0].loglog(lafferty_flux['ENERGY'],
                    ((lafferty_flux['ENERGY'] ** 2) * lafferty_flux['DIFF_FLUX']),
                    marker='D', color='k', linewidth=0, ms=5,
                    label='Lafferty Method')
    residuals_lafferty = (lafferty_flux['DIFF_FLUX']
                          - function(lafferty_flux['ENERGY'])) / function(lafferty_flux['ENERGY']) * 100
    axarr[0].loglog(log_flux['ENERGY'],
                    (log_flux['ENERGY'] ** 2) * log_flux['DIFF_FLUX'],
                    marker='D', color='r', linewidth=0, ms=5,
                    label='Log Center Method')
    axarr[0].legend(loc='lower left', fontsize=10)
    residuals_log = (log_flux['DIFF_FLUX'] -
                     function(log_flux['ENERGY'])) / function(log_flux['ENERGY']) * 100
    axarr[1].semilogx(lafferty_flux['ENERGY'], residuals_lafferty, marker='D',
                      color='k', linewidth=0, ms=5)
    axarr[1].semilogx(log_flux['ENERGY'], residuals_log, marker='D',
                      color='r', linewidth=0, ms=5)
    indices = np.arange(len(energy_min))
    for index in indices:
        axarr[0].axvline(energy_min[index], 0, 1e6, color='k',
                         linestyle=':')
        axarr[1].axvline(energy_min[index], 0, 1e6, color='k',
                         linestyle=':')
    axarr[1].axhline(0, 0, 10, color='k')
    axarr[0].set_ylabel('E^2 * Differential Flux')
    axarr[1].set_ylabel('Residuals/%')
    axarr[1].set_xlabel('Energy')
    axarr[0].set_xlim([0.1, 10000])
    axarr[0].set_ylim([1e-6, 1e1])
    return plt

def plot_plaw():
    # Define the function
    x = np.arange(0.1, 100000, 0.1)
    spectral_model = my_spectrum(x)
    spectral_model_function = lambda x: my_spectrum(x)
    y = spectral_model
    # Set the x-bins
    energy_min = [0.1, 1, 10, 100, 1000]
    energy_max = [1, 10, 100, 1000, 10000]
    energies = np.array(_x_lafferty(energy_min, energy_max,
                                    spectral_model_function))
    diff_fluxes = spectral_model_function(energies)
    indices = np.array([0, 1, 2, 3, 4])
    int_flux = power_law_integral_flux(diff_fluxes, (indices + 1),
                                       energies, energy_min, energy_max)
    special = lambda x: np.log(x)
    int_flux[0] = np.abs(_integrate([energy_min[0]],
                                    [energy_max[0]], special)[0])
    # Put data into table
    table = Table()
    table['ENERGY_MIN'] = energy_min
    table['ENERGY_MAX'] = energy_max
    table['INT_FLUX'] = int_flux
    plt = plot_flux_points(table, x, y, spectral_model_function,
                           energy_min, energy_max, 'power_law')
    plt.legend()

max_stars_repo_name: zblz/gammapy

max_issues_repo_name: zblz/gammapy

max_forks_repo_name: zblz/gammapy

reason: import numpy,from astropy


text:
import time
import numpy as np
from skopt import dump
from hyperspace import hyperband
from hyperpoints.ppo.hyperband import hyperband_objective

def main():
    start = time.time()
    space = [(2,10), (2,8), (3,6), (0.00, .99), (0.001, 0.1), (0.5, 0.99), (2.5e-4, 0.1), (0.5, 0.9)]
    res = hyperband(hyperband_objective, space, n_evaluations=20, max_iter=200, eta=3, verbose=True, random_state=0)
    dump(res, 'result200.pkl')
    print(f'Runtime: {time.time() - start}')

if __name__ == '__main__':
    main()

max_stars_repo_name: yngtodd/hyperpoints

max_issues_repo_name: yngtodd/hyperpoints

max_forks_repo_name: yngtodd/hyperpoints


text:
import os, sys, re
import numpy as np
import argparse

atac_columns = [[3,4,5], [6,7,8], [9,10,11,12,13], [14,15,16], [17,18,19]]
data = []
cluster = []
markers = []
marker_genes = 'ATML1', 'SPCH', 'MUTE', 'FAMA', 'GASA9'
with open(sys.argv[1]) as fi:
    items = fi.readline().strip().split('\t')
    columns = items[3:]
    if len(columns) > 10:
        columns = list(marker_genes) + columns[-5:]
    locations = []
    for line in fi:
        avals = []
        evals = []
        items = line.strip().split('\t')
        for cols in atac_columns:
            avals.append(np.mean([float(items[col]) for col in cols]))
        for item in items[-5:]:
            if item == '.':
                evals = None
                break
            evals.append(np.log(float(item) + .1))
        if avals is not None and evals is not None:
            for m in marker_genes:
                if items[1].find('({})'.format(m)) >= 0:
                    markers.append((len(locations), m))
                    # print(m)
                    break
            locations.append(items[1])
            data.append(avals + evals)
            # print(avals, evals)
            cluster.append(4 - int(items[0]))

import umap
import pandas as pd
import sklearn.decomposition

matrix = pd.DataFrame(data, columns=columns, index=locations)
u = umap.UMAP(n_components=3)
res = u.fit_transform(matrix)

# ipca = sklearn.decomposition.IncrementalPCA(n_components=3)
# res = ipca.fit_transform(matrix)

#print(res.shape)
import plotly
import plotly.graph_objs as go
s = 4
traces = [go.Scatter3d(x=res[:,0], y=res[:,1], z=res[:,2], text=matrix.index, hoverinfo='text', mode='markers', marker=dict(size=s, colorscale='Viridis', cmin=0, cmax=max(cluster), opacity=0.8, color=cluster))]

layout = go.Layout(margin=dict(l=0, r=0, b=0, t=0), scene=dict(xaxis=dict(title='UMAP1'), yaxis=dict(title='UMAP2'), zaxis=dict(title='UMAP3')))

for i, m in markers:
    pos = res[i:i+1].T
    # print(cluster[i:i+1])
    print(i, cluster[i], m)
    trace = go.Scatter3d(x=pos[0], y=pos[1], z=pos[2], mode='markers', marker=dict(size=s * 2, colorscale='Viridis', color=[cluster[i],], cmin=0, cmax=max(cluster), line=dict(width=2, color='rgb(192,64,128)'), symbol='diamond'), name=m)
    traces.append(trace)

fig = go.Figure(data=traces, layout=layout)
plotly.offline.plot(fig, filename='scatter.html')
#for i in range(matrix.shape[0]):

max_stars_repo_name: takaho/toriilab

max_issues_repo_name: takaho/toriilab

max_forks_repo_name: takaho/toriilab


text:
import json
import os
import pandas as pd
import geopandas as gpd
import pyproj
import glob
import numpy as np
from poi_conflation_tool.googlemap import GoogleMapScrapper
from poi_conflation_tool.heremap import HereMapScrapper
from poi_conflation_tool.onemap import OneMap
from poi_conflation_tool.osm import OSM
from poi_conflation_tool.sla import SLA
from shapely.geometry import Point
from shapely.ops import transform
from poi_conflation_tool.model import Model
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from fuzzywuzzy.fuzz import token_set_ratio, ratio
from joblib import load
from shapely import geometry
from poi_conflation_tool.util import divide_bounding_box, pixelise_region
from copy import deepcopy

# load config file
with open(os.path.join(os.path.dirname(__file__), '../../config.json')) as f:
    config = json.load(f)

class POIConflationTool:
    """
    Performs POI conflation based on 5 different data sources (i.e., OSM, OneMap, SLA, Google Maps
    and HERE Map).
    """
    def __init__(self, subzones=None):
        """
        Checks if the OSM, OneMap, and SLA datasets are formatted. If any of the datasets are
        not formatted, the appropriate functions will be triggered to begin formatting the dataset.
        Also checks if machine learning model for identifying POI duplicates are trained.
        """
        # load country shapefiles for filtering out POIs based on subzones
        country_shp = gpd.read_file(os.path.join(os.path.dirname(__file__), config['country_shapefile']),
                                    encoding='utf-8')
        country_shp = country_shp.to_crs("EPSG:4326")
        if subzones is not None:
            country_shp = country_shp[country_shp['PLN_AREA_N'].isin(subzones)].reset_index(drop=True)
        self.country_shp = country_shp

        # load locally cached POIs that was conflated in the past
        print('Loading conflated data from local directory...')
        self.conflated_data = None

        # load formatted OneMap data. If it does not exist, format and load data.
        print('Loading OneMap data from local directory...')
        if not os.path.exists(os.path.join(os.path.dirname(__file__), config['onemap_cache'])):
            OneMap().format_data()
        onemap_data = self._load_json_as_geopandas(os.path.join(os.path.dirname(__file__), config['onemap_cache']))
        if subzones is not None:
            self.onemap_data = onemap_data[onemap_data.intersects(
                country_shp.loc[0, 'geometry'])].reset_index(drop=True)
        else:
            self.onemap_data = onemap_data

        # load formatted SLA data. If it does not exist, format and load data.
        print('Loading SLA data from local directory...')
        if not os.path.exists(os.path.join(os.path.dirname(__file__), config['sla_cache'])):
            SLA().format_data()
        sla_data = self._load_json_as_geopandas(os.path.join(os.path.dirname(__file__), config['sla_cache']))
        if subzones is not None:
            self.sla_data = sla_data[sla_data.intersects(country_shp.loc[0, 'geometry'])].reset_index(drop=True)
        else:
            self.sla_data = sla_data

        # load formatted OSM data. If it does not exist, format and load data.
        print('Loading OSM data from local directory...')
        if not os.path.exists(os.path.join(os.path.dirname(__file__), config['osm_cache'])):
            OSM().format_data()
        osm_data = self._load_json_as_geopandas(os.path.join(os.path.dirname(__file__), config['osm_cache']))
        if subzones is not None:
            self.osm_data = osm_data[osm_data.intersects(country_shp.loc[0, 'geometry'])].reset_index(drop=True)
        else:
            self.osm_data = osm_data

        # load formatted Google data. If it does not exist, save as None.
        print('Loading Google data from local directory...')
        self.google_scrapper = GoogleMapScrapper(config['search_radius'])
        if subzones is None:
            if os.path.exists(os.path.join(os.path.dirname(__file__), config['google_cache'])):
                self.google_data = self._load_json_as_geopandas(os.path.join(os.path.dirname(__file__),
                                                                             config['google_cache']))
            else:
                self.google_data = None
        else:
            if not os.path.exists(os.path.join(os.path.dirname(__file__), config['google_area_cache'])):
                self.google_data = self.google_scrapper.extract_area(subzones=subzones)
            google_data = self._load_json_as_geopandas(os.path.join(os.path.dirname(__file__),
                                                                    config['google_area_cache']))
            self.google_data = google_data[google_data.intersects(
                country_shp.loc[0, 'geometry'])].reset_index(drop=True)

        # load formatted HERE Map data. If it does not exist, save as None.
        print('Loading HERE Map data from local directory...')
        self.here_scrapper = HereMapScrapper(config['search_radius'])
        if subzones is None:
            if os.path.exists(os.path.join(os.path.dirname(__file__), config['here_cache'])):
                self.here_data = self._load_json_as_geopandas(os.path.join(os.path.dirname(__file__),
                                                                           config['here_cache']))
            else:
                self.here_data = None
        else:
            if not os.path.exists(os.path.join(os.path.dirname(__file__), config['here_area_cache'])):
                self.here_data = self.here_scrapper.extract_area(subzones=subzones)
            here_data = self._load_json_as_geopandas(os.path.join(os.path.dirname(__file__),
                                                                  config['here_area_cache']))
            self.here_data = here_data[here_data.intersects(country_shp.loc[0, 'geometry'])].reset_index(drop=True)

        # check if machine learning model is trained. If not, train model.
        model = Model()
        if not os.listdir(os.path.join(os.path.dirname(__file__), config['models_directory'])):
            model.train_model()
        model_filenames = glob.glob(os.path.join(os.path.dirname(__file__),
                                                 config['models_directory'] + 'model_?.joblib'))
        self.models = [load(filename) for filename in model_filenames]

    def _load_json_as_geopandas(self, cache_directory):
        """
        This function loads a local JSON file and converts it into a Geodataframe.

        :param cache_directory: str
            Contains the directory for the cache POI data file.
        :return:
        data: geopandas
            Contains the cached POI data file formatted as a Geodataframe.
        """
        with open(os.path.join(os.path.dirname(__file__), cache_directory)) as file:
            data_json = json.load(file)
        data = pd.json_normalize(data_json['features'])
        data = gpd.GeoDataFrame(data,
                                geometry=gpd.points_from_xy(data['geometry.lng'],
                                                            data['geometry.lat']))
        return data

    def _buffer_in_meters(self, lng, lat, radius):
        """
        Converts a latitude, longitude coordinate pair into a buffer with user-defined radius.s

        :param lng: float
            Contains the longitude information.
        :param lat: float
            Contains the latitude information.
        :param radius: float
            Contains the buffer radius in metres.
        :return:
        buffer_latlng: Polygon
            Contains the buffer.
        """
        proj_meters = pyproj.CRS('EPSG:3414')  # EPSG for Singapore
        proj_latlng = pyproj.CRS('EPSG:4326')

        project_to_metres = pyproj.Transformer.from_crs(proj_latlng, proj_meters, always_xy=True).transform
        project_to_latlng = pyproj.Transformer.from_crs(proj_meters, proj_latlng, always_xy=True).transform
        pt_meters = transform(project_to_metres, Point(lng, lat))
        buffer_meters = pt_meters.buffer(radius)
        buffer_latlng = transform(project_to_latlng, buffer_meters)
        return buffer_latlng

    def conflate_area_poi(self):
        """
        Performs POI conflation within a certain user defined area and saves the conflated data locally.
        """
        # pixelise region based on shapefile
        coordinate_list = divide_bounding_box(max_lat=config['max_lat'], min_lat=config['min_lat'],
                                              max_lng=config['max_lng'], min_lng=config['min_lng'],
                                              querybox_dim=config['search_radius']*2)
        coordinate_list = pixelise_region(coordinate_list, self.country_shp)

        # extract POI in the region and perform conflation
        i = 1
        for coordinate in coordinate_list:
            print('Processing query {}/{}'.format(i, len(coordinate_list)))
            i += 1
            centre_lat = (coordinate[2] + coordinate[0]) / 2
            centre_lng = (coordinate[1] + coordinate[3]) / 2

            # create circular buffer around centre
            buffer = self._buffer_in_meters(centre_lng, centre_lat, config['search_radius'])

            # extracts neighbouring POIs from OSM
            osm_pois = self.osm_data[self.osm_data.intersects(buffer)]

            # extract neighbouring POIs from OneMap
            onemap_pois = self.onemap_data[self.onemap_data.intersects(buffer)]

            # extract neighbouring POIs from SLA
            sla_pois = self.sla_data[self.sla_data.intersects(buffer)]

            # extract neighbouring POIs from GoogleMap either locally or using API
            assert self.google_data is not None
            google_pois = self.google_data[self.google_data.intersects(buffer)]

            # extract neighbouring POIs from HERE Map either locally or using API
            assert self.here_data is not None
            here_pois = self.here_data[self.here_data.intersects(buffer)]

            # perform conflation
            combined_pois = pd.concat([osm_pois, onemap_pois, sla_pois,
                                       google_pois, here_pois], ignore_index=True)
            conflated_pois = self._perform_conflation(combined_pois)

            # cache conflated POIs
            if (self.conflated_data is None) and (conflated_pois is None):
                continue
            elif self.conflated_data is None:
                self.conflated_data = conflated_pois
            else:
                self.conflated_data = pd.concat([self.conflated_data, conflated_pois], ignore_index=True)

            if self.conflated_data is not None:
                if 'duplicates' in self.conflated_data.columns:
                    self.conflated_data.drop(columns=['duplicates'], inplace=True)
                self.conflated_data.to_file(os.path.join(os.path.dirname(__file__), config['conflated_area_cache']),
                                            encoding='utf-8')

    def extract_poi(self, lat, lng, stop_id=None):
        """
        Extracts the neighbouring POIs around a location based on the geographical coordinates
        and performs POI conflation.

        :param lat: float
            Contains the latitudinal information.
        :param lng: float
            Contains the longitudinal information.
        :param stop_id: str
            Contains the unique ID of a particular stop.
        :return:
        conflated_pois: Geodataframe
            Contains the conflated POIs.
        """
        # create circular buffer around POI
        buffer = self._buffer_in_meters(float(lng), float(lat), config['search_radius'])

        # extracts neighbouring POIs from OSM
        osm_pois = self.osm_data[self.osm_data.intersects(buffer)]

        # extract neighbouring POIs from OneMap
        onemap_pois = self.onemap_data[self.onemap_data.intersects(buffer)]

        # extract neighbouring POIs from SLA
        sla_pois = self.sla_data[self.sla_data.intersects(buffer)]

        # extract neighbouring POIs from GoogleMap either locally or using API
        if (self.google_data is not None) and (stop_id in self.google_data['stop'].tolist()):
            google_pois = self.google_data[self.google_data['stop'] == stop_id]
        else:
            google_pois = self.google_scrapper.extract_poi(lat, lng, stop_id)
            if self.google_data is None:
                self.google_data = google_pois
            else:
                self.google_data = pd.concat([self.google_data, google_pois], ignore_index=True)

        # extract neighbouring POIs from HERE Map either locally or using API
        if (self.here_data is not None) and (stop_id in self.here_data['stop'].tolist()):
            here_pois = self.here_data[self.here_data['stop'] == stop_id]
        else:
            here_pois = self.here_scrapper.extract_poi(lat, lng, stop_id)
            if self.here_data is None:
                self.here_data = here_pois
            else:
                self.here_data = pd.concat([self.here_data, here_pois], ignore_index=True)

        # perform conflation
        combined_pois = pd.concat([osm_pois, onemap_pois, sla_pois,
                                   google_pois, here_pois], ignore_index=True)
        # combined_pois = pd.concat([osm_pois, onemap_pois, sla_pois,
        #                            here_pois], ignore_index=True)  # TODO remove
        # combined_pois = osm_pois  # TODO remove

        if len(combined_pois) == 0:
            return None

        conflated_pois = self._perform_conflation(combined_pois)

        # return combined_pois
        return conflated_pois

    def _perform_conflation(self, potential_duplicates):
        """
        Performs POIs conflation by identifying the duplicates among a group of POIs.

        :param potential_duplicates: Geodataframe
            Contains the group of POIs found in a particular region. Some of which could be duplicates.

        :return:
        conflated_pois: Geodataframe
            Contains the conflated POIs.
        """
        # remove POIs with no name information
        potential_duplicates.dropna(subset=['properties.name'], inplace=True)
        potential_duplicates = potential_duplicates[~potential_duplicates['properties.name'].isin(['None'])]
        potential_duplicates.reset_index(drop=True, inplace=True)

        # vectorise address information
        if len(potential_duplicates) > 1:
            address_corpus = potential_duplicates['properties.address'].fillna('Singapore').tolist()
            address_corpus = [address if address != '' else 'Singapore' for address in address_corpus]
            address_vectorizer = TfidfVectorizer(analyzer='char_wb', ngram_range=(2, 3))
            address_matrix = address_vectorizer.fit_transform(address_corpus)

            # identify duplicates using ML models
            duplicate_idx = []
            for i in range(len(potential_duplicates)):
                duplicate_idx.append(self._identify_duplicates(potential_duplicates, i, address_matrix))

            assert len(potential_duplicates) == len(duplicate_idx)
            potential_duplicates['duplicates'] = duplicate_idx

            # conflate duplicated POIs
            conflated_pois = self._conflate(potential_duplicates)

            return conflated_pois
        else:
            return potential_duplicates

    def _identify_duplicates(self, data, centroid_idx, address_matrix):
        """
        Identifies duplicated POI points to a centroid POI by passing the name similarity and address
        similarity values into machine learning models for classification.

        :param data: Geodataframe
            Contains information about the neighbouring POIs and the centroid POI.
        :param centroid_idx: int
            Contains the index of the centroid POI in data.
        :param address_matrix: np.array
            Contains the TFIDF-vectorised address matrix.

        :return:
        duplicated_id: list
            Contains a list of IDs for the duplicated POIs
        """
        if len(data) > 1:  # presence of neighbouring POIs
            data.reset_index(drop=True, inplace=True)
            neighbour_idx = list(data.index)
            neighbour_idx.remove(centroid_idx)
            neighbour_data = data.iloc[neighbour_idx]

            address_similarity = cosine_similarity(address_matrix[neighbour_idx, :],
                                                   address_matrix[centroid_idx, :]).reshape(-1, 1)

            address_str_similarity = np.array(
                [token_set_ratio(data.loc[centroid_idx, 'properties.address'].lower(),
                                 neighbour_address.lower())
                 if not pd.isnull(neighbour_address) else 0.0
                 for neighbour_address
                 in neighbour_data['properties.address'].tolist()]
            ).reshape(-1, 1)

            name_similarity = np.array(
                [token_set_ratio(data.loc[centroid_idx, 'properties.name'].lower(),
                                 neighbour_name.lower())
                 if not pd.isnull(neighbour_name) else 0.0
                 for neighbour_name
                 in neighbour_data['properties.name'].tolist()]
            ).reshape(-1, 1)

            # Pass name and address similarity values into ML models
            predict_prob = np.zeros((len(neighbour_idx), 2))
            for model in self.models:
                predict_prob += model.predict_proba(np.hstack((address_similarity,
                                                               address_str_similarity,
                                                               name_similarity)))
            temp_idx = list(np.where(np.argmax(predict_prob, axis=1) == 1)[0])
            # POIs are only considered as duplicates if they come from different sources
            duplicate_id = [data.loc[neighbour_idx[idx], 'id'] for idx in temp_idx
                            if data.loc[neighbour_idx[idx], 'properties.source'] !=
                            data.loc[centroid_idx, 'properties.source']]

            # identify duplicates through exact name matches too
            exact_match_id = [data.loc[idx, 'id']
                              for idx in neighbour_idx
                              if ratio(data.loc[centroid_idx, 'properties.name'].lower(),
                                       data.loc[idx, 'properties.name'].lower()) == 100]

            return list(set(duplicate_id + exact_match_id))
        else:  # no neighbours
            return []

    def _conflate(self, data):
        """
        Performs conflation on the duplicated POIs.

        :param data: Geodataframe
            Contains the duplicated POIs.

        :return:
        conflated_pois: Geodataframe
            Contains the conflated POIs.
        """
        conflated_pois = gpd.GeoDataFrame()
        processed_id = []

        for i in range(len(data)):
            if data.loc[i, 'id'] in processed_id:  # ignore POIs that has been merged
                continue

            if len(data.loc[i, 'duplicates']) > 0:
                duplicate_ids, duplicates = self._find_all_duplicates(data.loc[i, 'duplicates'] + [data.loc[i, 'id']],
                                                                      data)
                conflated_pois = conflated_pois.append(self._merge_duplicates(duplicates), ignore_index=True)
                processed_id += duplicate_ids
            else:
                conflated_pois = conflated_pois.append(data.iloc[i], ignore_index=True)
                processed_id.append(data.loc[i, 'id'])

        return conflated_pois

    def _find_all_duplicates(self, duplicate_ids, data):
        """
        Searches for all identified duplicates and return their IDs and attribute information.

        :param duplicate_ids: list
            Contains a list of IDs that are identified to be duplicates.
        :param data: Geodataframe
            Contains the list of neighbouring POIs.

        :return:
        temp_ids: list
            Contains the full list of IDs that are identified as duplicates
        data[data['id'].isin(duplicate_ids)]: Geodataframe:
            Contains the POIs that are identified as duplicates.
        """
        temp_ids = deepcopy(duplicate_ids)
        all_duplicates_not_found = True
        while all_duplicates_not_found:
            temp_data = data[data['id'].isin(temp_ids)].reset_index(drop=True)
            id_list = list(set([item
                                for sublist in temp_data['duplicates'].tolist()
                                for item in sublist] + temp_ids))

            if sorted(id_list) == sorted(temp_ids):
                all_duplicates_not_found = False

            temp_ids = id_list

        return temp_ids, data[data['id'].isin(temp_ids)]

    def _extract_trusted_source_idx(self, data):
        """
        Extract the indices of the POIs that come from the most trusted source.
        OneMap > SLA > GoogleMap > HereMap > OpenstreetMap

        :param data: Geodataframe
            Contain all of the neighbouring POIs in the area.

        :return: list
            Contains the list of indices belonging to POIs from the most trusted source.
        """
        source_list = data['properties.source'].tolist()

        if 'OneMap' in source_list:
            return list(data[data['properties.source'] == 'OneMap'].index)
        elif 'SLA' in source_list:
            return list(data[data['properties.source'] == 'SLA'].index)
        elif 'GoogleMap' in source_list:
            return list(data[data['properties.source'] == 'GoogleMap'].index)
        elif 'HereMap' in source_list:
            return list(data[data['properties.source'] == 'HereMap'].index)
        elif 'OpenStreetMap' in source_list:
            return list(data[data['properties.source'] == 'OpenStreetMap'].index)
        else:
            raise ValueError('{} does not fall into any one of the POI sources.'.format(data['properties.source']))

    def _merge_duplicates(self, duplicates):
        """
        Merges the attributes of all duplicated POIs identified to form a new POI.

        :param duplicates: Geodataframe
            Contains the full list of duplicated POIs.

        :return:
        merged_poi: geoseries
            Contains the merged POI.
        """
        duplicates.reset_index(drop=True, inplace=True)
        trusted_idx = self._extract_trusted_source_idx(duplicates)
        merged_poi = gpd.GeoSeries(crs=4326)
        columns_processed = []

        for column in list(duplicates.columns):
            if column in columns_processed:
                continue

            # feature type
            if column == 'type':
                merged_poi[column] = 'Feature'
                columns_processed.append(column)

            # geometry coordinates
            elif 'geometry' in column:
                if len(duplicates) > 2:
                    centroid = geometry.Polygon([[p.x, p.y] for p in duplicates['geometry'].tolist()]).centroid
                else:
                    centroid = geometry.LineString([[p.x, p.y] for p in duplicates['geometry'].tolist()]).centroid
                merged_poi['geometry.lat'] = centroid.y
                merged_poi['geometry.lng'] = centroid.x
                merged_poi['geometry'] = centroid
                columns_processed += ['geometry', 'geometry.lat', 'geometry.lng']

            # address
            elif column == 'properties.address':
                merged_poi['properties.address'] = max(duplicates.loc[trusted_idx, 'properties.address'].tolist(),
                                                       key=len)
                columns_processed.append(column)

            # name
            elif column == 'properties.name':
                merged_poi['properties.name'] = max(duplicates.loc[trusted_idx, 'properties.name'].tolist(),
                                                    key=len)
                columns_processed.append(column)

            # place type
            elif column == 'properties.place_type':  # store all place types in a list
                temp_list = list(set('; '.join(duplicates[column].tolist()).split('; ')))
                merged_poi[column] = '; '.join(temp_list)
                columns_processed.append(column)

            # tags
            elif 'properties.tags' in column:
                tags = list(set([item for item in duplicates[column].tolist() if not pd.isnull(item)]))
                if len(tags) > 0:
                    merged_poi[column] = '; '.join(tags)
                columns_processed.append(column)

            # source
            elif column == 'properties.source':  # store all sources in a list
                source_list = list(set(duplicates[column].tolist()))
                merged_poi[column] = '; '.join(source_list)
                columns_processed.append(column)

            # requires_verification
            elif 'properties.requires_verification' in column:
                if 'Yes' in duplicates['properties.requires_verification.summary'].tolist():
                    merged_poi['properties.requires_verification.summary'] = 'Yes'
                    merged_poi['properties.requires_verification.reason'] = '; '.join(list(set(
                        [reason for reason in duplicates['properties.requires_verification.reason'].tolist()
                         if not pd.isnull(reason)])))
                else:
                    merged_poi['properties.requires_verification.summary'] = 'No'
                columns_processed += ['properties.requires_verification.summary',
                                      'properties.requires_verification.reason']

            # id information
            elif column == 'id':  # store all ids in a list
                merged_poi[column] = '; '.join(duplicates[column].tolist())
                columns_processed.append(column)

            # stop ids
            elif column == 'stop':
                stop_ids = list(set([item for item in duplicates[column].tolist() if not pd.isnull(item)]))
                if len(stop_ids) > 0:
                    merged_poi[column] = '; '.join(tags)
                columns_processed.append(column)

            # extraction date information
            elif column == 'extraction_date':  # the latest date will be chosen
                extraction_dates = [int(date) for date in duplicates[column].tolist() if not pd.isnull(date)]

                if extraction_dates:
                    merged_poi[column] = str(max(extraction_dates))
                else:
                    raise ValueError('Extraction date information is missing.')
                columns_processed.append(column)

            elif column == 'duplicates':  # ignore duplicate field
                columns_processed.append(column)
                continue

            else:
                raise ValueError('{} is not considered!'.format(column))

        assert set(list(duplicates.columns)).issubset(set(columns_processed))
        return merged_poi

# if __name__ == '__main__':
    # extracting POIs based on subzones
    # tool = POIConflationTool(subzones=['JURONG EAST'])
    # tool.conflate_area_poi()
    # data = tool.conflated_data

    # extracting POIs on the fly
    # tool = POIConflationTool()
    # data = pd.read_excel('../../data/processed_data/combined_stop_data.xlsx')
    # for i in range(len(data)):
    #     print('Processing {}/{}'.format(i+1, len(data)))
    #     tool.extract_poi(data.loc[i, 'StopLat'],
    #                      data.loc[i, 'StopLon'],
    #                      str(data.loc[i, 'StopID']))


text:
# Natural Language Processing

# Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import re
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
# Importing the dataset
dataset = pd.read_csv('Airline.csv')
y=dataset.iloc[:,5].values
y=np.array(y)
x1=dataset.iloc[:,7].values
x2=dataset.iloc[:,14].values
x1=np.asarray(x1,dtype='str')
x2=np.asarray(x2,dtype='str')
Y=np.zeros((len(y),3))
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
labelencoder = LabelEncoder()
yt = labelencoder.fit_transform(y)
onehotencoder = OneHotEncoder(categorical_features ='all')
yt=yt.reshape(-1,1)
yt = onehotencoder.fit_transform(yt).toarray()

x=[]
for i in range(0,len(x1)):
    x.append(x1[i]+x2[i])
# Cleaning the texts

corpus = []
for i in range(0, len(x)):
    review = re.sub('[^a-zA-Z]', ' ', x[i])
    review = review.lower()
    review = review.split()
    ps = PorterStemmer()
    review = [ps.stem(word) for word in review if not word in (set(stopwords.words('english')),'nan')]
    review = ' '.join(review)
    corpus.append(review)

# Creating the Bag of Words model
from sklearn.feature_extraction.text import CountVectorizer
cv = CountVectorizer(max_features = 2200)
X = cv.fit_transform(corpus).toarray()

# Splitting the dataset into the Training set and Test set
from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, yt, test_size = 0.20, random_state = 0)
ytt=np.zeros((len(y_test),1))
for i in range(0,len(y_test)):
    t=y_test[i][0]
    ind=0
    for j in range(0,3):
        if(y_test[i][j]>t):
            t=y_test[i][j]
            ind=j
    ytt[i]=ind
import keras
from keras.models import Sequential
from keras.layers import Dense
classifier = Sequential()
classifier.add(Dense(output_dim = 2200, init = 'uniform', activation = 'relu', input_dim = 2200))
classifier.add(Dense(output_dim = 2200, init = 'uniform', activation = 'relu'))
classifier.add(Dense(output_dim = 1100, init = 'uniform', activation = 'relu'))
classifier.add(Dense(output_dim = 1100, init = 'uniform', activation = 'relu'))
classifier.add(Dense(output_dim = 3, init = 'uniform', activation = 'softmax'))
classifier.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])
classifier.fit(X_train, y_train, batch_size = 20, nb_epoch = 50)
y_pred = classifier.predict(X_test)
ypt=np.zeros((len(y_pred),1))
for i in range(0,len(y_pred)):
    t=y_pred[i][0]
    ind=0
    for j in range(0,3):
        if(y_pred[i][j]>t):
            t=y_pred[i][j]
            ind=j
    ypt[i]=ind

from sklearn.metrics import confusion_matrix
cm = confusion_matrix(ytt, ypt)

max_stars_repo_name: CBringiton/Airline-sentiment-prediction

max_issues_repo_name: CBringiton/Airline-sentiment-prediction

max_forks_repo_name: CBringiton/Airline-sentiment-prediction


text:
# Modified by: Dr. Smruti Panigrahi

import numpy as np
import cv2
import time
from transformation import *

# Apply the above functions in succession and update the Rover state accordingly
def perception_step(Rover):
    # Perform perception steps to update Rover()
    # TODO:
    # NOTE: camera image is coming to you in Rover.img
    # 1) Define source and destination points for perspective transform
    # 2) Apply perspective transform
    # 3) Apply color threshold to identify navigable terrain/obstacles/rock samples
    # 4) Update Rover.vision_image (this will be displayed on left side of screen)
        # Example: Rover.vision_image[:,:,0] = obstacle color-thresholded binary image
        #          Rover.vision_image[:,:,1] = rock_sample color-thresholded binary image
        #          Rover.vision_image[:,:,2] = navigable terrain color-thresholded binary image

    # 5) Convert map image pixel values to rover-centric coords
    # 6) Convert rover-centric pixel values to world coordinates
    # 7) Update Rover worldmap (to be displayed on right side of screen)
        # Example: Rover.worldmap[obstacle_y_world, obstacle_x_world, 0] += 1
        #          Rover.worldmap[rock_y_world, rock_x_world, 1] += 1
        #          Rover.worldmap[navigable_y_world, navigable_x_world, 2] += 1

    # 8) Convert rover-centric pixel positions to polar coordinates
    # Update Rover pixel distances and angles
        # Rover.nav_dists = rover_centric_pixel_distances
        # Rover.nav_angles = rover_centric_angles

    # 1) Receive image frame from Rover camera
    image = Rover.img

    # 2) Apply perspective transform
    warped_image, mask = perspect_transform(image)

    # 3) Apply color threshold to identify navigable terrain/obstacles/rock samples
    navigable_warped_threshed = color_thresh(warped_image, (161, 161, 161), (255, 255, 255))
    #obstacle_warped_threshed = np.absolute(np.float32(navigable_warped_threshed) - 1)*mask
    obstacle_warped_threshed = color_thresh(warped_image, (0, 0, 0), (160, 160, 160))*mask
    rock_warped_threshed = color_thresh(warped_image, (100, 100, 0), (255, 255, 50))

    # 4) Update Rover.vision_image (this will be displayed on left side of screen)
    Rover.vision_image[:,:,0] = obstacle_warped_threshed*255
    Rover.vision_image[:,:,1] = rock_warped_threshed*255
    Rover.vision_image[:,:,2] = navigable_warped_threshed*255

    # 5) Convert warped and thresholded map image pixel values to rover-centric coords
    # Calculate pixel values in rover-centric coords and distance/angle to all pixels
    navigable_xpix, navigable_ypix = rover_coords(navigable_warped_threshed)
    obstacle_xpix, obstacle_ypix = rover_coords(obstacle_warped_threshed)
    rock_xpix, rock_ypix = rover_coords(rock_warped_threshed)

    # 5.1) To increase fidelity limit the distance to which the Rover can see clearly
    navigable_xpix, navigable_ypix = pix_in_range(navigable_xpix, navigable_ypix)
    obstacle_xpix, obstacle_ypix = pix_in_range(obstacle_xpix, obstacle_ypix)
    rock_xpix, rock_ypix = pix_in_range(rock_xpix, rock_ypix, range=50) #To not care about rocks on the other side of the road

    # 6) Convert rover-centric pixel values to world coords
    # Get Rover position and orientation in world coords
    scale = 10
    world_size = Rover.worldmap.shape[0]
    xpos = Rover.pos[0]
    ypos = Rover.pos[1]
    yaw = Rover.yaw
    # Get navigable pixel positions in world coords
    navigable_x_world, navigable_y_world = pix_to_world(navigable_xpix, navigable_ypix,
                                    xpos, ypos, yaw, world_size, scale)
    obstacle_x_world, obstacle_y_world = pix_to_world(obstacle_xpix, obstacle_ypix,
                                    xpos, ypos, yaw, world_size, scale)
    rock_x_world, rock_y_world = pix_to_world(rock_xpix, rock_ypix,
                                    xpos, ypos, yaw, world_size, scale)

    # 7) Update worldmap (to be displayed on right side of screen)
    # Add pixel positions of navigable terrain/obstacles/samples to worldmap
    # For better fidelity update map only if roll and pitch angles are minimal (near 0/360)
    if (Rover.pitch < Rover.pitch_min or Rover.pitch > Rover.pitch_max) and (Rover.roll < Rover.roll_min or Rover.roll > Rover.roll_max):
        Rover.worldmap[obstacle_y_world, obstacle_x_world, 0] += 1
        Rover.worldmap[rock_y_world, rock_x_world,1]  += 1
        Rover.worldmap[navigable_y_world, navigable_x_world, 2] += 10
        # remove overlap mesurements
        navigable_pix = Rover.worldmap[:,:,2] > 0 #if blue channel is > 0 then set it as navigable terrain
        Rover.worldmap[navigable_pix,0] = 0 # Make obstacle channels ZERO corresponding to navigable fixels
        # clip to avoid overflow
        Rover.worldmap = np.clip(Rover.worldmap, 0, 255)

    # 8) Convert rover-centric pixel positions to polar coordinates
    # Update Rover pixel distances and angles
    Rover.nav_dists, Rover.nav_angles = to_polar_coords(navigable_xpix, navigable_ypix)
    Rover.obs_dists, Rover.obs_angles = to_polar_coords(obstacle_xpix, obstacle_ypix)
    Rover.rock_dists, Rover.rock_angles = to_polar_coords(rock_xpix, rock_ypix)

    navigable_mean_dir = np.mean(Rover.nav_angles)
    obstacle_mean_dir = np.mean(Rover.obs_angles)
    rock_mean_dir = np.mean(Rover.rock_angles)

    # 9) Get on mission to look for gold rock samples
    # When rock sample is in sight: approach, stop when near rock, collect it and move on
    if len(Rover.rock_dists) > 0: #if rock_warped_threshed.any():
        Rover.pickup_counter += 1
        rock_idx = np.argmin(Rover.rock_dists) #Returns the indices of the minimum rock distance.
        rock_xcen = rock_x_world[rock_idx] #Make min. Rock dist. in Rover as center of Rock in worldmap
        rock_ycen = rock_y_world[rock_idx]
        Rover.rock_pos = (rock_xcen, rock_ycen)
        #if len(Rover.rock_dists) > 0: #if any Rock pixels detected
        Rover.nav_dists = Rover.rock_dists
        Rover.nav_angles = Rover.rock_angles
        Rover.worldmap[rock_ycen, rock_xcen, 1] = 255
        Rover.vision_image[:, :, 1] = rock_warped_threshed*255
        Rover.mode = 'pursuit'
        print('Rover detected gold rock sample')

    else:
        Rover.pickup_counter = 0
        #print('Rover does not see any golden rock')
        Rover.vision_image[:, :, 1] = 0
        #Rover.nav_dists = Rover.nav_dists
        #Rover.nav_angles = Rover.nav_angles
        if Rover.mode == 'pursuit':
            Rover.mode = 'forward'

    return Rover

max_stars_repo_name: DrPanigrahi/RoboND-Rover-Project

max_issues_repo_name: DrPanigrahi/RoboND-Rover-Project

max_forks_repo_name: DrPanigrahi/RoboND-Rover-Project


text:
import numpy as np
import matplotlib.pyplot as plt

def generate_linerp_plot(linerp_vals_train, linerp_vals_test, title:str=''):
    xs_train = np.linspace(0, 1, len(linerp_vals_train))
    xs_test = np.linspace(0, 1, len(linerp_vals_test))

    fig = plt.figure(figsize=(8, 5))
    if title != '': plt.title(title)
    plt.plot(xs_train, linerp_vals_train, label='Train')
    plt.plot(xs_test, linerp_vals_test, label='Test')
    plt.legend()
    plt.xlabel('alpha')
    plt.grid()

    return fig

def generate_acts_entropy_linerp_plot(linerp_values):
    linerp_values = np.array(linerp_values)
    xs = np.linspace(0, 1, linerp_values.shape[1])
    colors = plt.cm.jet(np.linspace(0, 1, linerp_values.shape[0]))

    fig = plt.figure(figsize=(12, 7))
    for i, layer_entropies in enumerate(linerp_values):
        plt.plot(xs, layer_entropies, label='Layer #%d' % i, color=colors[i])
    plt.legend()
    plt.xlabel('alpha')
    plt.grid()

    return fig

def generate_weights_entropy_linerp_plot(values):
    xs = np.linspace(0, 1, len(values))

    fig = plt.figure(figsize=(7, 5))
    plt.plot(xs, values)
    plt.xlabel('alpha')
    plt.grid()

    return fig

max_stars_repo_name: universome/loss-patterns

max_issues_repo_name: universome/loss-patterns

max_forks_repo_name: universome/loss-patterns


text:
"""
This script compares the png test image given in the second argument against
the known good baseline given in the third. Alternate known good images
(with _number.png in the filename) are also compared against.
"""

import math
import numpy
import vtk
import os
import os.path
import re
import sys
import logging
import vcs

defaultThreshold=10.0

def init(*args, **kwargs):
    testingDir = os.path.join(os.path.dirname(__file__), "..")
    sys.path.append(testingDir)

    if ((('bg' in kwargs and kwargs['bg']) or ('bg' not in kwargs))):
        vcsinst = vcs.init(*args, **dict(kwargs, bg=1))
        if ('geometry' not in kwargs):
            vcsinst.setbgoutputdimensions(1200, 1091, units="pixels")
        else:
            xy = kwargs['geometry']
            vcsinst.setbgoutputdimensions(xy[0], xy[1], units="pixels")
    else:
        vcsinst = vcs.init(*args, **dict(kwargs, bg=0))

    vcsinst.setantialiasing(0)
    vcsinst.drawlogooff()
    return vcsinst

def run(vcsinst, fname, baseline=sys.argv[1], threshold=defaultThreshold):
    """Export plot to a png and exit after comparsion."""
    vcsinst.png(fname)
    sys.exit(check_result_image(fname, baseline, threshold))

def run_wo_terminate(vcsinst, fname, baseline=sys.argv[1], threshold=defaultThreshold):
    """Export plot to a png and return comparison with baseline."""
    vcsinst.png(fname)
    return check_result_image(fname, baseline, threshold)

def image_compare(testImage, baselineImage):
    imageDiff = vtk.vtkImageDifference()
    imageDiff.SetInputData(testImage)
    imageDiff.SetImageData(baselineImage)
    imageDiff.Update()
    return (imageDiff.GetThresholdedError(), imageDiff.GetOutput())

def dump_image_to_file(fname,img):
    wr = vtk.vtkPNGWriter()
    wr.SetFileName(fname)
    wr.SetInputData(img)
    wr.Write()

def image_from_file(fname):
    try:
        rd = vtk.vtkPNGReader()
        rd.SetFileName(fname)
        removeAlpha = vtk.vtkImageExtractComponents()
        removeAlpha.SetComponents(0, 1, 2)
        removeAlpha.SetInputConnection(rd.GetOutputPort())
        removeAlpha.Update()
        return removeAlpha.GetOutput();
    except Exception,err:
        print "Problem opening file '%s': %s"%(fname,err)
        return None

# find alternate baselines for fname of the form basename_d.ext
# where fname = basename.ext and d is a digit between 1 and 9
def find_alternates(fname):
    dirname = os.path.dirname(fname)
    prefix, ext = os.path.splitext(os.path.split(fname)[1])
    files = os.listdir(dirname)
    results = [fname]
    for i in files:
        if (re.match(prefix + '_[1-9]' + ext, i)):
            results.append(os.path.join(dirname, i))
    return results

def check_result_image(fname, baselinefname=sys.argv[1], threshold=defaultThreshold,
                       baseline=True, cleanup=True, update_baselines = False, suffix=""):
    testImage = image_from_file(fname)
    if testImage is None:
        print "Testing image missing, test failed."
        return -1

    if baseline:
        baselinefnames = find_alternates(baselinefname)
    else:
        baselinefnames = [baselinefname,]

    print "Found Baselines:"
    for baselinefname in baselinefnames:
        print "- %s"%baselinefname

    bestImage = None
    bestFilename = None
    bestDiff = None
    bestDiffImage = None
    for baselineFilename in baselinefnames:
        sys.stdout.write("Comparing %s  %s ..."%(fname, baselineFilename))
        baselineImage = image_from_file(baselineFilename)
        if baselineImage is None:
            continue

        diff, diffImage = image_compare(testImage, baselineImage)

        sys.stdout.write("diff=%f"%diff)

        if bestDiff is None or diff < bestDiff:
            sys.stdout.write(", New best!")
            bestDiff = diff
            bestFilename = baselineFilename
            bestImage = baselineImage
            bestDiffImage = diffImage
        sys.stdout.write("\n")

    if bestImage is None:
      print "No baseline images found. Test failed."
      return -1

    if bestDiff < threshold:
        print "Baseline '%s' is the best match with a difference of %f."%(bestFilename, bestDiff)
        if cleanup:
            print "Deleting test image '%s'..."%(fname)
            os.remove(fname)
        return 0

    print "All baselines failed! Lowest error (%f) exceeds threshold (%f)."%(bestDiff, threshold)

    if update_baselines:
        bestFilename2=bestFilename[:-4]+suffix+".png"
        print "Update baselines is ON so we are assuming you know what you're doing"
        print "Replacing baseline %s with new baseline from %s" % (bestFilename2, fname)
        import shutil
        shutil.copy2(fname, bestFilename2)

    sp = fname.split(".")
    diffFilename = ".".join(sp[:-1])+"_diff."+sp[-1]
    print "Saving image diff at '%s'."%diffFilename
    dump_image_to_file(diffFilename, bestDiffImage)

    # Print metadata for CDash image upload:
    def printDart(name, type, value, suff=""):
      print '<DartMeasurement%s name="%s" type="%s">%s</DartMeasurement%s>'%(
        suff, name, type, value, suff)
    printDart("ImageError", "numeric/double", "%f"%bestDiff)
    printDart("TestImage", "image/png", os.path.abspath(fname), "File")
    printDart("DifferenceImage", "image/png", os.path.abspath(diffFilename), "File")
    printDart("ValidImage", "image/png", os.path.abspath(bestFilename), "File")
    return -1

def main():
    if len(sys.argv) != 4:
        print "Error:"
        print "Called with: " + str(sys.argv)
        print "Call with " + sys.argv[0] + " test_directory/test_filename.png  baseline_directory/baseline_filename.png threshold"
        sys.exit(-1)

    ret = check_result_image(sys.argv[1], sys.argv[2], float(sys.argv[3]))
    sys.exit(ret)

if __name__ == "__main__":
    main()

max_stars_repo_name: xylar/cdat

max_issues_repo_name: xylar/cdat

max_forks_repo_name: CDAT/uvcdat


text:
import numpy as np
import cv2
# from opencv.video import create_capture

# Once the camera has detected two faces, it will swap the bounding box of the faces. Rudimentary face swap app!
# There are many improvements that should be made for it to be a good looking face swap

def detect(img, cascade):
    rects = cascade.detectMultiScale(img, scaleFactor=1.3, minNeighbors=4, minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE)
    if len(rects) == 0:
        return []
    rects[:,2:] += rects[:,:2]
    return rects

def get_cam_frame(cam):
    ret, img = cam.read()
    # smaller frame size - things run a lot smoother than a full screen img
    img = cv2.resize(img, (800, 600))
    return img

def main():
    # Camera 0 is usually the built in webcam camera... also most people only have 1 webcam on their laptop
    cam = cv2.VideoCapture(0)

    # lbp_classifier = "../data/lbpcascade_frontalface.xml"
    haar_classifier = "../data/haarcascade_frontalface_default.xml"

    # use the haar classifier for now, it seems to work a little bit better
    cascade = cv2.CascadeClassifier(haar_classifier)

    while True:
        img = get_cam_frame(cam)

        # classifier wants things in black and white
        bw = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        bw = cv2.equalizeHist(bw)

        rects = detect(bw, cascade)
        final = img.copy()

        # Mostly useful for debugging
        # for x1, y1, x2, y2 in rects:
        #     cv2.rectangle(img, (x1, y1), (x2, y2), (0,255,0), 2)

        # Have we detected at least 2 faces?
        if len(rects) > 1:
            # A better solution would be to get the 2 largest rectangles from the list, because
            # sometimes we get false positives
            ax1, ay1, ax2, ay2 = rects[0]
            bx1, by1, bx2, by2 = rects[1]

            # Swap the pixels of each face
            faceA = cv2.resize(img[ay1:ay2, ax1:ax2].copy(), (bx2 - bx1, by2 - by1))
            faceB = cv2.resize(img[by1:by2, bx1:bx2].copy(), (ax2 - ax1, ay2 - ay1))
            final[ay1:ay2, ax1:ax2] = faceB
            final[by1:by2, bx1:bx2] = faceA

        cv2.imshow('face swap', final)

        # Esc key quits
        if 0xFF & cv2.waitKey(1) == 27:
            break
    cv2.destroyAllWindows()

if __name__ == '__main__':
    main()

max_stars_repo_name: rustielin/cvision

max_issues_repo_name: rustielin/cvision

max_forks_repo_name: rustielin/cvision


text:
import numpy as np
from matplotlib import pyplot as plt
import sys
import os

def run(seq='00'):
    pose_file = "/media/l/yp2/KITTI/odometry/dataset/poses/"+seq+".txt"
    poses = np.genfromtxt(pose_file)
    poses = poses[:, [3, 11]]
    inner = 2*np.matmul(poses, poses.T)
    xx = np.sum(poses**2, 1, keepdims=True)
    dis = xx-inner+xx.T
    dis = np.sqrt(np.abs(dis))
    id_pos = np.argwhere(dis < 3)
    id_neg = np.argwhere(dis > 20)
    # id_pos=id_pos[id_pos[:,0]-id_pos[:,1]>50]
    id_neg = id_neg[id_neg[:, 0] > id_neg[:, 1]]
    id_pos = np.concatenate(
        [id_pos, (id_pos[:, 0]*0+1).reshape(-1, 1)], axis=1)
    id_neg = np.concatenate([id_neg, (id_neg[:, 0]*0).reshape(-1, 1)], axis=1)
    print(id_pos.shape)
    np.savez(seq+'.npz', pos=id_pos, neg=id_neg)

def run_sigmoid(seq='00'):
    pose_file = "/media/l/yp2/KITTI/odometry/dataset/poses/"+seq+".txt"
    poses = np.genfromtxt(pose_file)
    poses = poses[:, [3, 11]]
    inner = 2*np.matmul(poses, poses.T)
    xx = np.sum(poses**2, 1, keepdims=True)
    dis = xx-inner+xx.T
    dis = np.sqrt(np.abs(dis))
    score = 1.-1./(1+np.exp((10.-dis)/1.5))
    score[dis < 3] = 1
    # plt.imshow(score)
    # plt.show()
    id = np.argwhere(dis > -1)
    id = id[id[:, 0] >= id[:, 1]]
    label = score[(id[:, 0], id[:, 1])]
    label = label.reshape(-1, 1)
    out = np.concatenate((id, label), 1)
    out_pos = out[out[:, 2] > 0.1]
    out_neg = out[out[:, 2] <= 0.1]
    print(out_pos.shape)
    print(out_neg.shape)
    np.savez(seq+'.npz', pos=out_pos, neg=out_neg)

if __name__ == '__main__':
    seq = "00"
    if len(sys.argv) > 1:
        seq = sys.argv[1]
    run_sigmoid(seq)
    # run(seq)

max_stars_repo_name: lilin-hitcrt/RINet

max_issues_repo_name: lilin-hitcrt/RINet

max_forks_repo_name: lilin-hitcrt/RINet


text:
import argparse
import os
import re
import shutil

import h5py
import nibabel as nib
import numpy as np
import SimpleITK as sitk
import torch
from medpy import metric
from scipy.ndimage import zoom
from scipy.ndimage.interpolation import zoom
from tqdm import tqdm

# from networks.efficientunet import UNet
from networks.net_factory import net_factory

parser = argparse.ArgumentParser()
parser.add_argument('--root_path', type=str,
                    default='../data/ACDC', help='Name of Experiment')
parser.add_argument('--exp', type=str,
                    default='ACDC/Ours', help='experiment_name')
parser.add_argument('--model', type=str,
                    default='unet_cct', help='model_name')
parser.add_argument('--fold', type=str,
                    default='fold5', help='fold')
parser.add_argument('--num_classes', type=int,  default=4,
                    help='output channel of network')
parser.add_argument('--sup_type', type=str, default="scribble",
                    help='label')

def get_fold_ids(fold):
    all_cases_set = ["patient{:0>3}".format(i) for i in range(1, 101)]
    fold1_testing_set = [
        "patient{:0>3}".format(i) for i in range(1, 21)]
    fold1_training_set = [
        i for i in all_cases_set if i not in fold1_testing_set]

    fold2_testing_set = [
        "patient{:0>3}".format(i) for i in range(21, 41)]
    fold2_training_set = [
        i for i in all_cases_set if i not in fold2_testing_set]

    fold3_testing_set = [
        "patient{:0>3}".format(i) for i in range(41, 61)]
    fold3_training_set = [
        i for i in all_cases_set if i not in fold3_testing_set]

    fold4_testing_set = [
        "patient{:0>3}".format(i) for i in range(61, 81)]
    fold4_training_set = [
        i for i in all_cases_set if i not in fold4_testing_set]

    fold5_testing_set = [
        "patient{:0>3}".format(i) for i in range(81, 101)]
    fold5_training_set = [
        i for i in all_cases_set if i not in fold5_testing_set]
    if fold == "fold1":
        return [fold1_training_set, fold1_testing_set]
    elif fold == "fold2":
        return [fold2_training_set, fold2_testing_set]
    elif fold == "fold3":
        return [fold3_training_set, fold3_testing_set]
    elif fold == "fold4":
        return [fold4_training_set, fold4_testing_set]
    elif fold == "fold5":
        return [fold5_training_set, fold5_testing_set]
    else:
        return "ERROR KEY"

def calculate_metric_percase(pred, gt, spacing):
    pred[pred > 0] = 1
    gt[gt > 0] = 1
    dice = metric.binary.dc(pred, gt)
    asd = metric.binary.asd(pred, gt, voxelspacing=spacing)
    hd95 = metric.binary.hd95(pred, gt, voxelspacing=spacing)
    return dice, hd95, asd

def test_single_volume(case, net, test_save_path, FLAGS):
    h5f = h5py.File(FLAGS.root_path +
                    "/ACDC_training_volumes/{}".format(case), 'r')
    image = h5f['image'][:]
    label = h5f['label'][:]
    prediction = np.zeros_like(label)
    for ind in range(image.shape[0]):
        slice = image[ind, :, :]
        x, y = slice.shape[0], slice.shape[1]
        slice = zoom(slice, (256 / x, 256 / y), order=0)
        input = torch.from_numpy(slice).unsqueeze(
            0).unsqueeze(0).float().cuda()
        net.eval()
        with torch.no_grad():
            out_aux1, out_aux2 = net(input)
            out_aux1_soft = torch.softmax(out_aux1, dim=1)
            out_aux2_soft = torch.softmax(out_aux2, dim=1)
            out = torch.argmax(out_aux1_soft, dim=1).squeeze(0)
            # out = torch.argmax((out_aux1_soft+out_aux2_soft)/2.0, dim=1).squeeze(0)
            out = out.cpu().detach().numpy()
            pred = zoom(out, (x / 256, y / 256), order=0)
            prediction[ind] = pred
    case = case.replace(".h5", "")
    org_img_path = "../data/ACDC_training/{}.nii.gz".format(case)
    org_img_itk = sitk.ReadImage(org_img_path)
    spacing = org_img_itk.GetSpacing()

    first_metric = calculate_metric_percase(
        prediction == 1, label == 1, (spacing[2], spacing[0], spacing[1]))
    second_metric = calculate_metric_percase(
        prediction == 2, label == 2, (spacing[2], spacing[0], spacing[1]))
    third_metric = calculate_metric_percase(
        prediction == 3, label == 3, (spacing[2], spacing[0], spacing[1]))

    img_itk = sitk.GetImageFromArray(image.astype(np.float32))
    img_itk.CopyInformation(org_img_itk)
    prd_itk = sitk.GetImageFromArray(prediction.astype(np.float32))
    prd_itk.CopyInformation(org_img_itk)
    lab_itk = sitk.GetImageFromArray(label.astype(np.float32))
    lab_itk.CopyInformation(org_img_itk)
    sitk.WriteImage(prd_itk, test_save_path + case + "_pred.nii.gz")
    sitk.WriteImage(img_itk, test_save_path + case + "_img.nii.gz")
    sitk.WriteImage(lab_itk, test_save_path + case + "_gt.nii.gz")
    return first_metric, second_metric, third_metric

def Inference(FLAGS):
    train_ids, test_ids = get_fold_ids(FLAGS.fold)
    all_volumes = os.listdir(
        FLAGS.root_path + "/ACDC_training_volumes")
    image_list = []
    for ids in test_ids:
        new_data_list = list(filter(lambda x: re.match(
            '{}.*'.format(ids), x) != None, all_volumes))
        image_list.extend(new_data_list)
    snapshot_path = "../model/{}_{}/{}".format(
        FLAGS.exp, FLAGS.fold, FLAGS.sup_type)
    test_save_path = "../model/{}_{}/{}/{}_predictions/".format(
        FLAGS.exp, FLAGS.fold, FLAGS.sup_type, FLAGS.model)
    if os.path.exists(test_save_path):
        shutil.rmtree(test_save_path)
    os.makedirs(test_save_path)
    net = net_factory(net_type=FLAGS.model, in_chns=1,
                      class_num=FLAGS.num_classes)
    save_mode_path = os.path.join(
        snapshot_path, 'iter_60000.pth')

    net.load_state_dict(torch.load(save_mode_path))
    print("init weight from {}".format(save_mode_path))
    net.eval()

    first_total = 0.0
    second_total = 0.0
    third_total = 0.0
    for case in tqdm(image_list):
        print(case)
        first_metric, second_metric, third_metric = test_single_volume(
            case, net, test_save_path, FLAGS)
        first_total += np.asarray(first_metric)
        second_total += np.asarray(second_metric)
        third_total += np.asarray(third_metric)
    avg_metric = [first_total / len(image_list), second_total /
                  len(image_list), third_total / len(image_list)]
    print(avg_metric)
    print((avg_metric[0] + avg_metric[1] + avg_metric[2]) / 3)
    return ((avg_metric[0] + avg_metric[1] + avg_metric[2]) / 3)[0]

if __name__ == '__main__':
    FLAGS = parser.parse_args()
    total = 0.0
    for i in [1, 2, 3, 4, 5]:
        # for i in [5]:
        FLAGS.fold = "fold{}".format(i)
        print("Inference fold{}".format(i))
        mean_dice = Inference(FLAGS)
        total += mean_dice
    print(total/5.0)

reason: import numpy,from scipy


text:
# -*- coding: utf-8 -*-
import numpy as np
from numpy.testing import assert_array_almost_equal
from nilearn.input_data import NiftiMasker
import nibabel
import pytest
from fmralign.tests.utils import random_niimg
from fmralign._utils import _make_parcellation, _hierarchical_k_means
import nilearn
from packaging import version

def test_hierarchical_k_means():
    X = [[10, -10, 30], [12, -8, 24]]
    truth_labels = np.tile([0, 1, 2], 5)
    X = np.tile(X, 5).T
    test_labels = _hierarchical_k_means(X, 3)
    truth_labels = np.tile([test_labels[0], test_labels[1], test_labels[2]], 5)
    assert_array_almost_equal(test_labels, truth_labels)

def test_make_parcellation():
    # make_parcellation is built on Nilearn which already has several test for its Parcellation class
    # here we test just the call of the API is right on a simple example
    img, mask_img = random_niimg((7, 6, 8, 5))
    indexes = np.arange(img.shape[-1])
    masker = NiftiMasker(mask_img=mask_img).fit()

    methods = ["kmeans", "ward", "hierarchical_kmeans"]

    # check rena only if nilearn version allow it
    if version.parse(nilearn.__version__) <= version.parse("0.5.2"):
        with pytest.raises(Exception):
            assert _make_parcellation(img, "rena", n_pieces, masker)
    else:
        methods.append("rena")

    for clustering_method in methods:
        # check n_pieces = 1 gives out ones of right shape
        assert (_make_parcellation(
            img, indexes, clustering_method, 1, masker) == masker.transform(mask_img)).all()

        # check n_pieces = 2 find right clustering
        labels = _make_parcellation(
            img, indexes, clustering_method, 2, masker)
        assert(len(np.unique(labels)) == 2)

        # check that not inputing n_pieces yields problems
        with pytest.raises(Exception):
            assert _make_parcellation(
                img, indexes, clustering_method, 0, masker)

    clustering = nibabel.Nifti1Image(
        np.hstack([np.ones((7, 3, 8)), 2 * np.ones((7, 3, 8))]), np.eye(4))

    # check 3D Niimg clusterings
    for n_pieces in [0, 1, 2]:
        labels = _make_parcellation(
            img, indexes, clustering, n_pieces, masker)
        assert(len(np.unique(labels)) == 2)

    # check warning if a parcel is too big
    with pytest.warns(UserWarning):
        clustering = nibabel.Nifti1Image(
            np.hstack([np.ones(2000), 4 * np.ones(800)]), np.eye(4))
        _make_parcellation(
            img, indexes, clustering_method, n_pieces, masker)

max_stars_repo_name: hugorichard/fmralign

max_issues_repo_name: hugorichard/fmralign

max_forks_repo_name: hugorichard/fmralign

reason: import numpy,from numpy


text:
import io
import sys
import os
import json
import base64
import numpy as np
import tensorflow as tf

from PIL import Image
from helpers import download_model

models_url = 'https://www.dropbox.com/s/emee1vxmoch4sbu/models.zip?raw=1'
checkpoint = 'mobilenet_v2_1.0_224'

class syndicai(object):
    def __init__(self):
        self._model = None
        download_model(models_url, os.getcwd())
        sys.path.append(os.path.abspath('./models/tensorflow/research/slim'))

    def predict(self, X, features_names=None):

        img = np.array(Image.open(io.BytesIO(base64.b64decode(X))).resize((224, 224))).astype(np.float) / 128 - 1

        gd = tf.GraphDef.FromString(open( "models/mobilenet/"+ checkpoint +'_frozen.pb', 'rb').read())
        inp, predictions = tf.import_graph_def(gd,  return_elements = ['input:0', 'MobilenetV2/Predictions/Reshape_1:0'])

        with tf.Session(graph=inp.graph):
            y = predictions.eval(feed_dict={inp: img.reshape(1, 224,224, 3)})

        with open('models/imagenet-simple-labels.json') as f:
            labels = json.load(f)

        return [labels[y.argmax()]]

    def metrics(self):
        return [{"type": "COUNTER", "key": "mycounter", "value": 1}]