text:
import numpy as np
import pandas as pd
from os import system, listdir
from time import time
from sys import exit
from math import ceil
from keras.models import Model, load_model
from keras.utils import np_utils
from keras.applications.resnet50 import ResNet50
from keras.preprocessing.image import load_img, img_to_array, ImageDataGenerator
from keras.layers import GlobalAveragePooling2D, Dense, Dropout, Activation, Flatten, Input
from keras.applications.imagenet_utils import preprocess_input
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split

# def acc_top5(y_true, y_pred):
# 	return metrics.top_k_categorical_accuracy(y_true, y_pred, k=5)

def get_list(name):
	string_to_get = './../data/' + name + '.txt'

	file_to_get = open(string_to_get)
	ls = [val.strip() for val in file_to_get]

	return ls

def get_data(name, ls, flag=True):
	if name == 'movements':
		relevant_col = ['style', 'new_filename']
		match_string = 'style'
	else:
		relevant_col = ['artist', 'new_filename']
		match_string = 'artist'

	base_path_data = './../'
	base_path_file = './../data/'
	raw_data = pd.read_csv(base_path_file + 'all_data_info.csv', dtype=object)
	data = pd.DataFrame(raw_data)
	images = listdir(base_path_data + 'train/')
	# print(images)

	new_data = data[relevant_col]

	del raw_data
	del data

	new_data = new_data.loc[new_data['new_filename'].isin(images), ]
	x_train = []
	y_train = []
	ref_dict = {}
	counter = 0

	if not flag:
		string_to_get = './../data/dict_' + name + '.txt'
		file_to_get = open(string_to_get)
		ref_dict = eval(file_to_get.read())

	for i in images:
		print(i)
		if not ((new_data.loc[new_data['new_filename'] == i][match_string]).empty):
			tmp_string = new_data.loc[new_data['new_filename'] == i][match_string].values[0]
			print(tmp_string)
			if tmp_string in ls:
				tmp_img = load_img(base_path_data + 'train/' + i, target_size=(224, 224))
				tmp_img = img_to_array(tmp_img)
				tmp_img = np.expand_dims(tmp_img, axis=0)
				tmp_img = preprocess_input(tmp_img)
				x_train.append(tmp_img)

				if tmp_string not in ref_dict:
					ref_dict[tmp_string] = counter
					counter = counter + 1

				y_train.append(ref_dict.get(tmp_string))

	print(ref_dict)

	#string_to_write = './../data/dict_' + name + '.txt'
	#file_to_write = open(string_to_write, 'w')
	#file_to_write.write(str(ref_dict))
	#file_to_write.close()

	del new_data

	x_train_data = np.array(x_train)
	x_train_data = np.rollaxis(x_train_data, 1, 0)
	x_train_data = x_train_data[0]

	num_classes = counter
	y_train_data = np_utils.to_categorical(y_train, num_classes)

	del x_train
	del y_train

	# x, y = shuffle(x_train_data, y_train_data, random_state=random_seed)

	# del x_train_data
	# del y_train_data

	x_t, x_v, y_t, y_v = train_test_split(x_train_data, y_train_data, test_size= 0.1)

	del x_train_data
	del y_train_data

	# del x
	# del y

	return (x_t, x_v, y_t, y_v, num_classes)

def train_model(x_t, x_v, y_t, y_v, num_classes):
	img_input = Input(shape=(224, 224, 3))

	model = ResNet50(input_tensor=img_input, include_top=True)
	last_layer = model.get_layer('avg_pool').output
	x = Flatten(name='flatten')(last_layer)
	out = Dense(num_classes, activation='softmax', name='output_layer')(x)
	resnet_model = Model(inputs=img_input, outputs=out)

	for layer in resnet_model.layers[:-1]:
		layer.trainable = False

	resnet_model.layers[-1].trainable

	# resnet_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy', metrics.top_k_categorical_accuracy, acc_top5])
	resnet_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

	datagen = ImageDataGenerator(rotation_range=10, width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True)

	datagen.fit(x_t)

	t = time()
	resnet_model.fit_generator(datagen.flow(x_t, y_t, batch_size=1), steps_per_epoch=100, epochs=100)
	print('training time %s' % (t- time()))
	(loss, acc) = resnet_model.evaluate(x_v, y_v, batch_size=10, verbose=1)

	print('loss={:.4f}, accuracy: {:.4f}%'.format(loss,acc * 100))

	text_file = open('./../data/results_mov.txt', 'w')
	text_file.write('acc %.4f' % acc)
	text_file.close()

	resnet_model.save('./../data/resnet_model_mov.h5')

	print('saved')

def load_and_train(x_t, x_v, y_t, y_v):
	resnet_model = load_model('./../data/resnet_model_artists_tmp.h5')

	datagen = ImageDataGenerator(rotation_range=10, width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True)

	datagen.fit(x_t)

	t = time()
	resnet_model.fit_generator(datagen.flow(x_t, y_t, batch_size=32), steps_per_epoch=100, epochs=100)
	print('training time %s' % (t- time()))
	(loss, acc) = resnet_model.evaluate(x_v, y_v, batch_size=10, verbose=1)

	print('loss={:.4f}, accuracy: {:.4f}%'.format(loss,acc * 100))

	text_file = open('./../data/results_artists_tmp_two.txt', 'w')
	text_file.write('acc %.4f' % acc)
	text_file.close()

	resnet_model.save('./../data/resnet_model_artists_tmp_two.h5')

	print('saved')

def main():
	system('clear')

	movements_ls = get_list('movements')
	artists_ls = get_list('artists')
	x = int(input('Enter 1 to train new movements, 2 to train existing movements, 3 train new artists, 4 to train existing artists: '))

	if x == 1:
		x_t, x_v, y_t, y_v, num_classes = get_data('movements', movements_ls)
		train_model(x_t, x_v, y_t, y_v, num_classes)
	elif x == 2:
		x_t, x_v, y_t, y_v, num_classes = get_data('movements', movements_ls, False)
		load_and_train(x_t, x_v, y_t, y_v)
	elif x == 3:
		x_t, x_v, y_t, y_v, num_classes = get_data('artists', artists_ls)
		train_model(x_t, x_v, y_t, y_v, num_classes)
	elif x == 4:
		x_t, x_v, y_t, y_v, num_classes = get_data('artists', artists_ls, False)
		load_and_train(x_t, x_v, y_t, y_v)
	else:
		main()

if __name__ == '__main__':
	main()

max_stars_repo_name: satvikshukla/curate

max_issues_repo_name: satvikshukla/curate

max_forks_repo_name: satvikshukla/curate


text:
import pymc3 as pm

def estimate_mean_difference_one_sample(obs_measures, delta_cauchy_kwargs,
                                        sigma_cauchy_kwargs, side=None):
    """PyMC3 implementation of one sample mean comparison

    Args:
        - obs_measures: numpy array, observed measurements.
        - delta_cauchy_kwargs: dict, parameters of a Cauchy distribution, it
            is half Cauchy if one_side = True.
        - sigma_cauchy_kwargs: dict, parameters of an half Cauchy distribution.
        - one_side: bool, whther the test is one or two sided.

    Returns:
        - model
    """
    with pm.Model() as model:

        if side is not None:
            side_multiplier = -1 if side == 'left' else 1
            delta = pm.Deterministic(
                'effect_size',
                side_multiplier * pm.HalfCauchy(
                    'effect_size_cauchy',
                    **delta_cauchy_kwargs
                )
            )
            if side == 'left':
                delta = delta * - 1
        else:
            delta = pm.Cauchy(
                'effect_size',
                **delta_cauchy_kwargs
            )

        sigma = pm.HalfCauchy(
            'standard_deviation',
            **sigma_cauchy_kwargs
        )

        mu = pm.Deterministic(
            'mu',
            delta*sigma
        )

        observed_measures = pm.Normal(
            'observed_measures',
            mu=mu,
            sigma=sigma,
            observed=obs_measures
        )
    return model

def estimate_mean_difference_two_samples(obs_measure, delta_cauchy_kwargs,
                                         sigma_cauchy_kwargs,
                                         mu_normal_kwargs):
    """
    """
    group_1 = obs_measures[:, 0]
    group_2 = obs_measures[:, 1]

    with pm.Model() as model:

        delta = pm.Cauchy(
            'effect_size',
            **delta_cauchy_kwargs
        )
        sigma = pm.HalfCauchy(
            'standard_deviation',
            **sigma_cauchy_kwargs
        )

        alpha = pm.Deterministic(
            'mean_difference',
            delta * sigma
        )

        mean = pm.Normal(
            'mu',
            **mu_normal_kwargs
        )

        group_1_obs = pm.Normal(
            'group_1',
            mu=mean+alpha,
            sigma=sigma,
            observed=group_1
        )
        group_2_obs = pm.Normal(
            'group_1',
            mu=mean-alpha,
            sigma=sigma,
            observed=group_2
        )

    return model

ext: py

reason: import pymc3


text:
import numpy as np

np.random.seed(1000) # シードの固定

print(np.random.randint(100, 1000)) # 100 以上 1000 未満の整数一様乱数

'''
535
'''

print(np.random.randint(100, 1000, 10)) # 100 以上 1000 未満の整数一様乱数を 10 個

'''
[699 171 804 351 450 448 869 545 740 957]
'''

max_stars_repo_name: skillup-ai/tettei-engineer

max_issues_repo_name: skillup-ai/tettei-engineer

max_forks_repo_name: skillup-ai/tettei-engineer

reason: import numpy


text:
import pandas as pd
import numpy as np

TARGETS = [
    'negative_exam_for_pe',
    'qa_motion',
    'qa_contrast',
    'flow_artifact',
    'rv_lv_ratio_gte_1',
    'rv_lv_ratio_lt_1',
    'leftsided_pe',
    'chronic_pe',
    'true_filling_defect_not_pe',
    'rightsided_pe',
    'acute_and_chronic_pe',
    'central_pe',
    'indeterminate',
]

df = pd.read_csv('../../data/train/train_kfold.csv')

# 1- Percent positive for each target
for t in TARGETS:
    tmp_df = df[['SeriesInstanceUID', t]].drop_duplicates()
    print(f'{t} : {tmp_df[t].mean()*100:.2f}%')

# 2- Distribution of number of slices/series
for p in [5,10,25,50,75,90,95]:
    print(f'{p:02d}th percentile : {int(np.percentile(df.num_slices, p))}')

# 3- For positive studies, what is the distribution of proportion
#    of positive slices?
pos_series = df[['SeriesInstanceUID', 'negative_exam_for_pe']].drop_duplicates()
pos_series = pos_series[pos_series.negative_exam_for_pe == 0]
pos_series = pos_series.SeriesInstanceUID.unique()
pos_df = df[df.SeriesInstanceUID.isin(pos_series)]
proportion_pos_slices = [_df.pe_present_on_image.mean() for series_id, _df in pos_df.groupby('SeriesInstanceUID')]
for p in [5,10,25,50,75,90,95]:
    print(f'{p:02d}th percentile : {np.percentile(proportion_pos_slices, p)*100:0.2f}%')

# 4- # of bilateral PEs
np.unique((df.rightsided_pe + df.leftsided_pe + df.central_pe == 1).astype('int'), return_counts=True)
df['rightcentral_pe'] = (df.rightsided_pe + df.central_pe == 2).astype('int')
df['leftcentral_pe'] = (df.leftsided_pe + df.central_pe == 2).astype('int')
df['bilateral_pe'] = (df.rightsided_pe + df.leftsided_pe - df.central_pe == 2).astype('int')
df['diffuse_pe'] = (df.rightsided_pe + df.leftsided_pe + df.central_pe == 3).astype('int')

df.rightcentral_pe.value_counts(normalize=True)*100
df.leftcentral_pe.value_counts(normalize=True)*100
df.bilateral_pe.value_counts(normalize=True)*100
df.diffuse_pe.value_counts(normalize=True)*100

# 5- Explore RV/LV ratio more
series_df = df[['SeriesInstanceUID'] + TARGETS].drop_duplicates()
pos_series = series_df[series_df.negative_exam_for_pe == 0]
neg_series = series_df[series_df.negative_exam_for_pe == 1]
# For negative series, RV/LV ratio is ALWAYS 0 for both values ...
# For positive series:
#   RV/LV ratio >= 1 -- 39.7%
#   RV/LV ratio <  1 -- 53.7%
# So there are still some positive series where RV/LV ratio is 0 for both ...

max_stars_repo_name: i-pan/kaggle-rsna-pe

max_issues_repo_name: i-pan/kaggle-rsna-pe

max_forks_repo_name: i-pan/kaggle-rsna-pe


text:
from peakfinding.curvature_detection import RealSpectrumPeakFinderFromNoiseFloor, threshold_curve_function_generator
import pandas as pd
from numpy import array as ary
import sys
import matplotlib.pyplot as plt
import numpy as np

assert not sys.argv[-1].endswith(".Spe"), "Must provide a file name (that doesn't include '.csv') as the output location to save the .csv to."
spectrum = RealSpectrumPeakFinderFromNoiseFloor.from_multiple_files(*sys.argv[1:-1])
spectrum.show_sqrt_scale()
print("Type in the calibration coefficients (comma separated) ; or press enter to fit the peaks manually.")
values = input()
spectrum.fwhm_cal = ary([0.3524835463387513, 0.002308561222141246])
# 1.5853449048830446, 0.0020776090022188573
# if values.strip()=="":
#     spectrum.fit_fwhm_cal_interactively()
# else:
#     spectrum.fwhm_cal = ary([float(i) for i in values.split(",")])

DEMONSTRATE_DECISION_MAKING = True
if DEMONSTRATE_DECISION_MAKING:
    pass
raise NotImplementedError("Need to copy from /home/ocean/Documents/PhD/ExperimentalVerification/data/get_peaks.py")
while True:
    answer = input("Acceptable? (y/n) (y=save to file {})\n".format(sys.argv[-1])).lower()
    if answer=="y":
        df.to_csv(sys.argv[-1], index_label="bin")
        break
    elif answer=="n":
        pass
        break

max_stars_repo_name: OceanNuclear/PeakFinding

max_issues_repo_name: OceanNuclear/PeakFinding

max_forks_repo_name: OceanNuclear/PeakFinding

reason: import numpy,from numpy


text:
import xarray as xr
import numpy as np
import scipy.ndimage
from skimage.measure import regionprops
from skimage.measure import label as label_np
import dask.array as dsa

def _morphological_operations(da, radius=8):
    '''Converts xarray.DataArray to binary, defines structuring element, and performs morphological closing then opening.
    Parameters
    ----------
    da     : xarray.DataArray
            The data to label
    radius : int
            Length of grid spacing to define the radius of the structing element used in morphological closing and opening.

    '''

    # Convert images to binary. All positive values == 1, otherwise == 0
    bitmap_binary = da.where(da>0, drop=False, other=0)
    bitmap_binary = bitmap_binary.where(bitmap_binary==0, drop=False, other=1)

    # Define structuring element
    diameter = radius*2
    x = np.arange(-radius, radius+1)
    x, y = np.meshgrid(x, x)
    r = x**2+y**2
    se = r<radius**2

    def binary_open_close(bitmap_binary):
        bitmap_binary_padded = np.pad(bitmap_binary,
                                      ((diameter, diameter), (diameter, diameter)),
                                      mode='wrap')
        s1 = scipy.ndimage.binary_closing(bitmap_binary_padded, se, iterations=1)
        s2 = scipy.ndimage.binary_opening(s1, se, iterations=1)
        unpadded= s2[diameter:-diameter, diameter:-diameter]
        return unpadded

    mo_binary = xr.apply_ufunc(binary_open_close, bitmap_binary,
                               input_core_dims=[['lat', 'lon']],
                               output_core_dims=[['lat', 'lon']],
                               output_dtypes=[bitmap_binary.dtype],
                               vectorize=True,
                               dask='parallelized')

    return mo_binary

def _apply_mask(mask, binary_images):
    binary_images_with_mask = binary_images.where(mask==1, drop=False, other=0)
    return binary_images_with_mask

def _filter_area(binary_images, min_size_quartile):
    '''calculatre area with regionprops'''

    def get_labels(binary_images):
        blobs_labels = _label_either(binary_images, background=0)
        return blobs_labels

    labels = xr.apply_ufunc(get_labels, binary_images,
                            input_core_dims=[['lat', 'lon']],
                            output_core_dims=[['lat', 'lon']],
                            output_dtypes=[binary_images.dtype],
                            vectorize=True,
                            dask='parallelized')

    labels = xr.DataArray(labels, dims=binary_images.dims, coords=binary_images.coords)
    labels = labels.where(labels>0, drop=False, other=np.nan)

    # The labels are repeated each time step, therefore we relabel them to be consecutive
    for i in range(1, labels.shape[0]):
        labels[i,:,:] = labels[i,:,:].values + labels[i-1,:,:].max().values

    labels = labels.where(labels>0, drop=False, other=0)
    labels_wrapped, N_initial = _wrap(np.array(labels))

    # Calculate Area of each object and keep objects larger than threshold
    props = regionprops(labels_wrapped.astype('int'))
    labelprops = [p.label for p in props]
    labelprops = xr.DataArray(labelprops, dims=['label'], coords={'label': labelprops})
    area = xr.DataArray([p.area for p in props], dims=['label'], coords={'label': labelprops})  # Number of pixels of the region.
    min_area = np.percentile(area, min_size_quartile*100)
    print('minimum area: ', min_area)
    keep_labels = labelprops.where(area>=min_area, drop=True)
    keep_where = np.isin(labels_wrapped, keep_labels)
    out_labels = xr.DataArray(np.where(keep_where==False, 0, labels_wrapped), dims=binary_images.dims, coords=binary_images.coords)

    # Convert images to binary. All positive values == 1, otherwise == 0
    binary_labels = out_labels.where(out_labels==0, drop=False, other=1)

    return area, min_area, binary_labels, N_initial

def _label_either(data, **kwargs):
    if isinstance(data, dsa.Array):
        try:
            from dask_image.ndmeasure import label as label_dask
            def label_func(a, **kwargs):
                ids, num = label_dask(a, **kwargs)
                return ids
        except ImportError:
            raise ImportError(
                "Dask_image is required to use this function on Dask arrays. "
                "Either install dask_image or else call .load() on your data."
            )
    else:
        label_func = label_np
    return label_func(data, **kwargs)

def _wrap(labels):
    ''' Impose periodic boundary and wrap labels'''
    first_column = labels[..., 0]
    last_column = labels[..., -1]

    unique_first = np.unique(first_column[first_column>0])

    # This loop iterates over the unique values in the first column, finds the location of those values in
    # the first columnm and then uses that index to replace the values in the last column with the first column value
    for i in enumerate(unique_first):
        first = np.where(first_column == i[1])
        last = last_column[first[0], first[1]]
        bad_labels = np.unique(last[last>0])
        replace = np.isin(labels, bad_labels)
        labels[replace] = i[1]

    labels_wrapped = np.unique(labels, return_inverse=True)[1].reshape(labels.shape)

    # recalculate the total number of labels
    N = np.max(labels_wrapped)

    return labels_wrapped, N

def track(da, mask, radius=8, min_size_quartile=0.75):
    '''Image labeling and tracking.

    Parameters
    ----------
    da : xarray.DataArray
        The data to label.

    radius : int
        size of the structuring element used in morphological opening and closing. Radius specified by the number of grid units.

    area_quantile : float
        quantile used to define the threshold of the smallest area object retained in tracking. Value should be between 0 and 1.

    mask : xarray.DataArray
        The mask of ponts to ignore. Must be binary where 1 = true point and 0 = background to be ignored.

    Returns
    -------
    labels : xarray.DataArray
        Integer labels of the connected regions.
    '''

    # Convert data to binary, define structuring element, and perform morphological closing then opening
    binary_images = _morphological_operations(da, radius=radius)

    # Apply mask
    binary_images_with_mask  = _apply_mask(mask, binary_images)

    # Filter area
    area, min_area, binary_labels, N_initial = _filter_area(binary_images_with_mask, min_size_quartile)

    # Label objects
    labels, num = _label_either(binary_labels, return_num= True, connectivity=3)

    # Wrap labels
    labels_wrapped, N_final = _wrap(labels)

    # Final labels to DataArray
    new_labels = xr.DataArray(labels_wrapped, dims=da.dims, coords=da.coords)
    new_labels = new_labels.where(new_labels!=0, drop=False, other=np.nan)

    ## Metadata

    # Calculate Percent of total object area retained after size filtering
    sum_tot_area = int(np.sum(area.values))

    reject_area = area.where(area<=min_area, drop=True)
    sum_reject_area = int(np.sum(reject_area.values))
    percent_area_reject = (sum_reject_area/sum_tot_area)

    accept_area = area.where(area>min_area, drop=True)
    sum_accept_area = int(np.sum(accept_area.values))
    percent_area_accept = (sum_accept_area/sum_tot_area)

    new_labels = new_labels.rename('labels')
    new_labels.attrs['inital objects identified'] = int(N_initial)
    new_labels.attrs['final objects tracked'] = int(N_final)
    new_labels.attrs['radius'] = radius
    new_labels.attrs['size quantile threshold'] = min_size_quartile
    new_labels.attrs['min area'] = min_area
    new_labels.attrs['percent area reject'] = percent_area_reject
    new_labels.attrs['percent area accept'] = percent_area_accept

    print('inital objects identified \t', int(N_initial))
    print('final objects tracked \t', int(N_final))

    return new_labels

max_stars_repo_name: jbusecke/ocetrac

max_issues_repo_name: jbusecke/ocetrac

max_forks_repo_name: jbusecke/ocetrac

reason: import numpy,import scipy


text:
import numpy as np
import eigen_matrix as tes

x = np.arange(32).reshape(8, 4).astype(np.float64)
print(x)
tes.block_test(x,10)
x = x.reshape(2,4,4)
print(x)


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Dec  9 08:21:38 2021
Carlos Armando De Castro Payares - cadecastro.com
"""
def analisis_arima(x):
    import numpy as np, pandas as pd
    from statsmodels.graphics.tsaplots import plot_acf, plot_pacf
    import matplotlib.pyplot as plt
    from pandas.plotting import register_matplotlib_converters
    register_matplotlib_converters()
    import statsmodels
    import pmdarima as pm
    plot_acf(x)
    plot_pacf(x)
    #Modelo Autoarima:
    modelo1 = pm.auto_arima(x, start_p=1, start_q=1,
                      test='adf',       # use adftest to find optimal 'd'
                      max_p=3, max_q=3, # maximum p and q
                      m=1,              # frequency of series
                      d=None,           # let model determine 'd'
                      seasonal=False,   # No Seasonality
                      start_P=0,
                      D=0,
                      trace=True,
                      error_action='ignore',
                      suppress_warnings=True,
                      stepwise=True)

    print(modelo1.summary())
    modelo1.plot_diagnostics()
    #Predicción:
    n_periods=int(input('Períodos a predecir = '))
    fc, confint = modelo1.predict(n_periods=n_periods, return_conf_int=True)
    index_of_fc = np.arange(len(x), len(x)+n_periods)

    # make series for plotting purpose
    fc_series = pd.Series(fc, index=index_of_fc)
    lower_series = pd.Series(confint[:, 0], index=index_of_fc)
    upper_series = pd.Series(confint[:, 1], index=index_of_fc)

    # Plot
    plt.figure(4)
    plt.plot(np.arange(len(x)),x,'b')
    plt.plot(fc_series, color='red')
    plt.fill_between(lower_series.index,
                 lower_series,
                 upper_series,
                 color='yellow', alpha=.75)

    plt.title("Predicción Auto ARIMA")
    plt.xlabel("Períodos - cadecastro.com")
    plt.grid(True,'both','both')
    #Modelo manual ARIMA:
    print('MODELO MANUAL ARIMA:')
    p=int(input('p = '))
    d=int(input('d = '))
    q=int(input('q = '))
    modelo2=statsmodels.tsa.arima.model.ARIMA(x, order=(p,d,q))
    resultados2= modelo2.fit()
    print(resultados2.summary())
    # Gráfica de errores residuales:
    residuales = pd.DataFrame(resultados2.resid)
    fig, ax = plt.subplots(1,2)
    residuales.plot(title="Residuales", ax=ax[0])
    residuales.plot(kind='kde', title='Densidad de probabilidad', ax=ax[1])
    #Ajuste y predicción:
    n=n_periods
    predic2=resultados2.predict(0,len(x)+n)
    indice1=range(0,len(x))
    indice2=range(0,len(x)+n+1)
    plt.figure(6)
    plt.plot(indice1,x,'b')
    plt.plot(indice2,predic2,'r')
    plt.title("Predicción ARIMA manual")
    plt.xlabel("Períodos - cadecastro.com")
    plt.grid(True,'both','both')

reason: import numpy,import statsmodels,from statsmodels


text:
#!/usr/bin/env python3

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import contextlib
import sys
import tempfile
import pathlib

import pytest
import numpy as np

import tvm
import tvm.testing
from tvm import te
from tvm.tir.stmt_functor import post_order_visit
from tvm.contrib.hexagon.build import HexagonLauncher

from .conftest import requires_hexagon_toolchain
from .infrastructure import allocate_hexagon_array

# Needed to register the link_shared packedfunc.
import tvm.contrib.hexagon

dtype = tvm.testing.parameter("int8")
batch_size = tvm.testing.parameter(
    16,
    2,
)
input_channels = tvm.testing.parameter(
    32,
)
input_image_shape = tvm.testing.parameter(
    by_dict={
        "8x8": (8, 8),
        "32x32": (32, 32),
    }
)

input_layout = tvm.testing.parameter(
    "nhwc",
    "nchw-8h8w32c-1d",
)
output_layout = tvm.testing.parameter(
    "nhwc",
    "nchw-8h8w32c-1d",
)
working_layout, working_scope = tvm.testing.parameters(
    ("nhwc", "global"),
    ("nhwc", "global.vtcm"),
    ("nchw-8h8w32c-1d", "global"),
    ("nchw-8h8w32c-1d", "global.vtcm"),
    # 2-d memory may only occur in vtcm memory
    ("nchw-8h8w32c-2d", "global.vtcm"),
)

@tvm.testing.fixture
def target_host(target):
    target = tvm.target.Target(target)

    if target.kind.name == "hexagon":
        # Shouldn't have to modify the target here, current
        # workaround.  In the future, should move the parameter
        # handling from tvm.target to target_kind.cc.
        target = tvm.target.hexagon("v68", link_params=True)
        host = target
    else:
        host = None
    return tvm.target.Target(target, host=host)

@tvm.testing.fixture
def input_shape(batch_size, input_channels, input_image_shape):
    return [batch_size, *input_image_shape, input_channels]

def transform_shape(shape, layout):
    if layout == "nhwc":
        return shape
    elif layout in ["nchw-8h8w32c-1d", "nchw-8h8w32c-2d"]:
        N, H, W, C = shape
        return [N, (C + 31) // 32, (H + 7) // 8, (W + 7) // 8, 8, 8, 32]
    else:
        raise RuntimeError(f"Unexpected layout '{layout}'")

def transform_numpy(arr_np, layout):
    if layout == "nhwc":
        return arr_np
    elif layout in ["nchw-8h8w32c-1d", "nchw-8h8w32c-2d"]:
        N, H, W, C = arr_np.shape
        return arr_np.reshape([N, H // 8, 8, W // 8, 8, C // 32, 32]).transpose(0, 5, 1, 3, 2, 4, 6)
    else:
        raise RuntimeError(f"Unexpected layout '{layout}'")

@tvm.testing.fixture
def transformed_input_shape(input_shape, input_layout):
    return transform_shape(input_shape, input_layout)

@tvm.testing.fixture
def transformed_output_shape(output_shape, output_layout):
    return transform_shape(output_shape, output_layout)

@tvm.testing.fixture
def input_np(input_shape, dtype):
    return (100 * np.random.uniform(size=input_shape)).astype(dtype)

@tvm.testing.fixture
def transformed_input_np(input_np, input_layout):
    return transform_numpy(input_np, input_layout)

@tvm.testing.fixture
def transformed_expected_output_np(expected_output_np, output_layout):
    return transform_numpy(expected_output_np, output_layout)

def layout_transform_1d(n, h, w, c):
    return [
        n,
        c // 32,
        h // 8,
        w // 8,
        h % 8,
        w % 8,
        c % 32,
    ]

def layout_transform_2d(n, h, w, c):
    return [
        n,
        c // 32,
        h // 8,
        w // 8,
        te.AXIS_SEPARATOR,
        h % 8,
        w % 8,
        c % 32,
    ]

def extract_buffers(stmt):
    buffers = []

    def visitor(node):
        if isinstance(node, (tvm.tir.BufferLoad, tvm.tir.BufferStore, tvm.tir.BufferRealize)):
            buffers.append(node.buffer)

    post_order_visit(stmt, visitor)
    return buffers

class TestElementWise:
    @tvm.testing.fixture
    def expected_output_np(self, input_np):
        return 2 * input_np

    @tvm.testing.fixture
    def output_shape(self, input_shape):
        return input_shape

    @tvm.testing.fixture
    def schedule_args(
        self,
        input_shape,
        dtype,
        input_layout,
        output_layout,
        working_layout,
        working_scope,
    ):
        InputTensor = te.placeholder(input_shape, dtype, name="Input")
        OutputTensor = te.compute(
            shape=InputTensor.shape,
            fcompute=lambda *indices: (2 * InputTensor[indices]).astype(dtype),
            name="Output",
        )
        schedule = te.create_schedule(OutputTensor.op)

        WriteCache = schedule.cache_write(OutputTensor, working_scope)
        ReadCache = schedule.cache_read(InputTensor, working_scope, [WriteCache])

        def apply_transform(tensor, layout):
            if layout == "nhwc":
                pass
            elif layout == "nchw-8h8w32c-1d":
                return schedule[tensor].transform_layout(layout_transform_1d)
            elif layout == "nchw-8h8w32c-2d":
                return schedule[tensor].transform_layout(layout_transform_2d)
            else:
                raise RuntimeError(f"Unexpected layout '{layout}'")

        apply_transform(InputTensor, input_layout)
        compute_loopnest = apply_transform(OutputTensor, output_layout) or OutputTensor.op.axis
        schedule[WriteCache].compute_at(schedule[OutputTensor], compute_loopnest[0])

        apply_transform(ReadCache, working_layout)
        apply_transform(WriteCache, working_layout)

        return [schedule, [InputTensor, OutputTensor]]

    @tvm.testing.fixture
    def ir_module(self, schedule_args):
        # If the two buffers are accessed with the same indices, CSE
        # will replace them with a Let binding.  Since this makes it
        # harder to test what the transformed indices are, disabling
        # the CSE pass for this test.
        with tvm.transform.PassContext(disabled_pass=["tir.CommonSubexprElimTIR"]):
            return tvm.lower(*schedule_args)

    @tvm.testing.fixture
    def uses_unsupported_physical_dimensions(
        self, target_host, input_layout, working_layout, output_layout
    ):
        uses_2d_memory = "nchw-8h8w32c-2d" in [input_layout, working_layout, output_layout]
        can_handle_2d_memory = target_host.kind.name == "hexagon"

        return uses_2d_memory and not can_handle_2d_memory

    def test_param_shapes(self, ir_module, transformed_input_shape, transformed_output_shape):
        func = ir_module["main"]
        primfunc_input_shape, primfunc_output_shape = [
            list(func.preflattened_buffer_map[param].shape) for param in func.params
        ]
        assert primfunc_input_shape == transformed_input_shape
        assert primfunc_output_shape == transformed_output_shape

    def test_cache_shape(self, ir_module, input_layout, working_layout, output_layout):
        func = ir_module["main"]
        for buffer in extract_buffers(func.body):
            buffer_layout = {
                "Input": input_layout,
                "Input.global": working_layout,
                "Output.global": working_layout,
                "Input.global.vtcm": working_layout,
                "Output.global.vtcm": working_layout,
                "Output": output_layout,
            }[buffer.name]

            expected_physical_dimensions = {
                "nhwc": 1,
                "nchw-8h8w32c-1d": 1,
                "nchw-8h8w32c-2d": 2,
            }[buffer_layout]

            assert len(buffer.shape) == expected_physical_dimensions

    def test_lower(self, schedule_args):
        return tvm.lower(*schedule_args)

    @requires_hexagon_toolchain
    def test_build(self, schedule_args, target_host, input_layout, working_layout, output_layout):
        # contextlib.nullcontext wasn't added until python3.7, and the
        # CI currently runs on python3.6.  Therefore, using ExitStack
        # to manage an optional context instead.
        stack = contextlib.ExitStack()

        with stack:
            is_hexagon = target_host.kind.name == "hexagon"
            uses_2d_memory = "nchw-8h8w32c-2d" in [input_layout, working_layout, output_layout]
            if uses_2d_memory and not is_hexagon:
                stack.enter_context(pytest.raises(tvm.TVMError))

            tvm.build(*schedule_args, target=target_host)

    @tvm.testing.fixture
    def runtime_module(self, schedule_args, target_host):
        if target_host.kind.name != "hexagon":
            pytest.skip("Only running on hexagon")

        return tvm.build(*schedule_args, target=target_host)

    @requires_hexagon_toolchain
    def test_execute(
        self,
        runtime_module,
        transformed_input_np,
        transformed_expected_output_np,
        input_layout,
        output_layout,
        hexagon_session,
    ):
        if hexagon_session is None:
            pytest.skip(msg="Skip hardware test, ANDROID_SERIAL_NUMBER is not set.")

        if input_layout == "nchw-8h8w32c-2d":
            input_axis_separators = [4]
        else:
            input_axis_separators = []

        if output_layout == "nchw-8h8w32c-2d":
            output_axis_separators = [4]
        else:
            output_axis_separators = []

        input_arr = allocate_hexagon_array(
            hexagon_session.device,
            data=transformed_input_np,
            axis_separators=input_axis_separators,
        )
        output_arr = allocate_hexagon_array(
            hexagon_session.device,
            data=np.zeros_like(transformed_expected_output_np),
            axis_separators=output_axis_separators,
        )

        mod = hexagon_session.load_module(runtime_module)

        mod(input_arr, output_arr)
        output_np = output_arr.numpy()

        np.testing.assert_array_equal(output_np, transformed_expected_output_np)

if __name__ == "__main__":
    sys.exit(pytest.main(sys.argv))


text:
import numpy as np
import pandas as pd

if __name__ == '__main__':
    parm = 534  # PCA降维后得到的数据
    df = pd.read_csv('CSV/weibo_normal_data_clean.csv')
    df_key_words = pd.read_csv('CSV/tf_idf_weight.csv')
    key_words = df_key_words.loc[:parm - 1]['tag']
    bw_matrix_normal = np.zeros(shape=(len(df), parm))

    # print(key_words)
    for i in range(len(df)):
        # 遍历所有文章
        word_list = df.loc[i, 'content'].split()  # 取得文章内容
        print(word_list)

        for w in word_list:
            # 判断文章每个词是否在1000个关键字里
            if w not in key_words:
                continue
            else:
                # 若为关键字，则矩阵相应位置加一
                bw_matrix_normal[i][key_words.index(w)] += 1

    bw_matrix_df = pd.DataFrame(data=bw_matrix_normal, columns=key_words)
    bw_matrix_df.to_csv('CSV/bw_matrix_normal.csv', index=False)  # 该csv即基于TF-IDF加权的词袋模型矩阵


text:
import pandas as pd
import numpy as np
from functools import singledispatch
import itertools

from ..siu import Symbolic, symbolic_dispatch

def _coerce_to_str(x):
    if isinstance(x, (pd.Series, np.ndarray)):
        return x.astype(str)
    elif not np.ndim(x) < 2:
        raise ValueError("np.ndim must be less than 2, but is %s" %np.ndim(x))

    return pd.Series(x, dtype = str)

@symbolic_dispatch(cls = pd.Series)
def str_c(x, *args, sep = "", collapse = None):
    all_args = itertools.chain([x], args)
    strings = list(map(_coerce_to_str, all_args))

    return np.sum(strings, axis = 0)

max_stars_repo_name: sethvargo/siuba

max_issues_repo_name: sethvargo/siuba

max_forks_repo_name: sethvargo/siuba


text:
# -*- coding: utf-8 -*-
#
# Copyright (c) 2017, the cclib development team
#
# This file is part of cclib (http://cclib.github.io) and is distributed under
# the terms of the BSD 3-Clause License.

"""Parser for MOPAC output files"""

# Based on parser in RMG-Py by Greg Magoon
# https://github.com/ReactionMechanismGenerator/RMG-Py/blob/master/external/cclib/parser/mopacparser.py
# Also parts from Ben Albrecht
# https://github.com/ben-albrecht/cclib/blob/master/cclib/parser/mopacparser.py
# Merged and modernized by Geoff Hutchison

from __future__ import print_function

import re
import math

import numpy

from cclib.parser import data
from cclib.parser import logfileparser
from cclib.parser import utils

def symbol2int(symbol):
    t = utils.PeriodicTable()
    return t.number[symbol]

class MOPAC(logfileparser.Logfile):
    """A MOPAC20XX output file."""

    def __init__(self, *args, **kwargs):

        # Call the __init__ method of the superclass
        super(MOPAC, self).__init__(logname="MOPAC", *args, **kwargs)

    def __str__(self):
        """Return a string representation of the object."""
        return "MOPAC log file %s" % (self.filename)

    def __repr__(self):
        """Return a representation of the object."""
        return 'MOPAC("%s")' % (self.filename)

    def normalisesym(self, label):
        """MOPAC does not require normalizing symmetry labels."""

    def before_parsing(self):
        #TODO

        # Defaults
        charge = 0
        self.set_attribute('charge', charge)
        mult = 1
        self.set_attribute('mult', mult)

        # Keep track of whether or not we're performing an
        # (un)restricted calculation.
        self.unrestricted = False
        self.is_rohf = False

        # Keep track of 1SCF vs. gopt since gopt is default
        self.onescf = False
        self.geomdone = False

        # Compile the dashes-and-or-spaces-only regex.
        self.re_dashes_and_spaces = re.compile('^[\s-]+$')

        self.star = ' * '
        self.stars = ' *******************************************************************************'

        self.spinstate = {'SINGLET': 1,
                          'DOUBLET': 2,
                          'TRIPLET': 3,
                          'QUARTET': 4,
                          'QUINTET': 5,
                          'SEXTET': 6,
                          'HEPTET': 7,
                          'OCTET': 8,
                          'NONET': 9}

    def extract(self, inputfile, line):
        """Extract information from the file object inputfile."""

        # Extract the atomic numbers and coordinates from the optimized geometry
        # note that cartesian coordinates section occurs multiple times in the file, and we want to end up using the last instance
        # also, note that the section labeled cartesian coordinates doesn't have as many decimal places as the one used here
        # Example 1 (not used):
        #          CARTESIAN COORDINATES
        #
        #    NO.       ATOM               X         Y         Z
        #
        #     1         O                  4.7928   -0.8461    0.3641
        #     2         O                  5.8977   -0.3171    0.0092
        # ...
        # Example 2 (used):
        #   ATOM   CHEMICAL          X               Y               Z
        #  NUMBER    SYMBOL      (ANGSTROMS)     (ANGSTROMS)     (ANGSTROMS)
        #
        #     1       O          4.79280259  *  -0.84610232  *   0.36409474  *
        #     2       O          5.89768035  *  -0.31706418  *   0.00917035  *
        # ... etc.
        if line.split() == ["NUMBER", "SYMBOL", "(ANGSTROMS)", "(ANGSTROMS)", "(ANGSTROMS)"]:

            self.updateprogress(inputfile, "Attributes", self.cupdate)

            self.inputcoords = []
            self.inputatoms = []

            blankline = inputfile.next()

            atomcoords = []
            line = inputfile.next()
            while len(line.split()) > 6:
                # MOPAC Version 14.019L 64BITS suddenly appends this block with
                # "CARTESIAN COORDINATES" block with no blank line.
                tokens = line.split()
                self.inputatoms.append(symbol2int(tokens[1]))
                xc = float(tokens[2])
                yc = float(tokens[4])
                zc = float(tokens[6])
                atomcoords.append([xc, yc, zc])
                line = inputfile.next()

            self.inputcoords.append(atomcoords)

            if not hasattr(self, "natom"):
                self.atomnos = numpy.array(self.inputatoms, 'i')
                self.natom = len(self.atomnos)

        if 'CHARGE ON SYSTEM =' in line:
            charge = int(line.split()[5])
            self.set_attribute('charge', charge)

        if 'SPIN STATE DEFINED' in line:
            # find the multiplicity from the line token (SINGLET, DOUBLET, TRIPLET, etc)
            mult = self.spinstate[line.split()[1]]
            self.set_attribute('mult', mult)

        # Read energy (in kcal/mol, converted to eV)
        #       Example:           FINAL HEAT OF FORMATION =       -333.88606 KCAL =   -1396.97927 KJ
        if line[0:35] == '          FINAL HEAT OF FORMATION =':
            if not hasattr(self, "scfenergies"):
                self.scfenergies = []
            self.scfenergies.append(utils.convertor(self.float(line.split()[5]), "kcal", "eV"))

        #molecular mass parsing (units will be amu)
        #Example:          MOLECULAR WEIGHT        =
        if line[0:35] == '          MOLECULAR WEIGHT        =':
            self.molmass = self.float(line.split()[3])

        #rotational constants
        #Example:
        #          ROTATIONAL CONSTANTS IN CM(-1)
        #
        #          A =    0.01757641   B =    0.00739763   C =    0.00712013
        # could also read in moment of inertia, but this should just differ by a constant: rot cons= h/(8*Pi^2*I)
        # note that the last occurence of this in the thermochemistry section has reduced precision,
        # so we will want to use the 2nd to last instance
        if line[0:40] == '          ROTATIONAL CONSTANTS IN CM(-1)':
            blankline = inputfile.next()
            rotinfo = inputfile.next()
            if not hasattr(self, "rotcons"):
                self.rotcons = []
            broken = rotinfo.split()
            # leave the rotational constants in Hz
            a = float(broken[2])
            b = float(broken[5])
            c = float(broken[8])
            self.rotcons.append([a, b, c])

        # Start of the IR/Raman frequency section.
        # Example:
        # VIBRATION    1    1A       ATOM PAIR        ENERGY CONTRIBUTION    RADIAL
        # FREQ.        15.08        C 12 --  C 16           +7.9% (999.0%)     0.0%
        # T-DIPOLE    0.2028        C 16 --  H 34           +5.8% (999.0%)    28.0%
        # TRAVEL      0.0240        C 16 --  H 32           +5.6% (999.0%)    35.0%
        # RED. MASS   1.7712        O  1 --  O  4           +5.2% (999.0%)     0.4%
        # EFF. MASS7752.8338
        #
        # VIBRATION    2    2A       ATOM PAIR        ENERGY CONTRIBUTION    RADIAL
        # FREQ.        42.22        C 11 --  C 15           +9.0% (985.8%)     0.0%
        # T-DIPOLE    0.1675        C 15 --  H 31           +6.6% (843.6%)     3.3%
        # TRAVEL      0.0359        C 15 --  H 29           +6.0% (802.8%)    24.5%
        # RED. MASS   1.7417        C 13 --  C 17           +5.8% (792.7%)     0.0%
        # EFF. MASS1242.2114
        if line[1:10] == 'VIBRATION':
            self.updateprogress(inputfile, "Frequency Information", self.fupdate)

            # get the vib symmetry
            if len(line.split()) >= 3:
                sym = line.split()[2]
                if not hasattr(self, 'vibsyms'):
                    self.vibsyms = []
                self.vibsyms.append(sym)

            line = inputfile.next()
            if 'FREQ' in line:
                if not hasattr(self, 'vibfreqs'):
                    self.vibfreqs = []
                freq = float(line.split()[1])
                self.vibfreqs.append(freq)

            line = inputfile.next()
            if 'T-DIPOLE' in line:
                if not hasattr(self, 'vibirs'):
                    self.vibirs = []
                tdipole = float(line.split()[1])
                # transform to km/mol
                self.vibirs.append(math.sqrt(tdipole))

        # Orbital eigenvalues, e.g.
        #           ALPHA EIGENVALUES
        #            BETA EIGENVALUES
        # or just "EIGENVALUES" for closed-shell
        if 'EIGENVALUES' in line:
            if not hasattr(self, 'moenergies'):
                self.moenergies = [] # list of arrays

            energies = []
            line = inputfile.next()
            while len(line.split()) > 0:
                energies.extend([float(i) for i in line.split()])
                line = inputfile.next()
            self.moenergies.append(energies)

        # todo:
        # Partial charges and dipole moments
        # Example:
        # NET ATOMIC CHARGES

if __name__ == "__main__":
    import doctest
    import sys
    from cclib.parser import mopacparser

    if len(sys.argv) == 1:
        doctest.testmod(mopacparser, verbose=False)

    if len(sys.argv) >= 2:
        parser = mopacparser.MOPAC(sys.argv[1])
        data = parser.parse()

    if len(sys.argv) > 2:
        for i in range(len(sys.argv[2:])):
            if hasattr(data, sys.argv[2 + i]):
                print(getattr(data, sys.argv[2 + i]))

max_stars_repo_name: adabbott/cclib

max_issues_repo_name: adabbott/cclib

max_forks_repo_name: adabbott/cclib


text:
"""Base classes for OpenCV-based computer vision."""

import numpy as np

try:
  import cv2
except ImportError:
  print "You need OpenCV to use vision modules, sorry."
  sys.exit(1)

from util import log

class FrameProcessor:
  """Processes a sequence of images (frames)."""

  def __init__(self, options):
    self.gui = options.get('gui', False)
    self.debug = options.get('debug', False)
    self.active = False  # set to True once initialized
    # NOTE Subclasses should call FrameProcessor.__init__(self, options) and process options

  def initialize(self, imageIn, timeNow):
    self.image = imageIn
    self.imageSize = (self.image.shape[1], self.image.shape[0])
    self.imageOut = self.image.copy() if self.gui else None
    self.logd("initialize", "Image size = " + str(self.imageSize))
    self.active = True

  def process(self, imageIn, timeNow):
    self.image = imageIn
    if self.gui: self.imageOut = self.image
    return True, self.imageOut

  def onKeyPress(self, key, keyChar=None):
    return True  # indicates stop vision loop if False
    # TODO Change return behavior to indicate if event is consumed, and not to signal stop (or better, return both!)

  def loge(self, func, msg):
    log(self, func, msg)

  def logi(self, func, msg):
    log(self, func, msg)

  def logd(self, func, msg):
    if self.debug:
      log(self, func, msg)

class FrameProcessorPool:
  """Abstract base class for all collections of FrameProcessors."""

  def getProcessorByType(self, processorType):
    raise NotImplementedError("FrameProcessorPool.getProcessorByType() is abstract; must be implmented by subclasses.")

class DependentFrameProcessor(FrameProcessor):
  """A FrameProcessor that depends on the output of one or more other processors."""

  def __init__(self, options, processorPool):
    FrameProcessor.__init__(self, options)
    # NOTE Sublasses should call through to DependentFrameProcessor.__init__(self, options, processorPool) and find desired processor(s) in processorPool

class FrameProcessorPipeline(FrameProcessorPool):
  """An ordered pipeline of FrameProcessor instances."""

  def __init__(self, options, processorTypes):
    """Create a list of FrameProcessors given appropriate types."""
    self.options = options
    self.gui = self.options['gui']
    self.debug = self.options['debug']

    self.processors = []
    for processorType in processorTypes:
      if issubclass(processorType, FrameProcessor):
        processor = processorType(options, self) if issubclass(processorType, DependentFrameProcessor) else processorType(options)
        self.processors.append(processor)
        #log(self, "__init__", "Added {0} instance.".format(processor.__class__.__name__))
      else:
        log(self, "__init__", "Warning: {0} is not a FrameProcessor; will not instantiate.".format(processorType.__name__))

  def initialize(self, imageIn, timeNow):
    for processor in self.processors:
      processor.initialize(imageIn, timeNow)

  def process(self, imageIn, timeNow):
    keepRunning = True
    imageOut = None
    for processor in self.processors:
      if processor.active:
        keepRunning, imageOut = processor.process(imageIn, timeNow)
        if self.gui and self.debug and imageOut is not None:  # show individual processor outputs only if gui and debug are true
        	cv2.imshow("Output: {}".format(processor.__class__.__name__), imageOut)
        if not keepRunning:
          break  # break out of this for loop (no further processors get to process this image)
    return keepRunning, imageOut

  def onKeyPress(self, key, keyChar=None):
    keepRunning = True
    for processor in self.processors:
      if processor.active:
        keepRunning = processor.onKeyPress(key, keyChar)  # pass along key-press to processor
        if not keepRunning:
          break  # break out of this for loop (no further processors get the key event)
    return keepRunning

  def activateProcessors(self, processorTypes=None, active=True):  # if None, activate all
    for processor in self.processors:
      if processorTypes is None or processor.__class__ in processorTypes:
        processor.active = active

  def deactivateProcessors(self, processorTypes=None):  # if None, deactivate all
    self.activateProcessors(processorTypes, False)

  def getProcessorByType(self, processorType):
    """Returns the first processor found that is an instance of processorType."""
    for processor in self.processors:
      if isinstance(processor, processorType):
        return processor
    return None

  def __str__(self):
    desc = "[" + ", ".join(("" if processor.active else "~") + processor.__class__.__name__ for processor in self.processors) + "]"
    return desc

max_stars_repo_name: IEEERobotics/high-level

max_issues_repo_name: IEEERobotics/high-level

max_forks_repo_name: IEEERobotics/high-level


text:
import numpy as np
import math

def visita_vetor(vetor_visitados, estado_visitado):
    for i in vetor_visitados:
        if estado_visitado == i:
            return False
    return True

def set_estado_inicial(high, weigth):
    estado = []
    for i in range(high):
        estado.append(np.random.randint(weigth))
    return estado

def count_heuristica(estado):
    count_h = 0
    for i in range(len(estado) - 1):
        for j in range(i + 1, len(estado)):
            if estado[i] + i == estado[j] + j:
                count_h = count_h + 1
            if estado[i] - i == estado[j] - j:
                count_h = count_h + 1
            if estado[i] == estado[j]:
                count_h = count_h + 1
    return count_h

if __name__ == '__main__':

    high = 8
    weigth = 8
    estado_inicial = set_estado_inicial(high, weigth)
    print(estado_inicial)
    #estado_inicial = [0, 1, 0, 1]
    estado_atual = estado_inicial
    vetor_visitados = []

    while(count_heuristica(estado_atual) != 0):
        estado_aux = estado_atual

        if visita_vetor(vetor_visitados, estado_aux) == True:
            vetor_visitados.append(estado_aux)
        else:
            estado_aux = set_estado_inicial(high, weigth)

        menor = math.factorial(len(estado_aux))
        for j in range(high):
            for i in range(weigth):
                estado_aux[j] = i
                if menor > count_heuristica(estado_aux):
                    menor = count_heuristica(estado_aux)
                    i_aux = i
                    j_aux = j

            estado_aux[j_aux] = i_aux

        estado_atual = estado_aux
    print("Estado Ideal:\n", estado_atual)

max_stars_repo_name: Matelaa/LearningPython

max_issues_repo_name: Matelaa/LearningPython

max_forks_repo_name: Matelaa/LearningPython


text:
#!/usr/bin/env python
# coding: utf-8

# In[1]:

get_ipython().system('python --version')

# #### Installation of pandas

# In[2]:

get_ipython().system('pip install pandas')

# In[3]:

get_ipython().system('pip install numpy')

# #### Installation of matplotlib

# In[4]:

get_ipython().system('pip install matplotlib')

# In[5]:

get_ipython().system('pip install seaborn')

# #### Pandas version

# In[6]:

import pandas as pd

pd.__version__

# #### Matplotlib version

# In[7]:

import matplotlib

matplotlib.__version__

# #### Numpy version

# In[8]:

import numpy as np

np.__version__

# ### Seaborn Version

# In[9]:

import seaborn

seaborn.__version__

# <b>Dataset</b> https://www.kaggle.com/vjchoudhary7/customer-segmentation-tutorial-in-python
# * Gender - Gender of the customer
# * Age - Age of the customer
# * Annual Income (k$) - Annual Income of the customee
# * Spending Score (1-100) - Score assigned by the mall based on customer behavior and spending nature

# #### Reading data using pandas python

# In[10]:

## TODO: While recording please show the file structure first

# In[11]:

mall_data = pd.read_json('datasets/Mall_Customers.json')

mall_data.head(5)

# In[12]:

average_income = mall_data['annual_income'].mean()

average_income

# In[13]:

mall_data['above_average_income'] = (mall_data['annual_income'] - average_income) > 0

mall_data.sample(5)

# * We can save it in any forms like normal text file, csv or json. It will save it in the location where you prefer as well

# In[14]:

mall_data.to_csv('datasets/mall_data_processed.csv', index=False)

# In[15]:

get_ipython().system('ls datasets/')

# #### Read the csv file for the clarification

# In[16]:

mall_data_updated = pd.read_csv('datasets/mall_data_processed.csv', index_col=0)

mall_data_updated.head(5)

# In[17]:

## TODO: While recording please show the files after each save

# In[18]:

mall_data.to_json('datasets/mall_data_column_oriented.json', orient='columns')

# In[19]:

mall_data.to_json('datasets/mall_data_records_oriented.json', orient='records')

# In[20]:

mall_data.to_json('datasets/mall_data_index_oriented.json', orient='index')

# In[21]:

mall_data.to_json('datasets/mall_data_values_oriented.json', orient='values')

# In[22]:

get_ipython().system('ls datasets/')

# In[ ]:

max_stars_repo_name: hossainlab/statswithpy

max_issues_repo_name: hossainlab/statswithpy

max_forks_repo_name: hossainlab/statswithpy


text:
import argparse
parser = argparse.ArgumentParser(description='control RDNSR')
parser.add_argument('--gpu', action="store",dest="gpu", default=-1)
parser.add_argument('--chk',action="store",dest="chk",default=-1)
# parser.add_argument('--test_sample', action="store",dest="test_sample",default=190)
parser.add_argument('--scale', action='store' , dest = 'scale' , default = 2)
parser.add_argument('--test_image', action = 'store' , dest = 'test_image' , default = 'test.png')
#parser.add_argument('--progressive_zoom',action='store',dest = 'pzoom' , default = False)

values = parser.parse_args()
gpu=int(values.gpu)
# test_sample = int(values.test_sample)
scale = int(values.scale)
chk = int(values.chk)
test_image = values.test_image

if gpu >= 0:
    os.environ["CUDA_VISIBLE_DEVICES"]=str(gpu)

import sys
import numpy as np
import matplotlib.pyplot as plt
from SRIP_DATA_BUILDER import DATA
from keras.models import Model
from keras.layers import Input,MaxPool2D,Deconvolution2D ,Convolution2D , Add, Dense , AveragePooling2D , UpSampling2D , Reshape , Flatten , Subtract , Concatenate
from keras.optimizers import Adam
from keras.callbacks import LearningRateScheduler
from keras import backend as k
from keras.utils import multi_gpu_model
import tensorflow as tf
from keras.utils import plot_model
from subpixel import Subpixel
from PIL import Image
import cv2

class SRResnet:
    def L1_loss(self , y_true , y_pred):
        return k.mean(k.abs(y_true - y_pred))

    #def L1_plus_PSNR_loss(self,y_true, y_pred):
        #return self.L1_loss(y_true , y_pred) - 0.0001*PSNRLoss(y_true , y_pred)

    def RDBlocks(self,x,name , count = 6 , g=32):
        ## 6 layers of RDB block
        ## this thing need to be in a damn loop for more customisability
            li = [x]
            pas = Convolution2D(filters=g, kernel_size=(3,3), strides=(1, 1), padding='same' , activation='relu' , name = name+'_conv1')(x)

            for i in range(2 , count+1):
                li.append(pas)
                out =  Concatenate(axis = self.channel_axis)(li) # conctenated out put
                pas = Convolution2D(filters=g, kernel_size=(3,3), strides=(1, 1), padding='same' , activation='relu', name = name+'_conv'+str(i))(out)

            # feature extractor from the dense net
            li.append(pas)
            out = Concatenate(axis = self.channel_axis)(li)
            feat = Convolution2D(filters=64, kernel_size=(1,1), strides=(1, 1), padding='same',activation='relu' , name = name+'_Local_Conv')(out)

            feat = Add()([feat , x])
            return feat

    def visualize(self):
            plot_model(self.model, to_file='model.png' , show_shapes = True)

    def get_model(self):
        return self.model

    def __init__(self , channel = 3 , lr=0.0001 , patch_size=32 , RDB_count=20 ,chk = -1 , scale = 2):
            self.channel_axis = 3
            inp = Input(shape = (patch_size , patch_size , channel))

            pass1 = Convolution2D(filters=64, kernel_size=(3,3), strides=(1, 1), padding='same' , activation='relu')(inp)

            pass2 = Convolution2D(filters=64, kernel_size=(3,3), strides=(1, 1), padding='same' , activation='relu')(pass1)

            RDB = self.RDBlocks(pass2 , 'RDB1')
            RDBlocks_list = [RDB,]
            for i in range(2,RDB_count+1):
                RDB = self.RDBlocks(RDB ,'RDB'+str(i))
                RDBlocks_list.append(RDB)
            out = Concatenate(axis = self.channel_axis)(RDBlocks_list)
            out = Convolution2D(filters=64 , kernel_size=(1,1) , strides=(1,1) , padding='same')(out)
            out = Convolution2D(filters=64 , kernel_size=(3,3) , strides=(1,1) , padding='same')(out)

            output = Add()([out , pass1])

            if scale >= 2:
                output = Subpixel(64, (3,3), r = 2,padding='same',activation='relu')(output)
            if scale >= 4:
                output = Subpixel(64, (3,3), r = 2,padding='same',activation='relu')(output)
            if scale >= 8:
                output = Subpixel(64, (3,3), r = 2,padding='same',activation='relu')(output)

            output = Convolution2D(filters =3 , kernel_size=(3,3) , strides=(1 , 1) , padding='same')(output)

            model = Model(inputs=inp , outputs = output)
            adam = Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=None, decay=lr/2, amsgrad=False)

            ## multi gpu setting

            #if gpu < 0:
               #model = multi_gpu_model(model, gpus=2)
            ## Modification of adding PSNR as a loss factor
            model.compile(loss=self.L1_loss, optimizer=adam , metrics=[PSNRLoss,SSIM])

            if chk > 0 :
        	print("Give a defined checkpoint to load from!!!")
		exit()

	    print("loading existing weights !!!")
	    model.load_weights('model_'+str(scale)+'x_iter'+str(chk)+'.h5')
            self.model = model

    def fit(self , X , Y ,batch_size=16 , epoch = 1000 ):
            # with tf.device('/gpu:'+str(gpu)):
            hist = self.model.fit(X, Y , batch_size = batch_size , verbose =1 , nb_epoch=epoch)
            return hist.history


text:
import numpy as np

from brl_gym.estimators.estimator import Estimator
from brl_gym.envs.herb_table import HerbTable, Action, GridCellState, ACTION_NAME

class BayesHerbTableEstimator(Estimator):
    """
    This class estimates herb_table given a history of observations
    It assumes that the beliefs and targets occupy only single grid
    """
    def __init__(self):
        env = HerbTable()
        # probability grids
        self.grid_shape = env.grid.shape
        super(BayesHerbTableEstimator, self).__init__(env.observation_space, env.action_space)

        # First layer is for beliefs
        # Second is for targets
        self.belief_high = np.ones((2, env.grid.shape[0], env.grid.shape[1]), dtype=np.float32)
        self.belief_low = np.zeros((2, env.grid.shape[0], env.grid.shape[1]), dtype=np.float32)
        self.nominal_env = env

    def reset(self):
        # Each column sums to 1
        self.belief = np.ones(self.grid_shape, dtype=np.float32) / self.grid_shape[0]
        return self.get_belief()

    def estimate(self, action, observation, **kwargs):
        """
        Given the latest (action, observation), update belief
        """
        if action is not None:
            action_type, x, y = self.nominal_env._decode_action(action)
            if action_type == Action.MOVE_LEFT:
                if kwargs['action_success']:
                    self.belief[:-1, y] = self.belief[1:, y]
                    self.belief[-1, y] = 0.0
                elif x > 0:
                    # Failure due to the cell being empty
                    self.belief[x, y] = 0.0

            if action_type == Action.MOVE_RIGHT:
                if kwargs['action_success']:
                    self.belief[1:, y] = self.belief[:-1, y]
                    self.belief[0, y] = 0.0
                elif x < self.grid_shape[0] - 1:
                    # Failure due to the cell being empty
                    self.belief[x, y] = 0.0

            if action_type == Action.GRASP:
                if kwargs['action_success']:
                    self.belief[:, y] = 0.0
                else:
                    # Failure due to the cell being empty
                    self.belief[x, y] = 0.0

            if np.sum(self.belief[:, y]) > 0:
                self.belief[:, y] /= np.sum(self.belief[:, y])

        # Any concrete observations are used to update the grids
        for i in range(self.belief.shape[1]):
            if np.any(observation[:, i] == GridCellState.MOVABLE):
                self.belief[:, i] =  0.0
                self.belief[observation[:, i] == GridCellState.MOVABLE, i] =  1.0

            if np.any(observation[:, i] == GridCellState.TARGET):
                self.belief[:, i] =  0.0
                self.belief[observation[:, i] == GridCellState.TARGET, i] =  1.0

        for i in range(self.belief.shape[1]):
            if np.any(self.belief[:, i] == 1.0):
                continue
            self.belief[observation[:, i] == GridCellState.EMPTY, i] = 0.0
            if np.sum(self.belief[:, i]) > 0:
                self.belief[:, i] /= np.sum(self.belief[:, i])

        return self.get_belief()

    def get_belief(self):
        movable = self.belief.copy()
        movable[:, 3:] = 0.0
        target = self.belief.copy()
        target[:, :3] = 0.0
        return np.array([movable, target])

if __name__ == "__main__":
    env = HerbTable()
    estimator = BayesHerbTableEstimator()
    obs = env.reset()
    belief = estimator.reset()
    print(obs)
    print(belief)
    belief = estimator.estimate(None, obs)

    import IPython; IPython.embed()


text:
import numpy as np

import os, sys
import re
#import pickle
import numpy as np
import copy
#from matplotlib.backends.backend_pdf import PdfPages
from matplotlib import pyplot as plt
import seaborn as sns

class mlbMarkov:
    def __init__(self, vbose=0, nbases=3, nouts=3):
        self.max_score = 15
        self.vbose = vbose
        self.nbases = nbases
        self.nouts = nouts
        self.state2int = {}
        self.int2state = {}
        self.initEnumerateStates()
        self.sz = len(self.allStates)
        self.initTransitionMatrix()
        self.probs = {}
        self.probs = {}
        for i in range(10):
            self.probs[i] = 0
        self.init_probs()
        self.solvedSystem = None

        self.transitionMatrix = self.initTransitionMatrix()
        self.valueMatrix = self.initValueMatrix()
        self.runsMatrix = np.zeros((self.max_score+1, self.sz))
        self.makeTransitionMatrix()
        assert np.all(abs(self.transitionMatrix.sum(0)-1)<1e-6)

        self.makeValueMatrix()
        self.summary_keys = ['man1', 'man2', 'man3', 'out0', 'out1', 'out2', 'out3']
        for i in range(self.max_score+1):
            self.summary_keys.append('run%d' % i)
        self.v0 = np.zeros((self.sz, 1))
        self.v0[0] = 1

    def init_probs(self):
        self.probs[1] = 0.15+0.08
        self.probs[2] = 0.05
        self.probs[3] = 0.005
        self.probs[4] = 0.025
        self.probs[0] = 1-(self.probs[1]+self.probs[2]+self.probs[3]+self.probs[4])

    def reNorm(self, a, norm=1.0):
        sum = 0.0
        for k in a:
            sum += a[k]
        v = norm/sum
        for k in a:
            a[k] *= v

        return a

    def initEnumerateStates(self):
        nb = self.nbases
        nstate = 0
        for i in range(2**(self.nbases)):
            s = bin(i).split('b')[1]
            for j in range(self.nbases-len(s)+1-1):
                s = '0'+s
            for o in range(self.nouts+1):
                for r in range(self.max_score+1):
                    k = s + '_%02d_%02d' % (o, r)
                    if self.vbose>=1:
                        print nstate, i, s, k
                    self.state2int[k] = nstate
                    self.int2state[nstate] = k
                    nstate += 1

        allStates = self.state2int.keys()
        allStates.sort()
        self.allStates = allStates

    def getNewState(self, nbaseHit, oldState):
        nb, no, rr = self.stateToInfo(oldState)
        if self.vbose>=1:
            print 'old nb', oldState, nbaseHit, nb, no

        if no==self.nouts:
            # cant transition if all outs already used up
            return oldState

        # a new state comes from ,
        # multiply by 2 nb times.
        # dont forget to add 1 the first time
        if nbaseHit == 0:
            return self.infoToState(nb, min(no+1, self.nouts), rr)

        newNb = nb
        for i in range(nbaseHit):

            newNb *= 2
            if i==0:
                newNb += 1
            if self.vbose>=1:
                print i, nb, newNb

        newNb = newNb % (2**(self.nbases))
        newState = self.infoToState(newNb, no)
        rr = int(rr) + self.getValue(oldState, newState)
        rr = min(rr, self.max_score)
        return self.infoToState(newNb, no, rr)

    def infoToState(self, nb, no, rr=None):

        s = bin(nb).split('b')[1]
        if self.vbose>=2:
            print nb, no, s, len(s), nb-len(s)+1
        for j in range(self.nbases-len(s)+1-1):
            s = '0'+s

        k = s + '_%02d' % no
        if rr is not None:
            k += '_%02d' % rr
        if self.vbose>=2:
            print 'infoToState---------', nb, no, k
        return k

    def stateToInfo(self, s):
        assert s.count('_') in [1,2], s
        if s.count('_')==2:
            bb, oo, rr = s.split('_')
            rr = int(rr)
        else:
            bb, oo = s.split('_')
            rr = None
        ii = int(bb, base=2)

        return ii, int(oo), rr

    def getNOnBase(self, state):
        s = state.split('_')[0]
        if self.vbose>=1:
            print 's', s
        sum = 0
        for i in s:
            sum += int(i)
            if self.vbose>=1:
                print 'sum', i, sum
        return sum

    def getValue(self, oldState, newState):
        oldb, oldo, oldr = self.stateToInfo(oldState)
        newb, newo, newr = self.stateToInfo(newState)
        if newo>oldo or newo==3:
            return 0
        # value is number that scored
        # this is, n_start + 1 = n_end + n_score
        # nscore = n_start + 1 - n_end
        n_start = self.getNOnBase(oldState)
        n_end = self.getNOnBase(newState)
        n_score = n_start+1-n_end
        return n_score

    def initTransitionMatrix(self):
        sz = len(self.int2state)
#        del self.transitionMatrix
        self.transitionMatrix = np.zeros((sz, sz))

    def initValueMatrix(self):
        sz = len(self.int2state)
#        del self.transitionMatrix
        self.valueMatrix = np.zeros((sz, sz))

    def makeTransitionMatrix(self, vbose=None):

        self.initTransitionMatrix()
        allStates = self.allStates

        for i, oldState in enumerate(allStates):
            # now, for each prob, we compute the prob to transition to new state
            assert oldState in allStates
            iold = self.state2int[oldState]
            for nb in range(self.nbases+2):
                if self.vbose>=1:
                    print '** makeTM *******'
                if not nb in self.probs:
                    self.probs[nb] = 0
                v = self.probs[nb]

                newState = self.getNewState(nb, oldState)
                assert newState in allStates, newState
                if not newState in self.allStates:
                    if self.vbose>=1:
                        print 'makeTM', oldState, nb, newState, iold, 'xxx'
                    continue
#                newB, newO = self.stateToInfo(
                inew = self.state2int[newState]
                self.transitionMatrix[inew][iold] += v
                if self.vbose>=1 or vbose>=1:
                    print 'makeTM', oldState, nb, newState, iold, inew, self.transitionMatrix[inew][iold]

        last = self.transitionMatrix.shape[0]-1
        self.transitionMatrix[last][last] = 1

    def makeValueMatrix(self):

        self.initValueMatrix()
        allStates = self.allStates

        for i, oldState in enumerate(allStates):
            iold = self.state2int[oldState]
            for j, newState in enumerate(allStates):
                inew = self.state2int[newState]
                self.valueMatrix[inew][iold] = self.getValue(oldState, newState)
                if self.vbose>=1:
                    print '** makeVM *******'
                    print iold, inew, self.valueMatrix[inew][iold]

    def solveSystem(self):
        nrow, ncol = np.shape(self.transitionMatrix)
        if not nrow==ncol:
            raise Exception

        m1 = np.identity(nrow)
        mt = self.transitionMatrix
        mv = self.valueMatrix
        vr = np.diag(np.dot(np.transpose(mv),mt))
        ans = np.linalg.solve(np.transpose(m1-mt), vr)
        self.solvedSystem = ans
        return ans

    def printSolution(self, state_vector, printProbs=True):
        if self.vbose>=1:
            print 'idx state prob'
        for i, v in enumerate(state_vector):
            s = self.int2state[i]
            if self.vbose>=1:
                print '%3d %s %.6f' % (i, s, v)

    def parse_state(self, state, n=0):
        m3, m2, m1 = state.split('_')[0][:]
        nouts = int(state.split('_')[1][:])
        m3 = int(m3)
        m2 = int(m2)
        m1 = int(m1)
        nruns = int(state.split('_')[2][:])
        return {'man3': m3, 'man2': m2, 'man1': m1, 'nouts': nouts, 'nruns': nruns}

    def state_vector_to_summary(self, stateVector, n=0):

        data = {}
        for k in self.summary_keys:
            data[k] = 0

        # data['out0_runs'] = dict([('run%d' % i, 0) for i in range(self.max_score+1)])
        # data['out1_runs'] = dict([('run%d' % i, 0) for i in range(self.max_score+1)])
        # data['out2_runs'] = dict([('run%d' % i, 0) for i in range(self.max_score+1)])
        # data['out3_runs'] = dict([('run%d' % i, 0) for i in range(self.max_score+1)])

        if self.vbose>=1:
            print data
        for i, p in enumerate(stateVector):
            s = self.int2state[i]
            ans = self.parse_state(s)
            if self.vbose>=1:
                print s, ans, p
            if p==0:
                continue
            for k in ['man1', 'man2', 'man3']:
                data[k] += p[0]*ans[k]
            assert ans['nruns']>=0 or p<1e-6
            k = 'run%d' % ans['nruns']
            data[k] += p[0]
            k = 'out%d' % ans['nouts']
            data[k] += p[0]

        return data

    def generate_sequence(self, v0, nseq=10):
        seq = []

        for i in range(nseq):
            if i==0:
                v = copy.copy(v0)
            else:
                v = self.transitionMatrix.dot(v)

            summary = self.state_vector_to_summary(v, n=i)
            print summary
            seq.append(summary)
        return seq

    def bar_chart_from_summary(self, summary, n=0, ysc='linear'):
        xx = []
        yy = []
        plt.clf()
        for i, k in enumerate(self.summary_keys):
            xx.append(i)
            yy.append(summary[k])

        plt.bar(np.array(xx)-0.3, yy, color='steelblue')
        plt.xticks(xx, self.summary_keys, rotation='vertical', fontsize=10)
        plt.yscale(ysc)
        plt.ylim(0.01,1)
        plt.xlim(-0.5, self.max_score+8)
        plt.text(self.max_score+7, 0.9, 'PA= %03d' % n, ha='right')
        plt.ylabel('probability')
        plt.title('baseball markov chain')

    def transitionMatrixOutputArray(self, threshold=1e-6):
        ans = []
        for j in range(self.sz):
            for i in range(self.sz):
                if i>j:
                    pass
#                    continue
                p = self.transitionMatrix[j][i]
                if p>threshold:
                    ans.append((i, self.int2state[i],
                               j, self.int2state[j],
                               self.transitionMatrix[j][i]
                               ))
        return ans

    def printTransitionMatrix(self, threshold=1e-6):
        ans = self.transitionMatrixOutputArray(threshold=threshold)
        for line in ans:
            print line

    def printStateVector(self, v, threshold=1e-6):
        assert len(v)==self.sz
        s = 0.0
        counter = 0
        maxp = -1
        maxi = None
        for i, p in enumerate(v):
            if p<threshold:
                continue
            print '%03d %s %.6f ' % (i, self.int2state[i], p[0])
            s += p[0]
            counter += 1
            if p[0]>maxp:
                maxp = p[0]
                maxi = i

        assert maxi is not None
        print '---------------------'
        descrpition = '%d of %d' % (counter, self.sz)
        print '%13s %.6f' % (descrpition, s)
        print 'max %s %.4f' % (self.int2state[maxi], maxp)
def main(nbases=3, nouts=3, vbose=0, probs=None):
    m = mlbMarkov(nbases=nbases, nouts=nouts, vbose=vbose)

    for k in probs:
        m.probs[k] = probs[k]

    m.probs = m.reNorm(m.probs, norm=1.0)
    return m

###################
if __name__=='__main__':
    printProbs = True
    vbose = 0
    nbases = 3
    nouts = 3
    ninn = 9

    probs = {}
    probs[1] = 0.15+0.08
    probs[2] = 0.05
    probs[3] = 0.005
    probs[4] = 0.025
    probs[0] = 1-(probs[1]+probs[2]+probs[3]+probs[4])
    doLinearWeights = False

    for ia, a in enumerate(sys.argv):
        if a=='-nbases' or a=='-nbase':
            nbases = int(sys.argv[ia+1])
        if a=='-nouts' or a=='-nout':
            nouts = int(sys.argv[ia+1])
        if a[0:2]=='-p':
            m = re.search('-p([0-9]+)', a)
            ib = int(m.group(1))
            v= float(sys.argv[ia+1])
            probs[ib] = v
        if a=='-vbose':
            vbose = int(sys.argv[ia+1])

        if a=='-printProb' or a=='-printProbs':
            printProbs = bool(int(sys.argv[ia+1]))

        if a=='-doLinearWeights':
            doLinearWeights = bool(int(sys.argv[ia+1]))

    for i in range(nbases+2):
        if not i in probs:
            probs[i] = 0

    m = main(nbases=nbases, nouts=nouts, vbose=vbose, probs=probs)
    m.probs = m.reNorm(m.probs)
    m.makeTransitionMatrix()
    m.makeValueMatrix()


text:
import numpy as np
from PIL import ImageGrab
import cv2
import time
import pyautogui

pyautogui.PAUSE = 0.001

last_time = time.time()
start_time = time.time()
kernelOpen=np.ones((5,5))
kernelClose=np.ones((20,20))
lower_range = np.array([90,100,50], np.uint8)
upper_range = np.array([100,250,250], np.uint8)

while(time.time() - start_time < 119):
    print(time.time() - start_time)
    screen = np.array(ImageGrab.grab(bbox=(637,380,1590,860)))
    screen = cv2.cvtColor(screen, cv2.COLOR_BGR2HSV)

    print("FPS: {}".format(1/(time.time() - last_time)))
    last_time = time.time()

    mask = cv2.inRange(screen, lower_range, upper_range)
    maskOpen=cv2.morphologyEx(mask,cv2.MORPH_OPEN,kernelOpen)
    maskClose=cv2.morphologyEx(maskOpen,cv2.MORPH_CLOSE,kernelClose)

    _, conts,h=cv2.findContours(maskClose.copy(),cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_NONE)
    cv2.drawContours(screen,conts,-1,(255,0,0),3)

    for i in range(len(conts)):
            x,y,w,h=cv2.boundingRect(conts[i])
            pyautogui.click(x + 637, y + 405, clicks=5)
            pyautogui.press('1')
time.sleep(5)
pyautogui.click(970, 580)

ext: pyw


text:
import pickle
from pathlib import Path

import matplotlib.font_manager as fm
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.offsetbox import AnchoredText
from mpl_toolkits.axes_grid1.anchored_artists import AnchoredSizeBar
from ScanImageTiffReader import ScanImageTiffReader
from skimage import exposure
from tqdm import tqdm

from .simple_guis import openfilegui

class SItiff:
    def __init__(self, path) -> None:
        """
        Loads a ScanImage tiff from the specified path.

        Args:
            path (str or Path): filepath to tiff
        """
        self.path = str(path)

        self.metadata = metadata_to_dict(self.path)
        self.nchannels = self._get_nchannels()
        self.zs = self._get_zs()
        self.nplanes = len(self.zs)
        self.fr = self._eval_numeric_metadata('scanVolumeRate')

        with ScanImageTiffReader(self.path) as reader:
            self.data = reader.data()

    def _eval_numeric_metadata(self, key):
        return eval(self.metadata[key].replace(' ',',').replace(';',','))

    def _get_nchannels(self):
        chans = self._eval_numeric_metadata('channelSave')
        if isinstance(chans, int):
            nchannels = 1
        else:
            nchannels = len(chans)
        return nchannels

    def _get_zs(self):
        zs = self._eval_numeric_metadata('zs')
        if isinstance(zs, (int, float)):
            zs = [zs]
        return zs

    def extract(self, z, ch):
        """
        Get the underlying data for a specified zplane and channel.

        Args:
            z (int): Index of zplane to extract.
            ch (int): Index of channel to extract. Use 0 for channel 1/green PMT, 1 for channel 2/red PMT.

        Returns:
            nframes x N x N numpy array slice of data
        """
        tslice = get_tslice(z, ch, self.nchannels, self.nplanes)
        img = self.data[tslice,:,:]
        return img

    def mean_img(self, z_idx, channel, scaling=None, as_rgb=False, rgb_ch=None, blue_as_cyan=True):
        """
        Get the mean image of a specified z-plane and channel. Takes the mean and does a minimum
        subtract. Optionally rescales the LUT intensity. Optionally returns as RGB-compatible array.

        Args:
            z_idx (int): Index of zplane to extract.
            channel (int): Index of channel to extract. Use 0 for channel 1/green PMT, 1 for channel 2/red PMT.
            scaling (tuple of ints, optional): LUT range to use. Defaults to None for no scaling.
            as_rgb (bool, optional): Whether to return a NxNx3 RGB image. Defaults to False.
            rgb_ch (int, optional): Specifies the RGB channel to use. Defaults to None.
            blue_as_cyan (bool, optional): [description]. Defaults to True.

        Returns:
            NxN numpy array of image (or NxNx3 if RGB)
        """
        tslice = get_tslice(z_idx, channel, self.nchannels, self.nplanes)
        mimg = np.mean(self.data[tslice,:,:], axis=0)
        mimg -= mimg.min()

        if scaling:
            mimg = exposure.rescale_intensity(mimg, scaling)

        if as_rgb:
            rgb_im = np.zeros((*mimg.shape,3))
            if not rgb_ch:
                if channel == 1:
                    rgb_ch = 0
                elif channel == 0:
                    rgb_ch = 1
                else:
                    rgb_ch = channel
            rgb_im[:,:,rgb_ch] = mimg

            if blue_as_cyan and rgb_ch == 2:
                rgb_im[:,:,1] = mimg

            return rgb_im

        else:
            return mimg

    def merge_mean(self, z_idx, gscale=None, rscale=None, green_as_cyan=False):
        green_ch = self.mean_img(z_idx, 0, scaling=gscale)
        red_ch = self.mean_img(z_idx, 1, scaling=rscale)

        rgb_im = np.zeros((*green_ch.shape, 3))
        rgb_im[:,:,0] = red_ch
        rgb_im[:,:,1] = green_ch

        if green_as_cyan:
            rgb_im[:,:,2] = green_ch

        return rgb_im

    def show(self, z_idx, ch_idx, scaling=None, as_rgb=False, rgb_ch=None, ax=None,
             ch_label_txt=None, ch_label_color='white', **kwargs):

        if ax is None:
            ax = plt.gca()

        mean_img = self.mean_img(z_idx, ch_idx, scaling=scaling, as_rgb=as_rgb, rgb_ch=rgb_ch, **kwargs)

        # note I don't think this will work will multiple colors since you can't specify per line
        fontdict = {
            'size':18,
            'color':ch_label_color
        }
        label = AnchoredText(ch_label_txt, loc='upper left', prop=fontdict,
                             frameon=False, pad=0.2, borderpad=0.2)

        fontprops = fm.FontProperties(size=18)
        scalebar = AnchoredSizeBar(ax.transData,
                                128,
                                f'200 $\mu$m',
                                'lower right',
                                pad=0.2,
                                color='white',
                                frameon=False,
                                size_vertical=8,
                                fontproperties=fontprops)
        ax.add_artist(scalebar)
        ax.add_artist(label)
        ax.imshow(mean_img)
        ax.axis('off')

        return ax

    @classmethod
    def load(cls, rootdir='d:/frankenrig/experiments'):
        path = openfilegui(rootdir=rootdir, title='Select Tiff')
        if not path:
            return
        print(f'Loading tiff: {path}')
        return cls(path)

def metadata_to_dict(file):
    """Read the SI metadata and turn in into a dict. Does not cast/eval values."""

    with ScanImageTiffReader(file) as reader:
        meta = reader.metadata()

    # split at the new line marker
    meta = meta.split('\n')
    # filter out the ROI data by keeping fields that start with SI.
    meta = list(filter(lambda x: 'SI.' in x, meta))
    # make dictionary by splitting at equals, and only keep the last part of the fieldname as the key
    d = {k.split('.')[-1]:v for k,v in (entry.split(' = ') for entry in meta)}

    return d

def get_tslice(z_idx, ch_idx, nchannels, nplanes):
    return slice((z_idx*nchannels)+ch_idx, None, nplanes*nchannels)

def slice_movie(mov_path, x_slice, y_slice, t_slice) -> np.ndarray:
    """
    Slice a single tiff along x, y, and time dims. Time dim must account for number of channels and
    z-planes. slice((z_idx*nchannels)+channel, -1, nplanes*nchannels)

    Args:
        mov_path (str): path to movie
        x_slice (slice): slice along x-axis
        y_slice (slice): slice along y-axis
        t_slice (slice): slice along t-axis

    Returns:
        np.array: array of sliced movie
    """
    with ScanImageTiffReader(mov_path) as reader:
        data = reader.data()
        data = data[t_slice, y_slice, x_slice]
    return data

def count_tiff_lengths(movie_list, save=False):
    """
    Counts the length of tiffs for a single plane and channel to get trial lenths. Optionally save
    the data to dist as a pickle.

    Args:
        movie_list (list): list of str or Path pointing to the tiffs to count.
        save (bool or str, optional): Whether to save the file. can either set to True to save in
                                      in the folder with the counted tiffs or specify save location
                                      with a string. Defaults to False.

    Returns:
        numpy array of tiff/trial lengths
    """

    first_tiff = SItiff(movie_list[0])
    t_slice = get_tslice(0, 0, first_tiff.nchannels, first_tiff.nplanes)
    nframes = [slice_movie(str(mov), slice(None), slice(None), t_slice).shape[0] for mov in tqdm(movie_list, desc="Counting Tiffs: ")]

    if save:
        if isinstance(save, str):
            save_path = Path(save, 'tiff_lengths.pickle')
        else:
            save_path = Path(first_tiff.path).parent/'tiff_lengths.pickle'

        with open(save_path, 'wb') as f:
            pickle.dump(nframes, f)

    return np.array(nframes)

def tiffs2array(movie_list, x_slice, y_slice, t_slice):
    data = [slice_movie(str(mov), x_slice, y_slice, t_slice) for mov in movie_list]
    return np.concatenate(data)

def get_crop_mask(Cx, Cy, bb):
    """
    Returns a mask to crop an image around a centered x,y point. Uses advanced indexing.

    Args:
        Cx (int, float): center X location
        Cy (int, float): center Y location
        bb (int): bounding box on either side of Cx, Cy
    """
    xs = np.arange(Cx-bb, Cx+bb, dtype=int)
    ys = np.arange(Cy-bb, Cy+bb, dtype=int)
    mask = np.ix_(ys,xs)
    return mask


text:
# coding: utf-8

# We have a bunch of probably chess x-corners output by our ML detector. Let's do the following routine:
#
# 1. Take 4 random points, (check no 3 points are colinear, else retry) then warp them to a unity grid.
# 2. Count distances of each point to closest integer grid corner, sum of these will be the score.
# 3. Do this for an exhaustive set of random points?
#
# The best score will likely be the correct transform to a chessboard.

# In[731]:

import numpy as np
from IPython.display import Image, display
import PIL.Image
import matplotlib.pyplot as plt
import scipy.ndimage
import cv2 # For Sobel etc
import glob
import SaddlePoints
from scipy.spatial import Delaunay
from functools import wraps
import time

def timed(f):
  @wraps(f)
  def wrapper(*args, **kwds):
    start = time.time()
    result = f(*args, **kwds)
    elapsed = time.time() - start
    print "%s took %.2f ms to finish" % (f.__name__, elapsed*1e3)
    return result
  return wrapper

# # RANSAC to find the best 4 points signifying a chess tile.
# Idea to find quads from triangles. For each triangle, the triangle that completes the quad should always be on the longest edge of the triangle and shares the two vertices of that edge. No image gradient checks needed.

# Build up a list of quads from input delaunay triangles, returns an Nx4 list of indices on the points used.
def getAllQuads(tri):
    pairings = set()
    quads = []
    # Build up a list of all possible unique quads from triangle neighbor pairings.
    # In general the worst common case with a fully visible board is 6*6*2=36 triangles, each with 3 neighbor
    # so around ~100 quads.
    for i, neighbors in enumerate(tri.neighbors):
        for k in range(3): # For each potential neighbor (3, one opposing each vertex of triangle)
            nk = neighbors[k]
            if nk != -1:
                # There is a neighbor, create a quad unless it already exists in set
                pair = (i, nk)
                reverse_pair = (nk, i)
                if reverse_pair not in pairings:
                    # New pair, add and create a quad
                    pairings.add(pair)
                    b = tri.simplices[i]
                    d = tri.simplices[nk]
                    nk_vtx = (set(d) - set(b)).pop()
                    insert_mapping = [2,3,1]
                    b = np.insert(b,insert_mapping[k], nk_vtx)
                    quads.append(b)
    return np.array(quads)

def countHits(given_pts, x_offset, y_offset):
    # Check the given integer points (in unity grid coordinate space) for matches
    # to an ideal chess grid with given initial offsets
    pt_set = set((a,b) for a,b in given_pts)
    [X,Y] = np.meshgrid(np.arange(7) + x_offset,np.arange(7) + y_offset)
    matches = 0
    # count matching points in set
    matches = sum(1 for x,y in zip(X.flatten(), Y.flatten()) if (x,y) in pt_set)
    return matches

def getBestBoardMatchup(given_pts):
    best_score = 0
    best_offset = None
    # scores = np.zeros([7,7])
    for i in range(7):
        for j in range(7):
            # Offsets from -6 to 0 for both
            score = countHits(given_pts, i-6, j-6)
            # scores[i,j] = score
            if score > best_score:
                best_score = score
                best_offset = [i-6, j-6]
    # print scores
    return best_score, best_offset

def scoreQuad(quad, pts, prevBestScore=0):
    idealQuad = np.array([[0,1],[1,1],[1,0],[0,0]],dtype=np.float32)
    M = cv2.getPerspectiveTransform(quad.astype(np.float32), idealQuad)

    # warped_to_ideal = cv2.perspectiveTransform(np.expand_dims(quad.astype(float),0), M)

    # Warp points and score error
    pts_warped = cv2.perspectiveTransform(np.expand_dims(pts.astype(float),0), M)[0,:,:]

    # Get their closest idealized grid point
    pts_warped_int = pts_warped.round().astype(int)

    # Refine
    M_refined, _ = cv2.findHomography(pts, pts_warped_int, cv2.RANSAC)
    if (M_refined is None):
      M_refined = M

    # Re-Warp points with the refined M and score error
    pts_warped = cv2.perspectiveTransform(np.expand_dims(pts.astype(float),0), M_refined)[0,:,:]

    # Get their closest idealized grid point
    pts_warped_int = pts_warped.round().astype(int)

    # Count matches
    score, offset = getBestBoardMatchup(pts_warped_int)
    if score < prevBestScore:
        return score, None, None, None

    # Sum of distances from closest integer value for each point
    # Use this error score for tie-breakers where number of matches is the same.
    error_score = np.sum(np.linalg.norm((pts_warped - pts_warped_int), axis=1))

    return score, error_score, M, offset

# brutesacChessboard ~752ms
# getAllQuads 12ms

@timed
def brutesacChessboard(xcorner_pts):
  # Build a list of quads to try.
  tri = Delaunay(xcorner_pts)
  quads = getAllQuads(tri) #

  # For each quad, keep track of the best fitting chessboard.
  best_score = 0
  best_error_score = None
  best_M = None
  best_quad = None
  best_offset = None
  for quad in xcorner_pts[quads]:
    score, error_score, M, offset = scoreQuad(quad, xcorner_pts, best_score)
    if score > best_score or (score == best_score and error_score < best_error_score):
      best_score = score
      best_error_score = error_score
      best_M = M
      best_quad = quad
      best_offset = offset
      if best_score > (len(xcorner_pts)*0.8):
        break

  return best_M, best_quad, best_offset, best_score, best_error_score

# @timed
def refineHomography(pts, M, best_offset):
  pts_warped = cv2.perspectiveTransform(np.expand_dims(pts.astype(float),0), M)[0,:,:]
  a = pts_warped.round() - best_offset
  b = pts

  # Only use points from within chessboard region.
  outliers = np.any((a < 0) | (a > 7), axis=1)
  a = a[~outliers]
  b = b[~outliers]

  # Least-Median robust homography
  M_homog, _ = cv2.findHomography(b,a, cv2.LMEDS)

  return M_homog

@timed
def predictOnTiles(tiles, predict_fn):
  predictions = predict_fn(
    {"x": tiles})

  # Return array of probability of tile being an xcorner.
  # return np.array([p[1] for p in predictions['probabilities']])
  return np.array([p[1] for p in predictions['probabilities']])

@timed
def predictOnImage(pts, img_gray, predict_fn, WINSIZE = 10):
  # Build tiles to run classifier on. (23 ms)
  tiles = getTilesFromImage(pts, img_gray, WINSIZE=WINSIZE)

  # Classify tiles. (~137ms)
  probs = predictOnTiles(tiles, predict_fn)

  return probs

@timed
def getTilesFromImage(pts, img_gray, WINSIZE=10):
  # NOTE : Assumes no point is within WINSIZE of an edge!

  # Points Nx2, columns should be x and y, not r and c.
  # Build tiles
  img_shape = np.array([img_gray.shape[1], img_gray.shape[0]])
  tiles = np.zeros([len(pts), WINSIZE*2+1, WINSIZE*2+1])
  for i, pt in enumerate(np.round(pts).astype(np.int64)):
    tiles[i,:,:] = img_gray[pt[1]-WINSIZE:pt[1]+WINSIZE+1, pt[0]-WINSIZE:pt[0]+WINSIZE+1]

  return tiles

@timed
def classifyPoints(pts, img_gray, predict_fn, WINSIZE = 10):
  tiles = getTilesFromImage(pts, img_gray, WINSIZE=WINSIZE)

  # Classify tiles.
  probs = predictOnTiles(tiles, predict_fn)

  return probs

@timed
def classifyImage(img_gray, predict_fn, WINSIZE = 10, prob_threshold=0.5):
  spts = SaddlePoints.getFinalSaddlePoints(img_gray, WINSIZE)

  return spts[predictOnImage(spts, img_gray, predict_fn, WINSIZE) > prob_threshold, :]

@timed
def processFrame(gray):
  # and return M for chessboard from image
  pts = classifyImage(gray)

  # pts = np.loadtxt('example_pts.txt')
  pts = pts[:,[1,0]] # Switch rows/cols to x/y for plotting on an image

  if (len(pts)) < 6:
      print("Probably not enough points (%d) for a good fit, skipping" % len(pts))
      return None, None

  raw_M, best_quad, best_offset, best_score, best_error_score = brutesacChessboard(pts)

  # The chess outline is only a rough guess, which we can easily refine now that we have an understanding of which points are meant for which chess corner. One could do distortion correction with the points available (as normal chessboard algorithms are used for in computer vision). If we wanted to get fancy we could apply a warp flow field on a per-tile basis.

  # ## Refine with Homography on all points
  # Now we can refine our best guess using all the found x-corner points to their ideal points with opencv findHomography, using the regular method with all points.
  if (raw_M is not None):
    M_homog = refineHomography(pts, raw_M, best_offset)
  else:
    M_homog = None

  return M_homog, pts

if __name__ == '__main__':
  np.set_printoptions(suppress=True) # Better printing of arrays
  plt.rcParams['image.cmap'] = 'jet' # Default colormap is jet

  # img = PIL.Image.open('frame900.png')
  # img = PIL.Image.open('input/img_35.jpg')
  filenames = glob.glob('input_yt/*.jpg')
  filenames.extend(glob.glob('input_yt/*.png'))
  filenames.extend(glob.glob('input/*.jpg'))
  filenames.extend(glob.glob('input/*.png'))
  # filenames.extend(glob.glob('input_bad/*.png'))
  # filenames.extend(glob.glob('input_bad/*.jpg'))
  filename = np.random.choice(filenames,1)[0]
  filename= 'weird.jpg'
  print(filename)
  img = PIL.Image.open(filename)
  if (img.size[0] > 800):
      img = img.resize([600,400])
  gray = np.array(img.convert('L'))

  M_homog, pts = processFrame(gray)

  if M_homog is None:
    print("Failed on image, not enough points")
    exit()

  pts_warped2 = cv2.perspectiveTransform(np.expand_dims(pts.astype(float),0), M_homog)[0,:,:]

  fig = plt.figure(figsize=(10,5))
  ax = fig.add_subplot(1, 2, 1)

  # Major ticks every 20, minor ticks every 5
  major_ticks = np.arange(-2, 9, 2)
  minor_ticks = np.arange(-2, 9, 1)

  ax.axis('square')
  ax.set_xticks(major_ticks)
  ax.set_xticks(minor_ticks, minor=True)
  ax.set_yticks(major_ticks)
  ax.set_yticks(minor_ticks, minor=True)

  # And a corresponding grid
  ax.grid(which='both')

  # Or if you want different settings for the grids:
  ax.grid(which='minor', alpha=0.2)
  ax.grid(which='major', alpha=0.5)

  ideal_grid_pts = np.vstack([np.array([0,0,1,1,0])*8-1, np.array([0,1,1,0,0])*8-1]).T

  plt.plot(ideal_grid_pts[:,0], ideal_grid_pts[:,1], 'r:')
  plt.plot(pts_warped2[:,0], pts_warped2[:,1], 'r.')

  plt.subplot(122)
  plt.imshow(img)
  plt.plot(pts[:,0], pts[:,1], 'ro')

  # Refined via homography of all valid points
  unwarped_ideal_chess_corners_homography = cv2.perspectiveTransform(
      np.expand_dims(ideal_grid_pts.astype(float),0), np.linalg.inv(M_homog))[0,:,:]

  plt.plot(unwarped_ideal_chess_corners_homography[:,0], unwarped_ideal_chess_corners_homography[:,1], 'r-', lw=4);
  plt.show()

max_stars_repo_name: Elucidation/ChessboardDetect

max_issues_repo_name: AnkaChan/ChessboardDetect

max_forks_repo_name: AnkaChan/ChessboardDetect

reason: import numpy,import scipy,from scipy