text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""ML Walk-Through"""
# pylint: disable=R0916, W0104, unused-argument, W0621

from __future__ import (division, absolute_import, print_function,
                        unicode_literals)

import os

# Pandas and numpy for data manipulation
import numpy as np
import pandas as pd

# Seaborn for visualization
import seaborn as sns

# Matplotlib visualization
import matplotlib.pyplot as plt

# Internal ipython tool for setting figure size
from IPython.core.pylabtools import figsize

# Splitting data into training and testing
from sklearn.model_selection import train_test_split

# No warnings about setting value on copy of slice
pd.options.mode.chained_assignment = None

# Display up to 60 columns of a dataframe
pd.set_option('display.max_columns', 60)

# Set default font size
plt.rcParams['font.size'] = 24

# Seaborn Font Size
sns.set(font_scale=2)

# Get File Directory
WORK_DIR = os.path.dirname((os.path.realpath(__file__)))

# Read in data into a dataframe
data = pd.read_csv(WORK_DIR + "/data/Energy_and_Water_Data_Disclosure_for\
_Local_Law_84_2017__Data_for_Calendar_Year_2016_.csv")

# Display top of dataframe
print(data.head())

# See the column data types and non-missing values
data.info()

# Replace all occurrences of Not Available with numpy not a number
data = data.replace({'Not Available': np.nan})

# Iterate through the columns
for col in list(data.columns):
    # Select columns that should be numeric
    if ('ft²' in col or 'kBtu' in col or 'Metric Tons CO2e' in col
            or 'kWh' in col or 'therms' in col or 'gal' in col
            or 'Score' in col):
        # Convert the data type to float
        data[col] = data[col].astype(float)

# Statistics for each column
print(data.describe())

# Missing Values (From https://stackoverflow.com/questions/26266362)
# Function to calculate missing values by column

def missing_values_table(df):
    # Total missing values
    mis_val = df.isnull().sum()

    # Percentage of missing values
    mis_val_percent = 100 * df.isnull().sum() / len(df)

    # Make a table with the results
    mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)

    # Rename the columns
    mis_val_table_ren_columns = mis_val_table.rename(columns={
        0: 'Missing Values',
        1: '% of Total Values'
    })

    # Sort the table by percentage of missing descending
    mis_val_table_ren_columns = mis_val_table_ren_columns[
        mis_val_table_ren_columns.iloc[:, 1] != 0].sort_values(
            '% of Total Values', ascending=False).round(1)

    # Print some summary information
    print("Your selected dataframe has " + str(df.shape[1]) + " columns.\n"
          "There are " + str(mis_val_table_ren_columns.shape[0]) +
          " columns that have missing values.")

    # Return the dataframe with missing information
    return mis_val_table_ren_columns

missing_values = missing_values_table(data)

# Get the columns with > 50% missing
missing_df = missing_values_table(data)
print(missing_df)
missing_columns = list(missing_df[missing_df['% of Total Values'] > 50].index)
print('We will remove %d columns.' % len(missing_columns))

# Drop the columns
data = data.drop(columns=list(missing_columns))

figsize(8, 8)

# Rename the score
data = data.rename(columns={'ENERGY STAR Score': 'score'})

# Histogram of the Energy Star Score
plt.style.use('fivethirtyeight')
plt.hist(data['score'].dropna(), bins=100, edgecolor='k')
plt.xlabel('Score')
plt.ylabel('Number of Buildings')
plt.title('Energy Star Score Distribution')
plt.show()

# Histogram Plot of Site EUI
figsize(8, 8)
plt.hist(data['Site EUI (kBtu/ft²)'].dropna(), bins=20, edgecolor='black')
plt.xlabel('Site EUI')
plt.ylabel('Count')
plt.title('Site EUI Distribution')
plt.show()

print(data['Site EUI (kBtu/ft²)'].describe())

print(data['Site EUI (kBtu/ft²)'].dropna().sort_values().tail(10))

# One building is clearly far above the rest.
data.loc[data['Site EUI (kBtu/ft²)'] == 869265, :]

# Calculate first and third quartile
first_quartile = data['Site EUI (kBtu/ft²)'].describe()['25%']
third_quartile = data['Site EUI (kBtu/ft²)'].describe()['75%']

# Interquartile range
iqr = third_quartile - first_quartile

# Remove outliers
data = data[(data['Site EUI (kBtu/ft²)'] > (first_quartile - 3 * iqr))
            & (data['Site EUI (kBtu/ft²)'] < (third_quartile + 3 * iqr))]

# Histogram Plot of Site EUI
figsize(8, 8)
plt.hist(data['Site EUI (kBtu/ft²)'].dropna(), bins=20, edgecolor='black')
plt.xlabel('Site EUI')
plt.ylabel('Count')
plt.title('Site EUI Distribution')
plt.show()

# Create a list of buildings with more than 100 measurements
types = data.dropna(subset=['score'])
types = types['Largest Property Use Type'].value_counts()
types = list(types[types.values > 100].index)

# Plot of distribution of scores for building categories
figsize(12, 10)

# Plot each building
for b_type in types:
    # Select the building type
    subset = data[data['Largest Property Use Type'] == b_type]

    # Density plot of Energy Star scores
    sns.kdeplot(subset['score'].dropna(), label=b_type, shade=False, alpha=0.8)

# label the plot
plt.xlabel('Energy Star Score', size=20)
plt.ylabel('Density', size=20)
plt.title('Density Plot of Energy Star Scores by Building Type', size=28)
plt.show()

# Create a list of boroughs with more than 100 observations
boroughs = data.dropna(subset=['score'])
boroughs = boroughs['Borough'].value_counts()
boroughs = list(boroughs[boroughs.values > 100].index)

# Plot of distribution of scores for boroughs
figsize(12, 10)

# Plot each borough distribution of scores
for borough in boroughs:
    # Select the building type
    subset = data[data['Borough'] == borough]

    # Density plot of Energy Star scores
    sns.kdeplot(subset['score'].dropna(), label=borough)

# label the plot
plt.xlabel('Energy Star Score', size=20)
plt.ylabel('Density', size=20)
plt.title('Density Plot of Energy Star Scores by Borough', size=28)
plt.show()

# Find all correlations and sort
correlations_data = data.corr()['score'].sort_values()

# Print the most negative correlations
print(correlations_data.head(15), '\n')

# Print the most positive correlations
print(correlations_data.tail(15))

# Select the numeric columns
numeric_subset = data.select_dtypes('number')

# Create columns with square root and log of numeric columns
for col in numeric_subset.columns:
    # Skip the Energy Star Score column
    if col == 'score':
        next
    else:
        numeric_subset['sqrt_' + col] = np.sqrt(numeric_subset[col])
        numeric_subset['log_' + col] = np.log(numeric_subset[col])

# Select the categorical columns
categorical_subset = data[['Borough', 'Largest Property Use Type']]

# One hot encode
categorical_subset = pd.get_dummies(categorical_subset)

# Join the two dataframes using concat
# Make sure to use axis = 1 to perform a column bind
features = pd.concat([numeric_subset, categorical_subset], axis=1)

# Drop buildings without an energy star score
features = features.dropna(subset=['score'])

# Find correlations with the score
correlations = features.corr()['score'].dropna().sort_values()

# Display most negative correlations
print(correlations.head(15))

# Display most positive correlations
print(correlations.tail(15))

figsize(12, 10)

# Extract the building types
features['Largest Property Use Type'] = data.dropna(
    subset=['score'])['Largest Property Use Type']

# Limit to building types with more than 100 observations (from previous code)
features = features[features['Largest Property Use Type'].isin(types)]

# Use seaborn to plot a scatterplot of Score vs Log Source EUI
sns.lmplot('Site EUI (kBtu/ft²)',
           'score',
           hue='Largest Property Use Type',
           data=features,
           scatter_kws={
               'alpha': 0.8,
               's': 60
           },
           fit_reg=False,
           size=12,
           aspect=1.2)

# Plot labeling
plt.xlabel("Site EUI", size=28)
plt.ylabel('Energy Star Score', size=28)
plt.title('Energy Star Score vs Site EUI', size=36)
plt.show()

# Extract the columns to  plot
plot_data = features[[
    'score', 'Site EUI (kBtu/ft²)', 'Weather Normalized Source EUI (kBtu/ft²)',
    'log_Total GHG Emissions (Metric Tons CO2e)'
]]

# Replace the inf with nan
plot_data = plot_data.replace({np.inf: np.nan, -np.inf: np.nan})

# Rename columns
plot_data = plot_data.rename(
    columns={
        'Site EUI (kBtu/ft²)': 'Site EUI',
        'Weather Normalized Source EUI (kBtu/ft²)': 'Weather Norm EUI',
        'log_Total GHG Emissions (Metric Tons CO2e)': 'log GHG Emissions'
    })

# Drop na values
plot_data = plot_data.dropna()

# Function to calculate correlation coefficient between two columns
def corr_func(x, y, **kargs):
    r = np.corrcoef(x, y)[0][1]
    ax = plt.gca()
    ax.annotate("r = {:.2f}".format(r),
                xy=(.2, .8),
                xycoords=ax.transAxes,
                size=20)

# Create the pairgrid object
grid = sns.PairGrid(data=plot_data, size=3)

# Upper is a scatter plot
grid.map_upper(plt.scatter, color='red', alpha=0.6)

# Diagonal is a histogram
grid.map_diag(plt.hist, color='red', edgecolor='black')

# Bottom is correlation and density plot
grid.map_lower(corr_func)
grid.map_lower(sns.kdeplot, cmap=plt.cm.Reds)

# Title for entire plot
plt.suptitle('Pairs Plot of Energy Data', size=36, y=1.02)

plt.show()

# Copy the original data
features = data.copy()

# Select the numeric columns
numeric_subset = data.select_dtypes('number')

# Create columns with log of numeric columns
for col in numeric_subset.columns:
    # Skip the Energy Star Score column
    if col == 'score':
        next
    else:
        numeric_subset['log_' + col] = np.log(numeric_subset[col])

# Select the categorical columns
categorical_subset = data[['Borough', 'Largest Property Use Type']]

# One hot encode
categorical_subset = pd.get_dummies(categorical_subset)

# Join the two dataframes using concat
# Make sure to use axis = 1 to perform a column bind
features = pd.concat([numeric_subset, categorical_subset], axis=1)

print(features.shape)

plot_data = data[[
    'Weather Normalized Site EUI (kBtu/ft²)', 'Site EUI (kBtu/ft²)'
]].dropna()

plt.plot(plot_data['Site EUI (kBtu/ft²)'],
         plot_data['Weather Normalized Site EUI (kBtu/ft²)'], 'bo')
plt.xlabel('Site EUI')
plt.ylabel('Weather Norm EUI')
plt.title('Weather Norm EUI vs Site EUI, R = %0.4f' % np.corrcoef(
    data[['Weather Normalized Site EUI (kBtu/ft²)', 'Site EUI (kBtu/ft²)'
          ]].dropna(),
    rowvar=False)[0][1])
plt.show()

def remove_collinear_features(x, threshold):
    '''
    Objective:
        Remove collinear features in a dataframe with a correlation coefficient
        greater than the threshold. Removing collinear features can help a
        model to generalize and improves the interpretability of the model.

    Inputs:
        threshold: any features with correlations greater than this value are
        removed

    Output:
        dataframe that contains only the non-highly-collinear features
    '''

    # Dont want to remove correlations between Energy Star Score
    y = x['score']
    x = x.drop(columns=['score'])

    # Calculate the correlation matrix
    corr_matrix = x.corr()
    iters = range(len(corr_matrix.columns) - 1)
    drop_cols = []

    # Iterate through the correlation matrix and compare correlations
    for i in iters:
        for j in range(i):
            item = corr_matrix.iloc[j:(j + 1), (i + 1):(i + 2)]
            col = item.columns
            row = item.index
            val = abs(item.values)

            # If correlation exceeds the threshold
            if val >= threshold:
                # Print the correlated features and the correlation value
                print(col.values[0], "|", row.values[0], "|",
                      round(val[0][0], 2))
                drop_cols.append(col.values[0])

    # Drop one of each pair of correlated columns
    drops = set(drop_cols)
    x = x.drop(columns=drops)
    x = x.drop(columns=[
        'Weather Normalized Site EUI (kBtu/ft²)',
        'Water Use (All Water Sources) (kgal)',
        'log_Water Use (All Water Sources) (kgal)',
        'Largest Property Use Type - Gross Floor Area (ft²)'
    ])

    # Add the score back in to the data
    x['score'] = y

    return x

features = remove_collinear_features(features, 0.6)

# Remove any columns with all na values
features = features.dropna(axis=1, how='all')
features.shape

# Extract the buildings with no score and the buildings with a score
no_score = features[features['score'].isna()]
score = features[features['score'].notnull()]

print(no_score.shape)
print(score.shape)

# Separate out the features and targets
features = score.drop(columns='score')
targets = pd.DataFrame(score['score'])

# Replace the inf and -inf with nan (required for later imputation)
features = features.replace({np.inf: np.nan, -np.inf: np.nan})

# Split into 70% training and 30% testing set
X, X_test, y, y_test = train_test_split(features,
                                        targets,
                                        test_size=0.3,
                                        random_state=42)

print(X.shape)
print(X_test.shape)
print(y.shape)
print(y_test.shape)

# Function to calculate mean absolute error
def mae(y_true, y_pred):
    return np.mean(abs(y_true - y_pred))

baseline_guess = np.median(y)

print('The baseline guess is a score of %0.2f' % baseline_guess)
print("Baseline Performance on the test set: MAE = %0.4f" %
      mae(y_test, baseline_guess))

# Save the no scores, training, and testing data
no_score.to_csv(WORK_DIR + '/data/no_score.csv', index=False)
X.to_csv(WORK_DIR + '/data/training_features.csv', index=False)
X_test.to_csv(WORK_DIR + '/data/testing_features.csv', index=False)
y.to_csv(WORK_DIR + '/data/training_labels.csv', index=False)
y_test.to_csv(WORK_DIR + '/data/testing_labels.csv', index=False)


text:
import numpy as np
import pandas as pa
from scipy.sparse import csr_matrix, isspmatrix
import pyensembl
from pyensembl import EnsemblRelease

def normalizeUMIWithscalefactor(data, scale_factor=10e6):
    cellNormalizedData=np.log(1+(data/np.sum(data,axis=1)[0]))*scale_factor
    return(cellNormalizedData)

def gene_Length(list_of_genes, ensembl_id=True, Ensembl_Release=75):
    gn=EnsemblRelease(Ensembl_Release)
    if ensembl_id:
        gene_pos=list(map(gn.locus_of_gene_id, list_of_genes))
        gene_pos_end=[i.end for i in gene_pos]
        gene_pos_start=[i.start for i in gene_pos]

    else:
        gene_pos=list(map(gn.loci_of_gene_names, list_of_genes))
        gene_pos_end=[i[0].end for i in gene_pos]
        gene_pos_start=[i[0].start for i in gene_pos]

    gene_len=np.array(gene_pos_end)-np.array(gene_pos_start)
    return pa.DataFrame({'gene_names':list_of_genes, "gene_length":gene_len/1000})

def getTPM(rowCountData, gene_Names=None, index_column=None, ensembol_gene=False, Ensembl_release=75):

    if(isspmatrix(rowCountData)):
        rowCountData=pa.DataFrame(rowCountData.toarray())
        rowCountData.index=gene_Names

    else:
        if index_column is not None:
            rowCountData.index=rowCountData[index_column]
            rowCountData=rowCountData.drop([index_column], axis=1)

    if ensembol_gene:
        known_genes=list(map(pyensembl.common.is_valid_ensembl_id,gene_Names))
        rowCountData=rowCountData.iloc[known_genes,:]

        rowCountData.index=np.array(gene_Names)[np.array(known_genes)]

        gene_length=gene_Length(gene_Names, ensembl_id=ensembol_gene, Ensembl_Release=Ensembl_release)
    else:

        gene_length=gene_Length(gene_Names, ensembl_id=ensembol_gene, Ensembl_Release=Ensembl_release)

    if(gene_length.shape[0]==rowCountData.shape[0]):

        count_data_RKB=rowCountData.div(list(gene_length["gene_length"]), axis=0)

        count_data_TPM=np.array(count_data_RKB)/ np.array(rowCountData.sum(axis=0)).reshape((1,len(rowCountData.sum(axis=0))))

        count_data_TPM=pa.DataFrame(count_data_TPM*10e6)
        count_data_TPM.index=list(rowCountData.index)
        count_data_TPM.columns=list(rowCountData.columns)

        count_data_TPM=count_data_TPM[(count_data_TPM.T != 0).any()].dropna(axis=0)
    else:
        print("one or more gene ids are not annoatated under", Ensembl_Release, "Please use different release")

    return count_data_TPM.T, list(count_data_TPM.index)

def ENSEMBLID_to_geneSymbol(ENSEMBL, Ensembl_Release=75):
    data=EnsemblRelease(Ensembl_Release)
    if type(ENSEMBL) is list:
        Genes=list(map(data.gene_name_of_gene_id,ENSEMBL))
    else:
        Genes=data.gene_name_of_gene_id(ENSEMBL)
    return Genes

ext: py

max_stars_repo_name: insilicolife/micti

max_issues_repo_name: insilicolife/micti

max_forks_repo_name: insilicolife/micti

reason: import numpy,from scipy


text:
"""
Example of running PyTorch implementation of DDPG on HalfCheetah.
"""
import gym

from railrl.data_management.her_replay_buffer import HerReplayBuffer
from railrl.envs.multitask.point2d import MultitaskPoint2DEnv
from railrl.envs.multitask.reacher_7dof import Reacher7DofXyzGoalState
from railrl.envs.wrappers import NormalizedBoxEnv
from railrl.state_distance.tdm_sac import TdmSac
import railrl.torch.pytorch_util as ptu
from railrl.state_distance.tdm_networks import TdmQf, TdmVf, StochasticTdmPolicy
import railrl.misc.hyperparameter as hyp
from railrl.launchers.launcher_util import run_experiment
import numpy as np

def experiment(variant):
    # env = NormalizedBoxEnv(Reacher7DofXyzGoalState())
    env = NormalizedBoxEnv(MultitaskPoint2DEnv())
    vectorized=True
    policy = StochasticTdmPolicy(
        env=env,
        **variant['policy_kwargs']
    )
    qf = TdmQf(
        env=env,
        vectorized=vectorized,
        norm_order=2,
        **variant['qf_kwargs']
    )
    vf = TdmVf(
        env=env,
        vectorized=vectorized,
        **variant['vf_kwargs']
    )
    replay_buffer_size = variant['algo_params']['base_kwargs']['replay_buffer_size']
    replay_buffer = HerReplayBuffer(replay_buffer_size, env)
    algorithm = TdmSac(
        env,
        qf,
        vf,
        variant['algo_params']['sac_kwargs'],
        variant['algo_params']['tdm_kwargs'],
        variant['algo_params']['base_kwargs'],
        supervised_weight=variant['algo_params']['supervised_weight'],
        policy=policy,
        replay_buffer=replay_buffer,
    )
    if ptu.gpu_enabled():
        algorithm.cuda()

    algorithm.train()

if __name__ == "__main__":
    # noinspection PyTypeChecker
    variant = dict(
        qf_kwargs=dict(
            hidden_sizes=[300, 300],
        ),
        policy_kwargs=dict(
            hidden_sizes=[300, 300],
        ),
        vf_kwargs=dict(
            hidden_sizes=[300, 300],
        ),
        algo_params=dict(
            base_kwargs=dict(
                num_epochs=100,
                num_steps_per_epoch=100,
                num_steps_per_eval=1000,
                max_path_length=100,
                num_updates_per_env_step=1,
                batch_size=128,
                discount=1,
                reward_scale=100,
                replay_buffer_size=1000000,
                render=True,
            ),
            tdm_kwargs=dict(
                max_tau=10,
            ),
            sac_kwargs=dict(
                soft_target_tau=0.01,
                policy_lr=3E-4,
                qf_lr=3E-4,
                vf_lr=3E-4,
            ),
        ),
    )
    search_space = {
        'algo_params.base_kwargs.reward_scale': [
            1,
            # 10,
            # 100,
        ],
        'algo_params.tdm_kwargs.max_tau': [
            1,
            # 15,
            # 20,
        ],
        'algo_params.supervised_weight':[
             # 0,
            .2,
            # .4,
            # .6,
            # .8,
        ]
    }
    sweeper = hyp.DeterministicHyperparameterSweeper(
        search_space, default_parameters=variant,
    )
    for exp_id, variant in enumerate(sweeper.iterate_hyperparameters()):
        run_experiment(
            experiment,
            seed=np.random.randint(1, 10004),
            variant=variant,
            exp_id=exp_id,
            # exp_prefix='tdm_rl_supervised_combo',
            exp_prefix='tdm_rl_supervised_combo',
            mode='local',
        )

reason: import numpy


text:
try:
    from astropy.io import fits as pyfits
except ImportError:
    import pyfits

from demo_helper import pyfits_card_fromstring

import matplotlib.pyplot as plt

import pyregion

# read in the image
def demo_header():
    cards = pyfits.CardList()
    for l in open("sample_fits02.header"):
        card = pyfits_card_fromstring(l.strip())
        cards.append(card)
    h = pyfits.Header(cards)
    return h

header = demo_header() # sample fits header
shape = (header["NAXIS1"], header["NAXIS2"])

reg_name = "test.reg"
r = pyregion.open(reg_name).as_imagecoord(header)
m = r.get_mask(shape=shape)

fig = plt.figure(1, figsize=(7,5))
ax = plt.subplot(121)
plt.imshow(m, origin="lower")

patch_list, text_list = r.get_mpl_patches_texts()
for p in patch_list:
    ax.add_patch(p)
for t in text_list:
    ax.add_artist(t)

# another region

reg_name = "test02.reg"
r = pyregion.open(reg_name).as_imagecoord(header)
m = r.get_mask(shape=shape)

ax = plt.subplot(122)
plt.imshow(m, origin="lower")

patch_list, text_list = r.get_mpl_patches_texts()
for p in patch_list:
    ax.add_patch(p)
for t in text_list:
    ax.add_artist(t)

plt.show()

max_stars_repo_name: astrofrog/pyregion

max_issues_repo_name: astrofrog/pyregion

max_forks_repo_name: astrofrog/pyregion

reason: from astropy


text:
import uuid

class Pedestrian:

  def __init__(self,bbox,label,confidence):
    self.id = str(uuid.uuid4())
    self.bbox = bbox
    self.label = label
    self.centroid = find_centroid(bbox)
    self.confidence = confidence

  def find_centroid(self,bbox):
    '''This function computes the coordinates of the center of a given bbox'''

    x_centroid = int((bbox[0]+bbox[2])/2)  #x coordinate
    y_centroid = int((bbox[1]+bbox[3])/2)  #y coordinate
    centroid = [x_centroid,y_centroid]     #list

    return centroid

  def update_ped(self,matched_bbox):
    self.bbox = matched_bbox
    self.centroid = find_centroid(matched_bbox)

class Scene:

  def __init__(self,ped_list):
    self.ped_list = ped_list
    self.unmatched_tracker = []

  def count(self):
    return len(self.ped_list), len(self.unmatched_tracker)

  def update(self,new_ped_list,new_unmatched_tracker):
    self.ped_list = new_ped_list
    self.unmatched_tracker = new_unmatched_tracker

'''
Implement and test tracker
'''
import numpy as np
from numpy import dot
from scipy.linalg import inv, block_diag

class KalmanTracker(): # class for Kalman Filter-based tracker
    def __init__(self):
        # Initialize parametes for tracker (history)
        self.id = 0  # tracker's id
        self.box = [] # list to store the coordinates for a bounding box
        self.hits = 0 # number of detection matches
        self.no_losses = 0 # number of unmatched tracks (track loss)

        # Initialize parameters for Kalman Filtering
        # The state is the (x, y) coordinates of the detection box
        # state: [up, up_dot, left, left_dot, down, down_dot, right, right_dot]
        # or[up, up_dot, left, left_dot, height, height_dot, width, width_dot]
        self.x_state=[]
        self.dt = 1.   # time interval

        # Process matrix, assuming constant velocity model
        self.F = np.array([[1, self.dt, 0,  0,  0,  0,  0, 0],
                           [0, 1,  0,  0,  0,  0,  0, 0],
                           [0, 0,  1,  self.dt, 0,  0,  0, 0],
                           [0, 0,  0,  1,  0,  0,  0, 0],
                           [0, 0,  0,  0,  1,  self.dt, 0, 0],
                           [0, 0,  0,  0,  0,  1,  0, 0],
                           [0, 0,  0,  0,  0,  0,  1, self.dt],
                           [0, 0,  0,  0,  0,  0,  0,  1]])

        # Measurement matrix, assuming we can only measure the coordinates

        self.H = np.array([[1, 0, 0, 0, 0, 0, 0, 0],
                           [0, 0, 1, 0, 0, 0, 0, 0],
                           [0, 0, 0, 0, 1, 0, 0, 0],
                           [0, 0, 0, 0, 0, 0, 1, 0]])

        # Initialize the state covariance
        self.L = 10.0
        self.P = np.diag(self.L*np.ones(8))

        # Initialize the process covariance
        self.Q_comp_mat = np.array([[self.dt**4/4., self.dt**3/2.],
                                    [self.dt**3/2., self.dt**2]])
        self.Q = block_diag(self.Q_comp_mat, self.Q_comp_mat,
                            self.Q_comp_mat, self.Q_comp_mat)

        # Initialize the measurement covariance
        self.R_scaler = 1.0
        self.R_diag_array = self.R_scaler * np.array([self.L, self.L, self.L, self.L])
        self.R = np.diag(self.R_diag_array)

    def update_R(self):
        R_diag_array = self.R_scaler * np.array([self.L, self.L, self.L, self.L])
        self.R = np.diag(R_diag_array)

    def kalman_filter(self, z):
        '''
        Implement the Kalman Filter, including the predict and the update stages,
        with the measurement z
        '''
        x = self.x_state
        # Predict
        x = dot(self.F, x)
        self.P = dot(self.F, self.P).dot(self.F.T) + self.Q

        #Update
        S = dot(self.H, self.P).dot(self.H.T) + self.R
        K = dot(self.P, self.H.T).dot(inv(S)) # Kalman gain
        y = z - dot(self.H, x) # residual
        x += dot(K, y)
        self.P = self.P - dot(K, self.H).dot(self.P)
        self.x_state = x.astype(int) # convert to integer coordinates
                                     #(pixel values)

    def predict_only(self):
        '''
        Implment only the predict stage. This is used for unmatched detections and
        unmatched tracks
        '''
        x = self.x_state
        # Predict
        x = dot(self.F, x)
        self.P = dot(self.F, self.P).dot(self.F.T) + self.Q
        self.x_state = x.astype(int)

reason: import numpy,from numpy,from scipy


text:
import logging
import numpy as np
import openpnm
from openpnm.utils import SettingsAttr

logger = logging.getLogger(__name__)

__all__ = ['Workspace']

class WorkspaceSettings(SettingsAttr):
    r"""
    loglevel : int
        Sets the threshold for the severity of logger message which appear.
        Ranges are as follows:

        ======= ==============================================================
        Level   Description
        ======= ==============================================================
        10      DEBUG: Detailed information, typically of interest only when
                diagnosing problems.
        20      INFO: Confirmation that things are working as expected.
        30      WARNING: An indication that something unexpected happened, or
                indicative of some problem in the near future (e.g. ‘disk
                space low’). The software is still working as expected.
        40      ERROR: Due to a more serious problem, the software has not
                been able to perform some function.
        50      CRITICAL: A serious error, indicating that the program itself
                may be unable to continue running.
        ======= ==============================================================
    """
    default_solver = 'PardisoSpsolve'

    @property
    def loglevel(self):
        logger = logging.getLogger()
        return logger.level

    @loglevel.setter
    def loglevel(self, value):
        logger = logging.getLogger()
        logger.setLevel(value)

class Workspace(dict):
    r"""
    The Workspace object provides the highest level of adminstrative
    control over active OpenPNM sessions.

    It is a ``dictionary`` that stores a list of all open Projects by name.

    This class is a
    `singleton <https://en.wikipedia.org/wiki/Singleton_pattern>`_ so that
    whenever and wherever a Workspace is instantiated, the same instance is
    obtained. This allows it to maintain a definitive record of all open
    Projects.

    See Also
    --------
    Project

    Notes
    -----
    The Workspace object contains a variety of functions that one might
    expect from the 'file-menu' in a typical GUI.

    """

    __instance__ = None

    def __new__(cls, *args, **kwargs):
        if Workspace.__instance__ is None:
            Workspace.__instance__ = dict.__new__(cls)
        return Workspace.__instance__

    def __init__(self):
        super().__init__()
        self._projects = {}
        self.settings = WorkspaceSettings()
        self.settings.loglevel = 30

    def __setitem__(self, name, project):
        if name is None:
            name = self._gen_name()
        if name in self.keys():
            # To ensure renaming a Project to its current name is allowed
            if self.__instance__[name] is not project:
                raise Exception(f"A project named {name} already exists")
        if project in self.values():
            self.pop(project.name, None)
        if not isinstance(project, openpnm.utils.Project):
            project = openpnm.utils.Project(project, name=name)
        super().__setitem__(name, project)

    def copy(self):
        """Brief explanation of 'copy'"""
        raise Exception('Cannot copy Workspace, only one can exist at a time')

    def _create_console_handles(self, project):
        r"""
        Adds all objects in the given project to the console as variables
        with handle names taken from each object's name.
        """
        import __main__
        for item in project:
            __main__.__dict__[item.name] = item

    @property
    def version(self):
        return openpnm.__version__

    def save_workspace(self, filename=None):
        r"""
        Saves all projects in the current workspace as a single file

        Parameters
        ----------
        filename : str
            The filename to use when saving.  If not provided, the present
            date and time are used.

        Notes
        -----
        The file is actually zip archive containing ``pnm`` files, one for
        each project in the workspace. This archive can be extracted and each
        ``pnm`` file can be loaded manually using ``load_project`` or the
        ``openpnm.io.PNM`` class.

        """
        from datetime import datetime
        if filename is None:
            dt = datetime.now()
            filename = dt.strftime("%Y_%m_%d_%H_%M_%S")
        from zipfile import ZipFile
        with ZipFile(filename + '.wrk', 'w') as z:
            for prj in self.values():
                prj.save_project()
                z.write(prj.name + '.pnm')

    def load_workspace(self, filename):
        r"""
        Loads project(s) from a saved workspace into current workspace

        Parameters
        ----------
        filename : str or Path
            The filename containing the saved workspace

        Notes
        -----
        ??

        """
        from zipfile import ZipFile
        with ZipFile(filename, 'r') as z:
            logger.info('Loading projects contained in ' + filename)
            files = z.filelist
            for f in files:
                self.load_project(f.orig_filename)

    def save_project(self, project, filename=None):
        r"""
        Saves given Project to a ``pnm`` file

        This will include all of associated objects, including algorithms.

        Parameters
        ----------
        project : Project
            The project to save.
        filename : str, optional
            If no filename is given, the given project name is used. See Notes
            for more information.

        See Also
        --------
        save_workspace

        Notes
        -----
        The filename can be a string such as 'saved_file.pnm'.  The string can
        include absolute path such as 'C:\networks\saved_file.pnm', or can
        be a relative path such as '..\..\saved_file.pnm', which will look
        2 directories above the current working directory.  Can also be a
        path object object such as that produced by ``pathlib`` or
        ``os.path`` in the Python standard library.

        """
        from openpnm.io import PNM
        PNM.save_project(project=project, filename=filename)

    def load_project(self, filename, overwrite=False):
        r"""
        Loads a Project from the specified 'pnm' file

        The loaded project is added to the Workspace. This will *not* delete
        any existing Projects in the Workspace and will rename any Projects
        being loaded if necessary.

        Parameters
        ----------
        filename : str or Path
            The name of the file to open. See Notes for more information.
        overwrite : bool
            Brief explanation of 'overwrite'

        See Also
        --------
        load_workspace

        Notes
        -----
        The filename can be a string such as 'saved_file.pnm'. The string can
        include absolute path such as 'C:\networks\saved_file.pnm', or can
        be a relative path such as '..\..\saved_file.pnm', which will look
        2 directories above the current working directory.  Can also be a
        path object object such as that produced by ``pathlib`` or
        ``os.path`` in the Python standard library.

        """
        from openpnm.io import PNM
        proj = PNM.load_project(filename=filename)
        return proj

    def close_project(self, project):
        r"""
        Removes the specified Project from the Workspace

        This does not save the project, so any changes will be lost.

        Parameters
        ----------
        project : Project
            The Project object to be copied

        """
        del self[project.name]

    def copy_project(self, project, name=None):
        r"""
        Makes a copy of an existing Project

        Parameters
        ----------
        project : Project
            The Project object to be copied
        name : str, optional
            A name for the new copy of the project. If not supplied, then
            one will be generated (e.g. 'proj_02')

        Returns
        -------
        proj : list
            A handle to the new Project

        """
        proj = project.copy(name)
        return proj

    def new_project(self, name=None):
        r"""
        Creates a new empty Project object

        Parameters
        ----------
        name : str, optional
            The unique name to give to the project. If none is given, one
            will be automatically generated (e.g. 'proj_01`)

        Returns
        -------
        proj : list
            An empty Project object, suitable for passing into a Network
            generator

        """
        proj = openpnm.utils.Project(name=name)
        return proj

    def _gen_name(self):
        r"""
        Generates a valid name for projects
        """
        n = [0]
        for item in self.keys():
            if item.startswith('proj_'):
                n.append(int(item.split('proj_')[1]))
        name = 'proj_'+str(max(n)+1).zfill(2)
        return name

    def _gen_ids(self, size):
        r"""
        Generates a sequence of integers of the given ``size``, starting at 1
        greater than the last produced value.

        The Workspace object keeps track of the most recent value, which
        persists until the current python session is restarted, so the
        returned array contains unique values for the given session.

        Parameters
        ----------
        size : int
            The number of values to generate.

        Returns
        -------
        A Numpy array of the specified size, containing integer values starting
        from the last used values.

        Notes
        -----
        When a new Workspace is created the

        """
        if not hasattr(self, '_next_id'):
            # If _next_id has not been set, then assign it
            self._next_id = 0
            # But check ids in any objects present first
            for proj in self.values():
                if len(proj) > 0:
                    if 'pore._id' in proj.network.keys():
                        Pmax = proj.network['pore._id'].max() + 1
                        Tmax = proj.network['throat._id'].max() + 1
                        self._next_id = max([Pmax, Tmax, self._next_id])
        ids = np.arange(self._next_id, self._next_id + size, dtype=np.int64)
        self._next_id += size
        return ids

    def __str__(self):
        s = []
        hr = '―'*78
        s.append(hr)
        s.append('OpenPNM Version ' + openpnm.__version__ + ' Workspace')
        s.append(hr)
        for item in self.values():
            s.append(' ' + item.name)
            s.append(item.__str__())
        return '\n'.join(s)

max_stars_repo_name: xu-kai-xu/OpenPNM

max_issues_repo_name: xu-kai-xu/OpenPNM

max_forks_repo_name: xu-kai-xu/OpenPNM


text:
#
# simple program to read E,S(E) and calculate mean and variance of energy with
# various filers. Input parameters are energy and take-off angle, the filter material
# name density and thickness and the thickness of th CsI detector.
# e.g
# pythom meanrms.py  50 1 Sn 7.3 0.4 0.3
# for 50KeV beam at 1deg take off through Sn filter of 7.3 density and .4 width (cgs)
# with a .3 thickness of CsI. At the moment an additional 1mm of Al is used as a pre
# filter to prove that beam softening is possible under some conditions.
# This program must be run in the "test" directory of the bhc code to ensure it
# has access to the spectra data for W and the xcom data for attenuation.
#
import os
import sys
import numpy as np

sums=0.
sume=0.
sume2=0.
# default if nothing specified on cmd line, but other parts will fail.
filename="spectra/W/070220.spc"
if len(sys.argv)>2:
    if int(sys.argv[2])>9:
        if int(sys.argv[1])<100:
            filename="spectra/W/0"+str(sys.argv[1])+str(sys.argv[2])+"0.spc"
        else:
            filename="spectra/W/"+str(sys.argv[1])+str(sys.argv[2])+"0.spc"
    else:
        if int(sys.argv[1])<100:
            filename="spectra/W/0"+str(sys.argv[1])+"0"+str(sys.argv[2])+"0.spc"
        else:
            filename="spectra/W/"+str(sys.argv[1])+"0"+str(sys.argv[2])+"0.spc"

print "filename=",filename
if not os.path.isfile(filename):
    print "file not found: ", filename, " in specData"
else:
    with open(filename, 'r') as fl:
        en, amp = np.loadtxt(fl, unpack=True)
if len(sys.argv)>3:
    filename="xcom/"+sys.argv[3]+".txt"
    density=float(sys.argv[4])
    width=float(sys.argv[5])
    if not os.path.isfile(filename):
        print "file not found: ", filename, " in xcom"
    else:
        with open(filename, 'r') as fl:
            en2, mu = np.loadtxt(fl, unpack=True)
        slen=len(en)
        mut=mu[0:slen]*density*width

if True:
    # add Al filter
    filename="xcom/Al.txt"
    density=2.64
    width=2.0
    if not os.path.isfile(filename):
        print "file not found: ", filename, " in xcom"
    else:
        with open(filename, 'r') as fl:
            ena, mua = np.loadtxt(fl, unpack=True)
        slen=len(en)
        muat=mua[0:slen]*density*width
    print("applying AL pre filter at "+str(width)+"cm")
    amp=amp*np.exp(-muat)

filename="xcom/CsI.txt"
if not os.path.isfile(filename):
    print "file not found: ", filename, " in xcom"
else:
    with open(filename, 'r') as fl:
        en3, mucsi = np.loadtxt(fl, unpack=True)
    slen=len(en)
    density=4.51
    if len(sys.argv)>5:
        width=float(sys.argv[6])
    else:
        width=0.02
    print "width=",str(width)
    mucsit=mucsi[0:slen]*density*width
sumAmp = sum(amp)
sumE = sum(en*amp)
sumE2 = sum(en*en*amp)
meanE = sumE/sumAmp
var2 = sumE2/sumAmp-meanE*meanE
print "file = ",filename
print "mean E = ",str(meanE)
print "var = ",str(var2)
print "RMS = ",str(np.sqrt(var2))

ampF=amp*np.exp(-mut)
sumAmpF = sum(ampF)
sumEF = sum(en*ampF)
sumEF2 = sum(en*en*ampF)
meanEF = sumEF/sumAmpF
varF2 = sumEF2/sumAmpF-meanEF*meanEF
print "mean E F = ",str(meanEF)
print "var F = ",str(varF2)
print "RMS F = ",str(np.sqrt(varF2))
print "ratio of Amps = ",str(sumAmp/sumAmpF)

ampR = ampF*en*(1-np.exp(-mucsit))
sumAmpR = sum(ampR)
sumER = sum(en*ampR)
sumER2 = sum(en*en*ampR)
meanER= sumER/sumAmpR
varR2 = sumER2/sumAmpR-meanER*meanER
print "mean E R = ",str(meanER)
print "var F = ",str(varR2)
print "RMS F = ",str(np.sqrt(varR2))
print "ratio of Amps = ",str(sumAmp/sumAmpR)

max_stars_repo_name: TomasKulhanek/CCPi-PreProcessing

max_issues_repo_name: TomasKulhanek/CCPi-PreProcessing

max_forks_repo_name: TomasKulhanek/CCPi-PreProcessing


text:
'''
Cross-compare blobs with incremental adjacency, within a frame
'''

from class_cluster import ClusterStructure, NoneType
import numpy as np
import cv2

class CderBlob(ClusterStructure):

    _blob = object
    mB = int
    dB = int
    dI = int
    mI = int
    dA = int
    mA = int
    dG = int
    mG = int
    dM = int
    mM = int

class CBblob(ClusterStructure):

    DerBlob = object
    blob_ = list

ave_mB = 20000  # ave can't be negative
ave_rM = .7  # average relative match at rL=1: rate of ave_mB decay with relative distance, due to correlation between proximity and similarity

def cross_comp_blobs(frame):
    '''
    root function of comp_blob: cross compare blobs with their adjacent blobs in frame.blob_, including sub_layers
    '''
    blob_ = frame.blob_

    for blob in blob_:  # each blob forms derBlob per compared adj_blob and accumulates adj_blobs'derBlobs:
        blob.DerBlob = CderBlob()
        comp_blob_recursive(blob, blob.adj_blobs[0], derBlob_=[], derBlob_id_=[])
        # derBlob_ and derBlob_id_ are local and frame-wide

    bblob_ = form_bblob_(blob_)  # form blobs of blobs, connected by mutual match

    visualize_cluster_(bblob_, blob_, frame)

    return bblob_

def comp_blob_recursive(blob, adj_blob_, derBlob_, derBlob_id_):
    '''
    called by cross_comp_blob to recursively compare blob to adj_blobs in incremental layers of adjacency
    '''
    for adj_blob in adj_blob_:

        id = blob.id; _id = adj_blob.id  # pairing function generates unique number from each comparand_pair, frame-wide, same for derBlob_:
        derBlob_id = 0.5 * (id+_id) * (id+_id+1) + _id  # 0.5*(a + b)(a + b + 1) + b: https://en.wikipedia.org/wiki/Pairing_function
        _derBlob_id = -derBlob_id     # different-order a and b yields different results, so we need both
        fderBlob = 0

        # existing derBlob
        if derBlob_id in derBlob_id_ :
            derBlob = derBlob_[derBlob_id_.index(derBlob_id)]
            accum_derBlob(blob, derBlob)  # also adj_blob.rdn += 1?
            fderBlob = 1
        elif _derBlob_id in derBlob_id_:
            derBlob = derBlob_[derBlob_id_.index(_derBlob_id)]
            accum_derBlob(blob, derBlob)  # also adj_blob.rdn += 1?
            fderBlob = 1

        # new derBlob
        elif adj_blob is not blob:  # adj of adj blob could be the blob itself
            derBlob = comp_blob(blob, adj_blob)  # compare blob and adjacent blob
            accum_derBlob(blob, derBlob)  # from all compared blobs, regardless of mB sign
            derBlob_id_.append(derBlob_id)  # unique comparand_pair identifier
            derBlob_.append(derBlob)  # also frame-wide
            fderBlob = 1

        if fderBlob: # derBlob is not exist if feval = 0
            if derBlob.mB > 0:
                # replace blob with adj_blob for continuing adjacency search:
                if not isinstance(adj_blob.DerBlob, CderBlob):  # if adj_blob.DerBlob: it's already searched in previous call
                    # but this search could be to a different depth?
                    adj_blob.DerBlob = CderBlob()
                    comp_blob_recursive(adj_blob, adj_blob.adj_blobs[0], derBlob_, derBlob_id_)

            elif blob.Dert.M + blob.neg_mB+ derBlob.mB > ave_mB:  # neg mB but positive comb M,
                # extend blob comparison to adjacents of adjacent, depth-first
                blob.neg_mB += derBlob.mB  # mB and distance are accumulated over comparison scope
                blob.distance += np.sqrt(adj_blob.A)
                comp_blob_recursive(blob, adj_blob.adj_blobs[0], derBlob_, derBlob_id_)

def comp_blob(blob, _blob):
    '''
    cross compare _blob and blob
    '''
    (_I, _Dy, _Dx, _G, _M, _Dyy, _Dyx, _Dxy, _Dxx, _Ga, _Ma, _Mdx, _Ddx), _A = _blob.Dert.unpack(), _blob.A
    (I, Dy, Dx, G, M, Dyy, Dyx, Dxy, Dxx, Ga, Ma, Mdx, Ddx), A = blob.Dert.unpack(), blob.A

    dI = _I - I  # d is always signed
    mI = min(_I, I)
    dA = _A - A
    mA = min(_A, A)
    dG = _G - G
    mG = min(_G, G)
    dM = _M - M
    mM = min(_M, M)

    mB = mI + mA + mG + mM - ave_mB * (ave_rM ** ((1+blob.distance) / np.sqrt(A)))
    # deviation from average blob match at current distance
    dB = dI + dA + dG + dM

    derBlob  = CderBlob(_blob=_blob, mB=mB, dB=dB)  # blob is core node, _blob is adjacent blob

    if _blob.fsliced and blob.fsliced:
        pass

    return derBlob

def form_bblob_(blob_):
    '''
    form blob of blobs as a cluster of blobs with positive adjacent derBlob_s, formed by comparing adj_blobs
    '''
    checked_ids = []  # checked for clustering, frame-wide vs. per blob
    bblob_ = []

    for blob in blob_:
        if blob.DerBlob.mB > 0 and blob.id not in checked_ids:  # init bblob with current blob
            checked_ids.append(blob.id)

            bblob = CBblob(DerBlob=CderBlob())
            accum_bblob(bblob, blob)
            form_bblob_recursive(bblob_, bblob, checked_ids)

    return bblob_

def form_bblob_recursive(bblob_, bblob, checked_ids):

    fsearch = 0  # there are blobs to check for inclusion in bblob

    for blob in bblob.blob_: # search blob' derBlob's blobs to get potential border clustering blob
        if (blob.DerBlob.mB > 0) and (blob.id not in checked_ids):  # positive mB
            for derBlob in blob.derBlob_:
                # if blob is in bblob.blob_, but derBlob._blob is not in bblob_blob_
                # so if sum of derBlob._blob's mB with blob's mB > 0 , pack the derBlob._blob into bblob.blob_
                if (derBlob._blob not in bblob.blob_) and (derBlob._blob.DerBlob.mB + blob.DerBlob.mB >0):
                    accum_bblob(bblob, derBlob._blob)
                    checked_ids.append(blob.id)
                    fsearch = 1

    if fsearch:
        form_bblob_recursive(bblob_, bblob, checked_ids)

    bblob_.append(bblob) # pack bblob after scanning all possible derBlobs

def accum_derBlob(blob, derBlob):

    blob.DerBlob.accumulate(**{param:getattr(derBlob, param) for param in blob.DerBlob.numeric_params})
    blob.derBlob_.append(derBlob)

def accum_bblob(bblob, blob):

    # accumulate derBlob
    bblob.DerBlob.accumulate(**{param:getattr(blob.DerBlob, param) for param in bblob.DerBlob.numeric_params})

    if blob not in bblob.blob_:
    # isn't that checked before calling accum_bblob?
        bblob.blob_.append(blob) # pack blob into bblob.blob_

    for derBlob in blob.derBlob_: # pack adjacent blobs of blob into bblob
        if derBlob._blob not in bblob.blob_:
            bblob.blob_.append(derBlob._blob)

'''
    cross-comp among sub_blobs in nested sub_layers:
    _sub_layer = bblob.sub_layer[0]
    for sub_layer in bblob.sub_layer[1:]:
        for _sub_blob in _sub_layer:
            for sub_blob in sub_layer:
                comp_blob(_sub_blob, sub_blob)
        merge(_sub_layer, sub_layer)  # only for sub-blobs not combined into new bblobs by cross-comp
'''

def visualize_cluster_(bblob_, blob_, frame):

    colour_list = []  # list of colours:
    colour_list.append([200, 130, 1])  # blue
    colour_list.append([75, 25, 230])  # red
    colour_list.append([25, 255, 255])  # yellow
    colour_list.append([75, 180, 60])  # green
    colour_list.append([212, 190, 250])  # pink
    colour_list.append([240, 250, 70])  # cyan
    colour_list.append([48, 130, 245])  # orange
    colour_list.append([180, 30, 145])  # purple
    colour_list.append([40, 110, 175])  # brown
    colour_list.append([192, 192, 192])  # silver
    colour_list.append([255, 255, 0])  # blue2
    colour_list.append([34, 34, 178])  # red2
    colour_list.append([0, 215, 255])  # yellow2
    colour_list.append([50, 205, 50])  # green2
    colour_list.append([114, 128, 250])  # pink2
    colour_list.append([255, 255, 224])  # cyan2
    colour_list.append([0, 140, 255])  # orange2
    colour_list.append([204, 50, 153])  # purple2
    colour_list.append([63, 133, 205])  # brown2
    colour_list.append([128, 128, 128])  # silver2

    # initialization
    ysize, xsize = frame.dert__[0].shape
    blob_img = np.zeros((ysize, xsize,3)).astype('uint8')
    cluster_img = np.zeros((ysize, xsize,3)).astype('uint8')
    img_separator = np.zeros((ysize,3,3)).astype('uint8')

    # create mask
    blob_mask = np.zeros_like(frame.dert__[0]).astype('uint8')
    cluster_mask = np.zeros_like(frame.dert__[0]).astype('uint8')

    blob_colour_index = 1
    cluster_colour_index = 1

    cv2.namedWindow('blobs & clusters', cv2.WINDOW_NORMAL)

    # draw blobs
    for blob in blob_:
        cy0, cyn, cx0, cxn = blob.box
        blob_mask[cy0:cyn,cx0:cxn] += (~blob.mask__ * blob_colour_index).astype('uint8')
        blob_colour_index += 1

    # draw blob cluster of bblob
    for bblob in bblob_:
        for blob in bblob.blob_:
            cy0, cyn, cx0, cxn = blob.box
            blob_colour_index += 1
            cluster_mask[cy0:cyn,cx0:cxn] += (~blob.mask__ *cluster_colour_index).astype('uint8')

        # increase colour index
        cluster_colour_index += 1

        # insert colour of blobs
        for i in range(1,blob_colour_index):
            blob_img[np.where(blob_mask == i)] = colour_list[i % 10]

        # insert colour of clusters
        for i in range(1,cluster_colour_index):
            cluster_img[np.where(cluster_mask == i)] = colour_list[i % 10]

        # combine images for visualization
        img_concat = np.concatenate((blob_img, img_separator,
                                    cluster_img, img_separator), axis=1)

        # plot cluster of blob
        cv2.imshow('blobs & clusters',img_concat)
        cv2.resizeWindow('blobs & clusters', 1920, 720)
        cv2.waitKey(10)

    cv2.waitKey(0)
    cv2.destroyAllWindows()


text:
# -*- coding: utf-8 -*-
"""
Created on Mon Feb 18 02:43:27 2019

@author: z
"""

import numpy as np
import scipy.stats as ss
import matplotlib.pyplot as plt

from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split

h = 1
sd = 1
n = 50
n_n = 1000
def gen_data(n, h, sd1, sd2):
    """ xxx """
    x1 = ss.norm.rvs(-h, sd1, n)
    y1 = ss.norm.rvs(0, sd1, n)

    x2 = ss.norm.rvs(h, sd2, n)
    y2 = ss.norm.rvs(0, sd2, n)
    return (x1, y1, x2, y2)

def plot_data(x1, y1, x2, y2):
    plt.figure()
    plt.plot(x1, y1, "o", ms=2)
    plt.plot(x2, y2, "o", ms=2)
    plt.xlabel("$X_1$")
    plt.ylabel("$X_2$")

def plot_probs(ax, clf, class_no):
    xx1, xx2 = np.meshgrid(np.arange(-5, 5, 0.1), np.arange(-5, 5, 0.1))
    probs = clf.predict_proba(np.stack((xx1.ravel(), xx2.ravel()), axis=1))
    Z = probs[:,class_no]
    Z = Z.reshape(xx1.shape)
    CS = ax.contourf(xx1, xx2, Z)
    cbar = plt.colorbar(CS)
    plt.xlabel("$X_1$")
    plt.ylabel("$X_2$")

(x1, y1, x2, y2) = gen_data(n_n, 1.5, 1, 1.5)
X = np.vstack((np.vstack((x1,y1)).T, np.vstack((x2,y2)).T))
y = np.hstack((np.repeat(1, n_n), np.repeat(2, n_n)))
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.5, random_state=1)

clf = LogisticRegression()
clf.fit(X_train, y_train)
print("测试数据：",clf.score(X_test, y_test))
print("单点分类概率：",clf.predict_proba(np.array([-2, 0]).reshape(1, -1)));
print("单点分类：",clf.predict(np.array([-2, 0]).reshape(1, -1)));

plt.figure(figsize=(5,8))
ax = plt.subplot(211)
plot_probs(ax, clf, 0)
plt.title("Pred. prob for class 1")
ax = plt.subplot(212)
plot_probs(ax, clf, 1)
plt.title("Pred. prob for class 2");
plot_data(x1, y1, x2, y2)

max_stars_repo_name: linghtiin/test

max_issues_repo_name: linghtiin/test

max_forks_repo_name: linghtiin/test

reason: import numpy,import scipy


text:
import numpy as np
from typing import Tuple

import plotly.graph_objects

from IMLearn.metalearners.adaboost import AdaBoost
from IMLearn.learners.classifiers import DecisionStump
from utils import *
import plotly.graph_objects as go
from plotly.subplots import make_subplots

def generate_data(n: int, noise_ratio: float) -> Tuple[np.ndarray, np.ndarray]:
    """
    Generate a dataset in R^2 of specified size

    Parameters
    ----------
    n: int
        Number of samples to generate

    noise_ratio: float
        Ratio of labels to invert

    Returns
    -------
    X: np.ndarray of shape (n_samples,2)
        Design matrix of samples

    y: np.ndarray of shape (n_samples,)
        Labels of samples
    """
    '''
    generate samples X with shape: (num_samples, 2) and labels y with shape (num_samples).
    num_samples: the number of samples to generate
    noise_ratio: invert the label for this ratio of the samples
    '''
    X, y = np.random.rand(n, 2) * 2 - 1, np.ones(n)
    y[np.sum(X ** 2, axis=1) < 0.5 ** 2] = -1
    y[np.random.choice(n, int(noise_ratio * n))] *= -1
    return X, y

def return_title(text: str):
    '''
    Returns title with global config with given text
    '''
    return {
        'text': text,
        'x': 0.5,
        'y': 0.99,
        'font': {'size': 35}
    }

def get_marker_shapes_list(test_y: np.ndarray):
    '''
    Helper function for styling markers in scatter plots
    '''
    return ['circle' if test_y[i] == -1 else 'x' for i in range(test_y.shape[0])]

def Q1_plot(training_losses: [int], testing_losses: [int]):
    '''
    Plots the classification loss as a function on adaboost ensemble size
    '''
    fig = go.Figure()

    # Adding training and testing losses
    fig.add_trace(go.Scatter(x=[i for i in range(1, 251)], y=training_losses, name="train errors"))
    fig.add_trace(go.Scatter(x=[i for i in range(1, 251)], y=testing_losses, name="test errors"))

    fig.update_layout(
        title=return_title("Classification loss as a function of ensemble size"),
        xaxis_title="Adaboost iterations",
        yaxis_title="Classification Error for iteration"
    )
    fig.show()

def Q2_plot(adaboost_classifier: AdaBoost, test_X: np.ndarray, test_y: np.ndarray, lims: [float]):
    """
    Creates subplots of the dataset and decision boundaries for a list of adaboost ensembles of increasing size
    """
    T = [5, 50, 100, 250]
    traces = [(1, 1), (1, 2), (2, 1), (2, 2)] # Used to place the subplots

    fig = make_subplots(rows=2, cols=2, subplot_titles=[f"Decision boundary for {t}" for t in T])

    for i in range(4):
        fig.add_trace(
            go.Scatter(
                x=test_X[:, 0],
                y=test_X[:, 1],
                mode='markers',
                marker=dict(
                    size=15,
                    color=test_y,
                    symbol=get_marker_shapes_list(test_y),
                    line_width=2,
                    line_color='black'
                )
            ), col=traces[i][0], row=traces[i][1])

        partial_predict = lambda test_data: adaboost_classifier.partial_predict(test_data, T[i])

        fig.add_trace(decision_surface(partial_predict, lims[0], lims[1], showscale=False),
                      col=traces[i][0], row=traces[i][1])

    fig.update_layout(title=return_title('Decision Boundaries of Adaboost ensembles'),showlegend=False)
    fig.show()

def Q3_plot(adaboost_classifier: AdaBoost, test_X: np.ndarray, testing_losses: [float], test_y: np.ndarray,
            lims: [int]):
    '''
    PLots the dataset and decision boundary for the best performing ensemble
    '''
    best_performing_ensemble = np.argmin(testing_losses)
    accuracy = 1 - testing_losses[best_performing_ensemble]
    fig = go.Figure()
    fig.add_trace(go.Scatter(
        x=test_X[:, 0],
        y=test_X[:, 1],
        mode='markers',
        marker=dict(
            size=15,
            color=test_y,
            symbol=get_marker_shapes_list(test_y),
            line_width=2,
            line_color='black'
        )
    ))
    partial_predict = lambda test_data: adaboost_classifier.partial_predict(test_data, 238)
    fig.add_trace(decision_surface(partial_predict, lims[0], lims[1], showscale=False))
    fig.update_layout(
        title=return_title(f'Decision boundary ensemble of size 238 '
                           f'with accuracy {accuracy}')
    )
    fig.show()

def Q4_plot(adaboost_classifier: AdaBoost, train_X: np.ndarray, train_y: np.ndarray, lims: [int]):
    '''
    Plot of weighted training samples by their respective weights in the last iteration of the adaboost
    algorithm
    '''
    max_D_t = np.max(adaboost_classifier.D)

    # I have chosen to multiply by 20 to increase graph visibility whilst still maintaining the marker
    # sizes proportional to their weight in the final iteration
    normalized_list = (adaboost_classifier.D / max_D_t) * 20

    fig = go.Figure()
    fig.add_trace(
        go.Scatter(
            x=train_X[:, 0],
            y=train_X[:, 1],
            mode='markers',
            marker=dict(
                size=normalized_list,
                color=train_y,
                symbol=get_marker_shapes_list(train_y),
                line_width=5,
                line_color='black'
            )
        )
    )
    fig.add_trace(decision_surface(adaboost_classifier.predict, lims[0], lims[1], showscale=False))
    fig.update_layout(
        title=return_title('Plot with weighted training samples')
    )
    fig.show()

def fit_and_evaluate_adaboost(noise, n_learners=250, train_size=5000, test_size=500):
    (train_X, train_y), (test_X, test_y) = generate_data(train_size, noise), generate_data(test_size, noise)

    adaboost_classifier = AdaBoost(DecisionStump, n_learners)
    adaboost_classifier.fit(train_X, train_y)

    training_losses = []
    # Training
    for i in range(adaboost_classifier.iterations_):
        training_losses.append(adaboost_classifier.partial_loss(train_X, train_y, i))

    # Testing
    testing_losses = []
    for i in range(adaboost_classifier.iterations_):
        testing_losses.append(adaboost_classifier.partial_loss(test_X, test_y, i))

    lims = np.array([np.r_[train_X, test_X].min(axis=0), np.r_[train_X, test_X].max(axis=0)]).T + np.array(
        [-.1, .1])

    # Question 1: Train- and test errors of AdaBoost in noiseless case
    Q1_plot(training_losses, testing_losses)

    # Question 2: Plotting decision surfaces
    if noise == 0:
        Q2_plot(adaboost_classifier, test_X, test_y, lims)

    # Question 3: Decision surface of best performing ensemble
    if noise == 0:
        Q3_plot(adaboost_classifier, test_X, testing_losses, test_y, lims)

    # Question 4: Decision surface with weighted samples
    Q4_plot(adaboost_classifier, train_X, train_y, lims)

if __name__ == '__main__':
    np.random.seed(0)
    fit_and_evaluate_adaboost(0)
    fit_and_evaluate_adaboost(0.4)


text:
# -*- coding: utf-8 -*-
import pytest
import os
import numpy
from numpy.testing import assert_almost_equal
from planetaryimage.pds3image import PDS3Image, Pointer
from pvl import Units

DATA_DIR = os.path.join(os.path.dirname(__file__), 'data/')
filename = os.path.join(DATA_DIR, 'pds3_1band.IMG')
filename_3bands = os.path.join(DATA_DIR, 'pds3_3bands.IMG')
filename_float = os.path.join(DATA_DIR, 'pds3_1band_float.IMG')
gzipped_filename = os.path.join(DATA_DIR, 'pds3_1band.IMG.gz')
bz2_filename = os.path.join(DATA_DIR, 'pds3_1band.IMG.bz2')

@pytest.fixture
def expected():
    return numpy.loadtxt(os.path.join(DATA_DIR, 'pds3_1band.txt')).reshape(
        (1, 10, 10)
    )

def test_pds3_1band_labels(expected):
    image = PDS3Image.open(filename)

    assert image.filename == filename
    assert image.bands == 1
    assert image.lines == 10
    assert image.samples == 10
    assert image.format == 'BAND_SEQUENTIAL'
    assert image.dtype == numpy.dtype('>i2')
    assert image.start_byte == 640
    assert image.shape == (1, 10, 10)
    # FIXME: Doublecheck that consolidating pixel_type and byte order
    #        is actually OK for PDS images.  I think here at the object level
    #        its OK even though in the PDS labels the information is separate.
    assert image.size == 100
    assert image.compression is None

    # Testing .label
    assert image.label['FILE_RECORDS'] == 42
    assert image.label['IMAGE']['SAMPLE_TYPE'] == 'MSB_INTEGER'

    # Testing .data
    assert image.data.shape == (1, 10, 10)
    assert image.data.dtype == numpy.dtype('>i2')
    assert_almost_equal(image.data, expected)

def test_gz_pds3_1band_labels(expected):
    image = PDS3Image.open(gzipped_filename)
    assert image.filename == gzipped_filename
    assert image.bands == 1
    assert image.lines == 10
    assert image.samples == 10
    assert image.format == 'BAND_SEQUENTIAL'
    assert image.dtype == numpy.dtype('>i2')
    assert image.start_byte == 640
    assert image.shape == (1, 10, 10)
    assert image.size == 100
    assert image.compression == 'gz'

    # Testing .label
    assert image.label['FILE_RECORDS'] == 42
    assert image.label['IMAGE']['SAMPLE_TYPE'] == 'MSB_INTEGER'

    # Testing .data
    assert image.data.shape == (1, 10, 10)
    assert image.data.dtype == numpy.dtype('>i2')
    assert_almost_equal(image.data, expected)

def test_image_save():
    image = PDS3Image.open(filename)
    image.save('Temp_Image.IMG')
    new_image = PDS3Image.open('Temp_Image.IMG')
    assert image.bands == new_image.bands
    assert image.lines == new_image.lines
    assert image.samples == new_image.samples
    assert image.format == new_image.format
    assert image.dtype == new_image.dtype
    assert image.start_byte == new_image.start_byte
    assert image.shape == new_image.shape
    assert image.size == new_image.size

    # Testing .label
    assert image.label['FILE_RECORDS'] == new_image.label['FILE_RECORDS']
    label_sample_type = image.label['IMAGE']['SAMPLE_TYPE']
    assert label_sample_type == new_image.label['IMAGE']['SAMPLE_TYPE']

    # Testing .data
    assert image.data.shape == new_image.data.shape
    assert image.data.dtype == image.data.dtype
    os.remove('Temp_Image.IMG')

def test_image_save_overwrite():
    image = PDS3Image.open(filename)
    image.save('Temp_Image.IMG')
    image_temp = PDS3Image.open('Temp_Image.IMG')
    image_temp.save(overwrite=True)
    new_image = PDS3Image.open(image_temp.filename)
    assert image_temp.filename == new_image.filename
    assert image_temp.bands == new_image.bands
    assert image_temp.lines == new_image.lines
    assert image_temp.samples == new_image.samples
    assert image_temp.format == new_image.format
    assert image_temp.dtype == new_image.dtype
    assert image_temp.start_byte == new_image.start_byte
    assert image_temp.shape == new_image.shape
    assert image_temp.size == new_image.size

    # Testing .label
    assert image_temp.label['FILE_RECORDS'] == new_image.label['FILE_RECORDS']
    label_sample_type = image_temp.label['IMAGE']['SAMPLE_TYPE']
    assert label_sample_type == new_image.label['IMAGE']['SAMPLE_TYPE']

    # Testing .data
    assert image_temp.data.shape == new_image.data.shape
    assert image_temp.data.dtype == image.data.dtype
    os.remove('Temp_Image.IMG')

def test_image_save_samefile():
    image = PDS3Image.open(filename)
    with pytest.raises(IOError):
        image.save(image.filename)

def test_image_save_lines_linesamples():
    image = PDS3Image.open(filename)
    image.data = image.data[0, 3:8, 3:8]
    image.save('Temp_Image.IMG')
    image_temp = PDS3Image.open('Temp_Image.IMG')
    assert image_temp.lines == 5
    assert image_temp.samples == 5
    os.remove('Temp_Image.IMG')

def test_image_save_3bands():
    image = PDS3Image.open(filename)
    temp_data = numpy.array([image.data, image.data, image.data])
    image.data = temp_data.reshape(3, 10, 10)
    image.save('Temp_Image.IMG')
    image_temp = PDS3Image.open('Temp_Image.IMG')
    assert image_temp.bands == 3
    os.remove('Temp_Image.IMG')

def test_image_save_1band():
    image = PDS3Image.open(filename_3bands)
    image.data = image.data[0, :, :]
    image.save('Temp_Image.IMG')
    image_temp = PDS3Image.open('Temp_Image.IMG')
    assert image_temp.bands == 1
    os.remove('Temp_Image.IMG')

def test_image_save_int_to_float():
    image = PDS3Image.open(filename)
    ref_image = PDS3Image.open(filename_float)
    image.data = image.data.astype('>f4')
    image.data *= 1.5
    image.save('Temp_Image.IMG')
    image_temp = PDS3Image.open('Temp_Image.IMG')
    assert image_temp.dtype == ref_image.dtype
    assert_almost_equal(image.data, ref_image.data)
    os.remove('Temp_Image.IMG')

def test_image_save_float_to_int():
    image = PDS3Image.open(filename_float)
    ref_image = PDS3Image.open(filename)
    image.data /= 1.5
    image.data = image.data.astype('>i2')
    image.save('Temp_Image.IMG')
    image_temp = PDS3Image.open('Temp_Image.IMG')
    assert image_temp.dtype == ref_image.dtype
    assert_almost_equal(image.data, ref_image.data)
    os.remove('Temp_Image.IMG')

def test_numpy_array_save_i2():
    array = numpy.arange(100, dtype='>i2')
    array = array.reshape(1, 10, 10)
    temp = PDS3Image(array)
    temp.save('Temp_Image.IMG')
    image_temp = PDS3Image.open('Temp_Image.IMG')
    assert image_temp.bands == 1
    assert image_temp.lines == 10
    assert image_temp.samples == 10
    assert image_temp.format == 'BAND_SEQUENTIAL'
    assert image_temp.dtype == '>i2'
    assert image_temp.shape == (1, 10, 10)
    assert image_temp.size == 100
    assert_almost_equal(image_temp.data, array)
    os.remove('Temp_Image.IMG')

def test_numpy_array_save_f4():
    array = numpy.arange(100)
    array = array.reshape(1, 10, 10)
    array = array * 1.5
    array = array.astype('>f4')
    temp = PDS3Image(array)
    temp.save('Temp_Image.IMG')
    image_temp = PDS3Image.open('Temp_Image.IMG')
    assert image_temp.bands == 1
    assert image_temp.lines == 10
    assert image_temp.samples == 10
    assert image_temp.format == 'BAND_SEQUENTIAL'
    assert image_temp.dtype == '>f4'
    assert image_temp.shape == (1, 10, 10)
    assert image_temp.size == 100
    assert_almost_equal(image_temp.data, array)
    os.remove('Temp_Image.IMG')

def test_bz2_pds3_1band_labels(expected):
    image = PDS3Image.open(bz2_filename)
    assert image.filename == bz2_filename
    assert image.bands == 1
    assert image.lines == 10
    assert image.samples == 10
    assert image.format == 'BAND_SEQUENTIAL'
    assert image.dtype == numpy.dtype('>i2')
    assert image.start_byte == 640
    assert image.shape == (1, 10, 10)
    assert image.size == 100
    assert image.compression == 'bz2'

    # Testing .label
    assert image.label['FILE_RECORDS'] == 42
    assert image.label['IMAGE']['SAMPLE_TYPE'] == 'MSB_INTEGER'

    # Testing .data
    assert image.data.shape == (1, 10, 10)
    assert image.data.dtype == numpy.dtype('>i2')
    assert_almost_equal(image.data, expected)

def test_parse_pointer():
    # ^PTR = nnn
    # Example tests/mission_data/1p432690858esfc847p2111l2m1.img
    assert Pointer.parse(56, 640) == Pointer(None, 35200)

    # ^PTR = nnn <BYTES>
    assert Pointer.parse(Units(101337, 'BYTES'), 0) == Pointer(None, 101337)

    # ^PTR = "filename"
    assert Pointer.parse('W1782844276_1.IMG', 1024) == Pointer('W1782844276_1.IMG', 0)

    # ^PTR = ("filename")
    assert Pointer.parse(['W1782844276_1.IMG'], 1024) == Pointer('W1782844276_1.IMG', 0)

    # ^PTR = ("filename", nnn)
    # Example tests/mission_data/W1782844276_1.LBL
    assert Pointer.parse(['W1782844276_1.IMG', 5], 1024) == Pointer('W1782844276_1.IMG', 4096)

    # ^PTR = ("filename", nnn <BYTES>)
    assert Pointer.parse(['W1782844276_1.IMG', Units(101337, 'BYTES')], 1024) == Pointer('W1782844276_1.IMG', 101337)

    # Test bad type
    with pytest.raises(ValueError):
        Pointer.parse(None, 64)

    # Test wrong sized arrays
    with pytest.raises(ValueError):
        Pointer.parse([], 64)

    with pytest.raises(ValueError):
        Pointer.parse(['W1782844276_1.IMG', 5, 6], 64)

max_stars_repo_name: planetarypy/planetaryimage

max_issues_repo_name: planetarypy/planetaryimage

max_forks_repo_name: planetarypy/planetaryimage

reason: import numpy,from numpy


text:
#!/usr/bin/env python
#------------------------------------------------------------------------------
#
#   This tool clips data to mask. Based on the mask value, the code
#   performs following pixel operations:
#       mask no-data value (0x00) -> sets pixel to a given no-data value
#       mask data value (0xFF)    -> copies pixel from the original image
#
#   This tool extracts subset of the image specified by the row/column
#   offset of the upper-left corner and row/column size of extracted
#   block. The tool takes care of preserving the geo-metadata.
#
# Author: Martin Paces <martin.paces@eox.at>
#
#-------------------------------------------------------------------------------
# Copyright (C) 2013 EOX IT Services GmbH
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies of this Software or works derived from this Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#-------------------------------------------------------------------------------

import sys
from os.path import basename
from numpy import dtype
from img import (
    FormatOptions, ImageFileReader, create_geotiff, DEF_GEOTIFF_FOPT,
    Progress, Block, execute, Point2,
)
from img.algs import clip_to_mask
from img.cli import error

def usage():
    """Print a short command usage help."""
    exe = basename(sys.argv[0])
    out = sys.stderr
    print >>out, (
        "USAGE: %s <input image> <mask> <output TIF> <no data value or list>"
        % exe
    )
    print >>out, "EXAMPLE: %s input.tif mask.tif output.tif 255,255,255" % exe
    print >>out, "EXAMPLE: %s input.tif mask.tif output.tif 0" % exe

def process(tile, img_in, img_mask, img_out, nodata, clipped_mask_value=0):
    """ Process one tile. """
    # pylint: disable=too-many-arguments
    tile = tile & img_out # clip tile to the image extent
    b_in = img_in.read(Block(img_in.dtype, tile))
    b_mask = img_mask.read(Block(img_mask.dtype, tile))
    b_out = clip_to_mask(b_in, b_mask, nodata, clipped_mask_value)
    img_out.write(b_out)

if __name__ == "__main__":
    FOPTS = FormatOptions(DEF_GEOTIFF_FOPT) # default format options
    MASKBG = 0x00
    MASKFG = 0xFF
    try:
        INPUT = sys.argv[1]
        MASK = sys.argv[2]
        OUTPUT = sys.argv[3]
        NODATA = sys.argv[4].split(",")
        #anything else treated as a format option
        FOPTS.set_options(sys.argv[5:])
    except IndexError:
        error("Not enough input arguments!")
        usage()
        sys.exit(1)

    # open input image
    IMG_IN = ImageFileReader(INPUT)

    # convert no-data values to the image's data type
    if len(NODATA) == 1 and len(IMG_IN) > 1:
        NODATA = NODATA * len(IMG_IN)
    NODATA = [dtype(dt).type(nd) for dt, nd in zip(IMG_IN.dtypes, NODATA)]

    # open the mask image
    IMG_MASK = ImageFileReader(MASK)

    # check mask properties
    if IMG_MASK.size.z > 1:
        error("Multi-band masks are not supported!")
        sys.exit(1)

    if IMG_MASK.dtype != 'uint8':
        error("Unsupported mask data type '%s'!" % IMG_MASK.dtype)
        sys.exit(1)

    if Point2(IMG_IN.size) != Point2(IMG_MASK.size):
        error(
            "Input mask and image must have the same pixel"
            " size! image: %dx%d  mask: %dx%d" %
            (IMG_IN.size.y, IMG_IN.size.x, IMG_MASK.size.y, IMG_MASK.size.x)
        )
        sys.exit(1)

    # creation parameters
    PARAM = {
        'path':   OUTPUT,
        'nrow':   IMG_IN.size.y,
        'ncol':   IMG_IN.size.x,
        'nband':  IMG_IN.size.z,
        'dtype':  IMG_IN.dtype,
        'options' : FOPTS.options,
        'nodata': NODATA,
    }
    PARAM.update(IMG_IN.geocoding) # add geo-coding

    # open output image
    IMG_OUT = create_geotiff(**PARAM)

    # block size
    TILE_SIZE = (int(FOPTS["BLOCKXSIZE"]), int(FOPTS["BLOCKYSIZE"]))

    print "Clipping image by a mask ..."
    execute(
        IMG_OUT.tiles(TILE_SIZE), process, (
            IMG_IN, IMG_MASK, IMG_OUT, NODATA, MASKBG,
        ),
        progress=Progress(sys.stdout, IMG_OUT.tile_count(TILE_SIZE)),
    )

max_stars_repo_name: DREAM-ODA-OS/tools

max_issues_repo_name: DREAM-ODA-OS/tools

max_forks_repo_name: DREAM-ODA-OS/tools

reason: from numpy


text:
#!/usr/bin/python
# -*- encoding: utf-8 -*-

import cv2
import numpy as np
from PIL import Image

class PostProcess:
    def __init__(self, config):
        self.denoise = config.POSTPROCESS.WILL_DENOISE
        self.img_size = config.DATA.IMG_SIZE

    def __call__(self, source: Image, result: Image):
        # TODO: Refract -> name, resize
        source = np.array(source)
        result = np.array(result)

        height, width = source.shape[:2]
        small_source = cv2.resize(source, (self.img_size, self.img_size))
        laplacian_diff = source.astype(
            np.float) - cv2.resize(small_source, (width, height)).astype(np.float)
        result = (cv2.resize(result, (width, height)) +
                  laplacian_diff).round().clip(0, 255).astype(np.uint8)
        if self.denoise:
            result = cv2.fastNlMeansDenoisingColored(result)
        result = Image.fromarray(result).convert('RGB')
        return result

max_stars_repo_name: CGCL-codes/AMT-GAN

max_issues_repo_name: CGCL-codes/AMT-GAN

max_forks_repo_name: CGCL-codes/AMT-GAN


text:
#!/usr/bin/env python
## Author: Alan Tavares
## Date: August, 12, 2019
# Purpose: Ros node to detect objects using tensorflow
import os
import sys
import cv2
import numpy as np
import math

# ROS related imports
import rospy
from geometry_msgs.msg import Twist, Vector3

# Navigation
class Navigation:

  def __init__(self):

        self.VERBOSE = True
        self.msg_hough = Twist()

        #############################################################################
        # Publishers
        self.nav_pub = rospy.Publisher('navigation/nav_position', Twist, queue_size=10)

        # Subscribers
        self.hough_sub = rospy.Subscriber("hough/nav_hough_lines", Twist, self.houghCallback, queue_size=10)
        self.rnn_sub = rospy.Subscriber("RNN/nav_direction", Vector3, self.rcnnCallback, queue_size=10)

  #############################################################################

  def houghCallback(self, data):
        # recive data
        self.msg_hough = data

  #############################################################################

  def rcnnCallback(self, data):
        # recive data
        moviment = data.x
        rotation = data.y

        velocity = Twist()

        # controll y and yaw only
        if moviment == 1:
            if self.VERBOSE == True:
                rospy.logdebug("Curva")

            if rotation == 1:
                velocity.linear.x = 0
                velocity.linear.y = 1
                velocity.linear.z = 0

                velocity.angular.x = 0
                velocity.angular.y = 0
                velocity.angular.z = math.radians(15)
                if self.VERBOSE == True:
                    rospy.logdebug("...Left-yaw: %f deg/s",velocity.angular.z*(180/np.pi))
                    rospy.logdebug("-------------------------")

            else:
                velocity.linear.x = 0
                velocity.linear.y = -1
                velocity.linear.z = 0

                velocity.angular.x = 0
                velocity.angular.y = 0
                velocity.angular.z = math.radians(-15)
                if self.VERBOSE == True:
                    rospy.logdebug("...Right-yaw: %f deg/s",velocity.angular.z*(180/np.pi))
                    rospy.logdebug("-------------------------")
        else:
            if self.VERBOSE == True:
                rospy.logdebug("Reta")
            velocity.linear.x = 0.03
            velocity.linear.y = -self.msg_hough.linear.y
            velocity.linear.z = 0

            velocity.angular.x = 0
            velocity.angular.y = 0
            velocity.angular.z = self.msg_hough.angular.z
            if self.VERBOSE == True:
                rospy.logdebug('Y-raw  : %f', velocity.linear.y)
                rospy.logdebug('Yaw-raw: %f', math.degrees(velocity.angular.z))
                rospy.logdebug("-------------------------")

        self.nav_pub.publish(velocity)

###############################################################################

def main(args):
  #-- Name of node
  rospy.init_node('navigation_node',log_level=rospy.DEBUG)
  ic = Navigation()

  try:
      rospy.spin()
  except KeyboardInterrupt:
      print("Shutting down")

###############################################################################

if __name__ == '__main__':
  main(sys.argv)


text:
import cv2
import argparse
import numpy as np
from utils import *

def main():
    args = parse.parse_args()
    video_path = args.video_path
    output_path = args.output_path
    channel_num = args.channel_num
    method = args.method
    thresh = args.threshold
    thresh_val = args.threshold_val

    cap = cv2.VideoCapture(video_path)
    ret, current_frame = cap.read()
    previous_frame = current_frame

    # Obtain frame size information using get() method
    frame_width = int(cap.get(3))
    frame_height = int(cap.get(4))
    frame_size = (frame_width, frame_height)
    fps = cap.get(5)

    # Initialize video writer object
    if channel_num == 3:
        output = cv2.VideoWriter(output_path, cv2.VideoWriter_fourcc(*'MJPG'), fps, frame_size)
    elif channel_num == 1:
        output = cv2.VideoWriter(output_path, cv2.VideoWriter_fourcc(*'MJPG'), fps, frame_size, False)
    else:
        sys.exit('Wrong channel number')

    while ret:
        frame_diff = cal_diff(method, current_frame, previous_frame, thresh, thresh_val)
        output.write(frame_diff)

        previous_frame = current_frame.copy()
        ret, current_frame = cap.read()

    cap.release()
    cv2.destroyAllWindows()

if __name__ == '__main__':
    parse = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parse.add_argument('--video_path', type=str, default='you forgot to set video path')
    parse.add_argument('--output_path', type=str, default='you forgot to set output path')
    parse.add_argument('--method', type=str, default='abs')
    parse.add_argument('--channel_num', type=int, default=3)
    parse.add_argument('--threshold', action='store_true')
    parse.add_argument('--threshold_val', type=int, default=127)

    main()

max_stars_repo_name: xuyingzhongguo/Video-diff

max_issues_repo_name: xuyingzhongguo/Video-diff

max_forks_repo_name: xuyingzhongguo/Video-diff


text:
import os
import sys
import time
import copy
import resource
import numpy as np
import pandas as pd
import torch.optim as optim
import torch.nn.functional as F
import torchvision.transforms as transforms

from image_util import *
from data_util import *
from env import *
from dqn_agent import *
import model_creator as mCreator

"""
#G10 Server
ROOT_DATA_PATH = "/home/g10/medical/data/rsna_pneumonia/"
S1_TRAIN_IMG_PATH = ROOT_DATA_PATH + "stage_1_train_images"
S1_TRAIN_ANNO_PATH = ROOT_DATA_PATH + "stage_1_train_labels.csv"
"""

ROOT_DATA_PATH = "/home/ubuntu/richielo/data/pneumo_data/"
S1_TRAIN_IMG_PATH = ROOT_DATA_PATH + "stage_1_train_images"
S1_TRAIN_ANNO_PATH = ROOT_DATA_PATH + "stage_1_train_labels.csv"

#Hyperparameters
EPISODES = 200  # number of episodes
EPS_START = 1.0  # e-greedy threshold start value
GUIDED_EPS = 0.5
EPS_MIN = 0.1  # e-greedy threshold end value
EPS_DECAY = 0.99  # e-greedy threshold decay
GAMMA = 0.99  # Q-learning discount factor
LR = 1e-6  # NN optimizer learning rate
LR_DECAY = 0.01
HIDDEN_LAYER = 512  # NN hidden layer size
BATCH_SIZE = 48  # Q-learning batch size
MEM_SIZE = 4000
ITER_UPDATE_TARGET = 500

#Global Variables
NUM_ACTIONS = 9
ACTION_THRES = 0.2
TRIGGER_THRES = 0.6
TRIGGER_REWARD = 10
INIT_BB_THRES = 0.8
steps_done = 0
INFER_TIMESTEP = 20

#@profile
def run_episodes(num_epoch, parsed_dict, train_loader, test_loader, global_features_net, local_features_net, use_chexnet=False, use_gpu=True):
    since = time.time()
    best_reward = None

    train_batches = len(train_loader)
    test_batches = len(test_loader)
    train_transforms = transforms.Compose([transforms.ToPILImage(), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
    #normalize = transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])

    #Initialize dqn agent
    dqn_agent = DQN_Agent(global_features_net, local_features_net, num_epoch, NUM_ACTIONS, HIDDEN_LAYER, LR, LR_DECAY, GAMMA, EPS_START, EPS_MIN, EPS_DECAY, GUIDED_EPS, MEM_SIZE, BATCH_SIZE, ITER_UPDATE_TARGET, use_gpu, use_chexnet, train_transforms)
    best_model_wts = copy.deepcopy(dqn_agent.target_net.state_dict())

    learn_freq = 10

    #Training
    for epoch in range(num_epoch):
        dqn_agent.set_training(True)
        num_train_step = 0
        train_return_list = []
        epoch_start = time.time()
        print("Epoch {}/{}".format(epoch, num_epoch))
        print('-' * 10)
        for i, data in enumerate(train_loader):
            if i % 100 == 0:
                print("Training batch {}/{}".format(i, train_batches), end='\n', flush=True)
            patient_ids = data
            for j in range(len(patient_ids)):
                env = PneumoEnv(parsed_dict[patient_ids[j]], True, ACTION_THRES, TRIGGER_THRES, TRIGGER_REWARD, INIT_BB_THRES)
                R = 0
                done = False
                while not done:
                    curr_bb = env.extract_bound_box_image(env.bb)
                    action = dqn_agent.select_action(env, curr_bb)
                    state_bb, action, reward, next_state_bb, done = env.step(action)
                    num_train_step += 1
                    dqn_agent.store_transitions((env.full_env, curr_bb), action, reward, (env.full_env, env.extract_bound_box_image(next_state_bb)), done)
                    R += reward
                    if(num_train_step % learn_freq == 0):
                        dqn_agent.learn()
                    if done:
                        train_return_list.append(R)
                        print("Epoch:", epoch, "Epsilon:", dqn_agent.epsilon, "R:", R)
                        #print("Length of memory" + str(len(dqn_agent.memory)))
                        #print("mem usage: " + str(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss))
	                    #Reset global features for new environment
                        sys.stdout.flush()
                        dqn_agent.global_features = None
        epoch_end = time.time()
        print("Epoch time: " + str(epoch_end-epoch_start))

        #Done one epoch
        dqn_agent.decay_epsilon_linear()
        #Validation episodes
        if(epoch+1 % 5 == 0):
            dqn_agent.set_training(False)
            with torch.no_grad():
                test_return_list = []
                for i, data in enumerate(test_loader):
                    if i % 100 == 0:
                        print("Training batch {}/{}".format(i, test_batches), end='\n', flush=True)
                    patient_ids = data
                    for j in range(len(patient_ids)):
                        env = PnemoEnv(parsed_dict[patient_ids[j]], True, ACTION_THRES, TRIGGER_THRES, TRIGGER_REWARD, INIT_BB_THRES)
                        R = 0
                        done = False
                        running_steps = 0
                        while not done:
                            action = dqn_agent.select_action_infer(env, env.extract_bound_box_image(env.bb))
                            state_bb, action, reward, next_state_bb, done = env.step(action)
                            running_steps += 1
                            R += reward
                            if done:
                                test_return_list.append(R)
                                print("Test Episode:", i, "R:", R)
                                #Reset global features for new environment
                                dqn_agent.global_features = None

                avg_train_reward = sum(train_return_list)/float(len(train_return_list))
                avg_test_reward = sum(test_return_list)/float(len(test_return_list))
                print("Average training reward: " + str(avg_train_reward))
                print("Average testing reward: " + str(avg_test_reward))
                if(best_reward is None):
                    best_model_wts = copy.deepcopy(dqn_agent.target_net.state_dict())
                    best_reward = avg_test_reward
                elif(avg_test_reward > best_reward):
                    best_model_wts = copy.deepcopy(dqn_agent.target_net.state_dict())
                    best_reward = avg_test_reward

    cwd = os.getcwd()
    torch.save(dqn_agent.target_net.state_dict(), cwd+'/' + "train_1_end_net.pt")
    dqn_agent.target_net.load_state_dict(best_model_wts)
    return dqn_agent

if __name__ == '__main__':
    cwd = os.getcwd()
    split_ratio = 0.05
    num_epoch = 25
    use_chexnet = False

    #Load train image paths and annotations
    anns = pd.read_csv(open(os.path.join(ROOT_DATA_PATH, 'stage_1_train_labels.csv'), 'r'))
    parsed_training_data_dict = parse_dataset(S1_TRAIN_IMG_PATH, anns)
    full_dataset = PneumoLocalizationDataset(parsed_training_data_dict)
    train_size = int(0.8 * len(full_dataset))
    test_size = len(full_dataset) - train_size
    train_dataset, test_dataset = torch.utils.data.random_split(full_dataset, [train_size, test_size])

    train_loader = DataLoader(train_dataset, batch_size=4, num_workers=2, pin_memory=True)
    test_loader = DataLoader(test_dataset, batch_size=4, num_workers=2, pin_memory=True)

    #Get pretrained global and local features net
    if(use_chexnet):
        global_features_net = mCreator.get_chexnet(14, True, "chexnet_pretrained.pth.tar")
        local_features_net = mCreator.get_chexnet(14, True, "chexnet_pretrained.pth.tar")
        print("Loaded pretrained chestxnet")
    else:
        global_features_net = mCreator.get_resnet50_model(2, True, 0, "pretrain_global2_b24_f0_e20_001_9_imgnet_normalize.pt", True)
        local_features_net = mCreator.get_resnet50_model(2, True, 0, "pretrain_local1_b4_f0_e60_00001_9_imgnet_normalize.pt", False)
        print("Loaded pretrained resnet")
    #Training
    trained_agent = run_episodes(num_epoch, parsed_training_data_dict, train_loader, test_loader, global_features_net, local_features_net, use_chexnet, True)
    #epoch-batch size-trigger reward thres-lr-decay-hidden layer size
    model_name = "dqn_train1_e{}_b{}_t0.6_1e6_99_{}_epslinear_resnet_pretrained.pt".format(num_epoch, BATCH_SIZE, HIDDEN_LAYER)
    torch.save(trained_agent.target_net.state_dict(), cwd+'/' + model_name)

    """
    #model = mCreator.get_resnet50_model(2, True, 0)
    #model2 = mCreator.get_resnet50_model(2, True, 0)
    #dqn = mCreator.Bi_DQN(9, model, model2, 512)
    #optimizer_ft = optim.SGD(model.parameters(), lr=0.00001, momentum=0.9)
    #new_env = PneumoEnv(testDataDict, True, 0.1, 0.6, 10, 0.7)
    #print(np.argmax(dqn(new_env.full_env, new_env.extract_bound_box_image()).detach().numpy()))
    #print(torch.argmax(dqn(new_env.full_env, new_env.extract_bound_box_image())))
    trained_agent = run_episodes(num_epoch, train_loader, test_loader, global_features_net, local_features_net)
    model_name = "dqn_train1_b{}_0001_99_{}_chexnet_pretrained.pt".format(BATCH_SIZE, HIDDEN_LAYER)
    torch.save(trained_agent.target_net.state_dict(), cwd+'/' + model_name)

    #Load train image paths and annotations
    anns = pd.read_csv(open(os.path.join(ROOT_DATA_PATH, 'stage_1_train_labels.csv'), 'r'))
    parsed_training_data = parse_dataset(S1_TRAIN_IMG_PATH, anns)
    print("Number of train images: " + str(len(parsed_training_data)))
    for k, v in parsed_training_data.items():
        print(k)
        print(v)
        testImg = get_dicom_image_data(parsed_training_data[k]['dicom'])
        print(np.amax(testImg))
        print(np.amin(testImg))
        print(testImg.shape)
        break
    """


text:
# das2py example 10:
#    Creating datasets manually and observing automatic dataset shape
#    adjustments.
#
# Make a test dataset in the style of a sweep frequency radar sounder.
# This dataset will have values occupying the parameter spaces:
#
#     Time, Pulse Frequency, Range, and Spectral Density
#
# That will be mapped to dataset indicies in the following manner:
#
#         A    +--------------+
#         x  R |\  Spectral    \
#         i  a | \   Density    \
#         s  n |  \      at      \
#            g |   \    Antenna   \
#         2  e |    \              \
#              |     +--------------+
#              +     |              |
#          A  T \    |              |
#           x  i \   |              |
#            i  m \  |              |
#             s  e \ |              |
#                   \|              |
#               0    +--------------+
#                      Pulse Freq &
#                      Offset Time
#                        Axis 1

import numpy as np
import das2

ds = das2.Dataset('sounder')
print("Initial dataset shape: %s"%list(ds.shape))

# State that the dataset has a time dimension, but don't define any variables
# in the new dimension yet.

time = ds.coord('time')

# Add a reference point in the time dimension that correlates with dataset
# axis 0.  This is the start of the pulse train

pulse0 = ['2015-08-27','2015-08-26','2015-08-29','2015-08-26']
ref = time.reference(pulse0, 'UTC')
print("Shape with start times: %s"%list(ds.shape))

# Add an offset variable in the time dimension that correlates with dataset
# axis 1.  This is the time between pulses in milliseconds.  Since the new
# array is orthogonal to the first, the dataset becomes rank 2.

offset = time.offset( [0,10,20], 'ms', axis=1)
print("Shape with orthogonal time offsets: %s"%list(ds.shape))

# Add a center point variable in the pulse_freq dimension who's values should
# map along dataset axis 1.  Make these out of order in offset time.
#
# We will now have two arrays that 'tag' axis 1.  With ISTP metadata CDFs this
# is a bit of an issue since you can't have two 'DEPEND_1' variables, but here
# it's no problem.  Note, it's not a problem in the base CDF specification
# either, just the ISTP metadata.

ds.coord('pulse_freq').center([4,3,5], 'MHz', axis=1)
print("Shape with pulse frequencies: %s"%list(ds.shape))

# Add center points in the range dimension.  Have thes values map along
# dateset axis 2.  Since the new array is orthoganal to both of the existing
# arrays the dataset becomes rank 3.

ds.coord('range').center(np.arange(5)*10 + 25, 'km',  axis=2)
print("Shape with echo return ranges: %s"%list(ds.shape))

# Add center points in the spectral density dimension.  We're adding a
# rank 3 array to the dataset without an axis shift so the shape remains
# unchanged.
ds.data('spec_dens').center(
   [   [  [321,322,323,324,325],
         [311,312,313,314,315],
         [331,332,333,334,335]   ],
      [   [121,122,123,124,125],
         [111,112,113,114,115],
         [131,132,133,134,135]   ],
      [  [421,422,423,424,425],
         [411,412,413,414,415],
         [431,432,433,434,435]   ],
      [   [221,222,223,224,225],
         [211,212,213,214,215],
         [231,232,233,234,235]   ]
   ],
   'V**2 m**-2 Hz**-1', fill=0.0
)
print("Shape with return intensities: %s"%list(ds.shape))

# Print a single slice of a single ionogram.  The same indices can be used
# with every array since all variables use numpy array broadcasting to match
# the shape of the overall dataset.

print("The 2nd pulse from 2nd Ionogram...")
for i in range(ds.shape[2]):
   print("%s, %s, %s, %s"%(
      ds['time']['center'][2,2,i],
      ds['pulse_freq']['center'][2,2,i],
      ds['range']['center'][2,2,i],
      ds['spec_dens']['center'][2,2,i]
   ))

# Sort on pulse0 times and then range
ds.sort('time:reference','range')


text:
"""
"""
import sys
import os
import math
import numpy as np
from VectorKMeans import VectorsClustered

class NearestClusterClassifier(object):
    """
    """
    def __init__(self, train_samples, val_samples):
        """
        """
        self._val_samples = val_samples
        self._min_n_class_samples = min(map(len, train_samples))
        print("_min_n_class_samples: ", self._min_n_class_samples)
        self._classes = [VectorsClustered(_samples)
                         for _samples in train_samples]
        self.n_classes = len(self._classes)

    def nClustersTrain(self, n_clusters):
        """
        """
        for _class in self._classes:
            _class.kmeansCluster(n_clusters)

    def scaledClustersTrain(self, n_clusters):
        """
        """
        for _class in self._classes:
            _scale = _class.n_samples // self._min_n_class_samples
            print("Scale: ", _scale)
            _class.kmeansCluster(n_clusters * _scale)

    def classify(self, sample):
        """
        """
        _min_d = sys.maxsize
        _classification = None
        for _i, _class in enumerate(self._classes):
            _class_dist = _class.distanceToCenters(sample)
            if _class_dist < _min_d:
                _min_d = _class_dist
                _classification = _i
        return _classification

    def validate(self):
        """
        """
        return self.test(self._val_samples)

    def test(self, val_samples):
        """
        """
        _n_correct_list = []
        _acc_list = []
        _n_val_samples = sum(map(len, val_samples))
        print(_n_val_samples)
        for _i, _val_class_samples in enumerate(val_samples):
            _n_correct = 0
            for _sample in _val_class_samples:
                if _i == self.classify(_sample):
                    _n_correct += 1
            _n_correct_list.append(_n_correct)
            _acc_list.append(float(_n_correct) /
                             float(len(_val_class_samples)))
        _n_correct = sum(_n_correct_list)
        print("Accuracy of classes: ", _acc_list)
        return float(_n_correct) / float(_n_val_samples)


text:
from mynn.initializers.glorot_uniform import glorot_uniform
from mynn.activations.relu import relu
from mynn.layers.conv import conv
from mygrad.nnet.layers import max_pool
from mygrad.nnet.losses import softmax_crossentropy
from mynn.layers.dense import dense
import numpy as np
import mygrad as mg

class Model:
    def __init__(self, input_dim, f1, f2, d1, num_classes):
        '''
        Parameters
        ----------
        input_dim:
            size of input based on training data
        n1 : int
            The number of neurons in the first hidden layer
        num_classes : int
            The number of classes predicted by the model'''
        init_kwargs = {'gain': np.sqrt(2)}
        self.conv1 = conv(input_dim, f1, 5, 5,
                          weight_initializer=glorot_uniform,
                          weight_kwargs=init_kwargs)
        self.conv2 = conv(f1, f2, 5, 5,
                          weight_initializer=glorot_uniform,
                          weight_kwargs=init_kwargs)
        self.dense1 = dense(f2 * 37 * 37, d1,
                            weight_initializer=glorot_uniform,
                            weight_kwargs=init_kwargs)
        self.dense2 = dense(d1, num_classes,
                            weight_initializer=glorot_uniform,
                            weight_kwargs=init_kwargs)

    def __call__(self, x):
        """ Performs a "forward-pass" of data through the network.

        This allows us to conveniently initialize a model `m` and then send data through it
        to be classified by calling `m(x)`.

        Parameters
        ----------
        x : Union[numpy.ndarray, mygrad.Tensor], shape=(M, ?)
            A batch of data consisting of M pieces of data,
            each with a dimentionality of ? (the number of
            values among all the pixels in a given image).

        Returns
        -------
        mygrad.Tensor, shape-(M, num_class)
            The model's prediction for each of the M images in the batch,
        """
        x = relu(self.conv1(x))
        x = max_pool(x, (2, 2), 2)
        x = relu(self.conv2(x))
        x = max_pool(x, (2, 2), 2)
        x = relu(self.dense1(x.reshape(x.shape[0], -1)))
        return self.dense2(x)

    @property
    def parameters(self):
        """ A convenience function for getting all the parameters of our model.

        Returns
        -------
        List[mygrad.Tensor]
            A list of all of the model's trainable parameters
        """
        params = []
        for layer in (self.conv1, self.conv2, self.dense1, self.dense2):
            params += list(layer.parameters)
        return params

    def save_model(self, path):
        """Path to .npz file where model parameters will be saved."""
        with open(path, "wb") as f:
            np.savez(f, *(x.data for x in self.parameters))

    def load_model(self, path):
        with open(path, "rb") as f:
            for param, (name, array) in zip(self.parameters, np.load(f).items()):
                param.data[:] = array

# accuracy function!!
def accuracy(predictions, truth):
    """
    Returns the mean classification accuracy for a batch of predictions.

    Parameters
    ----------
    predictions : Union[numpy.ndarray, mg.Tensor], shape=(M, D)
        The scores for D classes, for a batch of M data points

    truth : numpy.ndarray, shape=(M,)
        The true labels for each datum in the batch: each label is an
        integer in [0, D)

    Returns
    -------
    float
    """
    if isinstance(predictions, mg.Tensor):
        predictions = predictions.data
    return np.mean(np.argmax(predictions, axis=1) == truth)

def convert_data(images):
    '''
    returns xtrain,xtest as a tuple

    Parameters:
    ----------
    images: numpy array of image vectors

    Returns:
    --------
    A tuple containing the normalized images into (xtrain,xtest)
    '''
    images = images[:,np.newaxis,:,:]

    images = images.astype(np.float32)
    images /= 255.

    divide = 4 * len(images) // 5
    return (images[:divide], images[divide:])

max_stars_repo_name: AlphaInBinary/are-you-social-distancing

max_issues_repo_name: AlphaInBinary/are-you-social-distancing

max_forks_repo_name: AlphaInBinary/are-you-social-distancing