text:
#!/usr/bin/env python
# coding: utf-8

# In[1]:

# To support both python 2 and python 3
from __future__ import division, print_function, unicode_literals

# Common imports
import numpy as np
import os

# to make this notebook's output stable across runs
np.random.seed(42)

# To plot pretty figures
get_ipython().run_line_magic('matplotlib', 'inline')
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rc('axes', labelsize=14)
mpl.rc('xtick', labelsize=12)
mpl.rc('ytick', labelsize=12)

# Where to save the figures
PROJECT_ROOT_DIR = "."
CHAPTER_ID = "classification"

def save_fig(fig_id, tight_layout=True):
    path = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID, fig_id + ".png")
    print("Saving figure", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(path, format='png', dpi=300)

# Saving and loading sklearn objects to pickle files in hard disk
import os.path, joblib

def save_pkl(obj, name):
    joblib.dump(obj, name)
    print(name + 'saved!')

def load_pkl(name):
    name = name + '.pkl'
    if os.path.exists(name):
        return joblib.load(name)
    print('there is no' + name)
    return False

# In[3]:

# Some crazy stuff about ordering the data set so I get the same results as in the book
def sort_by_target(mnist):
    reorder_train = np.array(sorted([(target, i) for i, target in enumerate(mnist.target[:60000])]))[:, 1]
    reorder_test = np.array(sorted([(target, i) for i, target in enumerate(mnist.target[60000:])]))[:, 1]
    mnist.data[:60000] = mnist.data[reorder_train]
    mnist.target[:60000] = mnist.target[reorder_train]
    mnist.data[60000:] = mnist.data[reorder_test + 60000]
    mnist.target[60000:] = mnist.target[reorder_test + 60000]

# In[4]:

# Loading MINST dataset
mnist = load_pkl('mnist')
#Fetching MNISTdataset
if not mnist:
    from sklearn.datasets import fetch_openml
    mnist = fetch_openml('mnist_784', version=1, cache=True)
    save_pkl(mnist,'mnist')

# In[5]:

mnist.target = mnist.target.astype(np.int8) # fetch_openml() returns targets as strings
sort_by_target(mnist) # fetch_openml() returns an unsorted dataset

# In[6]:

X, y = mnist['data'], mnist['target']
print(X.shape,y.shape)

# In[7]:

#grab an instance’s feature vector, reshape it to a 28×28 array, and display it
some_digit = X[36000]
some_digit_image = some_digit.reshape(28,28)

# In[8]:

plt.imshow(some_digit_image)

# In[9]:

# The label corresponding to some_digit is indeed a 5
y[36000]

# In[10]:

#training and testing set
X_train, X_test, y_train, y_test = X[:60000], X[60000:], y[:60000], y[60000:]
#shuffling to guarantee cross validation folds will be similar
shuffle_index = np.random.permutation(60000)
#generates an array of indexes permutated
X_train, y_train= X_train[shuffle_index], y_train[shuffle_index]

# # Training a binary classifier: 5-detector

# In[11]:

#creating labels for this classification task
y_train_5 = (y_train == 5)
y_test_5 = (y_test == 5)

# In[15]:

sgd_clf = load_pkl('sgd_clf')

if not sgd_clf:
    from sklearn.linear_model import SGDClassifier
    sgd_clf = SGDClassifier(random_state=42)
    sgd_clf.fit(X_train, y_train_5)
    # Output a pickle file for the model
    save_pkl(sgd_clf,'sgd_clf')

# In[16]:

# Let us verify it works with the digit choosed before
sgd_clf.predict([some_digit]) == 5

# # Model performance

# ## Measuring Accuracy Using Cross-Validation

# In[17]:

from sklearn.model_selection import cross_val_score

# In[18]:

sgd_clf_accuracy_cv = cross_val_score(sgd_clf, X_train, y_train_5, cv=3, scoring="accuracy")

# In[39]:

sgd_clf_accuracy_cv

# In[19]:

np.savetxt('sgd_clf_accuracy_cv.np',sgd_clf_accuracy_cv, fmt='%f')

# Seems impressive that our SGD model has an accuracy over 96%, but with the next example we must check that this result is not to be taken seriously since it has some flaws related to the very same measure of acurracy and science case on a binary classifier for the MNIST data constitution :O

# In[20]:

from sklearn.base import BaseEstimator
# with BaseEstimator we create a classifier from scratch that says any feature from MNIST is not a 5

class Never5Classifier(BaseEstimator):

    # All hard work usually must be implemented on the fit method, but since the classifier only has to
    # say any X is labeled with a 5, it is left empty

    def fit(self, X, y=None):
        pass
    # predict method just outputs our y vector which is pretty dumb in this case: True, all the time.
    def predict(self, X):
        return np.zeros((len(X), 1), dtype=bool)

# In[21]:

# Now it is time to implement our Mighty Binary classifier XD and check its accuracy
never_5_clf = Never5Classifier()
cross_val_score(never_5_clf, X_train, y_train_5, cv=3, scoring="accuracy")

# This result shows why accuracy is not the ultimate word on how well the ML model performs and it is related to the very nature of the task and the data:
#
# 1. It is a binary classifier
# 2. The statistical distribution of the data is such that 5 is present 10% of the time
#
# Regarding item (1), if it were a multi classifier then the accuracy of the Mighty-non-5 classifier wouldn't be over 90%. Regarding item (2), the data has the asymmetry of having 1/10 of its Xs being 5s; if the asymmetry were 1/100 5s, then the accuracy will be higher or if it were 5s all the time, then the accuracy will be 0%.
#
# These two points are quite important and must be considered when training a model, and a reason why another performance measures must be considered.

# ## Confusion Matrix

# From the book:
# "
# The general idea is to count the number of times instances of class A are classified as class B. To compute the confusion matrix, you first need to have a set of predictions, so they can be compared to the actual targets.
# "

# In[24]:

if os.path.exists('y_train_pred.np'):
    y_train_pred = np.loadtxt('y_train_pred.np', dtype=float)
else:
    from sklearn.model_selection import cross_val_predict
    y_train_pred = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3)
    np.savetxt('y_train_pred.np',y_train_pred, fmt='%f')

# In[25]:

from sklearn.metrics import confusion_matrix
CM = confusion_matrix(y_train_5, y_train_pred)

# In[26]:

print(CM)

# "Each row in a confusion matrix represents an actual class, while each column represents a predicted class."
#
# In this case, the first row represents the negative class, ie, the true negatives; meaning that CM[0,0] is the number of non-5s classified correctly while CM[0,1] represents the number false positives, ie, the non-5s classified as 5s. The second class represents the possitive class. CM[1,0] is the number of false negatives, ie, the number of 5s classified as non-5s while CM[1,1] represents the number of true positives, ie, the number of 5s classified as 5s.
#
# "A perfect classifier would have only true positives and true negatives"

# ## Metrics derived from the confusion matrix

# ### Accuracy of the positive predictions: the precision of the classifier

# $$precision = \frac{TP}{TP+FP}$$

# ### Recall: ratio of positive instances that are correctly detected by the classifier

# $$recall = \frac{TP}{TP+FN}$$

# In[27]:

from sklearn.metrics import precision_score, recall_score
precision = precision_score(y_train_5, y_train_pred)
recall = recall_score(y_train_5, y_train_pred)

# In[28]:

print(precision,recall)

# Here we go nuts :O since this means that our classifier is correct only 88% of the time when it says an image is a 5 (precision). On the other hand, it only detects 68% of the 5s (recall).

# ### F$_1$ score: the harmonic mean

# $$F_1 = \frac{TP}{TP+\frac{FN+FP}{2}}= 2 \frac{precision \times recall}{precision + recall}$$

# In[29]:

from sklearn.metrics import f1_score
F_1 = f1_score(y_train_5, y_train_pred)

# In[30]:

print(F_1)

# https://towardsdatascience.com/beyond-accuracy-precision-and-recall-3da06bea9f6c
#
# "We use the harmonic mean instead of a simple average because it punishes extreme values. A classifier with a precision of 1.0 and a recall of 0.0 has a simple average of 0.5 but an F1 score of 0. The F1 score gives equal weight to both measures and is a specific example of the general Fβ metric where β can be adjusted to give more weight to either recall or precision... If we want to create a balanced classification model with the optimal balance of recall and precision, then we try to maximize the F1 score."

# ## Precision/Recall Tradeoff

# It is clear that there is correlation between precision and recall and that according to our science case, we should chose the combination that gives us the best results for our purposes. With that in mind, lets, examine the the precision and recal from our SGDClassifier as a function the threshold it uses to decide when classifying (an hyperparameter of the model).

# In[38]:

# Let's obtain the decition scares our classifier uses
from sklearn.model_selection import cross_val_predict

if os.path.exists('y_scores.np'):
    y_scores = np.loadtxt('y_scores.np', dtype=float)
else:
    y_scores = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3, method= "decision_function")
    np.savetxt('y_scores.np',y_scores, fmt='%f')

# In[40]:

# Now with the decision scores, we proceed to compute teh precision and recall curve
from sklearn.metrics import precision_recall_curve

precisions, recalls, thresholds = precision_recall_curve(y_train_5, y_scores)

# In[41]:

def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):
    plt.plot(thresholds, precisions[:-1], "b--", label="Precision")
    plt.plot(thresholds, recalls[:-1], "g-", label="Recall")
    plt.xlabel("Threshold")
    plt.legend(loc="upper left")
    plt.ylim([0, 1])
    plt.xlim([-25000, 25000])

# In[42]:

plot_precision_recall_vs_threshold(precisions, recalls, thresholds)
plt.show()

# ### The ROC Curve:  receiver operating characteristic (ROC) curve

# Fom the book:
# "the ROC curve plots the true positive rate (another name for recall) against the false positive rate. The FPR is the ratio of negative instances that are incorrectly classified as positive. It is equal to one minus the true negative rate, which is the ratio of negative instances that are correctly classified as negative. The TNR is also called specificity. Hence the ROC curve plots sensitivity (recall) versus 1 – specificity."

# In[43]:

from sklearn.metrics import roc_curve

fpr, tpr, thresholds = roc_curve(y_train_5, y_scores)

# In[44]:

def plot_roc_curve(fpr, tpr, label=None):
    plt.plot(fpr, tpr, linewidth=2, label=label)
    plt.plot([0, 1], [0, 1], 'k--')
    plt.axis([0, 1, 0, 1])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')

# In[45]:

plot_roc_curve(fpr, tpr)
plt.show()

# "Once again there is a tradeoff: the higher the recall (TPR), the more false positives (FPR) the classifier produces. The dotted line represents the ROC curve of a purely random classifier; a good classifier stays as far away from that line as possible (toward the top-left corner)."

# ### Multiclass Classification

# There are ML algorithms that are able to handle multiclass classification, but ohers than are able only to do binary classification. Nonetheless binary classifier can be used to generate multiclass classifiers using the one versus all strategy, where as many bianry classifiers as classes to classify we have, then we compare the classification scores and chose the one with the higest score. There are different alternatives too, depending on the dataset and on the problem to handle...

# "Scikit-Learn detects when you try to use a binary classification algorithm for a multiclass classification task, and it automatically runs OvA"

# In[46]:

multi_sgd_clf = load_pkl('multi_sgd_clf')
if not multi_sgd_clf:
    multi_sgd_clf.fit(X_train, y_train)
    save_pkl(multi_sgd_clf,'multi_sgd_clf')

# In[52]:

multi_sgd_clf.predict([some_digit])

# In[53]:

some_digit_scores = multi_sgd_clf.decision_function([some_digit])

# In[54]:

np.argmax(some_digit_scores)

# In[55]:

multi_sgd_clf.classes_[5]

# "When a classifier is trained, it stores the list of target classes in its classes_ attribute, ordered by value. In this case, the index of each class in the classes_ array conveniently matches the class itself (e.g., the class at index 5 happens to be class 5), but in general you won’t be so lucky."

# Now let's evaluate the classifier using cross validation:

# In[56]:

multi_cv_scores = cross_val_score(multi_sgd_clf, X_train, y_train, cv=3, scoring="accuracy")

# In[58]:

np.savetxt('multi_cv_scores.np',multi_cv_scores, fmt='%f')

# In[59]:

multi_cv_scores

# This is a good performance, meaning the data is well behaved on how its classes are distributed

# ## Error Analysis

# The goal here is to tweak the model for improving it by exploring the different erros it makes. To do so, let's build its confussion matrix but scaliting the imput parameters first, because this enhances our classifier.

# In[65]:

# sacating input vectors
from sklearn.preprocessing import StandardScaler
if os.path.exists('X_train_scaled.np'):
    X_train_scaled = np.loadtxt('X_train_scaled.np', dtype=float)
else:
    scaler = StandardScaler()
    X_train_scaled = scaler.fit_transform(X_train.astype(np.float64))
    np.savetxt('X_train_scaled.np',X_train_scaled, fmt='%f')

# In[67]:

# Training the model on the scaled input training vector
if os.path.exists('multi_y_train_pred.np'):
    multi_y_train_pred = np.loadtxt('multi_y_train_pred.np', dtype=float)
else:
    multi_y_train_pred = cross_val_predict(multi_sgd_clf, X_train_scaled, y_train, cv=3)
    np.savetxt('multi_y_train_pred.np',multi_y_train_pred, fmt='%f')

# In[68]:

# Confusion matrix
multi_CM = confusion_matrix(y_train, multi_y_train_pred)

# plt.matshow(multi_CM, cmap=plt.cm.gray)
# plt.show()

# In[71]:

# Computing error rates
row_sums = multi_CM.sum(axis=1, keepdims=True)
norm_multi_CM = multi_CM / row_sums
#diagonal with zeros to keep only the errors
np.fill_diagonal(norm_multi_CM, 0)
plt.matshow(norm_multi_CM, cmap=plt.cm.gray)
plt.show()

# In[ ]:

# ## Multilabel classification

# In[ ]:

max_stars_repo_name: ed-ortizm/learningML

max_issues_repo_name: ed-ortizm/learningML

max_forks_repo_name: ed-ortizm/learningML


text:
import numpy as np
import pandas as pd
import os
from pathlib import Path

folder_path = Path("/home/tvhahn/Tool-Wear-Files/temp_csv")

i = 0
for file in os.listdir(folder_path):
    if file.endswith(".csv"):
        if i == 0:
            file_path = folder_path / file
            df = pd.read_csv(file_path, na_filter=False) # load csv and replace NaNs with blanks
            i += 1
        else:
            file_path = folder_path / file
            df = df.append(pd.read_csv(file_path, na_filter=False))
        print(file)

# replace blanks in "failed" column with zeros
df['failed'] = df['failed'].replace(to_replace='',value=0).astype(dtype='int16')

# save as a new csv
df.to_csv(('low_level_labels_created_2020.03.11_v3.csv'),index=False)

ext: py

max_stars_repo_name: tvhahn/Tool-Wear-Files

max_issues_repo_name: tvhahn/Tool-Wear-Files

max_forks_repo_name: tvhahn/Tool-Wear-Files

reason: import numpy


text:
# Importing our liberaries
import pandas as pd
import numpy as np
import time

# Asking the user for the city to do analysis?
# Checked correctly
def choose_city():
	Our_cites = {'chicago':'chicago.csv','washington':'washington.csv','NY':'new_york_city.csv'}
	while True:
		city= input('What city you wnat to look in? "use the same format as [chicago,washington,NY]":\n')
		if city in Our_cites.keys():
			return Our_cites[city]
		else:
			print('wrong iput Please try again\n')
			continue

# Asking for the month filter?
# Checked correctly
def choose_month():
	while True:
		month = input('choose month to filter with "first six month of the year" or cohoose "all" for all month:\n')
		months = ['all','january', 'february', 'march', 'april', 'may', 'june']
		if month.lower() in months:
			return month.title()
		else:
			print('\nincorrect input  Please choose a month or choose all')
			continue

# Asking for the day to filter with?
# Checked correctly
def choose_day():
	days = ['Tuesday','Monday','Friday','Thursday','Wednesday','Saturday','Sunday','All']
	while True:
		day = input('Choose day to filter with or choose all for all days:\n')
		if day.title() in days:
			return day.title()
		else:
			print('\nincorrect input Please try again')
			continue

# loadig Data
# Checked correctly
def data_load(city,month,day):
	"""
	data_load function loads the data and performes data preprossing steps:
		- removes unlabled data
		- converting time strings into timestamp
		- fills the gabs (NaN values) with forward fill method

	arg:
		city(str): city data file in csv
		month(str): optional if you want to filter the data if not inster it as 'All'
		day(str): optional if you want to filter the data with specific day if not insert it as 'All'

	returns the data ready for analysis
	"""

	start_time = time.time()
	df = pd.read_csv(city)

	# Removing unnecessary data (unnammed data)
	try:
		df.pop('Unnamed: 0')
	except:
		pass

	#filling the Nan values if present
	df = df.fillna(method='ffill')
	#converting time string into time stamp useing pd.to_datetime
	df['Start Time'] = pd.to_datetime(df['Start Time'])
	df['End Time'] = pd.to_datetime(df['End Time'])

	# Making the new columns for analysis
	df['month'] = df['Start Time'].apply(lambda x:x.month_name())
	df['start_day_name'] = df['Start Time'].apply(lambda x:x.day_name())
	df['start_hour'] = df['Start Time'].apply(lambda x:x.hour)
	df['trip']=df['Start Station'] + ' --to--> '+ df['End Station']

	# Performing the filteration
	if month != 'All':
		df = df[df['month']==month]

	if day != 'All':
		df = df[df['start_day_name']==day]

	print("\nThis took %s seconds to load the data with filters." % (time.time() - start_time))
	print('-'*80,'\n')
	print('\nstarting the analysis....../')
	print('-'*60)
	return df

# Peforming Data analysis
# Checked correctly
def data_analysis(DF,city):
	'''
	Calculating the required statistics for the data
	'''
	start_time = time.time()

	# The most common month
	print('Which month is the most common ')
	print('-'*20)
	print('Calculating statistics..../')
	msc_month = DF['month'].value_counts().head(1).index[0]
	msc_month_counts = DF['month'].value_counts().head(1).values[0]
	print(f'the most common month is {msc_month} and count: {msc_month_counts}\n')
	print('-'*40)

	# The most common day of the week
	print('which day is more busy?')
	print('-'*20)
	print('Calculating statistics..../')
	msc_day = DF['start_day_name'].value_counts().head(1).index[0]
	msc_day_counts = DF['start_day_name'].value_counts().head(1).values[0]
	print(f'the most common day of the week is {msc_day} and count: {msc_day_counts}\n')
	print('-'*40)

	# The most common hour of the day
	print('Which hour Our Cutomers prefer?')
	print('-'*20)
	print('Calculating statistics..../')
	msc_hour=DF['start_hour'].value_counts().head(1).index[0]
	msc_hour_counts=DF['start_hour'].value_counts().head(1).values[0]
	print(f'the most common hour is {msc_hour} and count: {msc_hour_counts}\n')
	print('-'*40)

	# The most common Stations?
	print('Which start station is more preffered among the Cutomers?')
	print('-'*20)
	print('Calculating statistics..../')
	msc_start_station=DF['Start Station'].value_counts().head(1).index[0]
	msc_start_station_count=DF['Start Station'].value_counts().head(1).values[0]
	print(f'the most common Start Station is: {msc_start_station}, count: {msc_start_station_count}\n')

	print('\nWhcih destination is more used?')
	print('-'*20)
	msc_end_station=DF['End Station'].value_counts().head(1).index[0]
	msc_end_station_count=DF['End Station'].value_counts().head(1).values[0]
	print(f'the most common End Station is: {msc_end_station}, count: {msc_end_station_count}\n')

	print('Whcih trip is more used [start --to--> end]?')
	print('-'*20)
	msc_trip=DF['trip'].value_counts().head(1).index[0]
	msc_trip_count=DF['trip'].value_counts().head(1).values[0]
	print(f'the most common Trip is from: {msc_trip} and count: {msc_trip_count}\n')
	print('-'*40)

	# The trip duration information
	print('What about the trip duration?')
	print('-'*20)
	print('Calculating statistics..../')
	mask = DF[DF['trip']== DF['trip'].mode().values[0]]
	total_travel_time = mask['Trip Duration'].sum()
	average_travel_time = mask['Trip Duration'].mean()
	print(f'"{msc_trip}" "the most used trip" total travel duration is: {total_travel_time} seconds\nand the average travel time is: {round(average_travel_time)} seconds\n')
	print('-'*40)

	# The users info
	print('And Cutomers....?')
	print('-'*20)
	print('Calculating statistics..../')
	user_info = DF['User Type'].value_counts()
	print(f'the User counts are:\n{user_info.index[0]} count: {user_info.values[0]} , {user_info.index[1]} count: {user_info.values[1]}\n')
	print('-'*40)

	# The user Gender info
	if city !='washington':
		print(f'This users gender info for {city} city')
		print('-'*20)
		print('Calculating statistics..../')
		user_gender = DF['Gender'].value_counts()
		user_birth_date = DF['Birth Year'].value_counts()
		print(f'the Users Genders are: {user_gender.index[0]} count: {user_gender.values[0]} and {user_gender.index[1]} count: {user_gender.values[1]}\n')
		print(f'the Most recent year of birth is: {int(user_birth_date.index.max())}\nthe earliest year of birth is: {int(user_birth_date.index.min())}\nthe most common year is: {int(user_birth_date.head(1).index[0])}\n')

	print("\nThis took %s seconds to  calculate the statistics.\n" % (time.time() - start_time))
	print('-'*80,'\n')
# Showing the Raw Data?
# Checked correctly
def raw_data(data):
	step =5
	while True:
		raw = input('Do you want to see the raw data(Y/N)?\n')
		if raw == 'y' or raw ==  'Y':
			view = data[step-5:step]
			# As the screen only shows 2 columns we use for loop to show all data frame
			for i in range(0,len(view.columns)-2):
				print(view[[view.columns[i],view.columns[i+1]]],'\n')
			step +=5
		elif raw == 'n' or raw == 'N':
			print('-'*80,'\n')
			break
		else:
			print('wrong input!!')
			continue

# Performing another analysis?
# Checked correctly
def restart():
	while True:
		restart = input('Want another analysis(Y/N)?\n')
		if restart == 'Y' or restart=='y':
			return True
		elif restart == 'n' or restart=='N':
			return False
		else:
			print('wrong input Please try again\n')
			continue

# The main Function
def main():
	print("\nWelcome!! let's analyse some data\n")
	analyse = True
	while analyse:
		city_file = choose_city()
		month = choose_month()
		day = choose_day()
		print('\nloading data.../')
		print('-'*20,'\n')
		data_frame = data_load(city_file,month,day)
		city = city_file.split('.')[0]
		data_analysis(DF=data_frame,city=city)
		raw_data(data=data_frame)
		analyse=restart()

if __name__=='__main__':
	main()


text:
import numpy as np
import inspect  # Used for storing the input
from .aquifer_parameters import param_maq
from .constant import ConstantStar

class AquiferData:
    def __init__(self, model, kaq, c, z, npor, ltype):
        # All input variables except model should be numpy arrays
        # That should be checked outside this function
        self.model = model
        # Needed for heads
        self.kaq = np.atleast_1d(kaq)
        self.naq = len(kaq)
        self.c = np.atleast_1d(c)
        self.hstar = None
        # Needed for tracing
        self.z = np.atleast_1d(z)
        self.Hlayer = self.z[:-1] - self.z[1:]  # thickness of all layers
        self.nlayers = len(self.z) - 1
        self.npor = np.atleast_1d(npor)
        self.ltype = np.atleast_1d(ltype)
        # tag indicating whether an aquifer is Laplace (confined on top)
        if self.ltype[0] == 'a':
            self.ilap = 1
        else:
            self.ilap = 0
        #
        self.area = 1e200  # Smaller than default of ml.aq so that inhom is found
        self.layernumber = np.zeros(self.nlayers, dtype='int')
        self.layernumber[self.ltype == 'a'] = np.arange(self.naq)
        self.layernumber[self.ltype == 'l'] = np.arange(self.nlayers - self.naq)
        if self.ltype[0] == 'a':
            self.layernumber[self.ltype == 'l'] += 1  # first leaky layer below first aquifer layer
        self.zaqtop = self.z[:-1][self.ltype == 'a']
        self.zaqbot = self.z[1:][self.ltype == 'a']
        self.Haq = self.zaqtop - self.zaqbot
        self.T = self.kaq * self.Haq
        self.Tcol = self.T[:, np.newaxis]
        self.zlltop = self.z[:-1][self.ltype == 'l']
        self.zllbot = self.z[1:][self.ltype == 'l']
        if self.ltype[0] == 'a':
            self.zlltop = np.hstack((self.z[0], self.zlltop))
            self.zllbot = np.hstack((self.z[0], self.zllbot))
        self.Hll = self.zlltop - self.zllbot
        self.nporaq = self.npor[self.ltype == 'a']
        if self.ltype[0] == 'a':
            self.nporll =  np.ones(len(self.npor[self.ltype == 'l']) + 1)
            self.nporll[1:] = self.npor[self.ltype == 'l']
        else:
            self.nporll = self.npor[self.ltype == 'l']

    def initialize(self):
        self.elementlist = []  # Elementlist of aquifer
        d0 = 1.0 / (self.c * self.T)
        d0[:-1] += 1.0 / (self.c[1:] * self.T[:-1])
        dp1 = -1.0 / (self.c[1:] * self.T[1:])
        dm1 = -1.0 / (self.c[1:] * self.T[:-1])
        A = np.diag(dm1, -1) + np.diag(d0, 0) + np.diag(dp1, 1)
        w, v = np.linalg.eig(A)
        # sort lab in decending order, hence w in ascending order
        index = np.argsort(abs(w))
        w = w[index]
        v = v[:, index]
        if self.ilap:
            self.lab = np.zeros(self.naq)
            self.lab[1:] = 1.0 / np.sqrt(w[1:])
            self.zeropluslab = self.lab  # to be deprecated when new lambda is fully implemented
            v[:, 0] = self.T / np.sum(self.T)  # first column is normalized T
        else:
            self.lab = 1.0 / np.sqrt(w)
        self.eigvec = v
        self.coef = np.linalg.solve(v, np.diag(np.ones(self.naq))).T

    def add_element(self, e):
        self.elementlist.append(e)
        if isinstance(e, ConstantStar):
            self.hstar = e.hstar

    def isinside(self, x, y):
        raise Exception('Must overload AquiferData.isinside()')

    def storeinput(self, frame):
        self.inputargs, _, _, self.inputvalues = inspect.getargvalues(frame)

    def findlayer(self, z):
        '''
        Returns layer-number, layer-type and model-layer-number'''
        if z > self.z[0]:
            modellayer, ltype = -1, 'above'
            layernumber = None
        elif z < self.z[-1]:
            modellayer, ltype = len(self.layernumber), 'below'
            layernumber = None
        else:
            modellayer = np.argwhere((z <= self.z[:-1]) & (z >= self.z[1:]))[0, 0]
            layernumber = self.layernumber[modellayer]
            ltype = self.ltype[modellayer]
        return layernumber, ltype, modellayer

class Aquifer(AquiferData):
    def __init__(self, model, kaq, c, z, npor, ltype):
        AquiferData.__init__(self, model, kaq, c, z, npor, ltype)
        self.inhomlist = []
        self.area = 1e300  # Needed to find smallest inhom

    def initialize(self):
        # cause we are going to call initialize for inhoms
        AquiferData.initialize(self)
        for inhom in self.inhomlist:
            inhom.initialize()
        for inhom in self.inhomlist:
            inhom.create_elements()

    def add_inhom(self, inhom):
        self.inhomlist.append(inhom)
        return len(self.inhomlist) - 1  # returns number in the list

    def find_aquifer_data(self, x, y):
        rv = self
        for inhom in self.inhomlist:
            if inhom.isinside(x, y):
                if inhom.area < rv.area:
                    rv = inhom
        return rv
        # Not used anymore I think 5 Nov 2015
        # def find_aquifer_number(self, x, y):
        #    rv = -1
        #    for i,inhom in enumerate(self.inhomlist):
        #        if inhom.isinside(x, y):
        #            if inhom.area < rv.area:
        #                rv = i
        #    return rv

max_stars_repo_name: d-brakenhoff/timml

max_issues_repo_name: d-brakenhoff/timml

max_forks_repo_name: d-brakenhoff/timml


text:
#!/usr/bin/env python3
from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.layers import Dense
from random import randint
from numpy import array
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import tensorflow as tf

####################################################
# PARAMETERS
####################################################

VALUE_COUNT_TRAIN = 10000
MAX_VALUE_TRAIN = 500
MAX_VALUE_TEST = 500
EPOCHS = 50
BATCH_SIZE = 128
VERBOSE = 1
ACTIVATION = "linear"
MODEL_FILE_NAME = "trained_model.h5"

####################################################

# Create Numpy ndarray triples with two numbers and sum¶
def create_triples (count, max):
    numbers = list ()
    result = list ()
    for n in range (count):
        numbers.append ([randint (0, max), randint (0, max)])
        result.append (sum (numbers[n]))
    numbers = array (numbers)
    result = array (result)
    return numbers, result

# Denormalize numbers
def denormalize (value, max):
    return value * float (max)

# Normalize number to range
def normalize (value, max):
    return value.astype ('float') / float (max)

# Model create
def model_create ():
    model = Sequential()
    model.add (Dense (3, input_dim = 2, activation = 'linear'))
    model.add (Dense (5, activation = 'linear'))
    model.add (Dense (1, activation = 'linear'))
    model.compile (loss='mean_squared_error', optimizer='adam')
    return model

# Train model
def model_train (model):
        x, y = create_triples (VALUE_COUNT_TRAIN, MAX_VALUE_TRAIN)
        x2 = normalize (x, MAX_VALUE_TRAIN)
        y2 = normalize (y, MAX_VALUE_TRAIN)
        model.fit (x2,
               y2,
               epochs = EPOCHS,
               batch_size = BATCH_SIZE,
               verbose = VERBOSE)

def save_model (m, filename):
    model.save (filename)

# Load or Create model
load_model_status = False

if load_model_status:
    print ("Load model\n")
    model = load_model (MODEL_FILE_NAME)
else:
    print ("Create model")
    model = model_create ()
    print (model.summary ())
    model_train (model)
    model.save (MODEL_FILE_NAME)
    print ("Save_model\n")

# Prediction
x, y = create_triples (10, MAX_VALUE_TEST)
x2 = normalize (x, MAX_VALUE_TEST)
testresult = model.predict (x2, verbose = 0)

# Show result
print ('Count      Number1        Number2       Total            Right           Error')
for i in range (len (testresult)):
    number1 = denormalize (x2[i][0], MAX_VALUE_TEST)
    number2 = denormalize (x2[i][1], MAX_VALUE_TEST)
    total = denormalize (testresult[i][0], MAX_VALUE_TEST)
    print ('{:4d}. {:12.3f} + {:12.3f} = {:12.3f} === {:12.3f} {:12.3f}'.format (
		i + 1, number1, number2, total, number1 + number2, abs(total - (number1 + number2)))
	)

reason: from numpy


text:
###########################################################################
###########################################################################
#                             SPyH
###########################################################################
###########################################################################
#Authors :  yohan lanier and ruben persicot
#Version : SPyH
###########################################################################
# Kernel Wendland
import numpy as np
from numba import njit
from src.sphvar import *
from src.state import *
from src.contrib import *
from src.spyh import *

@njit
def computeForcesFluidSolid(partMOBILESOLID,partSPID,partPos,partVel,partRho,listNeibSpace,\
                        aW,h,m,ms,B,rhoF,rhoS,gamma,grav,solidAcc,mu,d=2):
    '''
        Compute the forces using Morris viscous forces
        and the RHS for the continuity equation
        input :
            - partMOBILESOLID : True if a MOBILESOLID particle
            - partSPID :  particle space ID
            - partPos : particle position
            - partVel : particle velocity
            - partRho : particle density
            - listNeibSpace : list of the particle influencing a particle in a space spId
            - aW : constant of kernel
            - h : smoothing length
            - m : particle mass
            - ms : solid particle mass
            - B : state constant
            - rhoF : reference density
            - rhoS : solid reference density
            - gamma : polytropic gas constant
            - grav : gravitational acceleration
            - solidAcc : the acceleration of center of mass of the solid
            - mu : viscosity
            - d : dimension
        return :
            - forces : table of the forces on each solid particle
    '''
    forces = np.zeros_like(partVel)
    drhodt = np.zeros_like(partRho)
    nPart = len(partMOBILESOLID)
    for i in range(nPart):
        if partMOBILESOLID[i]:
            spid_i = int(partSPID[i])
            #list neib
            listnb = listNeibSpace[spid_i,:]
            listnb = listnb[listnb>-1]
            listnb = listnb[listnb!=i] #no self contribution
            #---------------------------------------------------
            #removing non fluid particle from the neighbours list
            listnb = listnb[partMOBILESOLID[listnb]==False]
            #---------------------------------------------------
            #Position, norm  and er
            rPos = partPos[listnb][:]-partPos[i,:]
            rNorm = (rPos[:,0]*rPos[:,0]+rPos[:,1]*rPos[:,1])**.5
            q=rNorm/h
            dwdr = Fw(q,aW,h)
            w_ij = wend(q,aW,h)
            er = np.zeros_like(rPos)
            er[:,0] = rPos[:,0]/rNorm
            er[:,1] = rPos[:,1]/rNorm
            #velocity
            v_s = partVel[i,:]
            v_f = partVel[listnb][:]
            rVel = v_f-v_s
            #pressure contrib
            rho_s=partRho[i]
            rho_f=partRho[listnb]
            vol_f=m/rho_f
            P_f=pressure(rho_f,B,rhoF,gamma)
            #solid particle pressure calculated according to  Cf Adami, Hu & Adams, 2012
            P_s = pressureInterpolationContrib(rho_f, P_f, vol_f,rPos,w_ij,grav,solidAcc)
            P_s[P_s<0] = 0
            FFluidSolid = FFluidSolidContrib(P_f, P_s, mu, rho_f, rho_s,dwdr,rVel,rPos,m,ms)
            # We sum the contrib for all fluid particles
            forces[i,:] = np.sum(FFluidSolid,0)
    return forces

@njit
def IntegrateCenterOfMassMovement(partMOBILESOLID,partSPID,partPos,partVel,partRho,listNeibSpace,\
                        aW,h,m,ms,B,rhoF,rhoS,gamma,grav,mu,OG, V_OG,A_OG,part,nSolid,dt):
    '''
        Computes the movement of the center of mass
        input :
            - partMOBILESOLID : True if a MOBILESOLID particle
            - partSPID :  particle space ID
            - partPos : particle position
            - partVel : particle velocity
            - partRho : particle density
            - listNeibSpace : list of the particle influencing a particle in a space spId
            - aW : constant of kernel
            - h : smoothing length
            - m : particle mass
            - ms : solid particle mass
            - B : state constant
            - rhoF : reference density
            - gamma : polytropic gas constant
            - grav : gravitational acceleration
            - OG : the current position of the center of mass
            - V_OG : the current velocity of the center of mass
            - A_OG : the current acceleration of the center of mass
            - mu : viscosity
            - d : dimension
        return :
            - doG : the delta of displacement of the center of mass
            - dV_OG : the delta of velocity of the center of mass
            - A_OG : the new acceleration of the center of mass
    '''
    nPart = len(partMOBILESOLID)
    centerOfMassPosX, centerOfMassPosY = 0.0,0.0
    #compute the center of mass position
    for i in range(nPart):
        if partMOBILESOLID[i]:
            centerOfMassPosX += partPos[i,0]
            centerOfMassPosY += partPos[i,1]
    centerOfMassPos = 1/nSolid*np.array([centerOfMassPosX, centerOfMassPosY])
    #if we are above the free surface, no forces are computed
    if (centerOfMassPos[1]-0.2)>1:
        F=np.array([0.0,0.0])
    else :
        F = np.sum(computeForcesFluidSolid(partMOBILESOLID,partSPID,partPos,partVel,partRho,listNeibSpace,aW,h,m,ms,B,rhoF,       rhoS,gamma,grav,A_OG,mu),0)
    A_OG = grav + F/(ms*nSolid)
    V_OG = V_OG + A_OG*dt
    dOG = V_OG*dt
    return dOG, V_OG, A_OG

@njit
def MoveSolidParticles(partMOBILESOLID, partPos, partVel, dOG, dV_OG):
    '''
    computes the global movement of the solid which is considered rigid
    inputs :
            - partMOBILESOLID : True if a MOBILESOLID particle
            - partPos : particle position
            - partVel : particle velocity
            - doG : the delta of displacement of the center of mass
            - dV_OG : the delta of velocity of the center of mass
    outputs :
            -partPos : the updated table of position
            -partVel : the updated table of velocity
    '''
    nPart = len(partMOBILESOLID)
    for i in range(nPart):
        if partMOBILESOLID[i]:
            partPos[i,0]=partPos[i,0]+dOG[0]
            partPos[i,1]=partPos[i,1]+dOG[1]
            partVel[i,0]=partVel[i,0]+dV_OG[0]
            partVel[i,1]=partVel[i,1]+dV_OG[1]
    return partPos, partVel

@njit
def interpolateMobileSolidBoundary(partMOBILESOLID,partSPID,partPos,partVel,partRho,listNeibSpace,\
                        aW,h,m,B,rhoF,gamma,grav,solidVel,solidAcc,shepardMin = 10**(-6),d=2):
    '''
    interpolate the pressure and velocity on the mobile solid
    input :
        - partMOBILESOLID : table of True,False showing which particle is a MOBILESOLID
        - partSPID : table of particles SPIDs
        - partPos : table of particles positions
        - partVel : table of particles velocities
        - partRho : table of particles density
        - partSpace : table of particle per spaces (-1 are no values)
        - neibSpace : table of spaces neibs
        - aW,d,h : parameters for the wendland
        - m : particle mass
        - B, rhoF, gamma : state equation parameters
        - grav : gravital field acceleration
        - solidVel : velocity of the solid particles --> constant as the solid is considered rigid
        - solidAcc : acceleration of the solid particles
        - shepardMin : threshold for the shepard
    output :
        - partRho : updated table of density with interpolated pressure
        - partVel : new mobile solid velocity
    '''
    nPart = len(partMOBILESOLID)
    for i in range(nPart):
            if partMOBILESOLID[i]:
                spid_i = int(partSPID[i])
                #list neib
                listnb = listNeibSpace[spid_i,:]
                listnb = listnb[listnb>-1]
                listnb = listnb[listnb!=i] #no self contribution
                # keep only the fluid particles
                listnb = listnb[partMOBILESOLID[listnb]==False]
                rPos = partPos[i,:]-partPos[listnb][:]
                rNorm = (rPos[:,0]*rPos[:,0]+rPos[:,1]*rPos[:,1])**.5
                q=rNorm/h
                w_ij = wend(q,aW,h)
                rho_j=partRho[listnb]
                P_j= pressure(rho_j,B,rhoF,gamma)
                rho_j[rho_j<rhoF] = rhoF
                vol_j = m/rho_j
                rho_j= partRho[listnb]
                PressInt = pressureInterpolationContrib(rho_j, P_j, vol_j,rPos,w_ij,grav,solidAcc)
                PressInt[PressInt<0] = 0
                shepard = shepardContrib(vol_j,w_ij)
                shepard = max(np.sum(shepard,0), shepardMin)
                #VELOCITY INTERPOLATION FOR MOBILE BOUNDARIES according to S. Adami's work
                VTildeInt_x = solidVel[0]*vol_j*w_ij
                VTildeInt_y = solidVel[1]*vol_j*w_ij
                partVel[i,0] = 2*solidVel[0] - np.sum(VTildeInt_x,0)/shepard
                partVel[i,1] = 2*solidVel[1] - np.sum(VTildeInt_y,0)/shepard
                pres = np.sum(PressInt,0)/shepard
                partRho[i] = density(pres,B,rhoF,gamma)
    return partRho,partVel

reason: import numpy,from numba


text:
import numpy as np
import numpy.testing as npt
import unittest

from analysis_functions import calculate_recall_success
from network import BCPNNFast, NetworkManager, Protocol

class TestConvergence(unittest.TestCase):
    """
    This should test all the convergence properties of the network under
    the conditions of the free recall paper (parameter values)
    """

    def test_basic_convergence(self):
        """
        This tests a simple network for recall percentage.
        """

        # Patterns parameters
        hypercolumns = 4
        minicolumns = 20

        # Manager properties
        dt = 0.001
        T_recall = 4.0
        values_to_save = ['o']

        # Protocol
        training_time = 0.1
        inter_sequence_interval = 2.0
        inter_pulse_interval = 0.0
        epochs = 1

        # Build the network
        nn = BCPNNFast(hypercolumns, minicolumns)
        nn.k_inner = False

        # Build the manager
        manager = NetworkManager(nn=nn, dt=dt, values_to_save=values_to_save)

        # Build the protocol for
        patterns = [0, 1, 2, 3, 4]
        protocol = Protocol()
        protocol.simple_protocol(patterns_indexes=patterns, training_time=training_time,
                                 inter_pulse_interval=inter_pulse_interval,
                                 inter_sequence_interval=inter_sequence_interval,
                                 epochs=epochs)

        # Train
        manager.run_network_protocol(protocol, verbose=False, values_to_save_epoch=None, reset=True, empty_history=True)

        manager.run_network_recall(T_recall=T_recall, T_cue=0.1, I_cue=0, reset=True, empty_history=True)

        success = calculate_recall_success(manager, T_recall=T_recall, I_cue=0, T_cue=0.100, n=25,
                                           patterns_indexes=patterns)

        self.assertAlmostEqual(success, 100.0)

if __name__ == '__main__':
    unittest.main()


text:
#!/usr/bin/env python
# coding=utf-8
#
# Author: liufr
# Github: https://github.com/Fengrui-Liu
# LastEditTime: 2021-01-11 14:38:40
# Copyright 2021 liufr
# Description:
#

from os.path import dirname, join
import numpy as np
import pandas as pd

class MultivariateDS(object):
    """
    Load multivariate dataset.
    synthetic data point classes
    CLASSES = [
    range(0, 1000),  # sparse benign cluster
    range(1000, 3000),  # dense benign cluster
    range(3000, 3050),  # clustered anomalies
    range(3050, 3075),  # sparse anomalies
    range(3076, 3082),  # local anomalies
    range(3075, 3076),# single anomaly
    ]
    """

    def __init__(self) -> None:
        super().__init__()
        module_path = dirname(__file__)
        data_path = join(module_path, "data", "multiDS.csv")
        self.data = np.loadtxt(data_path)
        self.label = np.array([0] * 3000 + [1] * 82)

class UnivariateDS(object):
    """
    Load univariate dataset.
    """

    def __init__(self) -> None:
        super().__init__()
        module_path = dirname(__file__)
        data_path = join(module_path, "data", "uniDS.csv")
        data = pd.read_csv(data_path)
        self.data = data["value"].to_numpy()
        self.label = data["is_anomaly"].to_numpy()


text:
# This code is part of Qiskit.
#
# (C) Copyright IBM 2018, 2020.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

""" Test NumPy Eigen solver """

import unittest
from test.aqua import QiskitAquaTestCase
import numpy as np
from qiskit.aqua import AquaError
from qiskit.aqua.algorithms import NumPyEigensolver
from qiskit.aqua.operators import WeightedPauliOperator

class TestNumPyEigensolver(QiskitAquaTestCase):
    """ Test NumPy Eigen solver """
    def setUp(self):
        super().setUp()
        pauli_dict = {
            'paulis': [{"coeff": {"imag": 0.0, "real": -1.052373245772859}, "label": "II"},
                       {"coeff": {"imag": 0.0, "real": 0.39793742484318045}, "label": "ZI"},
                       {"coeff": {"imag": 0.0, "real": -0.39793742484318045}, "label": "IZ"},
                       {"coeff": {"imag": 0.0, "real": -0.01128010425623538}, "label": "ZZ"},
                       {"coeff": {"imag": 0.0, "real": 0.18093119978423156}, "label": "XX"}
                       ]
        }
        self.qubit_op = WeightedPauliOperator.from_dict(pauli_dict)

    def test_ce(self):
        """ Test basics """
        algo = NumPyEigensolver(self.qubit_op, aux_operators=[])
        result = algo.run()
        self.assertEqual(len(result.eigenvalues), 1)
        self.assertEqual(len(result.eigenstates), 1)
        self.assertAlmostEqual(result.eigenvalues[0], -1.85727503 + 0j)

    def test_ce_k4(self):
        """ Test for k=4 eigenvalues """
        algo = NumPyEigensolver(self.qubit_op, k=4, aux_operators=[])
        result = algo.run()
        self.assertEqual(len(result.eigenvalues), 4)
        self.assertEqual(len(result.eigenstates), 4)
        np.testing.assert_array_almost_equal(result.eigenvalues.real,
                                             [-1.85727503, -1.24458455, -0.88272215, -0.22491125])

    def test_ce_fail(self):
        """ Test no operator """
        algo = NumPyEigensolver()
        with self.assertRaises(AquaError):
            _ = algo.run()

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: MartenSkogh/qiskit-aqua

max_issues_repo_name: MartenSkogh/qiskit-aqua

max_forks_repo_name: MartenSkogh/qiskit-aqua


text:
import time
from absl import app, flags, logging
from absl.flags import FLAGS

import numpy as np
import cv2
import tensorflow as tf
from yolov3_tf2.models import (
    YoloV3, YoloV3Tiny
)
from yolov3_tf2.dataset import transform_images
from yolov3_tf2.utils import draw_outputs

import zetane

flags.DEFINE_string('classes', './data/coco.names', 'path to classes file')
flags.DEFINE_string('weights', './checkpoints/yolov3.tf',
                    'path to weights file')
flags.DEFINE_boolean('tiny', False, 'yolov3 or yolov3-tiny')
flags.DEFINE_integer('size', 416, 'resize images to')
flags.DEFINE_string('video', './data/video.mp4',
                    'path to video file or number for webcam)')
flags.DEFINE_string('output', None, 'path to output video')
flags.DEFINE_string('output_format', 'XVID', 'codec used in VideoWriter when saving video to file')
flags.DEFINE_integer('num_classes', 80, 'number of classes in the model')

def make_io_panels(zctxt):
    input_panel = zctxt.panel('Input', width=0.25, height=0.3, screen_x=0.0, screen_y=0.7, navigation='2d').set_camera(position=(
        1, 0.75, 30), aim=(1, 0.75, 0)).set_background_color(rgb=(0.025, 0.02, 0.045)).border(3).set_border_alpha(0.05).update()
    output_panel = zctxt.panel('Output', width=0.25, height=0.3, screen_x=0.0, screen_y=0.0, navigation='2d').set_camera(position=(
        1, 0.75, 30), aim=(1, 0.75, 0)).set_background_color(rgb=(0.025, 0.02, 0.045)).border(3).set_border_alpha(0.05).update()
    zctxt.text("Input").font_size(0.1).position(y=-.45).send_to(input_panel).update()
    zctxt.text("Output").font_size(.1).position(y=-.45).send_to(output_panel).update()
    return input_panel, output_panel

def main(_argv):
    physical_devices = tf.config.experimental.list_physical_devices('GPU')
    for physical_device in physical_devices:
        tf.config.experimental.set_memory_growth(physical_device, True)

    if FLAGS.tiny:
        yolo = YoloV3Tiny(classes=FLAGS.num_classes)
    else:
        yolo = YoloV3(classes=FLAGS.num_classes)

    yolo.load_weights(FLAGS.weights)
    logging.info('weights loaded')

    class_names = [c.strip() for c in open(FLAGS.classes).readlines()]
    logging.info('classes loaded')

    times = []

    try:
        vid = cv2.VideoCapture(int(FLAGS.video))
    except:
        vid = cv2.VideoCapture(FLAGS.video)

    out = None

    ctxt = zetane.Context()
    ctxt.clear_universe()
    input_panel, output_panel = make_io_panels(ctxt)

    counter = 0

    if FLAGS.output:
        # by default VideoCapture returns float instead of int
        width = int(vid.get(cv2.CAP_PROP_FRAME_WIDTH))
        height = int(vid.get(cv2.CAP_PROP_FRAME_HEIGHT))
        fps = int(vid.get(cv2.CAP_PROP_FPS))
        codec = cv2.VideoWriter_fourcc(*FLAGS.output_format)
        out = cv2.VideoWriter(FLAGS.output, codec, fps, (width, height))

    while True:
        _, img = vid.read()

        if img is None:
            logging.warning("Empty Frame")
            time.sleep(0.1)
            continue

        img_in = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        img_in = tf.expand_dims(img_in, 0)
        img_in = transform_images(img_in, FLAGS.size)

        image_np = np.transpose(img_in.numpy(), (1, 2, 3, 0))
        if counter == 0:
            to_fit = 0.15 / image_np.shape[2]
            zinput = ctxt.image().data(image_np).scale(to_fit, to_fit).send_to(input_panel).update()
            zmodel = ctxt.model().keras(yolo).inputs(img_in.numpy()).update()
        else:
            zinput.data(image_np).update()
            zmodel.inputs(img_in.numpy()).update()

        t1 = time.time()
        #boxes, scores, classes, nums = yolo.predict(img_in)
        bbox, confidence, class_probs, scores = yolo(img_in)
        boxes, scores, classes, nums = tf.image.combined_non_max_suppression(
            boxes=tf.reshape(bbox, (tf.shape(bbox)[0], -1, 1, 4)),
            scores=tf.reshape(
                scores, (tf.shape(scores)[0], -1, tf.shape(scores)[-1])),
            max_output_size_per_class=FLAGS.yolo_max_boxes,
            max_total_size=FLAGS.yolo_max_boxes,
            iou_threshold=FLAGS.yolo_iou_threshold,
            score_threshold=FLAGS.yolo_score_threshold
        )

        t2 = time.time()
        times.append(t2-t1)
        times = times[-20:]

        out_img = draw_outputs(img/255.0, (boxes, scores, classes, nums), class_names)
        out_img = cv2.putText(out_img, "Time: {:.2f}ms".format(sum(times)/len(times)*1000), (0, 30), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 0, 255), 2)

        if counter == 0:
            to_fit = 0.15 / out_img.shape[2]
            zoutput = ctxt.image().data(out_img).scale(to_fit, to_fit).send_to(output_panel).update()
        else:
            zoutput.data(out_img).debug();

        #if FLAGS.output:
        #    out.write(img)
        #cv2.imshow('output', img)

        counter += 1
        if cv2.waitKey(1) == ord('q'):
            break

    ctxt.disconnect()
    cv2.destroyAllWindows()

if __name__ == '__main__':
    try:
        app.run(main)
    except SystemExit:
        pass


text:
# -*- coding: utf-8 -*-
"""
Created on Sun Apr 15 12:40:58 2018

@author: Abhishek
"""

from keras.models import load_model
from keras.preprocessing import image
import numpy as np
import cv2
model = load_model('agrimodel.h5')
model.compile(loss='categorical_crossentropy',
                      optimizer='adam',
                      metrics=['accuracy'])
from skimage.io import imread
from skimage.transform import resize

def detect(frame):
    try:
        img = resize(frame,(64,64))
        img = np.expand_dims(img,axis=0)
        if(np.max(img)>1):
            img = img/255.0
        prediction = model.predict(img)
        print(prediction)
        p=np.argmax(prediction)

        if(p==0):
            print ("DAISY")
            return "daisy"
            print ("ROSE")
            return "rose"
        elif(p==1):
            print("DANDELION")
            return "dandelion"
        elif(p==2):
            print ("ROSE")
            return "rose"
        elif(p==3):
            print("SUNFLOWER")
            return "sunflower"
        else:
            print("TULIP")
            return "tulip"
    except AttributeError:
        print("shape not found")

frame=cv2.imread("sunflower.jpg")
data = detect(frame)

max_stars_repo_path: flowers prediction using CNN.py

max_issues_repo_path: flowers prediction using CNN.py

max_forks_repo_path: flowers prediction using CNN.py


text:
"""
Deals with accessing the components of the xmile file, and
formatting them for the builder

James Houghton <james.p.houghton@gmail.com>
Alexey Prey Mulyukin <alexprey@yandex.ru> from sdCloud.io development team.

"""
from __future__ import absolute_import
from .SMILE2Py import SMILEParser
from lxml import etree
from ...py_backend import builder, utils

import numpy as np

def translate_xmile(xmile_file):
    """ Translate an xmile model file into a python class.
    Functionality is currently limited.

    """

    # process xml file
    xml_parser = etree.XMLParser(encoding="utf-8", recover=True)
    root = etree.parse(xmile_file, parser=xml_parser).getroot()
    NS = root.nsmap[None]  # namespace of the xmile document

    def get_xpath_text(node, path, ns=None, default=''):
        """ Safe access of occassionally missing elements """
        # defined here to take advantage of NS in default
        if ns is None:
            ns = {'ns': NS}
        try:
            return node.xpath(path, namespaces=ns)[0].text
        except:
            return default

    def get_xpath_attrib(node, path, attrib, ns=None, default=None):
        """ Safe access of occassionally missing elements """
        # defined here to take advantage of NS in default
        if ns is None:
            ns = {'ns': NS}
        try:
            return node.xpath(path, namespaces=ns)[0].attrib[attrib]
        except:
            return default

    def is_constant_expression(py_expr):
        try:
            val = float(py_expr)
            return True
        except ValueError:
            return False

    def parse_lookup_xml_node(node):
        ys_node = node.xpath('ns:ypts', namespaces={'ns': NS})[0]
        ys = np.fromstring(ys_node.text, dtype=np.float, sep=ys_node.attrib['sep'] if 'sep' in ys_node.attrib else ',')
        xscale_node = node.xpath('ns:xscale', namespaces={'ns': NS})
        if len(xscale_node) > 0:
            xmin = xscale_node[0].attrib['min']
            xmax = xscale_node[0].attrib['max']
            xs = np.linspace(float(xmin), float(xmax), len(ys))
        else:
            xs_node = node.xpath('ns:xpts', namespaces={'ns': NS})[0]
            xs = np.fromstring(xs_node.text, dtype=np.float, sep=xs_node.attrib['sep'] if 'sep' in xs_node.attrib else ',')

        type = node.attrib['type'] if 'type' in node.attrib else 'continuous'
        functions_map = {
            'continuous': 'functions.lookup',
            'extrapolation': 'functions.lookup_extrapolation',
            'discrete': 'functions.lookup_discrete'
        }
        lookup_function = functions_map[type] if type in functions_map else functions_map['continuous']

        return {
            'name': node.attrib['name'] if 'name' in node.attrib else '',
            'xs': xs,
            'ys': ys,
            'type': type,
            'function': lookup_function
        }

    # build model namespace
    namespace = {
        'TIME': 'time',
        'Time': 'time',
        'time': 'time'
    }  # namespace of the python model
    names_xpath = '//ns:model/ns:variables/ns:aux|' \
                  '//ns:model/ns:variables/ns:flow|' \
                  '//ns:model/ns:variables/ns:stock|' \
                  '//ns:model/ns:variables/ns:gf'

    for node in root.xpath(names_xpath, namespaces={'ns': NS}):
        name = node.attrib['name']
        _, namespace = utils.make_python_identifier(name, namespace)

    model_elements = []
    smile_parser = SMILEParser(namespace)

    # add aux and flow elements
    flaux_xpath = '//ns:model/ns:variables/ns:aux|//ns:model/ns:variables/ns:flow'
    for node in root.xpath(flaux_xpath, namespaces={'ns': NS}):
        name = node.attrib['name']
        units = get_xpath_text(node, 'ns:units')
        lims = (get_xpath_attrib(node, 'ns:range', 'min'), get_xpath_attrib(node, 'ns:range', 'max'))
        lims = str(tuple(float(x) if x is not None else x for x in lims))
        doc = get_xpath_text(node, 'ns:doc')
        py_name = namespace[name]
        eqn = get_xpath_text(node, 'ns:eqn')

        element = {
            'kind': 'component',
            'real_name': name,
            'unit': units,
            'doc': doc,
            'eqn': eqn,
            'lims': lims,
            'py_name': py_name,
            'subs': [],  # Todo later
            'arguments': '',
        }

        tranlation, new_structure = smile_parser.parse(eqn, element)
        element.update(tranlation)
        if is_constant_expression(element['py_expr']):
            element['kind'] = 'constant'

        model_elements += new_structure

        gf_node = node.xpath("ns:gf", namespaces={'ns': NS})
        if len(gf_node) > 0:
            gf_data = parse_lookup_xml_node(gf_node[0])

            element.update({
                'kind': 'lookup',
                # This lookup declared as inline, so we should implement inline mode for flow and aux
                'arguments': "x = None",
                'py_expr': "%(function)s(%(x)s, [%(xs)s], [%(ys)s]) if x is None else %(function)s(x, [%(xs)s], [%(ys)s])" % {
                    'function': gf_data['function'],
                    'xs': ','.join("%10.3f" % x for x in gf_data['xs']),
                    'ys': ','.join("%10.3f" % x for x in gf_data['ys']),
                    'x': element['py_expr']
                },
            })

        model_elements.append(element)

    # add gf elements
    gf_xpath = '//ns:model/ns:variables/ns:gf'
    for node in root.xpath(gf_xpath, namespaces={'ns': NS}):
        name = node.attrib['name']
        py_name = namespace[name]

        units = get_xpath_text(node, 'ns:units')
        doc = get_xpath_text(node, 'ns:doc')

        gf_data = parse_lookup_xml_node(node)

        element = {
            'kind': 'lookup',
            'real_name': name,
            'unit': units,
            'lims': None,
            'doc': doc,
            'eqn': '',
            'py_name': py_name,
            'py_expr': "%(function)s(x, [%(xs)s], [%(ys)s])" % {
                    'function': gf_data['function'],
                    'xs': ','.join("%10.3f" % x for x in gf_data['xs']),
                    'ys': ','.join("%10.3f" % x for x in gf_data['ys']),
                },
            'arguments': 'x',
            'subs': [],  # Todo later
        }
        model_elements.append(element)

    # add stock elements
    stock_xpath = '//ns:model/ns:variables/ns:stock'
    for node in root.xpath(stock_xpath, namespaces={'ns': NS}):
        name = node.attrib['name']
        units = get_xpath_text(node, 'ns:units')
        lims = (get_xpath_attrib(node, 'ns:range', 'min'), get_xpath_attrib(node, 'ns:range', 'max'))
        lims = str(tuple(float(x) if x is not None else x for x in lims))
        doc = get_xpath_text(node, 'ns:doc')
        py_name = namespace[name]

        # Extract input and output flows equations
        inflows = [n.text for n in node.xpath('ns:inflow', namespaces={'ns': NS})]
        outflows = [n.text for n in node.xpath('ns:outflow', namespaces={'ns': NS})]

        eqn = ' + '.join(inflows) if inflows else ''
        eqn += (' - ' + ' - '.join(outflows)) if outflows else ''

        element = {
            'kind': 'component' if inflows or outflows else 'constant',
            'real_name': name,
            'unit': units,
            'doc': doc,
            'eqn': eqn,
            'lims': lims,
            'py_name': py_name,
            'subs': [],  # Todo later
            'arguments': ''
        }

        # Parse each flow equations
        py_inflows = []
        for inputFlow in inflows:
            translation, new_structure = smile_parser.parse(inputFlow, element)
            py_inflows.append(translation['py_expr'])
            model_elements += new_structure

        # Parse each flow equations
        py_outflows = []
        for outputFlow in outflows:
            translation, new_structure = smile_parser.parse(outputFlow, element)
            py_outflows.append(translation['py_expr'])
            model_elements += new_structure

        py_ddt = ' + '.join(py_inflows) if py_inflows else ''
        py_ddt += (' - ' + ' - '.join(py_outflows)) if py_outflows else ''

        # Read the initial value equation for stock element
        initial_value_eqn = get_xpath_text(node, 'ns:eqn')
        translation, new_structure = smile_parser.parse(initial_value_eqn, element)
        py_initial_value = translation['py_expr']
        model_elements += new_structure

        py_expr, new_structure = builder.add_stock(identifier=py_name,
                                                   subs=[],  # Todo later
                                                   expression=py_ddt,
                                                   initial_condition=py_initial_value,
                                                   subscript_dict={},  # Todo later
                                                   )
        element['py_expr'] = py_expr
        model_elements.append(element)
        model_elements += new_structure

    # remove timestamp pieces so as not to double-count
    model_elements_parsed = []
    for element in model_elements:
        if element['real_name'].lower() not in ['initial time', 'final time', 'time step', 'saveper']:
            model_elements_parsed.append(element)
    model_elements = model_elements_parsed

    # Add timeseries information

    # Read the start time of simulation
    sim_spec_node = root.xpath('//ns:sim_specs', namespaces={'ns': NS});
    time_units = sim_spec_node[0].attrib['time_units'] if (len(sim_spec_node) > 0 and 'time_units' in sim_spec_node[0].attrib) else ""

    tstart = root.xpath('//ns:sim_specs/ns:start', namespaces={'ns': NS})[0].text
    element = {
        'kind': 'constant',
        'real_name': 'INITIAL TIME',
        'unit': time_units,
        'lims': None,
        'doc': 'The initial time for the simulation.',
        'eqn': tstart,
        'py_name': 'initial_time',
        'subs': None,
        'arguments': '',
    }
    translation, new_structure = smile_parser.parse(tstart, element)
    element.update(translation)
    model_elements.append(element)
    model_elements += new_structure

    # Read the final time of simulation
    tstop = root.xpath('//ns:sim_specs/ns:stop', namespaces={'ns': NS})[0].text
    element = {
        'kind': 'constant',
        'real_name': 'FINAL TIME',
        'unit': time_units,
        'lims': None,
        'doc': 'The final time for the simulation.',
        'eqn': tstart,
        'py_name': 'final_time',
        'subs': None,
        'arguments': '',
    }

    translation, new_structure = smile_parser.parse(tstop, element)
    element.update(translation)
    model_elements.append(element)
    model_elements += new_structure

    # Read the time step of simulation
    dt_node = root.xpath('//ns:sim_specs/ns:dt', namespaces={'ns': NS})

    # Use default value for time step if `dt` is not specified in model
    dt_eqn = "1.0"
    if len(dt_node) > 0:
        dt_node = dt_node[0]
        dt_eqn = dt_node.text
        # If reciprocal mode are defined for `dt`, we should inverse value
        if ("reciprocal" in dt_node.attrib and dt_node.attrib["reciprocal"].lower() == "true"):
            dt_eqn = "1/" + dt_eqn

    element = {
        'kind': 'constant',
        'real_name': 'TIME STEP',
        'unit': time_units,
        'lims': None,
        'doc': 'The time step for the simulation.',
        'eqn': dt_eqn,
        'py_name': 'time_step',
        'subs': None,
        'arguments': '',
    }
    translation, new_structure = smile_parser.parse(dt_eqn, element)
    element.update(translation)
    model_elements.append(element)
    model_elements += new_structure

    # Add the SAVEPER attribute to the model
    model_elements.append({
        'kind': 'constant',
        'real_name': 'SAVEPER',
        'unit': time_units,
        'lims': None,
        'doc': 'The time step for the simulation.',
        'eqn': dt_eqn,
        'py_name': 'saveper',
        'py_expr': 'time_step()',
        'subs': None,
        'arguments': '',
    })

    outfile_name = xmile_file.replace('.xmile', '.py')

    builder.build(elements=model_elements,
                  subscript_dict={},
                  namespace=namespace,
                  outfile_name=outfile_name)

    return outfile_name

max_stars_repo_name: arnislektauers/pysd

max_issues_repo_name: arnislektauers/pysd

max_forks_repo_name: arnislektauers/pysd


text:
#From https://gist.github.com/EndingCredits/b5f35e84df10d46cfa716178d9c862a3
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import state_ops
from tensorflow.python.framework import ops
from tensorflow.python.training import optimizer
import tensorflow as tf
import hyperchamber as hc
import numpy as np
import inspect
from operator import itemgetter
from hypergan.train_hooks.base_train_hook import BaseTrainHook

class GradientPenaltyTrainHook(BaseTrainHook):
  def __init__(self, gan=None, config=None, trainer=None, name="GradientPenaltyTrainHook"):
    super().__init__(config=config, gan=gan, trainer=trainer, name=name)
    if hasattr(self.gan, 'x0'):
        gan_inputs = self.gan.x0
    else:
        gan_inputs = self.gan.inputs.x

    self._lambda = self.gan.configurable_param(config['lambda'] or 1)

    if self.config.target:
        v = getattr(gan, self.config.target)
    else:
        v = gan.discriminator
    target = v.sample
    if "components" in self.config:
        target_vars = []
        for component in self.config.components:
            c = getattr(gan, component)
            target_vars += c.variables()
    else:
        target_vars = self.gan.variables()

    gd = tf.gradients(target, target_vars)
    gds = [tf.square(_gd) for _gd in gd if _gd is not None]
    if self.config.flex:
        if isinstance(self.config.flex,list):
            gds = []
            for i,flex in enumerate(self.config.flex):
                split_target = tf.split(target, len(self.config.flex))
                gd = tf.gradients(split_target, target_vars)
                fc = self.gan.configurable_param(flex)
                gds += [tf.square(tf.nn.relu(tf.abs(_gd) - fc)) for _gd in gd if _gd is not None]
        else:
            gds = [tf.square(tf.nn.relu(tf.abs(_gd) - flex)) for _gd in gd if _gd is not None]
    self.loss = tf.add_n([self._lambda * tf.reduce_mean(_r) for _r in gds])
    self.gds = gds
    self.gd = gd
    self.target_vars = target_vars
    self.target = target
    self.gan.add_metric('gp', self.loss)

  def losses(self):
    if self.config.loss == "g_loss":
        return [None, self.loss]
    else:
        return [self.loss, None]

  def after_step(self, step, feed_dict):

    if self.config.debug:
        for _v, _g in zip(self.target_vars, self.gd):
            if(_g is not None and np.mean(self.gan.session.run(_g)) > 0.1):
                print(' -> ' + _v.name,  _v, _g)
                print(" in target?", _v in self.target_vars)
                print(" in dvars? ",_v in self.gan.d_vars())
                print(" in t_dvars? ",_v in self.gan.trainable_d_vars())
                print(" in gvars? ",_v in self.gan.g_vars())
    pass

  def before_step(self, step, feed_dict):
    pass

max_issues_repo_name: KonradLinkowski/HyperGAN

max_forks_repo_name: KonradLinkowski/HyperGAN


text:
import numpy as np
import pytest

import eniric.snr_normalization as snrnorm
import eniric.utilities as utils
from scripts.phoenix_precision import convolve_and_resample

xfail = pytest.mark.xfail

@xfail(raises=ModuleNotFoundError, reason="Issue with Starfish install.")
@pytest.mark.parametrize("desired_snr", [100.0, 150.0])
@pytest.mark.parametrize("band", ["J", "Y", "VIS"])
def test_snr_normalization(desired_snr, band, testing_spectrum):
    """Test SNR after normalizing function is the desired value.

    Testing on middle of J band.
    """
    wav, flux = testing_spectrum
    band_mid = utils.band_middle(band)

    # Searching for the closest index to 1.25
    index_reference = np.searchsorted(wav, [band_mid])[0]
    snr_estimate = np.sqrt(np.sum(flux[index_reference - 1 : index_reference + 2]))

    assert round(snr_estimate, 1) != desired_snr  # Assert SNR is not correct

    norm_const = snrnorm.snr_constant_band(wav, flux, snr=desired_snr, band=band)

    new_flux = flux / norm_const
    new_snr_estimate = np.sqrt(
        np.sum(new_flux[index_reference - 1 : index_reference + 2])
    )

    assert round(new_snr_estimate, 0) == desired_snr

@xfail(raises=ModuleNotFoundError, reason="Issue with Starfish install.")
@pytest.mark.parametrize("desired_snr", [50.0, 200.0])
@pytest.mark.parametrize("band", ["H", "GAP", "K"])
def test_snr_normalization_constant(desired_snr, band, testing_spectrum):
    """Test snr_constant_band and snr_constant_wav produce same result."""
    wav, flux = testing_spectrum
    band_mid = utils.band_middle(band)

    assert snrnorm.snr_constant_band(
        wav, flux, band=band, snr=desired_snr
    ) == snrnorm.snr_constant_wav(wav, flux, band_mid, snr=desired_snr)

@xfail(raises=ModuleNotFoundError, reason="Issue with Starfish install.")
@pytest.mark.parametrize("sampling", [3, 4.1])
def test_band_snr_norm(testing_spectrum, sampling):
    """Compared to wav snr norm."""
    wav, flux = testing_spectrum
    wav, flux = convolve_and_resample(
        wav, flux, vsini=1, R=100_000, band="J", sampling=sampling
    )
    assert snrnorm.snr_constant_band(
        wav, flux, band="J", snr=100, sampling=sampling
    ) == snrnorm.snr_constant_wav(wav, flux, wav_ref=1.25, snr=100, sampling=sampling)

    assert snrnorm.snr_constant_band(
        wav, flux, band="J", snr=100, sampling=sampling, verbose=True
    ) != snrnorm.snr_constant_wav(
        wav, flux, wav_ref=1.24, snr=100, sampling=sampling, verbose=True
    )

def test_sampling_index():
    """Some hard coded examples of sampling index."""
    # odd number
    assert snrnorm.sampling_index(6, 1) == [6]
    assert np.all(snrnorm.sampling_index(100, 3) == [99, 100, 101])
    assert np.all(snrnorm.sampling_index(5, 5) == [3, 4, 5, 6, 7])

    # even number
    assert np.all(snrnorm.sampling_index(10, 4) == [8, 9, 10, 11])
    assert np.all(snrnorm.sampling_index(10, 2) == [9, 10])
    # number that is at end of array.

@pytest.mark.parametrize("sample", [1, 2, 3, 4, 5, 7, 10, 15])
def test_sampling_size_and_type(sample):
    # Check number of values correct.
    for sample in [1, 2, 3, 4, 5, 7, 10, 15]:
        assert len(snrnorm.sampling_index(20, sample)) == sample
        assert isinstance(
            snrnorm.sampling_index(20, sample)[0], (int, np.integer)
        )  # index values must be int

def test_sampling_index_array():
    """Sampling index when array_length is given, or when index goes out of bounds."""
    assert np.all(snrnorm.sampling_index(100, 3, array_length=200) == [99, 100, 101])

    with pytest.raises(ValueError):
        snrnorm.sampling_index(3, 10)  # index will be < 0
    with pytest.raises(ValueError):
        snrnorm.sampling_index(3, 9, array_length=50)  # index will be < 0
    with pytest.raises(ValueError):
        snrnorm.sampling_index(
            46, 10, array_length=50
        )  # an index will be > (array_length - 1)

@pytest.mark.parametrize("band", ["VIS", "Z", "NIR", "J", "Y"])
def test_snr_constant_band_returns_mid_value_const(band):
    size = 100
    np.random.seed(40)
    flux = 500 * np.random.rand(
        size
    )  # To give a random spectrum (but consistent between tests)
    lim = utils.band_limits(band)
    wav = np.linspace(lim[0], lim[1], size)

    band_const = snrnorm.snr_constant_band(wav, flux, band=band)
    wav_const = snrnorm.snr_constant_wav(wav, flux, wav_ref=utils.band_middle(band))

    assert isinstance(band_const, float)
    assert isinstance(wav_const, float)
    assert band_const == wav_const  # Since band calls wave at midpoint

@pytest.mark.parametrize("band", ["VIS", "K", "H"])
@pytest.mark.parametrize("verbose", [True, False])
def test_snr_normalization_logic(band, verbose):
    """Testing direct value.

    snr = sqrt(sum(3 pixels))
    const = (snr/ref_snr)**2
    if pixel value = 3 then  the normalization constant will be 1.
    """
    size = 100
    band = "K"
    lim = utils.band_limits(band)
    wav = np.linspace(lim[0], lim[1], size)
    flux = 3 * np.ones(size)
    band_const = snrnorm.snr_constant_band(wav, flux, snr=3, band=band, verbose=verbose)
    wav_const = snrnorm.snr_constant_wav(
        wav, flux, snr=3, wav_ref=(lim[0] + lim[1]) / 2, verbose=verbose
    )
    assert band_const == 1
    assert wav_const == 1

@pytest.mark.parametrize(
    "wav,band",
    [
        (np.linspace(0.8, 1, 50), "VIS"),  # "VIS": (0.38, 0.78)
        (np.linspace(2, 3, 50), "J"),  # "J": (1.17, 1.33)
        (np.linspace(2.0, 2.1, 50), "K"),  # "K": (2.07, 2.35)
        (np.linspace(2.25, 2.4, 50), "K"),  # "K": (2.07, 2.35)
    ],
)
def test_snr_constant_band_with_invalid_wavelength(wav, band):
    with pytest.raises(ValueError):
        snrnorm.snr_constant_band(wav, np.ones(50), band=band)

@pytest.mark.parametrize("wav_ref", [0.5, 4])
def test_snr_constant_wav_ref_outside_wav(wav_ref):
    """Wav-ref outside bounds of wav should raise ValueError"""
    wav = np.linspace(1, 3, 60)
    flux = np.random.randn(len(wav))

    with pytest.raises(ValueError):
        snrnorm.snr_constant_wav(wav, flux, wav_ref)

max_stars_repo_name: sourcery-ai-bot/eniric

max_issues_repo_name: sourcery-ai-bot/eniric

max_forks_repo_name: sourcery-ai-bot/eniric


text:
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Privacy computation for DP-FTRL with given data order."""

from absl import app
import numpy as np
from typing import List
from collections import Counter
import json

def convert_gaussian_renyi_to_dp(sigma, delta, verbose=True):
    """
    Convert from RDP to DP for a Gaussian mechanism.
    :param sigma: the algorithm guarantees (alpha, alpha/(2*sigma^2))-RDP
    :param delta: target DP delta
    :param verbose: whether to print message
    :return: the DP epsilon
    """
    alphas = np.arange(1, 200, 0.1)[1:]
    epss = alphas / 2 / sigma**2 - (np.log(delta*(alphas - 1)) - alphas * np.log(1 - 1/alphas)) / (alphas - 1)
    idx = np.nanargmin(epss)
    if verbose and idx == len(alphas) - 1:
        print('The best alpha is the last one. Consider increasing the range of alpha.')
    eps = epss[idx]
    return eps

def get_total_sensitivity_sq_given_order(order):
    """
    Get the squared sensitivity for a given order of batches.
    Can be viewed as a general case for get_total_sensitivity_sq_same_order.
    This function is not used in the privacy computation, as we operated in the case where the
    data order is the same for every epoch.

    :param order: a list representing the order of the batches, e.g. [0,1,2,1] means we use batch indexed with 0,1,2,1.
                  -1 indicates virtual step.
    :return: squared sensitivity, squared_sensitivity with respect to all every batch
    """
    # get first layer as a list of counters
    layer = [Counter({node: 1}) for node in order]

    # sensitivity_sq[i] will record the total sensitivity wrt batch i
    sensitivity_sq_all = [0] * (max(order) + 1)

    # update sensitivity_sq with a given layer
    def update_sensitivity_sq(current_layer):
        for node in current_layer:
            for ss in node:
                if ss != -1:
                    sensitivity_sq_all[ss] += node[ss] ** 2

    update_sensitivity_sq(layer)  # get sensitivity for the first layer
    while len(layer) > 1:
        layer_new = []  # merge every two consecutive nodes to get the next layer
        length = len(layer)
        for i in range(0, length, 2):
            if i + 1 < length:
                layer_new.append(layer[i] + layer[i + 1])
        layer = layer_new
        update_sensitivity_sq(layer)
    return max(sensitivity_sq_all), sensitivity_sq_all

def get_total_sensitivity_sq_same_order(steps_per_epoch, epochs, extra_steps, mem_fn=None):
    """
    Get the squared sensitivty for a tree where we fix the order of batches for all epochs.

    :param steps_per_epoch: number of steps per epoch
    :param epochs: number of epochs in the tree
    :param extra_steps: number of virtual steps
    :param mem_fn: if set, will write result to the file
    :return: squared sensitivity, squared sensivity assuming no virtual steps,
             squared sensitivity with respect to every batch

    e.g. steps_per_epochs = 3 and epochs = 2, extra_steps = 2 means we have three batches b1, b2, b3,
    and train w/ [b1, b2, b3, b1, b2, b3, +, +] where + means the extra steps.
    We will enumerate through all nodes layer by layer in list "layer", and  compute the sensitivty
    with respect to every node in "sensitivity_sq".
    """
    # to record the result to save computation
    mem = json.load(open(mem_fn)) if mem_fn else {}
    key = f'{steps_per_epoch},{epochs},{extra_steps}'
    key_no_extra = f'{steps_per_epoch},{epochs},{0}'
    if key in mem and key_no_extra in mem:
        return mem[key], mem[key_no_extra], None

    # get first layer as a list of counters, the keys are batches (indexed with non-negative numbers), counts are
    # number of times the batch appears in the node
    layer = []
    for _ in range(epochs):
        layer += [Counter({ss: 1}) for ss in range(steps_per_epoch)]
    layer += [Counter({-1: 1}) for _ in range(extra_steps)]  # extra steps denoted as -1

    # sensitivity_sq[i] will record the total sensitivity wrt batch i
    sensitivity_sq_all = [0] * steps_per_epoch
    sensitivity_sq_all_no_extra = [0] * steps_per_epoch  # will also compute sensitivity without extra

    # update sensitivity_sq with a given layer
    def update_sensitivity_sq(current_layer):
        for node in current_layer:
            has_extra = -1 in node
            for ss in node:
                if ss != -1:
                    sensitivity_sq_all[ss] += node[ss] ** 2
                    if not has_extra:
                        sensitivity_sq_all_no_extra[ss] += node[ss] ** 2

    update_sensitivity_sq(layer)  # get sensitivity for the first layer
    while len(layer) > 1:
        layer_new = []  # merge every two consecutive nodes to get the next layer
        length = len(layer)
        for i in range(0, length, 2):
            if i + 1 < length:
                layer_new.append(layer[i] + layer[i + 1])
        del layer
        layer = layer_new
        update_sensitivity_sq(layer)

    # save to file
    if mem_fn:
        mem[key] = max(sensitivity_sq_all)
        mem[key_no_extra] = max(sensitivity_sq_all_no_extra)
        with open(mem_fn, 'w') as f:
            json.dump(mem, f, indent=4)
    return max(sensitivity_sq_all), max(sensitivity_sq_all_no_extra), sensitivity_sq_all

def compute_epsilon_tree_restart_rdp_same_order_extra(num_batches: int, epochs_between_restarts: List[int],
                                                      noise: float, tree_completion: bool = True,
                                                      mem_fn: str = None):
    """
    Compute the effective noise for DP-FTRL.

    :param num_batches: number of batches per epoch
    :param epochs_between_restarts: number of epochs between each restart, e.g. [2, 1] means epoch1, epoch2, restart, epoch3
    :param noise: noise multiplier for each step
    :param tree_completion: if true, use the tree completion trick which adds virtual steps to complete the binary tree
    :param mem_fn: if set, will write result to the file
    :return: the effective noise for DP-FTRL
    """
    if noise < 1e-20:
        return float('inf')

    mem = {}  # to record result to avoid computing the same setting twice
    sensitivity_sq = 0  # total sensitivity^2, which is the sum over all "intervals" between each restarting
    for i, epochs in enumerate(epochs_between_restarts):
        if epochs == 0:
            continue
        if tree_completion and i < len(epochs_between_restarts) - 1:
            # compute number of virtual steps
            extra_steps = 2 ** (num_batches * epochs - 1).bit_length() - num_batches * epochs
        else:
            extra_steps = 0
        key = (num_batches, epochs, extra_steps)
        mem[key] = mem.get(key,
                           get_total_sensitivity_sq_same_order(num_batches, epochs, extra_steps, mem_fn)[0])
        sensitivity_sq += mem[key]
    effective_sigma = noise / np.sqrt(sensitivity_sq)
    return effective_sigma

def compute_epsilon_tree(num_batches: int, epochs_between_restarts: List[int], noise: float, delta: float,
                         tree_completion: bool,
                         verbose=True, mem_fn=None):
    """
    Compute epsilon value for DP-FTRL.

    :param num_batches: number of batches per epoch
    :param epochs_between_restarts: number of epochs between each restart, e.g. [2, 1] means epoch1, epoch2, restart, epoch3
    :param noise: noise multiplier for each step
    :param delta: target DP delta
    :param tree_completion: if true, use the tree completion trick which adds virtual steps to complete the binary tree
    :param verbose: whether to print message
    :param mem_fn: if set, will write result to the file
    :return: the DP epsilon for DP-FTRL
    """

    if noise < 1e-20:
        return float('inf')

    effective_sigma = compute_epsilon_tree_restart_rdp_same_order_extra(num_batches, epochs_between_restarts, noise,
                                                                        tree_completion, mem_fn)
    eps = convert_gaussian_renyi_to_dp(effective_sigma, delta, verbose)
    return eps

def main(_):
    # An example for CIFAR-10 (50000 samples) with batch=500, restarting every 20 epochs for 100 epochs in total,
    # noise=46.3, using the tree completion trick.
    n = 50000
    delta = 1e-5
    batch = 500
    epochs = 100
    restart_every = 20
    noise = 46.3
    tree_completion = True

    num_batches = n // batch
    epochs_between_restarts = [restart_every] * (epochs // restart_every)

    eps = compute_epsilon_tree(num_batches, epochs_between_restarts, noise, delta, tree_completion)

    print(f'n={n}, batch={batch}, epochs={epochs} with restarting every {restart_every} epochs',
          f'noise={noise}, tree_completion={tree_completion}',
          f'gives ({eps:.2f}, {delta})-DP')

if __name__ == '__main__':
    app.run(main)

max_stars_repo_name: google-research/DP-FTRL

max_issues_repo_name: google-research/DP-FTRL

max_forks_repo_name: google-research/DP-FTRL


text:
import numpy as np
import matplotlib.pyplot as plt
import util as ut
import ImgUtil as iu

class ImgViewer:

    def __init__(self, w=4, h=4, rows=1, cols=1, title = "", svr=None, auto_save=False):
        self.enabled = True
        self.img_obj_list = []
        self.w = w
        self.h = h
        self.title = title
        self.rows = rows
        self.cols = cols
        self.svr = svr
        self.auto_save = auto_save

    def set_enabled(self, enabled):
        self.enabled = enabled

    def bce_set_enabled(self, enabled): #"bce" means "everything" in russian
        self.set_enabled(enabled)
        if self.svr:
            self.svr.set_enabled(enabled)

    def push(self, img,  debug=False):
        if not self.enabled:
            return
        assert(type(img) is iu.Image)

        self.img_obj_list.append(img)

        if (debug):
            img.show()

        if self.svr and self.auto_save:
            self.svr.save(img)

    def pop(self):
        return self.img_obj_list.pop()

    def flush(self):
        self.img_obj_list = []

    def show_1_grid(self, start):
        L = self.img_obj_list
        n_imgs = len(L)

        plt.figure(figsize=(self.w, self.h))
        for i in range(self.rows * self.cols):
            ix = start + i
            if ix >= n_imgs:
                break
            plt.subplot(self.rows, self.cols, i + 1)
            L[ix].plot(_plt=plt)
            plt.xticks([], [])
            plt.yticks([], [])
        plt.show()

    def show(self, clear=False):
        if not self.enabled:
            return
        L = self.img_obj_list
        n_imgs = len(L)
        grid_size = self.rows * self.cols

        for i in range(0, n_imgs, grid_size):
            self.show_1_grid(i)
        if clear:
            self.flush()

    def show_immed(self, img_list, title=""):
        # note: you can use this via pdb: vwr.show_immed(img)
        # ex: vwr.show_immed([in_img, top_down])
        # ex: cache_dict['viewer'].show_immed([hls_binary_l])
        for img in img_list:
            assert(type(img) is iu.Image)
            plt.figure()
            plt.title(img.title)
            plt.imshow(img.img_data, cmap=img.cmap)
            plt.show()

    def show_immed_ndarray(self, img=None, title=None, img_type=None):
        assert(type(img) is np.ndarray)
        tmp = iu.Image(img_data = img, title=title, img_type=img_type)
        self.show_immed(tmp, title)

def _view(vwr, img, title):
    # turn off viewing by passing None as viewer
    if vwr:
        vwr.show_immed(img, title)

def _push(vwr, img_obj):
    assert(type(img_obj) is iu.Image)
    if vwr:
        vwr.push(img_obj)

def _push_deprecated(vwr, img_obj):
    ut.brk("you didn't mean that")
    raise Exception("I TOLD you that you didn't mean that")

def _flush(vwr):
    if vwr:
        vwr.flush()

def _pop(vwr):
    if vwr:
        vwr.pop()

def _show(vwr, clear=False):
    if vwr:
        vwr.show(clear)

max_stars_repo_name: evtHsa/sdcAdvLaneFinding

max_issues_repo_name: evtHsa/sdcAdvLaneFinding

max_forks_repo_name: evtHsa/sdcAdvLaneFinding


text:
# 검색 엔진
import os
import re

import django
import numpy as np
import pandas as pd
import requests
import random
from bs4 import BeautifulSoup

from .rq_class import *

### 데이터 처리 메소드
def findTag(lst, tag):
    return lst[lst.index(tag)+1]

# 모든 검색결과 파싱
def parseAll(req):
    result = []

    tmp = requests.get(req)
    soup = BeautifulSoup(tmp.content, 'html.parser')

    # 파싱할 페이지 수 카운트
    c_tmp = list(re.split('[<>]', str(soup.select('totalcount'))))
    try:
        page_count = int(findTag(c_tmp, 'totalcount'))
    except:
        page_count = 1

    if page_count < 10:
        page_count = 1
    else:
        page_count = (page_count // 10) + 1

    # 내용 파싱
    for i in range(1, page_count+1):
        req_tmp = req+'&{0}={1}'.format('pageNo', i)

        res_tmp = requests.get(req_tmp)
        soup = BeautifulSoup(res_tmp.content, 'html.parser')
        data = soup.find_all('item')

        if data:
            result.extend(data)
        else:
            break

    return result # 결과 리스트 반환

# 한 페이지만 파싱
def parseOne(req):
    result = []

    req = req + '&pageNo=1'
    tmp = requests.get(req)
    soup = BeautifulSoup(tmp.content, 'html.parser')
    data = soup.find_all('item')

    for elm in data:
        result.append(elm)

    return result

# 태그 낀 객체에서 검색결과 분해하기
def getInfos(tag_obj):
    # 태그 <> / 문자 등 제거
    tmp = re.split('[<>]', str(tag_obj))

    for elm in tmp:
        if '/' in elm or elm == 'item':
            if 'http' in elm: continue
            else: tmp.remove(elm)
    while '' in tmp: tmp.remove('')
    while '[' in tmp: tmp.remove('[')
    while ']' in tmp: tmp.remove(']')
    while '\n' in tmp: tmp.remove('\n')

    # 요소 정리
    infos = {}

    try:
        for i in range(0, len(tmp), 2):
            infos[tmp[i]] = tmp[i+1]
        return infos # 딕셔너리 형태로 반환
    except:
        return 'num of contents is not an even number. check it out.'

# 중복 제거 후 반환(리스트, 리스트 원소=딕셔너리 일 때)
def checkInfos(lst, key):
    hash_space = set()
    new_lst = []
    for elm in lst:
        if hash(elm[key]) not in hash_space:
            hash_space.add(hash(elm[key]))
            new_lst.append(elm)
        else: continue
    return new_lst

# 지역코드값 대치
# 코드값 받아오면 한글 명칭으로 바꿔줌
def findGeo(df, dic):
    if not geo_file: return

    try:
        result = {'code1':'', 'code2':'',}

        val1 = int(dic['areacode'])
        val2 = int(dic['sigungucode'])

        row = df[ (df['city_code'] == val1) & (df['sigungu_code'] == val2) ]

        result['code1'] = row.iloc[0, 1]
        result['code2'] = row.iloc[0, 3]

        dic['areacode'] = result['code1']
        dic['sigungucode'] = result['code2']
    except:
        return

# 서비스코드값 대치
# 코드값 받아오면 한글 명칭으로 바꿔줌
def findSer(df, dic):
    if not ser_file: return

    try:
        result = {'code1':'', 'code2':'', 'code3':'',}
        val1 = dic['cat1']
        val2 = dic['cat2']
        val3 = dic['cat3']

        row = df[df['code3'] == str(val3)]

        result['code1'] = row.iloc[0, 1]
        result['code2'] = row.iloc[0, 3]
        result['code3'] = row.iloc[0, 5]

        dic['cat1'] = result['code1']
        dic['cat2'] = result['code2']
        dic['cat3'] = result['code3']
    except:
        return

### 데이터 처리 메소드 -끝-

### 검색 메소드

# 키워드로 검색
def searchByKeyword(word, apiInfo):
    req = tourReq('ETC', 'AppTest', apiInfo.mykey)
    req.addPara('keyword', word)
    return parseAll(req.makeReq(apiInfo.url, 'searchKeyword'))

# 무장애 정보 조회
def searchBF(id, apiInfo):
    req = tourReq('ETC', 'AppTest', apiInfo.mykey)
    req.addPara('contentId', id)
    tmp = parseAll(req.makeReq(apiInfo.url, 'detailWithTour'))
    result = getInfos(tmp)
    return result

# 상세 정보 조회
def searchDetails(contentId, typeId, apiInfo):
    req = tourReq('ETC', 'AppTest', apiInfo.mykey)
    req.addPara('contentId', contentId)

    # 이미지갤러리
    images = []
    tmp3 = parseAll(req.makeReq(apiInfo.url, 'detailImage'))
    for items in tmp3:
        images.append(getInfos(items))

    # 반복내용조회
    repetitive = []
    req.addPara('contentTypeId', typeId)
    tmp1 = parseAll(req.makeReq(apiInfo.url, 'detailInfo'))
    for items in tmp1:
        repetitive.append(getInfos(items))

    # 소개정보조회
    introduction = []
    tmp2 = parseAll(req.makeReq(apiInfo.url, 'detailIntro'))
    for items in tmp2:
        introduction.append(getInfos(items))

    infos = {'img':images, 'rep':repetitive, 'intro':introduction}

    return infos

# 인기장소 검색(n개, 사진 필수/무방 옵션)
def popularSites(site_num, pic_option, apiInfo):
    req = tourReq('ETC', 'AppTest', apiInfo.mykey)
    req.addPara('numOfRows', site_num)

    if pic_option:
        req.addPara('arrange', 'P')
    else:
        req.addPara('arrange', 'B')

    tmp = parseOne(req.makeReq(apiInfo.url, 'areaBasedList'))

    sites = []
    for elm in tmp:
        sites.append(getInfos(elm))

    return sites

# 0개의 장소 평가를 가지고 있는 초기 사용자에게 top8 장소 평가 제공
def initialSites(site_num, pic_option, apiInfo):
    req = tourReq('ETC', 'AppTest', apiInfo.mykey)
    req.addPara('numOfRows', site_num)

    rn_arrange = random.randint(0,3)
    rn_geocode = random.randint(1,39)

    if pic_option:
        req.addPara('arrange', 'P')
    else:
        req.addPara('arrange', 'B')

    tmp = parseOne(req.makeReq(apiInfo.url, 'areaBasedList'))

    results_list = []
    for elm in tmp:
        elm_tmp = getInfos(elm)
        dic_tmp = {'title':elm_tmp.get('title'), 'addr':elm_tmp.get('addr1'), 'contentId':elm_tmp.get('contentid'),
        'firstimage':elm_tmp.get('firstimage')}
        results_list.append(dic_tmp)

    return results_list

# 무작위장소 검색(n개, 사진 필수/무방 옵션)
def randomSites(site_num, pic_option, apiInfo):
    req = tourReq('ETC', 'AppTest', apiInfo.mykey)
    req.addPara('numOfRows', site_num)

    rn_arrange = random.randint(0,3)
    rn_geocode = random.randint(1,39)

    if pic_option:
        if rn_arrange == 0:
            req.addPara('arrange', 'O')
        elif rn_arrange == 1:
            req.addPara('arrange', 'P')
        elif rn_arrange == 2:
            req.addPara('arrange', 'Q')
        elif rn_arrange == 3:
            req.addPara('arrange', 'R')
    else:
        if rn_arrange == 0:
            req.addPara('arrange', 'A')
        elif rn_arrange == 1:
            req.addPara('arrange', 'B')
        elif rn_arrange == 2:
            req.addPara('arrange', 'C')
        elif rn_arrange == 3:
            req.addPara('arrange', 'D')

    req.addPara('areaCode', rn_geocode)

    tmp = parseOne(req.makeReq(apiInfo.url, 'areaBasedList'))

    results_list = []
    for elm in tmp:
        elm_tmp = getInfos(elm)
        dic_tmp = {'title':elm_tmp.get('title'), 'addr':elm_tmp.get('addr1'), 'contentId':elm_tmp.get('contentid'),
        'firstimage':elm_tmp.get('firstimage')}
        results_list.append(dic_tmp)

    return results_list

def simUserSites(words, site_num, pic_option, apiInfo):
    req = tourReq('ETC', 'AppTest', apiInfo.mykey)
    req.addPara('numOfRows', site_num)

    if pic_option:
        req.addPara('arrange', 'P')
    else:
        req.addPara('arrange', 'B')

    sites = []
    for i in words :
        req.addPara('keyword', i)
        try:
            elm = parseOne(req.makeReq(apiInfo.url, 'searchKeyword'))[0]
        except:
            pass
        sites.append(getInfos(elm))

    return sites

### 검색 메소드 -끝-

### 데이터 처리용 변수

# tag값 대응 위한 딕셔너리
tag_names ={
    'addr1' : '주소',
    'areacode' : '지역',
    'cat1' : '대분류',
    'cat2' : '중분류',
    'cat3' : '소분류',
    'contenttypeid' : '장소유형',
    'sigungucode' : '시군구',
    'title' : '장소명',
    'BF' : '무장애 정보'
}

# 무장애 정보 태그용 딕셔너리
bf_tags ={
    'parking' : '주차',
    'route' : '대중교통',
    'publictransport' : '접근로',
    'ticketoffice' : '매표소',
    'promotion' : '홍보물',
    'wheelchair' : '휠체어',
    'exit' : '출입통로',
    'elevator' : '엘레베이터',
    'restroom': '화장실',
    'auditorium': '관람석',
    'room' : '객실',
    'handicapetc' : '지체장애 편의시설',
    'braileblock' : '점자블록',
    'helpdog' : '보조견동반',
    'guidehuman' : '안내요원',
    'audioguide' : '오디오 가이드',
    'bigprint' : '큰활자 홍보물',
    'brailepromotion' : '점자홍보물 및 점자표지판',
    'guidesystem' : '유도안내설비',
    'blindhandicapetc' : '시각장애 편의시설',
    'signguide' : '수화안내',
    'videoguide' : '영상자막안내',
    'hearingroom' : '청각 객실',
    'hearinghandicapetc' : '청각장애 편의시설',
    'stroller' : '유모차',
    'lactationroom' : '수유실',
    'babysparechair' : '유아용 보조의자',
    'infantsfamilyetc' : '영유아가족 기타상세',
}

# 지역, 서비스 코드값 대응 위한 csv 로드
geo_file = False
ser_file = False
try:
    geo_df = pd.read_csv("./search/resources/geo_code.csv", encoding='euc-kr')
    ser_df = pd.read_csv("./search/resources/service_code.csv", encoding='euc-kr')
    geo_file = True
    ser_file = True
except:
    print('err: code csv files do not exist')
    print(os.getcwd())

### 데이터 처리용 변수 -끝-

# x, y좌표 취합
def getGeoInfos(contentId):
    apiInfo = ApiInfo('1a%2FLc1roxNrXp8QeIitbwvJdfpUYIFTcrbii4inJk3m%2BVpFvZSWjHFmOfWiH9T7TMbv07j5sDnJ5yefVDqHXfA%3D%3D', 'http://api.visitkorea.or.kr/openapi/service/rest/KorWithService/')
    req = tourReq('ETC', 'AppTest', apiInfo.mykey)
    req.addPara('contentId', contentId)
    req.addPara('mapinfoYN', 'Y')
    req.addPara('defaultYN', 'Y')
    tmp = parseOne(req.makeReq(apiInfo.url, 'detailCommon'))
    result = getInfos(tmp)

    geo_dict = {'mapx':float(result.get('mapx')),'mapy':float(result.get('mapy')), 'siteId':contentId, 'siteName':result.get('title'),}

    return geo_dict

max_stars_repo_name: OneUne/weltrip-web

max_issues_repo_name: OneUne/weltrip-web

max_forks_repo_name: OneUne/weltrip-web