text:
"""
@author:
    David J. Cox, PhD, MSB, BCBA-D
    dcox33@jhmi.edu
    https://www.researchgate.net/profile/David_Cox26
"""

# Packages!!
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from sklearn import metrics
import seaborn as sns

#Show current working directory.
dirpath = os.getcwd()
print(dirpath)

#Set current working directory to the folder that contains your data.
# Home PC
os.chdir('C:/Users/coxda/Dropbox/Projects/Current Project Manuscripts/Empirical/Are there different types of discounting/Data')
# Work Mac
os.chdir('/Users/dcox/Dropbox/Projects/Current Project Manuscripts/Empirical/Are there different types of discounting/Data')

# Import data.
data_all = pd.read_csv('ALLDiscountRates.csv', low_memory=False) #All data.
data_cocaine = pd.read_csv('DiscountRatesCocaine.csv', low_memory=False) #Cocaine data.
data_control = pd.read_csv('DiscountRatesControl.csv', low_memory=False) #Control data.
data_cocaine_ml = pd.read_csv('DiscountRatesCocaineML.csv', low_memory=False) #Cocaine data.
data_control_ml = pd.read_csv('DiscountRatesControlML.csv', low_memory=False) #Control data.

#Check the headers to make sure everything looks right.
data_all.head()
data_cocaine_ml.head()
data_control_ml.head()

#Drop task names from first column for analysis.
data_cocaine_ready = data_cocaine_ml.drop(['Unnamed: 0'], axis=1)
data_cocaine_ready.head()

data_control_ready = data_control_ml.drop(['Unnamed: 0'], axis=1)
data_control_ready.head()

#Check the shape of the data
print(data_all.shape)
print(data_cocaine.shape)
print(data_control.shape)

#Check to make sure we have all participants accounted for in each group.
print(len(data_all))
print(len(data_cocaine))
print(len(data_control))

# Create pairplot. Shows all discounting rates as a function of all other discounting rates.
sns.pairplot(data_cocaine)
sns.pairplot(data_control)

"""""""""""""""""""""""""""""""""
    Commence k-means clustering.
"""""""""""""""""""""""""""""""""
# Build clustering models for cocaine group.
kmeans_2 = KMeans(n_clusters =2)
kmeans_2.fit(data_cocaine_ready)

kmeans_3 = KMeans(n_clusters =3)
kmeans_3.fit(data_cocaine_ready)

kmeans_4 = KMeans(n_clusters =4)
kmeans_4.fit(data_cocaine_ready)

kmeans_5 = KMeans(n_clusters =5)
kmeans_5.fit(data_cocaine_ready)

kmeans_6 = KMeans(n_clusters =6)
kmeans_6.fit(data_cocaine_ready)

kmeans_7 = KMeans(n_clusters =7)
kmeans_7.fit(data_cocaine_ready)

kmeans_8 = KMeans(n_clusters =8)
kmeans_8.fit(data_cocaine_ready)

kmeans_9 = KMeans(n_clusters =9)
kmeans_9.fit(data_cocaine_ready)

kmeans_10 = KMeans(n_clusters =10)
kmeans_10.fit(data_cocaine_ready)

kmeans_11 = KMeans(n_clusters =11)
kmeans_11.fit(data_cocaine_ready)

kmeans_12 = KMeans(n_clusters =12)
kmeans_12.fit(data_cocaine_ready)

kmeans_13 = KMeans(n_clusters =13)
kmeans_13.fit(data_cocaine_ready)

kmeans_14 = KMeans(n_clusters =14)
kmeans_14.fit(data_cocaine_ready)

kmeans_15 = KMeans(n_clusters =15)
kmeans_15.fit(data_cocaine_ready)

kmeans_16 = KMeans(n_clusters =16)
kmeans_16.fit(data_cocaine_ready)

kmeans_17 = KMeans(n_clusters =17)
kmeans_17.fit(data_cocaine_ready)

kmeans_18 = KMeans(n_clusters =18)
kmeans_18.fit(data_cocaine_ready)

kmeans_19 = KMeans(n_clusters =19)
kmeans_19.fit(data_cocaine_ready)

kmeans_20 = KMeans(n_clusters =20)
kmeans_20.fit(data_cocaine_ready)

print("2 Cluster memberships:\n{}".format(kmeans_2.labels_))
print("3 Cluster memberships:\n{}".format(kmeans_3.labels_))
print("4 Cluster memberships:\n{}".format(kmeans_4.labels_))
print("5 Cluster memberships:\n{}".format(kmeans_5.labels_))
print("6 Cluster memberships:\n{}".format(kmeans_6.labels_))
print("7 Cluster memberships:\n{}".format(kmeans_7.labels_))
print("8 Cluster memberships:\n{}".format(kmeans_8.labels_))
print("9 Cluster memberships:\n{}".format(kmeans_9.labels_))
print("10 Cluster memberships:\n{}".format(kmeans_10.labels_))
print("11 Cluster memberships:\n{}".format(kmeans_11.labels_))
print("12 Cluster memberships:\n{}".format(kmeans_12.labels_))
print("13 Cluster memberships:\n{}".format(kmeans_13.labels_))
print("14 Cluster memberships:\n{}".format(kmeans_14.labels_))
print("15 Cluster memberships:\n{}".format(kmeans_15.labels_))
print("16 Cluster memberships:\n{}".format(kmeans_16.labels_))
print("17 Cluster memberships:\n{}".format(kmeans_17.labels_))
print("18 Cluster memberships:\n{}".format(kmeans_18.labels_))
print("19 Cluster memberships:\n{}".format(kmeans_19.labels_))
print("20 Cluster memberships:\n{}".format(kmeans_20.labels_))

# Build clustering models for control group.
kmeans_2 = KMeans(n_clusters =2)
kmeans_2.fit(data_control_ready)

kmeans_3 = KMeans(n_clusters =3)
kmeans_3.fit(data_control_ready)

kmeans_4 = KMeans(n_clusters =4)
kmeans_4.fit(data_control_ready)

kmeans_5 = KMeans(n_clusters =5)
kmeans_5.fit(data_control_ready)

kmeans_6 = KMeans(n_clusters =6)
kmeans_6.fit(data_control_ready)

kmeans_7 = KMeans(n_clusters =7)
kmeans_7.fit(data_control_ready)

kmeans_8 = KMeans(n_clusters =8)
kmeans_8.fit(data_control_ready)

kmeans_9 = KMeans(n_clusters =9)
kmeans_9.fit(data_control_ready)

kmeans_10 = KMeans(n_clusters =10)
kmeans_10.fit(data_control_ready)

kmeans_11 = KMeans(n_clusters =11)
kmeans_11.fit(data_control_ready)

kmeans_12 = KMeans(n_clusters =12)
kmeans_12.fit(data_control_ready)

kmeans_13 = KMeans(n_clusters =13)
kmeans_13.fit(data_control_ready)

kmeans_14 = KMeans(n_clusters =14)
kmeans_14.fit(data_control_ready)

kmeans_15 = KMeans(n_clusters =15)
kmeans_15.fit(data_control_ready)

kmeans_16 = KMeans(n_clusters =16)
kmeans_16.fit(data_control_ready)

kmeans_17 = KMeans(n_clusters =17)
kmeans_17.fit(data_control_ready)

kmeans_18 = KMeans(n_clusters =18)
kmeans_18.fit(data_control_ready)

kmeans_19 = KMeans(n_clusters =19)
kmeans_19.fit(data_control_ready)

print("2 Cluster memberships:\n{}".format(kmeans_2.labels_))
print("3 Cluster memberships:\n{}".format(kmeans_3.labels_))
print("4 Cluster memberships:\n{}".format(kmeans_4.labels_))
print("5 Cluster memberships:\n{}".format(kmeans_5.labels_))
print("6 Cluster memberships:\n{}".format(kmeans_6.labels_))
print("7 Cluster memberships:\n{}".format(kmeans_7.labels_))
print("8 Cluster memberships:\n{}".format(kmeans_8.labels_))
print("9 Cluster memberships:\n{}".format(kmeans_9.labels_))
print("10 Cluster memberships:\n{}".format(kmeans_10.labels_))
print("11 Cluster memberships:\n{}".format(kmeans_11.labels_))
print("12 Cluster memberships:\n{}".format(kmeans_12.labels_))
print("13 Cluster memberships:\n{}".format(kmeans_13.labels_))
print("14 Cluster memberships:\n{}".format(kmeans_14.labels_))
print("15 Cluster memberships:\n{}".format(kmeans_15.labels_))

"""""""""
    Identify optimal number of clusters for cocaine group from Rousseeuw, P.J. (1987).
    Silhouettes: a Graphical Aid to the Interpretation and Validation of Cluster Analysis.
    Computational and Applied Mathematics 20: 53–65. doi:10.1016/0377-0427(87)90125-7.
"""""""""
Sum_of_squared_distances_cocaine = []
Silhouette_Score_cocaine = []
K_cocaine = range(2, 20)
for k in K_cocaine:
    km = KMeans(n_clusters = k, random_state=1).fit(data_cocaine_ready)
    labels = km.labels_
    silhouette = metrics.silhouette_score(data_cocaine_ready, labels, metric='euclidean')
    Sum_of_squared_distances_cocaine.append(km.inertia_)
    Silhouette_Score_cocaine.append(silhouette)

# Identify optimal number of clusters for control group
Sum_of_squared_distances_control = []
Silhouette_Score_control = []
K_control = range(2, 16)
for k in K_control:
    km = KMeans(n_clusters = k, random_state=1).fit(data_control_ready)
    labels = km.labels_
    silhouette = metrics.silhouette_score(data_control_ready, labels, metric='euclidean')
    Sum_of_squared_distances_control.append(km.inertia_)
    Silhouette_Score_control.append(silhouette)

# Create elbow plot for cocaine group.
plt.plot(K_cocaine, Sum_of_squared_distances_cocaine, 'o-', color='black')
plt.xlabel('Number of Clusters')
plt.xticks(np.arange(0, 20, step=2))
plt.yticks(np.arange(0, 9000, step=2000))
plt.ylabel('Inertia')
plt.title('Cocaine Group')

# Create plot of silhouette scores for cocaine group
plt.plot(K_cocaine, Silhouette_Score_cocaine, 'o-', color='black')
plt.xlabel('Number of Clusters')
plt.xticks(np.arange(0, 20, step=2))
plt.yticks(np.arange(0, 0.6, step=0.1))
plt.ylabel('Silhouette Score')
plt.title('Cocaine Group')

# Create elbow plot for control group.
plt.plot(K_control, Sum_of_squared_distances_control, 'o-', color='black')
plt.xlabel('Number of Clusters')
plt.xticks(np.arange(0, 20, step=2))
plt.ylabel('Inertia')
plt.yticks(np.arange(0, 9000, step=2000))
plt.title('Control Group')

# Create plot of silhouette scores for control group
plt.plot(K_control, Silhouette_Score_control, 'o-', color='black')
plt.xlabel('Number of Clusters')
plt.xticks(np.arange(0, 20, step=2))
plt.yticks(np.arange(0, 0.6, step=0.1))
plt.ylabel('Silhouette Score')
plt.title('Control Group')

# Show cluster scores.
list(Silhouette_Score_cocaine)
list(Silhouette_Score_control)


text:
import numpy as np
import pandas as pd

from bokeh import mpl
from bokeh.plotting import output_file, show

ts = pd.Series(np.random.randn(1000), index=pd.date_range('1/1/2000', periods=1000))
ts = ts.cumsum()

df = pd.DataFrame(np.random.randn(1000, 4), index=ts.index, columns=list('ABCD'))
df = df.cumsum()
df.plot(legend=False)

output_file("pandas_dataframe.html", title="pandas_dataframe.py example")

show(mpl.to_bokeh())

ext: py

max_stars_repo_name: quasiben/bokeh

max_issues_repo_name: quasiben/bokeh

max_forks_repo_name: quasiben/bokeh

reason: import numpy


text:
"""
Main file for viewing data
"""

import os
import optparse
import time
import cPickle as pickle
import numpy as np
from skimage import color
import scipy.misc as sm
import skimage.draw as draw
import cv, cv2
from pylab import *

from pyKinectTools.dataset_readers.KinectPlayer import KinectPlayer, display_help
from pyKinectTools.dataset_readers.BasePlayer import BasePlayer
from pyKinectTools.utils.SkeletonUtils import *
from pyKinectTools.algs.smij import smij
# from pyKinectTools.dataset_readers.CADPlayer import CADPlayer

import mlpy
# from scipy.interpolate import interp1d
from scipy.interpolate import *
from scipy.signal import resample

""" Debugging """
from IPython import embed

# 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14
CAD_DISABLED = [7,9,8,10,13,14]
# -------------------------MAIN------------------------------------------

def get_skels_per_sequence(skels, labels):

	skel_segs = {}
	for i,lab in labels.items():
		start = lab['start']
		stop = lab['stop']
		name = lab['subaction']
		if name in skel_segs.keys():
			skel_segs[name] += [skels[start:stop]]
		else:
			skel_segs[name] = [skels[start:stop]]

	return skel_segs

def main():
	# Setup kinect data player

	record = False
	DIR = '/Users/colin/Data/CAD_120/'
	# cam = CADPlayer(base_dir=DIR, get_color=False, get_skeleton=True, subjects=[1], actions=range(10), instances=[1])
	cam = CADPlayer(base_dir=DIR, get_color=False, get_depth=True, get_skeleton=True,
					subjects=[1,3,4], actions=[1], instances=[0,1,2])

	if record:
		writer = cv2.VideoWriter(filename="CAD_action_{}.avi".format(2), fps=30,
							frameSize=(640,480),fourcc=cv.CV_FOURCC('I','4','2','0'), isColor=0)

	framerate = 1
	temporal_window = 1*(14) # 14 FPS
	skel_angles_all = []
	subactivities = []
	skels_subactivity = {}
	while cam.next(framerate):
		# window_start = np.maximum(0, cam.frame-temporal_window)

		# Get joint angles
		if cam.frame == 1:
			skels = cam.skel_stack['pos']
			skel_angles = np.array([get_CAD_skel_angles(x) for x in skels])
			skel_angles = np.nan_to_num(skel_angles)

			tmp = get_skels_per_sequence(skel_angles, cam.subactivity)
			for t in tmp:
				if t not in skels_subactivity:
					skels_subactivity[t] = []
				skels_subactivity[t] += tmp[t]

			skel_angles_all += [skel_angles]
			subactivities += [cam.subactivity]
			cam.next_sequence()
		# break

		# Compute variance over the past second
		# cam.skels_pos_conf[CAD_DISABLED] = 0
		# smij_features, smij_variance = smij(cam.skel_stack['pos'][window_start:cam.frame], confidence=cam.skels_pos_conf)
		# smij_features = smij_features[:3]
		# print smij_features

		# Add annotations
		cam.depthIm = np.repeat(cam.depthIm[: ,: ,None], 3, -1)
		max_val  = cam.depthIm.max()
		# Label actions
		cv2.putText(cam.depthIm, "Action: "+cam.action, (20,60), cv2.FONT_HERSHEY_DUPLEX, 1, (max_val,0,0), thickness=2)
		cv2.putText(cam.depthIm, "Subaction: "+cam.subaction, (20,85), cv2.FONT_HERSHEY_DUPLEX, 1, (max_val,0,0), thickness=2)
		# Label object affordances
		for o in cam.objects:
			pt1 = tuple(o['topleft'].astype(np.int))
			pt2 = tuple(o['bottomright'].astype(np.int))
			if not all(pt1) or not all(pt2):
				continue
			cv2.rectangle(cam.depthIm, pt1, pt2, (0,0,max_val))
			pt1 = (pt1[0], pt1[1]+15)
			cv2.putText(cam.depthIm, cam.object_names[o["ID"]], pt1, cv2.FONT_HERSHEY_DUPLEX, .6, (0,0,max_val), thickness=1)
		# Label skeleton
		if cam.depth_stack is not None:
			# Hilight active joints
			cam.depthIm = display_skeletons(cam.depthIm, cam.users_uv[0], skel_type='CAD', color=(0,max_val,0), confidence=cam.skels_pos_conf, alt_color=(0,max_val/5,max_val/4))

			# Highlight SMIJ joints
			# smij_top = np.ones(cam.users_uv[0].shape[0], np.int)
			# smij_top[smij_features] = 0
			# cam.depthIm = display_skeletons(cam.depthIm, cam.users_uv[0], skel_type='CAD', color=(0,max_val/4,0), confidence=smij_top, alt_color=(0,max_val,0))

		cam.visualize(color=True, depth=True)#, depth_bounds=[0,5000])

		if record:
			writer.write((cam.depthIm/cam.depthIm.max()*255).astype(np.uint8))

	for it in range(len(skel_angles_all)):
		for i,angs in enumerate(skel_angles_all[it].T):
			figure(i)
			plot(angs)
			title(CAD_JOINTS[i])
	# from scipy.signal import cspline1d, cspline1d_eval

	''' it appears to be matching to the wrong subactivity? in DTW '''
	for i_key,key in enumerate(skels_subactivity):
		figure(key)
		# title(key)
		print "{} iterations of {}".format(len(skels_subactivity[key]), key)
		for i_iter in range(len(skels_subactivity[key])):
			print 'iter', i_iter
			for i,ang in enumerate(skels_subactivity[key][i_iter].T):
				x = skels_subactivity[key][0][:,i]
				y = ang
				y = resample(y, len(x))
				error, dtw_mat, y_ind = mlpy.dtw.dtw_std(x, y, dist_only=False)
				error, dtw_mat, y_ind = mlpy.dtw.dtw_subsequence(y, x)

				subplot(3,4,i+1)
				y_new = y[y_ind[0]]
				x_new = np.linspace(0, 1, len(y_new))
				# poly = polyfit(x_new, y_new, 5)
				# y_spline_ev = poly1d(poly)(x_new)

				nknots = 4
				idx_knots = (np.arange(1,len(x_new)-1,(len(x_new)-2)/np.double(nknots))).astype('int')
				knots = x_new[idx_knots]
				y_spline = splrep(x_new, y_new, t=knots)
				y_spline_ev = splev(np.linspace(0, 1, len(y_new)), y_spline)
				# plot(y_new)
				plot(y_spline_ev)
				# show()
				# plot(y[y_ind[0]])

				# subplot(3,10,i+1 + i_iter*10)
				# plot(x[y_ind[1]])
				# plot(y[y_ind[0]])
				print i,":", len(ang), len(x), len(y[y_ind[0]])
				title(CAD_JOINTS[i])

				if i == 10:
					break
	show()

# Use bezier curves?

# import mlpy
# x = skel_angles_all[0].T[1]
# y = skel_angles_all[1].T[1]
# error, dtw_mat, y_ind = mlpy.dtw.dtw_std(y, x, dist_only=False)
# plot(y[y_ind[0]-1])
# plot(x[y_ind[1]-1])

# if record:
# 	writer.release()

if __name__=="__main__":

	parser = optparse.OptionParser()
	parser.add_option('-a', '--anon', dest='anon', action="store_true", default=False, help='Enable anonomization')
	(opt, args) = parser.parse_args()

	main(opt.anon)

max_stars_repo_name: colincsl/pyKinectTools

max_issues_repo_name: colincsl/pyKinectTools

max_forks_repo_name: colincsl/pyKinectTools

reason: import numpy,import scipy,from scipy


text:
'''
Random utility functions I don't know where to put yet..
'''

from numba import jit
import numpy as np
import pyemma
from sklearn.cluster import SpectralClustering

def total_variation(p, q):
    '''
    Computes the total variation distance between distributions p and q
    (assumes p and q are discrete distributions: normalized)
    '''
    return 0.5 * np.linalg.norm(p - q, 1)

def crisp_assign_using_hmm(i, hmm):
    return np.argmax(hmm.observation_probabilities[:,i])

@jit
def cg_T(microstate_T, microstate_pi, cg_map):
    '''
    Coarse-grain a microstate transition matrix by applying cg_map

    Parameters
    ----------
    microstate_T : (N,N), array-like, square
        microstate transition matrix
    microstate_pi : (N,), array-like
        microstate stationary distribution
    cg_map : (N,), array-like
        assigns each microstate i to a macrostate cg_map[i]

    Returns
    -------
    T : numpy.ndarray, square
        macrostate transition matrix
    '''

    n_macrostates = np.max(cg_map) + 1
    n_microstates = len(microstate_T)

    # compute macrostate stationary distribution
    macrostate_pi = np.zeros(n_macrostates)
    for i in range(n_microstates):
        macrostate_pi[cg_map[i]] += microstate_pi[i]
    macrostate_pi /= np.sum(macrostate_pi)

    # accumulate macrostate transition matrix
    T = np.zeros((n_macrostates, n_macrostates))
    for i in range(n_microstates):
        for j in range(n_microstates):
            T[cg_map[i], cg_map[j]] += microstate_pi[i] * microstate_T[i, j]

    # normalize
    for a in range(n_macrostates):
        T[a] /= macrostate_pi[a]

    return T

def create_merged_assignment_vector(cg_map, pair_to_merge):
    '''

    Parameters
    ----------

    cg_map : length-N vector, with entries in range(m)

    pair_to_merge : i,j in range(m)

    Returns
    -------

    merged_cg_map : length-N vector, with entries in range(m-1)

    '''
    merged_cg_map = np.array(cg_map)

    # later assume j > i
    i, j = sorted(pair_to_merge)

    # assign anything from cluster j to cluster i
    merged_cg_map[merged_cg_map == j] = i

    # fill in indices, if necessary
    merged_cg_map[merged_cg_map == np.max(cg_map)] = j

    return merged_cg_map

def score_merge(T, pi, assignment, pair_to_merge):
    merged_cg_map = create_merged_assignment_vector(assignment, pair_to_merge)
    new_T = cg_T(T, pi, merged_cg_map)
    return np.trace(new_T)

def hierarchical_T_clustering(T, pi):
    cg_map = np.arange(len(T))
    new_cg_map = np.arange(len(T))

    metastabilities = [np.trace(T)]
    merge_matrices = []
    all_cg_maps = [cg_map]

    for t in range(1, len(T) - 1):
        best = - np.inf
        n_macro = np.max(cg_map) + 1
        merge_matrix = np.zeros((n_macro, n_macro))

        for i in range(n_macro):
            for j in range(i):
                merged_cg_map = create_merged_assignment_vector(cg_map, (i, j))
                new_T = cg_T(T, pi, merged_cg_map)
                trace = np.trace(new_T)
                merge_matrix[i, j] = trace

                if trace > best:
                    best = trace
                    new_cg_map = merged_cg_map

        merge_matrices.append(merge_matrix + merge_matrix.T)
        cg_map = new_cg_map
        all_cg_maps.append(new_cg_map)
        metastabilities.append(best)

    return merge_matrices, metastabilities, all_cg_maps

def compute_mean_exit_times(msm):
    mean_exit_times = np.zeros(msm.nstates)

    for i in range(msm.nstates):
        not_i = list(set(range(msm.nstates)) - set([i]))
        mean_exit_times[i] = msm.mfpt(i, not_i)
    return mean_exit_times

def get_min_mean_exit_time_cg(diffusion_distance_matrix, dtrajs, n_states):
    clust = SpectralClustering(n_clusters=n_states, affinity='precomputed')
    clust.fit(1.0/(diffusion_distance_matrix+0.1))
    cg_dtrajs = [np.array([clust.labels_[i] for i in dtraj]) for dtraj in dtrajs]
    cg_msm = pyemma.msm.estimate_markov_model(cg_dtrajs, 10)
    mean_exit_times = compute_mean_exit_times(cg_msm)
    return min(mean_exit_times)

max_stars_repo_name: maxentile/cgtools

max_issues_repo_name: maxentile/cgtools

max_forks_repo_name: maxentile/cgtools

reason: import numpy,from numba


text:
import serial
import sys
import math
import struct
import numpy as np
from matplotlib import pyplot as plt
from matplotlib import animation

class MP:
    packer = struct.Struct('b')
    ser = serial.Serial(sys.argv[1], 115200, timeout=1)

    def load(self, data):
        self.ser.write(self.packer.pack(int(data)))

    def unload(self):
        return self.packer.unpack(self.ser.read(1))[0]

fig = plt.figure()
ax1 = fig.add_subplot(211, xlim=(0, 30), ylim=(-100, 100))
line1, = ax1.plot([], [], 'o')
ax2 = fig.add_subplot(212, xlim=(0, 30), ylim=(-100, 100))
line2, = ax2.plot([], [], 'o')

mp = MP()

y1 = np.array([], dtype=int)
y2 = np.array([], dtype=int)

for i in range(16):
    mp.load(0)
    mp.unload()

def init():
    line1.set_data([], [])
    line2.set_data([], [])
    return line1, line2,

def animate(i):
    global y1, y2
    if i == 1:
        y1 = np.array([], dtype=int)
        y2 = np.array([], dtype=int)
    x = np.arange(i)
    val = 0
    if i > 5:
        val = 79
    mp.load(val)
    y1 = np.append(y1, val)
    y2 = np.append(y2, mp.unload())
    line1.set_data(x, y1)
    line2.set_data(x, y2)
    return line1, line2,

frames = range(1, 30)

anim = animation.FuncAnimation(fig, animate, init_func=init,
                               frames=frames, interval=100, blit=True, repeat=False)
plt.show()


text:
"""
#from matplotlib.pyplot import pyplot as plt
#from matplotlib import style
from matplotlib import pyplot as plt
import csv
import sys
from numpy import genfromtxt
#style.use('ggplot')
path='../files/'+sys.argv[1]
x=[]
y=[]
plt.plot(x,y, marker='o')
plt.title('prime number generation rate')
plt.ylabel('Y axis')
plt.xlabel('X axis')
plt.show()
"""
import pandas as pd
from glob import glob
import matplotlib.pyplot as plt
import plotly.graph_objects as go

numFiles = 1 #Number of CSV files in your directory
separator = "," #Character that separates each value inside file
fExtension = ".csv" #Extension of the file storing the data
dataframe = sorted(glob('prime*.csv'))
pd.concat((pd.read_csv(file).assign(filename = file)
           for file in dataframe), ignore_index  = True)
plt.plot(list(str(dataframe)) )
plt.show()

max_stars_repo_path: Requirement 2/codes for csv file and text file/primeplot.py

max_stars_repo_name: Dhaval-B-Patel/DIGITAL-SECURITY-PROJECT

max_issues_repo_name: Dhaval-B-Patel/DIGITAL-SECURITY-PROJECT

max_forks_repo_name: Dhaval-B-Patel/DIGITAL-SECURITY-PROJECT

reason: from numpy


text:
# flake8: noqa
# yapf: disable

# __tf_setup_begin__

import numpy as np
import tensorflow as tf

def mnist_dataset(batch_size):
    (x_train, y_train), _ = tf.keras.datasets.mnist.load_data()
    # The `x` arrays are in uint8 and have values in the [0, 255] range.
    # You need to convert them to float32 with values in the [0, 1] range.
    x_train = x_train / np.float32(255)
    y_train = y_train.astype(np.int64)
    train_dataset = tf.data.Dataset.from_tensor_slices(
        (x_train, y_train)).shuffle(60000).repeat().batch(batch_size)
    return train_dataset

def build_and_compile_cnn_model():
    model = tf.keras.Sequential([
        tf.keras.layers.InputLayer(input_shape=(28, 28)),
        tf.keras.layers.Reshape(target_shape=(28, 28, 1)),
        tf.keras.layers.Conv2D(32, 3, activation='relu'),
        tf.keras.layers.Flatten(),
        tf.keras.layers.Dense(128, activation='relu'),
        tf.keras.layers.Dense(10)
    ])
    model.compile(
        loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
        optimizer=tf.keras.optimizers.SGD(learning_rate=0.001),
        metrics=['accuracy'])
    return model

# __tf_setup_end__

# __tf_single_begin__

def train_func():
    batch_size = 64
    single_worker_dataset = mnist_dataset(batch_size)
    single_worker_model = build_and_compile_cnn_model()
    single_worker_model.fit(single_worker_dataset, epochs=3, steps_per_epoch=70)

# __tf_single_end__

# __tf_distributed_begin__

import json
import os

def train_func_distributed():
    per_worker_batch_size = 64
    # This environment variable will be set by Ray Train.
    tf_config = json.loads(os.environ['TF_CONFIG'])
    num_workers = len(tf_config['cluster']['worker'])

    strategy = tf.distribute.MultiWorkerMirroredStrategy()

    global_batch_size = per_worker_batch_size * num_workers
    multi_worker_dataset = mnist_dataset(global_batch_size)

    with strategy.scope():
        # Model building/compiling need to be within `strategy.scope()`.
        multi_worker_model = build_and_compile_cnn_model()

    multi_worker_model.fit(multi_worker_dataset, epochs=3, steps_per_epoch=70)

# __tf_distributed_end__

if __name__ == "__main__":
    # __tf_single_run_begin__

    train_func()

    # __tf_single_run_end__

    # __tf_trainer_begin__

    from ray.train import Trainer

    trainer = Trainer(backend="tensorflow", num_workers=4)
    trainer.start()
    results = trainer.run(train_func_distributed)
    trainer.shutdown()

    # __tf_trainer_end__

max_stars_repo_name: daobook/ray

max_issues_repo_name: daobook/ray

max_forks_repo_name: daobook/ray


text:
import matplotlib.pyplot as plt
import torch
import numpy as np
from torchvision import models
from torchvision import transforms
from torchvision import models
from PIL import Image
from torch.utils.data import DataLoader
from net.fcn import FCN32,FCN16,FCN8
from net.linknet import LINKNET
from net.segnet import SEGNET
from net.pspnet import PSPNET
from net.icnet import ICNET, ICLoss
from net.unet import UNET
import torch.nn as nn
import torch.nn.functional as F
import tqdm
from torchvision.utils import make_grid
import time
from torchsummary import summary
from data import myCityscapes
from config import device
from PIL import ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True

#设置太大的BATCH会爆显存
BATCH = 8

def labelTransform(x):
    x = transforms.Resize(256)(x)
    x = np.array(x)
    x = torch.LongTensor(x)
    #第7类为道路
    mask = torch.LongTensor([[7]]).expand_as(x)
    y = (x==mask).long()
    return y

myTransform = transforms.Compose([
    transforms.Resize(256),
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])

#mode = 'foggy' or 'fine'
dataset = myCityscapes('./datasets/', split='train', mode='fine', target_type='semantic',transform=myTransform,target_transform=labelTransform)
dataloader = DataLoader(dataset, batch_size=BATCH,shuffle=True)
validset = myCityscapes('./datasets/', split='val', mode='fine', target_type='semantic',transform=myTransform,target_transform=labelTransform)
validloader = DataLoader(validset, batch_size=BATCH, shuffle=True)

#fcn = models.segmentation.fcn_resnet50(pretrained=True).to(device)

class SegModel():
    def __init__(self):
        #demo使用很少的训练轮次
        self.epoch = 10
        self.lr = 1e-4

        #FCN/SEGNET基于vgg16进行初始化
        self.model = FCN8().to(device)
        self.model.init_vgg16_params()
        #self.model = SEGNET().to(device)
        #self.model.init_vgg16_params()

        #使用残差卷积的LNINNET
        #self.model = LINKNET().to(device)

        #用resnet作为backbone并加入金字塔的PSPNET
        #self.model = PSPNET().to(device)

        #self.model = UNET().to(device)
        self.critirion = nn.CrossEntropyLoss()

        #self.model = ICNET().to(device)
        #self.critirion = ICLoss()

        self.optimizer =  torch.optim.Adam(self.model.parameters(),lr=self.lr)

    def train(self,dataloader,validloader):
        train_acces = []
        valid_acces = []
        losses = []
        for e in range(self.epoch):
            Loss = 0

            #共2975张图片
            for i, data in tqdm.tqdm(enumerate(dataloader)):
                #demo用比较少图片的训练
                if i > 50:
                    break

                self.optimizer.zero_grad()
                x,y = data
                x = x.to(device)
                y = y.to(device)

                out = self.model(x)

                loss = self.critirion(out,y)
                loss.backward()
                self.optimizer.step()

                Loss += loss

            Loss /= len(dataloader)
            losses.append(Loss)
            print("epoch: ",e, "loss: ",Loss.item())
            train_acc,_,_ = self.getMetrics(dataloader)
            valid_acc,_,_ = self.getMetrics(validloader)
            train_acces.append(train_acc)
            valid_acces.append(valid_acc)

            savepath = "results/" + str(e) + ".png"
            self.getPicture(dataloader, savepath)

        plt.figure()
        plt.plot(train_acces)
        plt.plot(valid_acces)
        plt.legend(["train","valid"])
        plt.savefig("results/acc.png")

        plt.figure()
        plt.plot(losses)
        plt.savefig("results/loss.png")

    def test(self,dataloader):
        acc,miou,fps = self.getMetrics(dataloader)
        print("accuracy: ", acc, "MIou: ", miou, "fps: ",fps)

    @torch.no_grad()
    def getMetrics(self,dataloader):
        correct = 0
        tot_pixel = 0
        tot_time = 0
        tot_num = 0

        tot_intersect = 0
        tot_union = 0

        for i,data in enumerate(dataloader):
            #选取部分计算评价指标
            if i > 5:
                break
            x,y = data
            B = x.shape[0]
            x = x.to(device)
            y = y.to(device)

            tot_num += 1

            start_time = time.time()
            #ICNET取第一个元素为输出
            out = self.model(x)
            if isinstance(out,list):
                out = out[0]
            end_time = time.time()
            tot_time += end_time - start_time

            pred = torch.argmax(out,dim=1)

            correct += torch.sum(pred ==  y)
            tot_pixel += torch.sum((y>=0))

            pred = pred.bool()
            y = y.bool()
            tot_intersect += torch.sum(pred & y)
            tot_union += torch.sum(pred | y)

        acc = (correct / tot_pixel).item()
        miou = (tot_intersect / tot_union).item()
        fps = (tot_num*BATCH) / tot_time

        return acc,miou,fps

    def getPicture(self,dataloader,path):
        for i,data in enumerate(dataloader):
            if i>0:
                break
            x,y = data
            x = x.to(device)

            plt.figure()
            plt.subplot(2,1,1)
            out = self.model(x)
            if isinstance(out,list):
                out = out[0]
            pred = torch.argmax(out,dim=1).float().detach().cpu()
            pred = pred.unsqueeze(dim=1)
            pic = make_grid(pred,padding=2).permute(1,2,0)
            plt.imshow(pic)
            plt.axis('off')

            plt.subplot(2,1,2)
            y = y.float().detach().cpu()
            y = y.unsqueeze(dim=1)
            pic = make_grid(y,padding=2).permute(1,2,0)
            plt.imshow(pic)
            plt.axis('off')
            plt.savefig(path)

seg = SegModel()
#summary(seg.model, (3,256,512))
seg.train(dataloader,validloader)
seg.test(validloader) #使用valloader而不使用testloader，猜测testdata标注有误
torch.save(seg.model.state_dict(), 'pth/segnet.pth')

max_stars_repo_name: caiyancheng/heyhey


text:
import matplotlib.pyplot as plt
import numpy as np

fig, axes = plt.subplots(nrows=2,ncols=1)

m = 3
b = 1
x = np.array([0,1,2,3])
y = np.array([(m*0+b),(m*1+b),(m*2+b),(m*3+b)])

axes[0].plot(x, y, color='b',ls='-',lw=4)

t = np.linspace(0, 2*np.pi,25)
a = np.sin(t)

axes[1].plot(t, a, color='r',ls='--',lw=2)

plt.show()

max_stars_repo_name: jaakjensen/PythonDSP

max_issues_repo_name: jaakjensen/PythonDSP

max_forks_repo_name: jaakjensen/PythonDSP


text:
# --------------------------------------------------------------------------
# Pelion Device Management Python SDK
# (C) COPYRIGHT 2017,2019 Arm Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# --------------------------------------------------------------------------
"""Part of the CI process"""

from collections import namedtuple
from collections import OrderedDict
import copy
import logging
import os
import unittest

# networkx: graph library for generating workflow
import networkx

import yaml

from tag_and_release import ReleaseTarget
from tag_and_release import release_target_map

LOG = logging.getLogger(__name__)
PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))
container_config_root = os.path.join(PROJECT_ROOT, 'container')
script_templates_root = os.path.join(PROJECT_ROOT, 'scripts', 'templates')
author_file = os.path.relpath(__file__, PROJECT_ROOT)
cache_dir = f'~/caches'
testrunner_cache = f'testrunner.tar'

base_structure = OrderedDict(
    version=2,
    jobs=dict(),
    workflows=dict(
        version=2,
        python_sdk_workflow=dict(
            jobs=[],
        ),
    ),
)

CloudHost = namedtuple('CloudHost', [
    'name',
    'public_name',
    'envvar_host',
    'envvar_key'
])

mbed_cloud_hosts = dict(
    osii=CloudHost('OS2', 'staging', 'MBED_CLOUD_API_HOST_OS2', 'MBED_CLOUD_API_KEY_OS2'),
    production=CloudHost('PROD', 'production', 'MBED_CLOUD_API_HOST_PROD', 'MBED_CLOUD_API_KEY_PROD'),
)

openssl_install = """
# openssl install
RUN apk add g++
RUN apk add libffi-dev
RUN apk add openssl-dev
RUN apk add openssl
"""

py2_openssl_cp = """
# openssl install
COPY --from=PY_SDK_BUILDER /usr/bin/openssl /usr/bin/openssl
COPY --from=PY_SDK_BUILDER /etc/ssl/ /etc/ssl/
COPY --from=PY_SDK_BUILDER /lib/ /lib/
"""

PyVer = namedtuple('PyVer', [
    'name',
    'docker_base',
    'tag',
    'docker_file',
    'compose_file',
    'py2_pre',
    'py2_post'
])
python_versions = dict(
    two=PyVer(
        'py2',
        'python:2.7.14-alpine3.6',
        'mbed_sdk_py2:latest',
        'py2.Dockerfile',
        'py2-compose.yml',
        openssl_install,
        py2_openssl_cp,
    ),
    three=PyVer(
        'py3',
        'python:3.7.3-alpine3.8',
        'mbed_sdk_py3:latest',
        'py3.Dockerfile',
        'py3-compose.yml',
        openssl_install,
        '',
    ),
)

def get_testrunner_image():
    """Load testrunner image from docker compose"""
    with open(os.path.join(PROJECT_ROOT, 'scripts', 'templates', 'docker-compose.yml')) as fh:
        docker_compose_spec = yaml.safe_load(fh)
    return docker_compose_spec['services']['testrunner']['image']

def new_base():
    """The high level structure of circle's config"""
    return copy.deepcopy(base_structure)

def new_preload():
    """Job running prior to builds - fetches TestRunner image

    This attempts to pull an image for the TestRunner with the same name as the branch which is being built for the
    Python SDK. If that cannot be found it falls back to the master build of TestRunner. Whichever is pulled is renamed
    with the tag `latest` which will be used by the docker compose.
    """
    branched_testrunner_image = '104059736540.dkr.ecr.us-west-2.amazonaws.com/mbed/sdk-testrunner:$(echo ${CIRCLE_BRANCH} | tr / -)'
    master_testrunner_image = '104059736540.dkr.ecr.us-west-2.amazonaws.com/mbed/sdk-testrunner:master'
    testrunner_image = get_testrunner_image()
    local_image = testrunner_image.rsplit(':')[-2].rsplit('/')[-1]
    version_file = 'testrunner_version.txt'
    template = yaml.safe_load(f"""
    machine:
      image: 'circleci/classic:latest'
    steps:
      - run:
          name: AWS login
          command: |-
            login="$(aws ecr get-login --no-include-email)"
            ${{login}}
      - run:
          name: Pull TestRunner Image for branch or fallback to master
          command: |-
            (docker pull {branched_testrunner_image} && docker tag {branched_testrunner_image} {testrunner_image}) || (docker pull {master_testrunner_image} && docker tag {master_testrunner_image} {testrunner_image})
      - run:
          name: Make cache directory
          command: mkdir -p {cache_dir}
      - run:
          name: Obtain Testrunner Version
          command: echo $(docker run {testrunner_image} python -m trunner --version) > {cache_dir}/{version_file}
      - run:
          name: Export docker image layer cache
          command: docker save -o {cache_dir}/{testrunner_cache} {testrunner_image}
      - persist_to_workspace: # workspace is used later in this same build
          root: {cache_dir}
          paths: '{testrunner_cache}'
      - persist_to_workspace: # workspace is used later in this same build
          root: {cache_dir}
          paths: '{version_file}'
      - store_artifacts:
          path: {cache_dir}/{version_file}
    """)
    return 'preload', template

def new_tpip():
    """Job for third party IP report generation"""
    template = yaml.safe_load("""
    steps:
      - checkout
      - run: sudo pip install -e .
      - run: python scripts/tpip.py python_tpip.csv
      - store_artifacts:
          path: python_tpip.csv
    docker:
      - image: circleci/python:3.6.1
    """)
    return 'tpip_report', template

def new_foundation_gen():
    """Job to generate the Foundation interface.

    If there are file changes caused by the generation then these are submitted back to github (which will trigger
    another build). The current build is then cancelled to avoid unnecessary builds and misleading test results (
    which would be with a pre-render code version).
    """
    foundation_dir = "src/mbed_cloud/foundation"
    template = yaml.safe_load(f"""
    steps:
      - checkout
      - run:
          name: Install pipenv
          command: sudo pip install pipenv
      - run:
          name: Install beta version of black for Python formatting
          command: pipenv install --pre black
      - run:
          name: Install SDK with dev dependencies
          command: pipenv install --dev .
      - run:
          name: Generate the Foundation interface code
          command: pipenv run python scripts/foundation/render_sdk.py
            api_specifications/public/sdk_foundation_definition.yaml  -vv
            -p python_definition.yaml
            -o {foundation_dir}
      - run:
          name: Commit code changes (cancel this build if commit made)
          command: |-
              git add -v {foundation_dir}/entities
              git add -v {foundation_dir}/enums
              git add -v {foundation_dir}/__init__.py
              git commit --message "Auto-generated code" || FILES_CHANGED=True
              git push -q https://${{GITHUB_TOKEN}}@github.com/ARMmbed/${{CIRCLE_PROJECT_REPONAME}}.git ${{CIRCLE_BRANCH}}
              if [ -z "$FILES_CHANGED" ]; then curl -X POST https://circleci.com/api/v1.1/project/github/ARMmbed/${{CIRCLE_PROJECT_REPONAME}}/${{CIRCLE_BUILD_NUM}}/cancel?circle-token=${{DOCS_CIRCLE_CI_TOKEN}}; fi
      - store_artifacts:
          path: python_definition.yaml
          when: always
    docker:
      - image: circleci/python:3.6.3
    """)
    return 'foundation_gen', template

def new_newscheck():
    """Job for checking newsfile existence"""
    template = yaml.safe_load("""
    steps:
      - checkout
      - run: python scripts/assert_news.py
    docker:
      - image: circleci/python:3.6.1
    """)
    return 'news_check', template

def new_build_documentation():
    """Job for checking updating SDK documentation"""
    template = yaml.safe_load("""
    steps:
      - run:
          name: Trigger documentation build
          command: >-
              curl -X POST --header "Content-Type:application/json"
              -d '{"branch":"master"}'
              https://circleci.com/api/v1.1/project/github/${GITHUB_DOCS_ORGANISATION}/${GITHUB_DOCS_PROJECT}/build?circle-token=${DOCS_CIRCLE_CI_TOKEN}
    docker:
      - image: circleci/node:jessie-browsers
    """)
    return 'build_documentation', template

def upload_reference_documentation():
    """Job for uploading reference documentation to S3"""

    # Build the documentation using python 3
    py_ver = python_versions["three"]

    cache_file = f'app_{py_ver.name}.tar'
    template = yaml.safe_load(f"""
    machine:
      image: circleci/classic:latest
    steps:
      - attach_workspace:
          at: {cache_dir}
      - checkout
      - run:
          name: Install prerequisites
          command: sudo pip install awscli
      - run:
          name: Load docker image layer cache
          command: docker load -i {cache_dir}/{cache_file}
      - run:
          name: Start a named container
          command: docker run --name=SDK {py_ver.tag}
      - run:
          name: Extract the documentation
          command: 'docker cp SDK:/build/built_docs ./built_docs'
      - run:
          name: Upload the documentation
          command: >-
            aws s3 sync --delete --cache-control
            max-age=3600 built_docs s3://mbed-cloud-sdk-python/${{CIRCLE_BRANCH}}-branch
    """)
    return 'reference_documentation', template

def build_name(py_ver: PyVer):
    """Name"""
    return f'build_{py_ver.name}'

def new_build(py_ver: PyVer):
    """Job for building/caching different docker images"""
    cache_file = f'app_{py_ver.name}.tar'
    cache_path = f'{cache_dir}/{cache_file}'
    cache_key = f'v3-{py_ver.name}-{{{{ .Branch }}}}'
    template = yaml.safe_load(f"""
    machine:
      image: 'circleci/classic:latest'
      docker_layer_caching: true
    steps:
      - checkout
      - restore_cache:
          keys: ['{cache_key}']
          paths: ['{cache_path}']
      - attach_workspace:
          at: {cache_dir}
      - run:
          name: Load docker image for TestRunner
          command: docker load -i {cache_dir}/{testrunner_cache}
      - run:
          name: Load docker image layer cache
          command: docker load -i {cache_path} || true  # silent failure if missing cache
      - run:
          name: Build application docker image
          command: >-
            docker build --cache-from={py_ver.tag}
            -t {py_ver.tag}
            --build-arg TESTRUNNER_VERSION=$(cat {cache_dir}/testrunner_version.txt)
            -f container/{py_ver.docker_file} .
      - run:
          name: Make cache directory
          command: mkdir -p {cache_dir}
      - run:
          name: Export docker image layer cache
          command: docker save -o {cache_path} {py_ver.tag}
      - save_cache: # cache is used between builds
          key: '{cache_key}'
          paths: ['{cache_path}']
      - persist_to_workspace: # workspace is used later in this same build
          root: {cache_dir}
          paths: '{cache_file}'

      # extracting documentation for review:
      - run:
          name: Start a named container
          command: docker run --name=SDK {py_ver.tag}
      - run:
          name: Extract the documentation
          command: 'docker cp SDK:/build/built_docs ./built_docs'
      - store_artifacts:
          path: built_docs
    """)
    return build_name(py_ver), template

def test_name(py_ver: PyVer, cloud_host: CloudHost):
    """Name"""
    return f'test_{py_ver.name}_{cloud_host.name}'

def new_test(py_ver: PyVer, cloud_host: CloudHost):
    """Job for running integration/coverage tests"""
    sdk_docker_cache = f'app_{py_ver.name}.tar'
    template = yaml.safe_load(f"""
    machine:
      image: circleci/classic:latest
    steps:
      - checkout
      - attach_workspace:
          at: {cache_dir}
      - run:
          name: Load docker image for SDK
          command: docker load -i {cache_dir}/{sdk_docker_cache}
      - run:
          name: Load docker image for TestRunner
          command: docker load -i {cache_dir}/{testrunner_cache}
      - run:
          name: Get docker-compose
          command: pip install docker-compose==1.23.2
      - run:
          name: Set testrunner parameters
          command: |-
            echo 'export TEST_RUNNER_DEFAULT_API_HOST=${{{cloud_host.envvar_host}}}' >> $BASH_ENV
            echo 'export MBED_CLOUD_SDK_HOST=${{{cloud_host.envvar_host}}}' >> $BASH_ENV
            echo 'export TEST_RUNNER_DEFAULT_API_KEY=${{{cloud_host.envvar_key}}}' >> $BASH_ENV
            echo 'export MBED_CLOUD_SDK_API_KEY=${{{cloud_host.envvar_key}}}' >> $BASH_ENV
      - run:
          name: Run all tests
          no_output_timeout: 15m
          command: >-
            docker-compose
            -f {os.path.relpath(container_config_root, PROJECT_ROOT)}/{py_ver.compose_file}
            up
            --exit-code-from=sdk_test_server
      - run:
          name: Generate summary
          command: sudo python scripts/ci_summary.py --noblock
          when: always
      - store_artifacts:
          path: results
      - run:
          name: Install codecov
          command: sudo pip install codecov
      - run:
          command: codecov --file=results/coverage.xml --flags {py_ver.name} {cloud_host.public_name}
          name: Upload code coverage results
    """)
    return test_name(py_ver, cloud_host), template

def deploy_name(py_ver: PyVer, release_target: ReleaseTarget):
    """Name"""
    return f'deploy_{py_ver.name}_{release_target.name}'

def deploy_docker_name(py_ver: PyVer, release_target: ReleaseTarget):
    """Name"""
    return f'deploy_docker_{py_ver.name}_{release_target.name}'

def new_deploy_docker(py_ver: PyVer, release_target: ReleaseTarget, is_release):
    """Job for deploying docker to aws ecr"""
    cache_file = f'app_{py_ver.name}.tar'
    version_file = 'testserver_version.txt'
    branched_testserver_raw_image='104059736540.dkr.ecr.us-west-2.amazonaws.com/mbed/sdk-testserver-python:'
    branched_testserver_tags = ['$(echo ${CIRCLE_BRANCH} | tr / -)', '${CIRCLE_SHA1}']
    if is_release:
        branched_testserver_tags.append('latest')
        branched_testserver_tags.append(f'$(cat {version_file})')
    job = f"""
    machine:
      image: circleci/classic:latest
    steps:
      - attach_workspace:
          at: {cache_dir}
      - checkout
      - run:
          name: Install prerequisites
          command: sudo pip install awscli
      - run:
          name: AWS login
          command: |-
            login="$(aws ecr get-login --no-include-email)"
            ${{login}}
      - run:
          name: Load docker image layer cache
          command: docker load -i {cache_dir}/{cache_file}
      - run:
          name: Store SDK's version
          command: docker run {py_ver.tag} > {version_file}
    """
    for tag in branched_testserver_tags:
        branched_testserver_image = branched_testserver_raw_image+tag
        job += f"""
      - run:
          name: Push TestServer docker image
          command: |-
            (docker tag {py_ver.tag} {branched_testserver_image}) && (docker push {branched_testserver_image})
    """
    template = yaml.safe_load(job)
    return deploy_docker_name(py_ver, release_target), template

def new_deploy(py_ver: PyVer, release_target: ReleaseTarget):
    """Job for deploying package to pypi"""
    cache_file = f'app_{py_ver.name}.tar'
    template = yaml.safe_load(f"""
    machine:
      image: circleci/classic:latest
    steps:
      - attach_workspace:
          at: {cache_dir}
      - checkout
      - run:
          name: Install prerequisites
          command: sudo pip install awscli
      - run:
          name: Load docker image layer cache
          command: docker load -i {cache_dir}/{cache_file}
      - run:
          name: Start a named container
          command: docker run --name=SDK {py_ver.tag}
      - run:
          name: Extract the documentation
          command: 'docker cp SDK:/build/built_docs ./built_docs'
      - run:
          name: Upload the documentation
          command: >-
            aws s3 sync --delete --cache-control
            max-age=3600 built_docs s3://mbed-cloud-sdk-python/${{CIRCLE_BRANCH}}-release
      - run:
          name: Tag and release
          command: >-
            docker run --env-file=scripts/templates/envvars.env
            -e TWINE_REPOSITORY={release_target.twine_repo}
            {py_ver.tag}
            sh -c "source .venv/bin/activate && python scripts/tag_and_release.py --mode={release_target.mode}"
    """)
    return deploy_name(py_ver, release_target), template

def release_name(release_target: ReleaseTarget):
    """Name"""
    return f'release_{release_target.name}'

def generate_circle_output():
    """Build sequence for Circle CI 2.0 config.yml

    builds the circleci structure
    also links individual jobs into a workflow graph
    """
    base = new_base()
    workflow = networkx.DiGraph()
    LOG.info('%s python versions', len(python_versions))
    LOG.info('%s Pelion Device Management hosts', len(mbed_cloud_hosts))

    job, content = new_tpip()
    base['jobs'].update({job: content})
    workflow.add_node(job)

    new_foundation_job, content = new_foundation_gen()
    base['jobs'].update({new_foundation_job: content})
    workflow.add_node(new_foundation_job)

    job, content = new_newscheck()
    base['jobs'].update({job: content})
    workflow.add_node(
        job,
        workflow=dict(
            filters=dict(
                branches=dict(
                    # we ignore this check for builds directly on the base branches
                    ignore=['master', 'integration']
                )
            )
        )
    )

    job, content = new_build_documentation()
    base['jobs'].update({job: content})
    workflow.add_node(
        job,
        workflow=dict(
            requires=['build_py2', 'build_py3'],
            filters=dict(
                branches=dict(
                    # Only update the documentation on release branches
                    only=['master', 'beta']
                )
            )
        )
    )

    job, content = upload_reference_documentation()
    base['jobs'].update({job: content})
    workflow.add_node(
        job,
        workflow=dict(
            requires=['build_py2', 'build_py3'],
            filters=dict(
                branches=dict(
                    # Only update the documentation on release branches
                    only=['master', 'beta']
                )
            )
        )
    )

    preload_job, content = new_preload()
    base['jobs'].update({preload_job: content})
    workflow.add_node(job)

    for py_ver in python_versions.values():
        build_job, build_content = new_build(py_ver=py_ver)
        base['jobs'].update({build_job: build_content})
        # Add requires to builds on preload and foundation_gen
        workflow.add_edge(preload_job, build_job)
        workflow.add_edge(new_foundation_job, build_job)

        for cloud_host in mbed_cloud_hosts.values():
            test_job, test_content = new_test(py_ver=py_ver, cloud_host=cloud_host)
            base['jobs'].update({test_job: test_content})
            workflow.add_edge(build_job, test_job)

    for release_target in release_target_map.values():
        deploy_job, deploy_content = new_deploy(
            py_ver=python_versions['three'],
            release_target=release_target
        )
        base['jobs'].update({deploy_job: deploy_content})
        deploy_docker_job, deploy_docker_content = new_deploy_docker(
            py_ver=python_versions['three'],
            release_target=release_target,
            is_release=release_target.mode in ['prod', 'beta']
        )
        base['jobs'].update({deploy_docker_job: deploy_docker_content})

    # wire up the release gates (clicky buttons)
    workflow.add_edge(
        test_name(python_versions['three'], mbed_cloud_hosts['osii']),
        release_name(release_target_map['beta']),
        type='approval',
    )
    workflow.add_edge(
        test_name(python_versions['three'], mbed_cloud_hosts['production']),
        release_name(release_target_map['prod']),
        type='approval',
        filters=dict(branches=dict(only='master')),
    )
    workflow.add_edge(
        test_name(python_versions['two'], mbed_cloud_hosts['production']),
        release_name(release_target_map['prod']),
    )
    workflow.add_edge(
        test_name(python_versions['three'], mbed_cloud_hosts['production']),
        deploy_docker_name(python_versions['three'], release_target_map['staging']),
        filters=dict(branches=dict(only=['api-contract/staging', 'master', 'integration', 'beta'])),
    )

    # we only want to deploy in certain conditions
    workflow.add_edge(
        release_name(release_target_map['beta']),
        deploy_name(python_versions['three'], release_target_map['beta'])
    )

    workflow.add_edge(
        release_name(release_target_map['beta']),
        deploy_docker_name(python_versions['three'], release_target_map['beta']),
    )

    workflow.add_edge(
        release_name(release_target_map['prod']),
        deploy_name(python_versions['three'], release_target_map['prod'])
    )

    workflow.add_edge(
        release_name(release_target_map['prod']),
        deploy_docker_name(python_versions['three'], release_target_map['prod']),
    )

    workflow_jobs = base['workflows']['python_sdk_workflow']['jobs']

    # build the workflow graph
    for job_name in networkx.topological_sort(workflow):
        job_config = {}
        per_node_config = workflow.nodes[job_name].get('workflow')
        if per_node_config:
            job_config.update(per_node_config)
        workflow_jobs.append({job_name: job_config})
        for edge in workflow.in_edges(job_name):
            job_config.update(workflow.get_edge_data(*edge))
            job_config.setdefault('requires', []).append(edge[0])

    LOG.info('%s circle jobs', len(base['jobs']))
    return dict(base)

def generate_docker_file(py_ver: PyVer):
    """Templated docker files"""
    with open(os.path.join(script_templates_root, 'Dockerfile')) as fh:
        return fh.read().format(py_ver=py_ver, author=author_file)

def generate_compose_file(py_ver: PyVer):
    """Templated docker-compose files"""
    with open(os.path.join(script_templates_root, 'docker-compose.yml')) as fh:
        return fh.read().format(py_ver=py_ver, author=author_file)

def generate_docker_targets():
    """Write all templated container engine files"""
    output = {}
    for py_ver in python_versions.values():
        filepath = os.path.join(container_config_root, py_ver.docker_file)
        output[filepath] = generate_docker_file(py_ver)
        filepath = os.path.join(container_config_root, py_ver.compose_file)
        output[filepath] = generate_compose_file(py_ver)
    return output

def main(output_path=None):
    """Writes out new python build system

    This is needed because CircleCI does not support build matrices
    nor parameterisation of cache paths or other aspects of their
    config

    There's also the added bonus of validating the yaml as we go.

    Additionally, we template and write Docker and docker-compose files
    for multiple python versions, as Docker `FROM` statements are also
    un-templatable using environment variables or similar.

    The bulk of the config structure is parsed templated yaml, which
    seems the most succinct way of building deeply nested dictionaries and lists,
    and also cleanly maps to the appearance of config.yml before & after templating.

    The main job blocks (build, test, deploy) are expanded as the product of python versions
    and Pelion Device Management environments, before being recombined into the job listing.

    Jobs are chained into a CircleCI workflow using a graph
    (in which nodes are job identifiers, and edges describe the dependencies
    and any additional parameters)
    """
    config_output_file = output_path or os.path.join(PROJECT_ROOT, '.circleci', 'config.yml')
    yaml_structure = generate_circle_output()
    with open(config_output_file, 'w') as fh:
        yaml_content = yaml.safe_dump(data=yaml_structure, default_flow_style=False)
        fh.write(
            f'#\n'
            f'# This file is autogenerated, do not modify manually. '
            f'See {author_file} for instructions.\n'
            f'#\n'
            f'{yaml_content}'
        )

    for path, content in generate_docker_targets().items():
        LOG.info('writing %s', path)
        with open(path, 'w') as fh:
            fh.write(content)

class Test(unittest.TestCase):
    """Test"""

    def setUp(self):
        """Make sure we have some logging running"""
        logging.basicConfig(level=logging.INFO)

    def test(self):
        """Runs build as if it were a test"""
        main()

if __name__ == '__main__':
    main()

max_stars_repo_name: GQMai/mbed-cloud-sdk-python

max_issues_repo_name: GQMai/mbed-cloud-sdk-python

max_forks_repo_name: GQMai/mbed-cloud-sdk-python

reason: import networkx


text:
from datetime import datetime
from math import floor
from multiprocessing import *
from project_lang_div import get_lang_div
from project_recur_co import get_recur_co
from project_team_famil import get_team_famil
from sqlalchemy import create_engine, MetaData, Table
from sqlalchemy.orm import sessionmaker
from sqlalchemy.sql import select
from utils import *
import logging
import numpy as np
import os
import pandas as pd
import pickle
import pymysql
import random
import time

# setup
get_fork = False
logging.basicConfig()
num_proc = 30
pswd = os.environ["SQLPW"]
id_file_name = "data/pid.list"
f = open(id_file_name)
pids = [int(l.strip()) for l in f.readlines()]
f.close()

f = open("data/big_repos.list")
big_repos = set([int(p.strip()) for p in f.readlines()])
pids = list(set(pids) - set(big_repos))
pids.sort()
print "Number of projects", len(pids)

# load contributors' projects
f = open("dict/contr_projs.dict")
cont_projs_dict = pickle.load(f)
f.close()

f = open("dict/proj_contrs_count.dict")
proj_contrs_count = pickle.load(f)
f.close()

f = open("dict/root_forks.dict")
root_forks = pickle.load(f)
f.close()

f = open("dict/fork_root.dict")
fork_root = pickle.load(f)
f.close()

f = open("dict/alias_map_b.dict")
alias_map = pickle.load(f)
f.close()

f = open("dict/reverse_alias_map_b.dict")
re_alias = pickle.load(f)
f.close()

langs = ["JavaScript", "Java", "Python", "CSS", "PHP", "Ruby", "C++",
  "C", "Shell", "C#", "Objective-C", "R", "VimL", "Go", "Perl",
  "CoffeeScript", "Tex", "Swift", "Scala", "Emacs Lisp", "Haskell",
  "Lua", "Clojure", "Matlab", "Arduino", "Groovy", "Puppet", "Rust",
  "PowerShell", "Erlang", "Visual Basic", "Processing", "Assembly", "Other"]
watchers = pd.read_csv("data/watchers_monthly_counts_win.csv")
print "Done setting up"

results = pd.DataFrame(columns=("p_id", "p_name", "p_lang", "p_owner",
                        "window", "p_age", "p_windows_active_to_date",
                        "p_num_stars", "p_num_users_to_date",
                        "p_num_commits_to_date", "r_team_size",
                        "p_recurring_co", "p_fam_no_decay", "p_langdiv_denom"))

url = "mysql://sophie:"+pswd+"@localhost/ghtorrent?charset=utf8mb4"
engine = create_engine(url, pool_size = num_proc, pool_recycle = 3600)
Session = sessionmaker(bind = engine)
metadata = MetaData(engine)
commits = Table("commits", metadata, autoload=True)
projects = Table("projects", metadata, autoload=True)
conns = engine.connect()
session = Session()

out = open("out", "w")
#for p_index, p in enumerate(pids):
def get_info(p):
  proc_id = os.getpid()

  # get the list of forks
  forks = root_forks[p]
  out.write(str(proc_id)+","+ str(p) +",forks,"+ str(len(forks))+","+str(datetime.now()))
  forks = tuple(forks)

  # get project basic info
  r = session.query(projects).filter(projects.c.id == p)
  p_name = r.first().name
  p_lang = r.first().language
  p_owner = r.first().owner_id
  p_owner = get_merged_id(re_alias, p_owner)

  # get project commits
  begin = datetime.strptime("2008-01-01", "%Y-%m-%d")
  end = datetime.strptime("2016-12-31", "%Y-%m-%d")
  r = session.query(commits).filter(commits.c.project_id.in_(forks),
        commits.c.created_at <= end,
        commits.c.created_at >= begin,
        commits.c.author_id != 6059)

  # get lists of contributors
  new_contr_win = {} # newly joined contributors per window
  contr_win = {} # list of contributors in each window
  all_contr = set() #
  num_commits_win = [0] * 36

  for i in range(36):
    new_contr_win[i] = set()
    contr_win[i] = set()

  for rr in r.all():
    win = floor((rr.created_at.month-1)/3+1)+ (rr.created_at.year-2008)*4
    win = int(win)

    merge_id = get_merged_id(re_alias, rr.author_id)
    if merge_id == 6059:
      continue # this id is for wrong email settings
    contr_win[int(win)-1].add(merge_id)
    if merge_id not in all_contr:
      new_contr_win[int(win)-1].add(merge_id)
    all_contr.add(merge_id)

    num_commits_win[win-1] += 1

  # active windows are those with commits
  # 0 indexed
  act_wins = [win for win in range(36) if num_commits_win[win] > 0]
  #print proc_id, "active windows", p, act_wins

  # count contributors
  contr_lens = []
  new_contr_lens = []
  for i in range(36):
    contr_lens.append(len(contr_win[i]))
    new_contr_lens.append(len(new_contr_win[i]))

  if max(contr_lens) > 1500:
    # otherwise it woule take tooooo long to calculate team familiarity
    return []
  #print proc_id, "contr", p, contr_lens

  # watchers
  cur_stars = watchers.loc[watchers["project_id"].isin(forks)]
  cur_stars = pd.DataFrame({"sum":cur_stars.groupby(["project_id",
                            "window"])["watchers"].sum()}).reset_index().sort_values(by=["window"])

  stars_count = [0] * 36
  for i in range(cur_stars.shape[0]):
    cur_win = int(cur_stars.iloc[i][["window"]])
    if cur_win > 0 and cur_win < 37:
      stars_count[cur_win-1] = int(cur_stars.iloc[i][["sum"]])
  #print proc_id, "star", p, stars_count

  # get users' projects per window to calculate team familiarity, lang
  # diversity, and recurring cohesion
  #contr_list_win = get_user_dict(p, helper.session)

  # create dataframe
  p_dicts = []
  for win_index, act_win in enumerate(act_wins):
    p_dict = {}
    p_dict["window_num"] = act_win + 1
    p_dict["window"] = windows[act_win]
    p_dict["p_id"] = np.int64(p)
    p_dict["p_lang"] = p_lang
    p_dict["p_owner"] = int(p_owner)
    p_dict["p_age"] = act_win - min(act_wins) # 0 based
    p_dict["p_windows_active_to_date"] = win_index + 1
    p_dict["p_team_size"] = contr_lens[act_win]
    p_dict["p_num_users_to_date"] = sum(new_contr_lens[:act_win])
    p_dict["p_num_stars"] = sum(stars_count[:act_win])
    p_dict["p_num_commits"] = num_commits_win[act_win]
    p_dict["p_num_commits_to_date"] = sum(num_commits_win[:act_win+1])

    # get contributors' projects in each window
    #contr_projs_win[act_win] = get_user_projs_all_win(session, commits, contr_win[act_win], act_win)
    #get_user_projs_all_win(contri_projs_win, session, commits, contr_win[act_win], act_win)
    contr_projs_win = get_user_projs_all_win(cont_projs_dict,
                                                      contr_win[act_win], act_win)

    team_famil = get_team_famil(p, session, commits, contr_win[act_win],
                 contr_projs_win, proj_contrs_count, act_win)
    recur_co = get_recur_co(p, contr_win[act_win], contr_projs_win, act_win)
    lang_div = get_lang_div(session, projects, contr_win[act_win],
                contr_projs_win, act_win, langs)

    # share of new comers
    new_comer = 0
    for contributor in contr_win[act_win]:
      # get the list of projects of this contributor
      this_contr_projs = cont_projs_dict[contributor]
      first_active_win = next((i for i, x in enumerate(this_contr_projs) if x), None)
      if first_active_win == act_win:
        new_comer += 1
    p_dict["p_sharenewcomers"] = new_comer * 1.0 / contr_lens[act_win]
    p_dict["p_sharenewcomers_this"] = new_contr_lens[act_win] * 1.0 / contr_lens[act_win]

    p_dict["p_fam_no_decay"] = team_famil
    p_dict["p_recurring_co"] = recur_co
    p_dict["p_div_langdenom"] = lang_div

    p_dicts.append(p_dict)
    #print proc_id, p, p_dict, datetime.now()
    out.write(str(p_dict)+",")
    out.write(str(datetime.now())+"\n")
  #out.write("\n".join([str(p_d) for p_d in p_dicts])+"\n")
  #results = pd.concat([results, pd.DataFrame(p_dicts)])
  #session.commit()

  return p_dicts

'''
for i in range(6):
  print "pid:", i*10000, (i+1)*10000
  p_ids_sub = pids[(i-1)*10000:i*10000]
  pool = Pool(num_proc)
  results = []
  results = pool.map(get_info, p_ids_sub)
  pool.close()
  pool.join()
  result_f = open("result_f"+str(i), "wb")
  pickle.dump(results, result_f)
  result_f.close()

  results = [dict_item for dict_lists in results for dict_item in dict_lists]
  results = pd.DataFrame(results)
  results.to_csv("data/proj_results"+str(i)+".csv", index = False)
'''

results = []
iter_size = 5000
num_iter = len(pids) / iter_size
print len(pids)
for i in range(num_iter+1):
  pool = Pool(num_proc)
  print datetime.now(), (i+1)*iter_size
  results_i = pool.map(get_info, pids[i*iter_size:(i+1)*iter_size])
  results_i = [dict_item for dict_lists in results_i for dict_item in dict_lists]
  results.extend(results_i)
  pool.close()
  pool.join()
  result_f = open("result_f_"+str(i), "wb")
  pickle.dump(results_i, result_f)
  result_f.close()

'''
results = []
for p in pids:
  results.append(get_info(p))
'''

results = pd.DataFrame(results)
results.to_csv("data/results_proj_big.csv", index = False, encoding = "utf-8")


text:
import torch
import torch.nn as nn
from torch.nn import init
import torch.nn.functional as F
import functools
from torch.optim import lr_scheduler
from memcnn.models.revop import ReversibleBlock
from torch.nn import Parameter
import numpy as np
import re

###############################################################################
# Helper Functions
###############################################################################

class Layer(nn.Module):
    def __init__(self):
        super(Layer, self).__init__()

    def forward(self, x):
        raise NotImplementedError

    def inverse(self, y):
        raise NotImplementedError

class Squeeze(Layer):
    def __init__(self, factor=2):
        super(Squeeze, self).__init__()
        assert factor > 1 and isinstance(factor, int), 'no point of using this if factor <= 1'
        self.factor = factor

    def squeeze_bchw(self, x):
        bs, c, h, w = x.size()
        assert h % self.factor == 0 and w % self.factor == 0, pdb.set_trace()

        # taken from https://github.com/chaiyujin/glow-pytorch/blob/master/glow/modules.py
        x = x.view(bs, c, h // self.factor, self.factor, w // self.factor, self.factor)
        x = x.permute(0, 1, 3, 5, 2, 4).contiguous()
        x = x.view(bs, c * self.factor * self.factor, h // self.factor, w // self.factor)

        return x

    def unsqueeze_bchw(self, x):
        bs, c, h, w = x.size()
        assert c >= 4 and c % 4 == 0

        # taken from https://github.com/chaiyujin/glow-pytorch/blob/master/glow/modules.py
        x = x.view(bs, c // self.factor ** 2, self.factor, self.factor, h, w)
        x = x.permute(0, 1, 4, 2, 5, 3).contiguous()
        x = x.view(bs, c // self.factor ** 2, h * self.factor, w * self.factor)
        return x

    def forward(self, x):
        if len(x.size()) != 4:
            raise NotImplementedError # Maybe ValueError would be more appropriate

        return self.squeeze_bchw(x)

    def inverse(self, x):
        if len(x.size()) != 4:
            raise NotImplementedError

        return self.unsqueeze_bchw(x)

class Unsqueeze(Layer):
    def __init__(self, factor=2):
        super(Unsqueeze, self).__init__()
        assert factor > 1 and isinstance(factor, int), 'no point of using this if factor <= 1'
        self.factor = factor

    def squeeze_bchw(self, x):
        bs, c, h, w = x.size()
        assert h % self.factor == 0 and w % self.factor == 0, pdb.set_trace()

        # taken from https://github.com/chaiyujin/glow-pytorch/blob/master/glow/modules.py
        x = x.view(bs, c, h // self.factor, self.factor, w // self.factor, self.factor)
        x = x.permute(0, 1, 3, 5, 2, 4).contiguous()
        x = x.view(bs, c * self.factor * self.factor, h // self.factor, w // self.factor)

        return x

    def unsqueeze_bchw(self, x):
        bs, c, h, w = x.size()
        assert c >= 4 and c % 4 == 0

        # taken from https://github.com/chaiyujin/glow-pytorch/blob/master/glow/modules.py
        x = x.view(bs, c // self.factor ** 2, self.factor, self.factor, h, w)
        x = x.permute(0, 1, 4, 2, 5, 3).contiguous()
        x = x.view(bs, c // self.factor ** 2, h * self.factor, w * self.factor)
        return x

    def forward(self, x):
        if len(x.size()) != 4:
            raise NotImplementedError # Maybe ValueError would be more appropriate

        return self.unsqueeze_bchw(x)

    def inverse(self, x):
        if len(x.size()) != 4:
            raise NotImplementedError

        return self.squeeze_bchw(x)

def get_norm_layer(norm_type='instance'):
    if norm_type == 'batch':
        norm_layer = functools.partial(nn.BatchNorm3d, affine=True)
    elif norm_type == 'instance':
        norm_layer = functools.partial(nn.InstanceNorm3d, affine=False, track_running_stats=True)
    elif norm_type == 'none':
        norm_layer = None
    else:
        raise NotImplementedError('normalization layer [%s] is not found' % norm_type)
    return norm_layer

def get_scheduler(optimizer, opt):
    if opt.lr_policy == 'lambda':
        def lambda_rule(epoch):
            lr_l = 1.0 - max(0, epoch + opt.epoch_count - opt.niter) / float(opt.niter_decay + 1)
            return lr_l
        scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_rule)
    elif opt.lr_policy == 'step':
        scheduler = lr_scheduler.StepLR(optimizer, step_size=opt.lr_decay_iters, gamma=0.1)
    elif opt.lr_policy == 'plateau':
        scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.2, threshold=0.01, patience=5)
    else:
        return NotImplementedError('learning rate policy [%s] is not implemented', opt.lr_policy)
    return scheduler

def init_weights(net, init_type='normal', gain=0.02):
    def init_func(m):
        classname = m.__class__.__name__
        if hasattr(m, 'weight') and (classname.find('Conv') != -1 or classname.find('Linear') != -1):
            if init_type == 'normal':
                init.normal_(m.weight.data, 0.0, gain)
            elif init_type == 'xavier':
                init.xavier_normal_(m.weight.data, gain=gain)
            elif init_type == 'kaiming':
                init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')
            elif init_type == 'orthogonal':
                init.orthogonal_(m.weight.data, gain=gain)
            else:
                raise NotImplementedError('initialization method [%s] is not implemented' % init_type)
            if hasattr(m, 'bias') and m.bias is not None:
                init.constant_(m.bias.data, 0.0)
        elif classname.find('BatchNorm3d') != -1:
            init.normal_(m.weight.data, 1.0, gain)
            init.constant_(m.bias.data, 0.0)

    print('initialize network with %s' % init_type)
    net.apply(init_func)

def init_net(net, init_type='normal', init_gain=0.02, gpu_ids=[]):
    if len(gpu_ids) > 0:
        assert(torch.cuda.is_available())
        print(gpu_ids)
        device = torch.device('cuda:{}'.format(gpu_ids[0])) if gpu_ids else torch.device('cpu')
        net.to(device)
        net = torch.nn.DataParallel(net, gpu_ids)
    init_weights(net, init_type, gain=init_gain)
    return net

def define_G(input_nc, output_nc, ngf, which_model_netG, norm='batch', use_dropout=False, init_type='normal', init_gain=0.02, gpu_ids=[], deconv='transposed', output_tanh=False, n_downsampling=2):
    netG = None
    norm_layer = get_norm_layer(norm_type=norm)

    if which_model_netG.startswith('srcnn_'):
        depth = int(re.findall(r'\d+', which_model_netG)[0])
        netG = SrcnnGenerator3d(input_nc, output_nc, depth, ngf)
    elif which_model_netG.startswith('edsr_'):
        depth = int(re.findall(r'\d+', which_model_netG)[0])
        netG = EdsrGenerator3d(input_nc, output_nc, depth, ngf)
    elif which_model_netG.startswith('edsrF_'):
        depth = int(re.findall(r'\d+', which_model_netG)[0])
        netG = EdsrFGenerator3d(input_nc, output_nc, depth, ngf)
    else:
        raise NotImplementedError('Generator model name [%s] is not recognized' % which_model_netG)
    return init_net(netG, init_type, init_gain, gpu_ids)

def define_D(input_nc, ndf, which_model_netD,
             n_layers_D=3, norm='batch', use_sigmoid=False, init_type='normal', init_gain=0.02, gpu_ids=[]):
    netD = None
    norm_layer = get_norm_layer(norm_type=norm)

    if which_model_netD == 'basic':
        netD = NLayerDiscriminator(input_nc, ndf, n_layers=2, norm_layer=norm_layer, use_sigmoid=use_sigmoid)
    elif which_model_netD == 'n_layers':
        netD = NLayerDiscriminator(input_nc, ndf, n_layers_D, norm_layer=norm_layer, use_sigmoid=use_sigmoid)
    elif which_model_netD == 'pixel':
        netD = PixelDiscriminator(input_nc, ndf, norm_layer=norm_layer, use_sigmoid=use_sigmoid)
    else:
        raise NotImplementedError('Discriminator model name [%s] is not recognized' %
                                  which_model_netD)
    return init_net(netD, init_type, init_gain, gpu_ids)

##############################################################################
# Classes
##############################################################################

# Defines the GAN loss which uses either LSGAN or the regular GAN.
# When LSGAN is used, it is basically same as MSELoss,
# but it abstracts away the need to create the target label tensor
# that has the same size as the input
class GANLoss(nn.Module):
    def __init__(self, use_lsgan=True, target_real_label=1.0, target_fake_label=0.0):
        super(GANLoss, self).__init__()
        self.register_buffer('real_label', torch.tensor(target_real_label))
        self.register_buffer('fake_label', torch.tensor(target_fake_label))
        if use_lsgan:
            self.loss = nn.MSELoss()
        else:
            self.loss = nn.BCELoss()

    def get_target_tensor(self, input, target_is_real):
        if target_is_real:
            target_tensor = self.real_label
        else:
            target_tensor = self.fake_label
        return target_tensor.expand_as(input)

    def __call__(self, input, target_is_real):
        target_tensor = self.get_target_tensor(input, target_is_real)
        return self.loss(input, target_tensor)

class ZeroInit(nn.Conv3d):
    def reset_parameters(self):
        self.weight.data.zero_()
        self.bias.data.zero_()

class ThickBlocknaive3d(nn.Module):
    def __init__(self, dim, use_bias):
        super(ThickBlocknaive3d, self).__init__()
        self.F = self.build_conv_block(dim, True)

    def build_conv_block(self, dim, use_bias):
        conv_block = []
        conv_block += [nn.InstanceNorm3d(dim)]
        conv_block += [nn.ReplicationPad3d(1)]
        conv_block += [nn.Conv3d(dim, dim, kernel_size=3, padding=0, bias=use_bias)]
        conv_block += [nn.InstanceNorm3d(dim)]
        conv_block += [nn.ReLU(True)]
        conv_block += [nn.ReplicationPad3d(1)]
        conv_block += [ZeroInit(dim, dim, kernel_size=3, padding=0, bias=use_bias)]

        return nn.Sequential(*conv_block)

    def forward(self, input):
        return self.F(input) + input

class ThickBlock3d(nn.Module):
    def __init__(self, dim, use_bias):
        super(ThickBlock3d, self).__init__()
        self.F = self.build_conv_block(dim // 2, True)
        self.G = self.build_conv_block(dim // 2, True)
        self.rev_block = ReversibleBlock(self.F, self.G, 'additive')

    def build_conv_block(self, dim, use_bias):
        conv_block = []
        conv_block += [nn.InstanceNorm3d(dim)]
        conv_block += [nn.ReplicationPad3d(1)]
        conv_block += [nn.Conv3d(dim, dim, kernel_size=3, padding=0, bias=use_bias)]
        conv_block += [nn.InstanceNorm3d(dim)]
        conv_block += [nn.ReLU(True)]
        conv_block += [nn.ReplicationPad3d(1)]
        conv_block += [ZeroInit(dim, dim, kernel_size=3, padding=0, bias=use_bias)]

        return nn.Sequential(*conv_block)

    def forward(self, x):
        return self.rev_block(x)

    def inverse(self, x):
        return self.rev_block.inverse(x)

class ThinBlock3d(nn.Module):
    def __init__(self, dim, use_bias):
        super(ThinBlock3d, self).__init__()
        self.F = self.build_conv_block(dim // 2, True)
        self.G = self.build_conv_block(dim // 2, True)
        self.rev_block = ReversibleBlock(self.F, self.G, 'additive')

    def build_conv_block(self, dim, use_bias):
        conv_block = []
        conv_block += [nn.ReplicationPad3d(1)]
        conv_block += [nn.Conv3d(dim, dim, kernel_size=3, padding=0, bias=use_bias)]
        conv_block += [nn.ReLU(True)]
        conv_block += [nn.ReplicationPad3d(1)]
        conv_block += [ZeroInit(dim, dim, kernel_size=3, padding=0, bias=use_bias)]

        return nn.Sequential(*conv_block)

    def forward(self, x):
        return self.rev_block(x)

    def inverse(self, x):
        return self.rev_block.inverse(x)

class RevBlock3d(nn.Module):
    def __init__(self, dim, use_bias, norm_layer):
        super(RevBlock3d, self).__init__()
        self.F = self.build_conv_block(dim // 2, True, norm_layer)
        self.G = self.build_conv_block(dim // 2, True, norm_layer)
        self.rev_block = ReversibleBlock(self.F, self.G, 'additive')

    def build_conv_block(self, dim, use_bias, norm_layer):
        conv_block = []
        conv_block += [nn.ReplicationPad3d(1)]
        conv_block += [nn.Conv3d(dim, dim, kernel_size=3, padding=0, bias=use_bias)]
        conv_block += [norm_layer(dim)]
        conv_block += [nn.ReLU(True)]
        conv_block += [nn.ReplicationPad3d(1)]
        conv_block += [ZeroInit(dim, dim, kernel_size=3, padding=0, bias=use_bias)]

        return nn.Sequential(*conv_block)

    def forward(self, x):
        return self.rev_block(x)

    def inverse(self, x):
        return self.rev_block.inverse(x)

class PixelUnshuffle3D(nn.Module):
    def __init__(self, upscale_factor):
        super(PixelUnshuffle3D, self).__init__()
        self.upscale_factor = upscale_factor

    def forward(self, input):
        return pixel_unshuffle_3d(input, self.upscale_factor)

    def __repr__(self):
        return self.__class__.__name__ + '(upscale_factor=' + str(self.upscale_factor) + ')'

def pixel_unshuffle_3d(input, upscale_factor):
    batch_size, channels, in_height, in_width, in_depth = input.size()

    out_channels = channels * upscale_factor ** 3
    out_height = in_height // upscale_factor
    out_width = in_width // upscale_factor
    out_depth = in_depth // upscale_factor

    fm_view = input.view(batch_size, channels, out_height, upscale_factor, out_width, upscale_factor, out_depth, upscale_factor)
    fm_perm = fm_view.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous()
    return fm_perm.view(batch_size, out_channels, out_height, out_width, out_depth)

class PixelShuffle3D(nn.Module):
    def __init__(self, upscale_factor):
        super(PixelShuffle3D, self).__init__()
        self.upscale_factor = upscale_factor

    def forward(self, input):
        return pixel_shuffle_3d(input, self.upscale_factor)

    def __repr__(self):
        return self.__class__.__name__ + '(upscale_factor=' + str(self.upscale_factor) + ')'

def pixel_shuffle_3d(input, upscale_factor):
    batch_size, channels, in_height, in_width, in_depth = input.size()
    channels //= upscale_factor ** 3

    out_height = in_height * upscale_factor
    out_width = in_width * upscale_factor
    out_depth = in_depth * upscale_factor

    input_view = input.contiguous().view(
        batch_size, channels, upscale_factor, upscale_factor, upscale_factor,
        in_height, in_width, in_depth)

    shuffle_out = input_view.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous()
    return shuffle_out.view(batch_size, channels, out_height, out_width, out_depth)

class EdsrFGenerator3d(nn.Module):
    def __init__(self, input_nc, output_nc, depth, ngf=64, deconv='nearest',
                 use_dropout=False):
        super(EdsrFGenerator3d, self).__init__()

        use_bias = True
        downconv_ab = [nn.ReplicationPad3d(2),
                       nn.Conv3d(input_nc, ngf, kernel_size=5,
                                 stride=1, padding=0, bias=use_bias),
                       nn.InstanceNorm3d(ngf),
                       nn.ReLU(True),
                       nn.Conv3d(ngf, ngf * 2, kernel_size=3,
                                 stride=2, padding=1, bias=use_bias),
                       nn.InstanceNorm3d(ngf * 2),
                       nn.ReLU(True)]
        downconv_ba = [nn.ReplicationPad3d(2),
                       nn.Conv3d(input_nc, ngf, kernel_size=5,
                                 stride=1, padding=0, bias=use_bias),
                       nn.InstanceNorm3d(ngf),
                       nn.ReLU(True),
                       nn.Conv3d(ngf, ngf * 2, kernel_size=3,
                                 stride=2, padding=1, bias=use_bias),
                       nn.InstanceNorm3d(ngf * 2),
                       nn.ReLU(True)]

        core = []
        for _ in range(depth):
            core += [ThickBlock3d(ngf * 2, use_bias)]

        upconv_ab = [nn.ConvTranspose3d(ngf * 2, ngf,
                                        kernel_size=3, stride=2,
                                        padding=1, output_padding=1,
                                       bias=use_bias),
                     nn.InstanceNorm3d(ngf),
                     nn.ReLU(True),
                     nn.ReplicationPad3d(2),
                     nn.Conv3d(ngf, output_nc, kernel_size=5, padding=0),
		             nn.Tanh()]
        upconv_ba = [nn.ConvTranspose3d(ngf * 2, ngf,
                                        kernel_size=3, stride=2,
                                        padding=1, output_padding=1,
                                       bias=use_bias),
                     nn.InstanceNorm3d(ngf),
                     nn.ReLU(True),
                     nn.ReplicationPad3d(2),
                     nn.Conv3d(ngf, output_nc, kernel_size=5, padding=0),
		             nn.Tanh()]

        self.downconv_ab = nn.Sequential(*downconv_ab)
        self.downconv_ba = nn.Sequential(*downconv_ba)
        self.core = nn.ModuleList(core)
        self.upconv_ab = nn.Sequential(*upconv_ab)
        self.upconv_ba = nn.Sequential(*upconv_ba)

    def forward(self, input, inverse=False):
        out = input

        if inverse:
            out = self.downconv_ba(out)
            for block in reversed(self.core):
                out = block.inverse(out)
            return self.upconv_ba(out)
        else:
            out = self.downconv_ab(out)
            for block in self.core:
                out = block(out)
            return self.upconv_ab(out)

class SrcnnGenerator3d(nn.Module):
    def __init__(self, input_nc, output_nc, depth, ngf=64, deconv='nearest',
                 norm_layer=nn.BatchNorm3d, use_dropout=False):
        super(SrcnnGenerator3d, self).__init__()

        use_bias = True
        downconv_ab = [nn.ReplicationPad3d(1),
                    nn.Conv3d(input_nc, ngf, kernel_size=3,
                              stride=1, padding=0, bias=use_bias)]
        downconv_ba = [nn.ReplicationPad3d(1),
                    nn.Conv3d(input_nc, ngf, kernel_size=3,
                              stride=1, padding=0, bias=use_bias)]

        core = []
        for _ in range(depth):
            core += [RevBlock3d(ngf, use_bias, norm_layer)]

        upconv_ab = [nn.Conv3d(ngf, output_nc, kernel_size=1,
                            stride=1, padding=0, bias=use_bias)]
        upconv_ba = [nn.Conv3d(ngf, output_nc, kernel_size=1,
                            stride=1, padding=0, bias=use_bias)]

        self.downconv_ab = nn.Sequential(*downconv_ab)
        self.downconv_ba = nn.Sequential(*downconv_ba)
        self.core = nn.ModuleList(core)
        self.upconv_ab = nn.Sequential(*upconv_ab)
        self.upconv_ba = nn.Sequential(*upconv_ba)

    def forward(self, input, inverse=False):
        orig_shape = input.shape[2:]
        out = input

        if inverse:
            out = self.downconv_ab(out)
            for block in reversed(self.core):
                out = block.inverse(out)
            out = self.upconv_ab(out)
        else:
            out = self.downconv_ba(out)
            for block in self.core:
                out = block(out)
            out = self.upconv_ba(out)
        return out

class UnetGenerator(nn.Module):
    def __init__(self, input_nc, output_nc, num_downs, ngf=64,
                 norm_layer=nn.BatchNorm3d, use_dropout=False):
        super(UnetGenerator, self).__init__()

        # construct unet structure
        unet_block = UnetSkipConnectionBlock(ngf * 4, ngf * 4, input_nc=None, submodule=None, norm_layer=norm_layer)
        unet_block = UnetSkipConnectionBlock(ngf * 2, ngf * 4, input_nc=None, submodule=unet_block, norm_layer=norm_layer)
        unet_block = UnetSkipConnectionBlock(ngf, ngf * 2, input_nc=None, submodule=unet_block, norm_layer=norm_layer)
        unet_block = UnetSkipConnectionBlock(output_nc, ngf, input_nc=input_nc, submodule=unet_block, outermost=True, norm_layer=norm_layer)

        self.model = unet_block

    def forward(self, input):
        return self.model(input)

class UnetSkipConnectionBlock(nn.Module):
    def __init__(self, outer_nc, inner_nc, input_nc=None,
                 submodule=None, outermost=False, innermost=False, norm_layer=nn.BatchNorm3d, use_dropout=False):
        super(UnetSkipConnectionBlock, self).__init__()
        self.outermost = outermost
        if type(norm_layer) == functools.partial:
            use_bias = norm_layer.func == nn.InstanceNorm3d
        else:
            use_bias = norm_layer == nn.InstanceNorm3d
        if input_nc is None:
            input_nc = outer_nc
        downconv = nn.Conv3d(input_nc, inner_nc, kernel_size=4,
                             stride=2, padding=1, bias=use_bias)
        downrelu = nn.LeakyReLU(0.2, True)
        downnorm = norm_layer(inner_nc)
        uprelu = nn.ReLU(True)
        upnorm = norm_layer(outer_nc)

        if outermost:
            upconv = nn.ConvTranspose3d(inner_nc * 2, outer_nc,
                                        kernel_size=4, stride=2,
                                        padding=1)
            down = [downconv]
            up = [uprelu, upconv, nn.Tanh()]
            model = down + [submodule] + up
        elif innermost:
            upconv = nn.ConvTranspose3d(inner_nc, outer_nc,
                                        kernel_size=4, stride=2,
                                        padding=1, bias=use_bias)
            down = [downrelu, downconv]
            up = [uprelu, upconv, upnorm]
            model = down + up
        else:
            upconv = nn.ConvTranspose3d(inner_nc * 2, outer_nc,
                                        kernel_size=4, stride=2,
                                        padding=1, bias=use_bias)
            down = [downrelu, downconv, downnorm]
            up = [uprelu, upconv, upnorm]

            if use_dropout:
                model = down + [submodule] + up + [nn.Dropout(0.5)]
            else:
                model = down + [submodule] + up

        self.model = nn.Sequential(*model)

    def forward(self, x):
        if self.outermost:
            return self.model(x)
        else:
            return torch.cat([x, self.model(x)], 1)

# Defines the PatchGAN discriminator with the specified arguments.
class NLayerDiscriminator(nn.Module):
    def __init__(self, input_nc, ndf=64, n_layers=2, norm_layer=nn.BatchNorm3d, use_sigmoid=False):
        super(NLayerDiscriminator, self).__init__()
        if type(norm_layer) == functools.partial:
            use_bias = norm_layer.func == nn.InstanceNorm3d
        else:
            use_bias = norm_layer == nn.InstanceNorm3d

        kw = 4
        padw = 1
        sequence = [
            nn.Conv3d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw),
            nn.LeakyReLU(0.2, True)
        ]

        nf_mult = 1
        nf_mult_prev = 1
        for n in range(1, n_layers):
            nf_mult_prev = nf_mult
            nf_mult = min(2**n, 8)
            sequence += [
                nn.Conv3d(ndf * nf_mult_prev, ndf * nf_mult,
                          kernel_size=kw, stride=2, padding=padw, bias=use_bias),
                norm_layer(ndf * nf_mult),
                nn.LeakyReLU(0.2, True)
            ]

        nf_mult_prev = nf_mult
        nf_mult = min(2**n_layers, 8)
        sequence += [
            nn.Conv3d(ndf * nf_mult_prev, ndf * nf_mult,
                      kernel_size=kw, stride=1, padding=padw, bias=use_bias),
            norm_layer(ndf * nf_mult),
            nn.LeakyReLU(0.2, True)
        ]

        sequence += [nn.Conv3d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)]

        if use_sigmoid:
            sequence += [nn.Sigmoid()]

        self.model = nn.Sequential(*sequence)

    def forward(self, input):
        return self.model(input)

class PixelDiscriminator(nn.Module):
    def __init__(self, input_nc, ndf=64, norm_layer=nn.BatchNorm3d, use_sigmoid=False):
        super(PixelDiscriminator, self).__init__()
        if type(norm_layer) == functools.partial:
            use_bias = norm_layer.func == nn.InstanceNorm3d
        else:
            use_bias = norm_layer == nn.InstanceNorm3d

        self.net = [
            nn.Conv3d(input_nc, ndf, kernel_size=1, stride=1, padding=0),
            nn.LeakyReLU(0.2, True),
            nn.Conv3d(ndf, ndf * 2, kernel_size=1, stride=1, padding=0, bias=use_bias),
            norm_layer(ndf * 2),
            nn.LeakyReLU(0.2, True),
            nn.Conv3d(ndf * 2, 1, kernel_size=1, stride=1, padding=0, bias=use_bias)]

        if use_sigmoid:
            self.net.append(nn.Sigmoid())

        self.net = nn.Sequential(*self.net)

    def forward(self, input):
        return self.net(input)

max_stars_repo_name: dr-benway/RevGAN

max_issues_repo_name: dr-benway/RevGAN

max_forks_repo_name: dr-benway/RevGAN


text:
import numpy as np

class Weno:
    def __init__(self):
        self.epsilon = 1e-6

    def weno(self, NumFl, Fl, L, In, Out):
        """ Interface reconstruction using WENO scheme. """

        # Built an extenday array with phantom cells to deal with periodicity
        #data = np.concatenate((In[-2:], In, In[0:2]))
        data = np.concatenate((In[-3:], In, In[0:2])) # FIXME: shift indexes so that there is no need to add 3 points at the left side

        # Squared derivatives of the input
        dp1m2p1 = np.square(np.convolve(data, np.flipud([1, -2, 1]), 'valid'))
        dp1m4p3 = np.square(np.convolve(data, np.flipud([1, -4, 3]), 'valid'))
        dp1p0m1 = np.square(np.convolve(data, np.flipud([1, 0, -1]), 'valid'))
        dp3m4p1 = np.square(np.convolve(data, np.flipud([3, -4, 1]), 'valid'))

        # Regularity indicators
        S1 = 13./12. * dp1m2p1 + 1./4. * dp1m4p3
        S2 = 13./12. * dp1m2p1 + 1./4. * dp1p0m1
        S3 = 13./12. * dp1m2p1 + 1./4. * dp3m4p1

        # Unweighted coefficients that can be reused for right reconstruction
        w1_tmp = np.reciprocal(np.square(self.epsilon + S1))
        w2_tmp = np.reciprocal(np.square(self.epsilon + S2))
        w3_tmp = np.reciprocal(np.square(self.epsilon + S3))

        # Weighted coefficients for left reconstruction
        w1 = 1./10. * w1_tmp[0:-3]
        w2 = 3./5. * w2_tmp[1:-2]
        w3 = 3./10. * w3_tmp[2:-1]

        # Coefficients sum
        atot = w1 + w2 + w3

        # Interpolations
        ip2m7p11 = np.convolve(data, np.flipud([2./6., -7./6., 11./6.]), 'valid')
        im1p5p2 = np.convolve(data, np.flipud([-1./6., 5./6., 2./6.]), 'valid')
        ip2p5m1 = np.convolve(data, np.flipud([2./6., 5./6., -1./6.]), 'valid')

        # Left reconstruction
        qm = np.divide(w1 * ip2m7p11[0:-3] + w2 * im1p5p2[1:-2] + w3 * ip2p5m1[2:-1], atot);

        # Note: for the right reconstruction, the input is processed like
        #   its order is reversed: for example, instead of dp1m4p3, we use
        #   dp3m4p1. But in fact, due to some symmetries, the main work has
        #   already be done during the left reconstruction.

        # Regularity indicators
        # S1 = 13./12. * dp1m2p1 + 1./4. * dp3m4p1 # == S3 from the first step
        # S2 = 13./12. * dp1m2p1 + 1./4. * dp1p0m1 # == S2 from the first step
        # S3 = 13./12. * dp1m2p1 + 1./4. * dp1m4p3 # == S1 from the first step

        # Weighted coefficients for right reconstruction
        w1 = 1./10. * w3_tmp[3:]
        w2 = 3./5. * w2_tmp[2:-1]
        w3 = 3./10. * w1_tmp[1:-2]

        # Coefficients sum
        atot = w1 + w2 + w3

        # Interpolations
        ip11m7p2 = np.convolve(data, np.flipud([11./6., -7./6., 2./6.]), 'valid')
        # ip2p5m1 = np.convolve(data, np.flipud([2./6., 5./6., -1./6.]), 'valid') # Already calculated
        # im1p5p2 = np.convolve(data, np.flipud([-1./6., 5./6., 2./6.]), 'valid') # Already calculated

        # Right reconstruction
        qp = np.divide(w1 * ip11m7p2[3:] + w2 * ip2p5m1[2:-1] + w3 * im1p5p2[1:-2], atot);

        # Numerical flux
        num_flux = np.empty(len(In)+1)
        num_flux[0:-1] = NumFl.NumFlux(Fl, qm, qp)
        num_flux[-1] = num_flux[0]

        # Derivative
        Out[:] = - (num_flux[1:] - num_flux[:-1]) / (L/len(In))

max_stars_repo_name: danielcort/benchmarks-python-julia-c

max_issues_repo_name: danielcort/benchmarks-python-julia-c

max_forks_repo_name: danielcort/benchmarks-python-julia-c


text:
# -*- coding: utf-8 -*-
import inspect
import numpy as np
import cv2
import os
import copy

__author__ = "Ulysse Rubens <urubens@uliege.be>"
__version__ = "0.1"

def check_params(fns, params, exceptions=None):
    if not exceptions:
        exceptions = []

    legal_params = []
    for fn in fns:
        legal_params += inspect.getargspec(fn)[0]
    legal_params = set(legal_params)

    for params_name in params:
        if params_name not in legal_params:
            if params_name not in exceptions:
                raise ValueError(
                        '{} is not a legal parameter'.format(params_name))

def merge_dicts(*dict_args):
    """
    Given any number of dicts, shallow copy and merge into a new dict,
    precedence goes to key value pairs in latter dicts.
    """
    result = {}
    for dictionary in dict_args:
        result.update(dictionary)
    return result

def files_in_directory(path):
    return [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))
            and not f.startswith(".")]

def make_dirs(path, remove_filename=False):
    if remove_filename:
        path = os.path.dirname(path)

    if not os.path.exists(path):
        os.makedirs(path)

def pad_image(image, padding=(0,0), mode='reflect'):
    if padding == (0,0):
        return image

    if image.ndim == 3:
        pad = ((padding[0], padding[0]), (padding[1], padding[1]), (0, 0))
    else:
        pad = ((padding[0], padding[0]), (padding[1], padding[1]))

    return np.pad(image, pad, mode=mode)

def open_image(filename, flag=None, padding=(0,0), padding_mode='reflect'):
    if not os.path.exists(filename):
        raise IOError("File " + filename + " does not exist !")

    if flag in ('L', cv2.IMREAD_GRAYSCALE):
        flag = cv2.IMREAD_GRAYSCALE
    elif flag in ('RGB', cv2.IMREAD_COLOR):
        flag = cv2.IMREAD_COLOR
    else:
        flag = cv2.IMREAD_UNCHANGED

    im = cv2.imread(filename, flag)

    if im is None:
        raise ValueError("Image " + filename + "unreadable !")

    return pad_image(im, padding, padding_mode)

def open_image_with_mask(filename, padding=(0, 0), splitted=True):
    image = open_image(filename)
    if image.shape[2] == 3:
        mask = pad_image(np.ones_like(image[:, :, 0]), padding=padding, mode='constant')
        image = pad_image(image, padding=padding, mode='reflect')
    elif image.shape[2] == 4:
        mask = pad_image(image[:, :, 3], padding=padding, mode='constant')
        image = pad_image(image[:, :, :3], padding=padding, mode='reflect')
    else:
        raise ValueError("Impossible to load image")

    if not splitted:
        return np.dstack((image, mask))
    else:
        return image, mask

def open_scoremap(filename, padding=(0, 0)):
    scoremap = open_image(filename, flag='L', padding=padding)
    scoremap /= np.max(scoremap)
    return scoremap.astype(np.float16)

def check_default(param, default_value=None, return_list=True):
    if param is None or len(param) == 0:
        if return_list:
            return [default_value]
        else:
            return default_value

    return param

def check_max_features(max_features):
    ret = []
    for mf in max_features:
        if '.' in mf:
            ret.append(float(mf))
        elif mf == 'sqrt':
            ret.append(mf)
        else:
            ret.append(int(mf))

    return ret

def params_remove_list(params):
    param_names = [p for p in dir(params) if not p.endswith('__')]

    for param_name in param_names:
        v = getattr(params, param_name, None)
        if isinstance(v, list):
            setattr(params, param_name, v[0])

    return params

def params_remove_none(params):
    ret = copy.deepcopy(params)
    param_names = [p for p in dir(ret) if not p.endswith('__')]

    for param_name in param_names:
        v = getattr(ret, param_name, None)
        if v is None:
            setattr(ret, param_name, " ")

    return ret

max_stars_repo_name: urubens/CellCounting

max_issues_repo_name: urubens/CellCounting

max_forks_repo_name: urubens/CellCounting