text:
import numpy as np
import pandas as pd
import scipy.stats as st
from plotnine import *
from mizani.breaks import *
from mizani.formatters import *
import seaborn as sns
from sklearn.linear_model import LinearRegression

#######################################################################
'''
Snippet for plotnine with thai font by @korakot
https://gist.github.com/korakot/01d181229b21411b0a20784e0ca20d3d
'''
import matplotlib
# !wget https://github.com/Phonbopit/sarabun-webfont/raw/master/fonts/thsarabunnew-webfont.ttf -q
# !cp thsarabunnew-webfont.ttf /usr/share/fonts/truetype/
matplotlib.font_manager._rebuild()
matplotlib.rc('font', family='TH Sarabun New')
theme_set(theme_minimal(11, 'TH Sarabun New'))
#######################################################################

#cleaning
def check_missing(df):
    per_missing = df.isnull().mean()
    missing_df = pd.DataFrame({'col_name': df.columns, 'per_missing': per_missing})
    missing_df = missing_df.sort_values('per_missing',ascending=False).reset_index(drop=True)
    missing_df['rnk'] = missing_df.index.map(lambda x: str(x).zfill(2)+'_') + missing_df.col_name
    return missing_df

def check_mode(df):
    mode_df = []
    for col in df.columns:
        x = df[col].value_counts()
        mode_df.append({'col':col, 'value':x.index[0], 'per_mode': list(x)[0]/df.shape[0],
                       'nb_value':len(x)})
    mode_df = pd.DataFrame(mode_df)[['col','value','per_mode','nb_value']]\
        .sort_values('per_mode',ascending=False)
    return mode_df.reset_index(drop=True)

def remove_outliers(df,col):
    q1 = np.percentile(df[col], 25)
    q3 = np.percentile(df[col], 75)
    iqr = q3-q1
    df = df[(df[col] < q3+1.5*iqr)&(df[col] > q1-1.5*iqr)]
    return df.reset_index(drop=True)

def value_dist(df,col):
    x = pd.DataFrame(df[col].value_counts()).reset_index()
    x.columns = ['value','cnt']
    x['per'] = x.cnt / x.cnt.sum()
    return x

def otherify(df,col, th=0.03, retain=['NA']):
    value_df = value_dist(df,col)
    other_cols = list(value_df[value_df.per<th].value)
    df[col] = df[col].map(lambda x: 'others' if (x in other_cols and x not in retain) else x)
    return df

def replace_dict(x, d):
    for key,value_list in d.items():
        for v in value_list:
            if v in str(x).lower():
                return key
    return 'others'

#visualization
def thai_text(font_size=11):
    #thanks to @korakot for Thai font with plotnine
    return theme(text=element_text(size=font_size,  family='TH Sarabun New'))

def cat_plot(df,col):
    g = (ggplot(df,aes(x=col)) +
         geom_bar(stat='bin', #histogram
                  binwidth=0.5, #histogram binwidth
                  bins=len(df[col].unique())) + #how many bins
         coord_flip()
#          theme(axis_text_x=element_blank())
        )
    return g

def numcat_plot(df,num,cat, no_outliers=True, geom=geom_boxplot()):
    if no_outliers:
        new_df = remove_outliers(df,num)
    else:
        new_df = df.copy()
    g = (ggplot(new_df, aes(x=cat,y=num)) +
         geom
        )
    return g

def numdist_plot(df, num,cat, geom=geom_density(alpha=0.5), no_outliers=True):
    if no_outliers:
        new_df = remove_outliers(df,num)
    else:
        new_df = df.copy()
    g = (ggplot(new_df,aes(x=num, fill=cat)) +
          geom
        )
    return g

def catcat_plot(df, cat_dep, cat_ind):
    plot_df = df.copy()
    plot_df['cnt'] = 1
    df_total = plot_df[[cat_ind,'cnt']].groupby(cat_ind).count().reset_index()
    df_agg = plot_df[[cat_dep,cat_ind,'cnt']].groupby([cat_dep,cat_ind]).count().reset_index()
    df_agg = df_agg.merge(df_total,on='color')
    df_agg['per'] = df_agg.cnt_x / df_agg.cnt_y
    g = (ggplot(df_agg,aes(x=cat_dep, y='per',fill=cat_dep)) +
         geom_col() +
         geom_text(aes(x=cat_dep,y='per',label='cnt_x')) +
         theme(axis_text_x = element_blank()) +
         facet_wrap(f'~{cat_ind}')) + theme(panel_spacing_x=0.5)
    return g

def value_dist_plot(df,bins=30):
    num_m = df.melt()
    g = (ggplot(num_m,aes(x='value')) +
         geom_bar(stat='bin', bins=bins) +
         facet_wrap('~variable', scales='free') + #facetting by variable
         theme_minimal() + theme(panel_spacing_x=0.5)
        )
    return g

def jointplot(df,col_x, col_y, no_outliers=True, kind='reg'): #'scatter','resid','reg','hex','kde','point'
    if no_outliers:
        new_df = remove_outliers(df,col_x)
        new_df = remove_outliers(new_df,col_y)
    else:
        new_df = df.copy()
    return sns.jointplot(new_df[col_x],new_df[col_y],kind=kind)

def qq_plot(df,col):
    qq, reg = calc_qq(df,col)
    g = (ggplot(qq,aes(x='theoretical_q',y='sample_q')) +
        geom_point() + #plot points
        geom_abline(slope=1,intercept=0,color='red') + #perfectly normal line
        stat_function(fun=lambda x: x*reg.coef_[0][0]) + #linear estimation
        ggtitle(f'y= {np.round(reg.coef_[0][0],2)} * x')+ #display equation
        labs(x='Theoretical Quantiles (normalized)', y='Sample Qunatiles (normalized)'))
    return g

def boxcox_plot(df, col, ls = [i/10 for i in range(-30,31,5)]):
    lamb_df = boxcox_lamb_df(df[col],ls)
    g = (ggplot(lamb_df, aes(x='lamb',y='coef',group=1)) +
         geom_point() + geom_line())
    return g

#transformation
def calc_qq(df,col):
    sample_qs = [(np.percentile(df[col],i)-np.mean(df[col]))/np.std(df[col]) for i in range(5,100,5)]
    theoretical_qs = [st.norm.ppf(i/100) for i in range(5,100,5)]
    qq = pd.DataFrame({'sample_q':sample_qs,'theoretical_q':theoretical_qs})
    reg = LinearRegression(fit_intercept=False).fit(np.array(qq['theoretical_q'])[:,None],
                                 np.array(qq['sample_q'])[:,None])
    return qq, reg

def boxcox(ser,lamb=0):
    ser+= 1 - ser.min()
    if lamb==0:
        return np.log(ser)
    else:
        return (ser**lamb - 1)/lamb

def boxcox_lamb_df(ser, ls = [i/10 for i in range(-30,31,5)]):
    coefs = []
    for l in ls:
        df = pd.DataFrame.from_dict({'val': boxcox(ser,l)})
        qq, reg = calc_qq(df,'val')
        coefs.append(reg.coef_.squeeze().item())
    return pd.DataFrame({'lamb':ls,'coef':coefs})

def boxcox_lamb(ser, ls = [i/10 for i in range(-30,31,5)]):
    df = boxcox_lamb_df(ser,ls)
    return df.lamb[df.coef.idxmax()]

#formatters
class k_format(custom_format):
    def k_ize(self,x):
        #format one tick
        return f'{int(x/1000)}k'
    def __call__(self,x):
        #format a list of ticks
        return [self.k_ize(i) for i in x]

class m_format(custom_format):
    def m_ize(self,x):
        #format one tick
        return f'{int(abs(x)/1e6)}M'
    def __call__(self,x):
        #format a list of ticks
        return [self.m_ize(i) for i in x]

class prettynum_format(currency_format):
    def __init__(self, prefix='', suffix='', digits=0, big_mark=','):
        self.prefix = prefix
        self.suffix = suffix
        self.digits = digits
        self.big_mark = big_mark

max_issues_repo_name: tanlull/viztech

max_forks_repo_name: tanlull/viztech

reason: import numpy,import scipy


text:
import numpy as np
import cv2, argparse, time, datetime

"""
example useage: python savingAVideo.py -r 10 -n testing -i 0
"""

beginningTime = time.time()

ap = argparse.ArgumentParser()
ap.add_argument("-n", "--videoName", help = "name of the video to be saved",required=True)
ap.add_argument("-i", "--inputStream", help = "input stream. typically 0 or 1. defaults to 1",default=1,type=int)
ap.add_argument("-r", "--recordingTime", help = "how many seconds do you want to the program to record for? defaults to 20 min",default=1200,type=int,required=True)
ap.add_argument("-w", "--waitingTime", help = "how many seconds do you want the program to wait for before it begins? defaults to 10 min",default=600,type=int,required=True)

args = ap.parse_args()

# print arguments to the screen
print("\n\n\tname of trial: {}".format(args.videoName))
print("\tinput: {}".format(args.inputStream))
print("\twaitingTime: {}".format(args.waitingTime))
print("\tlength of time to record: {}".format(args.recordingTime)+"\n")

args = vars(ap.parse_args())

# enter a check for acclimation time. just want to ensure that acclimation time is long enough that it doesn't interfere with calculating the background image
name = str(args["videoName"])
input = args["inputStream"]
recordingTime = args["recordingTime"]
waiting = args["waitingTime"]

cap = cv2.VideoCapture(input)

width = cap.get(3)
height = cap.get(4)

# Define the codec and create VideoWriter object
fourcc = cv2.cv.CV_FOURCC('m', 'p', '4', 'v')
out = cv2.VideoWriter(name + '.avi',fourcc, 5.0, (int(width),int(height)))

# spin your wheels until recording starts:
while(time.time() < beginningTime + waiting):
	print round(beginningTime + waiting - time.time(),1)
	time.sleep(1)

print "\n\n\n\n\n\tDONE WAITING"

enterTheLoop = time.time()

counter = 0
while(time.time()-enterTheLoop <= recordingTime):
	ret, frame = cap.read()
	counter += 1
	if ret==True:
    	# time stamp
		cv2.putText(frame,str(datetime.datetime.now().strftime("%D %H:%M:%S.%f")), (20,20),cv2.FONT_HERSHEY_PLAIN, 1.0,(255,255,255))
		# write the frame
		out.write(frame)
		#cv2.imshow('frame',frame)

		if counter % 100 == 0:
			print "still recording"

		if cv2.waitKey(1) & 0xFF == ord('q'):
			break
	else:
		break

# Release everything if job is finished
cap.release()
out.release()

print str(counter) + "frames total"
print "realized FPS: " + str(counter/(enterTheLoop-time.time()))

cv2.destroyAllWindows()

ext: py

max_stars_repo_name: lukereding/mateChoiceTracking

max_issues_repo_name: lukereding/mateChoiceTracking

max_forks_repo_name: lukereding/mateChoiceTracking

reason: import numpy


text:
"""
Paul Rozdeba (prozdeba@physics.ucsd.edu)
Department of Physics
University of California, San Diego
May 23, 2017

NeuralNetVA

Carry out variational annealing (VA) to train a neural network model.  Network
states and activation function parameters are estimated from training data
consisting of many input/output pair examples.

VA is a form of variational data assimilation that uses numerical continuation
to regularize the variational cost function, or "action", in a controlled way.
VA was first proposed by Jack C. Quinn in his Ph.D. thesis (2010) [1], and is
described by J. Ye et al. (2015) in detail in [2].

This code uses automatic differentiation to evaluate derivatives of the
action for optimization as implemented in ADOL-C, wrapped in Python code in a
package called PYADOLC (installation required for usage of VarAnneal).
PYADOLC is available at https://github.com/b45ch1/pyadolc.

To run the annealing algorithm using this code, instantiate an Annealer object
in your code using this module.  This object allows you to load in observation
data, set a model for the system, initial guesses for the states and parameters,
etc.  To get a good sense of how to use the code, follow along with the examples
included with this package, and the user guide (coming soon).

References:
[1] J.C. Quinn, "A path integral approach to data assimilation in stochastic
    nonlinear systems."  Ph.D. thesis in physics, UC San Diego (2010).
    Available at: https://escholarship.org/uc/item/0bm253qk

[2] J. Ye et al., "Improved variational methods in statistical data assimilation."
    Nonlin. Proc. in Geophys., 22, 205-213 (2015).
"""

import numpy as np
import adolc
import time
#import scipy.optimize as opt
from _autodiffmin import ADmin

class Annealer(ADmin):
    """
    Annealer is the main object type for performing variational data
    assimilation using VA.  It inherits the function minimization routines
    from ADmin, which uses automatic differentiation.
    """
    def __init__(self):
        """
        Constructor for the Annealer class.
        """
        self.taped = False
        self.annealing_initialized = False
        self.M = 0  # number of training examples
        self.structure = None

    def set_structure(self, structure):
        """
        Set up the network structure.
        structure - a length-N numpy array for an N-layer network, where each
        entry is the number of neurons in that layer.
        """
        self.structure = structure
        self.N = len(structure)
        if self.M > 0:
            self.NDnet = np.sum(structure)
            self.NDens = self.NDnet * self.M

    def set_activation(self, f):
        """
        Set activation for a neuron.
        f - activation function
        """
        self.f = f

    def set_input_data(self, data):
        """
        Set data for the input layer.
        """
        if data.ndim == 1:
            self.data_in = np.array([data])
        else:
            self.data_in = data

        if self.M == 0:
            self.M = data.shape[0]
        if self.structure is not None:
            self.NDnet = np.sum(self.structure)
            self.NDens = self.NDnet * self.M

    def set_output_data(self, data):
        """
        Set data for the output layer.
        """
        if data.ndim == 1:
            self.data_out = np.array([data])
        else:
            self.data_out = data

        if self.M == 0:
            self.M = data.shape[0]
        if self.structure is not None:
            self.NDnet = np.sum(self.structure)
            self.NDens = self.NDnet * self.M

    ############################################################################
    # Gaussian action
    ############################################################################
    def A_gaussian(self, XP):
        """
        Calculate the Gaussian action all in one go.
        """
        return self.me_gaussian(XP) + self.fe_gaussian(XP)

    def me_gaussian(self, XP):
        """
        Gaussian measurement error.
        """
        X = XP[:self.NDens]

        # Extract input and output layer states corresponding to the first example
        xmeas_in = X[:self.structure[0]][self.Lidx[0]]
        xmeas_out = X[(self.NDnet - self.structure[-1]):self.NDnet][self.Lidx[1]]

        # Calculate measurement error for the first example
        diff_in = xmeas_in - self.data_in[0]
        diff_out = xmeas_out - self.data_out[0]

        if type(self.RM) == np.ndarray:
            if self.RM.shape == (2,):
                merr = self.RM[0] * np.sum(diff_in*diff_in)
                merr = merr + self.RM[1] * np.sum(diff_out*diff_out)
            else:
                # Assume otherwise that you put in an array which looks like
                # [[RMin_1, RMin_2, ..., RMin_Lin], [RMout_1, RMout_2, ..., RMout_Lout]]
                merr = np.dot(diff_in, np.dot(self.RM[0], diff_in))
                merr = merr + np.dot(diff_out, np.dot(self.RM[1], diff_out))
            #else:
            #    print("ERROR: RM has an invalid shape. Exiting.")
            #    exit(1)
        else:
            merr = self.RM * (np.sum(diff_in*diff_in) + np.sum(diff_out*diff_out))

        # Loop through the remaining M-1 examples and add to the measurement error
        for m in xrange(1, self.M):
            # Get input and output layer states for example m+1
            i0 = m * self.NDnet
            i1 = (m+1) * self.NDnet
            xmeas_in = X[i0:i0 + self.structure[0]][self.Lidx[0]]
            xmeas_out = X[i1 - self.structure[-1]:i1][self.Lidx[1]]

            # Calculate error vectors and measurement error contribution
            diff_in = xmeas_in - self.data_in[m]
            diff_out = xmeas_out - self.data_out[m]

            if type(self.RM) == np.ndarray:
                if self.RM.shape == (2,):
                    merr = merr + self.RM[0] * np.sum(diff_in*diff_in)
                    merr = merr + self.RM[1] * np.sum(diff_out*diff_out)
                else:
                    # Assume otherwise that you put in an array which looks like
                    # [[RMin_1, RMin_2, ..., RMin_Lin], [RMout_1, RMout_2, ..., RMout_Lout]]
                    merr = merr + np.dot(diff_in, np.dot(self.RM[0], diff_in))
                    merr = merr + np.dot(diff_out, np.dot(self.RM[1], diff_out))
                #else:
                #    print("ERROR: RM has an invalid shape. Exiting.")
                #    exit(1)
            else:
                merr = merr + self.RM * (np.sum(diff_in*diff_in) + np.sum(diff_out*diff_out))

        return merr / np.float64(self.Ltot * self.M)

    def fe_gaussian(self, XP):
        """
        Gaussian model error.
        """
        # Extract state and parameters from XP.
        if self.NPest == 0:
            x = XP
            p = self.P
        elif self.NPest == self.NP:
            x = XP[:self.NDens]
            p = XP[self.NDens:]
        else:
            x = XP[:self.NDens]
            p = np.array(self.P, dtype=XP.dtype)
            p[self.Pidx] = XP[self.NDens:]
        p = np.array(p)

        # Start calculating the model error.
        # First get out the paramters, which are the same for EVERY example.
        W = []
        b = []
        W_i0 = 0
        W_if = self.structure[0]*self.structure[1]
        b_i0 = W_if
        b_if = b_i0 + self.structure[1]
        for n in xrange(self.N - 1):
            W.append(np.reshape(p[W_i0:W_if], (self.structure[n+1], self.structure[n])))
            b.append(p[b_i0:b_if])
            if n < self.N - 2:
                W_i0 = b_if
                W_if = W_i0 + self.structure[n+1]*self.structure[n+2]
                b_i0 = W_if
                b_if = b_i0 + self.structure[n+2]

        # Now loop over each copy of the network for each example
        for m in xrange(self.M):
            # Get entire network state for example m+1
            Xm = x[m*self.NDnet:(m+1)*self.NDnet]
            diff = np.zeros(np.sum(self.structure[1:]), dtype=x.dtype)
            diff_i0 = 0
            diff_if = self.structure[1]
            xn_i0 = 0
            xn_if = self.structure[0]
            xnp1_i0 = self.structure[0]
            xnp1_if = self.structure[0] + self.structure[1]
            #W_i0 = 0
            #W_if = self.structure[0]*self.structure[1]
            #b_i0 = W_if
            #b_if = b_i0 + self.structure[1]

            for n in xrange(self.N-1):
                xn = Xm[xn_i0:xn_if]
                xnp1 = Xm[xnp1_i0:xnp1_if]
                #W = np.reshape(p[W_i0:W_if], (self.structure[n+1], self.structure[n]))
                #b = p[b_i0:b_if]
                diff[diff_i0:diff_if] = xnp1 - self.disc(xn, W[n], b[n])

                if n < self.N - 2:
                    diff_i0 = diff_if
                    diff_if = diff_i0 + self.structure[n+2]
                    xn_i0 = xn_if
                    xn_if = xn_i0 + self.structure[n+1]
                    xnp1_i0 = xnp1_if
                    xnp1_if = xnp1_i0 + self.structure[n+2]
                    W_i0 = b_if
                    W_if = W_i0 + self.structure[n+1]*self.structure[n+2]
                    b_i0 = W_if
                    b_if = b_i0 + self.structure[n+2]
            if m == 0:
                ferr = self.RF * np.sum(diff * diff)
            else:
                ferr = ferr + self.RF * np.sum(diff * diff)
        #print diff
        #exit(0)
        #if isinstance(diff[0], adolc._adolc.adub):
        #    pass
        #else:
        #    print diff
        #    exit(0)
        #ferr = self.RF * np.sum(diff * diff)
        return ferr / np.float64((self.NDnet - self.structure[0]) * self.M)

    ############################################################################
    # Discretization routines
    ############################################################################
    def disc_forwardmap(self, x, W, b):
        """
        "Discretization" when f is a forward mapping, not an ODE.
        """
        return self.f(x, W, b)

    ############################################################################
    # Annealing functions
    ############################################################################
    def anneal(self, X0, P0, alpha, beta_array, RM, RF0, Pidx, Lidx=None,
               init_to_data=True, action='A_gaussian', disc='forwardmap',
               method='L-BFGS-B', bounds=None, opt_args=None, adolcID=0):
        """
        Convenience function to carry out a full annealing run over all values
        of beta in beta_array.
        """
        # initialize the annealing procedure, if not already done
        if self.annealing_initialized == False:
            self.anneal_init(X0, P0, alpha, beta_array, RM, RF0, Pidx, Lidx,
                             init_to_data, action, disc, method, bounds,
                             opt_args, adolcID)
        for i in beta_array:
            print('------------------------------')
            print('Step %d of %d'%(self.betaidx+1, len(self.beta_array)))
            print('beta = %d, RF = %.8e'%(self.beta, self.RF))
            print('')
            self.anneal_step()

    def anneal_init(self, X0, P0, alpha, beta_array, RM, RF0, Pidx, Lidx=None,
                    init_to_data=True, action='A_gaussian', disc='forwardmap',
                    method='L-BFGS-B', bounds=None, opt_args=None, adolcID=0):
        """
        Initialize the annealing procedure.
        """
        if method not in ('L-BFGS-B', 'NCG', 'LM', 'TNC'):
            print("ERROR: Optimization routine not recognized. Annealing not initialized.")
            return 1
        else:
            self.method = method

        # Store optimization bounds. Will only be used if the chosen
        # optimization routine supports it.
        if bounds is None:
            self.bounds = None
        else:
            self.bounds = np.array(bounds)

        # get optimization extra arguments
        self.opt_args = opt_args

        # set up parameters and determine if static or time series
        self.P = P0
        self.NP = len(P0)
        #if P0.ndim == 1:
        #    # Static parameters, so p is a single vector.
        #    self.NP = len(P0)
        #else:
        #    # Time-dependent parameters, so p is a time series of N values.
        #    self.NP = P0.shape[1]

        # get indices of parameters to be estimated by annealing
        self.Pidx = Pidx
        self.NPest = len(Pidx)

        # get indices of measured components
        if Lidx is None:
            self.Lidx = [np.linspace(0, self.structure[0]-1, self.structure[0]),
                         np.linspace(0, self.structure[-1]-1, self.structure[-1])]
        else:
            self.Lidx = Lidx

        self.L = [len(self.Lidx[0]), len(self.Lidx[1])]
        self.Ltot = self.L[0] + self.L[1]

        # Simply set RM here; measurement error function should decide how to handle it
        self.RM = RM

        ## get indices of measured components of f
        #self.Lidx = Lidx
        #self.L = len(Lidx)
        #
        ## properly set up the bounds arrays
        #if bounds is not None:
        #    bounds_full = []
        #    state_b = bounds[:self.D]
        #    param_b = bounds[self.D:]
        #    # set bounds on states for all N time points
        #    for n in xrange(self.N):
        #        for i in xrange(self.D):
        #            bounds_full.append(state_b[i])
        #    # set bounds on parameters
        #    if self.P.ndim == 1:
        #        # parameters are static
        #        for i in xrange(self.NPest):
        #            bounds_full.append(param_b[i])
        #    else:
        #        # parameters are time-dependent
        #        for n in xrange(self.N):
        #            for i in xrange(self.NPest):
        #                bounds_full.append(param_b[i])
        #else:
        #    bounds_full = None
        #
        ## Reshape RM and RF so that they span the whole time series.  This is
        ## done because in the action evaluation, it is more efficient to let
        ## numpy handle multiplication over time rather than using python loops.
        #if type(RM) == np.ndarray:
        #    if RM.shape == (self.L,):
        #        self.RM = np.resize(RM, (self.N, self.L))
        #    elif RM.shape == (self.L, self.L):
        #        self.RM = np.resize(RM, (self.N, self.L, self.L))
        #    elif RM.shape == (self.N, self.L) or RM.shape == np.resize(self.N, self.L, self.L):
        #        self.RM = RM
        #    else:
        #        print("ERROR: RM has an invalid shape. Exiting.")
        #        exit(1)
        #
        #else:
        #    self.RM = RM
        #
        #if type(RF0) == np.ndarray:
        #    if RF0.shape == (self.D,):
        #        self.RF0 = np.resize(RF0, (self.N - 1, self.D))
        #    elif RF0.shape == (self.D, self.D):
        #        self.RF0 = np.resize(RF0, (self.N - 1, self.D, self.D))
        #    elif RF0.shape == (self.N - 1, self.D) or RF0.shape == (self.N - 1, self.D, self.D):
        #        self.RF0 = RF0
        #    else:
        #        print("ERROR: RF0 has an invalid shape. Exiting.")
        #        exit(1)
        #
        #else:
        #    self.RF0 = RF0

        # set up beta array in RF = RF0 * alpha**beta
        self.alpha = alpha
        self.beta_array = beta_array
        self.betaidx = 0
        self.beta = self.beta_array[self.betaidx]
        self.Nbeta = len(self.beta_array)

        # set current RF
        if RF0 is not None:
            self.RF0 = RF0
        self.RF = self.RF0 * self.alpha**self.beta

        # set the desired action
        if self.method == 'LM':
            # Levenberg-Marquardt requires a "vector action"
            self.A = self.vecA_gaussian
        elif type(action) == str:
            exec 'self.A = self.%s'%(action)
        else:
            # Assumption: user has passed a function pointer
            self.A = action

        # set the discretization
        exec 'self.disc = self.disc_%s'%(disc,)

        # array to store minimizing paths
        self.minpaths = np.zeros((self.Nbeta, self.NDens + self.NP), dtype=np.float64)
        #if P0.ndim == 1:
        #    self.minpaths = np.zeros((self.Nbeta, self.N*self.D + self.NP), dtype=np.float64)
        #else:
        #    self.minpaths = np.zeros((self.Nbeta, self.N*(self.D + self.NP)), dtype=np.float64)

        # initialize observed state components to data if desired
        if init_to_data == True:
            for m in xrange(self.M):
                i0 = m * self.NDnet
                i1 = i0 + self.structure[0]
                i2 = (m+1) * self.NDnet - self.structure[-1]
                i3 = (m+1) * self.NDnet
                X0[i0:i1][self.Lidx[0]] = self.data_in[m]
                X0[i2:i3][self.Lidx[1]] = self.data_out[m]

        #if self.NPest > 0:
        #    if P0.ndim == 1:
        #        XP0 = np.append(X0.flatten(), P0)
        #    else:
        #        XP0 = np.append(X0.flatten(), P0.flatten())
        #else:
        #    XP0 = X0.flatten()

        XP0 = np.append(X0, P0)
        self.minpaths[0] = XP0

        # array to store optimization results
        self.A_array = np.zeros(self.Nbeta, dtype=np.float64)
        self.me_array = np.zeros(self.Nbeta, dtype=np.float64)
        self.fe_array = np.zeros(self.Nbeta, dtype=np.float64)
        self.exitflags = np.empty(self.Nbeta, dtype=np.int8)

        # set the adolcID
        self.adolcID = adolcID

        # Initialization successful, we're at the beta = beta_0 step now.
        self.initalized = True

    def anneal_step(self):
        """
        Perform a single annealing step. The cost function is minimized starting
        from the previous minimum (or the initial guess, if this is the first
        step). Then, RF is increased to prepare for the next annealing step.
        """
        # minimize A using the chosen method
        if self.method in ['L-BFGS-B', 'NCG', 'TNC']:#, 'LM']:
            if self.betaidx == 0:
                if self.NP == self.NPest:
                    XP0 = np.copy(self.minpaths[0])
                else:
                    X0 = self.minpaths[0][:self.NDens]
                    P0 = self.minpaths[0][self.NDens:][self.Pidx]
                    XP0 = np.append(X0, P0)
            else:
                if self.NP == self.NPest:
                    XP0 = np.copy(self.minpaths[self.betaidx-1])
                else:
                    X0 = self.minpaths[self.betaidx-1][:self.NDens]
                    P0 = self.minpaths[self.betaidx-1][self.NDens:][self.Pidx]
                    XP0 = np.append(X0, P0)

            if self.method == 'L-BFGS-B':
                XPmin, Amin, exitflag = self.min_lbfgs_scipy(XP0, self.gen_xtrace())
            elif self.method == 'NCG':
                XPmin, Amin, exitflag = self.min_cg_scipy(XP0, self.gen_xtrace())
            elif self.method == 'TNC':
                XPmin, Amin, exitflag = self.min_tnc_scipy(XP0, self.gen_xtrace())
            #elif self.method == 'LM':
            #    XPmin, Amin, exitflag = self.min_lm_scipy(XP0)
            else:
                print("You really shouldn't be here.  Exiting.")
                sys.exit(1)
        else:
            print("ERROR: Optimization routine not implemented or recognized.")
            sys.exit(1)

        # update optimal parameter values
        if self.NPest > 0:
            #if isinstance(XPmin[0], adolc._adolc.adouble):
            if isinstance(XPmin[0], adolc._adolc.adub):
                Pest_vals = [XPmin[-self.NPest + i].val for i in xrange(self.NPest)]
                self.P[self.Pidx] = np.array(Pest_vals)
            else:
                self.P[self.Pidx] = np.copy(XPmin[-self.NPest:])

        # store A_min and the minimizing path
        self.A_array[self.betaidx] = Amin
        self.me_array[self.betaidx] = self.me_gaussian(np.array(XPmin))
        self.fe_array[self.betaidx] = self.fe_gaussian(np.array(XPmin))
        self.minpaths[self.betaidx] = np.array(np.append(XPmin[:self.NDens], self.P))

        # increase RF
        if self.betaidx < len(self.beta_array) - 1:
            self.betaidx += 1
            self.beta = self.beta_array[self.betaidx]
            self.RF = self.RF0 * self.alpha**self.beta

        # set flags indicating that A needs to be retaped, and that we're no
        # longer at the beginning of the annealing procedure
        self.taped = False
        if self.annealing_initialized == True:
            # Indicate no longer at initial beta value
            self.initialized = False

    ################################################################################
    # Routines to save annealing results.
    ################################################################################
    def save_states(self, filename, dtype=np.float64, fmt="%.8e"):
        """
        Save minimizing neuron states (not including parameters).
        """
        #savearray = np.reshape(self.minpaths[:, :self.NDens], (self.Nbeta, self.NDens))

        savearray = []
        for m in xrange(self.M):
            savearray.append([])
            for b in xrange(self.Nbeta):
                savearray[m].append([])
                i0 = m * self.NDnet
                i1 = i0 + self.structure[0]
                for n in xrange(self.N):
                    savearray[m][b].append(self.minpaths[b, i0:i1])
                    i0 = i1
                    i1 = i0 + self.structure[n]
        savearray = np.array(savearray)

        if filename.endswith('.npy'):
            for m in xrange(self.M):
                for b in xrange(self.Nbeta):
                    for n in xrange(self.N):
                        savearray[m][b][n] = savearray[m][b][n].astype(dtype)
            np.save(filename, savearray)
        else:
            np.savetxt(filename, savearray, fmt=fmt)

    def save_io(self, filename, dtype=np.float64, fmt="%.8e"):
        """
        Save minimizing input/output neuron states (not including parameters).
        """
        #savearray = np.reshape(self.minpaths[:, :self.NDens], (self.Nbeta, self.NDens))

        savearray = []
        for m in xrange(self.M):
            savearray.append([])
            for b in xrange(self.Nbeta):
                savearray[m].append([])
                i0 = m * self.NDnet
                i1 = i0 + self.structure[0]
                savearray[m][b].append(self.minpaths[b, i0:i1])
                i0 = (m+1) * self.NDnet - self.structure[-1]
                i1 = (m+1) * self.NDnet
                savearray[m][b].append(self.minpaths[b, i0:i1])
        savearray = np.array(savearray)

        if filename.endswith('.npy'):
            for m in xrange(self.M):
                for b in xrange(self.Nbeta):
                    savearray[m][b][0] = savearray[m][b][0].astype(dtype)
                    savearray[m][b][1] = savearray[m][b][1].astype(dtype)
            np.save(filename, savearray)
        else:
            np.savetxt(filename, savearray, fmt=fmt)

    def save_params(self, filename, dtype=np.float64, fmt="%.8e"):
        """
        Save minimum action parameter values.
        """
        if self.NPest == 0:
            print("WARNING: You did not estimate any parameters.  Writing fixed " \
                  + "parameter values to file anyway.")

        # write fixed parameters to array
        savearray = np.resize(self.P, (self.Nbeta, self.NP))
        # write estimated parameters to array
        if self.NPest > 0:
            est_param_array = self.minpaths[:, self.NDnet:]
            savearray[:, self.Pidx] = est_param_array

        if filename.endswith('.npy'):
            np.save(filename, savearray.astype(dtype))
        else:
            np.savetxt(filename, savearray, fmt=fmt)

    def save_Wb(self, W_filename, b_filename, dtype=np.float64):
        """
        Save W and b in separate files.
        """
        # write fixed parameters to array
        W = []
        b = []
        for i in xrange(self.Nbeta - 1, self.Nbeta):
            W.append([])
            b.append([])
            p_i0 = self.NDens
            p_i1 = self.NDens + self.structure[0]*self.structure[1] + self.structure[1]
            for n in xrange(self.N-1):
                pn = self.minpaths[i, p_i0:p_i1]
                W[0].append(np.reshape(pn[:self.structure[n]*self.structure[n+1]],
                                       (self.structure[n+1], self.structure[n])))
                b[0].append(pn[-self.structure[n+1]:])

                if n < self.N - 2:
                    p_i0 = p_i1
                    p_i1 += self.structure[n+1]*self.structure[n+2] + self.structure[n+2]

        W = np.array(W)
        b = np.array(b)

        for i in xrange(self.Nbeta - 1, self.Nbeta):
            for n in xrange(self.N-1):
                W[0][n] = W[0][n].astype(dtype)
                b[0][n] = b[0][n].astype(dtype)
        np.save(W_filename, W)
        np.save(b_filename, b)

    def save_action_errors(self, filename, cmpt=0, dtype=np.float64, fmt="%.8e"):
        """
        Save beta values, action, and errors (with/without RM and RF) to file.
        cmpt sets which component of RF0 to normalize by.
        """
        savearray = np.zeros((self.Nbeta, 5))
        savearray[:, 0] = self.beta_array
        savearray[:, 1] = self.A_array
        savearray[:, 2] = self.me_array
        savearray[:, 3] = self.fe_array

        # Save model error / RF
        savearray[:, 4] = self.fe_array / (self.RF0 * self.alpha**self.beta_array)
        #if type(self.RF) == np.ndarray:
        #    if self.RF0.shape == (self.N - 1, self.D):
        #        savearray[:, 4] = self.fe_array / (self.RF0[0, 0] * self.alpha**self.beta_array)
        #    elif self.RF0.shape == (self.N - 1, self.D, self.D):
        #        savearray[:, 4] = self.fe_array / (self.RF0[0, 0, 0] * self.alpha**self.beta_array)
        #    else:
        #        print("RF shape currently not supported for saving.")
        #        return 1
        #else:
        #    savearray[:, 4] = self.fe_array / (self.RF0 * self.alpha**self.beta_array)

        if filename.endswith('.npy'):
            np.save(filename, savearray.astype(dtype))
        else:
            np.savetxt(filename, savearray, fmt=fmt)

    ############################################################################
    # AD taping & derivatives
    ############################################################################
    def gen_xtrace(self):
        """
        Define a random state vector for the AD trace.
        """
        xtrace = np.random.rand(self.NDens + self.NPest)
        return xtrace


text:
"""Tests for the weather_app main module."""
from datetime import datetime
import numpy as np
from random import randint

from weather_app.api import lookup_location, SMHIForecast
from weather_app import get_temperature_forecast

def test_temperature_forecast():
    """
    Ensure that forecast is correctly interpolated to given
    hours.
    """
    lat, lon = lookup_location()
    forecast = SMHIForecast(lat, lon)

    t_0 = (forecast.time[1] - forecast.reference_time).total_seconds() / 3600
    t_1 = (forecast.time[5] - forecast.reference_time).total_seconds() / 3600

    hours = np.linspace(t_0, t_1, randint(2, 20))
    temperature_forecast = get_temperature_forecast(hours)

    assert temperature_forecast[0] == forecast.temperature[1]
    assert temperature_forecast[-1] == forecast.temperature[5]

max_stars_repo_name: simonpf/weatherapp

max_issues_repo_name: simonpf/weatherapp

max_forks_repo_name: simonpf/weatherapp


text:
# -*- coding: utf-8 -*-

import os
import pytest

from Bio.Seq import Seq
import numpy as numpy

from neoRNA.io.secondary_structure_io import SecondaryStructureIO

parametrize = pytest.mark.parametrize

class TestSecondaryStructureIO(object):

    fileDir = os.path.dirname(os.path.realpath('__file__'))
    __EXAMPLE_FILENAME = 'tests/io/example_files/bprna_example.st'
    __EXAMPLE_FILE_PATH = os.path.join(fileDir, __EXAMPLE_FILENAME)

    def test_init(self):
        io = SecondaryStructureIO()
        secondary_structure_info = io.parse(self.__EXAMPLE_FILE_PATH)
        assert secondary_structure_info is not None

    def test_data_parsing(self):
        io = SecondaryStructureIO()
        secondary_structure_info = io.parse(self.__EXAMPLE_FILE_PATH)

        assert len(secondary_structure_info.elements) == 11


text:
import json
from pprint import pprint
import matplotlib.pyplot as plt
import numpy as np
from matplotlib import rcParams
from scipy.optimize import curve_fit

DATA_PATH = 'getHistory.html'
DATA_PATH_NEW = 'FAutoGlobalDailyList.json'

def read_data():
    data_processed = []
    # 0     1           2       3       4           5           6
    # date  confirmed   new     cured   deceased    deathrate   curerate
    data_x = []  # Date
    data_y = []  # Confirmed

    # Previous data (https://www.ncovchina.com/data/getHistory.html)
    # with open(DATA_PATH) as f:
    #     soup = BeautifulSoup(f.read())
    # data_raw = json.loads(soup.find('body').string)
    # for row in data_raw:
    #     month, day = row['date'].split('.')
    #     confirmed, suspected, cured, deceased = row['confirm'], row['suspect'], row['heal'], row['dead']
    #     int_date = int(month)*31 + int(day)
    #     data_processed.append([int_date, confirmed, suspected, cured, deceased])
    #     data_x.append(int_date)
    #     data_y.append(confirmed)
    # return data_processed, data_x, data_y

    # New data (https://api.inews.qq.com/newsqa/v1/automation/modules/list?modules=FAutoGlobalDailyList)
    with open(DATA_PATH_NEW) as f:
        data_raw = json.load(f)
    pprint(data_raw)
    for row in data_raw['data']['FAutoGlobalDailyList']:
        month, day = row['date'].split('.')
        confirmed, deceased, drate, cured, crate, new = row['all'].values()
        int_date = int(month)*31 + int(day)
        data_processed.append([int_date, confirmed, new, cured, deceased, drate, crate])
        data_x.append(int_date)
        data_y.append(confirmed)
    return data_processed, data_x, data_y

def plot_data(x, y, ax=plt, **kwargs):
    ax.plot(x, y, **kwargs)

def f(x, a):
    return np.power(a,x)

def func(x, a, b):
    return x**a + b

def main():
    rcParams['font.family'] = 'SimHei'
    fig, axes1 = plt.subplots()
    plt.xlabel('日期')
    plt.ylabel('确诊病例数')
    fig, axes2 = plt.subplots()
    plt.xlabel('日期')
    plt.ylabel('确诊病例数')
    fig, axes3 = plt.subplots()
    plt.xlabel('日期')
    plt.ylabel('确诊病例数')
    data, dx, dy = read_data()
    # dx, dy = dx[:50], dy[:50]
    pprint(dx, compact=True)
    pprint(dy, compact=True)

    axes1.set_title('全球新冠肺炎累计确诊数据')
    plot_data(dx, dy, ax=axes1, label='原始数据')

    # y1 = f(dx, 1.0788)
    # pprint(y1, compact=True)
    # plot_data(dx, y1, ax=axes, label='1.0788')
    #
    # y2 = f(dx, 1.08)
    # plot_data(dx, y2, ax=axes, label='1.08')
    #
    # y3 = f(dx, 1.07)
    # plot_data(dx, y3, ax=axes, label='1.07')
    #
    # y4 = f(dx, 1.0785)
    # plot_data(dx, y4, ax=axes, label='1.0785')

    axes2.set_title('新冠肺炎累计确诊数据拟合曲线')
    popt, pcov = curve_fit(func, dx, dy)
    y = [func(i, popt[0], popt[1]) for i in dx]
    axes2.plot(dx, dy, label='原始数据')
    axes2.plot(dx, y, 'r--', label='拟合数据')
    print(popt)

    axes3.set_title('新冠肺炎确诊数据预测')
    ex = np.arange(59, 300)
    popt, pcov = curve_fit(func, dx, dy)
    # popt数组中，三个值分别是待求参数a,b,c
    ey = [func(i, popt[0], popt[1]) for i in ex]
    axes3.plot(dx, dy, label='原始数据')
    axes3.plot(ex, ey, 'r--', label='预测数据')
    plt.legend()
    plt.show()

if __name__ == '__main__':
    main()

reason: import numpy,from scipy


text:
#  Copyright (c) 2022 Mira Geoscience Ltd.
#
#  This file is part of geoapps.
#
#  geoapps is distributed under the terms and conditions of the MIT License
#  (see LICENSE file at the root of this source code package).

from copy import deepcopy

import numpy as np
import pytest
from discretize import TreeMesh
from geoh5py.workspace import Workspace

from geoapps.drivers.components import (
    InversionData,
    InversionMesh,
    InversionTopography,
    InversionWindow,
)
from geoapps.io.MagneticVector import MagneticVectorParams
from geoapps.io.MagneticVector.constants import default_ui_json
from geoapps.utils.testing import Geoh5Tester

geoh5 = Workspace("./FlinFlon.geoh5")

def setup_params(tmp):
    geotest = Geoh5Tester(
        geoh5, tmp, "test.geoh5", deepcopy(default_ui_json), MagneticVectorParams
    )
    geotest.set_param("mesh", "{e334f687-df71-4538-ad28-264e420210b8}")
    geotest.set_param("data_object", "{538a7eb1-2218-4bec-98cc-0a759aa0ef4f}")
    geotest.set_param("topography_object", "{ab3c2083-6ea8-4d31-9230-7aad3ec09525}")
    geotest.set_param("tmi_channel_bool", True)
    geotest.set_param("tmi_channel", "{44822654-b6ae-45b0-8886-2d845f80f422}")
    geotest.set_param("topography", "{a603a762-f6cb-4b21-afda-3160e725bf7d}")
    geotest.set_param("out_group", "MVIInversion")
    return geotest.make()

def test_initialize(tmp_path):

    ws, params = setup_params(tmp_path)
    inversion_window = InversionWindow(ws, params)
    inversion_data = InversionData(ws, params, inversion_window.window)
    inversion_topography = InversionTopography(ws, params, inversion_window.window)
    inversion_mesh = InversionMesh(ws, params, inversion_data, inversion_topography)
    assert isinstance(inversion_mesh.mesh, TreeMesh)
    assert inversion_mesh.rotation["angle"] == 20

def test_collect_mesh_params(tmp_path):
    ws, params = setup_params(tmp_path)
    locs = ws.get_entity(params.data_object)[0].centroids
    window = {"center": [np.mean(locs[:, 0]), np.mean(locs[:, 1])], "size": [100, 100]}
    data = InversionData(ws, params, window)
    inversion_window = InversionWindow(ws, params)
    inversion_data = InversionData(ws, params, inversion_window.window)
    inversion_topography = InversionTopography(ws, params, inversion_window.window)
    inversion_mesh = InversionMesh(ws, params, inversion_data, inversion_topography)
    octree_params = inversion_mesh.collect_mesh_params(params)
    print(octree_params._free_param_dict)
    assert "refinement a" in octree_params._free_param_dict.keys()
    assert "refinement b" in octree_params._free_param_dict.keys()
    with pytest.raises(ValueError) as excinfo:
        params.u_cell_size = None
        octree_params = inversion_mesh.collect_mesh_params(params)
    assert "Cannot create OctreeParams" in str(excinfo.value)

def test_mesh_from_params(tmp_path):
    ws, params = setup_params(tmp_path)
    locs = ws.get_entity(params.data_object)[0].centroids
    window = {"center": [np.mean(locs[:, 0]), np.mean(locs[:, 1])], "size": [100, 100]}

    params.mesh_from_params = True
    params.mesh = None
    params.u_cell_size, params.v_cell_size, params.w_cell_size = 19, 25, 25
    inversion_window = InversionWindow(ws, params)
    inversion_data = InversionData(ws, params, window)
    inversion_topography = InversionTopography(ws, params, window)
    inversion_mesh = InversionMesh(ws, params, inversion_data, inversion_topography)
    assert all(inversion_mesh.mesh.h[0] == 19)

max_stars_repo_name: nwilliams-kobold/geoapps

max_issues_repo_name: nwilliams-kobold/geoapps

max_forks_repo_name: nwilliams-kobold/geoapps


text:
# -*- coding: utf-8 -*-
import numpy as np
from numpy import ndarray
from numba import njit

from dewloosh.math.array import atleast3d, repeat

from .space import frames_of_lines
from .rgrid import rgridMT as grid

__cache = True

__all__ = ['mesh1d_uniform']

@njit(nogil=True, parallel=False, fastmath=True, cache=__cache)
def _mesh1d_uniform_(coords: ndarray, topo: ndarray, eshape: ndarray,
                     N: int, frames: ndarray):
    origo = np.zeros(1)
    subcoords_, subtopo_ = grid((1,), N, eshape, origo, 0)
    num_node_sub = len(subcoords_)
    N_new = len(coords) + len(topo) * (N - 1)
    coords_new = np.zeros((N_new, coords.shape[1]), dtype=coords.dtype)
    coords_new[:len(coords)] = coords
    frames_new = dict()
    topo_new = dict()
    cN = len(coords)  # node counter
    for i in range(topo.shape[0]):
        subtopo = subtopo_ + cN - 1
        subtopo[0, 0] = topo[i, 0]
        subtopo[-1, -1] = topo[i, -1]
        p1 = coords[topo[i, 0]]
        p2 = coords[topo[i, -1]]
        for j in range(1, num_node_sub - 1):
            p = p1 * (1-subcoords_[j]) + p2 * subcoords_[j]
            coords_new[cN] = p
            cN += 1
        topo_new[i] = np.copy(subtopo)
        frames_new[i] = repeat(frames[i], N)
    return coords_new, topo_new, frames_new

def mesh1d_uniform(coords: ndarray, topo: ndarray, eshape: ndarray, *args,
                   N :int=2, refZ=None, return_frames=False, **kwargs):
    frames = atleast3d(frames_of_lines(coords, topo, refZ))
    coords, topo, frames = _mesh1d_uniform_(coords, topo, eshape, N, frames)
    if return_frames:
        return coords, topo, frames
    return coords, topo

max_stars_repo_name: dewloosh/dewloosh-geom

max_issues_repo_name: dewloosh/dewloosh-geom

max_forks_repo_name: dewloosh/dewloosh-geom

reason: import numpy,from numpy,from numba


text:
"""Performs face alignment and stores face thumbnails in the output directory."""

# MIT License
#
# Copyright (c) 2016 David Sandberg
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from scipy import misc
import sys
import os
import argparse
import random
import align_dlib
import facenet

def main(args):
    align = align_dlib.AlignDlib(os.path.expanduser(args.dlib_face_predictor))
    landmarkIndices = align_dlib.AlignDlib.OUTER_EYES_AND_NOSE
    output_dir = os.path.expanduser(args.output_dir)
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)
    # Store some git revision info in a text file in the log directory
    src_path,_ = os.path.split(os.path.realpath(__file__))
    facenet.store_revision_info(src_path, output_dir, ' '.join(sys.argv))
    dataset = facenet.get_dataset(args.input_dir)
    random.shuffle(dataset)
    # Scale the image such that the face fills the frame when cropped to crop_size
    scale = float(args.face_size) / args.image_size
    nrof_images_total = 0
    nrof_prealigned_images = 0
    nrof_successfully_aligned = 0
    for cls in dataset:
        output_class_dir = os.path.join(output_dir, cls.name)
        if not os.path.exists(output_class_dir):
            os.makedirs(output_class_dir)
        random.shuffle(cls.image_paths)
        for image_path in cls.image_paths:
            nrof_images_total += 1
            filename = os.path.splitext(os.path.split(image_path)[1])[0]
            output_filename = os.path.join(output_class_dir, filename+'.png')
            if not os.path.exists(output_filename):
                try:
                    img = misc.imread(image_path)
                except (IOError, ValueError, IndexError) as e:
                    errorMessage = '{}: {}'.format(image_path, e)
                    print(errorMessage)
                else:
                    if img.ndim == 2:
                        img = facenet.to_rgb(img)
                    if args.use_center_crop:
                        scaled = misc.imresize(img, args.prealigned_scale, interp='bilinear')
                        sz1 = scaled.shape[1]/2
                        sz2 = args.image_size/2
                        aligned = scaled[(sz1-sz2):(sz1+sz2),(sz1-sz2):(sz1+sz2),:]
                    else:
                        aligned = align.align(args.image_size, img, landmarkIndices=landmarkIndices,
                                              skipMulti=False, scale=scale)
                    if aligned is not None:
                        print(image_path)
                        nrof_successfully_aligned += 1
                        misc.imsave(output_filename, aligned)
                    elif args.prealigned_dir:
                        # Face detection failed. Use center crop from pre-aligned dataset
                        class_name = os.path.split(output_class_dir)[1]
                        image_path_without_ext = os.path.join(os.path.expanduser(args.prealigned_dir),
                                                              class_name, filename)
                        # Find the extension of the image
                        exts = ('jpg', 'png')
                        for ext in exts:
                            temp_path = image_path_without_ext + '.' + ext
                            image_path = ''
                            if os.path.exists(temp_path):
                                image_path = temp_path
                                break
                        try:
                            img = misc.imread(image_path)
                        except (IOError, ValueError, IndexError) as e:
                            errorMessage = '{}: {}'.format(image_path, e)
                            print(errorMessage)
                        else:
                            scaled = misc.imresize(img, args.prealigned_scale, interp='bilinear')
                            sz1 = scaled.shape[1]/2
                            sz2 = args.image_size/2
                            cropped = scaled[(sz1-sz2):(sz1+sz2),(sz1-sz2):(sz1+sz2),:]
                            print(image_path)
                            nrof_prealigned_images += 1
                            misc.imsave(output_filename, cropped)
                    else:
                        print('Unable to align "%s"' % image_path)

    print('Total number of images: %d' % nrof_images_total)
    print('Number of successfully aligned images: %d' % nrof_successfully_aligned)
    print('Number of pre-aligned images: %d' % nrof_prealigned_images)

def parse_arguments(argv):
    parser = argparse.ArgumentParser()

    parser.add_argument('input_dir', type=str, help='Directory with unaligned images.')
    parser.add_argument('output_dir', type=str, help='Directory with aligned face thumbnails.')
    parser.add_argument('--dlib_face_predictor', type=str,
        help='File containing the dlib face predictor.', default='../data/shape_predictor_68_face_landmarks.dat')
    parser.add_argument('--image_size', type=int,
        help='Image size (height, width) in pixels.', default=110)
    parser.add_argument('--face_size', type=int,
        help='Size of the face thumbnail (height, width) in pixels.', default=96)
    parser.add_argument('--use_center_crop',
        help='Use the center crop of the original image after scaling the image using prealigned_scale.', action='store_true')
    parser.add_argument('--prealigned_dir', type=str,
        help='Replace image with a pre-aligned version when face detection fails.', default='')
    parser.add_argument('--prealigned_scale', type=float,
        help='The amount of scaling to apply to prealigned images before taking the center crop.', default=0.87)
    return parser.parse_args(argv)

if __name__ == '__main__':
    main(parse_arguments(sys.argv[1:]))

reason: from scipy


text:
import socket
import threading
import sys
import cv2
import pickle
import numpy as np
import struct
import time
import argparse

# Default camera
height = 480
width = 640
dimen = 3

class ThreadedServer(threading.Thread):
    def __init__(self, host, port):
        self.host = host
        self.port = port
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.sock.bind((self.host, self.port))
        self.clientnum = 0
        self.framearray = []

        print(host, port)

        # Don't lock up main thread.  Is there a better way to do this
        thread = threading.Thread(target=self.listen, args=())
        thread.daemon = True
        thread.start()
        return

    def listen(self):
        self.sock.listen(5)
        while True:
            client, address = self.sock.accept()
            client.settimeout(60)
            # Likely not needed but I feel better preallocating memory
            self.framearray.append(np.zeros((height, width, dimen), np.uint8))
            t = threading.Thread(target = self.listenToClient, args = (client,address))
            t.start()
        return self

    def listenToClient(self, client, address):
        clientindex = self.clientnum
        # Add graceful way to deal with clients leaving later?
        self.clientnum += 1
        # Likely not needed but I feel better preallocating memory, yeah yeah I'm bad at python
        # But seriously, support tracking the loss of a client
        np.append(self.framearray, np.zeros((height, width, dimen), np.uint8))
        print("Client Listening, ", clientindex)
        size = 4096
        data = b''
        payload_size = struct.calcsize(">I")
        while True:
            while len(data) < payload_size:
                data += client.recv(4096)
            packed_msg_size = data[:payload_size]

            data = data[payload_size:]
            msg_size = struct.unpack(">I", packed_msg_size)[0]

            while len(data) < msg_size:
                data += client.recv(4096)
            frame_data = data[:msg_size]
            data = data[msg_size:]

            frame=pickle.loads(frame_data)

            # Possible race condition
            self.framearray[clientindex] = frame

        return

    def getFrames(self, clientindex):
        # Possible race condition
        return self.framearray[clientindex]

    def maxClients(self):
        return self.clientnum

if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("--host", type=str, required=False, default='0.0.0.0')
    ap.add_argument("--port", type=int, required=False, default=8000)
    args = vars(ap.parse_args())

    HOST = args.get("host")
    PORT = args.get("port")

    serv = ThreadedServer(HOST, PORT)

    while True:
        clientnumber = 0
        while clientnumber < serv.maxClients():
#           print(clientnumber, serv.maxClients())
            frame = serv.getFrames(clientnumber)
            # Change later to support one frame per client
            cv2.imshow("Frame", frame)
            clientnumber += 1
#       I clearly don't know how to use cv2.waitKey() correctly
        if cv2.waitKey(1) & 0xFF == ord('q'):
            # Something to stop server threads?
            break

    cv2.destroyAllWindows()

max_stars_repo_name: prak/opencv-client-server

max_issues_repo_name: prak/opencv-client-server

max_forks_repo_name: prak/opencv-client-server


text:
# py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking.
# https://github.com/cheind/py-motmetrics/
#
# MIT License
# Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others.
# See LICENSE file for terms.

"""Tests behavior of MOTAccumulator."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import pandas as pd
import pytest

import motmetrics as mm

def test_events():
    """Tests that expected events are created by MOTAccumulator.update()."""
    acc = mm.MOTAccumulator()

    # All FP
    acc.update([], [1, 2], [], frameid=0)
    # All miss
    acc.update([1, 2], [], [], frameid=1)
    # Match
    acc.update([1, 2], [1, 2], [[1, 0.5], [0.3, 1]], frameid=2)
    # Switch
    acc.update([1, 2], [1, 2], [[0.2, np.nan], [np.nan, 0.1]], frameid=3)
    # Match. Better new match is available but should prefer history
    acc.update([1, 2], [1, 2], [[5, 1], [1, 5]], frameid=4)
    # No data
    acc.update([], [], [], frameid=5)

    expect = mm.MOTAccumulator.new_event_dataframe()
    expect.loc[(0, 0), :] = ['RAW', np.nan, np.nan, np.nan]
    expect.loc[(0, 1), :] = ['RAW', np.nan, 1, np.nan]
    expect.loc[(0, 2), :] = ['RAW', np.nan, 2, np.nan]
    expect.loc[(0, 3), :] = ['FP', np.nan, 1, np.nan]
    expect.loc[(0, 4), :] = ['FP', np.nan, 2, np.nan]

    expect.loc[(1, 0), :] = ['RAW', np.nan, np.nan, np.nan]
    expect.loc[(1, 1), :] = ['RAW', 1, np.nan, np.nan]
    expect.loc[(1, 2), :] = ['RAW', 2, np.nan, np.nan]
    expect.loc[(1, 3), :] = ['MISS', 1, np.nan, np.nan]
    expect.loc[(1, 4), :] = ['MISS', 2, np.nan, np.nan]

    expect.loc[(2, 0), :] = ['RAW', np.nan, np.nan, np.nan]
    expect.loc[(2, 1), :] = ['RAW', 1, 1, 1.0]
    expect.loc[(2, 2), :] = ['RAW', 1, 2, 0.5]
    expect.loc[(2, 3), :] = ['RAW', 2, 1, 0.3]
    expect.loc[(2, 4), :] = ['RAW', 2, 2, 1.0]
    expect.loc[(2, 5), :] = ['MATCH', 1, 2, 0.5]
    expect.loc[(2, 6), :] = ['MATCH', 2, 1, 0.3]

    expect.loc[(3, 0), :] = ['RAW', np.nan, np.nan, np.nan]
    expect.loc[(3, 1), :] = ['RAW', 1, 1, 0.2]
    expect.loc[(3, 2), :] = ['RAW', 2, 2, 0.1]
    expect.loc[(3, 3), :] = ['TRANSFER', 1, 1, 0.2]
    expect.loc[(3, 4), :] = ['SWITCH', 1, 1, 0.2]
    expect.loc[(3, 5), :] = ['TRANSFER', 2, 2, 0.1]
    expect.loc[(3, 6), :] = ['SWITCH', 2, 2, 0.1]

    expect.loc[(4, 0), :] = ['RAW', np.nan, np.nan, np.nan]
    expect.loc[(4, 1), :] = ['RAW', 1, 1, 5.]
    expect.loc[(4, 2), :] = ['RAW', 1, 2, 1.]
    expect.loc[(4, 3), :] = ['RAW', 2, 1, 1.]
    expect.loc[(4, 4), :] = ['RAW', 2, 2, 5.]
    expect.loc[(4, 5), :] = ['MATCH', 1, 1, 5.]
    expect.loc[(4, 6), :] = ['MATCH', 2, 2, 5.]

    expect.loc[(5, 0), :] = ['RAW', np.nan, np.nan, np.nan]

    pd.util.testing.assert_frame_equal(acc.events, expect)

def test_max_switch_time():
    """Tests max_switch_time option."""
    acc = mm.MOTAccumulator(max_switch_time=1)
    acc.update([1, 2], [1, 2], [[1, 0.5], [0.3, 1]], frameid=1)  # 1->a, 2->b
    frameid = acc.update([1, 2], [1, 2], [[0.5, np.nan], [np.nan, 0.5]], frameid=2)  # 1->b, 2->a

    df = acc.events.loc[frameid]
    assert ((df.Type == 'SWITCH') | (df.Type == 'RAW') | (df.Type == 'TRANSFER')).all()

    acc = mm.MOTAccumulator(max_switch_time=1)
    acc.update([1, 2], [1, 2], [[1, 0.5], [0.3, 1]], frameid=1)  # 1->a, 2->b
    frameid = acc.update([1, 2], [1, 2], [[0.5, np.nan], [np.nan, 0.5]], frameid=5)  # Later frame 1->b, 2->a

    df = acc.events.loc[frameid]
    assert ((df.Type == 'MATCH') | (df.Type == 'RAW') | (df.Type == 'TRANSFER')).all()

def test_auto_id():
    """Tests auto_id option."""
    acc = mm.MOTAccumulator(auto_id=True)
    acc.update([1, 2, 3, 4], [], [])
    acc.update([1, 2, 3, 4], [], [])
    assert acc.events.index.levels[0][-1] == 1
    acc.update([1, 2, 3, 4], [], [])
    assert acc.events.index.levels[0][-1] == 2

    with pytest.raises(AssertionError):
        acc.update([1, 2, 3, 4], [], [], frameid=5)

    acc = mm.MOTAccumulator(auto_id=False)
    with pytest.raises(AssertionError):
        acc.update([1, 2, 3, 4], [], [])

def test_merge_dataframes():
    """Tests merge_event_dataframes()."""
    # pylint: disable=too-many-statements
    acc = mm.MOTAccumulator()

    acc.update([], [1, 2], [], frameid=0)
    acc.update([1, 2], [], [], frameid=1)
    acc.update([1, 2], [1, 2], [[1, 0.5], [0.3, 1]], frameid=2)
    acc.update([1, 2], [1, 2], [[0.2, np.nan], [np.nan, 0.1]], frameid=3)

    r, mappings = mm.MOTAccumulator.merge_event_dataframes([acc.events, acc.events], return_mappings=True)

    expect = mm.MOTAccumulator.new_event_dataframe()

    expect.loc[(0, 0), :] = ['RAW', np.nan, np.nan, np.nan]
    expect.loc[(0, 1), :] = ['RAW', np.nan, mappings[0]['hid_map'][1], np.nan]
    expect.loc[(0, 2), :] = ['RAW', np.nan, mappings[0]['hid_map'][2], np.nan]
    expect.loc[(0, 3), :] = ['FP', np.nan, mappings[0]['hid_map'][1], np.nan]
    expect.loc[(0, 4), :] = ['FP', np.nan, mappings[0]['hid_map'][2], np.nan]

    expect.loc[(1, 0), :] = ['RAW', np.nan, np.nan, np.nan]
    expect.loc[(1, 1), :] = ['RAW', mappings[0]['oid_map'][1], np.nan, np.nan]
    expect.loc[(1, 2), :] = ['RAW', mappings[0]['oid_map'][2], np.nan, np.nan]
    expect.loc[(1, 3), :] = ['MISS', mappings[0]['oid_map'][1], np.nan, np.nan]
    expect.loc[(1, 4), :] = ['MISS', mappings[0]['oid_map'][2], np.nan, np.nan]

    expect.loc[(2, 0), :] = ['RAW', np.nan, np.nan, np.nan]
    expect.loc[(2, 1), :] = ['RAW', mappings[0]['oid_map'][1], mappings[0]['hid_map'][1], 1]
    expect.loc[(2, 2), :] = ['RAW', mappings[0]['oid_map'][1], mappings[0]['hid_map'][2], 0.5]
    expect.loc[(2, 3), :] = ['RAW', mappings[0]['oid_map'][2], mappings[0]['hid_map'][1], 0.3]
    expect.loc[(2, 4), :] = ['RAW', mappings[0]['oid_map'][2], mappings[0]['hid_map'][2], 1.0]
    expect.loc[(2, 5), :] = ['MATCH', mappings[0]['oid_map'][1], mappings[0]['hid_map'][2], 0.5]
    expect.loc[(2, 6), :] = ['MATCH', mappings[0]['oid_map'][2], mappings[0]['hid_map'][1], 0.3]

    expect.loc[(3, 0), :] = ['RAW', np.nan, np.nan, np.nan]
    expect.loc[(3, 1), :] = ['RAW', mappings[0]['oid_map'][1], mappings[0]['hid_map'][1], 0.2]
    expect.loc[(3, 2), :] = ['RAW', mappings[0]['oid_map'][2], mappings[0]['hid_map'][2], 0.1]
    expect.loc[(3, 3), :] = ['TRANSFER', mappings[0]['oid_map'][1], mappings[0]['hid_map'][1], 0.2]
    expect.loc[(3, 4), :] = ['SWITCH', mappings[0]['oid_map'][1], mappings[0]['hid_map'][1], 0.2]
    expect.loc[(3, 5), :] = ['TRANSFER', mappings[0]['oid_map'][2], mappings[0]['hid_map'][2], 0.1]
    expect.loc[(3, 6), :] = ['SWITCH', mappings[0]['oid_map'][2], mappings[0]['hid_map'][2], 0.1]

    # Merge duplication
    expect.loc[(4, 0), :] = ['RAW', np.nan, np.nan, np.nan]
    expect.loc[(4, 1), :] = ['RAW', np.nan, mappings[1]['hid_map'][1], np.nan]
    expect.loc[(4, 2), :] = ['RAW', np.nan, mappings[1]['hid_map'][2], np.nan]
    expect.loc[(4, 3), :] = ['FP', np.nan, mappings[1]['hid_map'][1], np.nan]
    expect.loc[(4, 4), :] = ['FP', np.nan, mappings[1]['hid_map'][2], np.nan]

    expect.loc[(5, 0), :] = ['RAW', np.nan, np.nan, np.nan]
    expect.loc[(5, 1), :] = ['RAW', mappings[1]['oid_map'][1], np.nan, np.nan]
    expect.loc[(5, 2), :] = ['RAW', mappings[1]['oid_map'][2], np.nan, np.nan]
    expect.loc[(5, 3), :] = ['MISS', mappings[1]['oid_map'][1], np.nan, np.nan]
    expect.loc[(5, 4), :] = ['MISS', mappings[1]['oid_map'][2], np.nan, np.nan]

    expect.loc[(6, 0), :] = ['RAW', np.nan, np.nan, np.nan]
    expect.loc[(6, 1), :] = ['RAW', mappings[1]['oid_map'][1], mappings[1]['hid_map'][1], 1]
    expect.loc[(6, 2), :] = ['RAW', mappings[1]['oid_map'][1], mappings[1]['hid_map'][2], 0.5]
    expect.loc[(6, 3), :] = ['RAW', mappings[1]['oid_map'][2], mappings[1]['hid_map'][1], 0.3]
    expect.loc[(6, 4), :] = ['RAW', mappings[1]['oid_map'][2], mappings[1]['hid_map'][2], 1.0]
    expect.loc[(6, 5), :] = ['MATCH', mappings[1]['oid_map'][1], mappings[1]['hid_map'][2], 0.5]
    expect.loc[(6, 6), :] = ['MATCH', mappings[1]['oid_map'][2], mappings[1]['hid_map'][1], 0.3]

    expect.loc[(7, 0), :] = ['RAW', np.nan, np.nan, np.nan]
    expect.loc[(7, 1), :] = ['RAW', mappings[1]['oid_map'][1], mappings[1]['hid_map'][1], 0.2]
    expect.loc[(7, 2), :] = ['RAW', mappings[1]['oid_map'][2], mappings[1]['hid_map'][2], 0.1]
    expect.loc[(7, 3), :] = ['TRANSFER', mappings[1]['oid_map'][1], mappings[1]['hid_map'][1], 0.2]
    expect.loc[(7, 4), :] = ['SWITCH', mappings[1]['oid_map'][1], mappings[1]['hid_map'][1], 0.2]
    expect.loc[(7, 5), :] = ['TRANSFER', mappings[1]['oid_map'][2], mappings[1]['hid_map'][2], 0.1]
    expect.loc[(7, 6), :] = ['SWITCH', mappings[1]['oid_map'][2], mappings[1]['hid_map'][2], 0.1]

    pd.util.testing.assert_frame_equal(r, expect)


text:
import os
import zipfile
import shutil
import gdown
import cv2
import glob
import numpy as np
import matplotlib.pyplot as plt

DATA_DIR = 'data/'
DOCS_DIR = 'docs/'
LABELS_FILE = 'All_labels.txt'
DATA_URL = 'https://drive.google.com/uc?id=1w0TorBfTIqbquQVd6k3h_77ypnrvfGwf'
ZFILE = 'SCUT-FBP5500_v2.1.zip'

def extract_zipfile():
    with zipfile.ZipFile(ZFILE) as zip_file:
        for member in zip_file.namelist():
            filename = os.path.basename(member)
            # skip directories, all non-jpgs, except labels
            if filename.endswith(".jpg") or filename == LABELS_FILE:
                # copy file (taken from zipfile's extract)
                source = zip_file.open(member)
                target = open(os.path.join(DATA_DIR, filename), "wb")
                with source, target:
                    shutil.copyfileobj(source, target)

def download_data():
    # Download Dataset
    if os.path.isfile(ZFILE) or os.path.isfile(DATA_DIR+LABELS_FILE):
        print('data already downloaded')
    else:
        print ("data does not exist. downloading it.")
        gdown.download(DATA_URL, ZFILE, quiet=False)
    # Extract ZipFile
    if os.path.isfile(DATA_DIR+LABELS_FILE):
        print("data already extracted.")
    else:
        print("extracting data.")
        if not os.path.exists(DATA_DIR):
            os.mkdir(DATA_DIR)
        extract_zipfile()
        os.remove(ZFILE)

def preprocess_image(image,target_size):
    return cv2.resize(cv2.cvtColor(image, cv2.COLOR_BGR2RGB),target_size) / .255

def create_dataset(target_size):
    X = []
    y = []
    labels_dict = get_labels_dict()
    img_files = glob.glob(DATA_DIR+'*.jpg')
    print(f'reading {len(img_files)} images into dataset')
    for f in img_files:
        img = preprocess_image(cv2.imread(f), target_size)
        X.append(img)
        y.append(labels_dict[os.path.split(f)[-1]])
    return np.array(X), np.array(y)

def get_labels_dict():
    labels_dict = {}
    with open(DATA_DIR + LABELS_FILE) as fp:
        for line in fp:
            img,label = line.split(' ', 1)
            labels_dict[img] = float(label)
    return labels_dict

def plot_metrics(history, model_name, stage):
    f,(ax1) = plt.subplots(1, 1, figsize=(15,7))
    f.suptitle(f'Stage {stage} Model "{model_name}" training Metrics')
    ax1.plot(history.history["loss"], color='darkblue', label="Train")
    ax1.plot(history.history["val_loss"], color='darkred', label="Test")
    ax1.set_title('Loss (Mean Squared Error) over epoch')
    ax1.set_xlabel('Epoch')
    ax1.set_ylabel('Loss (MSE)')
    ax1.legend()
    plt.savefig(f'{DOCS_DIR}metrics_stage_{stage}_{model_name}')
    plt.show()

if __name__ == "__main__":
    download_data()


text:
import networkx as nx
import torch
from torch import nn
import torch.nn.functional as F
import pytorch_lightning as pl
from RandomCalmanTree import RandomCalmanTree
from typing import List

class GPDNN(pl.LightningModule):

    def __init__(self, rct: RandomCalmanTree):
        super().__init__()

        # construct linear ReLU architecture
        modules = []
        for i in range(len(rct.layers) - 1):
            in_features = rct.layers[i]
            out_features = rct.layers[i+1]
            modules += [nn.Linear(in_features, out_features)]
            if i < len(rct.layers) - 2:
                modules += [nn.ReLU()]
        self.net = nn.Sequential(*modules)

        # mask by connectivity
        self.masks = self.masks(rct)
        assert len(self.masks) == (len(self.net) // 2 + 1)
        for j, mask in enumerate(self.masks):
            matrix_idx = j * 2  # we do not mask the bias weights
            masked = self.net[matrix_idx].weight * mask
            with torch.no_grad():
                self.net[matrix_idx].weight = nn.Parameter(masked)

    @staticmethod
    def masks(rct: RandomCalmanTree) -> List[torch.tensor]:
        masks = []
        curr_node = 0
        for i in range(len(rct.layers) - 1):
            # define layer mask shape
            in_features = rct.layers[i]
            out_features = rct.layers[i+1]
            mask = torch.zeros(out_features, in_features)

            # detect connectivity from node to successors
            for node in range(out_features):
                node_idx = node + curr_node + in_features
                children = [x for x in rct.tree.successors(node_idx)]
                for child in children:
                    child_idx = child - curr_node
                    mask[node, child_idx] = 1

            # proceed to next layer
            curr_node += in_features
            masks.append(mask)

        return masks

    def forward(self, x):
        return self.net(x)

    def training_step(self, batch, batch_idx):
        x, y = batch
        y = y.type(torch.FloatTensor)  # necessary for MSE
        x = x.view(x.size(0), -1)
        y_hat = self.net(x)
        y_hat = torch.squeeze(y_hat)
        loss = F.mse_loss(y_hat, y)
        self.log("train_loss", loss, on_epoch=True)
        return loss

    def validation_step(self, val_batch, batch_idx):
        x, y = val_batch
        y = y.type(torch.FloatTensor)  # necessary for MSE
        x = x.view(x.size(0), -1)
        y_hat = self.net(x)
        y_hat = torch.squeeze(y_hat)
        val_loss = F.mse_loss(y_hat, y)
        self.log("val_loss", val_loss)
        return val_loss

    def configure_optimizers(self):
        optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)
        return optimizer

max_stars_repo_name: idocal/statslearning

max_issues_repo_name: idocal/statslearning

max_forks_repo_name: idocal/statslearning

reason: import networkx


text:
# -*- coding: utf-8 -*-
"""
Created on Thu Sep 12 14:53:36 2019

@author: rday
"""

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Mar 24 15:08:00 2019

@author: ryanday
"""

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.figure import Figure
from matplotlib.widgets import RadioButtons,Slider
from matplotlib import gridspec
import matplotlib.cm as cm

class interface:
    '''
    This interactive tool is intended for exploring the dataset associated
    with an ARPES simulation using chinook.
    The user can scan through the datacube in each dimension.
    This uses matplotlib natively rather than alternative gui systems in python like Tkinter,
    which makes it a bit more robust across platforms.
    '''

    def __init__(self,experiment):
        '''
        Initialize the interface. If necessary, the ARPES intensity is calculated
        in advance. The system is initialized as a cut in constant momentum along
        the y-direction.

        *args*:

            - **experiment**: ARPES experiment object, as defined in *chinook.ARPES_lib*

        ***
        '''
        self.experiment = experiment
        self.slice_dict = {'Fixed Kx':1,'Fixed Ky':0,'Fixed Energy':2}
        self.other_dims = {0:(2,1),1:(2,0),2:(1,0)} #relevant axes for rows, columns of each dataset
        try:
            _,self.Imat = self.experiment.spectral()
        except AttributeError:
            print('Matrix Elements have not yet been calculated. Running experiment.datacube() now.')
            self.experiment.datacube()
            print('Matrix element calculation complete.')
            _,self.Imat = self.experiment.spectral()

        self.axes = [np.linspace(*self.experiment.cube[1]),np.linspace(*self.experiment.cube[0]),np.linspace(*self.experiment.cube[2])]
        self.state_coords = np.array([self.axes[0][self.experiment.pks[:,1].astype(int)],self.axes[1][self.experiment.pks[:,2].astype(int)],self.experiment.pks[:,3]]) ## Y, X , Energy coordinates of states
        self.state_coords[2,:] = self.bin_energy() #energy is discretized according to the energy mesh values to allow for state selection from angular maps

        self.aspects = [(self.experiment.cube[self.other_dims[ii][0]][1]-self.experiment.cube[self.other_dims[ii][0]][0])/(self.experiment.cube[self.other_dims[ii][1]][1]-self.experiment.cube[self.other_dims[ii][1]][0]) for ii in range(3)]
        self.extents = [[self.axes[self.other_dims[ii][0]][0],self.axes[self.other_dims[ii][0]][-1],self.axes[self.other_dims[ii][1]][0],self.axes[self.other_dims[ii][1]][-1]] for ii in range(3)]
        self.deltas = [self.axes[0][1]-self.axes[0][0],self.axes[1][1]-self.axes[1][0],self.axes[2][1]-self.axes[2][0]]

        self.run_gui()

    def run_gui(self):

        '''
        Execution of the matplotlib gui. The figure is initialized, along with all widgets and
        chosen datasets. The user has access to both the slice of ARPES data plotted, in addition
        to the orbital projection plotted in upper right panel.

        '''

        self.fig = plt.figure(figsize=(11,11))
        self.fig.canvas.set_window_title('Chinook Orbital Mapper')
        self.ax1 = plt.subplot2grid((10,10),(0,0),rowspan=6,colspan=6,fig=self.fig) #main figure
        self.ax2 = plt.subplot2grid((10,10),(0,7),rowspan=6,colspan=3,fig=self.fig) #subfigure, right
        self.ax3 = plt.subplot2grid((10,10),(7,0),rowspan=3,colspan=6,fig=self.fig) #subfigure, bottom
        self.ax4 = plt.subplot2grid((10,10),(7,7),rowspan=1,colspan=3,fig=self.fig) #slider axis
        self.ax5 = plt.subplot2grid((10,10),(8,7),rowspan=3,colspan=3,fig=self.fig) #radio button axis

        self.dim = 0 #initialize looking at constant Y contour, midway through dataset
        self.indx = int(len(self.axes[0])/2)

        self.img = self.ax1.imshow(self.Imat[self.indx,:,:],cmap=cm.magma,extent=self.extents[self.dim],origin='lower')

        self.ax1.set_aspect(self.aspects[self.dim])

        self.slide_w = Slider(self.ax4,'',self.axes[0][0],self.axes[0][-1],valinit=self.axes[0][self.indx],valstep =self.deltas[self.dim],color='#1F1B33')

        self.radio_ax = RadioButtons(self.ax5,('Fixed Kx','Fixed Ky','Fixed Energy'),active=1,activecolor='#1F1B33')

        ## INITIALIZE THE ORBITAL PLOT ##

        ind1,ind2 = self.other_dims[self.dim]  ##rows, columns
        self.cursor_location =  np.array([self.axes[ind1][int(len(self.axes[ind1])/2)],self.axes[ind2][int(len(self.axes[ind2])/2)]]) ##row, column coordinates of centre, in phys. units
        self.rowdc, = self.ax2.plot(self.Imat[self.indx,:,int(len(self.axes[ind1])/2)],self.axes[ind2]) #all rows distribution curve, here fixed energy, all kx
        self.coldc, = self.ax3.plot(self.axes[ind1],self.Imat[self.indx,int(len(self.axes[ind2])/2),:]) #all column distribution curve, here fixed kx, all energy
        self.cursor, = self.ax1.plot([self.cursor_location[0]],[self.cursor_location[1]],marker='+',markersize=20,c='r')
        self.ax1.set_title('Cursor: E: {:0.04f}, X: {:0.04f}'.format(self.cursor_location[0],self.cursor_location[1]))

        self.plot_peaks = self.state_coords[(ind1,ind2),:][:,np.where(self.state_coords[self.dim,:]==self.axes[self.dim][self.indx])[0]]
        self.bands, = self.ax1.plot(self.plot_peaks[0,:],self.plot_peaks[1,:],marker='o',linestyle='',markersize=2,c='w',alpha=0.3)

        ## SLIDER FUNCTION ##
        def img_slide(val):

            '''
            User requests another slice of the image to be displayed using the
            slider widget. Image is updated with the requested dataset.

            *args*:

                - **val**: float, slider value chosen

            ***
            '''
            self.indx = int((val-self.axes[self.dim][0])/(self.axes[self.dim][1]-self.axes[self.dim][0]))
            self.plot_img()
            self.find_cursor()
            self.fig.canvas.draw()
            plt.show()

        ## SLICE SELECTION RADIO BUTTON FUNCTION ##
        def button_click(label):
            '''

            User requests plotting of intensity map along an alternative axis.
            The intensity map is regenerated with axes scaled to reflect the
            extent of data within the new plane.

            *args*:

                - **label**: string, radio-button label associated with the
                possible slice directions

            ***
            '''
            self.dim = self.slice_dict[label]

            ind1,ind2 = self.other_dims[self.dim]
            self.cursor_location = np.array([self.axes[ind1][int(len(self.axes[ind1])/2)],self.axes[ind2][int(len(self.axes[ind2])/2)]]) #column, row in physical units

            self.indx = int(len(self.axes[self.dim])/2)
            if self.dim==0:

                self.img = self.ax1.imshow(self.Imat[self.indx,:,:],cmap=cm.magma,extent=self.extents[self.dim],origin='lower')
                self.img.set_clim(vmin = self.Imat[self.indx,:,:].min()*0.8,vmax=self.Imat[self.indx,:,:].max()*1.2)

            elif self.dim==1:
                self.img = self.ax1.imshow(self.Imat[:,self.indx,:],cmap=cm.magma,extent=self.extents[self.dim],origin='lower')
                self.img.set_clim(vmin = self.Imat[:,self.indx,:].min()*0.8,vmax=self.Imat[:,self.indx,:].max()*1.2)

            elif self.dim==2:
                self.img = self.ax1.imshow(self.Imat[:,:,self.indx],cmap=cm.magma,extent=self.extents[self.dim],origin='lower')
                self.img.set_clim(vmin = self.Imat[:,:,self.indx].min()*0.8,vmax=self.Imat[:,:,self.indx].max()*1.2)

            self.find_cursor()

            self.plot_peaks = self.state_coords[(ind1,ind2),:][:,np.where(self.state_coords[self.dim,:]==self.axes[self.dim][self.indx])[0]]
            self.bands.set_data(self.plot_peaks[0,:],self.plot_peaks[1,:])

            self.slide_w.valmin = self.axes[self.dim][0]
            self.slide_w.valmax = self.axes[self.dim][-1]
            self.slide_w.ax.set_xlim(self.slide_w.valmin,self.slide_w.valmax)
            self.slide_w.delta = self.deltas[self.dim]
            self.slide_w.val = self.axes[self.dim][self.indx]

            self.ax1.set_aspect(self.aspects[self.dim])

            self.fig.canvas.draw()

        def onclick(event):

            '''
            User selects a point of interest in the intensity map, and the
            corresponding orbital eigenvector is displayed in the upper right
            panel of the window.

            *args*:

                - **event**: button_press_event associated with mouse query
                within the frame of self.ax1. All other mouse events are
                disregarded.

            ***
            '''
            if event.inaxes==self.ax1.axes:
                coords = np.array([event.xdata,event.ydata])
                ind1,ind2 = self.other_dims[self.dim]
                self.cursor_location = np.array([coords[0],coords[1]])
                self.find_cursor()

                self.fig.canvas.draw()
                plt.show()

        cid = self.fig.canvas.mpl_connect('button_press_event',onclick)
        self.slide_w.on_changed(img_slide)
        self.radio_ax.on_clicked(button_click)
        plt.tight_layout()
        plt.show()

    def find_cursor(self):

        '''
        Find nearest point to the desired cursor position, as clicked by the
        user. The cursor event coordinates are compared against the peak positions
        from the tight-binding calculation, and a best choice within the plotted
        slice is selected.

        *args*:

            - **cursor**: tuple of 2 float, indicating the column and row of the
            event, in units of the data-set scaling (e.g. 1/A or eV)

        ***
        '''
        ind1,ind2 = self.other_dims[self.dim] #columns,rows
        cursor = self.cursor_location

        axis1 = self.axes[ind1] #
        axis2 = self.axes[ind2]

        row_index = np.where((abs(axis1-cursor[0])==abs(axis1-cursor[0]).min()))[0][0]
        col_index = np.where((abs(axis2-cursor[1])==abs(axis2-cursor[1]).min()))[0][0]

        if self.dim == 0:
            rowdc = self.Imat[self.indx,:,row_index]
            coldc = self.Imat[self.indx,col_index,:]
            imax = self.Imat[self.indx,:,:].max()
            self.ax1.set_title('Cursor: E: {:0.04f}, Kx: {:0.04f}'.format(cursor[0],cursor[1]))

        if self.dim == 1:
            rowdc = self.Imat[:,self.indx,row_index]
            coldc = self.Imat[col_index,self.indx,:]
            imax = self.Imat[:,self.indx,:].max()
            self.ax1.set_title('Cursor: E: {:0.04f}, Ky: {:0.04f}'.format(cursor[0],cursor[1]))

        if self.dim == 2:
            rowdc = self.Imat[:,row_index,self.indx]
            coldc = self.Imat[col_index,:,self.indx]
            imax = self.Imat[:,:,self.indx].max()
            self.ax1.set_title('Cursor: Kx: {:0.04f}, Ky: {:0.04f}'.format(cursor[0],cursor[1]))

        self.rowdc.set_data(rowdc,axis2)
        self.coldc.set_data(axis1,coldc)
        self.cursor.set_data([cursor[0]],[cursor[1]])
        self.ax2.set_ylim(axis2.min(),axis2.max())
        self.ax2.set_xlim(rowdc.min(),imax)
        self.ax3.set_ylim(coldc.min(),imax)
        self.ax3.set_xlim(axis1.min(),axis1.max())

    def plot_img(self):

        '''
        Update the plotted intensity map slice. The plotted bandstructure states are
        also displayed.

        ***
        '''
        ind1,ind2 = self.other_dims[self.dim]
        self.plot_peaks = self.state_coords[(ind1,ind2),:][:,np.where(self.state_coords[self.dim,:]==self.axes[self.dim][self.indx])[0]]
        self.bands.set_data(self.plot_peaks[0,:],self.plot_peaks[1,:])
        if self.dim ==0:
            self.img.set_data(self.Imat[self.indx,:,:])
            self.img.set_clim(vmin = self.Imat[self.indx,:,:].min()*0.8,vmax=self.Imat[self.indx,:,:].max()*1.2)

        elif self.dim == 1:
            self.img.set_data(self.Imat[:,self.indx,:])
            self.img.set_clim(vmin = self.Imat[:,self.indx,:].min()*0.8,vmax=self.Imat[:,self.indx,:].max()*1.2)

        elif self.dim == 2:
            self.img.set_data(self.Imat[:,:,self.indx])
            self.img.set_clim(vmin = self.Imat[:,:,self.indx].min()*0.8,vmax=self.Imat[:,:,self.indx].max()*1.2)

    def bin_energy(self):

        '''
        Translate the exact energy value for the band peaks into the discrete
        binning of the intensity map, to allow for cursor queries to be processed.

        *return*:

            - **coarse_pts**: numpy array of float, same lengths as *self.state_coords*,
            but sampled over a discrete grid.

        ***
        '''

        fine_pts = self.state_coords[2,:]
        coarse_pts = np.array([self.axes[2][int((fine_pts[ii]-self.axes[2][0])/(self.axes[2][1]-self.axes[2][0]))] if (fine_pts[ii]<self.axes[2].max() and fine_pts[ii]>self.axes[2].min()) else -1 for ii in range(len(fine_pts))])
        return coarse_pts

if __name__ == "__main__":
    plotimg = interface(expmt)

max_stars_repo_name: jminar/chinook

max_issues_repo_name: jminar/chinook

max_forks_repo_name: Arfaouim/chinook


text:
import torch
import torchvision.datasets as dsets
import torchvision.transforms as transforms
from torch.autograd import Variable
import torch.nn.functional as F
import dataset
from torch.utils.data.sampler import SubsetRandomSampler
import numpy as np
from torch.utils.data import DataLoader
from visdom import Visdom
from model import CDSSM

neg_num = 2 # Number of random unclicked documents serving as negative examples for a query. See section 4.
n_epochs=100
batch_size=64  #no16
test_batch_size=64
log_interval=10
vali_split = .2
shuffle_dataset =True

def main():
    # device
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

    # dataset
    # train & validation
    train_data=dataset.MusicSet('datas_train.txt')

    # Creating data indices for training and validation splits:
    train_size = int((1-vali_split)* len(train_data))
    vali_size = len(train_data) - train_size
    train_dataset, vali_dataset = torch.utils.data.random_split(train_data, [train_size, vali_size])

    trainloader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)
    validationloader = DataLoader(dataset=vali_dataset, batch_size=batch_size, shuffle=True)

    # test
    test_dataset=dataset.MusicSet('datas_test.txt')
    testloader = DataLoader(dataset=test_dataset, batch_size=test_batch_size,shuffle=True)

    # model
    #model = CDSSM().to(device)
    model = CDSSM()

    # Loss and optimizer
    criterion = torch.nn.CrossEntropyLoss(reduction='mean')
    optimizer = torch.optim.SGD(model.parameters(), lr=1e-2, momentum=0.4)

    # draw
    vis = Visdom(env='music')

    for epoch in range(n_epochs):
        # train stage
        train_loss = train(device, model, trainloader, criterion, optimizer)
        message = 'Epoch: {}/{}. Train set: Average loss: {:.6f}\n'.format(epoch + 1, n_epochs, train_loss)

        # vali stage
        val_loss = test(device, model, validationloader, criterion, optimizer)
        message += 'Epoch: {}/{}. Validation set: Average loss: {:.6f}'.format(epoch + 1, n_epochs, val_loss)
        print(message)

        #vis.line(X=torch.FloatTensor([epoch+1]), Y=torch.FloatTensor([train_loss]), win='train_loss', update='append' if epoch+1 >0  else None,
        #     opts={'title': 'train loss'})
        vis.line(X=np.column_stack((np.array([epoch+1]),np.array([epoch+1]))), Y=np.column_stack((np.array([train_loss]),np.array([val_loss]))), win='loss', update='append' if epoch+1 >0  else None
            ,opts=dict(title='train&validation loss', legend=['train', 'validation']))

def train(device, model, trainloader, criterion, optimizer):
    model.train()
    total_loss=0
    losses=[]

    for batch_idx, (Q, P, N) in enumerate(trainloader):
        '''
        N_list=[]
        Q, P= Q.to(device), P.to(device)
        for i in range(J):
            tensor=N[i].to(device)
            N_list.append(tensor)
        model=model.double().to(device)
        y_pred = model(Q, P, N_list)
        '''
        model=model.float()
        y_pred = model(Q, P, N)
        #print(y_pred)
        n_correct=0
        for i in range(len(y_pred)):
            record=y_pred[i].resize(1,neg_num+1).float()
            _,idx=torch.max(record,dim=1)
            #print(record)
            #print(idx)
            if(idx==0):
                n_correct+=1
            if i==0:
                a=record
            else:
                b=record
                a=torch.cat((a,b),0)
            #y_pred_list.append(y_pred[i].resize(1,J+1))
        acc=float(n_correct)/len(y_pred)

        ### CrossEntropyLoss expects only the index as a long tensor
        y = np.ndarray(len(y_pred))
        y[:] = 0
        y = Variable(torch.from_numpy(y).long())
        #print(a)
        #print(y)
        loss = criterion(a, y) #Input&target
        #print(loss)

        losses.append(loss.item())
        total_loss+=loss.item()
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if batch_idx % log_interval == 0:
            #average loss
            message = 'Train: [{}/{} ({:.0f}%)]\tLoss: {:.6f}, Acc: {:.6f}'.format(
                batch_idx * len(Q), len(trainloader.dataset),
                100. * batch_idx / len(trainloader), np.mean(losses),acc)
            print(message)

    #print(total_loss)
    total_loss/=(batch_idx+1)
    return total_loss

def test(device, model, testloader, criterion, optimizer):
    with torch.no_grad():
        model.eval()
        val_loss=0
        losses=[]
        for batch_idx, (Q, P, N) in enumerate(testloader):
            model=model.float()
            y_pred = model(Q, P, N)
            n_correct=0
            for i in range(len(y_pred)):
                record=y_pred[i].resize(1,neg_num+1).float()
                _,idx=torch.max(record,dim=1)
                if(idx==0):
                    n_correct+=1
                if i==0:
                    a=record
                else:
                    b=record
                    a=torch.cat((a,b),0)
            acc=float(n_correct)/len(y_pred)

            ### CrossEntropyLoss expects only the index as a long tensor
            y = np.ndarray(len(y_pred))
            y[:] = 0
            y = Variable(torch.from_numpy(y).long())
            loss = criterion(a, y) #Input&target

            losses.append(loss.item())
            val_loss+=loss.item()

            if batch_idx % log_interval == 0:
                #average loss
                message = 'Test: [{}/{} ({:.0f}%)]\tLoss: {:.6f}, Acc: {:.6f}'.format(
                    batch_idx * len(Q), len(testloader.dataset),
                    100. * batch_idx / len(testloader), np.mean(losses),acc)
                print(message)

        val_loss/=(batch_idx+1)
    return val_loss

if __name__=='__main__':
    main()

max_stars_repo_name: sumfish/C-DSSM

max_issues_repo_name: sumfish/MusicSegmentMashUp

max_forks_repo_name: sumfish/MusicSegmentMashUp


text:
import tensorflow as tf
import argparse
import numpy as np
from network import RandWire
from dataset import iter_utils
import os

# main function for training
def main():
    # TODO
    tf.compat.v1.reset_default_graph()
    FOLDERNAME = 'cs231n/project/RandWire_tensorflow'
    code_directory = '/content/drive/My Drive/{}/'.format(FOLDERNAME)

    dataset_dir = 'dataset/cifar10/train.tfrecord/'

    args = argparse.ArgumentParser()
    args.__dict__['class_num'] = 10
    args.__dict__['image_shape'] = [32, 32, 3]
    args.__dict__['stages'] = 4
    args.__dict__['channel_count'] = 78
    args.__dict__['graph_model'] = 'ws'
    args.__dict__['graph_param'] = [32.0, 4.0, 0.75]
    args.__dict__['dropout_rate'] = 0.2
    args.__dict__['learning_rate'] = 1e-1
    args.__dict__['momentum'] = 0.9
    args.__dict__['weight_decay'] = 1e-4
    args.__dict__['train_set_size'] = 50000
    args.__dict__['val_set_size'] = 10000
    args.__dict__['batch_size'] = 100
    args.__dict__['epochs'] = 100
    args.__dict__['checkpoint_dir'] = code_directory+'checkpoint_model{}'.format(2)
    args.__dict__['checkpoint_name'] = 'randwire_cifar10'
    args.__dict__['train_record_dir'] = code_directory+'dataset/cifar10/train.tfrecord'
    args.__dict__['val_record_dir'] = code_directory+'dataset/cifar10/test.tfrecord'

    images = tf.compat.v1.placeholder('float32', shape=[None, *args.image_shape], name='images')  # placeholder for images
    labels = tf.compat.v1.placeholder('float32', shape=[None, args.class_num], name='labels')  # placeholder for labels
    training = tf.compat.v1.placeholder('bool', name='training')  # placeholder for training boolean (is training)
    global_step = tf.compat.v1.get_variable(name='global_step', shape=[], dtype='int64', trainable=False)  # variable for global step
    best_accuracy = tf.compat.v1.get_variable(name='best_accuracy', dtype='float32', trainable=False, initializer=0.0)

    steps_per_epoch = round(args.train_set_size / args.batch_size)
    learning_rate = tf.compat.v1.train.piecewise_constant(global_step, [round(steps_per_epoch * 0.5 * args.epochs),
                                                              round(steps_per_epoch * 0.75 * args.epochs)],
                                                [args.learning_rate, 0.1 * args.learning_rate,
                                                 0.01 * args.learning_rate])
    # output logit from NN
    # output = RandWire.my_small_regime(images, args.stages, args.channel_count, args.class_num, args.dropout_rate,
    #                             args.graph_model, args.graph_param, args.checkpoint_dir + '/' + 'graphs', False, training)
    # output = RandWire.my_regime(images, args.stages, args.channel_count, args.class_num, args.dropout_rate,
    #                             args.graph_model, args.graph_param, args.checkpoint_dir + '/' + 'graphs', False, training)
    output = RandWire.small_regime(images, args.stages, args.channel_count, args.class_num, args.dropout_rate,
                                args.graph_model, args.graph_param, args.checkpoint_dir + '/' + 'graphs', False,
                                training)
    # output = RandWire.regular_regime(images, args.stages, args.channel_count, args.class_num, args.dropout_rate,
    #                             args.graph_model, args.graph_param, args.checkpoint_dir + '/' + 'graphs', training)

    #loss and optimizer
    with tf.compat.v1.variable_scope('losses'):
        # loss = tf.nn.softmax_cross_entropy_with_logits_v2(labels=labels, logits=output)
        loss = tf.compat.v1.losses.softmax_cross_entropy(labels, output, label_smoothing=0.1)
        loss = tf.reduce_mean(loss, name='loss')
        l2_loss = tf.add_n([tf.nn.l2_loss(var) for var in tf.compat.v1.trainable_variables()], name='l2_loss')

    with tf.compat.v1.variable_scope('optimizers'):
        optimizer = tf.compat.v1.train.MomentumOptimizer(learning_rate=learning_rate, momentum=args.momentum, use_nesterov=True)
        #optimizer = tf.train.AdamOptimizer(learning_rate)

        update_ops = tf.compat.v1.get_collection(tf.GraphKeys.UPDATE_OPS)
        train_op = optimizer.minimize(loss + l2_loss * args.weight_decay, global_step=global_step)
        train_op = tf.group([train_op, update_ops], name='train_op')

    #accuracy
    with tf.compat.v1.variable_scope('accuracy'):
        output = tf.nn.softmax(output, name='output')
        prediction = tf.equal(tf.argmax(output, 1), tf.argmax(labels, 1), name='prediction')
        accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32), name='accuracy')

    # summary
    train_loss_summary = tf.compat.v1.summary.scalar("train_loss", loss)
    val_loss_summary = tf.compat.v1.summary.scalar("val_loss", loss)
    train_accuracy_summary = tf.compat.v1.summary.scalar("train_acc", accuracy)
    val_accuracy_summary = tf.compat.v1.summary.scalar("val_acc", accuracy)

    saver = tf.train.Saver()
    best_saver = tf.train.Saver(max_to_keep=1)

    with tf.compat.v1.Session() as sess:
        merged = tf.summary.merge_all()
        writer = tf.summary.FileWriter(args.checkpoint_dir + '/log', sess.graph)

        sess.run(tf.global_variables_initializer())
        augmentations = [lambda image, label: iter_utils.pad_and_crop(image, label, args.image_shape, 4), iter_utils.flip]
        train_iterator = iter_utils.batch_iterator(args.train_record_dir, args.epochs, args.batch_size, augmentations, True)
        train_images_batch, train_labels_batch = train_iterator.get_next()
        val_iterator = iter_utils.batch_iterator(args.val_record_dir, args.epochs, args.batch_size)
        val_images_batch, val_labels_batch = val_iterator.get_next()
        sess.run(train_iterator.initializer)
        if args.val_set_size != 0:
            sess.run(val_iterator.initializer)

        # restoring checkpoint
        try:
            checkpoint_filepath = tf.train.latest_checkpoint(args.checkpoint_dir+'/best')
            saver.restore(sess, checkpoint_filepath)
            print('checkpoint restored. train from checkpoint')
        except:
            print('failed to load checkpoint. train from the beginning')

        #get initial step
        gstep = sess.run(global_step)
        init_epoch = round(gstep / steps_per_epoch)
        init_epoch = int(init_epoch)

        for epoch_ in range(init_epoch + 1, args.epochs + 1):

            # train
            while gstep * args.batch_size < epoch_ * args.train_set_size:
                try:
                    train_images, train_labels = sess.run([train_images_batch, train_labels_batch])
                    train_labels = np.eye(args.class_num)[train_labels]
                    gstep, _, loss_, accuracy_, train_loss_sum, train_acc_sum = sess.run(
                        [global_step, train_op, loss, accuracy, train_loss_summary, train_accuracy_summary],
                        feed_dict={images: train_images, labels: train_labels, training: True})
                    print('[global step: ' + str(gstep) + ' / epoch ' + str(epoch_) + '] -> train accuracy: ',
                          accuracy_, ' loss: ', loss_)
                    writer.add_summary(train_loss_sum, gstep)
                    writer.add_summary(train_acc_sum, gstep)
                except tf.errors.OutOfRangeError:
                    break

            predictions = []

            # validation
            if args.val_set_size != 0:
                while True:
                    try:
                        val_images, val_labels = sess.run([val_images_batch, val_labels_batch])
                        val_labels = np.eye(args.class_num)[val_labels]
                        loss_, accuracy_, prediction_, val_loss_sum, val_acc_sum = sess.run(
                            [loss, accuracy, prediction, val_loss_summary, val_accuracy_summary],
                            feed_dict={images: val_images, labels: val_labels, training: False})
                        predictions.append(prediction_)
                        print('[epoch ' + str(epoch_) + '] -> val accuracy: ', accuracy_, ' loss: ', loss_)
                        writer.add_summary(val_loss_sum, gstep)
                        writer.add_summary(val_acc_sum, gstep)
                    except tf.errors.OutOfRangeError:
                        sess.run(val_iterator.initializer)
                        break

            predictions = np.concatenate(predictions)
            print('best: ', best_accuracy.eval(), '\ncurrent: ', np.mean(predictions))
            if best_accuracy.eval() < np.mean(predictions):
                print('save checkpoint')
                best_accuracy = tf.assign(best_accuracy, np.mean(predictions))
                best_saver.save(sess, args.checkpoint_dir + '/best/' + args.checkpoint_name, global_step=global_step)