text:
import numpy as np
import numpy.ma as ma
import math as mt
import datetime
import sklearn.metrics
import statsmodels.stats.inter_rater
import pandas as pd
from sklearn.metrics import roc_auc_score,confusion_matrix
from sklearn.model_selection import ShuffleSplit
from sklearn import tree,preprocessing
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import export_graphviz
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import BaggingClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import VotingClassifier
from sklearn.decomposition import PCA
from sklearn.preprocessing import binarize
import matplotlib.pyplot as plt
import seaborn as sns
#from sklearn.metrics import cohen_kappa_score
import warnings
warnings.filterwarnings("ignore")
pd.set_option('display.max_columns', 30)

import time
start = time.time()

def kappa(labels, predictions):
    sh=np.shape(predictions)
    if len(sh)>1:
        pred_round   = np.round(np.mean(predictions[:,:],axis=1))  # accepts multiple predictions per image (requires a 2D input)
    elif len(sh)==1:
        pred_round   = np.round(predictions)
    F1_score     = sklearn.metrics.f1_score(labels, pred_round)
    conf_matrix  = sklearn.metrics.confusion_matrix(labels,pred_round)
    kappa        = statsmodels.stats.inter_rater.cohens_kappa(conf_matrix, weights=None, return_results=True, wt=None)
    AUC=roc_auc_score(labels,pred_round)
    fp = conf_matrix.sum(axis=0) - np.diag(conf_matrix)  #false positives
    fn = conf_matrix.sum(axis=1) - np.diag(conf_matrix)   #false negatives
    tp = np.diag(conf_matrix)  #true positives
    tn = np.sum(conf_matrix,axis=(0,1)) - (fp + fn + tp)   #true negatives

    sensi=tp/(tp+fn)
    speci=tn/(tn+fp)
    PPV=tp/(tp+fp)  #positive predictive value
    NPV=tn/(tn+fn)  #negative predictive value
    return(F1_score, kappa,AUC,PPV[1],NPV[1],sensi[1],speci[1])

def con_lab(dframe):
    '''convert the relevant labels to binary'''
    column=dframe['SubjectDiabetesStatus']  #select the column with labels
    labels=[0,3]   #labels displaying healthy and T2D
    s1=(column==labels[0]) #indices of healthy patients
    s2=(column==labels[1])  #indices of T2D
    mask = [any(tup) for tup in zip(s1, s2)]  #add the relavant indices together
    y_true=column.loc[mask]  #mask the column to extract patients
    lut = dict(zip(labels, [0,1]))  #create dictionary
    y_true=y_true.map(lut)  #convert labels to binary
    return y_true,mask

def averaging(feats,y_true,num):
    '''average the probabilities subject-level and evaluate'''
    pred=feats[feats.columns[0:num]]   #only take the means in the DataFrame
    (rows,col)=pred.shape   #get shape
    means=np.zeros((rows))   #reserve memory for the averages
    for i in range(0,rows):   #go over the patients
        p=pred.iloc[i]   #get the patient row
        means[i]=np.mean(p.loc[p!=0])   #only take non-zero values and calculate the average
    (F1,kap,AUC,PPV,NPV,sensi,speci)=kappa(y_true,means)  #evaluate when averaging the predictions
    return F1,kap['kappa'],AUC,PPV,NPV,sensi,speci

def first_only(feats,y_true,num):
    '''Only take the first prediction for each subject and evaluate scores'''
    (F1,kap,AUC,PPV,NPV,sensi,speci)=kappa(y_true,feats[feats.columns[0]])  #evaluate when only taking the first mean
    return F1,kap['kappa'],AUC,PPV,NPV,sensi,speci

def major(feats,y_true,num):
    '''Majority Voting to determine predicted label and evaluate'''
    pred=feats[feats.columns[0:num]]  #only take the means in the DataFrame
    (rows,col)=pred.shape  #get shape
    preds=np.zeros((rows))   #reserve memory for predicted labels
    for i in range(0,rows):  #go over the patients
        p=pred.iloc[i]   #take the patient
        p=p.loc[p!=0]   #only take non-zero means
        rnd=np.round(p)   #round the means to binary
        sm=np.sum(rnd)   #sum binary values
        bol=(sm>=(np.size(rnd)/2))   #If number of positives is in excess or voting is equal than give a True value
        preds[i]=bol.astype(int)  #boolean to an integer, so binary
    (F1,kap,AUC,PPV,NPV,sensi,speci)=kappa(y_true,preds)  #calculate scores
    return F1,kap['kappa'],AUC,PPV,NPV,sensi,speci

def display_results(methods,**kwargs):
    '''display the scores for all methods'''
    if kwargs is not None:  #if any keyword arguments are give continue
        frame=pd.DataFrame(methods,columns=['Methods']) #set up a dataframe
        for metric, value in kwargs.items():   #unpack the dictionary of keyword arguments
            frame[metric.replace('_',' ')]=value #add new column to the frame
        #print(frame)
    return frame

def prepare_data(dframe,predictions,padding,normalize,method,certainty):
    '''preprocessing of the data to serve as features for the classifiers'''
    y,mask=con_lab(dframe)   #get the labels and indexes of correct patients
    predictions=predictions[mask,:]   #only take corresponding predictions
    dframe=dframe.loc[mask,:]     #only take correct indexes
    idx=dframe['SubjectID3'].value_counts()    #all the patient ID's and number of occurences in the list
    #get number of means which will be in list and names of features
    if method=='Eyes':
        mx=2  #number of eyes
        names=['Mean left','Mean right','Std left','Std right']  #feature names
    elif method=='All':
        mx=idx.max()  #maximum number of occurences
        names=['Prob 1','Prob 2','Prob 3','Prob 4','Prob 5','Prob 6','Prob 7','Prob 8','Prob 9','Prob 10','Prob 11','Prob 12','Std 1','Std 2','Std 3','Std 4','Std 5','Std 6','Std 7','Std 8','Std 9','Std 10','Std 11','Std 12']
        names=names[0:mx]+names[12:12+mx]
    elif method=='Center':
        mx=2   #OD or Fovea
        names=['Mean OD','Mean Fovea','Std OD','Std Fovea']
    elif method=='Both':
        mx=4  #number of OD/Fovea with left/right combinations
        names=['Mean OD/Left','Mean Fovea/Left','Mean OD/Right','Mean Fovea/Right','Std OD/Left','Std Fovea/Left','Std OD/Right','Std Fovea/Right']
    features=np.zeros((len(idx),2*mx))    #make space in memory for all means and std's of the predictions
    unq=dframe['SubjectID3'].unique()   #all unqiue ID's
    adt=0   #will be used to take the correct rows of the prediction matrix
    ages=np.zeros((len(idx),1))  #reserve memory for the ages
    y_true=np.zeros((len(idx),1))  #reserve memory for the true labels
    gender=np.zeros((len(idx),1))
    y_cert=np.zeros((len(idx),1))
    for index,pat in enumerate(unq):   #go over all patients
        part=dframe.loc[dframe['SubjectID3']==pat]    #only take the part of the DF of this patient
        ages[index]=int(part['SubjectAge'].iloc[0])   #include the age of this patient
        gender[index]=part['SubjectGender'].iloc[0]
        y_true[index]=int(y.loc[(dframe['SubjectID3']==pat)].iloc[0]) #set the correct label of this patient
        if certainty:
            preds=predictions[adt:adt+idx.loc[pat],:]   #all predictions of this patient
            top=np.sum(preds>0.8,axis=1)
            bot=np.sum(preds<0.2,axis=1)
            if np.sum(top>2)>0 and np.sum(bot>2)==0:
                y_cert[index]=1
            elif np.sum(bot>2)>0 and np.sum(top>2)==0:
                y_cert[index]=0
            else:
                y_cert[index]=2
        if method=='Eyes':  #mean and standard deviation of eyes seperately to serve as features
            preds=predictions[adt:adt+idx.loc[pat],:]   #all predictions of this patient
            il=(part['Eye']=='Left')   #indexes of left eye
            ir=(part['Eye']=='Right')   #indexes of right eye
            left=preds[il,:]   #predictions left eye
            right=preds[ir,:]   #predictions right eye
            means=[np.mean(np.mean(left,axis=1)),np.mean(np.mean(right,axis=1))]  #mean of left and right eye
            stands=[np.std(np.mean(left,axis=1)),np.std(np.mean(right,axis=1))]  #std of left and right eye
            if padding=='Average':
                means = [means[abs(i-1)] if mt.isnan(x) else x for i,x in enumerate(means)]   #replace Nan values with mean value other eye
                stands = [stands[abs(i-1)] if mt.isnan(x) else x for i,x in enumerate(stands)]  #replace Nan values with std value of other eye
            elif padding=='Zero':
                means = [0 if mt.isnan(x) else x for i,x in enumerate(means)]   #replace Nan values with 0
                stands = [0 if mt.isnan(x) else x for i,x in enumerate(stands)]  #replace Nan values with 0

        elif method=='Both':
            preds=predictions[adt:adt+idx.loc[pat],:]   #all predictions of this patient
            il=(part['Eye']=='Left')   #indexes of left eye
            ir=(part['Eye']=='Right')   #indexes of right eye
            iod=(part['ODorFoveaCentered']=='OD')   #indexes of OD center
            ifo=(part['ODorFoveaCentered']=='Fovea')   #indexes of Fovea center
            OD_L=(il & iod) #OD centering - left eye
            FOV_L=(il % ifo)   #Fovea Centering - Left eye
            OD_R=(ir % iod)   #OD centering - Right eye
            FOV_R=(ir & ifo)   #Fovea Centering - Right Eye
            means=[np.mean(np.mean(preds[OD_L,:],axis=1)),np.mean(np.mean(preds[FOV_L,:],axis=1)),np.mean(np.mean(preds[OD_R,:],axis=1)),np.mean(np.mean(preds[FOV_R,:],axis=1))]  #mean features
            stands=[np.std(np.mean(preds[OD_L,:],axis=1)),np.std(np.mean(preds[FOV_L,:],axis=1)),np.std(np.mean(preds[OD_R,:],axis=1)),np.std(np.mean(preds[FOV_R,:],axis=1))]   #std features
            if padding=='Zero':
                means=np.where(np.isnan(means),0,means)   #replace nan values with a 0
                stands=np.where(np.isnan(stands),0,stands)  #replace nan values with a 0

            elif padding=='Average':
                nans=np.isnan(means)   #places which contain Nan values
                means=np.where(np.isnan(means),np.mean(ma.masked_array(means,mask=nans)),means)   #replace Nan values with the mean of the means that are present
                stands=np.where(np.isnan(stands),np.mean(ma.masked_array(stands,mask=nans)),stands)  #replace Nan values with the mean of the std's that are present

        elif method=='Center':  #mean and standard deviation of centers seperately to serve as features
            preds=predictions[adt:adt+idx.loc[pat],:]   #all predictions of this patient
            iod=(part['ODorFoveaCentered']=='OD')   #indexes of OD center
            ifo=(part['ODorFoveaCentered']=='Fovea')   #indexes of Fovea center
            od=preds[iod,:]   #predictions OD center
            fov=preds[ifo,:]   #predictions Fovea center
            means=[np.mean(np.mean(od,axis=1)),np.mean(np.mean(fov,axis=1))]  #mean of OD and Fovea center
            stands=[np.std(np.mean(od,axis=1)),np.std(np.mean(fov,axis=1))]  #std of OD and Fovea center
            if padding=='Average':
                means = [means[abs(i-1)] if mt.isnan(x) else x for i,x in enumerate(means)]   #replace Nan values with mean value other eye
                stands = [stands[abs(i-1)] if mt.isnan(x) else x for i,x in enumerate(stands)]  #replace Nan values with std value of other eye
            elif padding=='Zero':
                means = [0 if mt.isnan(x) else x for i,x in enumerate(means)]   #replace Nan values with 0
                stands = [0 if mt.isnan(x) else x for i,x in enumerate(stands)]  #replace Nan values with 0

        elif method=='All':
            means=np.mean(predictions[adt:adt+idx.loc[pat],:],axis=1)  #take all of the mean predictions of this patient
            stands=np.std(predictions[adt:adt+idx.loc[pat],:],axis=1)  #take all of the std's of the patient predictions
        adt=adt+idx.loc[pat]  #update the variabel
        features[index,0:len(means)]=np.reshape(means,(len(means)))   #set the means of this patient in the matrix
        features[index,mx:mx+len(stands)]=np.reshape(stands,(len(stands))) #set the standard deviations of this patient in the second half of this patient
        if padding=='Average' and method=='All' and len(means)<mx:   #replace the zero values with the average mean/std values of that patient
            features[index,len(means):mx]=np.mean(means)
            features[index,mx+len(stands):]=np.mean(stands)

    features=pd.DataFrame(features,columns=names)  #features to a DataFrame
    ages=pd.DataFrame(ages,columns=['Age']) #ages array to DataFrame
    gender=pd.DataFrame(gender,columns=['Gender'])
    extra=pd.merge(ages,gender,right_index=True,left_index=True) #merge Frames
    feats=pd.merge(features,extra,right_index=True,left_index=True)   #merge the Frames
    features_normal=pd.merge(features,extra,right_index=True,left_index=True)  #is not affected by normalization
    y_true=pd.DataFrame(y_true,columns=['Label']) #labels array to DataFrame

    if normalize:   #normalization of the feature columns
        scaler = preprocessing.StandardScaler()   #initialize scaler
        feats[feats.columns]=scaler.fit_transform(feats.values)   #normalize the feature values

    return feats,y_true,mx,features_normal,y_cert

def cross_sc(clf,feat,y_true,y_cert,certainty):
    '''Use cross validation to determine mean and standard deviation of the statistics for the given classifier'''
    n=100   #number of folds to use
    ss = ShuffleSplit(n_splits=n,test_size=0.2)   #make a shuffle split generator
    #make lists to store statistics
    F1s=[]
    AUCs=[]
    kaps=[]
    PPVs=[]
    NPVs=[]
    sensis=[]
    specis=[]
    for train_index, test_index in ss.split(feat):   #loop over each fold
        clf.fit(feat.iloc[train_index],y_true.iloc[train_index])   #fit the classifier to the train data
        pred=clf.predict(feat.iloc[test_index])    #make predictions on the test data
        if certainty:
            pred[np.reshape(y_cert[test_index]==1,len(pred))]=1
            pred[np.reshape(y_cert[test_index]==0,len(pred))]=0

        (F1,kap,AUC,PPV,NPV,sensi,speci)=kappa(y_true.iloc[test_index],pred)   #compute statistics
        assert clf.classes_[1]==1   #ensure that the correct predictions are chosen
        #append the statistics
        F1s.append(F1)
        AUCs.append(AUC)
        kaps.append(kap['kappa'])
        PPVs.append(PPV)
        NPVs.append(NPV)
        sensis.append(sensi)
        specis.append(speci)
    return np.mean(kaps),np.std(kaps),np.mean(F1s),np.std(F1s),np.mean(AUCs),np.std(AUCs),np.mean(PPVs),np.std(PPVs),np.mean(NPVs),np.std(NPVs),np.mean(sensis),np.std(sensis),np.mean(specis),np.std(specis)  #calculate the mean and standard deviation of the statistics

def classifiers(dframe,predictions,algos,padding,normalize,method,certainty,basic,validation):
    '''Set-up all of the options for classification and calculate the scores'''
    features,y_true,num,features_normal,y_cert=prepare_data(dframe,predictions,padding,normalize,method,certainty)   #acquire the features and labels
    if basic:
        ending=num*2  #if not all features are wished to be taken into account
        features=features[features.columns[0:ending]]
        features_normal=features_normal[features_normal.columns[0:ending]]

    if validation:
        pred_val=np.load(path.validation_pred)  #load numpy array
        frame_val=pd.read_excel(path.validation_excel)  #load patient data

        X_val,y_val,_,features_normal,y_val_cert=prepare_data(frame_val,pred_val,padding,normalize,method,certainty)   #acquire the features and labels
        if basic:
            ending=num*2  #if not all features are wished to be taken into account
            X_val=X_val[X_val.columns[0:ending]]
            features_normal=features_normal[features_normal.columns[0:ending]]
    #lists for the scores
    AUCs=[]
    AUCs_std=[];
    F1s=[]
    F1_std=[]
    kaps=[]
    kap_std=[]
    PPVs=[]
    PPV_std=[]
    NPVs=[]
    NPV_std=[]
    sensis=[]
    sensi_std=[]
    specis=[]
    speci_std=[]
    classif=[]
    for clas in algos:   #go over all specified options
        disable=False
        if clas=='Decision Tree':
            clf = tree.DecisionTreeClassifier(criterion='entropy',min_samples_leaf=3,max_depth=3,min_samples_split=10,max_features=None)   #initialize the classifier
        elif clas=='Logistic Regression':
            clf = LogisticRegression(penalty='l2',solver='liblinear',dual=False)    #initialization of the classifier
        elif clas=='Naive Bayes':
            clf = GaussianNB()
        elif clas=='SVM':
            clf = SVC(C=5,gamma='auto')
        #elif clas=='Random Forest':
            #clf = RandomForestClassifier(n_estimators=100,criterion='gini')
        elif clas=='Nearest Neighbour':
            clf = KNeighborsClassifier(n_neighbors=25,weights='distance',algorithm='auto')
        elif clas =='First Only' and method=='All' and validation==False:
            F1,kap,auc,PPV,NPV,sensi,speci=first_only(features_normal,y_true,num)
        elif clas =='First Only' and method=='All' and validation==True:
            F1,kap,auc,PPV,NPV,sensi,speci=first_only(features_normal,y_val,num)
        elif clas =='Averaging' and validation==False:
            F1,kap,auc,PPV,NPV,sensi,speci=averaging(features_normal,y_true,num)
        elif clas =='Averaging' and validation==True:
            F1,kap,auc,PPV,NPV,sensi,speci=averaging(features_normal,y_val,num)
        elif clas =='Majority Vote' and validation==False:
            F1,kap,auc,PPV,NPV,sensi,speci=major(features_normal,y_true,num)
        elif clas =='Majority Vote' and validation==True:
            F1,kap,auc,PPV,NPV,sensi,speci=major(features_normal,y_val,num)
        elif clas=='AdaBoost':
            clf=AdaBoostClassifier(base_estimator=LogisticRegression(penalty='l2',solver='liblinear'),n_estimators=50,learning_rate=0.1)
        #elif clas=='Bagging':
            #clf=BaggingClassifier(KNeighborsClassifier(n_neighbors=20,algorithm='auto'),max_samples=0.5, max_features=0.5)
        #elif clas=='Extra Trees':
            #clf=ExtraTreesClassifier(n_estimators=10)
        #elif clas=='Gradient Boosting':
            #clf = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=1)
        elif clas=='Voting classifier':
            clf1=LogisticRegression(penalty='l2',solver='liblinear',dual=False)
            clf2=SVC(C=5,gamma='auto',probability=True)
            clf3=tree.DecisionTreeClassifier(criterion='entropy',min_samples_leaf=3,max_depth=3,min_samples_split=10,max_features=None)
            clf = VotingClassifier(estimators=[('lr', clf1), ('svc', clf2), ('dt', clf3)], voting='soft')
        else:
            disable=True   #to avoid the remaining processes if no correct classifiers are given

        if clas=='First Only' or clas=='Averaging' or clas =='Majority Vote':  #these do not have a standard deviation
            ks=0
            fs=0
            ast=0
            Ps=0
            Ns=0
            sss=0
            sfs=0
        elif validation==False and disable==False:
            kap,ks,F1,fs,auc,ast,PPV,Ps,NPV,Ns,sensi,sss,speci,sfs=cross_sc(clf,features,y_true,y_cert,certainty)    #compute cross-validation score
        elif disable==False and validation==True:
            kap,ks,F1,fs,auc,ast,PPV,Ps,NPV,Ns,sensi,sss,speci,sfs=validation_score(clf,features,y_true,X_val,y_val,y_val_cert,certainty,method,padding)
        #add the scores to the list
        if disable==False:
            AUCs.append(auc)
            AUCs_std.append(ast)
            F1s.append(F1)
            F1_std.append(fs)
            kaps.append(kap)
            kap_std.append(ks)
            PPVs.append(PPV)
            PPV_std.append(Ps)
            NPVs.append(NPV)
            NPV_std.append(Ns)
            sensis.append(sensi)
            sensi_std.append(sss)
            specis.append(speci)
            speci_std.append(sfs)
            classif.append(clas)
    return AUCs, AUCs_std, F1s, F1_std, kaps, kap_std,PPVs,PPV_std,NPVs,NPV_std,sensis,sensi_std,specis,speci_std, classif

def perform_pca(dframe,predictions,padding,normalize,method,certainty):
    '''Principal Component analysis'''
    pca=PCA()  #initialize PCA
    feat,y,num,fn,y_cert=prepare_data(dframe,predictions,padding,normalize,method,certainty)  #get features
    pca.fit(feat)   #fit data to PCA
    #print outcome
    ev=pca.explained_variance_
    print(ev)
    print(feat.columns)

    #calculate and print the ratio of eigenvalues
    ratios=np.zeros((len(ev),1))
    for i in range(0,len(ev)):
        ratios[i]=np.sum(ev[0:i+1])/np.sum(ev)
    print(ratios)

    return

def make_bar(frame,frame_basic,save,padding,method):
    '''make the bar plots for each metric with an option to save them'''
    names=frame.columns[1:]  #select the column names wich will have the scores and std's
    plt.rcParams.update(plt.rcParamsDefault)   #go back to default plt settings
    for name in names:  #go over columns
        if 'score' in name:  #if this is the name of a score continue
            metric=name.split(' ')[0]  #define which metric this is
            scores=frame[metric+' score']  #get the scores of this metric
            scoresb=frame_basic[metric+' score']
            stds=frame[metric+' std']   #get the std's of this metric
            stds_bas=frame_basic[metric+' std']

            #create the bar plot
            ind = np.arange(len(scores))  # the x locations for the groups
            width = 0.35  # the width of the bars

            f, ax = plt.subplots()
            rectn=ax.bar(ind - width/2,scores,width,yerr=stds,capsize=10,label='Image and Patient Info')
            rect_bas=ax.bar(ind + width/2,scoresb,width,yerr=stds_bas,capsize=10,label='Only Image Info')
            ax.set_ylabel(name)
            ax.set_title(name+' with settings padding: '+padding+', method: '+method)
            ax.set_xticks(ind)
            ax.set_xticklabels(frame['Methods'],rotation='vertical')
            mx=np.max(scores+stds)
            ax.set_ylim([0,mx+0.3])
            ax.legend(loc='upper left')

            autolabel(rectn,ax,stds,"center")
            autolabel(rect_bas,ax,stds_bas,"center")
            f.tight_layout()
            plt.show()

            if save:      #save the figure if wanted with a unique name to prevent overwriting files
                x=datetime.datetime.now()
                extra='_'.join([metric,str(x.year),str(x.month),str(x.day),str(x.hour),str(x.minute),str(x.second)])
                f.savefig('Bar_plot_'+extra+'.png',bbox_inches='tight')
    return

def autolabel(rects,ax,std, xpos='center'):
    """ Attach a text label above each bar, displaying its height."""

    ha = {'center': 'center', 'right': 'left', 'left': 'right'}
    offset = {'center': 0, 'right': 1, 'left': -1}

    for index,rect in enumerate(rects):
        height = rect.get_height()
        ax.annotate('{:0.2f}'.format(height),
                    xy=(rect.get_x() + rect.get_width() / 2, height+std[index]),
                    xytext=(offset[xpos], 3),  # use 3 points offset
                    textcoords="offset points",  # in both directions
                    ha=ha[xpos], va='bottom',rotation=90)
    return

def data_visual(dframe,predictions,padding,method):
    '''Visualize the distribution of the data'''
    features,y_true,num,_,_=prepare_data(dframe,predictions,padding,False,method,False)   #acquire the features and labels
    #make a pairplot
    sns.set_style("whitegrid")
    y_true[y_true==0]='Healthy'
    y_true[y_true==1]='Diabetes'
    feat=pd.merge(features,y_true,right_index=True,left_index=True)
    sns.pairplot(feat,hue='Label',size=3)
    plt.show()

    #make a heatmap of the correlations
    corr=features.corr()
    ax = sns.heatmap(corr, vmin=-1, vmax=1, center=0, cmap=sns.diverging_palette(20, 220, n=200), square=True)
    ax.set_xticklabels(ax.get_xticklabels(),rotation=45,horizontalalignment='right');
    plt.show()

    #show the distribution plost of the means
    if method!='All':
        for i in range(0,num):
            mean=features[features.columns[i]]
            sns.distplot(mean[np.reshape(y_true.values=='Healthy',len(y_true))], hist = False, kde = True,kde_kws = {'shade': True, 'linewidth': 3}, label = 'Healthy')
            sns.distplot(mean[np.reshape(y_true.values=='Diabetes',len(y_true))], hist = False, kde = True,kde_kws = {'shade': True, 'linewidth': 3}, label = 'Diabetes')
            plt.show()
    return

def validation_score(clf,X_train,y_train,X_val,y_val,y_cert_val,certainty,method,padding):
    tree_visual=False   #whether to visualize the DT
    n=10   #number of models made
    ss = ShuffleSplit(n_splits=n,test_size=0.2)   #make a shuffle split generator
    #clear memory for scores
    F1s=np.zeros((n))
    AUCs=np.zeros((n))
    kaps=np.zeros((n))
    PPVs=np.zeros((n))
    NPVs=np.zeros((n))
    sensis=np.zeros((n))
    specis=np.zeros((n))
    for i,(train_index, test) in enumerate(ss.split(X_train)):  #go over each fod
        clf.fit(X_train.iloc[train_index],y_train.iloc[train_index])   #train on 80% of the training dataset
        pred=clf.predict(X_val)  #predict on the entire validation dataset

        if certainty:   #correct mistakes of the labels which were certain
            pred[np.reshape(y_cert_val==1,len(pred))]=1
            pred[np.reshape(y_cert_val==0,len(pred))]=0
        assert clf.classes_[1]==1   #ensure that the correct predictions are chosen
        #calculate the scores and add to the corresponding lists
        (F1,kap,AUC,PPV,NPV,sensi,speci)=kappa(y_val,pred)
        F1s[i]=F1
        AUCs[i]=AUC
        kaps[i]=kap['kappa']
        PPVs[i]=PPV
        NPVs[i]=NPV
        sensis[i]=sensi
        specis[i]=speci

    #if the classifier is a Decision Tree and visualization is enabled, save the DT
    string=method+'_'+padding+'.dot' #filename of DT
    if type(clf)==sklearn.tree.tree.DecisionTreeClassifier and tree_visual==True:
        clf.fit(X_train,y_train)   #fit on the entire training dataset
        export_graphviz(clf, out_file=string, feature_names = X_val.columns,class_names = ['Healthy','T2D'],rounded = True, proportion = False, precision = 2, filled = True)

    return np.mean(kaps),np.std(kaps),np.mean(F1s),np.std(F1s),np.mean(AUCs),np.std(AUCs),np.mean(PPVs),np.std(PPVs),np.mean(NPVs),np.std(NPVs),np.mean(sensis),np.std(sensis),np.mean(specis),np.std(specis)

class path():
    '''contains the paths the the data'''
    training_pred='train_prediction_30.npy'
    training_excel='labels_trainingset.xlsx'
    validation_pred='val_prediction_30.npy'
    validation_excel='labels_validationset.xlsx'

save_bar=True   #option to save bar plots    options: True or False
normalize=True   #option to normalize features    options: True or False
method='All'  #method to use for data pre-processing    options: 'All' or 'Eyes' or 'Center' or 'Both'
padding='Zero'  #Padding method   options: 'Zero' or 'Average'
certainty=True    #options: True or False
visual=False   #visualze data distributions
Only_kappa=True   #Only show Kappa Score if set to True
validation=True   #Use the validation dataset or not

predictions=np.load(path.training_pred)  #load numpy array
dframe=pd.read_excel(path.training_excel)  #load patient data
if visual:
    data_visual(dframe,predictions,padding,method)
classi=['First Only','Averaging','Majority Vote','Decision Tree','Logistic Regression','Naive Bayes','SVM','Nearest Neighbour','AdaBoost','Voting classifier']
#Classification process with patient information
AUC,Ast,F1,F1st,Kap,Kst,PPV,Ps,NPV,Ns,sensi,Sstd,speci,spst,classif=classifiers(dframe,predictions,classi,padding,normalize,method,certainty,False,validation)
if Only_kappa:
    results_fr=display_results(classif,Kappa_score=Kap,Kappa_std=Kst)
else:
    results_fr=display_results(classif,F1_score=F1,F1_std=F1st,Kappa_score=Kap,Kappa_std=Kst,AUC_score=AUC,AUC_std=Ast,PPV_score=PPV,PPV_std=Ps,NPV_score=NPV,NPV_std=Ns,sensitivity_score=sensi,sensitivity_std=Sstd,specificity_score=speci,specificity_std=spst)

#Classification process without patient information
AUC,Ast,F1,F1st,Kap,Kst,PPV,Ps,NPV,Ns,sensi,Sstd,speci,spst,classif=classifiers(dframe,predictions,classi,padding,normalize,method,certainty,True,validation)
if Only_kappa:
    results_basic=display_results(classif,Kappa_score=Kap,Kappa_std=Kst)
else:
    results_basic=display_results(classif,F1_score=F1,F1_std=F1st,Kappa_score=Kap,Kappa_std=Kst,AUC_score=AUC,AUC_std=Ast,PPV_score=PPV,PPV_std=Ps,NPV_score=NPV,NPV_std=Ns,sensitivity_score=sensi,sensitivity_std=Sstd,specificity_score=speci,specificity_std=spst)
if len(results_fr)>0:  #if any selected classifiers have been used, display results
    make_bar(results_fr,results_basic,save_bar,padding,method)
else:
    print('Please select a classifier programmed in this code')

end = time.time()
print('time: '+ '{:0.1f}'.format(end - start)+' sec')

reason: import numpy,import statsmodels


text:
import numpy as np
import functools

'''
test Markov Chain
'''

custom_next_P = np.array([[.8,.1, .1], [.5, .1, .4], [.5, .3, .2]])
ini_status = np.array([.2,.4, .4])
custom_next_4 = functools.reduce(np.dot, [custom_next_P, custom_next_P, custom_next_P, custom_next_P])
p4 = np.dot(ini_status, custom_next_4)
print (p4)

ext: py

reason: import numpy


text:
import os
import numpy as np
from sklearn.preprocessing import LabelEncoder
import matplotlib.pyplot as plt
import cv2
import torch
import torchvision
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from sklearn.model_selection import train_test_split
from torch.optim.lr_scheduler import StepLR
from torch.utils.data import DataLoader, Dataset
from torchvision import datasets, transforms
import timm

def set_parameter_required_grad(model, required_grad):
    for param in model.parameters():
        param.required_grad = required_grad

def evaluate(output, label):
    c_True_Positive = torch.sum((output == 1) & ((label == 1)))
    c_False_Positive = torch.sum((output == 1) & ((label == 0)))
    c_True_Negative = torch.sum((output == 0) & ((label == 0)))
    c_False_Negative = torch.sum((output == 0) & ((label == 1)))
    c_Positive = torch.sum((label == 1))
    c_Negative = torch.sum((label == 0))
    return c_True_Positive, c_False_Positive, c_True_Negative, c_False_Negative, c_Positive, c_Negative

def eval_numpy(output, label):
    c_True_Positive = np.sum((output == 1) & ((label == 1)))
    c_False_Positive = np.sum((output == 1) & ((label == 0)))
    c_True_Negative = np.sum((output == 0) & ((label == 0)))
    c_False_Negative = np.sum((output == 0) & ((label == 1)))
    c_Positive = np.sum((label == 1))
    c_Negative = np.sum((label == 0))
    return c_True_Positive, c_False_Positive, c_True_Negative, c_False_Negative, c_Positive, c_Negative

def get_class_idx():
    class_to_idx= {'negative': 0, 'positive': 1}
    print(class_to_idx)
    idx_to_class={idx:cls for cls,idx in class_to_idx.items()}
    print(idx_to_class)
    return class_to_idx, idx_to_class

    # F1 = (True_Positive * False_Positive)
    # Recall =
    # Precision =
    # acc = (output.argmax(dim=1) == label).float().mean()
# def create_criterion(class):

max_stars_repo_name: Haiduongcable/Facial-Stroke-recognition

max_issues_repo_name: Haiduongcable/Facial-Stroke-recognition

max_forks_repo_name: Haiduongcable/Facial-Stroke-recognition


text:
"""Structural Agnostic Model.

Author: Diviyan Kalainathan, Olivier Goudet
Date: 09/3/2018
"""
import os
import numpy as np
import torch as th
import pandas as pd
from torch.utils.data import DataLoader
from tqdm import tqdm
from sklearn.preprocessing import scale
from .utils.linear3d import Linear3D
from .utils.graph import MatrixSampler, notears_constr
from .utils.batchnorm import ChannelBatchNorm1d
from .utils.parlib import parallel_run

class SAM_generators(th.nn.Module):
    """Ensemble of all the generators."""

    def __init__(self, data_shape, nh, skeleton=None, cat_sizes=None, linear=False):
        """Init the model."""
        super(SAM_generators, self).__init__()
        layers = []
        # Building skeleton
        self.sizes = cat_sizes
        self.linear = linear

        nb_vars = data_shape[1]
        self.nb_vars = nb_vars
        if skeleton is None:
            skeleton = 1 - th.eye(nb_vars + 1, nb_vars)  # 1 row for noise
        else:
            skeleton = th.cat([th.Tensor(skeleton), th.ones(1, nb_vars)], 1)
        if linear:
            self.input_layer = Linear3D((nb_vars, nb_vars + 1, 1))
        else:
            self.input_layer = Linear3D((nb_vars, nb_vars + 1, nh))
            layers.append(ChannelBatchNorm1d(nb_vars, nh))
            layers.append(th.nn.Tanh())
            self.output_layer = Linear3D((nb_vars, nh, 1))

        self.layers = th.nn.Sequential(*layers)
        self.register_buffer('skeleton', skeleton)

    def forward(self, data, noise, adj_matrix, drawn_neurons=None):
        """Forward through all the generators."""

        if self.linear:
            output = self.input_layer(data, noise, adj_matrix * self.skeleton)
        else:
            output = self.output_layer(self.layers(self.input_layer(data,
                                                                    noise,
                                                                    adj_matrix * self.skeleton)),
                                       drawn_neurons)

        return output.squeeze(2)

    def reset_parameters(self):
        if not self.linear:
            self.output_layer.reset_parameters()
            for layer in self.layers:
                if hasattr(layer, 'reset_parameters'):
                    layer.reset_parameters()
        self.input_layer.reset_parameters()

class SAM_discriminator(th.nn.Module):
    """SAM discriminator."""

    def __init__(self, nfeatures, dnh, hlayers=2, mask=None):
        super(SAM_discriminator, self).__init__()
        self.nfeatures = nfeatures
        layers = []
        layers.append(th.nn.Linear(nfeatures, dnh))
        layers.append(th.nn.BatchNorm1d(dnh))
        layers.append(th.nn.LeakyReLU(.2))
        for i in range(hlayers-1):
            layers.append(th.nn.Linear(dnh, dnh))
            layers.append(th.nn.BatchNorm1d(dnh))
            layers.append(th.nn.LeakyReLU(.2))

        layers.append(th.nn.Linear(dnh, 1))
        self.layers = th.nn.Sequential(*layers)

        if mask is None:
            mask = th.eye(nfeatures, nfeatures)
        self.register_buffer("mask", mask.unsqueeze(0))

    def forward(self, input, obs_data=None):
        if obs_data is not None:
            return [self.layers(i) for i in th.unbind(obs_data.unsqueeze(1) * (1 - self.mask)
                                                      + input.unsqueeze(1) * self.mask, 1)]
        else:
            return self.layers(input)

    def reset_parameters(self):
        for layer in self.layers:
            if hasattr(layer, 'reset_parameters'):
                layer.reset_parameters()

def run_SAM(in_data, skeleton=None, device="cpu",
            train=5000, test=1000,
            batch_size=-1, lr_gen=.001,
            lr_disc=.01, lambda1=0.001, lambda2=0.0000001, nh=None, dnh=None,
            verbose=True, losstype="fgan",
            dagstart=0, dagloss=False,
            dagpenalization=0.05, dagpenalization_increase=0.0,
            dag_threshold=0.5,
            linear=False, hlayers=2):

    list_nodes = list(in_data.columns)
    data = scale(in_data[list_nodes].values)
    nb_var = len(list_nodes)
    data = data.astype('float32')
    data = th.from_numpy(data).to(device)
    if batch_size == -1:
        batch_size = data.shape[0]
    rows, cols = data.size()
    # Get the list of indexes to ignore
    if skeleton is not None:
        skeleton = th.from_numpy(skeleton.astype('float32'))

    sam = SAM_generators((batch_size, cols), nh, skeleton=skeleton,
                         linear=linear).to(device)

    sam.reset_parameters()
    g_optimizer = th.optim.Adam(list(sam.parameters()), lr=lr_gen)

    if losstype != "mse":
        discriminator = SAM_discriminator(cols, dnh, hlayers).to(device)
        discriminator.reset_parameters()
        d_optimizer = th.optim.Adam(discriminator.parameters(), lr=lr_disc)
        criterion = th.nn.BCEWithLogitsLoss()
    else:
        criterion = th.nn.MSELoss()
        disc_loss = th.zeros(1)

    graph_sampler = MatrixSampler(nb_var, mask=skeleton,
                                  gumble=False).to(device)
    graph_sampler.weights.data.fill_(2)
    graph_optimizer = th.optim.Adam(graph_sampler.parameters(), lr=lr_gen)

    if not linear:
        neuron_sampler = MatrixSampler((nh, nb_var), mask=False,
                                       gumble=True).to(device)
        neuron_optimizer = th.optim.Adam(list(neuron_sampler.parameters()),
                                         lr=lr_gen)

    _true = th.ones(1).to(device)
    _false = th.zeros(1).to(device)
    output = th.zeros(nb_var, nb_var).to(device)

    noise = th.randn(batch_size, nb_var).to(device)
    noise_row = th.ones(1, nb_var).to(device)
    data_iterator = DataLoader(data, batch_size=batch_size,
                               shuffle=True, drop_last=True)

    # RUN
    if verbose:
        pbar = tqdm(range(train + test))
    else:
        pbar = range(train+test)
    for epoch in pbar:
        for i_batch, batch in enumerate(data_iterator):
            g_optimizer.zero_grad()
            graph_optimizer.zero_grad()

            if losstype != "mse":
                d_optimizer.zero_grad()

            if not linear:
                neuron_optimizer.zero_grad()

            # Train the discriminator

            if not epoch > train:
                drawn_graph = graph_sampler()
                if not linear:
                    drawn_neurons = neuron_sampler()
                else:
                    drawn_neurons = None
            noise.normal_()
            generated_variables = sam(batch, noise,
                                      th.cat([drawn_graph, noise_row], 0),
                                      drawn_neurons)

            if losstype == "mse":
                gen_loss = criterion(generated_variables, batch)
            else:
                disc_vars_d = discriminator(generated_variables.detach(), batch)
                disc_vars_g = discriminator(generated_variables, batch)
                true_vars_disc = discriminator(batch)

                if losstype == "gan":
                    disc_loss = sum([criterion(gen, _false.expand_as(gen)) for gen in disc_vars_d]) / nb_var \
                                     + criterion(true_vars_disc, _true.expand_as(true_vars_disc))
                    # Gen Losses per generator: multiply py the number of channels
                    gen_loss = sum([criterion(gen,
                                              _true.expand_as(gen))
                                    for gen in disc_vars_g])
                elif losstype == "fgan":

                    disc_loss = sum([th.mean(th.exp(gen - 1)) for gen in disc_vars_d]) / nb_var - th.mean(true_vars_disc)
                    gen_loss = -sum([th.mean(th.exp(gen - 1)) for gen in disc_vars_g])

                disc_loss.backward()
                d_optimizer.step()

            filters = graph_sampler.get_proba()

            struc_loss = lambda1*drawn_graph.sum()

            func_loss = 0 if linear else lambda2*drawn_neurons.sum()
            regul_loss = struc_loss + func_loss

            if dagloss and epoch > train * dagstart:
                dag_constraint = notears_constr(filters*filters)
                loss = gen_loss + regul_loss + (dagpenalization +
                                                (epoch - train * dagstart)
                                                * dagpenalization_increase) * dag_constraint
            else:
                loss = gen_loss + regul_loss
            if verbose and epoch % 20 == 0 and i_batch == 0:
                pbar.set_postfix(gen=gen_loss.item()/cols,
                                 disc=disc_loss.item(),
                                 regul_loss=regul_loss.item(),
                                 tot=loss.item())

            if epoch < train + test - 1:
                loss.backward(retain_graph=True)

            if epoch >= train:
                output.add_(filters.data)

            g_optimizer.step()
            graph_optimizer.step()
            if not linear:
                neuron_optimizer.step()

    return output.div_(test).cpu().numpy()

# def exec_sam_instance(data, skeleton=None, gpus=0,
#                       device='cpu', verbose=True, log=None,
#                       lr=0.001, dlr=0.01, lambda1=0.001, lambda2=0.0000001, nh=200, dnh=500,
#                       train=10000, test=1000, batchsize=-1,
#                       losstype="fgan", dagstart=0, dagloss=False,
#                       dagpenalization=0.001, dagpenalization_increase=0.0,
#                       dag_threshold=0.5, linear=False, hlayers=2):
#         out = run_SAM(data, skeleton=skeleton,
#                       device=device,lr_gen=lr, lr_disc=dlr,
#                       lambda1=lambda1, lambda2=lambda2,
#                       nh=nh, dnh=dnh,
#                       train=train,
#                       test=test, batch_size=batchsize,
#                       dagstart=dagstart,
#                       dagloss=dagloss,
#                       dagpenalization=dagpenalization,
#                       dagpenalization_increase=dagpenalization_increase,
#                       losstype=losstype,
#                       dag_threshold=dag_threshold,
#                       linear=linear,
#                       hlayers=hlayers
#                       )
#         if log is not None:
#             np.savetxt(log, out, delimiter=",")
#         return out

class SAM(object):
    """Structural Agnostic Model."""

    def __init__(self, lr=0.01, dlr=0.01, lambda1=0.01, lambda2=0.00001, nh=200, dnh=200,
                 train_epochs=10000, test_epochs=1000, batchsize=-1,
                 losstype="fgan", dagstart=0.5, dagloss=True, dagpenalization=0,
                 dagpenalization_increase=0.001, linear=False, hlayers=2):

        """Init and parametrize the SAM model.

        :param lr: Learning rate of the generators
        :param dlr: Learning rate of the discriminator
        :param l1: L1 penalization on the causal filters
        :param nh: Number of hidden units in the generators' hidden layers
           a
           ((cols,cols)
        :param dnh: Number of hidden units in the discriminator's hidden layer$
        :param train_epochs: Number of training epochs
        :param test_epochs: Number of test epochs (saving and averaging the causal filters)
        :param batchsize: Size of the batches to be fed to the SAM model.
        """
        super(SAM, self).__init__()
        self.lr = lr
        self.dlr = dlr
        self.lambda1 = lambda1
        self.lambda2 = lambda2
        self.nh = nh
        self.dnh = dnh
        self.train = train_epochs
        self.test = test_epochs
        self.batchsize = batchsize
        self.losstype = losstype
        self.dagstart = dagstart
        self.dagloss = dagloss
        self.dagpenalization = dagpenalization
        self.dagpenalization_increase = dagpenalization_increase
        self.linear = linear
        self.hlayers = hlayers

    def predict(self, data, skeleton=None, mixed_data=False, nruns=6, njobs=1,
                gpus=0, verbose=True, log=None):
        """Execute SAM on a dataset given a skeleton or not.

        :param data: Observational data for estimation of causal relationships by SAM
        :param skeleton: A priori knowledge about the causal relationships as an adjacency matrix.
                         Can be fed either directed or undirected links.
        :param nruns: Number of runs to be made for causal estimation.
                      Recommended: >5 for optimal performance.
        :param njobs: Numbers of jobs to be run in Parallel.
                      Recommended: 1 if no GPU available, 2*number of GPUs else.
        :param gpus: Number of available GPUs for the algorithm.
        :param verbose: verbose mode
        :param plot: Plot losses interactively. Not recommended if nruns>1
        :param plot_generated_pair: plots a generated pair interactively.  Not recommended if nruns>1
        :return: Adjacency matrix (A) of the graph estimated by SAM,
                A[i,j] is the term of the ith variable for the jth generator.
        """
        assert nruns > 0
        if nruns == 1:
            return run_SAM(data, skeleton=skeleton,
                           lr_gen=self.lr,
                           lr_disc=self.dlr,
                           verbose=verbose,
                           lambda1=self.lambda1, lambda2=self.lambda2,
                           nh=self.nh, dnh=self.dnh,
                           train=self.train,
                           test=self.test, batch_size=self.batchsize,
                           dagstart=self.dagstart,
                           dagloss=self.dagloss,
                           dagpenalization=self.dagpenalization,
                           dagpenalization_increase=self.dagpenalization_increase,
                           losstype=self.losstype,
                           dag_threshold=self.dag_threshold,
                           linear=self.linear,
                           hlayers=self.hlayers,
                           device='cuda:0' if gpus else 'cpu')
        else:
            list_out = []
            if log is not None:
                idx = 0
                while os.path.isfile(log + str(idx)):
                    list_out.append(np.loadtxt(log + str(idx), delimiter=","))
                    idx += 1
            results = parallel_run(run_SAM, data, skeleton=skeleton,
                                   nruns=nruns-len(list_out),
                                   njobs=njobs, gpus=gpus, lr_gen=self.lr,
                                   lr_disc=self.dlr,
                                   verbose=verbose,
                                   lambda1=self.lambda1, lambda2=self.lambda2,
                                   nh=self.nh, dnh=self.dnh,
                                   train=self.train,
                                   test=self.test, batch_size=self.batchsize,
                                   dagstart=self.dagstart,
                                   dagloss=self.dagloss,
                                   dagpenalization=self.dagpenalization,
                                   dagpenalization_increase=self.dagpenalization_increase,
                                   losstype=self.losstype,
                                   linear=self.linear,
                                   hlayers=self.hlayers)
            list_out.extend(results)
            list_out = [i for i in list_out if not np.isnan(i).any()]
            try:
                assert len(list_out) > 0
            except AssertionError as e:
                print("All solutions contain NaNs")
                raise(e)
            return sum(list_out)/len(list_out)

max_stars_repo_name: vishalbelsare/SAM

max_issues_repo_name: kassaad/SAM

max_forks_repo_name: kassaad/SAM


text:
"""Tensor Class."""
import functools
import operator

import numpy as np

# PyCUDA initialization

import pycuda.driver as cuda

import pycuda.autoinit
from pycuda.compiler import SourceModule

from .gpu_kernels import add, arithmetic
from .states import TensorState

ops = {"+": operator.add, "-": operator.sub, "*": operator.mul, "/": operator.truediv}

class GPUConnectMixin:
    """Mixin for GPU connect"""

    def _alloc_device_memory(self, shape):
        """_alloc_device_memory.
        Allocate memory  to device.

        Args:
            data:
        """
        _nbytes = np.prod(shape) * 4
        _device_data = cuda.mem_alloc(int(_nbytes))
        _device_data.shape = tuple(shape)
        _device_data.dtype = np.float32
        return _device_data

    def _memory_host_to_device(self, device_data, data):
        """_memory_host_to_device.
        Copy memory host to device(GPU).

        Args:
            data:
            device_data:
        """
        cuda.memcpy_htod(device_data, data)
        return

    @staticmethod
    def _idiv(a, b):
        return a // b + 1

    @staticmethod
    def get_kernel(kernel, function):
        """get_kernel.
        get the kernel.

        Args:
            kernel:
            function:
        """
        return kernel.get_function(function)

class GradientMixin:
    """ Gradient Mixin class with grad tools. """

    def _walk(self, leaf_out_node):
        """_walk.
        Reverse Graph Traversal with gradients.

        Args:
            leaf_out_node: Leaf Node.
            in_grad: Input Gradient.
        """
        self.visited.add(leaf_out_node)
        for node in leaf_out_node._child_nodes:
            if node not in self.visited:
                self._walk(node)
        self.nodes.append(leaf_out_node)
        return

    def backward(self):
        """backward.
        Backward Function with Input Gradient set 1.

        Args:
            out_node: Leaf Output Node.
        """
        self.visited = set()
        self.nodes = []
        self._walk(self)
        self.grad = 1.0
        for node in reversed(self.nodes):
            node._backward(node.grad)
        return self.grad

class Tensor(GPUConnectMixin, GradientMixin):
    """Tensor Class."""

    BLOCKSIZE = 256

    """
    The dict wastes a lot of RAM. Python can’t just allocate a static amount of memory at
    object creation to store all the attributes. Therefore it sucks a lot of RAM if you
    create a lot of objects (I am talking in thousands and millions).
    Still there is a way to circumvent this issue.
    It involves the usage of __slots__ to tell Python not to use a dict,
    and only allocate space for a fixed set of attributes.
    """

    __slots__ = (
        "_data",
        "_name",
        "_n",
        "_dtype",
        "_shape",
        "gpu",
        "state",
        "device_name",
    )

    def __init__(self, data, name=None, dtype=None):
        """__init__.
        Initializes Tensor Class.

        Args:
            data: list or np.array data.
            gpu: use gpu?
        ::

        Example:

        >> a = Tensor([1, 2])
        >> b = Tensor([2,3])
        >> print(a + b)
        (dp.Tensor, shape=(2,), dtype = int32, numpy:([3,5], dtype = int32)
        """
        self.state = TensorState.HOST
        if isinstance(data, (list, float, int)):
            data = np.array(data, dtype=dtype if dtype else np.float32)
        elif isinstance(data, pycuda._driver.DeviceAllocation):
            self.state = TensorState.DEVICE
        elif not (isinstance(data, np.ndarray) or isinstance(data, np.float32)):
            raise TypeError(f"numpy excepted but {type(data)} passed.")
        self._data = data
        self._dtype = data.dtype
        self._shape = data.shape
        self._name = name
        self.gpu = False
        self.grad = 0.0
        self._child_nodes = tuple()

        def _backward(in_grad=0.0):
            self.grad = in_grad
            return (in_grad,)

        self._backward = _backward
        self.device_name = "cpu:0"

    def detach(self):
        """detach.
        Detach state.
        """
        self.state = TensorState.DETACH
        # TODO(kartik4949) : Write ME.
        return Tensor(self._data)

    @property
    def shape(self):
        return self._shape

    @property
    def name(self):
        return self._name

    @property
    def data(self):
        return self._data

    @property
    def dtype(self):
        return self._dtype

    @property
    def where(self):
        return self._device()

    def _device(self):
        if self.state == TensorState.DEVICE:
            _cuda_device = "gpu"

        if self.state == TensorState.HOST:
            _cuda_device = "cpu"
        return _cuda_device

    def asarray(self, data: list = None, dtype: tuple = None):
        """asarray.
        convert array to DP array.

        Args:
            data (list): data
            dtype (tuple): dtype
        """
        # Depracted!
        return Tensor(np.asarray(data, dtype=dtype))

    def device(self, name: str = None):
        """device.
        register the data on device.

        Args:
            name (str): name of device
        """
        assert name.startswith("cpu") or name.startswith("gpu"), "Wrong Device!!"
        # set precision to float32.
        assert (
            self.dtype == np.float32
        ), "Only single precision is supported i.e float32"
        if self.state != TensorState.DEVICE:
            self.state = TensorState.DEVICE
            self.device_name = name
            data = self._alloc_device_memory(self.shape)
            self._memory_host_to_device(data, self._data)
            self._shape = self._data.shape
            self._dtype = self._data.dtype
            self._data = data
        return self

    def cpu(
        self,
    ):
        """cpu.
        copy buffer from device to cpu.
        """
        _host_out_arry = np.empty(self.shape, dtype=np.float32)
        cuda.memcpy_dtoh(_host_out_arry, self._data)
        cuda.Context.synchronize()
        return Tensor(_host_out_arry)

    def sigmoid(self):
        """Sigmoid function."""
        sig = 1 / (1 + np.exp(-self._data))
        ret = Tensor(sig)
        ret._child_nodes = (self,)

        def _backward(in_grad):
            self.grad += in_grad * (ret._data * (1 - ret._data))
            return self.grad

        ret._backward = _backward

        return ret

    def relu(self):
        """Relu function."""
        _data = np.maximum(self._data, 0)
        out = Tensor(_data)
        out._child_nodes = (self,)

        def _backward(in_grad):
            self.grad += (out._data > 0) * in_grad
            return (self.grad,)

        out._backward = _backward
        return out

    def tanh(self):
        """Tanh Function."""
        t2 = Tensor(
            np.zeros(self.shape, dtype=self.data.dtype) + 2,
        )
        t1 = Tensor(np.zeros(self.shape, dtype=self.data.dtype))
        return self.mul(t2).sigmoid().mul(t2) - t1  # 2*sigmoid(2*x)-1

    def add(self, tensor):
        """add.
        Vector Addition which adds Tensor with given Tensor.

        Args:
            tensor: Tensor class
        """

        def _backward(in_grad):
            self.grad += in_grad
            tensor.grad += in_grad
            return in_grad, in_grad

        return self.arithmetic(tensor, _backward, "+")

    def sub(self, tensor):
        """sub.
        Vector Addition which substracts Tensor with given Tensor.

        Args:
            tensor: Tensor class
        """

        def _backward(in_grad):
            self.grad += in_grad
            tensor.grad += -in_grad
            return in_grad, -in_grad

        return self.arithmetic(tensor, _backward, "-")

    def mul(self, tensor):
        """mul.
        Vector Addition which multiplies Tensor with given Tensor.

        Args:
            tensor: Tensor class
        """

        def _backward(in_grad):
            self_grad = in_grad * tensor._data
            tensor_grad = in_grad * self._data
            self.grad += self_grad
            tensor.grad += tensor_grad
            return self_grad, tensor_grad

        return self.arithmetic(tensor, _backward, "*")

    def arithmetic(self, tensor, backward=None, operation: str = "+"):
        """Arithmetic.
        Vector arithmetic operations on given Tensor.

        Args:
            tensor: Tensor class
        """
        if self.state != TensorState.DEVICE:
            ret = Tensor(ops[operation](self._data, tensor.data))
            ret._child_nodes = (self, tensor)
            if backward:
                ret._backward = backward
            return ret
        assert isinstance(
            tensor, self.__class__
        ), f"Tensor is required but passed {type(tensor)}"
        ret = self._alloc_device_memory(self.shape)
        N = max(self.shape)
        blockDim = (self.BLOCKSIZE, 1, 1)
        gridDim = (self._idiv(N, self.BLOCKSIZE), 1, 1)
        _vec_kernel = self.get_kernel(arithmetic(operation), "device_arithmetic")
        _vec_kernel(
            ret,
            self._data,
            tensor.data,
            np.int32(N),
            block=blockDim,
            grid=gridDim,
        )

        ret = Tensor(ret)
        ret._dtype = self.dtype
        ret._shape = self.shape
        return ret

    def __pow__(self, value):
        out = Tensor(self.data ** value)
        out._child_nodes = (self,)

        def _backward(in_grad):
            self.grad += (value * self._data ** (value - 1)) * in_grad
            return (self.grad,)

        out._backward = _backward
        return out

    def __add__(self, tensor):
        tensor = tensor if isinstance(tensor, Tensor) else Tensor(tensor)
        return self.add(tensor)

    def __radd__(self, tensor):
        return self + tensor

    def __mul__(self, tensor):
        return self.mul(tensor)

    def __sub__(self, tensor):
        return self.sub(tensor)

    def __neg__(self):
        return self * -1

    def __rsub__(self, tensor):
        return tensor + (-self)

    def __rmul__(self, tensor):
        return self * tensor

    def __truediv__(self, value):
        return self * value ** -1

    def __rtruediv__(self, vale):
        return value * self ** -1

    def __repr__(self):
        return "Tensor( %s shape: %s, numpy: (%s, dtype=%s), device: %s)" % (
            f"name: {self.name}, " if self.name else "",
            self.shape,
            self._data,
            self.dtype,
            self.where,
        )

reason: import numpy,import pycuda,from pycuda


text:
from asyncio import create_task
from datetime import datetime, timedelta
from typing import Tuple

from numpy import isclose
from src.data.database import Database
from src.data.histories import Histories
from src.data.symbols import Symbols
from src.logger import WithLogger
from src.utils.asyncio_utils import run_async_main, task_queue
from src.utils.console_utils import progress
from src.utils.date_utiles import LATEST_WEEKDAY

class Data(WithLogger):
    def __init__(self) -> None:
        WithLogger.__init__(self)
        self.__database = Database()
        self.__histories = Histories()
        self.__symbols = Symbols()

    async def init(self):
        await self.__database.init()

    async def read_symbols(self):
        return await self.__database.history.read_symbols()

    async def read_history(self, symbol: str):
        return await self.__database.history.read(symbol)

    async def backfill(self, *symbols: Tuple[str]):
        if not symbols:
            symbols = await self.__symbols.symbols()

        self.logger.info(f"{len(symbols)} symbols.")
        tasks = map(lambda s: create_task(self.__update(s), name=s), symbols)

        async for i in task_queue(tasks):
            progress(i, len(symbols), "Backfill")

        self.logger.info("done.")

    async def validate(self, resume: str = None, symbols: Tuple[str] = None):
        """
        :Parameter
            resume: str
                Last known valid symbol to resume the validation from.
                Default is None.
        """
        if not symbols:
            symbols = await self.__database.history.read_symbols()
            if resume:
                symbols = symbols[symbols.index(resume) + 1:]

        self.logger.info(f"{len(symbols)} symbols.")
        start = datetime.now()
        tasks = map(lambda s: create_task(self.__test(s), name=s), symbols)

        async for i in task_queue(tasks):
            progress(i, len(symbols), "Validate")

        self.logger.info(datetime.now() - start)

    async def delete(self, symbol: str, start: str = '2000-01-01'):
        pass

    async def __test(self, symbol: str):
        db = await self.__database.history.read_first(symbol)
        web = await self.__histories.history(symbol, start=str(db.index[0].date()), end=str(db.index[0].date() + timedelta(days=1)))

        def candidate(d):
            return d[['Open', 'High', 'Low', 'Close']]

        async def handle():
            self.logger.error("Invalid data for\n%s.\nData in db:\n%s.\nData from web:\n%s" % (symbol, db, web))
            await self.__database.history.delete(symbol)

        try:
            if not isclose(candidate(web), candidate(db)).all():
                await handle()
        except Exception as e:
            await handle()
            self.logger.exception(e)

    async def __update(self, symbol: str):
        last = await self.__database.history.read_last(symbol)
        if last.empty:
            history = await self.__histories.history(symbol)
            if not history.empty:
                await self.__database.history.insert(symbol, history)
        else:
            updated = last.index[0]
            if (updated >= LATEST_WEEKDAY):
                return

            history = await self.__histories.history(symbol, start=updated.date())
            ln = len(history)
            if ln < 2 or history.index[0] != updated:
                return

            factor = history['Close'][0] / last['Close'][0]
            if not isclose(factor, 1):
                await self.__database.history.update(symbol, factor)

            await self.__database.history.insert(symbol, history.tail(ln-1))

if __name__ == '__main__':
    async def main():
        ds = Data()
        await ds.init()

        # await ds.backfill('GIM')
        await ds.validate()

    run_async_main(main)

reason: from numpy


text:
import ase
import numpy as np
import spglib as spg
from custom_frame import frame2qe_format,get_standard_frame,frame2qe_format_new
from utils import get_kpts
from raw_info import Ang2au,z2eleminfo,dont_print_wyck,sgwyck2qemask
from SSSP_acc_PBE_info import PP_names,rhocutoffs,wfccutoffs

# List of the Space groups that do not need special care
NOPROBLEM = [
     3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22,
     23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
     41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58,
     59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76,
     77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94,
     95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110,
     111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125,
     126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140,
     141, 142, 143, 144, 145, 147, 149, 150, 151, 152, 153, 154,
     156, 157, 158, 159, 162, 163, 164, 165, 168, 169, 170,
     171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185,
     186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200,
     201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215,
     216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230
]

# List of spacegroups that need ibrav0 input type
ibrav0 = [1,2]

frame2change = {
                5:[146, 148, 155, 160, 161, 166, 167],
}

def makeQEInput_new(crystal,sites_z,symprec=1e-5,
                    rhocutoff = None,wfccutoff = None,
                calculation_type='"scf"',smearing=1e-2,
                pressure=0,press_conv_thr=0.5,cell_factor=2,
                etot_conv_thr=1e-4,forc_conv_thr=1e-3,nstep=150,
                scf_conv_thr=1e-6,print_forces=True,print_stress=False,
                restart=False,collect_wf=False,force_ibrav0=False,
                kpt = [2,2,2],Nkpt=None,kpt_offset = [0,0,0],
                ppPath='"./pseudo/SSSP_acc_PBE/"'):

    restart_mode = '"from_scratch"'
    if restart:
        restart_mode = '"restart"'

    sym_data = spg.get_symmetry_dataset(crystal,symprec=symprec)
    if np.unique(sym_data['equivalent_atoms']).size != len(sites_z):
        raise ValueError('The structure has different number of '
                         'sites than expected: detected {} / expected {}'.format(np.unique(sym_data['equivalent_atoms']).size,len(sites_z)))
    spaceGroupIdx = sym_data['number']
    if spaceGroupIdx in ibrav0:
        force_ibrav0 = True

    new_crystal,cell_par, positions_data = frame2qe_format_new(crystal,sites_z,sym_data,force_ibrav0)

    if wfccutoff is None:
        wfccutoff = np.max([wfccutoffs[zatom] for zatom in sites_z])
    if rhocutoff is None:
        rhocutoff = np.max([rhocutoffs[zatom]for zatom in sites_z])

    if Nkpt is not None:
        kpt = list(get_kpts(new_crystal,Nkpt=Nkpt))

    kwargs = dict(crystal=new_crystal,sites_z=sites_z,sym_data=sym_data, cell_par=cell_par, positions_data=positions_data,
                  etot_conv_thr=etot_conv_thr, forc_conv_thr=forc_conv_thr, nstep=nstep,
                  rhocutoff=rhocutoff, wfccutoff=wfccutoff,force_ibrav0=force_ibrav0,
                  scf_conv_thr=scf_conv_thr, print_forces=print_forces,
                  print_stress=print_stress, restart_mode=restart_mode,
                  pressure=pressure, press_conv_thr=press_conv_thr, cell_factor=cell_factor,
                  calculation_type=calculation_type, smearing=smearing,
                  kpt=kpt, kpt_offset=kpt_offset, ppPath=ppPath,
                  collect_wf=collect_wf)

    qeInput = makeQEInput_sg_new(**kwargs)

    return qeInput

def makeQEInput_sg_new(crystal, sites_z,cell_par,positions_data,sym_data,
                    rhocutoff=20 * 4, wfccutoff=20,force_ibrav0=False,
                   calculation_type='"scf"', restart_mode='"from_scratch"',
                   smearing=1e-2,
                   etot_conv_thr=1e-4, forc_conv_thr=1e-3, nstep=150,
                   scf_conv_thr=1e-6, print_forces=True, print_stress=False,
                   pressure=0, press_conv_thr=0.5, cell_factor=2, collect_wf=False,
                   kpt=[2, 2, 2], kpt_offset=[0, 0, 0], ppPath='"./pseudo/"'):

    spaceGroupIdx = sym_data['number']

    unique_sites_z = np.unique(sites_z)
    unique_species = []
    mass = []
    PP = []
    for site_z in unique_sites_z:
        info = z2eleminfo[site_z]
        unique_species.append(info['symbol'])
        mass.append(info['mass'])
        PP.append(PP_names[site_z])

    Ntype = len(unique_species)

    if force_ibrav0:
        ibrav = 0
        cellParam = np.ones((6,))
        spaceGroupIdx = None
        Natom = len(positions_data['positions'])

    else:
        ibrav = SG2ibrav(spaceGroupIdx)
        cellParam = cell_par
        Natom = len(unique_species)

    if ibrav == -12:
        uniqueb = '.TRUE.'
    elif ibrav == -13:
        uniqueb = '.TRUE.'
    else:
        uniqueb = '.FALSE.'

    # nbnd = get_number_of_bands(Natom=Natom, Nvalence=nvalence)

    d2r = np.pi / 180.

    tprnfor = '.false.'
    if print_forces:
        tprnfor = '.true.'
    tstress = '.false.'
    if print_stress or calculation_type == '"vc-relax"':
        tstress = '.true.'

    wf_collect = '.false.'
    outdir = '"./out/"'
    if collect_wf:
        wf_collect = '.true.'
    wfcdir = '"./wf_out/"'

    cosAB = np.cos(cellParam[3] * d2r)
    cosAC = np.cos(cellParam[4] * d2r)
    cosBC = np.cos(cellParam[5] * d2r)

    # define name list of QE
    control = {'calculation': calculation_type,
               'outdir': outdir,
               'wfcdir': wfcdir,
               'prefix': '"qe"',
               'disk_io': '"low"',
               'pseudo_dir': ppPath,
               'restart_mode': restart_mode,
               'verbosity': '"high"',
               'wf_collect': wf_collect,
               'tprnfor': tprnfor,
               'tstress': tstress,
               'nstep': '{:.0f}'.format(nstep),
               'etot_conv_thr': '{:.8f}'.format(etot_conv_thr * Natom),
               'forc_conv_thr': '{:.8f}'.format(forc_conv_thr),
               }
    controlKeys = ['calculation', 'outdir', 'wfcdir', 'prefix', 'pseudo_dir', 'disk_io', 'restart_mode',
                   'verbosity', 'wf_collect', 'tprnfor', 'tstress', 'nstep', 'etot_conv_thr', 'forc_conv_thr']

    system = {
        'ecutrho': '{:.0f}'.format(rhocutoff),
        'ecutwfc': '{:.0f}'.format(wfccutoff),
        'ibrav': str(ibrav),
        'nat': str(Natom),
        # 'nbnd' : str(nbnd),
        'ntyp': str(Ntype),
        'occupations': '"smearing"',
        'smearing': '"cold"',
        'degauss': '{:.6f}'.format(smearing),
        'space_group': str(spaceGroupIdx),
        'uniqueb': uniqueb,
        'A': str(cellParam[0]),
        'B': str(cellParam[1]),
        'C': str(cellParam[2]),
        'cosAB': '{:.5f}'.format(cosAB),
        'cosAC': '{:.5f}'.format(cosAC),
        'cosBC': '{:.5f}'.format(cosBC),
        'celldm(1)': str(cellParam[0] * Ang2au),
        'celldm(2)': str(cellParam[1] / cellParam[0]),
        'celldm(3)': str(cellParam[2] / cellParam[0]),
        'celldm(4)': '{:.5f}'.format(cosAB),
        'celldm(5)': '{:.5f}'.format(cosAC),
        'celldm(6)': '{:.5f}'.format(cosBC),
    }

    if ibrav in [-13,5]:
        syskeys = ['ecutrho', 'ecutwfc', 'ibrav', 'nat', 'ntyp',
                   'occupations', 'smearing', 'degauss', 'space_group', 'uniqueb',
                   'celldm(1)', 'celldm(2)', 'celldm(3)', 'celldm(4)', 'celldm(5)', 'celldm(6)']
    elif ibrav == 0:
        syskeys = ['ecutrho', 'ecutwfc', 'ibrav', 'nat', 'ntyp', #'nbnd',
                   'occupations', 'smearing', 'degauss']
    else:
        syskeys = ['ecutrho', 'ecutwfc', 'ibrav', 'nat', 'ntyp',
                   'occupations', 'smearing', 'degauss', 'space_group', 'uniqueb',
                   'A', 'B', 'C', 'cosAB', 'cosAC', 'cosBC']

    electrons = {'conv_thr': '{:.8f}'.format(scf_conv_thr)}

    # pressure in [KBar]
    cell = {
        'cell_factor': '{:.5f}'.format(cell_factor),
        'press': '{:.5f}'.format(pressure),
        'press_conv_thr': '{:.5f}'.format(press_conv_thr),
    }

    # define cards of QE
    optionkey = 'unit'
    atomic_sp = {'ATOMIC_SPECIES': {'species': unique_species,
                                    'mass': mass, 'PP': PP}, 'unit': ''}
    atspkeys = ['species', 'mass', 'PP']

    # if spaceGroupIdx in dont_print_wyck_dict.keys() and ibrav != 0:
    #     for it in range(len(positions_data['wyckoffs'])):
    #         if positions_data['wyckoffs'][it] in dont_print_wyck_dict[spaceGroupIdx]:
    #             positions_data['wyckoffs'][it] = ''

    if ibrav == 0:
        atomic_pos = {'ATOMIC_POSITIONS': positions_data,
                      'unit': 'angstrom'}
        atposkeys = ['species',  'positions']

        cell_param = {'CELL_PARAMETERS': {'cell': crystal.get_cell()}, 'unit': 'angstrom'}
        cellkeys = ['cell']
    else:

        atomic_pos = {'ATOMIC_POSITIONS': positions_data,
                      'unit': 'crystal_sg'}
        atposkeys = ['species','wyckoffs', 'positions']

    kpoints = {'K_POINTS': {'kpoints': np.array(kpt + kpt_offset,int).reshape((1,-1))}, 'unit': 'automatic'}
    kptkeys = ['kpoints']

    qeInput = ''

    qeInput += makeNamelist('&CONTROL', control, controlKeys)
    qeInput += makeNamelist('&SYSTEM', system, syskeys)
    qeInput += makeNamelist('&ELECTRONS', electrons)
    if calculation_type == '"vc-relax"':
        # seems like this algorithm is not as reliable as bfgs
        #cell.update({'cell_dynamics': '"damp-w"'})
        cell.update({'cell_dynamics': '"bfgs"'})
        ions = {'ion_dynamics': '"bfgs"', }
        qeInput += makeNamelist('&IONS', ions)

    qeInput += makeNamelist('&CELL', cell)

    qeInput += makeCard(atomic_sp, cardKeys=atspkeys, optionKey=optionkey, T=True)
    qeInput += makeCard(atomic_pos, cardKeys=atposkeys, optionKey=optionkey, T=True, sg=spaceGroupIdx)

    if ibrav == 0:
        qeInput += makeCard(cell_param, cardKeys=cellkeys, optionKey=optionkey, T=True)

    qeInput += makeCard(kpoints, cardKeys=kptkeys, optionKey=optionkey, T=False)

    return qeInput

def makeQEInput(crystal,spaceGroupIdx=None,WyckTable=None,SGTable=None,ElemTable=None,
                zatom = 14,rhocutoff = None,wfccutoff = None,
                calculation_type='"scf"',smearing=1e-2,
                pressure=0,press_conv_thr=0.5,cell_factor=2,
                etot_conv_thr=1e-4,forc_conv_thr=1e-3,nstep=150,
                scf_conv_thr=1e-6,print_forces=True,print_stress=False,
                restart=False,collect_wf=False,force_ibrav0=False,
                kpt = [2,2,2],Nkpt=None,kpt_offset = [0,0,0],
                ppPath='"./pseudo/SSSP_acc_PBE/"'):

    restart_mode = '"from_scratch"'
    if restart:
        restart_mode = '"restart"'

    if force_ibrav0:
        new_crystal,cell_par, inequivalent_pos = crystal.copy(),crystal.get_cell(),crystal.get_positions()
    else:
        new_crystal,cell_par, inequivalent_pos = frame2qe_format(crystal,spaceGroupIdx)

    PP = [PP_names[zatom]]

    if wfccutoff is None:
        wfccutoff = wfccutoffs[zatom]
    if rhocutoff is None:
        rhocutoff = rhocutoffs[zatom]

    if Nkpt is not None:
        kpt = list(get_kpts(new_crystal,Nkpt=Nkpt))
    # pressure in kBar and en_tot/forces in a.u.
    kwargs = dict(crystal=new_crystal,cell_par=cell_par, inequivalent_pos=inequivalent_pos,
                  spaceGroupIdx=spaceGroupIdx,WyckTable=WyckTable,
                  SGTable=SGTable,ElemTable=ElemTable,
                  etot_conv_thr=etot_conv_thr, forc_conv_thr=forc_conv_thr,nstep=nstep,
                 zatom = zatom,rhocutoff = rhocutoff,wfccutoff = wfccutoff,
                  scf_conv_thr=scf_conv_thr,print_forces=print_forces,
                  print_stress=print_stress,restart_mode=restart_mode,
                  pressure=pressure, press_conv_thr=press_conv_thr, cell_factor=cell_factor,
                 calculation_type=calculation_type,smearing=smearing,
                 kpt = kpt,kpt_offset = kpt_offset,ppPath=ppPath,
                 PP=PP,collect_wf=collect_wf)

    if force_ibrav0:
        qeInput = makeQEInput_ibrav0(**kwargs)
    else:
        if spaceGroupIdx in ibrav0:
            qeInput = makeQEInput_ibrav0(**kwargs)
        else:
            qeInput = makeQEInput_sg(**kwargs)

    return qeInput

def makeQEInput_sg(crystal,spaceGroupIdx,WyckTable,SGTable,ElemTable,
                 zatom = 14,rhocutoff = 20 * 4,wfccutoff = 20,
                 calculation_type='"scf"',restart_mode = '"from_scratch"',
                   smearing=1e-2,cell_par=None, inequivalent_pos=None ,
                etot_conv_thr=1e-4, forc_conv_thr=1e-3,nstep=150,
                scf_conv_thr=1e-6,print_forces=True,print_stress=False,
                 pressure=0,press_conv_thr=0.5,cell_factor=2,collect_wf=False,
                 kpt = [2,2,2],kpt_offset = [0,0,0],ppPath='"./pseudo/"',
                 PP=['Si.pbe-n-rrkjus_psl.1.0.0.UPF']):

    elemInfo = ElemTable[ElemTable['z'] == zatom]
    species = [elemInfo['sym'].values[0],]*1
    mass = [elemInfo['mass'].values[0]] # in atomic unit
    #PP = ['Si.pbe-n-rrkjus_psl.1.0.0.UPF']
    nvalence = elemInfo['val'].values[0]

    wyck = [SG2wyckoff(spaceGroupIdx,WyckTable)]

    ibrav = SG2ibrav(spaceGroupIdx)

    if cell_par is not None and inequivalent_pos  is not None:
        positions = inequivalent_pos
        cellParam = cell_par
    else:
        positions = crystal.get_scaled_positions()[0].reshape((1, -1))
        cellParam = crystal.get_cell_lengths_and_angles()

    cc = get_standard_frame(crystal, spaceGroupIdx, primitive=True)
    Natom = cc.get_number_of_atoms()

    if ibrav == -12:
        uniqueb = '.TRUE.'
    elif ibrav == -13:
        uniqueb = '.TRUE.'
    else:
        uniqueb = '.FALSE.'
    if ibrav == 5:
        rhombohedral = '.FALSE.'
    else:
        rhombohedral = '.TRUE.'

    nbnd = get_number_of_bands(Natom=Natom,Nvalence=nvalence)

    d2r = np.pi / 180.

    tprnfor = '.false.'
    if print_forces:
        tprnfor = '.true.'
    tstress = '.false.'
    if print_stress or calculation_type == '"vc-relax"':
        tstress = '.true.'

    wf_collect = '.false.'
    outdir = '"./out/"'
    if collect_wf:
        wf_collect = '.true.'
    wfcdir = '"./wf_out/"'

    cosAB = np.cos(cellParam[3] * d2r)
    cosAC = np.cos(cellParam[4] * d2r)
    cosBC = np.cos(cellParam[5] * d2r)

    # define name list of QE
    control = {'calculation' : calculation_type,
              'outdir' : outdir,
              'wfcdir': wfcdir,
              'prefix' : '"qe"',
               'disk_io':'"low"',
              'pseudo_dir' : ppPath,
              'restart_mode' : restart_mode,
              'verbosity' : '"high"',
              'wf_collect' : wf_collect,
               'tprnfor': tprnfor,
               'tstress': tstress,
              'nstep': '{:.0f}'.format(nstep),
              'etot_conv_thr' : '{:.8f}'.format(etot_conv_thr*Natom),
              'forc_conv_thr' : '{:.8f}'.format(forc_conv_thr),
        }
    controlKeys = ['calculation', 'outdir','wfcdir', 'prefix', 'pseudo_dir','disk_io', 'restart_mode',
                   'verbosity', 'wf_collect','tprnfor','tstress','nstep', 'etot_conv_thr', 'forc_conv_thr']

    system = {
              'ecutrho' :   '{:.0f}'.format(rhocutoff),
              'ecutwfc' :   '{:.0f}'.format(wfccutoff),
              'ibrav' : str(ibrav),
              'nat' : str(1),
              #'nbnd' : str(nbnd),
              'ntyp' : str(1),
              'occupations' : '"smearing"',
              'smearing' : '"cold"',
              'degauss' :   '{:.6f}'.format(smearing),
              'space_group' : str(spaceGroupIdx),
              'uniqueb':uniqueb,
              'rhombohedral':rhombohedral,
              'A' : str(cellParam[0]),
              'B' : str(cellParam[1]),
              'C' : str(cellParam[2]),
              'cosAB' : '{:.5f}'.format(cosAB),
              'cosAC' : '{:.5f}'.format(cosAC),
              'cosBC' : '{:.5f}'.format(cosBC),
              'celldm(1)': str(cellParam[0] * Ang2au),
              'celldm(2)': str(cellParam[1] / cellParam[0]),
              'celldm(3)': str(cellParam[2] / cellParam[0]),
              'celldm(4)': '{:.5f}'.format(cosAB),
              'celldm(5)': '{:.5f}'.format(cosAC),
              'celldm(6)': '{:.5f}'.format(cosBC),
        }
    if ibrav == -13:
        syskeys = ['ecutrho', 'ecutwfc', 'ibrav', 'nat', 'ntyp',
                   'occupations', 'smearing', 'degauss', 'space_group', 'uniqueb', 'rhombohedral',
                   'celldm(1)', 'celldm(2)', 'celldm(3)', 'celldm(4)', 'celldm(5)', 'celldm(6)']
    else:
        syskeys = ['ecutrho', 'ecutwfc', 'ibrav', 'nat', 'ntyp',
                   'occupations', 'smearing', 'degauss', 'space_group', 'uniqueb','rhombohedral',
                   'A', 'B', 'C', 'cosAB', 'cosAC', 'cosBC']

    # syskeys = [ 'ecutrho','ecutwfc','ibrav', 'nat','nbnd','ntyp' ,
    #            'occupations', 'smearing','degauss','space_group','uniqueb','A','B' ,'C', 'cosAB', 'cosAC', 'cosBC' ]
    electrons = {'conv_thr':'{:.8f}'.format(scf_conv_thr)}
    cellkeys = ['cell_factor', 'press', 'press_conv_thr']
    # pressure in [KBar]
    cell = {
        'cell_factor': '{:.5f}'.format(cell_factor),
        'press': '{:.5f}'.format(pressure),
        'press_conv_thr': '{:.5f}'.format(press_conv_thr),
    }

    # define cards of QE
    optionkey = 'unit'
    atomic_sp = {'ATOMIC_SPECIES':{'species':np.unique(species),
                                   'mass':mass,'PP':PP},'unit':''}
    atspkeys = ['species','mass','PP']
    if spaceGroupIdx in dont_print_wyck:
        atomic_pos = {'ATOMIC_POSITIONS': {'species': species, 'wyckoffs': '',
                                           'positions': list(positions)},
                      'unit': 'crystal_sg'}
    else:
        atomic_pos = {'ATOMIC_POSITIONS':{'species':species, 'wyckoffs':wyck,
                                      'positions':list(positions)},
                  'unit':'crystal_sg'}
    atposkeys = ['species', 'wyckoffs','positions']
    kpoints = {'K_POINTS':{'kpoints':kpt+kpt_offset},'unit':'automatic'}
    kptkeys = ['kpoints']

    qeInput = ''

    qeInput += makeNamelist('&CONTROL',control,controlKeys)
    qeInput += makeNamelist( '&SYSTEM' ,system,syskeys)
    qeInput += makeNamelist( '&ELECTRONS',electrons)
    if calculation_type == '"vc-relax"':
        ions = {'ion_dynamics': '"bfgs"',}
        qeInput += makeNamelist('&IONS', ions)
    qeInput += makeNamelist('&CELL', cell)

    qeInput += makeCard(atomic_sp,cardKeys = atspkeys,optionKey = optionkey,T = True)
    qeInput += makeCard(atomic_pos,cardKeys = atposkeys,optionKey = optionkey,T = True)
    qeInput += makeCard(kpoints,cardKeys = kptkeys,optionKey = optionkey,T = False)

    return qeInput

def makeQEInput_ibrav0(crystal,WyckTable,SGTable,ElemTable,spaceGroupIdx=10,
                 zatom = 14,rhocutoff = 20 * 4,wfccutoff = 20,smearing=1e-2,
                pressure=0,press_conv_thr=0.5,cell_factor=2,collect_wf=False,
                restart_mode = '"from_scratch"',cell_par=None, inequivalent_pos=None ,
                etot_conv_thr=1e-4,forc_conv_thr=1e-3,nstep=150,
                 scf_conv_thr=1e-6,print_forces=True,print_stress=False,
                kpt = [2,2,2],kpt_offset = [0,0,0],calculation_type='"scf"',
                ppPath='"./pseudo/"',PP=['Si.pbe-n-rrkjus_psl.1.0.0.UPF']):

    elemInfo = ElemTable[ElemTable['z'] == zatom]

    mass = [elemInfo['mass'].values[0]] # in atomic unit
    #PP = ['Si.pbe-n-rrkjus_psl.1.0.0.UPF']
    nvalence = elemInfo['val'].values[0]

    if cell_par is not None and inequivalent_pos  is not None:
        # get_ibrav0_atoms gives the positions and cell instead of inequivalent_pos and cell_par
        positions = inequivalent_pos
        lattice = cell_par
    else:
        positions = crystal.get_positions()
        lattice = crystal.get_cell()

    ibrav = 0
    Natom = crystal.get_number_of_atoms()
    species = [elemInfo['sym'].values[0],]*Natom

    nbnd = get_number_of_bands(Natom=Natom,Nvalence=nvalence)

    tprnfor = '.false.'
    if print_forces:
        tprnfor = '.true.'
    tstress = '.false.'
    if print_stress or calculation_type == '"vc-relax"':
        tstress = '.true.'

    wf_collect = '.false.'
    outdir  = '"./out/"'
    if collect_wf:
        wf_collect = '.true.'
    wfcdir = '"./wf_out/"'
    # define name list of QE
    control = {'calculation' : calculation_type,
              'outdir' : outdir,
              'wfcdir': wfcdir,
              'prefix' : '"qe"',
               'disk_io':'"low"',
              'pseudo_dir' : ppPath,
              'restart_mode' : restart_mode,
              'verbosity' : '"high"',
              'wf_collect' : wf_collect,
               'tprnfor': tprnfor,
               'tstress': tstress,
              'nstep':'{:.0f}'.format(nstep),
              'etot_conv_thr': '{:.8f}'.format(etot_conv_thr*Natom),
              'forc_conv_thr': '{:.8f}'.format(forc_conv_thr),
        }
    controlKeys = ['calculation', 'outdir', 'wfcdir', 'prefix', 'pseudo_dir', 'disk_io', 'restart_mode',
                   'verbosity', 'wf_collect', 'tprnfor', 'tstress', 'nstep', 'etot_conv_thr', 'forc_conv_thr']

    system = {
              'ecutrho' :   '{:.0f}'.format(rhocutoff),
              'ecutwfc' :   '{:.0f}'.format(wfccutoff),
              'ibrav' : str(ibrav),
              'nat' : str(Natom),
              'nbnd' : str(nbnd),
              'ntyp' : str(1),
              'occupations' : '"smearing"',
              'smearing' : '"cold"',
              'degauss' :   '{:.6f}'.format(smearing),
        }
    syskeys = [ 'ecutrho','ecutwfc','ibrav', 'nat','nbnd','ntyp' ,
               'occupations', 'smearing','degauss' ]
    electrons = {'conv_thr':'{:.8f}'.format(scf_conv_thr)}

    cellkeys = ['cell_factor', 'press', 'press_conv_thr']
    # pressure in [KBar]
    cell = {
        'cell_factor': '{:.5f}'.format(cell_factor),
        'press': '{:.5f}'.format(pressure) ,
        'press_conv_thr': '{:.5f}'.format(press_conv_thr),
    }

    # define cards of QE
    optionkey = 'unit'
    atomic_sp = {'ATOMIC_SPECIES':{'species':np.unique(species), 'mass':mass,'PP':PP},'unit':''}
    atspkeys = ['species','mass','PP']
    atomic_pos = {'ATOMIC_POSITIONS':{'species':species,'positions':positions},'unit':'angstrom'}
    atposkeys = ['species','positions']
    kpoints = {'K_POINTS':{'kpoints':kpt+kpt_offset},'unit':'automatic'}
    kptkeys = ['kpoints']
    cell_param = {'CELL_PARAMETERS':{'cell':lattice},'unit':'angstrom'}
    cellkeys = ['cell']

    qeInput = ''

    qeInput += makeNamelist('&CONTROL',control,controlKeys)
    qeInput += makeNamelist( '&SYSTEM' ,system,syskeys)
    qeInput += makeNamelist( '&ELECTRONS',electrons)
    if calculation_type == '"vc-relax"':
        ions = {
            'ion_dynamics': '"bfgs"',

        }
        qeInput += makeNamelist('&IONS', ions)
    qeInput += makeNamelist('&CELL', cell)
    qeInput += makeCard(atomic_sp,cardKeys = atspkeys,optionKey = optionkey,T = True)
    qeInput += makeCard(atomic_pos,cardKeys = atposkeys,optionKey = optionkey,T = True)
    qeInput += makeCard(kpoints,cardKeys = kptkeys,optionKey = optionkey,T = False)
    qeInput += makeCard(cell_param,cardKeys = cellkeys,optionKey = optionkey,T = True)

    inputDic = qeInput

    return inputDic

def makeCard(cardDict,cardKeys=None,optionKey='unit',
             T=False, endl = ' \n',eq = ' = ',stl = '  ',sg=None):
    """Transform a Card block in dict form into a string."""
    # from itertools import izip_longest
    # from  tabulate import tabulate
    cardStr = ''
    if cardKeys is None:
        keys =  cardDict.keys()
        keys.remove(optionKey)
        cardKeys = cardDict[keys[0]]

    if optionKey in cardDict.keys():
        keys =  cardDict.keys()
        keys.remove(optionKey)
        for key in keys:
            cardStr += key + stl
        cardStr += cardDict[optionKey] + endl

        for key,val in cardDict.items():
            if isinstance(val, dict):

                if 'wyckoffs' in cardKeys:
                    wyckoffs = val['wyckoffs']
                    positions = val['positions']
                    species = val['species']
                    # print positions
                    for wyck, pos, sp in zip(wyckoffs, positions, species):
                        mask = sgwyck2qemask[(sg, wyck)]

                        if (sg,wyck) in dont_print_wyck:
                            wyck = ' '
                        pp = pos[mask]
                        row_format = get_row_format(cardKeys,len(pp))
                        cardStr += stl + row_format.format(sp,wyck,*pp) + endl
                elif 'positions' in cardKeys and 'wyckoffs' not in cardKeys:
                    positions = val['positions']
                    species = val['species']
                    for  pos, sp in zip( positions, species):
                        row_format = get_row_format(cardKeys, 3)
                        cardStr += stl + row_format.format(sp,  *pos) + endl
                else:
                    lls = np.column_stack([val[cardKey] for cardKey in cardKeys])
                    for row in lls:
                        row_format = get_row_format(cardKeys,3)
                        cardStr += stl + row_format.format(*row) + endl
                # else:
                #     ll = np.column_stack([val[cardKey] for cardKey in cardKeys])
                #     cardStr += stl + tabulate(ll, tablefmt="plain", floatfmt=floatfmt).replace('\n',endl+stl) + endl
                # l = [val[cardKey] for cardKey in cardKeys]
                # ll = map(list, izip_longest(*l)) if T else l
                # cardStr += stl + tabulate(ll, tablefmt="plain",floatfmt=".6f")\
                #     .replace('[','').replace(']','').replace(',','').replace('\n',endl+stl) + endl

        return cardStr
    else:
        print 'cardDict has no key called: {} (optionKey)'.format(optionKey)

def get_row_format(cardKeys,len):
    cardKey_formating = {'species': '{}  ', 'positions': '{:>12.7f}', 'mass': '{}  ', 'PP': '{}  ',
                         'kpoints': '{:>5.0f}'*6, 'wyckoffs': '{} ', 'cell': '{:>12.7f}'*3}

    row_format = ''
    for cardKey in cardKeys:
        if cardKey == 'positions':
            row_format += cardKey_formating[cardKey] * len
        else:
            row_format += cardKey_formating[cardKey]

    return row_format

def makeNamelist(namelistName,namelistDict,namelistKeys=None,
                 endl = ' \n',eq = ' = ',stl = '  '):
    """Transform a Name List block in dict form into a string."""
    if namelistKeys is None:
        namelistKeys = namelistDict.keys()
    namelistStr = namelistName + endl
    for key in namelistKeys:
        namelistStr += stl + key + eq + namelistDict[key] + endl
    namelistStr += '/' + endl
    return namelistStr

def get_number_of_bands(Natom,Nvalence):
    if int(0.2*Natom*Nvalence/2.) > 4:
        nbnd = int(Natom*Nvalence/2.+int(0.2*Natom*Nvalence/2.))
    else:
        nbnd = int(Natom*Nvalence/2.+4)
    return nbnd

def SG2ibrav(spaceGroupIdx,SGTable=None):
    from raw_info import bravaisLattice2ibrav,SG2BravaisLattice

    bravaisLattice = SG2BravaisLattice[spaceGroupIdx]
    ibrav = bravaisLattice2ibrav[bravaisLattice]
    return ibrav

def SG2wyckoff(spaceGroupIdx,wyckTable):
    highestMultiplicitySite = wyckTable[spaceGroupIdx].loc[0,:]
    return str(highestMultiplicitySite['Multiplicity']) + \
                highestMultiplicitySite['Wyckoff letter']


text:
from collections import defaultdict

import numpy as np

from str_finder.utils import *

class DummyMatchFinder:
    def __init__(self, text, pattern):
        self._n = len(text)
        self._m = len(pattern)
        self._text = text + 'A' * (self._m - 1)
        self._pattern = pattern

    @staticmethod
    def _diff_count(a, b):
        return np.sum([
            int((y != '*') & (x != y))
            for x, y in zip(a, b)
        ])

    def find_fuzzy_matches(self):
        result = [0] * self._n
        for i in range(self._n):
            result[i] = self._diff_count(
                self._text[i:i + self._m], self._pattern)
        return result

class TemplateMatcher:

    class PatternMatch:
        def __init__(self, l, r, p_id):
            self._l = l
            self._r = r
            self._id = p_id

        def __str__(self):
            return f'{self._l}:{self._r}, {self._id}'

    def __init__(self, cfg):
        self._cfg = cfg
        self.__parse_config()

    def __parse_config(self):
        self._n = len(self._cfg['template_structure'])
        self._patterns = defaultdict(dict)
        for i, p_cfg in enumerate(self._cfg['template_structure']):
            self._patterns[i]['pattern'] = p_cfg['pattern']
            self._patterns[i]['len'] = len(p_cfg['pattern'])
            self._patterns[i]['is_delim'] = p_cfg['delim']
            self._patterns[i]['max_errors'] = p_cfg['max_errors']

    def __preprocess(self, text):
        for i in range(self._n):
            match_finder = DummyMatchFinder(
                text, self._patterns[i]['pattern'])
            self._patterns[i]['fuzzy_matches'] = match_finder.find_fuzzy_matches()

    def __search_for_start_positions(self):
        result = []
        for i, x in enumerate(self._patterns[0]['fuzzy_matches']):
            if x <= self._patterns[0]['max_errors']:
                result.append(i)
        return result

    def __check_template_match(self, start_position, text_len, text):
        def __add_match(result_matches, x, current_index):
            result_matches.append(
                TemplateMatcher.PatternMatch(
                    current_index,
                    current_index + x['len'] - 1,
                    cp
                )
            )

        current_index = start_position
        result_matches = []
        for cp in range(self._n):
            x = self._patterns[cp]
            if current_index + x['len'] - 1 >= text_len:
                return False, None
            # At least one match is required
            if x['fuzzy_matches'][current_index] > x['max_errors']:
                return False, None
            __add_match(result_matches, x, current_index)
            current_index += x['len']
            if x['is_delim']:
                continue
            while True:
                if current_index >= text_len or x['fuzzy_matches'][current_index] > x['max_errors']:
                    break
                __add_match(result_matches, x, current_index)
                current_index += x['len']

        return True, result_matches

    def __visualize_matches(self, text, matches):
        full_colors = ['black'] * len(text)

        for cp in range(self._n):
            print(f'{cp}: {self._patterns[cp]["pattern"]}')
            n_repeats = 0
            for m in matches:
                if m._id != cp:
                    continue
                match_ = text[m._l:m._r + 1]
                colors = ['green'] + ['blue'] * (len(match_) - 1)
                for i in range(m._l, m._r + 1):
                    full_colors[i] = 'red'
                display_seq(match_, colors)
                n_repeats += 1
            print(f'{n_repeats} repeats\n\n\n\n')

        display_seq(text, full_colors)

    def match(self, text, mode='greedy'):
        self.__preprocess(text)

        potential_start_positions = self.__search_for_start_positions()
        best_matches = []

        for i in potential_start_positions:
            if mode == 'greedy':
                is_match, matches = self.__check_template_match(
                    i, len(text), text)
            else:
                raise NotImplementedError('Only greedy mode is available :(')
            if is_match:
                if len(best_matches) < len(matches):
                    best_matches = matches

        if len(best_matches) == 0:
            print('No matches found')
        else:
            self.__visualize_matches(text, best_matches)

def visualize_matches(text, pattern, max_errors_limit=0):
    match_finder = DummyMatchFinder(text, pattern)
    fuzzy_matches = match_finder.find_fuzzy_matches()
    bins = categorize_matches(fuzzy_matches, len(pattern), max_errors_limit)
    colors = [COLORS_DICT[x] for x in bins]
    display_seq(text, colors)

max_stars_repo_name: dzemidkada/bsu-genomics

max_issues_repo_name: dzemidkada/bsu-genomics

max_forks_repo_name: dzemidkada/bsu-genomics


text:
import torch
import numpy as np

import argparse
import logging

import common
from const import *

class Dictionary(object):
    def __init__(self):
        self.word2idx = {
            WORD[BOS]: BOS,
            WORD[EOS]: EOS,
            WORD[PAD]: PAD,
            WORD[UNK]: UNK
        }
        self.idx = 4

    def add(self, word):
        if self.word2idx.get(word) is None:
            self.word2idx[word] = self.idx
            self.idx += 1

    def __call__(self, sents, min_count):
        words = [word for sent in sents for word in sent]
        word_count = {w: 0 for w in set(words)}
        for w in words:
            word_count[w] += 1

        ignored_word_count = 0
        for word, count in word_count.items():
            if count <= min_count:
                ignored_word_count += 1
                continue
            self.add(word)

        return ignored_word_count

    def __len__(self):
        return self.idx

    def __str__(self):
        return "%s(size = %d)".format(self.__class__.__name__, len(self.idx))

class Corpus(object):
    def __init__(self, save_data=f"{DATAPATH}/corpus.pt", min_word_count=1, max_len=184):
        self._save_data = save_data
        self.min_word_count = min_word_count
        self.max_len = max_len
        self.dict = Dictionary()
        self.parse()
        self.save()

    def parse(self):
        def parse_file(inf):
            src_texts, src_turn, tgt_indexs, tgt_texts = [], [], [], []
            with open(inf, encoding="utf8") as contexts:
                for line in contexts:
                    query1, query2, query3, target = line.strip().split("\t\t")

                    q1_words = common.split_char(query1)
                    turn1 = [1]*(len(q1_words))
                    q2_words = common.split_char(query2)
                    turn2 = [2]*(len(q2_words)+1)
                    q3_words = common.split_char(query3)
                    turn3 = [3]*(len(q3_words))

                    q_words = q1_words + q2_words + [WORD[EOS]] + q3_words
                    turns = turn1 + turn2 + turn3
                    if len(q_words) > self.max_len:
                        continue

                    assert len(q_words) == len(turns)
                    src_texts.append(q_words)
                    src_turn.append(turns)

                    tgt_words = common.split_char(target)
                    new_tgt_words = []
                    for word in tgt_words:
                        if word in q_words:
                            new_tgt_words.append(word)

                    tgt_texts.append([WORD[BOS]]+new_tgt_words)

                    new_tgt_words = new_tgt_words + [WORD[EOS]]
                    t_index = common.find_text_index(q_words, new_tgt_words)

                    tgt_indexs.append(t_index)

                return src_texts, src_turn, tgt_indexs, tgt_texts

        src_texts, src_turn, tgt_indexs, tgt_texts = parse_file(
            f"{DATAPATH}/corpus")
        print(
            f"Ignored word counts - {self.dict(src_texts, self.min_word_count)}")

        src_texts = np.asarray(common.texts2idx(src_texts, self.dict.word2idx))
        src_turn = np.asarray(src_turn)
        tgt_indexs = np.asarray(tgt_indexs)
        tgt_texts = np.asarray(common.texts2idx(tgt_texts, self.dict.word2idx))

        assert src_texts.shape == src_turn.shape
        assert tgt_indexs.shape == tgt_texts.shape

        index = np.arange(tgt_texts.shape[0])
        np.random.shuffle(index)
        src_texts = src_texts[index]
        src_turn = src_turn[index]
        tgt_indexs = tgt_indexs[index]
        tgt_texts = tgt_texts[index]

        self.src_texts_train = src_texts[2000:]
        self.src_turn_train = src_turn[2000:]
        self.tgt_indexs_train = tgt_indexs[2000:]
        self.tgt_texts_train = tgt_texts[2000:]

        self.src_texts_test = src_texts[:2000]
        self.src_turn_test = src_turn[:2000]
        self.tgt_indexs_test = tgt_indexs[:2000]
        self.tgt_texts_test = tgt_texts[:2000]

    def save(self):
        data = {
            'word2idx':  self.dict.word2idx,
            'max_len':  self.max_len,
            'train': {
                'src_texts': self.src_texts_train,
                'src_turn': self.src_turn_train,
                'tgt_indexs': self.tgt_indexs_train,
                'tgt_texts':  self.tgt_texts_train,
            },
            'valid': {
                'src_texts': self.src_texts_test,
                'src_turn': self.src_turn_test,
                'tgt_indexs': self.tgt_indexs_test,
                'tgt_texts':  self.tgt_texts_test,
            }
        }

        torch.save(data, self._save_data)
        print(f'corpora length - {len(self.dict)}')

if __name__ == "__main__":
    Corpus()