text:
# -*- coding: utf-8 -*-
"""
Created on Mon May 29 11:14:44 2017

@author: ning
"""

import pandas as pd
import os
import numpy as np
from collections import Counter

try:
    function_dir = 'D:\\NING - spindle\\Spindle_by_Graphical_Features'
    os.chdir(function_dir)
except:
    function_dir = 'C:\\Users\\ning\\OneDrive\\python works\\Spindle_by_Graphical_Features'
    os.chdir(function_dir)
import eegPipelineFunctions
try:
    file_dir = 'D:\\NING - spindle\\training set\\road_trip\\'
#    file_dir = 'D:\\NING - spindle\\training set\\road_trip_more_channels\\'
    os.chdir(file_dir)
except:
    file_dir = 'C:\\Users\\ning\\Downloads\\road_trip\\'
#    file_dir = 'C:\\Users\\ning\\Downloads\\road_trip_more_channels\\'
    os.chdir(file_dir)
################################### Random forest #################################
signal_features_indivisual_results,graph_features_indivisual_results,combine_features_indivisual_results={},{},{}
signal_features_indivisual_results = eegPipelineFunctions.cross_validation_report(signal_features_indivisual_results,0,
                                                                                clf_='RF',file_dir=file_dir,compute='signal')
graph_features_indivisual_results = eegPipelineFunctions.cross_validation_report(graph_features_indivisual_results,0,
                                                                                clf_='RF',file_dir=file_dir,compute='graph')
combine_features_indivisual_results = eegPipelineFunctions.cross_validation_report(combine_features_indivisual_results,0,
                                                                                clf_='RF',file_dir=file_dir,compute='combine')
signal_features_indivisual_results.to_csv(file_dir+'individual_signal_feature_RF.csv',index=False)
graph_features_indivisual_results.to_csv(file_dir+'individual_graph_feature_RF.csv',index=False)
combine_features_indivisual_results.to_csv(file_dir+'individual_combine_feature_RF.csv',index=False)
################################### xgb #################################
signal_features_indivisual_results,graph_features_indivisual_results,combine_features_indivisual_results={},{},{}
signal_features_indivisual_results = eegPipelineFunctions.cross_validation_report(signal_features_indivisual_results,0,
                                                                                clf_='xgb',file_dir=file_dir,compute='signal')
graph_features_indivisual_results = eegPipelineFunctions.cross_validation_report(graph_features_indivisual_results,0,
                                                                                clf_='xgb',file_dir=file_dir,compute='graph')
combine_features_indivisual_results = eegPipelineFunctions.cross_validation_report(combine_features_indivisual_results,0,
                                                                                clf_='xgb',file_dir=file_dir,compute='combine')
signal_features_indivisual_results.to_csv(file_dir+'individual_signal_feature_xgb.csv',index=False)
graph_features_indivisual_results.to_csv(file_dir+'individual_graph_feature_xgb.csv',index=False)
combine_features_indivisual_results.to_csv(file_dir+'individual_combine_feature_xgb.csv',index=False)
################################### support vector machine ###########################
signal_features_indivisual_results,graph_features_indivisual_results,combine_features_indivisual_results={},{},{}
signal_features_indivisual_results = eegPipelineFunctions.cross_validation_report(signal_features_indivisual_results,0,
                                                                                clf_='svm',file_dir=file_dir,compute='signal',n_estimators=1)
graph_features_indivisual_results = eegPipelineFunctions.cross_validation_report(graph_features_indivisual_results,0,
                                                                                clf_='svm',file_dir=file_dir,compute='graph',n_estimators=1)
combine_features_indivisual_results = eegPipelineFunctions.cross_validation_report(combine_features_indivisual_results,0,
                                                                                clf_='svm',file_dir=file_dir,compute='combine',n_estimators=1)
signal_features_indivisual_results.to_csv(file_dir+'individual_signal_feature_svm.csv',index=False)
graph_features_indivisual_results.to_csv(file_dir+'individual_graph_feature_svm.csv',index=False)
combine_features_indivisual_results.to_csv(file_dir+'individual_combine_feature_svm.csv',index=False)
################################## logistic regression ##################################################
signal_features_indivisual_results,graph_features_indivisual_results,combine_features_indivisual_results={},{},{}
signal_features_indivisual_results = eegPipelineFunctions.cross_validation_report(signal_features_indivisual_results,0,
                                                                                clf_='logistic',file_dir=file_dir,compute='signal')
graph_features_indivisual_results = eegPipelineFunctions.cross_validation_report(graph_features_indivisual_results,0,
                                                                                clf_='logistic',file_dir=file_dir,compute='graph')
combine_features_indivisual_results = eegPipelineFunctions.cross_validation_report(combine_features_indivisual_results,0,
                                                                                clf_='logistic',file_dir=file_dir,compute='combine')
signal_features_indivisual_results.to_csv(file_dir+'individual_signal_feature_logistic.csv',index=False)
graph_features_indivisual_results.to_csv(file_dir+'individual_graph_feature_logistic.csv',index=False)
combine_features_indivisual_results.to_csv(file_dir+'individual_combine_feature_logistic.csv',index=False)
################################### knn #################################
signal_features_indivisual_results,graph_features_indivisual_results,combine_features_indivisual_results={},{},{}
signal_features_indivisual_results = eegPipelineFunctions.cross_validation_report(signal_features_indivisual_results,0,
                                                                                clf_='knn',file_dir=file_dir,compute='signal',n_estimators=15)
graph_features_indivisual_results = eegPipelineFunctions.cross_validation_report(graph_features_indivisual_results,0,
                                                                                clf_='knn',file_dir=file_dir,compute='graph',n_estimators=15)
combine_features_indivisual_results = eegPipelineFunctions.cross_validation_report(combine_features_indivisual_results,0,
                                                                                clf_='knn',file_dir=file_dir,compute='combine',n_estimators=15)
signal_features_indivisual_results.to_csv(file_dir+'individual_signal_feature_knn.csv',index=False)
graph_features_indivisual_results.to_csv(file_dir+'individual_graph_feature_knn.csv',index=False)
combine_features_indivisual_results.to_csv(file_dir+'individual_combine_feature_knn.csv',index=False)
################################ TPOT ############################################
#from sklearn.pipeline import make_pipeline, make_union
#from sklearn.decomposition import PCA
#from sklearn.neighbors import KNeighborsClassifier
#from sklearn.preprocessing import FunctionTransformer,Normalizer
#from sklearn.naive_bayes import BernoulliNB,GaussianNB
#from copy import copy
#from sklearn.svm import LinearSVC
#from sklearn.ensemble import GradientBoostingClassifier, VotingClassifier
#from sklearn.linear_model import LogisticRegression
#from sklearn.tree import DecisionTreeClassifier
#from sklearn.feature_selection import VarianceThreshold,SelectPercentile, f_classif
#clfs_graph = {1.5:make_pipeline(
#    make_union(VotingClassifier([("est", GradientBoostingClassifier(max_depth=1,
#                                                                    max_features=0.2,
#                                                                    min_samples_leaf=5,
#                                                                    min_samples_split=2,
#                                                                    n_estimators=100,
#                                                                    subsample=0.45))]),
#                                                FunctionTransformer(copy)),
#                                                LogisticRegression(C=0.5)),
#        2.0:LogisticRegression(C=0.1, dual=False),
#        2.5:LinearSVC(C=0.001, loss="hinge", penalty="l2", tol=0.1),
#        3.0:make_pipeline(make_union(
#                                    Normalizer(norm="max"),
#                                    FunctionTransformer(copy)),
#                                    KNeighborsClassifier(n_neighbors=95, p=1)),
#        3.5:LogisticRegression(C=5.0),
#        4.0:make_pipeline(make_union(VotingClassifier([("est", BernoulliNB(alpha=100.0,
#                                                                                fit_prior=True))]),
#                                                FunctionTransformer(copy)),
#                                                LogisticRegression(C=0.5, penalty="l2")),
#        4.5:LogisticRegression(),
#        5.0:LogisticRegression(C=0.5, dual=False, penalty="l2")}
#
#clfs_signal = {1.5:make_pipeline(VarianceThreshold(threshold=0.5),
#                                                DecisionTreeClassifier(criterion="entropy",
#                                                                       max_depth=3,
#                                                                       min_samples_leaf=8,
#                                                                       min_samples_split=6)),
#        2.0:make_pipeline(make_union(VotingClassifier([("est", GaussianNB())]), FunctionTransformer(copy)),
#                                                                     LinearSVC(C=15.0, loss="hinge", penalty="l2", tol=0.01)),
#        2.5:LogisticRegression(C=0.01, dual=True),
#        3.0:LinearSVC(dual=True, loss="hinge", penalty="l2", tol=0.001),
#        3.5:LinearSVC(C=1.0, dual=True, loss="hinge", penalty="l2", tol=0.1),
#        4.0:make_pipeline(VarianceThreshold(threshold=0.9),
#                                         make_union(VotingClassifier([("est", GaussianNB())]),
#                                                                     FunctionTransformer(copy)),
#                                                                     KNeighborsClassifier(n_neighbors=100, p=1)),
#        4.5:make_pipeline(SelectPercentile(score_func=f_classif, percentile=33),
#                                         LogisticRegression(C=20.0, dual=False)),
#        5.0:make_pipeline(PCA(iterated_power=7, svd_solver="randomized"),GaussianNB())}
#
#signal_features_indivisual_results = {'subject':[],'day':[],'epoch_length':[],
#                                      'auc_score_mean':[],'auc_score_std':[],
#                                      'fpr':[],'tpr':[],
#                                      'precision':[],'recall':[],
#                                      'precision_mean':[],'precision_std':[],
#                                      'recall_mean':[],'recall_std':[],
#                                      'area_under_precision_recall':[],
#                      'matthews_corrcoef_mean':[],
#                      'matthews_corrcoef_std':[]}
#graph_features_indivisual_results = {'subject':[],'day':[],'epoch_length':[],
#                                      'auc_score_mean':[],'auc_score_std':[],
#                                      'fpr':[],'tpr':[],
#                                      'precision':[],'recall':[],
#                                      'precision_mean':[],'precision_std':[],
#                                      'recall_mean':[],'recall_std':[],
#                                      'area_under_precision_recall':[],
#                      'matthews_corrcoef_mean':[],
#                      'matthews_corrcoef_std':[]}
#combine_features_indivisual_results = {'subject':[],'day':[],'epoch_length':[],
#                                      'auc_score_mean':[],'auc_score_std':[],
#                                      'fpr':[],'tpr':[],
#                                      'precision':[],'recall':[],
#                                      'precision_mean':[],'precision_std':[],
#                                      'recall_mean':[],'recall_std':[],
#                                      'area_under_precision_recall':[],
#                      'matthews_corrcoef_mean':[],
#                      'matthews_corrcoef_std':[]}
#for directory_1 in [f for f in os.listdir(file_dir) if ('epoch_length' in f)]:
#    sub_dir = file_dir + directory_1 + '\\'
#    epoch_length = directory_1.split(' ')[1]
#    os.chdir(sub_dir)
#    #signal_features_indivisual_results[directory_1],graph_features_indivisual_results[directory_1]={},{}
#    #df_cc, df_pli, df_plv, df_signal,df_graph = [],[],[],[],[]
#    for sub_fold in os.listdir(sub_dir):
#        sub_fold_dir = sub_dir + sub_fold + '\\'
#        os.chdir(sub_fold_dir)
#        sub = sub_fold[:-4]
#        day = sub_fold[4:][-4:]
#        print(sub,day,epoch_length)
#
#        cc_features, pli_features, plv_features, signal_features = [pd.read_csv(f) for f in os.listdir(sub_fold_dir) if ('csv' in f)]
#        #df_cc.append(cc_features)
#        #df_pli.append(pli_features)
#        #df_plv.append(plv_features)
#        label = cc_features['label']
#        cc_features = eegPipelineFunctions.get_real_part(cc_features)
#        pli_features = eegPipelineFunctions.get_real_part(pli_features)
#        plv_features = eegPipelineFunctions.get_real_part(plv_features)
#        cc_features.columns = ['cc_'+name for name in cc_features]
#        pli_features.columns = ['pli_'+name for name in pli_features]
#        plv_features.columns = ['plv_'+name for name in plv_features]
#        cc_features = cc_features.drop('cc_label',1)
#        pli_features = pli_features.drop('pli_label',1)
#        plv_features = plv_features.drop('plv_label',1)
#        df_combine = pd.concat([cc_features,pli_features,plv_features],axis=1)
#        df_combine['label']=label
#        df_two = pd.concat([cc_features, pli_features, plv_features, signal_features],axis=1)
#        try:
#            signal_temp = eegPipelineFunctions.cross_validation_with_clfs(signal_features,clf_=clfs_signal[float(epoch_length)])
#            graph_temp = eegPipelineFunctions.cross_validation_with_clfs(df_combine,clf_=clfs_graph[float(epoch_length)])
#            two_temp = eegPipelineFunctions.cross_validation_with_clfs(df_two,clf_=clfs_graph[float(epoch_length)])
#            auc_score,fpr,tpr,precision,recall,precision_scores,recall_scores,average_scores,MCC=signal_temp
#            signal_features_indivisual_results['auc_score_mean'].append(np.nanmean(auc_score))
#            signal_features_indivisual_results['auc_score_std'].append(np.std(auc_score))
#            signal_features_indivisual_results['fpr'].append(fpr)
#            signal_features_indivisual_results['tpr'].append(tpr)
#            signal_features_indivisual_results['precision'].append(precision)
#            signal_features_indivisual_results['recall'].append(recall)
#            signal_features_indivisual_results['precision_mean'].append(np.nanmean(precision_scores))
#            signal_features_indivisual_results['precision_std'].append(np.std(precision_scores))
#            signal_features_indivisual_results['recall_mean'].append(np.nanmean(recall_scores))
#            signal_features_indivisual_results['recall_std'].append(np.std(recall_scores))
#            signal_features_indivisual_results['area_under_precision_recall'].append(average_scores)
#            signal_features_indivisual_results['matthews_corrcoef_mean'].append(np.nanmean(MCC))
#            signal_features_indivisual_results['matthews_corrcoef_std'].append(np.nanstd(MCC))
#            signal_features_indivisual_results['subject'].append(sub)
#            signal_features_indivisual_results['day'].append(int(day[-1]))
#            signal_features_indivisual_results['epoch_length'].append(float(epoch_length))
#            print(sub_fold,Counter(label),'signal:%.2f +/-%.2f'%(np.nanmean(auc_score),np.std(auc_score)))
#            auc_score,fpr,tpr,precision,recall,precision_scores,recall_scores,average_scores,MCC=graph_temp
#            graph_features_indivisual_results['auc_score_mean'].append(np.nanmean(auc_score))
#            graph_features_indivisual_results['auc_score_std'].append(np.std(auc_score))
#            graph_features_indivisual_results['fpr'].append(fpr)
#            graph_features_indivisual_results['tpr'].append(tpr)
#            graph_features_indivisual_results['precision'].append(precision)
#            graph_features_indivisual_results['recall'].append(recall)
#            graph_features_indivisual_results['precision_mean'].append(np.nanmean(precision_scores))
#            graph_features_indivisual_results['precision_std'].append(np.std(precision_scores))
#            graph_features_indivisual_results['recall_mean'].append(np.nanmean(recall_scores))
#            graph_features_indivisual_results['recall_std'].append(np.std(recall_scores))
#            graph_features_indivisual_results['area_under_precision_recall'].append(average_scores)
#            graph_features_indivisual_results['matthews_corrcoef_mean'].append(np.nanmean(MCC))
#            graph_features_indivisual_results['matthews_corrcoef_std'].append(np.nanstd(MCC))
#            graph_features_indivisual_results['subject'].append(sub)
#            graph_features_indivisual_results['day'].append(int(day[-1]))
#            graph_features_indivisual_results['epoch_length'].append(float(epoch_length))
#            print(sub_fold,Counter(label),'graph:%.2f +/-%.2f'%(np.nanmean(auc_score),np.std(auc_score)))
#            auc_score,fpr,tpr,precision,recall,precision_scores,recall_scores,average_scores,MCC=two_temp
#            combine_features_indivisual_results['auc_score_mean'].append(np.nanmean(auc_score))
#            combine_features_indivisual_results['auc_score_std'].append(np.std(auc_score))
#            combine_features_indivisual_results['fpr'].append(fpr)
#            combine_features_indivisual_results['tpr'].append(tpr)
#            combine_features_indivisual_results['precision'].append(precision)
#            combine_features_indivisual_results['recall'].append(recall)
#            combine_features_indivisual_results['precision_mean'].append(np.nanmean(precision_scores))
#            combine_features_indivisual_results['precision_std'].append(np.std(precision_scores))
#            combine_features_indivisual_results['recall_mean'].append(np.nanmean(recall_scores))
#            combine_features_indivisual_results['recall_std'].append(np.std(recall_scores))
#            combine_features_indivisual_results['area_under_precision_recall'].append(average_scores)
#            combine_features_indivisual_results['matthews_corrcoef_mean'].append(np.nanmean(MCC))
#            combine_features_indivisual_results['matthews_corrcoef_std'].append(np.nanstd(MCC))
#            combine_features_indivisual_results['subject'].append(sub)
#            combine_features_indivisual_results['day'].append(int(day[-1]))
#            combine_features_indivisual_results['epoch_length'].append(float(epoch_length))
#            print(sub_fold,Counter(label),'signal:%.2f +/-%.2f'%(np.nanmean(auc_score),np.std(auc_score)))
#        except:
#            print(sub_fold,Counter(label),'not enough samples')
#signal_features_indivisual_results = pd.DataFrame(signal_features_indivisual_results)
#graph_features_indivisual_results = pd.DataFrame(graph_features_indivisual_results)
#combine_features_indivisual_results = pd.DataFrame(combine_features_indivisual_results)
#signal_features_indivisual_results.to_csv(file_dir+'individual_signal_feature_TPOT.csv',index=False)
#graph_features_indivisual_results.to_csv(file_dir+'individual_graph_feature_TPOT.csv',index=False)
#combine_features_indivisual_results.to_csv(file_dir+'individual_combine_feature_TPOT.csv',index=False)

#signal_features_indivisual_results = {'subject':[],'day':[],'epoch_length':[],
#                                      'auc_score_mean':[],'auc_score_std':[],
#                                      'fpr':[],'tpr':[],
#                                      'precision':[],'recall':[],
#                                      'precision_mean':[],'precision_std':[],
#                                      'recall_mean':[],'recall_std':[],
#                                      'area_under_precision_recall':[]}
#graph_features_indivisual_results = {'subject':[],'day':[],'epoch_length':[],
#                                      'auc_score_mean':[],'auc_score_std':[],
#                                      'fpr':[],'tpr':[],
#                                      'precision':[],'recall':[],
#                                      'precision_mean':[],'precision_std':[],
#                                      'recall_mean':[],'recall_std':[],
#                                      'area_under_precision_recall':[]}
#for directory_1 in [f for f in os.listdir(file_dir) if ('epoch_length' in f)]:
#    sub_dir = file_dir + directory_1 + '\\'
#    epoch_length = directory_1.split(' ')[1]
#    os.chdir(sub_dir)
#    #signal_features_indivisual_results[directory_1],graph_features_indivisual_results[directory_1]={},{}
#    #df_cc, df_pli, df_plv, df_signal,df_graph = [],[],[],[],[]
#    for sub_fold in os.listdir(sub_dir):
#        sub_fold_dir = sub_dir + sub_fold + '\\'
#        os.chdir(sub_fold_dir)
#        sub = sub_fold[:-4]
#        day = sub_fold[4:][-4:]
#        print(sub,day,epoch_length)
#
#        cc_features, pli_features, plv_features, signal_features = [pd.read_csv(f) for f in os.listdir(sub_fold_dir) if ('csv' in f)]
#        #df_cc.append(cc_features)
#        #df_pli.append(pli_features)
#        #df_plv.append(plv_features)
#        label = cc_features['label']
#        cc_features = eegPipelineFunctions.get_real_part(cc_features)
#        pli_features = eegPipelineFunctions.get_real_part(pli_features)
#        plv_features = eegPipelineFunctions.get_real_part(plv_features)
#        cc_features.columns = ['cc_'+name for name in cc_features]
#        pli_features.columns = ['pli_'+name for name in pli_features]
#        plv_features.columns = ['plv_'+name for name in plv_features]
#        cc_features = cc_features.drop('cc_label',1)
#        pli_features = pli_features.drop('pli_label',1)
#        plv_features = plv_features.drop('plv_label',1)
#        df_combine = pd.concat([cc_features,pli_features,plv_features],axis=1)
#        df_combine['label']=label
#        try:
#            signal_temp = eegPipelineFunctions.cross_validation_with_clfs(signal_features,clf_='logistic')
#            graph_temp = eegPipelineFunctions.cross_validation_with_clfs(df_combine,clf_='logistic')
#            auc_score,fpr,tpr,precision,recall,precision_scores,recall_scores,average_scores=signal_temp
#            signal_features_indivisual_results['auc_score_mean'].append(np.nanmean(auc_score))
#            signal_features_indivisual_results['auc_score_std'].append(np.std(auc_score))
#            signal_features_indivisual_results['fpr'].append(fpr)
#            signal_features_indivisual_results['tpr'].append(tpr)
#            signal_features_indivisual_results['precision'].append(precision)
#            signal_features_indivisual_results['recall'].append(recall)
#            signal_features_indivisual_results['precision_mean'].append(np.nanmean(precision_scores))
#            signal_features_indivisual_results['precision_std'].append(np.std(precision_scores))
#            signal_features_indivisual_results['recall_mean'].append(np.nanmean(recall_scores))
#            signal_features_indivisual_results['recall_std'].append(np.std(recall_scores))
#            signal_features_indivisual_results['area_under_precision_recall'].append(average_scores)
#            signal_features_indivisual_results['subject'].append(sub)
#            signal_features_indivisual_results['day'].append(int(day[-1]))
#            signal_features_indivisual_results['epoch_length'].append(float(epoch_length))
#            print(sub_fold,Counter(label),'signal:%.2f +/-%.2f'%(np.nanmean(auc_score),np.std(auc_score)))
#            auc_score,fpr,tpr,precision,recall,precision_scores,recall_scores,average_scores=graph_temp
#            graph_features_indivisual_results['auc_score_mean'].append(np.nanmean(auc_score))
#            graph_features_indivisual_results['auc_score_std'].append(np.std(auc_score))
#            graph_features_indivisual_results['fpr'].append(fpr)
#            graph_features_indivisual_results['tpr'].append(tpr)
#            graph_features_indivisual_results['precision'].append(precision)
#            graph_features_indivisual_results['recall'].append(recall)
#            graph_features_indivisual_results['precision_mean'].append(np.nanmean(precision_scores))
#            graph_features_indivisual_results['precision_std'].append(np.std(precision_scores))
#            graph_features_indivisual_results['recall_mean'].append(np.nanmean(recall_scores))
#            graph_features_indivisual_results['recall_std'].append(np.std(recall_scores))
#            graph_features_indivisual_results['area_under_precision_recall'].append(average_scores)
#            graph_features_indivisual_results['subject'].append(sub)
#            graph_features_indivisual_results['day'].append(int(day[-1]))
#            graph_features_indivisual_results['epoch_length'].append(float(epoch_length))
#            print(sub_fold,Counter(label),'graph:%.2f +/-%.2f'%(np.nanmean(auc_score),np.std(auc_score)))
#        except:
#            print(sub_fold,Counter(label),'not enough samples')
#signal_features_indivisual_results = pd.DataFrame(signal_features_indivisual_results)
#graph_features_indivisual_results = pd.DataFrame(graph_features_indivisual_results)
#signal_features_indivisual_results.to_csv(file_dir+'individual_signal_feature_regression.csv',index=False)
#graph_features_indivisual_results.to_csv(file_dir+'individual_graph_feature_regression.csv',index=False)
#
#
#signal_features_indivisual_results = {'subject':[],'day':[],'epoch_length':[],
#                                      'auc_score_mean':[],'auc_score_std':[],
#                                      'fpr':[],'tpr':[],
#                                      'precision':[],'recall':[],
#                                      'precision_mean':[],'precision_std':[],
#                                      'recall_mean':[],'recall_std':[],
#                                      'area_under_precision_recall':[]}
#graph_features_indivisual_results = {'subject':[],'day':[],'epoch_length':[],
#                                      'auc_score_mean':[],'auc_score_std':[],
#                                      'fpr':[],'tpr':[],
#                                      'precision':[],'recall':[],
#                                      'precision_mean':[],'precision_std':[],
#                                      'recall_mean':[],'recall_std':[],
#                                      'area_under_precision_recall':[]}
#combine_features_indivisual_results = {'subject':[],'day':[],'epoch_length':[],
#                                      'auc_score_mean':[],'auc_score_std':[],
#                                      'fpr':[],'tpr':[],
#                                      'precision':[],'recall':[],
#                                      'precision_mean':[],'precision_std':[],
#                                      'recall_mean':[],'recall_std':[],
#                                      'area_under_precision_recall':[]}
#for directory_1 in [f for f in os.listdir(file_dir) if ('epoch_length' in f)]:
#    sub_dir = file_dir + directory_1 + '\\'
#    epoch_length = directory_1.split(' ')[1]
#    os.chdir(sub_dir)
#    #signal_features_indivisual_results[directory_1],graph_features_indivisual_results[directory_1]={},{}
#    #df_cc, df_pli, df_plv, df_signal,df_graph = [],[],[],[],[]
#    for sub_fold in os.listdir(sub_dir):
#        sub_fold_dir = sub_dir + sub_fold + '\\'
#        os.chdir(sub_fold_dir)
#        sub = sub_fold[:-4]
#        day = sub_fold[4:][-4:]
#        print(sub,day,epoch_length)
#        cc_features, pli_features, plv_features, signal_features = [pd.read_csv(f) for f in os.listdir(sub_fold_dir) if ('csv' in f)]
#        #df_cc.append(cc_features)
#        #df_pli.append(pli_features)
#        #df_plv.append(plv_features)
#        label = cc_features['label']
#        cc_features = eegPipelineFunctions.get_real_part(cc_features)
#        pli_features = eegPipelineFunctions.get_real_part(pli_features)
#        plv_features = eegPipelineFunctions.get_real_part(plv_features)
#        cc_features.columns = ['cc_'+name for name in cc_features]
#        pli_features.columns = ['pli_'+name for name in pli_features]
#        plv_features.columns = ['plv_'+name for name in plv_features]
#        cc_features = cc_features.drop('cc_label',1)
#        pli_features = pli_features.drop('pli_label',1)
#        plv_features = plv_features.drop('plv_label',1)
#        df_combine = pd.concat([cc_features,pli_features,plv_features],axis=1)
#        df_combine['label']=label
#        df_two = pd.concat([cc_features, pli_features, plv_features, signal_features],axis=1)
#        try:
#            signal_temp = eegPipelineFunctions.cross_validation_with_clfs(signal_features,clf_='RF')
#            graph_temp = eegPipelineFunctions.cross_validation_with_clfs(df_combine,clf_='RF')
#            two_temp = eegPipelineFunctions.cross_validation_with_clfs(df_two,clf_='RF')
#            auc_score,fpr,tpr,precision,recall,precision_scores,recall_scores,average_scores,MCC=signal_temp
#            signal_features_indivisual_results['auc_score_mean'].append(np.nanmean(auc_score))
#            signal_features_indivisual_results['auc_score_std'].append(np.std(auc_score))
#            signal_features_indivisual_results['fpr'].append(fpr)
#            signal_features_indivisual_results['tpr'].append(tpr)
#            signal_features_indivisual_results['precision'].append(precision)
#            signal_features_indivisual_results['recall'].append(recall)
#            signal_features_indivisual_results['precision_mean'].append(np.nanmean(precision_scores))
#            signal_features_indivisual_results['precision_std'].append(np.std(precision_scores))
#            signal_features_indivisual_results['recall_mean'].append(np.nanmean(recall_scores))
#            signal_features_indivisual_results['recall_std'].append(np.std(recall_scores))
#            signal_features_indivisual_results['area_under_precision_recall'].append(average_scores)
#            signal_features_indivisual_results['subject'].append(sub)
#            signal_features_indivisual_results['day'].append(int(day[-1]))
#            signal_features_indivisual_results['epoch_length'].append(float(epoch_length))
#            print(sub_fold,Counter(label),'signal:%.2f +/-%.2f'%(np.nanmean(MCC),np.std(MCC)))
#            auc_score,fpr,tpr,precision,recall,precision_scores,recall_scores,average_scores=graph_temp
#            graph_features_indivisual_results['auc_score_mean'].append(np.nanmean(auc_score))
#            graph_features_indivisual_results['auc_score_std'].append(np.std(auc_score))
#            graph_features_indivisual_results['fpr'].append(fpr)
#            graph_features_indivisual_results['tpr'].append(tpr)
#            graph_features_indivisual_results['precision'].append(precision)
#            graph_features_indivisual_results['recall'].append(recall)
#            graph_features_indivisual_results['precision_mean'].append(np.nanmean(precision_scores))
#            graph_features_indivisual_results['precision_std'].append(np.std(precision_scores))
#            graph_features_indivisual_results['recall_mean'].append(np.nanmean(recall_scores))
#            graph_features_indivisual_results['recall_std'].append(np.std(recall_scores))
#            graph_features_indivisual_results['area_under_precision_recall'].append(average_scores)
#            graph_features_indivisual_results['subject'].append(sub)
#            graph_features_indivisual_results['day'].append(int(day[-1]))
#            graph_features_indivisual_results['epoch_length'].append(float(epoch_length))
#            print(sub_fold,Counter(label),'graph:%.2f +/-%.2f'%(np.nanmean(auc_score),np.std(auc_score)))
#            auc_score,fpr,tpr,precision,recall,precision_scores,recall_scores,average_scores=two_temp
#            combine_features_indivisual_results['auc_score_mean'].append(np.nanmean(auc_score))
#            combine_features_indivisual_results['auc_score_std'].append(np.std(auc_score))
#            combine_features_indivisual_results['fpr'].append(fpr)
#            combine_features_indivisual_results['tpr'].append(tpr)
#            combine_features_indivisual_results['precision'].append(precision)
#            combine_features_indivisual_results['recall'].append(recall)
#            combine_features_indivisual_results['precision_mean'].append(np.nanmean(precision_scores))
#            combine_features_indivisual_results['precision_std'].append(np.std(precision_scores))
#            combine_features_indivisual_results['recall_mean'].append(np.nanmean(recall_scores))
#            combine_features_indivisual_results['recall_std'].append(np.std(recall_scores))
#            combine_features_indivisual_results['area_under_precision_recall'].append(average_scores)
#            combine_features_indivisual_results['subject'].append(sub)
#            combine_features_indivisual_results['day'].append(int(day[-1]))
#            combine_features_indivisual_results['epoch_length'].append(float(epoch_length))
#            print(sub_fold,Counter(label),'signal:%.2f +/-%.2f'%(np.nanmean(auc_score),np.std(auc_score)))
#        except:
#            print(sub_fold,Counter(label),'not enough samples')
#signal_features_indivisual_results = pd.DataFrame(signal_features_indivisual_results)
#graph_features_indivisual_results = pd.DataFrame(graph_features_indivisual_results)
#signal_features_indivisual_results.to_csv(file_dir+'individual_signal_feature_RF.csv',index=False)
#graph_features_indivisual_results.to_csv(file_dir+'individual_graph_feature_RF.csv',index=False)
##pickle.dump(signal_features_indivisual_results,open(file_dir+'individual_signal_feature_RF.p','wb'))
##pickle.dump(graph_features_indivisual_results,open(file_dir+'individual_graph_feature_RF.p','wb'))
#
#
#signal_features_indivisual_results = {'subject':[],'day':[],'epoch_length':[],
#                                      'auc_score_mean':[],'auc_score_std':[],
#                                      'fpr':[],'tpr':[],
#                                      'precision':[],'recall':[],
#                                      'precision_mean':[],'precision_std':[],
#                                      'recall_mean':[],'recall_std':[],
#                                      'area_under_precision_recall':[]}
#graph_features_indivisual_results = {'subject':[],'day':[],'epoch_length':[],
#                                      'auc_score_mean':[],'auc_score_std':[],
#                                      'fpr':[],'tpr':[],
#                                      'precision':[],'recall':[],
#                                      'precision_mean':[],'precision_std':[],
#                                      'recall_mean':[],'recall_std':[],
#                                      'area_under_precision_recall':[]}
#for directory_1 in [f for f in os.listdir(file_dir) if ('epoch_length' in f)]:
#    sub_dir = file_dir + directory_1 + '\\'
#    epoch_length = directory_1.split(' ')[1]
#    os.chdir(sub_dir)
#    #signal_features_indivisual_results[directory_1],graph_features_indivisual_results[directory_1]={},{}
#    #df_cc, df_pli, df_plv, df_signal,df_graph = [],[],[],[],[]
#    for sub_fold in os.listdir(sub_dir):
#        sub_fold_dir = sub_dir + sub_fold + '\\'
#        os.chdir(sub_fold_dir)
#        sub = sub_fold[:-4]
#        day = sub_fold[4:][-4:]
#        print(sub,day,epoch_length)
#
#        cc_features, pli_features, plv_features, signal_features = [pd.read_csv(f) for f in os.listdir(sub_fold_dir) if ('csv' in f)]
#        #df_cc.append(cc_features)
#        #df_pli.append(pli_features)
#        #df_plv.append(plv_features)
#        label = cc_features['label']
#        cc_features = eegPipelineFunctions.get_real_part(cc_features)
#        pli_features = eegPipelineFunctions.get_real_part(pli_features)
#        plv_features = eegPipelineFunctions.get_real_part(plv_features)
#        cc_features.columns = ['cc_'+name for name in cc_features]
#        pli_features.columns = ['pli_'+name for name in pli_features]
#        plv_features.columns = ['plv_'+name for name in plv_features]
#        cc_features = cc_features.drop('cc_label',1)
#        pli_features = pli_features.drop('pli_label',1)
#        plv_features = plv_features.drop('plv_label',1)
#        df_combine = pd.concat([cc_features,pli_features,plv_features],axis=1)
#        df_combine['label']=label
#        try:
#            signal_temp = eegPipelineFunctions.cross_validation_with_clfs(signal_features,clf_='svm')
#            graph_temp = eegPipelineFunctions.cross_validation_with_clfs(df_combine,clf_='svm')
#            auc_score,fpr,tpr,precision,recall,precision_scores,recall_scores,average_scores=signal_temp
#            signal_features_indivisual_results['auc_score_mean'].append(np.nanmean(auc_score))
#            signal_features_indivisual_results['auc_score_std'].append(np.std(auc_score))
#            signal_features_indivisual_results['fpr'].append(fpr)
#            signal_features_indivisual_results['tpr'].append(tpr)
#            signal_features_indivisual_results['precision'].append(precision)
#            signal_features_indivisual_results['recall'].append(recall)
#            signal_features_indivisual_results['precision_mean'].append(np.nanmean(precision_scores))
#            signal_features_indivisual_results['precision_std'].append(np.std(precision_scores))
#            signal_features_indivisual_results['recall_mean'].append(np.nanmean(recall_scores))
#            signal_features_indivisual_results['recall_std'].append(np.std(recall_scores))
#            signal_features_indivisual_results['area_under_precision_recall'].append(average_scores)
#            signal_features_indivisual_results['subject'].append(sub)
#            signal_features_indivisual_results['day'].append(int(day[-1]))
#            signal_features_indivisual_results['epoch_length'].append(float(epoch_length))
#            print(sub_fold,Counter(label),'signal:%.2f +/-%.2f'%(np.nanmean(auc_score),np.std(auc_score)))
#            auc_score,fpr,tpr,precision,recall,precision_scores,recall_scores,average_scores=graph_temp
#            graph_features_indivisual_results['auc_score_mean'].append(np.nanmean(auc_score))
#            graph_features_indivisual_results['auc_score_std'].append(np.std(auc_score))
#            graph_features_indivisual_results['fpr'].append(fpr)
#            graph_features_indivisual_results['tpr'].append(tpr)
#            graph_features_indivisual_results['precision'].append(precision)
#            graph_features_indivisual_results['recall'].append(recall)
#            graph_features_indivisual_results['precision_mean'].append(np.nanmean(precision_scores))
#            graph_features_indivisual_results['precision_std'].append(np.std(precision_scores))
#            graph_features_indivisual_results['recall_mean'].append(np.nanmean(recall_scores))
#            graph_features_indivisual_results['recall_std'].append(np.std(recall_scores))
#            graph_features_indivisual_results['area_under_precision_recall'].append(average_scores)
#            graph_features_indivisual_results['subject'].append(sub)
#            graph_features_indivisual_results['day'].append(int(day[-1]))
#            graph_features_indivisual_results['epoch_length'].append(float(epoch_length))
#            print(sub_fold,Counter(label),'graph:%.2f +/-%.2f'%(np.nanmean(auc_score),np.std(auc_score)))
#        except:
#            print(sub_fold,Counter(label),'not enough samples')
#signal_features_indivisual_results = pd.DataFrame(signal_features_indivisual_results)
#graph_features_indivisual_results = pd.DataFrame(graph_features_indivisual_results)
#signal_features_indivisual_results.to_csv(file_dir+'individual_signal_feature_svm.csv',index=False)
#graph_features_indivisual_results.to_csv(file_dir+'individual_graph_feature_svm.csv',index=False)


text:
# Copyright (c) Facebook, Inc. and its affiliates.

# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import numpy as np
import os
import torch

from foundations import paths
import models.registry
from pruning.mask import Mask
from testing import test_case

class TestMask(test_case.TestCase):
    def test_dict_behavior(self):
        m = Mask()
        self.assertEqual(len(m), 0)
        self.assertEqual(len(m.keys()), 0)
        self.assertEqual(len(m.values()), 0)

        m['hello'] = np.ones([2, 3])
        m['world'] = np.zeros([5, 6])
        self.assertEqual(len(m), 2)
        self.assertEqual(len(m.keys()), 2)
        self.assertEqual(len(m.values()), 2)
        self.assertEqual(set(m.keys()), set(['hello', 'world']))
        self.assertTrue(np.array_equal(np.ones([2, 3]), m['hello']))
        self.assertTrue(np.array_equal(np.zeros([5, 6]), m['world']))

        del m['hello']
        self.assertEqual(len(m), 1)
        self.assertEqual(len(m.keys()), 1)
        self.assertEqual(len(m.values()), 1)
        self.assertEqual(set(m.keys()), set(['world']))
        self.assertTrue(np.array_equal(np.zeros([5, 6]), m['world']))

    def test_create_mask_from_dict(self):
        m = Mask({'hello': np.ones([2, 3]), 'world': np.zeros([5, 6])})
        self.assertEqual(len(m), 2)
        self.assertEqual(len(m.keys()), 2)
        self.assertEqual(len(m.values()), 2)
        self.assertEqual(set(m.keys()), set(['hello', 'world']))
        self.assertTrue(np.array_equal(np.ones([2, 3]), m['hello']))
        self.assertTrue(np.array_equal(np.zeros([5, 6]), m['world']))

    def test_create_from_tensor(self):
        m = Mask({'hello': torch.ones([2, 3]), 'world': torch.zeros([5, 6])})
        self.assertEqual(len(m), 2)
        self.assertEqual(len(m.keys()), 2)
        self.assertEqual(len(m.values()), 2)
        self.assertEqual(set(m.keys()), set(['hello', 'world']))
        self.assertTrue(np.array_equal(np.ones([2, 3]), m['hello']))
        self.assertTrue(np.array_equal(np.zeros([5, 6]), m['world']))

    def test_bad_inputs(self):
        m = Mask()

        with self.assertRaises(ValueError):
            m[''] = np.ones([2, 3])

        with self.assertRaises(ValueError):
            m[6] = np.ones([2, 3])

        with self.assertRaises(ValueError):
            m['hello'] = [[0, 1], [1, 0]]

        with self.assertRaises(ValueError):
            m['hello'] = np.array([[0, 1], [2, 0]])

    def test_ones_like(self):
        model = models.registry.get(models.registry.get_default_hparams('cifar_resnet_20').model_hparams)
        m = Mask.ones_like(model)

        for k, v in model.state_dict().items():
            if k in model.prunable_layer_names:
                self.assertIn(k, m)
                self.assertEqual(list(m[k].shape), list(v.shape))
                self.assertTrue((m[k] == 1).all())
            else:
                self.assertNotIn(k, m)

    def test_save_load_exists(self):
        self.assertFalse(Mask.exists(self.root))
        self.assertFalse(os.path.exists(paths.mask(self.root)))

        m = Mask({'hello': np.ones([2, 3]), 'world': np.zeros([5, 6])})
        m.save(self.root)
        self.assertTrue(os.path.exists(paths.mask(self.root)))
        self.assertTrue(Mask.exists(self.root))

        m2 = Mask.load(self.root)
        self.assertEqual(len(m2), 2)
        self.assertEqual(len(m2.keys()), 2)
        self.assertEqual(len(m2.values()), 2)
        self.assertEqual(set(m2.keys()), set(['hello', 'world']))
        self.assertTrue(np.array_equal(np.ones([2, 3]), m2['hello']))
        self.assertTrue(np.array_equal(np.zeros([5, 6]), m2['world']))

test_case.main()

ext: py

reason: import numpy


text:
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt

plt.ion()
n_observations = 100
fig, ax = plt.subplots(1, 1)
xs = np.linspace(-3, 3, n_observations)
ys = np.sin(xs) + np.random.uniform(-0.5, 0.5, n_observations)
ax.scatter(xs, ys)
fig.show()
plt.draw()

# tf.placeholders for the input and output of the network. Placeholders are
# variables which we need to fill in when we are ready to compute the graph.
X = tf.placeholder(tf.float32)
Y = tf.placeholder(tf.float32)

Y_pred = tf.Variable(tf.random_normal([1]), name='bias')
for pow_i in range(1, 5):
    W = tf.Variable(tf.random_normal([1]), name='weight_%d' % pow_i)
    Y_pred = tf.add(tf.multiply(tf.pow(X, pow_i), W), Y_pred)

cost = tf.reduce_sum(tf.pow(Y_pred - Y, 2)) / (n_observations - 1)

learning_rate = 0.01
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)

n_epochs = 1000
with tf.Session() as sess:
    # Here we tell tensorflow that we want to initialize all
    # the variables in the graph so we can use them
    sess.run(tf.global_variables_initializer())

    # Fit all training data
    prev_training_cost = 0.0
    for epoch_i in range(n_epochs):
        for (x, y) in zip(xs, ys):
            sess.run(optimizer, feed_dict={X: x, Y: y})

        training_cost = sess.run(
            cost, feed_dict={X: xs, Y: ys})
        print(training_cost)

        if epoch_i % 100 == 0:
            ax.plot(xs, Y_pred.eval(
                feed_dict={X: xs}, session=sess),
                    'k', alpha=epoch_i / n_epochs)
            fig.show()
            plt.draw()

        if np.abs(prev_training_cost - training_cost) < 0.000001:
            break
        prev_training_cost = training_cost
ax.set_ylim([-3, 3])
fig.show()
plt.waitforbuttonpress()

max_stars_repo_name: LieJiang/einstein


text:
import warnings
import numpy as np
from scipy import linalg

class Model(object):
    name = 'Model'
    status_need_for_eval = 0
    """ Base class for a model. Actual models should inherit from this class.

    In this class the functions that should be implemented by each model are
    defined.

    Attributes
    ----------
    name : str
        Name of the model.

    status : int
        Indicates the status of the model:
            -1 : Instance created. Not filled with values yet.
             0 : Filled with values
             1 : Filled with values and x0 set (optional level).
    """
    def __init__(self):
        self.status = -1
        self.has_background = False

    def initialize(self):
        """This function should be called with all needed values. To actually
        fill all the models with values.
        """
        self.status = 0

    def evaluate(self):
        """Evaluates the model.

        Actual implementation of this functions should return:
            vec_g     : Observable vector
            vec_f     : Solution vector
            vec_f_reg : Vector used in the regularization

        """
        if self.status < 0 and self.status_need_for_eval == 0:
            raise RuntimeError("Model has to be intilized. "
                               "Run 'model.initialize' first!")
        if self.status < 1 and self.status_need_for_eval == 1:
            raise RuntimeError("Model has to be intilized and x0 has to be"
                               "set. Run 'model.initialize' and "
                               "'model.set_x0' first!")

    def set_model_x0(self):
        """Some models need to be set up with a x0 for the model. For those .
        models the class parameter 'status_need_for_eval' should be set to 1.
        """
        if self.status < 0:
            raise RuntimeError("Model has to be intilized, before setting x0. "
                               "Run 'model.initialize' first!")
        self.status = 1

    def generate_fit_x0(self):
        """The model should be able to return resonable starting values
        for the fitter.
        """
        if self.status < 0 and self.status_need_for_eval == 0:
            raise RuntimeError("Model has to be intilized. "
                               "Run 'model.initialize' first!")
        if self.status < 1 and self.status_need_for_eval == 1:
            raise RuntimeError("Model has to be intilized and x0 has to be"
                               "set. Run 'model.initialize' and "
                               "'model.set_x0' first!")

    def generate_fit_bounds(self):
        """The model should be able to return resonable bounds for the fitter.
        """
        if self.status < 0 and self.status_need_for_eval == 0:
            raise RuntimeError("Model has to be intilized. "
                               "Run 'model.initialize' first!")
        if self.status < 1 and self.status_need_for_eval == 1:
            raise RuntimeError("Model has to be intilized and x0 has to be"
                               "set. Run 'model.initialize' and "
                               "'model.set_x0' first!")

    def add_background(self):
        self.has_background = True

    def remove_background(self):
        """Disables the background vector. A stored background vector is
        not deleted.
        """
        self.has_background = False

class LinearModel(Model):
    name = 'LinearModel'
    status_need_for_eval = 0
    """ Basic Linear model:
    g = A * f

    Attributes
    ----------
    name : str
        Name of the model.

    status : int
        Indicates the status of the model:
            -1 : Instance created. Not filled with values yet.
             0 : Filled with values

    dim_g :
        Dimension of the histogrammed observable vector.

    dim_f :
        Dimension of the histogrammed truth vector.

    range_obs : tuple (int, int)
        Tuple containing the lowest and highest bin number used in
        the digitized observable vector. For performance reasons it is
        assumed that all numbers between min and max are used.

    range_truth : tuple (int, int)
        Tuple containing the lowest and highest bin number used in
        the digitized truth vector. For performance reasons it is
        assumed that all numbers between min and max are used.

    A : numpy.array shape=(dim_g, dim_f)
        Response matrix.

    vec_b : numpy.array, shape=(dim_f)
        Observable vector for the background.

    has_background : boolean
        Indicator if self.vec_b should be added to the model evaluationg
    """
    def __init__(self):
        super(LinearModel, self).__init__()
        self.range_obs = None
        self.range_truth = None
        self.A = None
        self.dim_f = None
        self.dim_g = None
        self.vec_b = None

    def initialize(self, digitized_obs, digitized_truth, sample_weight=None):
        """

        """
        super(LinearModel, self).initialize()
        self.range_obs = (min(digitized_obs), max(digitized_obs))
        self.range_truth = (min(digitized_truth), max(digitized_truth))
        self.dim_f = self.range_truth[1] - self.range_truth[0] + 1
        self.dim_g = self.range_obs[1] - self.range_obs[0] + 1
        binning_g, binning_f = self.__generate_binning__()
        self.A = np.histogram2d(x=digitized_obs,
                                y=digitized_truth,
                                bins=(binning_g, binning_f),
                                weights=sample_weight)[0]
        M_norm = np.diag(1 / np.sum(self.A, axis=0))
        self.A = np.dot(self.A, M_norm)

    def evaluate(self, vec_fit):
        """Evaluating the model for a given vector f

        Parameters
        ----------
        vec_fit : numpy.array, shape=(dim_f,)
            Vector f for which the model should be evaluated.

        Returns
        -------
        vec_g : nump.array, shape=(dim_g,)
            Vector containing the number of events in observable space.
            If background was added the returned vector is A * vec_f + vec_b.

        vec_f : nump.array, shape=(dim_f,)
            Vector used to evaluate A * vec_f

        vec_f_reg : nump.array, shape=(dim_f,)
            Vector that should be passed to the regularization. For the
            BasisLinearModel it is identical to f.
        """
        super(LinearModel, self).evaluate()
        vec_g = np.dot(self.A, vec_fit)
        if self.has_background:
            vec_g += self.vec_b
        return vec_g, vec_fit, vec_fit

    def generate_fit_x0(self, vec_g):
        """Generates a default seed for the minimization.
        The default seed vec_f_0 is a uniform distribution with
        sum(vec_f_0) = sum(vec_g). If background is present the default seed
        is: sum(vec_f_0) = sum(vec_g) - sum(self.vec_b).

        Parameters
        ----------
        vec_g : np.array, shape=(dim_g)
            Observable vector which should be used to get the correct
            normalization for vec_f_0.

        Returns
        -------
        vec_f_0 : np.array, shape=(dim_f)
            Seed vector of a minimization.
        """
        super(LinearModel, self).generate_fit_x0()
        n = self.A.shape[1]
        if self.has_background:
            vec_f_0 = np.ones(n) * (np.sum(vec_g) - np.sum(self.vec_b)) / n
        else:
            vec_f_0 = np.ones(n) * np.sum(vec_g) / n
        return vec_f_0

    def generate_fit_bounds(self, vec_g):
        """Generates a bounds for a minimization.
        The bounds are (0, sum(vec_g)) without background and
        (0, sum(vec_g - self.vec_b)) with background. The bounds are for
        each fit parameter/entry in f.

        Parameters
        ----------
        vec_g : np.array, shape=(dim_g)
            Observable vector which should be used to get the correct
            upper bound

        Returns
        -------
        bounds : list, shape=(dim_f)
            List of tuples with the bounds.
        """
        super(LinearModel, self).generate_fit_bounds()
        n = self.A.shape[1]
        if self.has_background:
            n_events = np.sum(vec_g) - np.sum(self.vec_b)
        else:
            n_events = np.sum(vec_g)
        bounds = [(0, n_events)] * n
        return bounds

    def set_model_x0(self):
        """The LinearModel has no referenz model_x0.
        """
        super(LinearModel, self).set_model_x0()
        warnings.warn('\tx0 has no effect for {}'.format(self.name))

    def evaluate_condition(self, normalize=True):
        """Returns an ordered array of the singular values of matrix A.

        Parameters
        ----------
        normalize : boolean (optional)
            If True the singular values return relativ to the largest
            value.

        Returns
        -------
        S_values : np.array, shape=(dim_f)
            Ordered array of the singular values.
        """
        if self.status < 0:
            raise RuntimeError("Model has to be intilized. "
                               "Run 'model.initialize' first!")
        U, S_values, V = linalg.svd(self.A)
        S_values = S_values / S_values[0]
        return S_values

    def __generate_binning__(self):
        if self.status < 0:
            raise RuntimeError("Model has to be intilized. "
                               "Run 'model.initialize' first!")
        binning_obs = np.linspace(self.range_obs[0],
                                  self.range_obs[1] + 1,
                                  self.dim_g + 1)
        binning_truth = np.linspace(self.range_truth[0],
                                    self.range_truth[1] + 1,
                                    self.dim_f + 1)
        return binning_obs, binning_truth

    def generate_vectors(self, digitized_obs=None, digitized_truth=None):
        """Returns vec_g, vec_f for digitized values. Either f, g or both
        can be provided to the function.

        Parameters
        ----------
        digitized_obs : np.intarray (optional)
            Array with digitized values form the observable space

        digitized_truth : np.intarray (optinal)
            Array with digitized values for the sought-after quantity.

        Returns
        -------
        vec_g : None or np.array shape=(dim_g)
            None if no digitized_obs was provided otherwise the histrogram
            of digitized_obs.

        vec_f : None or np.array shape=(dim_f)
            None if no digitized_truth was provided otherwise the histrogram
            of digitized_truth.
        """
        binning_obs, binning_truth = self.__generate_binning__()
        if digitized_obs is not None:
            vec_g = np.histogram(digitized_obs, bins=binning_obs)[0]
        else:
            vec_g = None
        if digitized_truth is not None:
            vec_f = np.histogram(digitized_truth, bins=binning_truth)[0]
        else:
            vec_f = None
        return vec_g, vec_f

    def add_background(self, vec_b):
        """Adds a background vector to the model.

        Parameters
        ----------
        vec_b : numpy.array, shape=(dim_g)
            Vector g which is added to the model evaluation.
        """
        super(LinearModel, self).add_background()
        self.vec_b = vec_b

class BiasedLinearModel(LinearModel):
    name = 'BiasedLinearModel'
    status_need_for_eval = 1
    """Extense the LinearModel with an bias distribtuion model_x0.
    the vec_f is interpreted as element-wise multiple of the model_x0.

    g = A * (model_x0 * vec_fit)

    Internally the model_x0 is normalize in a way that
    vec_fit = [1.] * dim_f
    is transformed to
     vec_f = model_x0 / sum(model_x0) * sum(vec_g).

    Attributes
    ----------
    name : str
        Name of the model.

    model_x0 : np.array, shape=(vec_f)
        Distribtuion which is element-wise multiplied with the vec_fit.

    status : int
        Indicates the status of the model:
            -1 : Instance created. Not filled with values yet.
             0 : Filled with values

    dim_g :
        Dimension of the histogrammed observable vector.

    dim_f :
        Dimension of the histogrammed truth vector.

    range_obs : tuple (int, int)
        Tuple containing the lowest and highest bin number used in
        the digitized observable vector. For performance reasons it is
        assumed that all numbers between min and max are used.

    range_truth : tuple (int, int)
        Tuple containing the lowest and highest bin number used in
        the digitized truth vector. For performance reasons it is
        assumed that all numbers between min and max are used.

    A : numpy.array shape=(dim_g, dim_f)
        Response matrix.

    vec_b : numpy.array, shape=(dim_f)
        Observable vector for the background.

    has_background : boolean
        Indicator if self.vec_b should be added to the model evaluationg
    """
    def __init__(self):
        super(LinearModel, self).__init__()
        self.range_obs = None
        self.range_truth = None
        self.A = None
        self.dim_f = None
        self.dim_g = None
        self.model_x0 = None
        self.model_factor_ = 1.
        self.background_factor_ = 0.
        self.vec_b = None

    def evaluate(self, vec_fit):
        """Evaluating the model for a given vector f

        Parameters
        ----------
        vec_fit : numpy.array, shape=(dim_f,)
            Vector f for which the model should be evaluated.

        Returns
        -------
        vec_g : nump.array, shape=(dim_g,)
            Vector containing the number of events in observable space.
            If background was added the returned vector is A * vec_f + vec_b.

        vec_f : nump.array, shape=(dim_f,)
            Vector used to evaluate A * vec_f

        vec_f_reg : nump.array, shape=(dim_f,)
            Vector that should be passed to the regularization. For the
            BasisLinearModel it is identical to f.
        """
        vec_f = self.transform_vec_fit(vec_fit)
        vec_g, _, _ = super(BiasedLinearModel, self).evaluate(vec_f)
        return vec_g, vec_f, vec_fit

    def transform_vec_fit(self, vec_fit):
        """Transforms the fit vector to the actual vec_f which is e.g.
        used to evaluate the model.

        Parameters
        ----------
        vec_fit : np.array, shape=(dim_f)
            Vector which should be transformed into an acutal vec_f.

        Returns
        -------
        vec_f : np.array, shape=(dim_f)
            Vector in the space of the sought-after quantity.
        """
        eff_factor = self.model_factor_ - self.background_factor_
        vec_f = self.model_x0 * vec_fit * eff_factor
        return vec_f

    def generate_fit_x0(self, vec_g):
        """Generates a default seed for the minimization.
        The default seed vec_f_0 is a uniform distribution with
        sum(vec_f_0) = sum(vec_g). If background is present the default seed
        is: sum(vec_f_0) = sum(vec_g) - sum(self.vec_b).

        Parameters
        ----------
        vec_g : np.array, shape=(dim_g)
            Observable vector which should be used to get the correct
            normalization for vec_f_0.

        Returns
        -------
        vec_f_0 : np.array, shape=(dim_f)
            Seed vector of a minimization.
        """
        super(LinearModel, self).generate_fit_x0()
        return np.ones(self.dim_f)

    def generate_fit_bounds(self, vec_g):
        """Generates a bounds for a minimization.
        The bounds are (0, sum(vec_g)) without background and
        (0, sum(vec_g - self.vec_b)) with background. The bounds are for
        each fit parameter/entry in f.

        Parameters
        ----------
        vec_g : np.array, shape=(dim_g)
            Observable vector which should be used to get the correct
            upper bound

        Returns
        -------
        bounds : list, shape=(dim_f)
            List of tuples with the bounds.
        """
        super(LinearModel, self).generate_fit_bounds()
        n = self.A.shape[1]
        if self.has_background:
            n_events = np.sum(vec_g) - np.sum(self.vec_b)
        else:
            n_events = np.sum(vec_g)
        bounds = [(0, n_events)] * n
        return bounds

    def set_model_x0(self, model_x0, vec_g):
        """Sets the model_x0. Also the vec_g for the unfolding is need,
        to get centralize the fit around 1.

        Parameters
        ----------
        model_x0 : np.array, shape=(dim_f)
            Distribtuion used as a bias. Internally it is nomalized to
            sum(model_x0) = 1.

        vec_g : np.array, shape=(dim_g)
            Observable vector which is used to get the fit centered around
            1.
        """
        super(LinearModel, self).set_model_x0()
        if len(model_x0) != self.dim_f:
            raise ValueError("'model_x0' has to be of the length as "
                             "vec_f!")
        self.model_factor = sum(vec_g)
        self.model_x0 = model_x0 / self.model_factor

    def add_background(self, vec_b):
        """Adds a background vector to the model.

        Parameters
        ----------
        vec_b : numpy.array, shape=(dim_g)
            Vector g which is added to the model evaluation.
        """
        super(LinearModel, self).add_background()
        self.vec_b = vec_b
        self.background_factor = np.sum(vec_b)

max_stars_repo_name: jacobbieker/funfolding-test

max_issues_repo_name: jacobbieker/funfolding-test

max_forks_repo_name: jacobbieker/funfolding-test

reason: import numpy,from scipy


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

try:
    from vtk import vtkActor
    from vtk import vtkAxesActor
    from vtk import vtkCamera
    from vtk import vtkCellArray
    from vtk import vtkColorTransferFunction
    from vtk import vtkCubeSource
    from vtk import vtkFixedPointVolumeRayCastMapper
    from vtk import vtkGlyph3DMapper
    from vtk import vtkIdList
    from vtk import vtkImageImport
    from vtk import vtkInteractorStyleTrackballCamera
    from vtk import vtkLine
    from vtk import vtkPiecewiseFunction
    from vtk import vtkPoints
    from vtk import vtkPolyData
    from vtk import vtkPolyDataMapper
    from vtk import vtkRenderer
    from vtk import vtkRenderWindow
    from vtk import vtkSphereSource
    from vtk import vtkUnsignedCharArray
    from vtk import vtkVolume
    from vtk import vtkVolumeProperty
    from vtk.qt.QVTKRenderWindowInteractor import QVTKRenderWindowInteractor
except ImportError:
    pass

try:
    from numpy import asarray
    from numpy import max
    from numpy import min
    from numpy import uint8
except ImportError:
    pass

try:
    from PyQt5.Qt import QApplication
    from PyQt5.Qt import QCheckBox
    from PyQt5.Qt import QComboBox
    from PyQt5.Qt import QDockWidget
    from PyQt5.Qt import QFrame
    from PyQt5.Qt import QLabel
    from PyQt5.Qt import QMainWindow
    from PyQt5.Qt import QSlider
    from PyQt5.Qt import QVBoxLayout
    from PyQt5.Qt import QWidget
    from PyQt5.QtCore import Qt
except ImportError:
    pass

import sys

__author__    = ['Andrew Liew <liew@arch.ethz.ch>']
__copyright__ = 'Copyright 2018, BLOCK Research Group - ETH Zurich'
__license__   = 'MIT License'
__email__     = 'liew@arch.ethz.ch'

__all__ = [
    'VtkViewer',
]

# ==============================================================================
# MainWindow
# ==============================================================================

class MainWindow(QMainWindow):

    def __init__(self, app, parent=None):
        QMainWindow.__init__(self, parent)

        # Camera

        x, y, z = app.camera_position
        u, v, w = app.camera_target

        self.camera = camera = vtkCamera()
        camera.SetViewUp(0, 0, 1)
        camera.SetPosition(x, y, z)
        camera.SetFocalPoint(u, v, w)

        # Renderer

        self.renderer = renderer = vtkRenderer()
        renderer.SetBackground(1.0, 1.0, 1.0)
        renderer.SetBackground2(0.8, 0.8, 0.8)
        renderer.GradientBackgroundOn()
        renderer.SetActiveCamera(camera)
        renderer.ResetCamera()
        renderer.ResetCameraClippingRange()
        renderer.UseFXAAOn()

        self.window = window = vtkRenderWindow()
        window.AddRenderer(renderer)

        # Widget

        self.frame = frame = QFrame()
        self.widget = widget = QVTKRenderWindowInteractor(frame, rw=window)
        self.layout = layout = QVBoxLayout()
        layout.addWidget(widget)

        frame.setLayout(layout)

        # Interactor

        self.interactor = interactor = widget.GetRenderWindow().GetInteractor()
        interactor.SetInteractorStyle(InteractorStyle())
        interactor.AddObserver('KeyPressEvent', app.keypress)
        interactor.Initialize()
        interactor.Start()

        self.resize(app.width, app.height)
        self.setCentralWidget(frame)

# ==============================================================================
# InteractorStyle
# ==============================================================================

class InteractorStyle(vtkInteractorStyleTrackballCamera):

    def __init__(self, parent=None):

        self.AddObserver('LeftButtonPressEvent',     self.LeftButtonPressEvent)
        self.AddObserver('LeftButtonReleaseEvent',   self.LeftButtonReleaseEvent)
        self.AddObserver('MiddleButtonPressEvent',   self.MiddleButtonPressEvent)
        self.AddObserver('MiddleButtonReleaseEvent', self.MiddleButtonReleaseEvent)
        self.AddObserver('RightButtonPressEvent',    self.RightButtonPressEvent)
        self.AddObserver('RightButtonReleaseEvent',  self.RightButtonReleaseEvent)

    def LeftButtonPressEvent(self, obj, event):

        self.OnLeftButtonDown()
        return

    def LeftButtonReleaseEvent(self, obj, event):

        self.OnLeftButtonUp()
        return

    def MiddleButtonPressEvent(self, obj, event):

        self.OnMiddleButtonDown()
        return

    def MiddleButtonReleaseEvent(self, obj, event):

        self.OnMiddleButtonUp()
        return

    def RightButtonPressEvent(self, obj, event):

        self.OnRightButtonDown()
        return

    def RightButtonReleaseEvent(self, obj, event):

        self.OnRightButtonUp()
        return

# ==============================================================================
# VtkViewer
# ==============================================================================

class VtkViewer(QApplication):

    def __init__(self, name='VTK Viewer', data={}, height=900, width=1440, datastructure=None):
        QApplication.__init__(self, sys.argv)

        if datastructure:

            i_k = datastructure.index_key()
            k_i = datastructure.index_key()
            data = {}

            data['vertices'] = {i: datastructure.vertex_coordinates(i_k[i])
                                for i in range(datastructure.number_of_vertices())}

            data['edges']    = [{'u': k_i[u], 'v': k_i[v]} for u, v in datastructure.edges()]

            data['fixed']    = [k_i[key] for key in datastructure.vertices()
                                if datastructure.vertex[key].get('is_fixed')]

            if datastructure.attributes['name'] == 'Mesh':
                data['faces']    = {i: {'vertices': datastructure.face[i]} for i in datastructure.faces()}

        self.camera_position = [10, -1, 10]
        self.camera_target   = [0, 0, 0]
        self.show_axes       = True
        self.vertex_size     = 20.0
        self.vertex_scale    = 0.001
        self.edge_width      = 20.0
        self.edge_scale      = 0.05

        self.name         = name
        self.data         = data
        self.height       = height
        self.width        = width
        self.keycallbacks = {}

        self.labels     = {}
        self.listboxes  = {}
        self.sliders    = {}
        self.checkboxes = {}

    # ==============================================================================
    # Setup
    # ==============================================================================

    def setup(self):

        self.main = MainWindow(app=self)
        self.main.show()

        self.setup_statusbar()
        self.setup_menubar()
        self.setup_sidebar()

        self.draw()

    # ==============================================================================
    # Start
    # ==============================================================================

    def start(self):

        sys.exit(self.exec_())

    # ==============================================================================
    # Keypress
    # ==============================================================================

    def keypress(self, obj, event):

        key = obj.GetKeySym()
        try:
            func = self.keycallbacks[key]
            func(self)
        except:
            pass

    # ==============================================================================
    # Draw
    # ==============================================================================

    def draw(self):

        self.vertices  = vtkPoints()
        self.polydata  = vtkPolyData()
        self.blocks    = vtkPolyData()
        self.locations = vtkPoints()
        self.bcs       = vtkPolyData()
        self.fixed     = vtkPoints()
        self.colors    = vtkUnsignedCharArray()
        self.vcolors   = self.data.get('vertex_colors', None)
        self.colors.SetNumberOfComponents(3)

        if self.data.get('vertices', None):
            for key, vertex in self.data['vertices'].items():
                self.vertices.InsertNextPoint(vertex)
            self.polydata.SetPoints(self.vertices)
            self.draw_vertices()

        if self.data.get('fixed', None):
            for key in self.data['fixed']:
                self.fixed.InsertNextPoint(self.data['vertices'][key])
            self.bcs.SetPoints(self.fixed)

        if self.data.get('edges', None):
            self.draw_edges()

        if self.data.get('faces', None):
            self.draw_faces()

        if self.data.get('blocks', None):
            self.draw_blocks()
        else:
            self.block_actor = None

        if self.data.get('voxels', None) is not None:
            self.draw_voxels()

        self.polydata.GetCellData().SetScalars(self.colors)

        if self.vcolors:
            self.vertex_colors = vtkUnsignedCharArray()
            self.vertex_colors.SetNumberOfComponents(3)
            for key in self.data['vertices']:
                self.vertex_colors.InsertNextTypedTuple(self.vcolors.get(key, [200, 200, 200]))
            self.polydata.GetPointData().SetScalars(self.vertex_colors)

        self.draw_axes()
        if not self.show_axes:
            self.axes.VisibilityOff()

        mapper = vtkPolyDataMapper()
        mapper.SetInputData(self.polydata)

        self.actor = actor = vtkActor()
        actor.SetMapper(mapper)
        actor.GetProperty().SetLineWidth(self.edge_width * self.edge_scale)
        if self.edge_width == 0:
            actor.GetProperty().EdgeVisibilityOff()
        else:
            actor.GetProperty().EdgeVisibilityOn()
        actor.GetProperty().SetEdgeColor([0, 0.5, 0])
        actor.GetProperty().SetInterpolationToGouraud()

        self.main.renderer.AddActor(actor)

    # ==============================================================================
    # Statusbar
    # ==============================================================================

    def setup_statusbar(self):

        self.status = self.main.statusBar()
        self.main.setStatusBar(self.status)
        self.status.showMessage('Status bar')

    def update_statusbar(self, text):

        self.status.showMessage(text)

    # ==============================================================================
    # Menus
    # ==============================================================================

    def setup_menubar(self):

        self.menu = self.main.menuBar()
        self.main.setMenuBar(self.menu)
        self.add_file_menu()
        self.add_view_menu()

    def add_file_menu(self):

        file_menu = self.menu.addMenu('&File')
        open_action = file_menu.addAction('&Open')
        open_action.triggered.connect(self.file_open)

    def add_view_menu(self):

        view_menu = self.menu.addMenu('&View')

    def file_open(self):

        print('FILE')

    # ==============================================================================
    # Sidebar
    # ==============================================================================

    def add_slider(self, name, value, minimum, maximum, interval, callback):

        self.sliders[name] = slider = QSlider(Qt.Horizontal)
        slider.setTickPosition(QSlider.TicksBelow)
        slider.setTickInterval(interval)
        slider.setValue(value)
        slider.setMinimum(minimum)
        slider.setMaximum(maximum)
        slider.valueChanged.connect(callback)
        self.layout.addWidget(slider)

    def add_label(self, name, text):

        self.labels[name] = label = QLabel()
        label.setText(text)
        self.layout.addWidget(label)

    def add_checkbox(self, name, text, checked, callback):

        self.checkboxes[name] = checkbox = QCheckBox()
        checkbox.setText(text)
        checkbox.setChecked(checked)
        checkbox.stateChanged.connect(callback)
        self.layout.addWidget(checkbox)

    def add_listbox(self, name, items, callback):

        self.listboxes[name] = listbox = QComboBox()
        listbox.addItems(items)
        listbox.currentIndexChanged.connect(callback)
        self.layout.addWidget(listbox)

    def add_splitter(self):

        pass
        # self.layout.setFrameShape(QFrame.HLine)

    def setup_sidebar(self):

        self.sidebar = sidebar = QDockWidget()
        sidebar.setFixedWidth(120)

        self.layout = layout = QVBoxLayout()
        layout.addStretch()

        if self.data.get('vertices', None):

            self.add_label(name='label_vertices', text='Vertex size: {0}'.format(self.vertex_size))
            self.add_slider(name='slider_vertices', value=self.vertex_size, minimum=0, maximum=100,
                            interval=10, callback=self.vertex_callback)

        if self.data.get('edges', None) or self.data.get('faces', None):

            self.add_label(name='label_edges', text='Edge width: {0}'.format(self.edge_width))
            self.add_slider(name='slider_edges', value=self.edge_width, minimum=0, maximum=100,
                            interval=10, callback=self.edge_callback)

            self.add_label(name='label_opacity', text='Opacity: {0}'.format(100))
            self.add_slider(name='slider_opacity', value=100, minimum=0, maximum=100,
                            interval=10, callback=self.opacity_callback)

        if self.data.get('voxels', None) is not None:

            self.add_label(name='label_gradient', text='Gradient: {0}'.format(0))
            self.add_slider(name='slider_gradient', value=0, minimum=0, maximum=100,
                            interval=10, callback=self.gradient_callback)

        widget = QWidget(self.sidebar)
        widget.setLayout(layout)

        self.sidebar.setWidget(widget)
        self.main.addDockWidget(Qt.LeftDockWidgetArea, sidebar)

    # ==============================================================================
    # Vertices
    # ==============================================================================

    def draw_vertices(self):

        self.vertex = vertex = vtkSphereSource()
        vertex.SetRadius(self.vertex_size * self.vertex_scale)
        vertex.SetPhiResolution(15)
        vertex.SetThetaResolution(15)

        mapper1 = vtkGlyph3DMapper()
        mapper1.SetInputData(self.polydata)
        mapper1.SetSourceConnection(vertex.GetOutputPort())
        mapper1.ScalingOff()
        mapper1.ScalarVisibilityOff()

        actor1 = vtkActor()
        actor1.SetMapper(mapper1)
        actor1.GetProperty().SetDiffuseColor(0.6, 0.6, 1.0)
        actor1.GetProperty().SetDiffuse(1.0)
        self.main.renderer.AddActor(actor1)

        if self.data.get('fixed', None):

            self.support = support = vtkSphereSource()
            support.SetRadius(self.vertex_size * 1.5 * self.vertex_scale)
            support.SetPhiResolution(15)
            support.SetThetaResolution(15)

            mapper2 = vtkGlyph3DMapper()
            mapper2.SetInputData(self.bcs)
            mapper2.SetSourceConnection(support.GetOutputPort())
            mapper2.ScalingOff()
            mapper2.ScalarVisibilityOff()

            actor2 = vtkActor()
            actor2.SetMapper(mapper2)
            actor2.GetProperty().SetDiffuseColor(1.0, 0.8, 0.0)
            actor2.GetProperty().SetDiffuse(1.0)
            self.main.renderer.AddActor(actor2)

        else:
            self.support = None

    def vertex_callback(self):

        value = self.sliders['slider_vertices'].value()
        self.labels['label_vertices'].setText('Vertex size: {0}'.format(value))
        self.vertex.SetRadius(value * self.vertex_scale)
        if self.support:
            self.support.SetRadius(value * 1.5 * self.vertex_scale)
        self.main.window.Render()

    def update_vertices_coordinates(self, coordinates):

        for key, xyz in coordinates.items():
            self.vertices.SetPoint(key, xyz)
        self.vertices.Modified()
        self.main.window.Render()

    def update_vertices_colors(self, colors):

        self.vertex_colors = vtkUnsignedCharArray()
        self.vertex_colors.SetNumberOfComponents(3)
        colors_ = [colors.get(i, [200, 200, 200]) for i in self.data['vertices'].keys()]
        for color in colors_:
            self.vertex_colors.InsertNextTypedTuple([int(round(i)) for i in color])
        self.polydata.GetPointData().SetScalars(self.vertex_colors)
        self.main.window.Render()

    # ==============================================================================
    # Edges
    # ==============================================================================

    def draw_edges(self):

        edges = vtkCellArray()

        for edge in self.data['edges']:

            line = vtkLine()
            line.GetPointIds().SetId(0, edge['u'])
            line.GetPointIds().SetId(1, edge['v'])
            edges.InsertNextCell(line)
            color = edge.get('color', [255, 100, 100])

            try:
                self.colors.InsertNextTypedTuple(color)
            except Exception:
                self.colors.InsertNextTupleValue(color)

        self.polydata.SetLines(edges)

    def edge_callback(self):

        value = self.sliders['slider_edges'].value()
        if value == 0:
            self.actor.GetProperty().EdgeVisibilityOff()
        else:
            self.actor.GetProperty().EdgeVisibilityOn()
        self.labels['label_edges'].setText('Edge width: {0}'.format(value))
        self.actor.GetProperty().SetLineWidth(value * self.edge_scale)
        self.main.window.Render()

    # ==============================================================================
    # Faces
    # ==============================================================================

    def draw_faces(self):

        faces = vtkCellArray()

        for fkey, face in self.data['faces'].items():

            vil = vtkIdList()
            for pt in face['vertices']:
                vil.InsertNextId(pt)
            faces.InsertNextCell(vil)
            color = face.get('color', [150, 255, 150])

            try:
                self.colors.InsertNextTypedTuple(color)
            except Exception:
                self.colors.InsertNextTupleValue(color)

        self.polydata.SetPolys(faces)

    def opacity_callback(self):

        value = self.sliders['slider_opacity'].value()
        self.labels['label_opacity'].setText('Opacity: {0}'.format(value))
        self.actor.GetProperty().SetOpacity(value * 0.01)
        if self.block_actor:
            self.block_actor.GetProperty().SetOpacity(value * 0.01)
        self.main.window.Render()

    # ==============================================================================
    # Blocks
    # ==============================================================================

    def draw_blocks(self):

        for location in self.data['blocks'].get('locations', []):
            self.locations.InsertNextPoint(location)
        self.blocks.SetPoints(self.locations)

        self.block = block = vtkCubeSource()
        size = self.data['blocks'].get('size', 1)
        block.SetXLength(size)
        block.SetYLength(size)
        block.SetZLength(size)

        mapper = vtkGlyph3DMapper()
        mapper.SetInputData(self.blocks)
        mapper.SetSourceConnection(block.GetOutputPort())

        self.block_actor = actor = vtkActor()
        actor.SetMapper(mapper)
        actor.GetProperty().SetDiffuseColor(0.7, 0.7, 1.0)
        actor.GetProperty().SetDiffuse(1.0)
        actor.GetProperty().EdgeVisibilityOn()
        self.main.renderer.AddActor(actor)

    # ==============================================================================
    # Voxels
    # ==============================================================================

    def draw_voxels(self):

        self.show_axes = False

        u = self.data['voxels']
        u /= (2 * max([abs(min(u)), abs(max(u))]))
        u += 0.5
        u *= 255

        nx, ny, nz = u.shape
        U = asarray(u, dtype=uint8)
        dstr = U.tostring()

        img = vtkImageImport()
        img.CopyImportVoidPointer(dstr, len(dstr))
        img.SetDataScalarTypeToUnsignedChar()
        img.SetNumberOfScalarComponents(1)
        img.SetDataExtent(0, nz - 1, 0, ny - 1, 0, nx - 1)
        img.SetWholeExtent(0, nz - 1, 0, ny - 1, 0, nx - 1)

        self.gradient = gradient = vtkPiecewiseFunction()
        gradient.AddPoint(1, 0.0)
        gradient.AddPoint(255, 0.2)

        self.cbar = cbar = vtkColorTransferFunction()
        cbar.AddRGBPoint(0.0, 0.0, 0.0, 1.0)
        cbar.AddRGBPoint(42.0, 0.0, 0.5, 1.0)
        cbar.AddRGBPoint(84.0, 0.0, 1.0, 0.5)
        cbar.AddRGBPoint(128.0, 0.0, 1.0, 0.0)
        cbar.AddRGBPoint(168.0, 0.5, 1.0, 0.0)
        cbar.AddRGBPoint(212.0, 1.0, 0.5, 0.0)
        cbar.AddRGBPoint(255.0, 1.0, 0.0, 0.0)

        self.volprop = volprop = vtkVolumeProperty()
        volprop.SetColor(cbar)
        volprop.SetScalarOpacity(gradient)
        volprop.ShadeOff()
        volprop.SetInterpolationTypeToLinear()

        mapper = vtkFixedPointVolumeRayCastMapper()
        mapper.SetInputConnection(img.GetOutputPort())

        volume = vtkVolume()
        volume.SetMapper(mapper)
        volume.SetProperty(volprop)
        self.main.renderer.AddVolume(volume)

        self.main.camera.SetPosition(0, -2 * max([nx, ny, nz]), nz)
        self.main.camera.SetFocalPoint(0.5 * nx, 0.5 * ny, 0.5 * nz)

    def gradient_callback(self):

        value = self.sliders['slider_gradient'].value()
        self.labels['label_gradient'].setText('Gradient: {0}'.format(value))

        self.gradient = gradient = vtkPiecewiseFunction()
        gradient.AddPoint(0, 0.2)
        if value:
            gradient.AddPoint(127 - value * 1.27, 0.0)
            gradient.AddPoint(127 + value * 1.27, 0.0)
        else:
            gradient.AddPoint(128, 0.2)
        gradient.AddPoint(255, 0.2)

        self.volprop.SetScalarOpacity(gradient)
        self.main.window.Render()

    # ==============================================================================
    # Axis
    # ==============================================================================

    def draw_axes(self):

        self.axes = axes = vtkAxesActor()
        axes.AxisLabelsOff()
        axes.SetConeRadius(0.2)
        self.main.renderer.AddActor(axes)

# ==============================================================================
# Main
# ==============================================================================

if __name__ == "__main__":

    # ==============================================================================
    # Mesh
    # ==============================================================================

    # def func(self):
    #     print('Callback test!')

    # data = {
    #     'vertices': {
    #         0: [-3, -3, 0],
    #         1: [+3, -3, 0],
    #         2: [+3, +3, 0],
    #         3: [-3, +3, 0],
    #         4: [-3, -3, 3],
    #         5: [+3, -3, 3],
    #         6: [+3, +3, 3],
    #         7: [-3, +3, 3],
    #     },
    #     'vertex_colors': {
    #         # turn on vertex coloring by uncommenting
    #         0: [255, 0, 255],
    #         1: [255, 0, 0],
    #         2: [255, 255, 0],
    #         3: [255, 255, 0],
    #         4: [0, 255, 0],
    #         5: [0, 255, 150],
    #         6: [0, 255, 255],
    #         7: [0, 0, 255],
    #     },
    #     'edges': [
    #         {'u': 0, 'v': 4, 'color': [0, 0, 0]},
    #         {'u': 1, 'v': 5, 'color': [0, 0, 255]},
    #         {'u': 2, 'v': 6, 'color': [0, 255, 0]},
    #         {'u': 3, 'v': 7}
    #     ],
    #     'faces': {
    #         0: {'vertices': [4, 5, 6], 'color': [250, 150, 150]},
    #         1: {'vertices': [6, 7, 4], 'color': [150, 150, 250]},
    #     },
    #     'fixed':
    #         [0, 1],
    #     'blocks': {
    #         'size': 1,
    #         'locations': [[0, 0, 3], [0, 0, 4]],
    #     }
    # }

    # viewer = VtkViewer(data=data)
    # viewer.show_axes = False
    # viewer.keycallbacks['s'] = func
    # viewer.setup()
    # viewer.start()

    # ==============================================================================
    # Voxels
    # ==============================================================================

    # from numpy import linspace
    # from numpy import meshgrid

    # r = linspace(-1, 1, 50)
    # x, y, z = meshgrid(r, r, r)

    # data = {
    #     'voxels': x + y + z,
    # }

    # viewer = VtkViewer(data=data)
    # viewer.setup()
    # viewer.start()

    # ==============================================================================
    # Datastructures
    # ==============================================================================

    from compas.datastructures import Network
    from compas.datastructures import Mesh

    import compas

    # datastructure = Network.from_obj(compas.get('saddle.obj'))
    datastructure = Mesh.from_obj(compas.get('quadmesh.obj'))

    viewer = VtkViewer(datastructure=datastructure)
    viewer.setup()
    viewer.start()

max_stars_repo_name: philianeles/compas

max_issues_repo_name: philianeles/compas

max_forks_repo_name: philianeles/compas

reason: from numpy


text:
from utils import rot_vec
from numpy import savetxt
import numpy as np
from scipy.spatial import distance
from scipy.spatial.transform import Rotation as R
import random
from enum import Enum

class Tasktype(Enum):
  RANDOM    = 0
  EASY      = 1
  FRONTIER  = 2

class RobotTask():
  def __init__(self,default_task,):
    super(RobotTask, self).__init__()

    self.robot_translation      = np.array(default_task['robot_pose']['translation']).astype(np.float)
    self.robot_rotation         = np.array(default_task['robot_pose']['rotation_yaw']).astype(np.float)

    self.dolly_translation      = np.array(default_task['dolly_pose']['translation']).astype(np.float)
    self.dolly_rotation         = np.array(default_task['dolly_pose']['rotation_yaw']).astype(np.float)

    self.obstacle_translation   = np.array(default_task['obstacle_pose']['translation']).astype(np.float)
    self.obstacle_rotation      = np.array(default_task['obstacle_pose']['rotation_yaw']).astype(np.float)

    self.obstacle_1_translation = np.array(default_task['obstacle_1_pose']['translation']).astype(np.float)
    self.obstacle_1_rotation    = np.array(default_task['obstacle_1_pose']['rotation_yaw']).astype(np.float)

    self.obstacle_2_translation = np.array(default_task['obstacle_2_pose']['translation']).astype(np.float)
    self.obstacle_2_rotation    = np.array(default_task['obstacle_2_pose']['rotation_yaw']).astype(np.float)

    self.obstacle_3_translation = np.array(default_task['obstacle_3_pose']['translation']).astype(np.float)
    self.obstacle_3_rotation    = np.array(default_task['obstacle_3_pose']['rotation_yaw']).astype(np.float)

    self.q_value                = 0
    self.task_type              = Tasktype.RANDOM

  def randomize_task(self, randomization_params):
    """
      randomizes the current task settings according to the randomization parameters (as a json)
    """
    num_obstacles                = random.randint(0,randomization_params['num_obstacles'])
    del_obstacle_translation     = np.asarray(randomization_params['del_obstacle_translation'])
    self.robot_rotation         += ( (-0.5 + np.random.uniform(0,1)) *np.array(randomization_params['robot_randomization']['rotation_rnd'])           )
    self.dolly_rotation         += ( (-0.5 + np.random.uniform(0,1)) *np.array(randomization_params['dolly_randomization']['rotation_rnd'])           )
    self.obstacle_rotation      += ( (-0.5 + np.random.uniform(0,1)) *np.array(randomization_params['obstacle_randomization']['rotation_rnd'])        )
    self.obstacle_1_rotation    += ( (-0.5 + np.random.uniform(0,1)) *np.array(randomization_params['obstacle_randomization']['rotation_rnd'])        )
    self.obstacle_2_rotation    += ( (-0.5 + np.random.uniform(0,1)) *np.array(randomization_params['obstacle_randomization']['rotation_rnd'])        )
    self.obstacle_3_rotation    += ( (-0.5 + np.random.uniform(0,1)) *np.array(randomization_params['obstacle_randomization']['rotation_rnd'])        )

    self.robot_translation      += ( (-0.5 + np.random.uniform(0,1)) *np.array(randomization_params['robot_randomization']['translation_rnd_xyz'])    )
    #self.dolly_translation      += ( (-0.5 + np.random.uniform(0,1)) *np.array(randomization_params['dolly_randomization']['translation_rnd_xyz'])    )

    min_dist_d_r                = randomization_params['min_dist_dolly_robot']
    max_dist_d_r                = randomization_params['max_dist_dolly_robot']
    dolly_pos_spline            = randomization_params['dolly_pos_spline_angle']

    dolly_distance              = min_dist_d_r + np.random.uniform(0,1)*(max_dist_d_r-min_dist_d_r)        #create a distance vector with min, max distance according to the config file
    dolly_vector                = np.array([1,0])*dolly_distance                                    #create the final dolly position as the robot position+distance vector
    dolly_pos_rot               = ( (-0.5 + np.random.uniform(0,1))) * dolly_pos_spline                    #rotate it around the origin
    dolly_vector                = self.robot_translation + np.hstack((rot_vec(dolly_vector,dolly_pos_rot),np.zeros(1)))

    self.dolly_translation      = dolly_vector

    min_dist_r_o                = randomization_params['min_dist_robot_obs']
    max_dist_r_o                = randomization_params['max_dist_robot_obs']

    min_dist_d_o                = randomization_params['min_dist_dolly_obs']
    max_dist_d_o                = randomization_params['max_dist_dolly_obs']
    obs_mean                    = randomization_params['obstacle_pos_spline_mean']
    obstacle_vectors            = []

    for i in range(2):
      obstacle_distance = min_dist_r_o + np.random.uniform(0,1)*(max_dist_r_o-min_dist_r_o)        #create a distance vector with min, max distance according to the config file
      obstacle_pos_rot  = 90 + np.random.uniform(-obs_mean/2,obs_mean/2)*90+(i*180)                                             #rotate it around the origin
      obstacle_vector   = np.array([1,0])*obstacle_distance                                 #create the final object position as the robot position+distance vector

      obstacle_vector   = self.robot_translation + np.hstack((rot_vec(obstacle_vector,obstacle_pos_rot),np.zeros(1)))
      obstacle_vectors.append(obstacle_vector)

    for i in range(2):
      obstacle_distance = min_dist_d_o + np.random.uniform(0,1)*(max_dist_d_o-min_dist_d_o)        #create a distance vector with min, max distance according to the config file
      obstacle_pos_rot  = 90 + np.random.uniform(-obs_mean/2,obs_mean/2)*90+(i*180)                                           #rotate it around the origin
      obstacle_vector   = np.array([1,0])*obstacle_distance                                 #create the final object position as the dolly position+distance vector

      obstacle_vector   = self.dolly_translation + np.hstack((rot_vec(obstacle_vector,obstacle_pos_rot),np.zeros(1)))
      obstacle_vectors.append(obstacle_vector)

    for remove_obj_idx in range(4-1, num_obstacles-1, -1):
      obstacle_vectors[remove_obj_idx] =  del_obstacle_translation

    # now randomly assign the created obstacle vectors to the target translations
    rnd_obstcl_indexes = np.random.permutation(np.array([0,1,2,3]))
    self.obstacle_translation   =obstacle_vectors[rnd_obstcl_indexes[0]]        #+= ( (-0.5 + np.random.uniform(0,1)) *np.array(randomization_params['obstacle_randomization']['translation_rnd_xyz']) )
    self.obstacle_1_translation =obstacle_vectors[rnd_obstcl_indexes[1]]        #+= ( (-0.5 + np.random.uniform(0,1)) *np.array(randomization_params['obstacle_randomization']['translation_rnd_xyz']) )
    self.obstacle_2_translation =obstacle_vectors[rnd_obstcl_indexes[2]]        #+= ( (-0.5 + np.random.uniform(0,1)) *np.array(randomization_params['obstacle_randomization']['translation_rnd_xyz']) )
    self.obstacle_3_translation =obstacle_vectors[rnd_obstcl_indexes[3]]        #+= ( (-0.5 + np.random.uniform(0,1)) *np.array(randomization_params['obstacle_randomization']['translation_rnd_xyz']) )

  def set_q_value(self, q_value):
    self.q_value = q_value

  def get_obstacle_translations_array(self):
    return np.array([self.obstacle_translation, self.obstacle_1_translation, self.obstacle_2_translation, self.obstacle_3_translation])

  def get_task_array(self):
      """
        returns the task as a numpy array in the form: [[robo_trans_x,robo_trans_y,robo_trans_z,robo_yaw_angle],
                                                        [dolly_trans_x,dolly_trans_y,dolly_trans_z,dolly_yaw_angle],
                                                        [obstacle_trans_x,obstacletrans_y,obstacletrans_z,obobstacleyaw_angle]
                                                        [obstacle_1_trans_x,obstacle_1_trans_y,obstacle_1_trans_z,obobstacle_1_yaw_angle]
                                                        [obstacle_2_trans_x,obstacle_2_trans_y,obstacle_2_trans_z,obobstacle_2_yaw_angle]
                                                        [obstacle_3_trans_x,obstacle_3_trans_y,obstacle_3_trans_z,obobstacle_3_yaw_angle]]
      """
      robot_part      = np.hstack((self.robot_translation,      self.robot_rotation))
      dolly_part      = np.hstack((self.dolly_translation,      self.dolly_rotation))
      obstacle_part   = np.hstack((self.obstacle_translation,   self.obstacle_rotation))
      obstacle_1_part = np.hstack((self.obstacle_1_translation, self.obstacle_1_rotation))
      obstacle_2_part = np.hstack((self.obstacle_2_translation, self.obstacle_2_rotation))
      obstacle_3_part = np.hstack((self.obstacle_3_translation, self.obstacle_3_rotation))

      task_array = np.array([robot_part,dolly_part,obstacle_part,obstacle_1_part,obstacle_2_part,obstacle_3_part])
      return task_array

  def from_task_array(self,task_array):
    self.robot_translation      = task_array[0,0:3]
    self.robot_rotation         = np.array([task_array[0][3]])
    self.dolly_translation      = task_array[1,0:3]
    self.dolly_rotation         = np.array([task_array[1][3]])
    self.obstacle_translation   = task_array[2,0:3]
    self.obstacle_rotation      = np.array([task_array[2][3]])
    self.obstacle_1_translation = task_array[3,0:3]
    self.obstacle_1_rotation    = np.array([task_array[3][3]])
    self.obstacle_2_translation = task_array[4,0:3]
    self.obstacle_2_rotation    = np.array([task_array[4][3]])
    self.obstacle_3_translation = task_array[5,0:3]
    self.obstacle_3_rotation    = np.array([task_array[5][3]])
    self.q_value                = 0

reason: import numpy,from numpy,from scipy