text:
# @Author KSASAN I317174

"""
This is the Entry point for all the ML Related Stuff
"""
from plotly.utils import numpy
from sklearn.preprocessing import LabelEncoder

import LinearRegressionUtility
from CreateLassoLinearRegressionWithKFold import CreateLassoLinearRegressionWithKFold
from CreateLinearRegressionModelWithKFold import CreateLinearRegressionModelWithKFold
from CreateRidgeLinearRegressionWithKFold import CreateRidgeLinearRegressionWithKFold
from NewRegressionApproach import NewRegressionApproach

if __name__ == '__main__':
    # Creating Graphs for Export Hana
    exportDataSet = LinearRegressionUtility.loadDataSet("../resources/DBPoolDataExport.csv", ",", False)
    LinearRegressionUtility.createMLModelAndGraphs(exportDataSet, 5)

    # Creating Graphs for Import Hana
    importDataSet = LinearRegressionUtility.loadDataSetOfDiffDataTypesAsString(
        "../resources/DBPoolDataImport.csv", ",", False)
    labelEncoder = LabelEncoder()
    importDataSet[:, 3] = labelEncoder.fit_transform(importDataSet[:, 3].astype('str'))
    importDataSet = importDataSet.astype(numpy.float64)

    # LinearRegressionUtility.removeColumns(importDataSet, 3)
    # LinearRegressionUtility.createMLModelAndGraphs(importDataSet, 5)

    # Classify and Regress Approach as Discussed with @Roopang
    importDataSet = LinearRegressionUtility.removeColumns(importDataSet, 3)
    newRegressionApproach = NewRegressionApproach(importDataSet, 2, 50)
    newRegressionApproach = NewRegressionApproach(exportDataSet, 2, 50)


text:
import numpy as np
from numpy.core.fromnumeric import shape
from sbgas.sbmga import DiscreteSBMGA

# test parameters
FFUNC = np.sum
POPSIZE = 5
GENESIZE = 3
MUTPROB = 0.01
INFECTPROB = 0.01
SBSIZE = 2
DORMPROB = 0.01
DEMESIZE = 3
ALPHABET = [0, 1]

ga = DiscreteSBMGA(FFUNC, POPSIZE, GENESIZE, MUTPROB, INFECTPROB, SBSIZE, DORMPROB, DEMESIZE, ALPHABET)

def test_init():
    # get a bunch of flags
    pop_shape = ga.population.shape == (POPSIZE, GENESIZE)
    gene_probs = ga.pmutate == MUTPROB and ga.pinfect == INFECTPROB
    sb_shape = ga.seed_bank.shape == (SBSIZE, GENESIZE)
    dorm_prob = ga.pdormant == DORMPROB
    deme = ga.deme_size == DEMESIZE
    alph = ga.alphabet == ALPHABET
    # make sure all flags are true
    assert pop_shape and gene_probs and sb_shape and dorm_prob and deme and alph

def test_tournament_basic():
    # set so genome 1 is better than genome 0
    ga.population[1] = np.array([0, 0, 1])
    winner_index, loser_index = ga._tournament(0, 1)
    assert winner_index == 1 and loser_index == 0

def test_tournament_mutation():
    # pass in a mutation probability of 1, other probs to zero
    ga_mut = DiscreteSBMGA(FFUNC, POPSIZE, GENESIZE, 1, 0, SBSIZE, 0, DEMESIZE, ALPHABET)
    ga_mut._tournament(0, 1)
    assert np.array_equal(ga_mut.population[1], np.ones(GENESIZE))

def test_tournament_infection():
    ga_inf = DiscreteSBMGA(FFUNC, POPSIZE, GENESIZE, 0, 1, SBSIZE, 0, DEMESIZE, ALPHABET)
    ga_inf.population[0] = np.ones(GENESIZE)
    ga_inf._tournament(0, 1)
    assert np.array_equal(ga_inf.population[1], np.ones(GENESIZE))

def test_tournament_dormancy():
    ga_sb = DiscreteSBMGA(FFUNC, POPSIZE, GENESIZE, 0, 0, SBSIZE, 1, DEMESIZE, ALPHABET)
    # set so 0 still wins but we can see the change in the seed bank when 1 enters
    ga_sb.population[0] = np.ones(GENESIZE)
    ga_sb.population[1] = np.array([0, 0, 1])
    ga_sb._tournament(0, 1)
    assert np.array_equal(ga_sb.population[1], np.zeros(GENESIZE)) and np.sum(ga_sb.seed_bank) == 1

def test_pick_competitors():
    ga_comp = DiscreteSBMGA(FFUNC, POPSIZE, GENESIZE, MUTPROB, INFECTPROB, SBSIZE, DORMPROB, 1, ALPHABET)
    i, j = ga_comp._pick_competitors(4)
    assert i == 4 and j == 0

def test_pick_competitors_stochastic():
    i, j = ga._pick_competitors(2)
    assert i == 2 and j in [3, 4, 0]

def test_run_evolution():
    history = ga.run_evolution(500)
    assert np.max(history[-1]) == 3

ext: py

max_stars_repo_name: godzilla-but-nicer/sbgas

max_issues_repo_name: godzilla-but-nicer/sbgas

max_forks_repo_name: godzilla-but-nicer/sbgas

reason: import numpy,from numpy


text:
# -*- coding: utf-8 -*-

import numpy as np
import copy
import statsmodels.api as sm
from pyorderedfuzzy.ofnumbers.ofnumber import OFNumber, flog
from pyorderedfuzzy.ofcandles.ofcandle import OFCandle

__author__ = "amarszalek"

class OFSeries(list):
    def __init__(self, ofns):
        list.__init__(self, ofns)

    def copy(self):
        return copy.deepcopy(self)

    def transform(self, method='diff'):
        arr = np.array(self, dtype=object)
        if method == 'diff':
            new_ofns = arr[1:]-arr[:-1]
        elif method == 'ret':
            new_ofns = (arr[1:]-arr[:-1])/arr[:-1]
        elif method == 'logret':
            new_ofns = map(flog, (arr[1:]/arr[:-1]))
        else:
            raise ValueError('method must be diff, ret or logret')
        return OFSeries(new_ofns)

    # Augmented Dickey-Fuller test from statsmodels.tsa
    def adfuller(self, verbose='max', s=0, e=None, **kwargs):
        if e is None:
            ofns = self[s:]
        else:
            ofns = self[s:e]
        n = len(ofns)
        dim = ofns[0].branch_f.dim
        res = []
        for i in range(dim):
            series = np.array([ofns[j].branch_f.fvalue_y[i] for j in range(n)])
            r = sm.tsa.adfuller(series, **kwargs)
            if verbose == 'full':
                print('f:', i, '  statistic:', r[0], '  pvalue:', r[1], '  1%:', r[4]['1%'], '  5%:', r[4]['5%'])
            res.append(r)
        dim = ofns[0].branch_g.dim
        for i in range(dim):
            series = np.array([ofns[j].branch_g.fvalue_y[i] for j in range(n)])
            r = sm.tsa.adfuller(series, **kwargs)
            if verbose == 'full':
                print('g:', i, '  statistic:', r[0], '  pvalue:', r[1], '  1%:', r[4]['1%'], '  5%:', r[4]['5%'])
            res.append(r)
        if verbose in ['max', 'full']:
            a = np.array([r[0] for r in res])
            print('max of DF-statistics:', np.max(a), '  1%:', res[0][4]['1%'], '  5%:', res[0][4]['5%'])
        return res

    # Partial autocorrelation estimated from statsmodels.tsa
    def pacf(self, nlags=20, method='ywunbiased', alpha=None, verbose='maxmin', s=0, e=None):
        if e is None:
            ofns = self[s:]
        else:
            ofns = self[s:e]
        n = len(ofns)
        dim = ofns[0].branch_f.dim
        res = []
        for i in range(dim):
            series = np.array([ofns[j].branch_f.fvalue_y[i] for j in range(n)])
            r = sm.tsa.pacf(series, nlags=nlags, method=method, alpha=alpha)
            if verbose == 'full':
                print('f:', i)
                print('pacf:', r)
            res.append(r)
        dim = ofns[0].branch_g.dim
        for i in range(dim):
            series = np.array([ofns[j].branch_g.fvalue_y[i] for j in range(n)])
            r = sm.tsa.pacf(series, nlags=nlags, method=method, alpha=alpha)
            if verbose == 'full':
                print('g:', i)
                print('pacf:', r)
            res.append(r)
        if verbose in ['maxmin', 'full']:
            a = np.array(res)
            print('max:', np.max(a, axis=0))
            print('min:', np.min(a, axis=0))
        return res

    # Returns information criteria for many ARMA models, arma_order_selctic from statsmodels.tsa
    def arma_select_order_ic(self, max_ar=5, max_ma=0, ic=['aic'], trend='c', verbose='max', s=0, e=None, **kwargs):
        if e is None:
            ofns = self[s:]
        else:
            ofns = self[s:e]
        n = len(ofns)
        dim = ofns[0].branch_f.dim
        res = []
        for i in range(dim):
            series = np.array([ofns[j].branch_f.fvalue_y[i] for j in range(n)])
            r = sm.tsa.arma_order_select_ic(series, max_ar=max_ar, max_ma=max_ma, ic=ic, trend=trend, **kwargs)
            if verbose == 'full':
                print('f:', i)
                for c in ic:
                    print(c, ':', r[str(c)+'_min_order'])
            res.append(r)
        dim = ofns[0].branch_g.dim
        for i in range(dim):
            series = np.array([ofns[j].branch_g.fvalue_y[i] for j in range(n)])
            r = sm.tsa.arma_order_select_ic(series, max_ar=max_ar, max_ma=max_ma, ic=ic, trend=trend, **kwargs)
            if verbose == 'full':
                print('g:', i)
                for c in ic:
                    print(c, ':', r[str(c)+'_min_order'])
            res.append(r)
        if verbose in ['max', 'full']:
            for c in ic:
                print('max '+str(c)+'_min_order:', np.max(np.array([res[i][str(c)+'_min_order']
                                                                    for i in range(len(res))])))
        return res

    def plot_ofseries(self, ax, s=0, e=None, color='black', shift=0, ord_method='expected'):
        if e is None:
            ofns = self[s:]
        else:
            ofns = self[s:e]
        n = len(ofns)
        for i in range(n):
            f = ofns[i].branch_f
            g = ofns[i].branch_g
            fg_y = np.append(f.fvalue_y,  g.fvalue_y[::-1])
            fg_x = np.append(f.domain_x, g.domain_x[::-1])*0.5 + i-0.25 + shift
            o = ofns[i].order(method=ord_method)
            if o >= 0:
                ax.fill_betweenx(fg_y, i+shift-0.25, fg_x, facecolor='white', edgecolor='black')
            else:
                ax.fill_betweenx(fg_y, i+shift-0.25, fg_x, facecolor=color, edgecolor='black')
            ax.plot([i+shift-0.25, i+shift-0.25], [np.min(fg_y), np.max(fg_y)], c='w', lw=2)
            ax.plot([i+shift-0.25, i+shift-0.25], [f.fvalue_y[0], g.fvalue_y[0]], c='black', lw=2)
        ax.set_xlim(-1+shift, n+shift)

    def histogram(self, level='f_0', bins=10, normed=False, s=0, e=None, **kwargs):
        lev = level.split('_')
        branch = lev[0]
        alpha = float(lev[1])
        if e is None:
            e = len(self)
        if 0.0 <= alpha <= 1.0:
            if branch == 'f':
                data = np.array([self[i].branch_f.acut(alpha) for i in range(s, e)])
            elif branch == 'g':
                data = np.array([self[i].branch_g.acut(alpha) for i in range(s, e)])
            else:
                raise ValueError('wrong level syntax')
        else:
            raise ValueError('wrong level synatx')
        return np.histogram(data, bins=bins, normed=normed, **kwargs), data

def init_from_stock_data(ctype, data, params):
    ofcans = []
    for i in range(len(data['price'])):
        c_data = {'price': data['price'][i], 'volume': data['volume'][i], 'spread': data['spread'][i]}
        ofcans.append(OFCandle(ctype, c_data, params).to_ofn())
    return OFSeries(ofcans)

def init_from_scalar_values(prices, dim):
    ofns = [OFNumber(np.ones(dim)*p, np.ones(dim)*p) for p in prices]
    return OFSeries(ofns)

max_stars_repo_name: amarszalek/PyOrderedFuzzyTools

max_issues_repo_name: amarszalek/PyOrderedFuzzyTools

max_forks_repo_name: amarszalek/PyOrderedFuzzyTools

reason: import numpy,import statsmodels,from statsmodels


text:
import numpy as np
import os

import argparse

from rankers import BayesianRanker

from initializers.bow_initializer import BoWInitializer
from users import LogitUser
from users import NullUser

from displays.ransam_display import RanSamDisplay
from displays import TopNDisplay
from displays import SOMDisplay

from users import IdealUser
from users import RanSamUser
from users import RanSamSmoothUser

parser = argparse.ArgumentParser()

parser.add_argument("--seed", default=42, type=int, help="Random seed.")
parser.add_argument("--quiet", default=False, action="store_true", help="Quiet")

parser.add_argument("-t", "--target_file", required=False, default="data/study_targets.csv", type=str,
                    help="File with defined targets. On each line should be 'target_id;text_query;iterations;displayType;likes'")
parser.add_argument("-o", "--output_file", required=False, default="data/study_targets.csv.out", type=str,
                    help="Name of output csv file.")

parser.add_argument("--dataset_path", default="v3c1", type=str,
                    help="Root to dataset path.")
parser.add_argument("--features_name", default="V3C1_20191228.w2vv.images.normed.128pca.viretfromat", type=str,
                    help="Name of file with image features.")

parser.add_argument("--keywords_list_name", default="word2idx.txt", type=str,
                    help="Name of file with keyword features.")
parser.add_argument("--kw_features_name", default="txt_weight-11147x2048floats.bin", type=str,
                    help="Name of file with keyword features.")
parser.add_argument("--kw_bias_name", default="txt_bias-2048floats.bin", type=str,
                    help="Name of file with keyword bias.")
parser.add_argument("--pca_matrix_name", default="V3C1_20191228.w2vv.pca.matrix.bin", type=str,
                    help="Name of file with pca matrix.")
parser.add_argument("--pca_mean_name", default="V3C1_20191228.w2vv.pca.mean.bin", type=str,
                    help="Name of file with pca mean.")

parser.add_argument("--user", default="pcu", type=str,
                    help="Name of user.")
parser.add_argument("--pickle_root", default="pickle", type=str,
                    help="Root of pickle models.")

def load_features(args):
    features = np.fromfile(os.path.join(args.dataset_path, args.features_name), dtype='float32')
    features = features[3:]
    features = features.reshape(int(features.shape[0] / 128), 128)
    return features

def quiet_log(*args, **kwargs):
    pass

def main(args):
    np.random.seed(args.seed)

    log = print
    if args.quiet:
        log = quiet_log

    features = load_features(args)
    kw_init = BoWInitializer(features,
        os.path.join(args.dataset_path, args.keywords_list_name),
        os.path.join(args.dataset_path, args.kw_features_name),
        os.path.join(args.dataset_path, args.kw_bias_name),
        os.path.join(args.dataset_path, args.pca_matrix_name),
        os.path.join(args.dataset_path, args.pca_mean_name)
        )
    with open(args.output_file, "w") as of:
        with open(args.target_file, "r") as f:
            content = f.readlines()
            counter = 0
            content_len = str(len(content))
            for line in content:
                line = line.strip()
                counter += 1
                log(str(counter) + "/" + content_len, line, end="")
                tokens = line.split(";")

                # Parse arguments
                target_id = int(tokens[0])
                text_query = tokens[1].strip()
                iterations = int(tokens[2])
                display_type = tokens[3].lower()
                num_of_likes = list(map(lambda x: int(x), filter(lambda x: len(x) > 0, tokens[4].split(","))))

                # Prepare search structures
                display_gen = TopNDisplay()
                if display_type == "som":
                    display_gen = SOMDisplay(features)

                ranker = BayesianRanker(features, features.shape[0])
                if text_query:
                    ranker._scores = kw_init.score(text_query)

                user_type = args.user.lower()
                if user_type == "pcu":
                    user = RanSamUser(features, target_id, 13)
                elif user_type == "ideal":
                    user = IdealUser(features, target_id)
                elif user_type == "null":
                    user = NullUser()
                elif user_type == "logit":
                    user = LogitUser(features, target_id, pickle_file = os.path.join(args.pickle_root, "smf.all.full.pickle"))

                # Generate first display
                found = -1

                for iteration in range(iterations + 1):
                    display = display_gen.generate(ranker.scores)
                    # Check if found
                    if target_id in display:
                        found = iteration
                        break

                    # Skip feedback on last iteration
                    if iteration < iterations:
                        user._count = num_of_likes[iteration]
                        likes = user.decision(display)
                        ranker.apply_feedback(likes, display)
                        log(".", end="", flush=True)

                of.write(line + ";" + str(found) + "\n")
                of.flush()
                log("DONE", flush=True)

    log("Simulations done")

if __name__ == "__main__":
    args = parser.parse_args([] if "__file__" not in globals() else None)
    main(args)

max_stars_repo_name: siret-junior/somhunter-simulator

max_issues_repo_name: siret-junior/somhunter-simulator

max_forks_repo_name: siret-junior/somhunter-simulator

reason: import numpy


text:
# -*- coding:utf-8 -*-

import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # 禁止显示系统信息

# cpu_count = 4

# 清晰的打印格式
def fancy_print(n=None, c=None, s='#'):
    print(s * 40)
    print(n)
    print(c)
    print(s * 40)
    print() # 避免混淆

# 引入全部模型
from model import *

from sklearn.metrics import roc_curve, auc, roc_auc_score, average_precision_score, f1_score
from sklearn.preprocessing import label_binarize
from sklearn.model_selection import cross_val_score

import tensorflow as tf
import numpy as np
# 打印完整信息
# np.set_printoptions(threshold=np.inf)

from keras.preprocessing.image import ImageDataGenerator
# one-hot encoding
from keras.utils.np_utils import to_categorical

# 因为服务器没有图形界面，所以必须这样弄
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

def test_cnn_dense_resnet(gen_name, model_name, target_name, gene_length):

    # 打印说明，方便检查
    fancy_print('gen_name', gen_name)
    fancy_print('model_name', model_name)
    fancy_print('target_name', target_name)

    ##############################
    #
    # Test set iterator
    #
    ##############################

    datagen = ImageDataGenerator(rescale = 1./255)

    BATCH_SIZE = 32 # 一次大小

    test_generator = datagen.flow_from_directory(directory = 'data/'+target_name+'/png_test/', target_size = (gene_length*2, 5),
                                                 color_mode = 'grayscale',
                                                 batch_size = BATCH_SIZE,
                                                 shuffle = False) # 不需要 shuffle

    ##############################
    #
    # Load the pre-trained model
    #
    ##############################

    if model_name == 'onehot_cnn_one_branch':
        clf = model_onehot_cnn_one_branch(gene_length)
    if model_name == 'onehot_embedding_dense':
        clf = model_onehot_embedding_dense(gene_length)

    if model_name == 'onehot_dense':
        clf = model_onehot_dense(gene_length)
    if model_name == 'onehot_resnet18':
        clf = model_onehot_resnet18(gene_length)
    if model_name == 'onehot_resnet34':
        clf = model_onehot_resnet34(gene_length)

    clf.load_weights('h5_weights/'+gen_name+'/'+model_name+'.h5')

    fancy_print('Load the model ...', 'h5_weights/'+gen_name+'/'+model_name+'.h5', '=')

    ##############################
    #
    # prediction
    #
    ##############################

    # Measure accuracy first
    # score = clf.evaluate_generator(generator = test_generator, steps = len(test_generator))
    # workers = cpu_count, use_multiprocessing = True)
    # 以上不能使用多进程，不然会出错
    # fancy_print('loss & acc', score)

    # Use model.predict to get the predicted probability of the test set
    y_prob = clf.predict_generator(generator = test_generator, steps = len(test_generator))
    # workers = cpu_count, use_multiprocessing = True)
    # 以上不能使用多进程，不然会出错
    # fancy_print('y_prob', y_prob, '.')
    # fancy_print('y_prob.shape', y_prob.shape, '-')

    # Get label
    label_test_tag = test_generator.class_indices
    label_test_name = test_generator.filenames

    # 没有反，这里是对的
    label_test = []
    for i in label_test_name:
        # windows系统
        # label = i.split('\\')[0] # Separate categories
        label = i.split('/')[0] # Separate categories
        # 这里是对的
        label_test.append( int(label_test_tag[label]) ) # Make it into number

    from scipy.stats import pearsonr

    pesr = pearsonr( label_test, [ i[1] for i in y_prob ] )

    label_test = to_categorical(label_test)

    # fancy_print('label_test', label_test, '.')
    # fancy_print('label_test.shape', label_test.shape, '-')

    # Calculate ROC curve and AUC for each category
    fpr = dict()
    tpr = dict()
    roc_auc = dict()

    # Two classification problem
    n_classes = label_test.shape[1] # n_classes = 2
    fancy_print('n_classes', n_classes) # n_classes = 2

    # Draw ROC curve using actual category and predicted probability
    for i in range(n_classes):
        fpr[i], tpr[i], _ = roc_curve(label_test[:, i], y_prob[:, i])
        roc_auc[i] = auc(fpr[i], tpr[i])

    # fancy_print('fpr', fpr)
    # fancy_print('tpr', tpr)
    fancy_print('roc_auc', roc_auc)

    plt.figure()
    lw = 2
    plt.plot(fpr[0], tpr[0], color = 'darkorange',
             lw = lw, label = 'ROC curve (area = %0.6f)' % roc_auc[1])
    plt.plot([0, 1], [0, 1], color = 'navy', lw = lw, linestyle = '--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate'); plt.ylabel('True Positive Rate')
    plt.title('Receiver operating characteristic (ROC)')
    plt.legend(loc = "lower right")

    plt.savefig('result/'+gen_name+'/'+model_name+'/'+target_name+'_rocauc.png')
    # plt.pause(0.01)
    # plt.show()

    fw = open('log.txt','a+')
    import time
    fw.write( time.asctime(time.localtime(time.time())) + '\n' )
    fw.write( gen_name + '\t' + target_name + '\t' + model_name + '\t' + str(roc_auc[1]) + '\t' + str(pesr[0]) + '\n' )
    fw.close()

    fancy_print('roc_auc[1]', roc_auc[1], '-')

def test_cnn_separate(gen_name, model_name, target_name, gene_length):

    ##############################
    #
    # 测试集迭代器
    #
    ##############################

    from keras.preprocessing.image import ImageDataGenerator

    BATCH_SIZE = 32 # 每次大小

    test_datagen = ImageDataGenerator(rescale = 1. / 255)

    # 这个是用来产生label的
    test_generator = test_datagen.flow_from_directory(directory = 'data/'+target_name+'/test_en/', target_size=(gene_length, 5),
                                                      color_mode = 'grayscale',
                                                      class_mode = 'categorical',
                                                      # "categorical"会返回2D的one-hot编码标签, "binary"返回1D的二值标签."sparse"返回1D的整数标签
                                                      batch_size = BATCH_SIZE,
                                                      shuffle = False)  # 不要乱

    def generator_two_test():
        test_generator1 = test_datagen.flow_from_directory(directory = 'data/'+target_name+'/test_en/', target_size = (gene_length, 5),
                                                          color_mode = 'grayscale',
                                                          class_mode = 'categorical', # "categorical"会返回2D的one-hot编码标签, "binary"返回1D的二值标签."sparse"返回1D的整数标签
                                                          batch_size = BATCH_SIZE,
                                                          shuffle = False) # 不要乱

        test_generator2 = test_datagen.flow_from_directory(directory = 'data/'+target_name+'/test_pr/', target_size = (gene_length, 5),
                                                          color_mode = 'grayscale',
                                                          class_mode = 'categorical', # "categorical"会返回2D的one-hot编码标签, "binary"返回1D的二值标签."sparse"返回1D的整数标签
                                                          batch_size = BATCH_SIZE,
                                                          shuffle = False) # 不要乱
        while True:
            out1 = test_generator1.next()
            out2 = test_generator2.next()
            yield [out1[0], out2[0]] # , out1[1]  # 返回两个的组合和结果

    ##############################
    #
    # Loading 已经训练好的模型
    #
    ##############################

    ##############################
    #
    # Model building
    #
    ##############################

    if model_name == 'onehot_cnn_two_branch':
        clf = model_onehot_cnn_two_branch(gene_length)

    # 跳过训练，直接Load the model ...
    from keras.models import load_model
    clf.load_weights('h5_weights/'+gen_name+'/'+model_name+'.h5')

    fancy_print('Load the model ...', 'h5_weights/'+gen_name+'/'+model_name+'.h5', '=')

    ##############################
    #
    # 预测
    #
    ##############################

    # 新加入内容，用来评估模型质量
    # 计算auc和绘制roc_curve
    from sklearn.metrics import roc_curve, auc
    import matplotlib.pyplot as plt

    # 先测算准确度
    # score = clf.evaluate_generator(generator = generator_two_test(), steps = len(test_generator))
    # fancy_print('loss & acc', score)

    # 打印所有内容
    # np.set_printoptions(threshold = np.inf)

    # 利用model.predict获取测试集的预测概率
    y_prob = clf.predict_generator(generator = generator_two_test(), steps = len(test_generator))
    # workers = cpu_count, use_multiprocessing = True)
    # 以上不能使用多进程，不然会出错
    # fancy_print('y_prob', y_prob, '.')
    # fancy_print('y_prob.shape', y_prob.shape, '-')

    # 获得label
    label_test_tag = test_generator.class_indices
    label_test_name = test_generator.filenames

    label_test = []
    for i in label_test_name:
        # windows系统
        # label = i.split('\\')[0] # Separate categories
        label = i.split('/')[0] # Separate categories
        label_test.append(int(label_test_tag[label])) # 弄成数字呗

    from scipy.stats import pearsonr

    # print(label_test)
    # print([ i[1] for i in y_prob ])

    # print(len(label_test))
    # print(len([ i[1] for i in y_prob ]))

    pesr = pearsonr( label_test, [ i[1] for i in y_prob ] )

    from keras.utils.np_utils import to_categorical
    label_test = to_categorical(label_test)

    # fancy_print('label_test', label_test, '.')
    # fancy_print('label_test.shape', label_test.shape, '-')

    # 为每个类别计算ROC曲线和AUC
    fpr = dict()
    tpr = dict()
    roc_auc = dict()

    # 二分类问题
    n_classes = label_test.shape[1] # n_classes = 2
    # fancy_print('n_classes', n_classes) # n_classes = 2

    # 使用实际类别和预测概率绘制ROC曲线
    for i in range(n_classes):
        fpr[i], tpr[i], _ = roc_curve(label_test[:, i], y_prob[:, i])
        roc_auc[i] = auc(fpr[i], tpr[i])

    # fancy_print('fpr', fpr)
    # fancy_print('tpr', tpr)
    # fancy_print('cnn_roc_auc', roc_auc)

    plt.figure()
    lw = 2
    plt.plot(fpr[0], tpr[0], color = 'darkorange',
             lw = lw, label = 'ROC curve (area = %0.6f)' % roc_auc[1])
    plt.plot([0, 1], [0, 1], color = 'navy', lw = lw, linestyle = '--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate'); plt.ylabel('True Positive Rate')
    plt.title('Receiver operating characteristic (ROC)')
    plt.legend(loc = "lower right")

    plt.savefig('result/'+gen_name+'/'+model_name+'/'+target_name+'_rocauc.png')
    # plt.pause(0.01)

    # plt.show()

    fw = open('log.txt','a+')
    import time
    fw.write( time.asctime(time.localtime(time.time())) + '\n' )
    fw.write( gen_name + '\t' + target_name + '\t' + model_name + '\t' + str(roc_auc[1]) + '\t' + str(pesr[0]) + '\n' )
    fw.close()

    fancy_print('roc_auc[1]', roc_auc[1], '-')

def test_embedding(gen_name, model_name, target_name):

    # 打印说明，方便检查
    fancy_print('gen_name', gen_name)
    fancy_print('model_name', model_name)

    physical_devices = tf.config.experimental.list_physical_devices('GPU')
    tf.config.experimental.set_memory_growth(physical_devices[0], True)

    ##############################
    #
    # Model building
    #
    ##############################

    if model_name == 'embedding_cnn_one_branch':
        model = model_embedding_cnn_one_branch()
    if model_name == 'embedding_cnn_two_branch':
        model = model_embedding_cnn_two_branch()
    if model_name == 'embedding_dense':
        model = model_embedding_dense()

    if model_name == 'onehot_embedding_cnn_one_branch':
        model = model_onehot_embedding_cnn_one_branch()
    if model_name == 'onehot_embedding_cnn_two_branch':
        model = model_onehot_embedding_cnn_two_branch()

    model.load_weights('h5_weights/'+gen_name+'/'+model_name+'.h5')

    fancy_print('Load the model ...', 'h5_weights/'+gen_name+'/'+model_name+'.h5', '=')

    # Data_dir = '/home/ycm/data/%s/' % name
    test = np.load('data/'+target_name+'/embedding_test.npz')
    fancy_print('Loading npz', 'data/'+target_name+'/embedding_test.npz', '=')
    X_en_tes, X_pr_tes, y_test = test['X_en_tes'], test['X_pr_tes'], test['y_tes']
    # fancy_print('y_test', y_test[:100], '=')

    ##############################
    #
    # prediction
    #
    ##############################

    # 利用model.predict获取测试集的预测值
    y_score = model.predict([X_en_tes, X_pr_tes])
    # workers = cpu_count, use_multiprocessing = True)
    # 以上不能使用多进程，不然会出错
    # fancy_print('y_score', y_score, '.')

    # 计算F1
    # y_pred = y_score.argmax(axis=-1)
    # fancy_print('f1', f1_score(y_test, y_pred, average='binary'), '.')

    # 为每个类别计算ROC曲线和AUC
    fpr = dict()
    tpr = dict()
    roc_auc = dict()

    # 二进制化输出
    # 转成独热码
    y_test = y_test.tolist()
    y_temp_1 = []; y_temp_0 = []
    for each in y_test:
        if each == 1: y_temp_1.append(1); y_temp_0.append(0)
        else: y_temp_1.append(0); y_temp_0.append(1)
    y_test = [y_temp_0, y_temp_1]; y_test = np.transpose(y_test)
    # fancy_print('y_test', y_test, '.')
    # fancy_print('y_test.shape', y_test.shape)

    # y_test = label_binarize(y_test, classes=[0, 1, 2])  # 二分类
    # fancy_print('y_test', y_test)

    # 二进制化输出
    # 转成独热码，不需要了，这次直接就是两组

    y_score = y_score.tolist()
    y_temp_1 = []; y_temp_0 = []
    for each in y_score:
        y_temp_1.append(float(each[0]))
        y_temp_0.append(1-float(each[0]))
    y_score = [y_temp_0, y_temp_1]; y_score = np.transpose(y_score)
    # fancy_print('y_score', y_score)
    # fancy_print('y_score.shape', y_score.shape)

    # y_score = label_binarize(y_score, classes=[0, 1, 2])  # 二分类
    # fancy_print('y_score', y_score)

    from scipy.stats import pearsonr

    pesr = pearsonr( [ i[1] for i in y_test ], [ i[1] for i in y_score ] )

    n_classes = y_test.shape[1] # n_classes = 2
    # fancy_print('n_classes', n_classes)

    for i in range(n_classes):
        fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])
        roc_auc[i] = auc(fpr[i], tpr[i])

    # fancy_print('fpr', fpr)
    # fancy_print('tpr', tpr)
    # fancy_print('roc_auc', roc_auc)

    plt.figure()
    lw = 2
    plt.plot(fpr[0], tpr[0], color='darkorange',
             lw=lw, label='ROC curve (area = %0.6f)' % roc_auc[1])
    plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver operating characteristic(ROC)')
    plt.legend(loc="lower right")

    plt.savefig('result/'+gen_name+'/'+model_name+'/'+target_name+'_rocauc.png')
    # plt.pause(0.01)

    fw = open('log.txt','a+')
    import time
    fw.write( time.asctime(time.localtime(time.time())) + '\n' )
    fw.write( gen_name + '\t' + target_name + '\t' + model_name + '\t' + str(roc_auc[1]) + '\t' + str(pesr[0]) + '\n' )
    fw.close()

    fancy_print('roc_auc[1]', roc_auc[1], '-')

########################################
#
#   本模块没有代码运行
#
########################################
if __name__ == '__main__':

    pass

max_stars_repo_name: lichen-lab/DeepChrInteract

max_issues_repo_name: lichen-lab/DeepChrInteract

max_forks_repo_name: lichen-lab/DeepChrInteract

reason: import numpy,from scipy


text:
import json
import torch
import numpy as np

from get_input_args import get_predict_args
from torchvision import models
from PIL import Image

alexnet = models.alexnet(pretrained=True)
vgg16 = models.vgg16(pretrained=True)
densenet121 = models.densenet121(pretrained=True)

models = {"alexnet": alexnet, "vgg": vgg16, "densenet": densenet121}

def load_checkpoint(filepath, model):
    checkpoint = torch.load(filepath)
    model.classifier = checkpoint['classifier']
    model.load_state_dict(checkpoint['state_dict'])

    return model

def process_image(image):
    im = Image.open(image)
    im.thumbnail((256, 256))

    # Center crop
    width, height = im.size
    desired_width, desired_height = 224, 224
    left = (width-desired_width) / 2
    upper = (height-desired_height) / 2
    right = (width+desired_width) / 2
    lower = (height+desired_height) / 2

    im = im.crop((left, upper, right, lower))

    # Normalize
    np_image = np.array(im)
    np_image = np_image / 255
    means = np.array([0.485, 0.456, 0.406])
    st_dev = np.array([0.229, 0.224, 0.225])
    np_image = np_image - means
    np_image = np_image / st_dev
    np_image = np_image.transpose(2, 0 , 1)
    return np_image

def predict(image_path, model, topk, gpu):
    device = torch.device("cuda" if torch.cuda.is_available() and gpu else "cpu")
    model = model.to(device)

    image = torch.tensor(process_image(image_path)).type(torch.FloatTensor)
    image.unsqueeze_(0)
    inputs = image.to(device)

    logps = model(inputs)
    ps = torch.exp(logps)
    top_ps, top_class = ps.topk(topk, dim=1)
    return image.squeeze(), top_ps, top_class

def get_class_name(top_class):
    with open('cat_to_name.json', 'r') as f:
        cat_to_name = json.load(f)

    return [cat_to_name[str(var)] for var in top_class.cpu().numpy()[0]]

def main():
    predict_args = get_predict_args()
    model = models[predict_args.arch]
    model = load_checkpoint(predict_args.checkpoint, model)

    image_tensor, top_ps, top_class = predict(predict_args.data_dir, model, predict_args.top_k, predict_args.gpu)

    class_name = get_class_name(top_class)

    print(f"Top classes: {class_name}")
    print(f"Top probabilities: {top_ps}")

if __name__ == "__main__":
    main()

max_stars_repo_name: matehr/ImageClassifier

max_issues_repo_name: matehr/ImageClassifier

max_forks_repo_name: matehr/ImageClassifier


text:
from __future__ import print_function
import matplotlib.pyplot as plt
from scipy.interpolate import interp1d
import os,sys
import hmvec as hm # Git clone and pip install as in readme from github.com/msyriac/hmvec
import numpy as np

zgalaxy = 0.6 # delta-function lens population
zsource = 1.0 # delta-function source population
ngal = 1e-4 # number density of lenses per mpc3 (to solve for stellar mass threshold in HOD)

# We set up the grid to integrate on
zmax = zsource + 1
numzs = 20
zs = np.linspace(0.01,zmax,numzs) # redshifts
ms = np.geomspace(2e10,1e17,100) # masses
ks = np.geomspace(1e-4,100,1001) # wavenumbers

# Initialize halo model
hcos = hm.HaloModel(zs,ks,ms=ms)
# We add an HOD which should be CMASS-like at z=zgalaxy
hcos.add_hod("g",ngal=ngal+zs*0.,corr="max")
print("Galaxy bias at zgalaxy=",zgalaxy, " is ",interp1d(zs,hcos.hods['g']['bg'])(zgalaxy))
# We add a gas profile
hcos.add_battaglia_profile("electron",family="AGN",xmax=50,nxs=30000)

"""
Galaxy-galaxy lensing
"""
Pgn = hcos.get_power("g","nfw",verbose=False)
Pge = hcos.get_power("g","electron",verbose=False )
Pgm = hcos.total_matter_galaxy_power_spectrum(Pgn,Pge)

# Compare P(k)
for i in [0,-1]:
    plt.plot(ks,Pgm[i]/Pgn[i],color="C%d" % (i+2))
plt.axhline(y=1,ls='--')
plt.xlabel('$k \\mathrm{Mpc}^{-1}$')
plt.ylabel('$P_{gm}^{\\rm{fb}}/P_{gm}^{\\rm{no-fb}}$')
plt.xscale('log')
plt.yscale('linear')
plt.savefig("pkgm.png",bbox_inches='tight')

# Do Limber integrals
ells = np.linspace(80,6000,100)
Ckg0 = hcos.C_kg(ells,zs,ks,Pgn,gzs=zgalaxy,lzs=zsource)
Ckg = hcos.C_kg(ells,zs,ks,Pgm,gzs=zgalaxy,lzs=zsource)

# Compare Ckg
plt.clf()
plt.plot(ells,Ckg/Ckg0)
plt.axhline(y=1,ls='--')
plt.xlabel('$\\ell$')
plt.ylabel('$C_{kg}^{\\rm{fb}}/C_{kg}^{\\rm{no-fb}}$')
plt.xscale('linear')
plt.yscale('linear')
plt.savefig("clkg.png",bbox_inches='tight')

"""
Cosmic Shear
"""

Pnn = hcos.get_power("nfw",verbose=False)
Pne = hcos.get_power("nfw","electron",verbose=False )
Pee = hcos.get_power("electron","electron",verbose=False )
Pmm = hcos.total_matter_power_spectrum(Pnn,Pne,Pee)

# Do Limber integral
Ckk0 = hcos.C_kk(ells,zs,ks,Pnn,lzs1=zsource,lzs2=zsource)
Ckk = hcos.C_kk(ells,zs,ks,Pmm,lzs1=zsource,lzs2=zsource)

# Compare Ckk
plt.clf()
plt.plot(ells,Ckk/Ckk0)
plt.axhline(y=1,ls='--')
plt.xlabel('$\\ell$')
plt.ylabel('$C_{kk}^{\\rm{fb}}/C_{kk}^{\\rm{no-fb}}$')
plt.xscale('linear')
plt.yscale('linear')
plt.savefig("clkk.png",bbox_inches='tight')


text:
import pickle
import numpy as np
from abc import ABC, abstractmethod
from typing import Dict, List
from enum import Enum
from nxs_types import DataModel

from nxs_types.infer import NxsInferRequest

class BackendDispatcherType(str, Enum):
    BASIC = "basic"
    BASIC_SLA = "basic_sla"
    BASIC_MONITORING = "basic_monitoring"

class BackendDispatcherExceptionInvalidType(Exception):
    pass

class DispatcherResult(DataModel):
    to_schedule: List[NxsInferRequest] = []
    to_delay: List[NxsInferRequest] = []
    to_drop: List[NxsInferRequest] = []

class BackendDispatcher(ABC):
    def __init__(self, extra_params={}) -> None:
        super().__init__()
        self.extra_params = extra_params

    @abstractmethod
    def dispatch(self, requests: List[NxsInferRequest]) -> DispatcherResult:
        raise NotImplementedError

    @abstractmethod
    def update_stats(self, stats: Dict = {}) -> None:
        raise NotImplementedError

    @abstractmethod
    def update_stats_from_global_dispatcher(self, stats: Dict = {}) -> None:
        raise NotImplementedError

    @abstractmethod
    def report_stats_to_global_dispatcher(self) -> Dict:
        raise NotImplementedError

    @abstractmethod
    def get_stats_summary(self) -> Dict:
        raise NotImplementedError

class BackendBasicDispatcher(BackendDispatcher):
    def __init__(self, extra_params={}) -> None:
        super().__init__(extra_params=extra_params)

    def dispatch(self, requests: List[NxsInferRequest]) -> DispatcherResult:
        return DispatcherResult(to_schedule=requests)

    def update_stats(self, stats: Dict = {}) -> None:
        self.last_stats = stats

    def update_stats_from_global_dispatcher(self, stats: Dict = {}) -> None:
        pass

    def report_stats_to_global_dispatcher(self) -> Dict:
        try:
            last_stats = self.last_stats
        except:
            last_stats = {}

        return last_stats

    def get_stats_summary(self) -> Dict:
        return {}

class BackendBasicMonitoringDispatcher(BackendBasicDispatcher):
    def __init__(self, extra_params={}, max_cache_size: int = 30) -> None:
        super().__init__(extra_params)
        self.max_cache_size = max_cache_size
        self.states_cache: List = []
        self.total_processed_reqs = 0

    def dispatch(self, requests: List[NxsInferRequest]) -> DispatcherResult:
        return DispatcherResult(to_schedule=requests)

    def update_stats(self, stats: Dict = {}) -> None:
        self.last_stats = stats

        self.total_processed_reqs += stats.get("num_reqs", 0)

        if len(self.states_cache) > self.max_cache_size:
            self.states_cache.pop(0)

        self.states_cache.append(stats)

    def update_stats_from_global_dispatcher(self, stats: Dict = {}) -> None:
        pass

    def report_stats_to_global_dispatcher(self) -> Dict:
        return {}

    def get_stats_summary(self) -> Dict:
        output_processes = {}

        for stats in self.states_cache:
            output_pid = stats["output_pid"]
            if output_pid not in output_processes:
                output_processes[output_pid] = []

            output_processes[output_pid].append(stats)

        fps = 0
        latency_stats = []
        for pid in output_processes:
            fps += np.mean([stats["fps"] for stats in output_processes[pid]])
            latency_stats.extend([stats["latency"] for stats in output_processes[pid]])

        latency = {
            "mean": np.mean([stats["mean"] for stats in latency_stats]),
            "min": np.mean([stats["min"] for stats in latency_stats]),
            "max": np.mean([stats["max"] for stats in latency_stats]),
        }

        response = {
            "total_reqs": self.total_processed_reqs,
            "fps": fps,
            "latency": latency,
        }

        return response

class BackendBasicSlaDispatcher(BackendDispatcher):
    def __init__(self, extra_params={}) -> None:
        super().__init__(extra_params=extra_params)
        self.input_process = extra_params["input_process"]

    def dispatch(self, requests: List[NxsInferRequest]) -> DispatcherResult:
        sla_requests: List[NxsInferRequest] = []
        slas: List[float] = []
        nonsla_requests: List[NxsInferRequest] = []

        for request in requests:
            try:
                # extra_params = json.loads(request.extra_params)
                extra_params = pickle.loads(request.extra_params)
                assert isinstance(extra_params, Dict)
            except:
                extra_params = {}

            if "sla" in extra_params:
                sla = 0
                try:
                    sla = float(extra_params.get("sla", 0))
                except:
                    sla = 0

                if sla > 0:
                    sla_requests.append(request)
                    slas.append(sla)
                else:
                    # treat invalid requests as non-sla requests
                    nonsla_requests.append(request)
            else:
                nonsla_requests.append(request)

        # sort sla-requests based on sla-requirements
        zipped_lists = zip(slas, sla_requests)
        sorted_pairs = sorted(zipped_lists)
        tuples = zip(*sorted_pairs)
        slas, sla_requests = tuples

        # schedule sla_requests first
        to_schedule = []
        to_schedule.extend(sla_requests)
        to_schedule.extend(nonsla_requests)

        return DispatcherResult(to_schedule=to_schedule)

    def update_stats(self, stats: Dict = {}) -> None:
        self.last_stats = stats

    def update_stats_from_global_dispatcher(self, stats: Dict = {}) -> None:
        pass

    def report_stats_to_global_dispatcher(self) -> Dict:
        try:
            last_stats = self.last_stats
        except:
            last_stats = {}

        return last_stats

    def get_stats_summary(self) -> Dict:
        return {}

class BackendDispatcherFactory:
    @staticmethod
    def create_dispatcher(type: BackendDispatcherType, **kwargs) -> BackendDispatcher:
        if type == BackendDispatcherType.BASIC:
            return BackendBasicDispatcher(**kwargs)
        elif type == BackendDispatcherType.BASIC_MONITORING:
            return BackendBasicMonitoringDispatcher(**kwargs)

        raise BackendDispatcherExceptionInvalidType

max_stars_repo_name: microsoft/nxs

max_issues_repo_name: microsoft/nxs

max_forks_repo_name: microsoft/nxs


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import, print_function, division

import os
from os import path
from dotenv import load_dotenv

import numpy as np
import tensorflow as tf

HOME_PATH = os.environ.get("HOME")
load_dotenv(path.join(HOME_PATH, '.nnlab.env'))

# Float precision
FLOAT_PRECISION = os.environ.get("FLOAT_PRECISION", "32")
if int(FLOAT_PRECISION) == 16:
    TF_FLOAT_TYPE = tf.float16
    FLOAT_TYPE = "float16"
elif int(FLOAT_PRECISION) == 64:
    TF_FLOAT_TYPE = tf.float64
    FLOAT_TYPE = "float64"
else:
    TF_FLOAT_TYPE = tf.float32
    FLOAT_TYPE = "float32"

# Int precision
INT_PRECISION = os.environ.get("INT_PRECISION", "64")
if int(INT_PRECISION) == 16:
    TF_INT_TYPE = tf.int16
    INT_TYPE = "int16"
elif int(INT_PRECISION) == 64:
    TF_INT_TYPE = tf.int64
    INT_TYPE = "int64"
else:
    TF_INT_TYPE = tf.int32
    INT_TYPE = "int32"

# Random vars
if "global_numpy_rand" not in globals():
    global_numpy_rand = np.random.RandomState(int(os.environ.get("RANDOM_SEED", "3")))

max_stars_repo_name: nlab-mpg/nnlab

max_issues_repo_name: nlab-mpg/nnlab

max_forks_repo_name: nlab-mpg/nnlab


text:
import multiprocessing
import pandas as pd
import numpy as np
from tqdm import tqdm
from gensim.models import Doc2Vec
from sklearn import utils
from gensim.models.doc2vec import TaggedDocument
import re
import nltk
from gensim.test.test_doc2vec import ConcatenatedDoc2Vec
nltk.download('punkt')

def tokenize_text(text):
    tokens = []
    for sent in nltk.sent_tokenize(text):
        for word in nltk.word_tokenize(sent):
            if len(word) < 2:
                continue
            tokens.append(word.lower())
    return tokens

def get_vectors(model, tagged_docs):
    sents = tagged_docs.values
    targets, regressors = zip(*[(doc.tags[0], model.infer_vector(doc.words, epochs=20)) for doc in sents])
    return targets, regressors

cores = multiprocessing.cpu_count()

training_set = pd.read_json('../processed_data/train_set.json')
test_set = pd.read_json('../processed_data/test_set.json')

train_tagged_summary = training_set.apply(
    lambda r: TaggedDocument(words=tokenize_text(r['summary']), tags=[r.label]), axis=1)
train_tagged_document = training_set.apply(
    lambda r: TaggedDocument(words=tokenize_text(r['document']), tags=[r.label]), axis=1)

model_dbow = Doc2Vec(dm=0, vector_size=300, negative=5, hs=0, min_count=2, sample = 0, workers=cores)

print("\n> Building model vocab")

model_dbow.build_vocab([x for x in tqdm(train_tagged_summary.values)] + [x for x in tqdm(train_tagged_document.values)])

print("\n> Training bag of words model")
for epoch in range(30):
    model_dbow.train(utils.shuffle([x for x in tqdm(train_tagged_summary.values, desc = 'epoch {}'.format(epoch))]), total_examples=len(train_tagged_summary.values), epochs=30)
    model_dbow.alpha -= 0.002
    model_dbow.min_alpha = model_dbow.alpha

model_dmm = Doc2Vec(dm=1, dm_mean=1, vector_size=300, window=10, negative=5, min_count=1, workers=cores, alpha=0.065, min_alpha=0.065)

print("\n> Building model vocab")

model_dmm.build_vocab([x for x in tqdm(train_tagged_summary.values)] + [x for x in tqdm(train_tagged_document.values)])

print("\n> Training Distributed memory model")
for epoch in range(30):
    model_dmm.train(utils.shuffle([x for x in tqdm(train_tagged_summary.values, desc = 'epoch {}'.format(epoch))]), total_examples=len(train_tagged_summary.values), epochs=30)
    model_dmm.alpha -= 0.002
    model_dmm.min_alpha = model_dmm.alpha

new_model = ConcatenatedDoc2Vec([model_dbow, model_dmm])

test_tagged_summary = test_set.apply(
    lambda r: TaggedDocument(words=tokenize_text(r['summary']), tags=[1]), axis=1)
test_tagged_document = test_set.apply(
    lambda r: TaggedDocument(words=tokenize_text(r['document']), tags=[1]), axis=1)

print('\n> Generating final vectors')
y_train, X_train = get_vectors(new_model, train_tagged_summary)
y_test, X_test = get_vectors(new_model, test_tagged_summary)

y_train, X_train_document = get_vectors(new_model, train_tagged_document)
y_test, X_test_document = get_vectors(new_model, test_tagged_document)

print('> Saving result')
training_result = pd.DataFrame()
training_result.insert(0, "summary_embedding", X_train)
training_result.insert(0, "document_embedding", X_train_document)
training_result.insert(2, "labels", training_set['label'])

testing_result = pd.DataFrame()
testing_result.insert(0, "summary_embedding", X_test)
testing_result.insert(0, "document_embedding", X_test_document)

training_result.to_json("../processed_data/d2v_training_20.json")
testing_result.to_json("../processed_data/d2v_testing_20.json")


text:
import numpy as np
import math

def gaussian_value(x, mu, sigma):
    # 2*pi is omitted
    det_sigma = np.linalg.det(sigma)
    sigma_inv = np.linalg.inv(sigma)
    exponent = -0.5*np.dot(np.transpose(x-mu), np.dot(sigma_inv, x-mu))
    return math.exp(exponent)/math.sqrt(det_sigma)

def gaussian_pos_prob(X, Mu, Sigma, Phi):
    '''
    GAUSSIAN_POS_PROB Posterior probability of GDA.
    Compute the posterior probability of given N data points X
    using Gaussian Discriminant Analysis where the K gaussian distributions
    are specified by Mu, Sigma and Phi.
    Inputs:
        'X'     - M-by-N numpy array, N data points of dimension M.
        'Mu'    - M-by-K numpy array, mean of K Gaussian distributions.
        'Sigma' - M-by-M-by-K  numpy array (yes, a 3D matrix), variance matrix of
                  K Gaussian distributions.
        'Phi'   - 1-by-K  numpy array, prior of K Gaussian distributions.
    Outputs:
        'p'     - N-by-K  numpy array, posterior probability of N data points
                with in K Gaussian distribsubplots_adjustutions.
    '''
    N = X.shape[1]
    K = Phi.shape[0]
    p = np.zeros((N, K))
    #Your code HERE

    # begin answer
    for i in range(N):
        # calculate P(x_i)
        P_xi = 0
        for j in range(K):
            # consider p[i][j]
            p[i][j] = gaussian_value(X[:,i], Mu[:,j], Sigma[:,:,j])*Phi[j]
            P_xi += p[i][j]
        # regularization
        p[i] = p[i] / P_xi
    # end answer

    return p

max_stars_repo_name: Junlin-Yin/Data-Mining-Homework

max_issues_repo_name: Junlin-Yin/Data-Mining-Homework

max_forks_repo_name: Junlin-Yin/Data-Mining-Homework


text:
import numpy as np
import cv2
from .image import img_write, img_visualize, img_resize, img_crop, img_pad

##############################################
# mask io
##############################################
def mask_parse(annotation, gt_masks, gt_mask_polys, gt_poly_lens, dataset):
    """
    Parse ground-truth mask in an annotation dict. There is no return in this
    function, because the `gt_masks`, `gt_mask_polys`, `gt_poly_lens` are
    lists, and if they have append element in this function, the change will
    be with them automatically, do not need to return.
    And because the mask is the whole image size, and actually, when we get the
    mask target of the object, we do not need the whole mask, we just need the
    exact piece of mask, so we do not seed the mask to GPU, we re-generate the
    mask target in the `mask_head` module, and seed it to the GPU where the
    `predicted_masks` is.

    Args:
        annotation (dict): The annotation dict for an image.
        gt_masks (list): The list of ground truth masks, each element in this
            list is will generate a ground truth binary mask
        gt_mask_polys (list): The list of ground truth poly coordinates, each
            element in this list is a couple of coordinates for each point in
            the polygons.
        gt_poly_lens (list): Each element in this list is number of points in
            the polygons.
        dataset (:obj:dataset): The dataset object, e.g. COCO dataset object.
    """

    # annToMask: first use `annToRLE` to compress polygons, and then use
    # `decode` convert the compressed polygon into a binary mask.
    gt_masks.append(dataset.annToMask(annotation))
    mask_polys = [
        p for p in annotation['segmentation'] if len(p) >= 6
    ]  # valid polygons have >= 3 points (6 coordinates)
    poly_lens = [len(p) for p in mask_polys]
    gt_mask_polys.append(mask_polys)
    gt_poly_lens.extend(poly_lens)

##############################################
# mask visualize
##############################################
def mask_visualize(img_array, masks, inds,
                   mask_color=(0, 255, 0), alpha=0.5,
                   show=True, win_name='', wait_time=0, out_file=None):
    """
    Visualize the predicted mask with `inds`

    Args:
        img_array (ndarray): The image with `bbox` and `text` to be displayed.
        masks (ndarray, np.uint8): The predicted masks for each `bbox`
        inds (ndarray of bool or `[]`): The `inds` get from
            `:func:bbox_visualize`
        mask_color (tuple): color of bbox lines.
        alpha (float): the opacity of the mask contour
        show (bool):Whether to show the image
        win_name (str): The window name
        wait_time (int): value of waiting time to display.
        out_file (str or None): The filename to write the image.
    """
    assert masks.ndim == 3
    assert inds.ndim == 1
    masks = masks.astype(np.uint8)

    output_img = img_array.copy()
    if len(inds) > 0:
        masks = masks[inds, ...]

    for mask in masks:
        _, contours, hierarchy = cv2.findContours(
            mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
        cv2.fillPoly(img_array, pts=contours, color=mask_color)
    # `:func:cv2.addWeighted`: dst = src1 * alpha + src2 * beta + gamma
    cv2.addWeighted(src1=img_array, alpha=alpha, src2=output_img,
                    beta=1 - alpha, gamma=0, dst=output_img)

    if show:
        img_visualize(output_img, win_name, wait_time)
    if out_file is not None:
        img_write(output_img, out_file)

##############################################
# mask resize
##############################################
def mask_resize(mask,
                scale_factor=None,
                return_scale=False,
                interpolation='nearest'):
    """
    Resize the binary mask according to the `scale_factor` that used to
    resize the image. Because `mask` is a binary image, so this function is
    the Same as `:func:img_resize`

    Args:
        mask (ndarray, np.uint8): Binary image for object, the shape of
        `mask` is `[h, w]`
        scale_factor: same as `:func:img_resize`
        return_scale: same as `:func:img_resize`
        interpolation: same as `:func:img_resize`

    Returns:
        resized_mask (ndarray, np.uint8): resized binary mask
    """
    assert mask.ndim == 2
    mask = mask.astype(np.uint8)

    resized_mask = img_resize(mask,
                              scale_factor=scale_factor,
                              return_scale=return_scale,
                              interpolation=interpolation)
    return resized_mask

##############################################
# mask flip
##############################################
def mask_flip(mask, flipped_flag=True, direction="horizontal"):
    """
    Flip the binary mask when the `flipped_flag` is True. And flip the
    binary mask in the `direction` of flipping image.

    Args:
        mask (ndarray, np.uint8): Binary image for object, the shape of
        `mask` is `[h, w]`
        flipped_flag (bool): Whether the image has flipped or not, this flag
            comes from the returns of `:func:img_flip`
        direction (str): same as the param in `:func:img_flip`

    Returns:
        flipped_mask (ndarray, np.uint8): flipped binary mask
    """
    assert mask.ndim == 2
    mask = mask.astype(np.uint8)

    if not flipped_flag:
        return mask
    else:
        if direction == 'horizontal':
            flipped_mask = np.flip(mask, 1)
        else:
            flipped_mask = np.flip(mask, 0)
    return flipped_mask

##############################################
# mask crop
##############################################
def mask_crop(mask, size_crop, min_w=0, min_h=0):
    """
    Crop the binary mask, same as `:func:img_crop`

    Args:
        mask (ndarray, np.uint8): Binary image for object, the shape of
        `mask` is `[h, w]`
        size_crop (tuple): same as `:func:img_crop`
        min_w (int): same as `:func:img_crop`
        min_h (int): same as `:func:img_crop`

    Returns:
        cropped_mask (ndarray, np.uint8): cropped binary mask
    """
    assert mask.ndim == 2
    mask = mask.astype(np.uint8)

    return img_crop(mask, size_crop, min_w=min_w, min_h=min_h)

##############################################
# mask pad
##############################################
def mask_pad(mask, expected_shape, pad_val=0):
    """
    Pad the binary mask, same as `:func:img_pad`

    Args:
        mask (ndarray, np.uint8): Binary image for object, the shape of
        `mask` is `[h, w]`
        expected_shape (tuple): same as `:func:img_pad`
        pad_val (number or sequence): same as `:func:img_pad`

    Returns:
        padded_mask (ndarray, np.uint8): padded binary mask
    """
    assert mask.ndim == 2
    mask = mask.astype(np.uint8)

    return img_pad(mask, expected_shape, pad_val=pad_val)


text:
"""Classification with abstention plotting functions."""

import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.colors
import seaborn as sns
import palettable
import numpy as np
from collections import Counter
import cartopy as ct
import cartopy.crs as ccrs
import cartopy.feature as cfeature
import numpy.ma as ma

import metrics

__author__ = "Elizabeth A. Barnes and Randal J. Barnes"
__date__ = "January 11, 2021"

mpl.rcParams['figure.facecolor'] = 'white'
mpl.rcParams['figure.dpi'] = 150
dpiFig = 300.

# ----------------------------------------------------
def plot_results(EXP_NAME, history, exp_info, saveplot=True, showplot=True):

    loss_type, n_epochs, abst_setpoint, spinup_epochs, hiddens, lr_init, lr_epoch_bound, batch_size, network_seed = exp_info

#     if exp_info[0].find('PID')>=0:
#         loss_type, n_epochs, abst_setpoint, spinup_epochs, hiddens, lr_init, lr_epoch_bound, batch_size, network_seed = exp_info
#     else:
#         raise ValueError('no such loss_type')

    trainColor = (117/255., 112/255., 179/255., 1.)
    valColor = (231/255., 41/255., 138/255., 1.)
    FS = 7
    plt.figure(figsize=(15, 7))

    #---------- plot loss -------------------
    plt.subplot(2,3,1)
    plt.plot(history.history['loss'], 'o', color=trainColor, label='training loss')
    plt.plot(history.history['val_loss'], 'o', color=valColor, label='validation loss')
    if 'prediction_loss' in history.history:
        plt.plot(history.history['prediction_loss'], 'x', color=trainColor, label='training nonabstaining loss')
        plt.plot(history.history['val_prediction_loss'], 'x', color=valColor, label='validation nonabstaining loss')
    plt.title('Loss')
    plt.xlabel('epoch')
    plt.grid(True)
    plt.legend(frameon=True, fontsize=FS)
    plt.xlim(-2, n_epochs+2)
#     plt.ylim(0,1.5)

    # ---------- plot abstention -------------------
    plt.subplot(2, 3, 2)
    if loss_type!='DNN':
        plt.axhline(abst_setpoint, linestyle='-', color='gray', linewidth=3.0)

    plt.plot(history.history['abstention_fraction'], 'o', color=trainColor, label='training')
    plt.plot(history.history['val_abstention_fraction'], 'o', color=valColor, label='validation')
    plt.title('Abstention Fraction')
    plt.xlabel('epoch')
    plt.legend(frameon=True, fontsize=FS)
    plt.ylim(0, 1.02)
    plt.grid(True)
    plt.xlim(-2, n_epochs+2)

    # ---------- plot accuracy -------------------
    plt.subplot(2, 3, 3)

    plt.plot(history.history['prediction_accuracy'], 'o', color=trainColor, label='training')
    plt.plot(history.history['val_prediction_accuracy'], 'o', color=valColor, label='validation')
    plt.title('Prediction Accuracy (nonabstention)')
    plt.xlabel('epoch')
    plt.legend(frameon=True, fontsize=FS)
    plt.ylim(0, 1.02)
    plt.grid(True)
    plt.xlim(-2, n_epochs+2)

    # ---------- report parameters -------------------
    plt.subplot(2, 3, 4)
    plt.ylim(0, 1)

    text = (
            "\n"
            + f"NETWORK PARAMETERS\n"
            + f"  hiddens        = {hiddens}\n"
            + f"  lr_init        = {lr_init}\n"
            + f"  lr_epoch_bound = {lr_epoch_bound}\n"
            + f"  n_epochs       = {n_epochs}\n"
            + f"  batch_size     = {batch_size}\n"
            + f"  network_seed   = {network_seed}\n"
    )

    if loss_type != 'DNN':
        text += (
            "\n"
            + f"PID ABSTENTION PARAMETERS\n"
            + f"  spinup_epochs  = {spinup_epochs}\n"
            + f"  abst_setpoint  = {abst_setpoint}\n"
        )
    else:
        text += "\nUNKNOWN LOSS TYPE\n"

    plt.text(0.01, 0.95, text, fontfamily='monospace', fontsize='small', va='top')

    plt.axis('off')

    # ---------- plot alpha -------------------
    if 'alpha_value' in history.history:
        plt.subplot(2, 3, 5)
        plt.plot(history.history['alpha_value'], 'o', color='k')
        plt.title('Alpha')
        plt.xlabel('epoch')
        plt.xlim(-2, n_epochs+2)

    # ---------- Make and save the plot -------------------
    plt.tight_layout()
    if saveplot==True:
        plt.savefig('figures/model_diagnostics/' + EXP_NAME + '.png', dpi=dpiFig)
    if showplot==False:
        plt.close('all')
    else:
        plt.show()

# ----------------------------------------------------
def plot_data(X, z, gameboard, title_str, marker_size=4, plot_type='pred'):

    # get colors and legends
    cmap = getattr(palettable.cartocolors.qualitative, 'Bold_' + str(np.max([gameboard.nlabel, 5])) + '_r')
    if plot_type == 'pred':
        colors = cmap.mpl_colors
        pal = {}
        leg_labels = {}
        keys = range(gameboard.nlabel+1)
        for i in keys:
            pal[i] = colors[i]
            leg_labels[str(i)] = 'class ' + str(i)
            if i == gameboard.nlabel:
                pal[i] = 'silver'
                leg_labels[str(i)] = 'abstain'
    elif plot_type == 'acc':
        colors = palettable.cartocolors.qualitative.Prism_7.mpl_colors
        pal = {-1: 'silver', 0: colors[-1], 1: 'black'}
        leg_labels = {'-1': 'abstain', '0': 'wrong', '1': 'correct'}
    else:
        raise ValueError('no such plot_type')

    # make figure
    plt.axis("equal")

    # plot background shading
    v_edge = np.linspace(0, gameboard.nrow, gameboard.nrank+1) - 0.5
    h_edge = np.linspace(0, gameboard.ncol, gameboard.nfile+1) - 0.5

    plt.pcolormesh(
        v_edge,
        h_edge,
        gameboard.board,
        alpha=0.20,
        edgecolors=None,
        cmap=matplotlib.colors.ListedColormap(cmap.mpl_colors[:gameboard.nlabel])
    )
    plt.clim(0, gameboard.nlabel-1)

    for i in range(gameboard.nrank):
        for j in range(gameboard.nfile):
            if gameboard.tranquil[i, j]:
                x = [h_edge[j], h_edge[j+1], h_edge[j+1], h_edge[j], h_edge[j]]
                y = [v_edge[i], v_edge[i], v_edge[i+1], v_edge[i+1], v_edge[i]]
                plt.plot(x, y, '-k', linewidth=0.25)

    # plot dots
    ax = sns.scatterplot(
        x=[xy[0] for xy in X],
        y=[xy[1] for xy in X],
        hue=z,
        palette=pal,
        zorder=10,
        s=marker_size
    )

    legend_labelhandles, labels = ax.get_legend_handles_labels()
    new_labels = [leg_labels.get(key) for key in labels]
    plt.legend(legend_labelhandles, new_labels, bbox_to_anchor=(1.0, 0.90), loc=2, borderaxespad=0)

    plt.title(title_str, fontsize='large')
    plt.axis('off')

# ----------------------------------------------------
# Make the prediction plots.

# TODO: Currently, many of these plots and statistics are legit ONLY for the "PID" controller.

def plot_predictions(exp_name, data, gameboard, saveplot=True, showplot=True):

    X_all, y_all, X_train, y_train, X_val, y_val, y_train_pred, y_val_pred, onehotlabels_val, abstain = data

    cat_train_pred = np.argmax(y_train_pred, axis=1)
    cat_val_pred = np.argmax(y_val_pred, axis=1)

    # make final predictions plot
    FS = 15
    plt.figure(figsize=(16, 24))

    # ------------------------------------------------
    # Plot the real, raw data
    plt.subplot(3, 2, 1)
    title_str = (
        f"{gameboard.nlabel}-label data with {gameboard.ntranquil} quiet cells "
        + f"and {gameboard.nnoisy} noisy cells\n"
        + f"({100*gameboard.pr_mislabel}% mislabeled in noisy cells)"
    )
    plot_data(X_all, y_all, gameboard, title_str)

    # ------------------------------------------------
    # Plot validation statistics
    num_validation = len(X_val)
    acc = metrics.compute_dac_accuracy(onehotlabels_val, y_val_pred, abstain)

    tr = gameboard.aretranquil(X_val)
    num_tranquil = np.sum(tr)
    acc_tr = metrics.compute_dac_accuracy(onehotlabels_val[tr, :], y_val_pred[tr, :], abstain)

    plt.subplot(3, 2, 2)
    plt.ylim(0, 1)
    text = (
        f"VALIDATION PREDICTIONS\n"
        + f"number = {num_validation}\n"
        + f"accuracy = {int(np.round(acc*100))}%\n"
        + f"\n"
        + f"TRANQUIL VALIDATION PREDICTIONS\n"
        + f"number = {num_tranquil}\n"
        + f"accuracy = {int(np.round(acc_tr*100))}%\n"
    )
    plt.text(.2, .5, text, fontsize=FS*1., va='top')
    plt.axis('off')

    # ------------------------------------------------
    # Plot the validation subset predictions
    plt.subplot(3, 2, 3)

    title_str = "Validation Subset Predictions"
    plot_data(X_val, cat_val_pred, gameboard, title_str, plot_type='pred')

    # ------------------------------------------------
    # Plot the validation subset accuracy
    accuracy_val = []
    for i, cat in enumerate(cat_val_pred):
        if cat == abstain:
            accuracy_val.append(-1)
        elif cat == y_val[i]:
            accuracy_val.append(1)
        else:
            accuracy_val.append(0)

    plt.subplot(3, 2, 4)
    title_str = "Validation Subset Accuracy"
    plot_data(X_val, accuracy_val, gameboard, title_str, plot_type='acc')

    # ------------------------------------------------
    # Plot the training subset predictions
    plt.subplot(3, 2, 5)
    title_str = "Training Subset Predictions"
    plot_data(X_train, cat_train_pred, gameboard, title_str, plot_type='pred')

    # ------------------------------------------------
    # Plot the training subset accuracy
    accuracy_train = []
    for i, y in enumerate(cat_train_pred):
        if y == abstain:
            accuracy_train.append(-1)
        elif y == y_train[i]:
            accuracy_train.append(1)
        else:
            accuracy_train.append(0)

    plt.subplot(3, 2, 6)
    title_str = "Training Subset Accuracy"
    plot_data(X_train, accuracy_train, gameboard, title_str, plot_type='acc')

    # ---------- Make and save the plot -------------------
    plt.tight_layout()
    if saveplot:
        plt.savefig('figures/model_predictions/' + exp_name + '.png', dpi=dpiFig)
    if not showplot:
        plt.close('all')
    else:
        plt.show()

    print(f"Validation = {Counter(accuracy_val)}")
    print(f"Training = {Counter(accuracy_train)}")

def drawOnGlobe(ax, map_proj, data, lats, lons, cmap='coolwarm', vmin=None, vmax=None, inc=None, cbarBool=True, contourMap=[], contourVals = [], fastBool=False, extent='both'):

    data_crs = ct.crs.PlateCarree()
    data_cyc, lons_cyc = add_cyclic_point(data, coord=lons) #fixes white line by adding point#data,lons#ct.util.add_cyclic_point(data, coord=lons) #fixes white line by adding point

    ax.set_global()
#     ax.coastlines(linewidth = 1.2, color='black')
#     ax.add_feature(cartopy.feature.LAND, zorder=0, scale = '50m', edgecolor='black', facecolor='black')
    land_feature = cfeature.NaturalEarthFeature(
        category='physical',
        name='land',
        scale='50m',
        facecolor='black',
        edgecolor = None
    )
    ax.add_feature(land_feature)
#     ax.GeoAxes.patch.set_facecolor('black')

    if(fastBool):
        image = ax.pcolormesh(lons_cyc, lats, data_cyc, transform=data_crs, cmap=cmap)
#         image = ax.contourf(lons_cyc, lats, data_cyc, np.linspace(0,vmax,20),transform=data_crs, cmap=cmap)
    else:
        image = ax.pcolor(lons_cyc, lats, data_cyc, transform=data_crs, cmap=cmap,shading='auto')

    if(np.size(contourMap) !=0 ):
        contourMap_cyc, __ = add_cyclic_point(contourMap, coord=lons) #fixes white line by adding point
        ax.contour(lons_cyc,lats,contourMap_cyc,contourVals, transform=data_crs, colors='fuchsia')

    if(cbarBool):
        cb = plt.colorbar(image, shrink=.5, orientation="horizontal", pad=.02, extend=extent)
        cb.ax.tick_params(labelsize=6)
    else:
        cb = None

    image.set_clim(vmin,vmax)

    return cb, image

def add_cyclic_point(data, coord=None, axis=-1):

    # had issues with cartopy finding utils so copied for myself

    if coord is not None:
        if coord.ndim != 1:
            raise ValueError('The coordinate must be 1-dimensional.')
        if len(coord) != data.shape[axis]:
            raise ValueError('The length of the coordinate does not match '
                             'the size of the corresponding dimension of '
                             'the data array: len(coord) = {}, '
                             'data.shape[{}] = {}.'.format(
                                 len(coord), axis, data.shape[axis]))
        delta_coord = np.diff(coord)
        if not np.allclose(delta_coord, delta_coord[0]):
            raise ValueError('The coordinate must be equally spaced.')
        new_coord = ma.concatenate((coord, coord[-1:] + delta_coord[0]))
    slicer = [slice(None)] * data.ndim
    try:
        slicer[axis] = slice(0, 1)
    except IndexError:
        raise ValueError('The specified axis does not correspond to an '
                         'array dimension.')
    new_data = ma.concatenate((data, data[tuple(slicer)]), axis=axis)
    if coord is None:
        return_value = new_data
    else:
        return_value = new_data, new_coord
    return return_value

def plot_stats_comparisons(df, savename, lines=False, shades=True):

    ABSTENTION_VAR = 'coverage'
    plt.figure(figsize=(6*2,4*3))
    plt.rcParams["axes.prop_cycle"] = plt.cycler("color", palettable.cartocolors.qualitative.Vivid_6.mpl_colors)

    #-------------------
    s0 = plt.subplot(3,2,1)
    plot_stats_panel(df, var='acc', abstention_var=ABSTENTION_VAR, legend=True, lines=lines, shades=shades)

    #-------------------
    s1 = plt.subplot(3,2,2)
    plot_stats_panel(df, var='acc_portion_tr', abstention_var=ABSTENTION_VAR, legend=False, lines=lines, shades=shades)
    plt.title('Contributions of Tranquil to Total Accuracy', fontsize=8)

    #-------------------
    s5 = plt.subplot(3,2,5)
    plot_stats_panel(df, var='frac_corr_tr', abstention_var=ABSTENTION_VAR, legend=False, lines=lines, shades=shades)
    plt.title('Fraction of those Correct that are Tranquil', fontsize=8)

    #-------------------
    s3 = plt.subplot(3,2,3)
    plot_stats_panel(df, var='acc_tr', abstention_var=ABSTENTION_VAR, legend=False, lines=lines, shades=shades)
    plt.title('Tranquil Accuracy', fontsize=8)

    #-------------------
    s4 = plt.subplot(3,2,4)
    plot_stats_panel(df, var='acc_ntr', abstention_var=ABSTENTION_VAR, legend=False, lines=lines, shades=shades)
    plt.title('Not-Tranquil Accuracy', fontsize=8)

    #-------------------
    s6 = plt.subplot(3,2,6)
    plot_stats_panel(df, var='frac_tr', abstention_var=ABSTENTION_VAR, legend=False, lines=lines, shades=shades)
    plt.title('Fraction Covered that are Tranquil', fontsize=8)

    #----------------------------------------------------------
    s0.set_title('Total Accuracy\n' + savename, fontsize=8)
    plt.tight_layout()

    return savename

def plot_stats_panel(df, var, abstention_var, legend=False, lines=False, shades=True):
    alpha = .2

    colors = palettable.cartocolors.qualitative.Bold_9_r.mpl_colors

    APP_LIST = np.unique(df['app_type'].values)
    APP_LIST = sorted(APP_LIST,reverse=False)
    for k,app_type in enumerate(APP_LIST):

        edgeclr='white'
        label=app_type
        if(app_type=='ORACLE'):
            clr = 'lightgray'
        elif((app_type=='DNN' or app_type=='ANN') and var=='acc'):
            clr = colors[0]
        elif((app_type=='DNN' or app_type=='ANN') and var=='acc_tr'):
            clr = colors[1]
            label='ANN: tranquil'
        elif((app_type=='DNN' or app_type=='ANN') and var=='acc_ntr'):
            clr = (.25,.25,.25,1.)
            label='ANN: not tranquil'
        elif((app_type=='CAN' or app_type=='DAC') and var=='acc_tr'):
            clr = colors[1]
            label='CAN: tranquil'
        elif((app_type=='CAN' or app_type=='DAC') and var=='acc_ntr'):
            clr=(.25,.25,.25,1.)
            label='CAN: not tranquil'
            edgeclr=(.25,.25,.25,1.)
        else:
            clr = colors[k]

        df_plot = df[df['app_type']==app_type]

        if(lines==True and shades==True):
            lines_label = None
            args = {'color':clr,}
        else:
            lines_label = app_type
            args={}

        if((app_type=='DNN' or app_type=='ANN') and var=='acc'):
            lines_label=None#'DNN median'

        if(app_type=='CAN' or app_type=='DAC'):
            ax1 = sns.scatterplot(data=df_plot, x=abstention_var, y=var,
                                 hue="setpoint",
                                 palette = palettable.cartocolors.qualitative.Vivid_5.mpl_colormap,
                                 label = label,
                                 legend = False,
                                 ci = None,
                                 markers=True,
                                 s = 15,
                                 style="app_type",
                                 alpha = .6,
                                 zorder = 100,
                                 edgecolor=edgeclr,
                                )
        else:
            if(lines==True):
                sns.lineplot(data=df_plot, x=abstention_var, y=var,
                                  label = lines_label,
                                  legend = False,
                                  ci = None,
                                  estimator = np.median,
                                  markers=False,
                                  style="app_type",
                                  linewidth=1.5,
                                  **args,
                                 )
            if(shades==True):
                df_stats = df_plot.groupby([abstention_var]).describe()
                plt.fill_between(x=df_stats.index, y1=df_stats[(var, 'min')], y2=df_stats[(var, 'max')], alpha = alpha, label=label, color=clr)

    #---------------------------------------

    plt.legend(loc=2,)
    if(legend is False):
        try:
            ax1.get_legend().remove()
        except:
            print('no such ax1')

    if(var=='frac_tr'):
        ax9 = sns.lineplot(data=df[df['app_type']=='ANN'], x=abstention_var, y='perf_frac_tr',
                          label = None,
                          legend = None,
                          ci = None,
                          estimator = np.median,
                          markers=False,
                          color = 'black',
                          dashes=[(2,2)],
                          style="app_type",
                          alpha = .2,
                         )
    xticks = np.arange(0,110,10)
    plt.xticks(xticks,map(str,xticks))
    plt.xlabel('coverage (\%)', fontsize=16)
    plt.ylim(-.02,1.01)
    plt.xlim(-1,100)
    plt.gca().invert_xaxis()
    ylabel = plt.gca().yaxis.get_label().get_text()
    if(ylabel.find('\_')==-1):
        plt.ylabel(ylabel.replace('_','\_'), fontsize=16)

    try:
        return ax1
    except:
        return None


text:
import numpy as np
import torch
from fed_common.config_functions import init_params

#fedAvg implementation
def average(in_params, set_sizes, class_imbalances, dataset, model_choice, use_sizes= False, use_imbalances = False) :

    #create size-based weights
    num_clients = set_sizes.size
    weights = np.ones_like(set_sizes)/num_clients

    if use_sizes:
        total_size = np.sum(set_sizes)
        weights = set_sizes / total_size

    #do averaging
    if isinstance(in_params[0], np.ndarray):
        parameters = np.zeros_like(in_params[0])
        for i in range (in_params.shape[1]):
            for j in range(num_clients):
                parameters[i] += weights[j] * in_params[j,i]
    else:
        parameters = init_params(dataset, model_choice, True)

        for param in parameters.keys():
            for i in range(num_clients):
                parameters[param] += weights[i] * in_params[i][param]

    return parameters

#scaffold implementation
def scaffold(dataset,model_choice, global_parameters, local_parameters, c, old_local_c, local_c, lr, use_c = True, key = None):

    #for sklearn-based implementations
    if isinstance(global_parameters, np.ndarray):
        num_clients = local_parameters.shape[0]

        param_agg = np.zeros_like(global_parameters)
        parameters = np.zeros_like(global_parameters)
        c_agg = np.zeros_like(global_parameters)

        for i in range(num_clients):
            c_agg += local_c[i][key] - old_local_c[i][key]
            param_agg += local_parameters[i] - global_parameters
        parameters = global_parameters + (lr/num_clients) * param_agg
        c[key] = c[key] + (1/num_clients) * c_agg

    #for pytorch-based implementations
    else:
        num_clients = local_parameters.size
        parameters = init_params(dataset, model_choice, True)
        for param in parameters.keys():
            param_agg = torch.clone(parameters[param])
            c_agg = torch.clone(param_agg)
            #calculate the sum of differences between the local and global model
            for i in range(num_clients):
                param_agg += local_parameters[i][param] - global_parameters[param]
                c_agg += local_c[i][param] - old_local_c[i][param]
            #calculate new weight value
            parameters[param] = global_parameters[param] + (lr/num_clients) * param_agg
            if use_c:
                c[param] = c[param] + (1/num_clients) * c_agg
    return parameters, c

max_stars_repo_name: swiergarst/FedvsCent

max_issues_repo_name: swiergarst/FedvsCent

max_forks_repo_name: swiergarst/FedvsCent


text:
import copy
import numpy as np
from ..grid import get_kij_from_node3d

def test_grid_eq(lpr_modelgrid):
    grid2 = copy.deepcopy(lpr_modelgrid)
    assert lpr_modelgrid == grid2

def test_get_kij_from_node3d():
    nlay, nrow, ncol = 3, 3, 4
    nodes_expected = [(0, (0, 0, 0)),
                      (2, (0, 0, 2)),
                      (ncol, (0, 1, 0)),
                      (nrow * ncol + 2, (1, 0, 2)),
                      (nrow * ncol*2 + 3, (2, 0, 3)),
                      (nrow*ncol*nlay-1, (2, nrow-1, ncol-1)),
                      (nrow * ncol * nlay, (nlay, 0, 0))
                      ]
    for node3d, expected in nodes_expected:
        k, i, j = get_kij_from_node3d(node3d, nrow, ncol)
        assert (k, i, j) == expected

max_stars_repo_name: mnfienen/modflow-export

max_issues_repo_name: mnfienen/modflow-export

max_forks_repo_name: mnfienen/modflow-export


text:
import functools
import tensorflow as tf
import numpy as np
from tensorflow import Tensor
from google_drive_downloader import GoogleDriveDownloader as gdd
from tensorflow.contrib import eager as tfe

def define_scope(function):
    attribute = '_cache_' + function.__name__

    @property
    @functools.wraps(function)
    def decorator(self):
        if not hasattr(self, attribute):
            with tf.variable_scope(function.__name__):
                setattr(self, attribute, function(self))
        return getattr(self, attribute)

    return decorator

def sigmoid(x):
    """
      Code for the sigmoid function. https://en.wikipedia.org/wiki/Logistic_function
    """
    calc = tf.div(1., (1. + tf.exp(-x)))
    return calc

class LogisticRegressionModel:

    def __init__(self, data: Tensor, target: Tensor, n_features: int, learning_rate=1e-1):
        self.data = data
        self.target = target
        self.n_features = n_features
        self.rate = learning_rate
        self.prediction_round
        self.prediction
        self.optimize
        self.accuracy
        self.loss

    @define_scope
    def prediction(self):
        weight = tf.get_variable(
            "weight",
            initializer=tf.random_normal((self.n_features, 1))
        )
        bias = tf.get_variable(
            "bias",
            initializer=tf.zeros((1))
        )

        incoming = tf.matmul(self.data, weight) + bias
        incoming = tf.squeeze(incoming)

        return sigmoid(incoming)

    @define_scope
    def loss(self):
        side_1 = self.target * tf.log(self.prediction)
        side_2 = (1 - self.target) * tf.log(1 - self.prediction)

        return - tf.reduce_mean(side_1 + side_2)

    @define_scope
    def optimize(self):
        optimizer = tf.train.GradientDescentOptimizer(self.rate)
        return optimizer.minimize(self.loss)

    @define_scope
    def prediction_round(self):
        return tf.round(self.prediction)

    @define_scope
    def accuracy(self):
        return tf.reduce_mean(
            tf.cast(
                tf.equal(self.target, self.prediction_round),
                tf.float32)
        )

class LogisticRegressionEagerModel:
    def __init__(self, n_features: int, learning_rate=1e-1):
        self.n_features = n_features
        self.rate = learning_rate
        self.weight = tfe.Variable(tf.random_normal((self.n_features, 1)))
        self.bias = tfe.Variable(0.)

    def prediction(self, data):

        incoming = tf.matmul(data, self.weight) + self.bias
        incoming = tf.squeeze(incoming)
        return sigmoid(incoming)

    def loss(self, label, prediction):
        side_1 = label * tf.log(prediction)
        side_2 = (1 - label) * tf.log(1 - prediction)

        return - tf.reduce_mean(side_1 + side_2)

    def prediction_round(self, target):
        return tf.round(self.prediction(target))

    def accuracy(self, data, target):
        return tf.reduce_mean(
            tf.cast(
                tf.equal(target, self.prediction_round(data)),
                tf.float32)
        )

    def train(self, data, target):
        optimizer = tf.train.GradientDescentOptimizer(self.rate)

        def loss_fn(x, y):
            return self.loss(y, self.prediction(x))

        grad_fn = tfe.implicit_value_and_gradients(loss_fn)
        loss, gradients = grad_fn(data, target)
        optimizer.apply_gradients(gradients)

        return loss

max_stars_repo_name: loribonna/EserciziMLPython

max_issues_repo_name: loribonna/EserciziMLPython

max_forks_repo_name: loribonna/EserciziMLPython


text:
#!/usr/bin/env python
""" some equations that are useful for plasmas """

from __future__ import print_function, division

import numpy as np
try:
    import numexpr as ne
    _HAS_NUMEXPR = True
except ImportError:
    _HAS_NUMEXPR = False

from viscid import field
from viscid import logger
# from viscid.calculator import calc

__all__ = ["calc_psi", "calc_beta"]

def calc_psi(B, rev=False):
    """Calc Flux function (only valid in 2d)

    Parameters:
        B (VectorField): magnetic field, should only have two
            spatial dimensions so we can infer the symmetry dimension
        rev (bool): since this integration doesn't like going
            through undefined regions (like within 1 earth radius of
            the origin for openggcm), you can use this to start
            integrating from the opposite corner.

    Returns:
        ScalarField: 2-D scalar flux function

    Raises:
        ValueError: If B has <> 2 spatial dimensions

    """
    # TODO: if this is painfully slow, i bet just putting this exact
    # code in a cython module would make it a bunch faster, the problem
    # being that the loops are in python instead of some broadcasting
    # numpy type thing

    B = B.slice_reduce(":")

    # try to guess if a dim of a 3D field is invariant
    reduced_axes = []
    if B.nr_sdims > 2:
        slcs = [slice(None)] * B.nr_sdims
        for i, nxi in enumerate(B.sshape):
            if nxi <= 2:
                slcs[i] = 0
                reduced_axes.append(B.crds.axes[i])
        slcs.insert(B.nr_comp, slice(None))
        B = B[slcs]

    # ok, so the above didn't work... just nip out the smallest dim?
    if B.nr_sdims == 3:
        slcs = [slice(None)] * B.nr_sdims
        i = np.argmin(B.sshape)
        slcs[i] = 0
        reduced_axes.append(B.crds.axes[i])
        logger.warning("Tried to get the flux function of a 3D field. "
                       "I can't do that, so I'm\njust ignoring the {0} "
                       "dimension".format(reduced_axes[-1]))
        slcs.insert(B.nr_comp, slice(None))
        B = B[slcs]

    if B.nr_sdims != 2:
        raise ValueError("flux function only implemented for 2D fields")

    comps = ""
    for comp in "xyz":
        if comp in B.crds.axes:
            comps += comp
    # ex: comps = "yz", comp_inds = [1, 2]
    comp_inds = [dict(x=0, y=1, z=2)[comp] for comp in comps]

    # Note: what follows says y, z, but it has been generalized
    # to any two directions, so hy isn't necessarily hy, but it's
    # easier to see at a glance if it's correct using a specific
    # example
    ycc, zcc = B.get_crds(comps)
    comp_views = B.component_views()
    hy, hz = comp_views[comp_inds[0]], comp_views[comp_inds[1]]
    dy = ycc[1:] - ycc[:-1]
    dz = zcc[1:] - zcc[:-1]
    ny, nz = len(ycc), len(zcc)

    A = np.empty((ny, nz), dtype=B.dtype)

    if rev:
        A[-1, -1] = 0.0
        for i in range(ny - 2, -1, -1):
            A[i, -1] = A[i + 1, -1] - dy[i] * 0.5 * (hz[i, -1] + hz[i + 1, -1])

        for j in range(nz - 2, -1, -1):
            A[:, j] = A[:, j + 1] + dz[j] * 0.5 * (hy[:, j + 1] + hy[:, j])
    else:
        A[0, 0] = 0.0
        for i in range(1, ny):
            A[i, 0] = A[i - 1, 0] + dy[i - 1] * 0.5 * (hz[i, 0] + hz[i - 1, 0])

        for j in range(1, nz):
            A[:, j] = A[:, j - 1] - dz[j - 1] * 0.5 * (hy[:, j - 1] + hy[:, j])

    psi = field.wrap_field(A, B.crds, name="psi", center=B.center,
                           pretty_name=r"$\psi$", parents=[B])
    if reduced_axes:
        slc = "..., " + ", ".join("{0}=None".format(ax) for ax in reduced_axes)
        psi = psi[slc]
    return psi

def calc_beta(pp, B, scale=1.0):
    """Calc plasma beta (2.0 * p / B^2)

    Parameters:
        pp (ScalarField or ndarray): pressure
        B (VectorField): magnetic field
        scale (float, optional): overall scale factor

    Returns:
        ScalarField: Plasma beta

    Note:
        For OpenGGCM, where pp is in pPa and B is in nT, scale should
        be 40.0.
    """
    two = np.array([2.0], dtype=pp.dtype)
    bx, by, bz = B.component_views()
    if _HAS_NUMEXPR:
        ldict = dict(scale=scale, bx=bx, by=by, bz=bz,
                     pp=pp, two=two)
        result = ne.evaluate("scale * two * pp / sqrt(bx**2 + by**2 + bz**2)",
                             local_dict=ldict)
    else:
        result = scale * two * pp / np.sqrt(bx**2 + by**2 + bz**2)
    context = dict(name="beta", pretty_name=r"$\beta_{pl}$")
    return pp.wrap(result, context=context)

##
## EOF
##

max_stars_repo_name: jimgreen/Viscid

max_issues_repo_name: jimgreen/Viscid

max_forks_repo_name: jimgreen/Viscid

reason: import numpy,import numexpr