text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Purpose: Perform Incremental PCA on QPESUMS data.
"""
import os, csv, logging, argparse, pickle, h5py
import numpy as np
import pandas as pd
from sklearn.decomposition import PCA, IncrementalPCA
import joblib

# Read precipitation data
tmp = pd.read_csv('../examples/data/t1hr.csv')
# Calculate maximal precipitation among 45 stations
dates = tmp['date']
t1hr = tmp.iloc[:,1:]
t1hr_max = pd.DataFrame({'timestamp':dates, 'prec':t1hr.max(axis=1)})

# Read projections
pfull = pd.read_csv('../ws.flt/log/proj_ln_full.csv')
pfp40 = pd.read_csv('../ws.flt/log/proj_ln_fp40.csv')
pftyw = pd.read_csv('../ws.flt/log/proj_ln_ftyw.csv')

# Create new column names
pfull.columns = ['full_'+i for i in pfull.columns]
pfp40.columns = ['fp40_'+i for i in pfp40.columns]
pftyw.columns = ['ftyw_'+i for i in pftyw.columns]
pfull.rename(columns={'full_timestamp':'timestamp'}, inplace=True)
pfp40.rename(columns={'fp40_timestamp':'timestamp'}, inplace=True)
pftyw.rename(columns={'ftyw_timestamp':'timestamp'}, inplace=True)

# Clean up
data = pd.merge(pd.merge(pfull.iloc[:,:11], pfp40.iloc[:,:11], on='timestamp'),pftyw.iloc[:,:11], on='timestamp')
data.index = list(data['timestamp'])
data = data.iloc[:,1:]

# Add logarithm
for i in range(data.shape[1]):
    data['squared_'+list(data.columns)[i]] = data.iloc[:,i]**2

# Check
print("Input data shape:")
print(data.shape)
print(list(data.columns))

tmp = pd.merge(data, t1hr_max, left_index=True, right_on='timestamp')
y = tmp['prec']

idx2015 = sum(tmp['timestamp']<=2015010101)
print('Data index before 2015')
print(idx2015)
idx2016 = sum(tmp['timestamp']<=2016010101)
print('Data index before 2016')
print(idx2016)

print("Output data shape:")
print(y.shape)

# Split data
x_train = data.iloc[:idx2015,:]
x_test = data.iloc[idx2015:idx2016,:]
y_train = y.iloc[:idx2015,]
y_test = y.iloc[idx2015:idx2016,]

print("Training data dimension:")
print(x_train.shape)
print(y_train.shape)
print("Test data dimension:")
print(x_test.shape)
print(y_test.shape)

#GLM
import statsmodels.api as sm

# Fit the classifier
glm = sm.GLM(y_train.reset_index(drop=True), x_train.fillna(0.).reset_index(drop=True))
glm_results = glm.fit()

print(glm_results.summary())

# RFECV
import matplotlib.pyplot as plt
from sklearn.svm import SVR
from sklearn.model_selection import KFold
from sklearn.feature_selection import RFECV
from sklearn.datasets import make_classification

# Select input variables
xvars = ['full_pc1', 'full_pc2', 'full_pc3', 'full_pc4', 'full_pc5', 'full_pc6', 'full_pc7', 'full_pc8', 'full_pc9', 'full_pc10', 'fp40_pc1', 'fp40_pc2', 'fp40_pc3', 'fp40_pc4', 'fp40_pc5', 'fp40_pc6', 'fp40_pc7', 'fp40_pc8', 'fp40_pc9', 'fp40_pc10', 'ftyw_pc1', 'ftyw_pc2', 'ftyw_pc3', 'ftyw_pc4', 'ftyw_pc5', 'ftyw_pc6', 'ftyw_pc7', 'ftyw_pc8', 'ftyw_pc9', 'ftyw_pc10', 'squared_full_pc1', 'squared_full_pc2', 'squared_full_pc3', 'squared_full_pc4', 'squared_full_pc5', 'squared_full_pc6', 'squared_full_pc7', 'squared_full_pc8', 'squared_full_pc9', 'squared_full_pc10', 'squared_fp40_pc1', 'squared_fp40_pc2', 'squared_fp40_pc3', 'squared_fp40_pc4', 'squared_fp40_pc5', 'squared_fp40_pc6', 'squared_fp40_pc7', 'squared_fp40_pc8', 'squared_fp40_pc9', 'squared_fp40_pc10', 'squared_ftyw_pc1', 'squared_ftyw_pc2', 'squared_ftyw_pc3', 'squared_ftyw_pc4', 'squared_ftyw_pc5', 'squared_ftyw_pc6', 'squared_ftyw_pc7', 'squared_ftyw_pc8', 'squared_ftyw_pc9', 'squared_ftyw_pc10']
# Build a classification task using 3 informative features
X = x_train.reset_index(drop=True).loc[:,xvars[:10]]
y = y_train.reset_index(drop=True)

print("Starting feature selection:")
print("  input data dimesnion: "+str(X.shape))
print("  output data dimesnion: "+str(y.shape))
# Create the RFE object and compute a cross-validated score.
svc = SVR(kernel="linear")
# The "accuracy" scoring is proportional to the number of correct
# classifications
rfecv = RFECV(estimator=svc, step=1, cv=KFold(5), scoring='neg_mean_squared_error', verbose=1, n_jobs=5)
rfecv.fit(X, y)

print("Optimal number of features : %d" % rfecv.n_features_)
joblib.dump(rfecv, 'rfecv_fitted.joblib')

# Plot number of features VS. cross-validation scores
plt.figure()
plt.xlabel("Number of features selected")
plt.ylabel("Cross validation score (nb of correct classifications)")
plt.plot(range(1, len(rfecv.grid_scores_) + 1), rfecv.grid_scores_)
plt.show()

max_stars_repo_name: tingsyo/qpetw

max_issues_repo_name: tingsyo/qpetw

max_forks_repo_name: tingsyo/qpetw

reason: import numpy,import statsmodels


text:
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
################################################################################
#
# Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved
#
################################################################################
"""
File: network.py
"""

import argparse

import numpy as np

import paddle.fluid as fluid
import paddle.fluid.framework as framework
from paddle.fluid.executor import Executor

from source.utils.utils import str2bool
from source.utils.utils import id_to_text
from source.utils.utils import init_embedding
from source.utils.utils import build_data_feed
from source.utils.utils import load_id2str_dict
from source.inputters.corpus import KnowledgeCorpus
from source.models.knowledge_seq2seq import knowledge_seq2seq

def model_config():
    """ model config """
    parser = argparse.ArgumentParser()

    # Data
    data_arg = parser.add_argument_group("Data")
    data_arg.add_argument("--data_dir", type=str, default="./data/")
    data_arg.add_argument("--data_prefix", type=str, default="demo")
    data_arg.add_argument("--save_dir", type=str, default="./models/")
    data_arg.add_argument("--vocab_path", type=str, default="./data/vocab.txt")
    data_arg.add_argument("--embed_file", type=str,
                          default="./data/sgns.weibo.300d.txt")

    # Network
    net_arg = parser.add_argument_group("Network")
    net_arg.add_argument("--embed_size", type=int, default=300)
    net_arg.add_argument("--hidden_size", type=int, default=800)
    net_arg.add_argument("--bidirectional", type=str2bool, default=True)
    net_arg.add_argument("--vocab_size", type=int, default=30004)
    net_arg.add_argument("--min_len", type=int, default=1)
    net_arg.add_argument("--max_len", type=int, default=500)
    net_arg.add_argument("--num_layers", type=int, default=1)
    net_arg.add_argument("--attn", type=str, default='dot',
                         choices=['none', 'mlp', 'dot', 'general'])

    # Training / Testing
    train_arg = parser.add_argument_group("Training")

    train_arg.add_argument("--stage", type=int, default="0")
    train_arg.add_argument("--run_type", type=str, default="train")
    train_arg.add_argument("--init_model", type=str, default="")
    train_arg.add_argument("--init_opt_state", type=str, default="")

    train_arg.add_argument("--optimizer", type=str, default="Adam")
    train_arg.add_argument("--lr", type=float, default=0.0005)
    train_arg.add_argument("--grad_clip", type=float, default=5.0)
    train_arg.add_argument("--dropout", type=float, default=0.3)
    train_arg.add_argument("--num_epochs", type=int, default=13)
    train_arg.add_argument("--pretrain_epoch", type=int, default=5)
    train_arg.add_argument("--use_bow", type=str2bool, default=True)
    train_arg.add_argument("--use_posterior", type=str2bool, default=False)

    # Geneation
    gen_arg = parser.add_argument_group("Generation")
    gen_arg.add_argument("--beam_size", type=int, default=10)
    gen_arg.add_argument("--max_dec_len", type=int, default=30)
    gen_arg.add_argument("--length_average", type=str2bool, default=True)
    gen_arg.add_argument("--output", type=str, default="./output/test.result")
    gen_arg.add_argument("--model_path", type=str, default="./models/best_model/")
    gen_arg.add_argument("--unk_id", type=int, default=1)
    gen_arg.add_argument("--bos_id", type=int, default=2)
    gen_arg.add_argument("--eos_id", type=int, default=3)

    # MISC
    misc_arg = parser.add_argument_group("Misc")
    misc_arg.add_argument("--use_gpu", type=str2bool, default=True)
    misc_arg.add_argument("--log_steps", type=int, default=10000)
    misc_arg.add_argument("--valid_steps", type=int, default=10000)
    misc_arg.add_argument("--batch_size", type=int, default=1)

    config = parser.parse_args()

    return config

def trace_fianl_result(final_score, final_ids, final_index, topk=1, EOS=3):
    """ trace fianl result """
    col_size = final_score.shape[1]
    row_size = final_score.shape[0]

    found_eos_num = 0
    i = row_size - 1

    beam_size = col_size
    score = final_score[-1]
    row_array = [row_size - 1] * beam_size
    col_array = [e for e in range(col_size)]

    while i >= 0:
        for j in range(col_size - 1, -1, -1):
            if final_ids[i, j] == EOS:
                repalce_idx = beam_size - (found_eos_num % beam_size) - 1
                score[repalce_idx] = final_score[i, j]
                found_eos_num += 1

                row_array[repalce_idx] = i
                col_array[repalce_idx] = j

        i -= 1

    topk_index = np.argsort(score,)[-topk:]

    trace_result = []
    trace_score = []

    for index in reversed(topk_index):
        start_i = row_array[index]
        start_j = col_array[index]
        ids = []
        for k in range(start_i, -1, -1):
            ids.append(final_ids[k, start_j])
            start_j = final_index[k, start_j]

        ids = ids[::-1]

        trace_result.append(ids)
        trace_score.append(score[index])

    return trace_result, trace_score

def load():
    """ load model for predict """
    config = model_config()
    config.vocab_size = len(open(config.vocab_path).readlines())
    final_score, final_ids, final_index = knowledge_seq2seq(config)

    final_score.persistable = True
    final_ids.persistable = True
    final_index.persistable = True

    main_program = fluid.default_main_program()

    if config.use_gpu:
        place = fluid.CUDAPlace(0)
    else:
        place = fluid.CPUPlace()

    exe = Executor(place)
    exe.run(framework.default_startup_program())

    fluid.io.load_params(executor=exe, dirname=config.model_path, main_program=main_program)

    processors = KnowledgeCorpus(
        data_dir=config.data_dir,
        data_prefix=config.data_prefix,
        vocab_path=config.vocab_path,
        min_len=config.min_len,
        max_len=config.max_len)

    # load dict
    id_dict_array = load_id2str_dict(config.vocab_path)

    model_handle = [exe, place, final_score, final_ids, final_index, processors, id_dict_array]
    return model_handle

def predict(model_handle, text):
    """ predict for text by model_handle """
    batch_size = 1

    [exe, place, final_score, final_ids, final_index, processors, id_dict_array] = model_handle

    data_generator = processors.preprocessing_for_lines([text], batch_size=batch_size)

    results = []
    for batch_id, data in enumerate(data_generator()):
        data_feed, sent_num = build_data_feed(data, place, batch_size=batch_size)
        out = exe.run(feed=data_feed,
                      fetch_list=[final_score.name, final_ids.name, final_index.name])

        batch_score = out[0]
        batch_ids = out[1]
        batch_pre_index = out[2]

        batch_score_arr = np.split(batch_score, batch_size, axis=1)
        batch_ids_arr = np.split(batch_ids, batch_size, axis=1)
        batch_pre_index_arr = np.split(batch_pre_index, batch_size, axis=1)

        index = 0
        for (score, ids, pre_index) in zip(batch_score_arr, batch_ids_arr, batch_pre_index_arr):
            trace_ids, trace_score = trace_fianl_result(score, ids, pre_index, topk=1, EOS=3)
            results.append(id_to_text(trace_ids[0][:-1], id_dict_array))

            index += 1
            if index >= sent_num:
                break

    return results[0]

def init_model(config, param_name_list, place):
    """ init model """
    stage = config.stage
    if stage == 0:
        for name in param_name_list:
            t = fluid.global_scope().find_var(name).get_tensor()
            init_scale = 0.05
            np_t = np.asarray(t)
            if str(name) == 'embedding':
                np_para = init_embedding(config.embed_file, config.vocab_path,
                                         init_scale, np_t.shape)
            else:
                np_para = np.random.uniform(-init_scale, init_scale, np_t.shape).astype('float32')
            t.set(np_para.astype('float32'), place)
    else:
        model_init_file = config.init_model
        try:
            model_init = np.load(model_init_file)
        except:
            print("load init model failed", model_init_file)
            raise Exception("load init model failed")

        print("load init model")
        for name in param_name_list:
            t = fluid.global_scope().find_var(name).get_tensor()
            t.set(model_init[str(name)].astype('float32'), place)

        # load opt state
        opt_state_init_file = config.init_opt_state
        if opt_state_init_file != "":
            print("begin to load opt state")
            opt_state_data = np.load(opt_state_init_file)
            for k, v in opt_state_data.items():
                t = fluid.global_scope().find_var(str(k)).get_tensor()
                t.set(v, place)
            print("set opt state finished")

    print("init model parameters finshed")

def train_loop(config,
               train_generator, valid_generator,
               main_program, inference_program,
               model_handle, param_name_list, opt_var_name_list):
    """ model train loop """
    stage = config.stage
    [exe, place, bow_loss, kl_loss, nll_loss, final_loss] = model_handle

    total_step = 0
    start_epoch = 0 if stage == 0 else config.pretrain_epoch
    end_epoch = config.pretrain_epoch if stage == 0 else config.num_epochs
    print("start end", start_epoch, end_epoch)

    best_score = float('inf')
    for epoch_idx in range(start_epoch, end_epoch):
        total_bow_loss = 0
        total_kl_loss = 0
        total_nll_loss = 0
        total_final_loss = 0
        sample_num = 0

        for batch_id, data in enumerate(train_generator()):
            data_feed = build_data_feed(data, place,
                                        batch_size=config.batch_size,
                                        is_training=True,
                                        bow_max_len=config.max_len,
                                        pretrain_epoch=epoch_idx < config.pretrain_epoch)

            if data_feed is None:
                break

            out = exe.run(main_program, feed=data_feed,
                          fetch_list=[bow_loss.name, kl_loss.name, nll_loss.name, final_loss.name])

            total_step += 1
            total_bow_loss += out[0]
            total_kl_loss += out[1]
            total_nll_loss += out[2]
            total_final_loss += out[3]
            sample_num += 1

            if batch_id > 0 and batch_id % config.log_steps == 0:
                print("epoch %d step %d | "
                      "bow loss %0.6f kl loss %0.6f nll loss %0.6f total loss %0.6f" % \
                      (epoch_idx, batch_id,
                       total_bow_loss / sample_num, total_kl_loss / sample_num, \
                       total_nll_loss / sample_num, total_final_loss / sample_num))

                total_bow_loss = 0
                total_kl_loss = 0
                total_nll_loss = 0
                total_final_loss = 0
                sample_num = 0

            if batch_id > 0 and batch_id % config.valid_steps == 0:
                eval_bow_loss, eval_kl_loss, eval_nll_loss, eval_total_loss = \
                    vaild_loop(config, valid_generator, inference_program, model_handle)
                # save model
                if stage != 0:
                    param_path = config.save_dir + "/" + str(total_step)
                    fluid.io.save_params(executor=exe, dirname=param_path,
                                         main_program=main_program)

                    if eval_nll_loss < best_score:
                        # save to best
                        best_model_path = config.save_dir + "/best_model"
                        print("save to best", eval_nll_loss, best_model_path)
                        fluid.io.save_params(executor=exe, dirname=best_model_path,
                                             main_program=main_program)
                        best_score = eval_nll_loss

        eval_bow_loss, eval_kl_loss, eval_nll_loss, eval_total_loss = \
            vaild_loop(config, valid_generator, inference_program, model_handle)

        if stage != 0:
            param_path = config.save_dir + "/" + str(total_step)
            fluid.io.save_params(executor=exe, dirname=param_path,
                                 main_program=main_program)
            if eval_nll_loss < best_score:
                best_model_path = config.save_dir + "/best_model"
                print("save to best", eval_nll_loss, best_model_path)
                fluid.io.save_params(executor=exe, dirname=best_model_path,
                                     main_program=main_program)
                best_score = eval_nll_loss

    if stage == 0:
        # save last model and opt_stat to npz for next stage init
        save_model_file = config.save_dir + "/model_stage_0"
        save_opt_state_file = config.save_dir + "/opt_state_stage_0"

        model_stage_0 = {}
        for name in param_name_list:
            t = np.asarray(fluid.global_scope().find_var(name).get_tensor())
            model_stage_0[name] = t
        np.savez(save_model_file, **model_stage_0)

        opt_state_stage_0 = {}
        for name in opt_var_name_list:
            t_data = np.asarray(fluid.global_scope().find_var(name).get_tensor())
            opt_state_stage_0[name] = t_data
        np.savez(save_opt_state_file, **opt_state_stage_0)

def vaild_loop(config, valid_generator, inference_program, model_handle):
    """ model vaild loop """
    [exe, place, bow_loss, kl_loss, nll_loss, final_loss] = model_handle
    valid_num = 0.0
    total_valid_bow_loss = 0.0
    total_valid_kl_loss = 0.0
    total_valid_nll_loss = 0.0
    total_valid_final_loss = 0.0
    for batch_id, data in enumerate(valid_generator()):
        data_feed = build_data_feed(data, place,
                                    batch_size=config.batch_size,
                                    is_training=True,
                                    bow_max_len=config.max_len,
                                    pretrain_epoch=False)

        if data_feed is None:
            continue

        val_fetch_outs = \
            exe.run(inference_program,
                    feed=data_feed,
                    fetch_list=[bow_loss.name, kl_loss.name, nll_loss.name, final_loss.name])

        total_valid_bow_loss += val_fetch_outs[0] * config.batch_size
        total_valid_kl_loss += val_fetch_outs[1] * config.batch_size
        total_valid_nll_loss += val_fetch_outs[2] * config.batch_size
        total_valid_final_loss += val_fetch_outs[3] * config.batch_size
        valid_num += config.batch_size

    print("valid dataset: bow loss %0.6f kl loss %0.6f nll loss %0.6f total loss %0.6f" % \
          (total_valid_bow_loss / valid_num, total_valid_kl_loss / valid_num, \
           total_valid_nll_loss / valid_num, total_valid_final_loss / valid_num))

    return [total_valid_bow_loss / valid_num, total_valid_kl_loss / valid_num, \
           total_valid_nll_loss / valid_num, total_valid_final_loss / valid_num]

def test(config):
    """ test """
    batch_size = config.batch_size
    config.vocab_size = len(open(config.vocab_path).readlines())
    final_score, final_ids, final_index = knowledge_seq2seq(config)

    final_score.persistable = True
    final_ids.persistable = True
    final_index.persistable = True

    main_program = fluid.default_main_program()

    if config.use_gpu:
        place = fluid.CUDAPlace(0)
    else:
        place = fluid.CPUPlace()

    exe = Executor(place)
    exe.run(framework.default_startup_program())

    fluid.io.load_params(executor=exe, dirname=config.model_path,
                         main_program=main_program)
    print("laod params finsihed")

    # test data generator
    processors = KnowledgeCorpus(
        data_dir=config.data_dir,
        data_prefix=config.data_prefix,
        vocab_path=config.vocab_path,
        min_len=config.min_len,
        max_len=config.max_len)
    test_generator = processors.data_generator(
        batch_size=config.batch_size,
        phase="test",
        shuffle=False)

    # load dict
    id_dict_array = load_id2str_dict(config.vocab_path)

    out_file = config.output
    fout = open(out_file, 'w')
    for batch_id, data in enumerate(test_generator()):
        data_feed, sent_num = build_data_feed(data, place, batch_size=batch_size)

        if data_feed is None:
            break

        out = exe.run(feed=data_feed,
                      fetch_list=[final_score.name, final_ids.name, final_index.name])

        batch_score = out[0]
        batch_ids = out[1]
        batch_pre_index = out[2]

        batch_score_arr = np.split(batch_score, batch_size, axis=1)
        batch_ids_arr = np.split(batch_ids, batch_size, axis=1)
        batch_pre_index_arr = np.split(batch_pre_index, batch_size, axis=1)

        index = 0
        for (score, ids, pre_index) in zip(batch_score_arr, batch_ids_arr, batch_pre_index_arr):
            trace_ids, trace_score = trace_fianl_result(score, ids, pre_index, topk=1, EOS=3)
            fout.write(id_to_text(trace_ids[0][:-1], id_dict_array))
            fout.write('\n')

            index += 1
            if index >= sent_num:
                break

    fout.close()

def train(config):
    """ model training """
    config.vocab_size = len(open(config.vocab_path).readlines())
    bow_loss, kl_loss, nll_loss, final_loss= knowledge_seq2seq(config)

    bow_loss.persistable = True
    kl_loss.persistable = True
    nll_loss.persistable = True
    final_loss.persistable = True

    main_program = fluid.default_main_program()
    inference_program = fluid.default_main_program().clone(for_test=True)

    fluid.clip.set_gradient_clip(
        clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=config.grad_clip))
    optimizer = fluid.optimizer.Adam(learning_rate=config.lr)

    if config.stage == 0:
        print("stage 0")
        optimizer.minimize(bow_loss)
    else:
        print("stage 1")
        optimizer.minimize(final_loss)

    opt_var_name_list = optimizer.get_opti_var_name_list()

    if config.use_gpu:
        place = fluid.CUDAPlace(0)
    else:
        place = fluid.CPUPlace()

    exe = Executor(place)
    exe.run(framework.default_startup_program())

    param_list = main_program.block(0).all_parameters()
    param_name_list = [p.name for p in param_list]

    init_model(config, param_name_list, place)

    processors = KnowledgeCorpus(
                        data_dir=config.data_dir,
                        data_prefix=config.data_prefix,
                        vocab_path=config.vocab_path,
                        min_len=config.min_len,
                        max_len=config.max_len)
    train_generator = processors.data_generator(
                        batch_size=config.batch_size,
                        phase="train",
                        shuffle=True)
    valid_generator = processors.data_generator(
                        batch_size=config.batch_size,
                        phase="dev",
                        shuffle=False)

    model_handle = [exe, place, bow_loss, kl_loss, nll_loss, final_loss]

    train_loop(config,
               train_generator, valid_generator,
               main_program, inference_program,
               model_handle, param_name_list, opt_var_name_list)

if __name__ == "__main__":
    config = model_config()
    run_type = config.run_type

    if run_type == "train":
        train(config)
    elif run_type == "test":
        test(config)

ext: py

max_stars_repo_name: zhangyimi/Research

reason: import numpy


text:
import os
from pyhdx import PeptideMasterTable, KineticsSeries
from pyhdx.fileIO import read_dynamx, csv_to_protein
from pyhdx.fitting import fit_rates_weighted_average, fit_gibbs_global, fit_gibbs_global_batch, fit_gibbs_global_batch_aligned
from pyhdx.models import HDXMeasurementSet
import numpy as np
import torch
import pandas as pd
import time
from dask.distributed import LocalCluster
import asyncio

directory = os.path.dirname(__file__)
np.random.seed(43)
torch.manual_seed(43)

class TestSecBDataFit(object):
    @classmethod
    def setup_class(cls):
        fpath_apo = os.path.join(directory, 'test_data', 'ecSecB_apo.csv')
        fpath_dimer = os.path.join(directory, 'test_data', 'ecSecB_dimer.csv')
        data = read_dynamx(fpath_apo, fpath_dimer)
        control = ('Full deuteration control', 0.167)

        cls.temperature, cls.pH = 273.15 + 30, 8.

        pf = PeptideMasterTable(data, drop_first=1, ignore_prolines=True, remove_nan=False)
        pf.set_control(control)
        cls.series_apo = KineticsSeries(pf.get_state('SecB WT apo'), temperature=cls.temperature, pH=cls.pH)
        cls.series_dimer = KineticsSeries(pf.get_state('SecB his dimer apo'), temperature=cls.temperature, pH=cls.pH)

        data = pf.get_state('SecB WT apo')
        reduced_data = data[data['end'] < 40]
        cls.reduced_series = KineticsSeries(reduced_data)

        cluster = LocalCluster()
        cls.address = cluster.scheduler_address

    def test_initial_guess(self):
        result = fit_rates_weighted_average(self.reduced_series)
        output = result.output

        assert output.size == 100

        # todo additional assertl, compare to stored values

    def test_global_fit(self):
        #kf = KineticsFitting(self.series_apo, bounds=(1e-2, 800), temperature=self.temperature, pH=self.pH)
        initial_rates = csv_to_protein(os.path.join(directory, 'test_data', 'ecSecB_guess.txt'))

        t0 = time.time()  # Very crude benchmarks
        gibbs_guess = self.series_apo.guess_deltaG(initial_rates['rate']).to_numpy()
        fr_global = fit_gibbs_global(self.series_apo, gibbs_guess, epochs=1000)
        t1 = time.time()

        assert t1 - t0 < 5
        out_deltaG = fr_global.output
        check_deltaG = csv_to_protein(os.path.join(directory, 'test_data', 'ecSecB_torch_fit.txt'))

        assert np.allclose(check_deltaG['deltaG'], out_deltaG['deltaG'], equal_nan=True, rtol=0.01)
        assert np.allclose(check_deltaG['covariance'], out_deltaG['covariance'], equal_nan=True, rtol=0.01)

    def test_batch_fit(self):
        hdx_set = HDXMeasurementSet([self.series_apo, self.series_dimer])
        guess = csv_to_protein(os.path.join(directory, 'test_data', 'ecSecB_guess.txt'))

        gibbs_guess = hdx_set.guess_deltaG([guess['rate'], guess['rate']])
        result = fit_gibbs_global_batch(hdx_set, gibbs_guess, epochs=1000)

        output = result.output

        check_protein = csv_to_protein(os.path.join(directory, 'test_data', 'ecSecB_batch.csv'), column_depth=2)
        states = ['SecB WT apo', 'SecB his dimer apo']

        for state in states:
            from pandas.testing import assert_series_equal
            #
            result = output[state]['deltaG']
            test = check_protein[state]['deltaG']

            assert_series_equal(result, test, rtol=0.1)

        mock_alignment = {
            'apo': 'MSEQNNTEMTFQIQRIYTKDI------------SFEAPNAPHVFQKDWQPEVKLDLDTASSQLADDVYEVVLRVTVTASLG-------------------EETAFLCEVQQGGIFSIAGIEGTQMAHCLGAYCPNILFPYARECITSMVSRG----TFPQLNLAPVNFDALFMNYLQQQAGEGTEEHQDA',
            'dimer': 'MSEQNNTEMTFQIQRIYTKDISFEAPNAPHVFQKDWQPEVKLDLDTASSQLADDVY--------------EVVLRVTVTASLGEETAFLCEVQQGGIFSIAGIEGTQMAHCLGA----YCPNILFPAARECIASMVARGTFPQLNLAPVNFDALFMNYLQQQAGEGTEEHQDA-----------------',
        }

        hdx_set.add_alignment(list(mock_alignment.values()))

        gibbs_guess = hdx_set.guess_deltaG([guess['rate'], guess['rate']])
        aligned_result = fit_gibbs_global_batch_aligned(hdx_set, gibbs_guess, r1=2, r2=5, epochs=1000)
        output = aligned_result.output
        check_protein = csv_to_protein(os.path.join(directory, 'test_data', 'ecSecB_batch_aligned.csv'), column_depth=2)
        states = ['SecB WT apo', 'SecB his dimer apo']

        for state in states:
            from pandas.testing import assert_series_equal
            result = output[state]['deltaG']
            test = check_protein[state]['deltaG']

            assert_series_equal(result, test, rtol=0.1)

max_stars_repo_name: sebaztiano/PyHDX

max_issues_repo_name: sebaztiano/PyHDX

max_forks_repo_name: sebaztiano/PyHDX


text:
"""
Unrolled VAE
Ishaan Gulrajani
"""

import os, sys
sys.path.append(os.getcwd())

try: # This only matters on Ishaan's computer
    import experiment_tools
    experiment_tools.wait_for_gpu(tf=True, n_gpus=1)
except ImportError:
    pass

import tflib as lib
import tflib.debug
import tflib.ops.kl_unit_gaussian
import tflib.ops.linear

import tflib.mnist_binarized

import tensorflow as tf
import numpy as np

DIM = 512
LATENT_DIM = 32

BATCH_SIZE = 1000
TRAIN_ITERS = 100*500
# TRAIN_ITERS = 10

# MODE = 'vanilla'
# UNROLL_STEPS = 0

# Most fair comparison is unroll_decoder vs unroll_decoder with a tf.stop_gradient in the middle
MODE = 'unroll_decoder'
UNROLL_STEPS = 3

# OPTIMIZER = 'sgd'
# LEARNING_RATE = 1e-2

OPTIMIZER = 'adam'
LEARNING_RATE = 1e-4

train_data, dev_data, test_data = lib.mnist_binarized.load(BATCH_SIZE, BATCH_SIZE)

lib.print_model_settings(locals().copy())

lib.ops.linear.enable_default_weightnorm()

with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as session:
    images = tf.placeholder(tf.float32, shape=[None, 1, 28, 28], name='images')
    flat_images = tf.reshape(images, [-1, 784])

    # For some reason, ReLU immediately gives nans in certain configurations.
    # LeakyReLU and ELU seem to work fine.
    def nonlinearity(x):
        return tf.nn.elu(x)

    def Encoder(inputs):
        output = lib.ops.linear.Linear('Encoder.1', 784, DIM, inputs)
        output = nonlinearity(output)
        output = lib.ops.linear.Linear('Encoder.2', DIM, DIM, output)
        output = nonlinearity(output)
        # lib.debug.print_stats('Encoder.3_Before', output)
        output = lib.ops.linear.Linear('Encoder.3', DIM, 2*LATENT_DIM, output)
        # lib.debug.print_stats('Encoder.3_After', output)
        return output

    def Decoder(latents):
        # lib.debug.print_stats('Decoder.1_Before', latents)
        output = lib.ops.linear.Linear('Decoder.1', LATENT_DIM, DIM, latents)
        # lib.debug.print_stats('Decoder.1_After', output)
        output = nonlinearity(output)
        # lib.debug.print_stats('Decoder.1_After2', output)
        output = lib.ops.linear.Linear('Decoder.2', DIM, DIM, output)
        output = nonlinearity(output)
        output = lib.ops.linear.Linear('Decoder.3', DIM, 784, output)
        return output

    def build_forward_graph(flat_images_split):
        # flat_images_split = flat_images
        mu, log_sigma = tf.split(1, 2, Encoder(flat_images_split))
        sigma = tf.nn.softplus(log_sigma)
        log_sigma = tf.log(sigma)
        # sigma = tf.exp(log_sigma)

        # lib.debug.print_stats('mu', mu)
        # lib.debug.print_stats('sigma', sigma)
        # lib.debug.print_stats('log_sigma', log_sigma)

        eps = tf.random_normal(tf.shape(mu))
        latents = mu + (eps * sigma)

        logits = Decoder(latents)

        kl_cost = tf.reduce_mean(
            lib.ops.kl_unit_gaussian.kl_unit_gaussian(
                mu,
                log_sigma,
                sigma
            )
        ) * float(LATENT_DIM)

        reconst_cost = tf.reduce_mean(
            tf.nn.sigmoid_cross_entropy_with_logits(logits, flat_images_split)
        ) * float(784)

        cost = reconst_cost + kl_cost
        return cost, kl_cost

    cost, kl_cost = build_forward_graph(flat_images[0::UNROLL_STEPS+1])

    def make_adam_vars(name, params):
        t = lib.param(name+'.t', np.float32(0.))
        ms, vs = [], []
        for i, param in enumerate(params):
            ms.append(lib.param(name+'.m_{}'.format(i), np.zeros(param.get_shape(), dtype='float32')))
            vs.append(lib.param(name+'.v_{}'.format(i), np.zeros(param.get_shape(), dtype='float32')))
        return (t, ms, vs)

    def apply_adam(params, grads, adam_vars):
        t, ms, vs = adam_vars

        beta1=0.9
        beta2=0.999
        epsilon=1e-8

        t = t + 1.
        a_t = LEARNING_RATE*tf.sqrt(1.-beta2**t)/(1.-beta1**t)

        new_ms = []
        new_vs = []
        new_params = {}

        for param, g_t, m_prev, v_prev in zip(params, grads, ms, vs):
            m_t = beta1*m_prev + (1.-beta1)*g_t

            if OPTIMIZER == 'adam':
                v_t = beta2*v_prev + (1.-beta2)*g_t**2
                step = a_t*m_t/(tf.sqrt(v_t) + epsilon)
            elif OPTIMIZER == 'adamax':
                v_t = tf.maximum(beta2*v_prev + epsilon, tf.abs(g_t))
                step = LEARNING_RATE * m_t / v_t
            elif OPTIMIZER == 'sgd':
                v_t = v_prev
                step = LEARNING_RATE * g_t
            else:
                raise Exception()

            new_ms.append(m_t)
            new_vs.append(v_t)
            new_params[param] = param - step

        return new_params, (t, new_ms, new_vs)

    def make_adam_var_update_ops(old, new):
        old_t, old_ms, old_vs = old
        new_t, new_ms, new_vs = new
        updates = []
        updates.append(tf.assign(old_t, new_t))
        for old, new in zip(old_ms + old_vs, new_ms + new_vs):
            updates.append(tf.assign(old, new))
        return updates

    def clip_grads(grads):
        return tf.clip_by_global_norm(grads, 5.0)[0]

    enc_params = lib.params_with_name('Encoder')
    dec_params = lib.params_with_name('Decoder')

    enc_adam_vars = make_adam_vars('Encoder_Adam', enc_params)
    dec_adam_vars = make_adam_vars('Decoder_Adam', dec_params)

    if MODE == 'vanilla':
        enc_grads = tf.gradients(cost, enc_params)
        dec_grads = tf.gradients(cost, dec_params)

        enc_updates, new_enc_adam_vars = apply_adam(enc_params, enc_grads, enc_adam_vars)
        dec_updates, new_dec_adam_vars = apply_adam(dec_params, dec_grads, dec_adam_vars)

        update_ops = []
        for old, new in enc_updates.items() + dec_updates.items():
            update_ops.append(tf.assign(old, new))

        update_ops.extend(make_adam_var_update_ops(enc_adam_vars, new_enc_adam_vars))
        update_ops.extend(make_adam_var_update_ops(dec_adam_vars, new_dec_adam_vars))

    elif MODE == 'unroll_decoder':

        dec_grads = clip_grads(tf.gradients(cost, dec_params))
        dec_updates, new_dec_adam_vars = apply_adam(dec_params, dec_grads, dec_adam_vars)

        all_costs = [cost]

        new_cost = cost
        new_dec_params = [dec_updates[p] for p in dec_params]
        new_dec_updates = dec_updates
        newer_dec_adam_vars = new_dec_adam_vars
        for i in xrange(UNROLL_STEPS - 1):
            lib.alias_params(new_dec_updates)
            new_cost, new_kl_cost = build_forward_graph(flat_images[i+1::UNROLL_STEPS])
            all_costs.append(new_cost)
            new_dec_updates, newer_dec_adam_vars = apply_adam(
                new_dec_params,
                clip_grads(tf.gradients(new_cost, new_dec_params)),
                newer_dec_adam_vars
            )
            new_dec_params = [new_dec_updates[p] for p in new_dec_params]

        lib.alias_params(new_dec_updates)
        new_cost, new_kl_cost = build_forward_graph(flat_images[UNROLL_STEPS::UNROLL_STEPS+1])

        enc_grads = clip_grads(tf.gradients(tf.reduce_mean(tf.pack(all_costs)), enc_params))
        enc_updates, new_enc_adam_vars = apply_adam(enc_params, enc_grads, enc_adam_vars)

        update_ops = []
        for old, new in enc_updates.items() + ({old:new for old,new in zip(dec_params, new_dec_params)}).items():
            update_ops.append(tf.assign(old, new))

        update_ops.extend(make_adam_var_update_ops(enc_adam_vars, new_enc_adam_vars))
        update_ops.extend(make_adam_var_update_ops(dec_adam_vars, newer_dec_adam_vars))

    else:
        raise Exception()

    session.run(tf.initialize_all_variables())

    def inf_train_gen():
        while True:
            for data in train_data():
                yield data

    train_iters = 0
    costs, kl_costs = [], []
    for (_images,) in inf_train_gen():
        lib.debug.print_all_stats({images:_images})
        _outputs = session.run([cost, kl_cost] + update_ops, feed_dict={images: _images})
        costs.append(_outputs[0])
        kl_costs.append(_outputs[1])
        # if True:
        if train_iters % 500 == 499:
            print "iter {}\tcost {}\tkl {}".format(train_iters, np.mean(costs), np.mean(kl_costs))
            costs, kl_costs = [], []
        train_iters += 1
        if train_iters == TRAIN_ITERS:
            break


text:
"""
Distributed multiclass meta-estimators
"""

import numpy as np
import pandas as pd
import warnings

from joblib import Parallel, delayed
from copy import copy
from collections.abc import Sequence
from scipy.sparse import vstack
from sklearn.base import BaseEstimator
from sklearn.multiclass import OneVsRestClassifier, OneVsOneClassifier
from sklearn.preprocessing import LabelBinarizer, MultiLabelBinarizer
from sklearn.preprocessing import normalize
from sklearn.utils import shuffle
from sklearn.utils.validation import check_X_y
from sklearn.utils.multiclass import check_classification_targets, type_of_target
from sklearn.model_selection import train_test_split, GroupKFold

from .validation import _check_estimator
from .utils import _safe_split
from .base import (
    _clone, _get_value, _parse_partitions
    )

__all__ = [
    "DistOneVsRestClassifier",
    "DistOneVsOneClassifier"
]

def _chunks(l, n):
    """ Yield successive n-sized chunks from input list """
    for i in range(0, len(l), n):
        yield l[i:i + n]

def _split_X(X, n_splits, sc):
    """ Split and broadcast a sparse matrix """
    if n_splits > 1:
        if isinstance(X, pd.DataFrame):
            inds = list(range(len(X)))
            inds_list = list(_chunks(inds, int(len(inds) / float(n_splits))))
            Xs = [X.ix[slc] for slc in inds_list]
        else:
            # X is a scipy sparse matrix
            inds = list(range(X.shape[0]))
            inds_list = list(_chunks(inds, int(len(inds) / float(n_splits))))
            Xs = [X[slc, :] for slc in inds_list]

        return [sc.broadcast(x) for x in Xs]
    else:
        return X

def _combine_Xs(Xs):
    """ Combine and access values of broadcasted, splitted sparse matrix """
    if isinstance(Xs, list):
        if isinstance(Xs[0].value, pd.DataFrame):
            df = pd.DataFrame()
            return df.append([x.value for x in Xs])
        else:
            return vstack([x.value for x in Xs])
    else:
        return Xs

def _use_best_estimator(x):
    """ Pull best estimator out of meta-estimator if available """
    estimator_ = x.best_estimator_ if hasattr(x, "best_estimator_") else x
    if hasattr(x, "cv_results_"):
        estimator_.cv_results_ = pd.DataFrame(x.cv_results_)
        for col in estimator_.cv_results_.columns:
            estimator_.cv_results_[col] = estimator_.cv_results_[col].astype(str)
        estimator_.cv_results_ = estimator_.cv_results_.to_dict("list")
    return estimator_

def _negatives_mask(X, y, max_negatives=None, random_state=None, method="ratio"):
    """ Limit the number of negative records in training set """
    if max_negatives is None:
        return [X, y]
    else:
        pos_mask = y == 1
        if method == "ratio":
            pass
        elif method == "multiplier":
            max_negatives = int(max_negatives * len(y[pos_mask]))
        else:
            raise ValueError("Unknown method. Options are 'ratio' or 'multiplier'.")
        if isinstance(max_negatives, int) and max_negatives >= len(y[~pos_mask]):
            return [X, y]
        max_negatives = (
            max_negatives
            if isinstance(max_negatives, float)
            else (max_negatives / float(len(y[~pos_mask]))))
        _, X_neg, _, y_neg = train_test_split(
            X[~pos_mask, :], y[~pos_mask],
            test_size=max_negatives, random_state=random_state)
        stack_func = vstack if hasattr(X, "nnz") else np.vstack
        return shuffle(
            stack_func([X[pos_mask, :], X_neg]), np.concatenate([y[pos_mask], y_neg]),
            random_state=random_state)

def _fit_binary(estimator, X, y, fit_params, classes=None, max_negatives=None, random_state=None, method="ratio"):
    """ Fit a single binary estimator """
    X = _combine_Xs(X)
    unique_y = np.unique(y)
    if len(unique_y) == 1:
        if classes is not None:
            if y[0] == -1:
                c = 0
            else:
                c = y[0]
            warnings.warn("Label %s is present in all training examples." %
                          str(classes[c]))
        est = _ConstantPredictor().fit(*_negatives_mask(
            (X.value if hasattr(X, "value") else X), unique_y,
            max_negatives=max_negatives, random_state=random_state,
            method=method))
    else:
        est = _clone(_get_value(estimator))
        est.fit(*_negatives_mask(
            (X.value if hasattr(X, "value") else X), y,
            max_negatives=max_negatives, random_state=random_state,
            method=method), **fit_params)
    return _use_best_estimator(est), classes[1]

def _fit_ovo_binary(estimator, X, y, i, j, fit_params):
    """ Fit a single binary estimator (one-vs-one) """
    cond = np.logical_or(y == i, y == j)
    y_ = y[cond]
    y_binary = np.empty(y_.shape, np.int)
    y_binary[y_ == i] = 0
    y_binary[y_ == j] = 1
    indcond = np.arange(X.shape[0])[cond]
    return _fit_binary(estimator,
                       _safe_split(estimator, X, None, indices=indcond)[0],
                       y_binary, fit_params, classes=[i, j]), indcond

class _ConstantPredictor(BaseEstimator):
    """ Predicts same labels as trained """
    def fit(self, X, y):
        self.y_ = y
        return self

    def predict(self, X):
        check_is_fitted(self, 'y_')
        return np.repeat(self.y_, X.shape[0])

    def decision_function(self, X):
        check_is_fitted(self, 'y_')
        return np.repeat(self.y_, X.shape[0])

    def predict_proba(self, X):
        check_is_fitted(self, 'y_')
        return np.repeat([np.hstack([1 - self.y_, self.y_])],
                         X.shape[0], axis=0)

class DistOneVsRestClassifier(OneVsRestClassifier):
    """
    Same as sklearn `OneVsRestClassifier` but with distributed
    training using spark. Additionally implements flexible
    ``predict_proba`` method with custom `norm` input
    designating the normalization method used after individual
    predictions are made.

    Args:
        estimator (sklearn estimator): An estimator object implementing fit and one of
            decision_function or predict_proba.
        sc (sparkContext): Spark context for spark broadcasting and rdd operations.
        norm (string): default None, Normalization method for predict_proba.
        partitions (int or 'auto'): default 'auto'
            Number of partitions to use for parallelization of parameter
            search space. Integer values or None will be used directly for `numSlices`,
            while 'auto' will set `numSlices` to the number required fits.
        max_negatives (int or float): default None
            Maximum number of negative records allowed for each binary
            estimator. Use int for hard maximum, or float for percentage
            of total negatives.
        random_state (int): default None
            Random state for limiting negatives (if max_negatives is not None).
        method (str): 'ratio' or 'multiplier'
            Method used to calculate true maximum number of negatives.
        n_splits (int): default 1
            Dials the number of splits for broadcasting
            X during fitting. Use values higher than 1 for large X.
        mlb_override (bool): pass over mlb step; this assumes
            that input `y` to `fit` is already in sparse (one-hot-encoded)
            format
        verbose (bool): print status messages
        **kwargs: Keyword arguments to be passed to `OneVsRestClassifier`.
    """
    def __init__(self, estimator, sc=None, norm=None, partitions='auto',
            max_negatives=None, random_state=None, method="ratio",
            n_splits=1, mlb_override=False, verbose=False, **kwargs):
        OneVsRestClassifier.__init__(
            self, estimator, **kwargs)
        self.norm = norm
        self.sc = sc
        self.partitions = partitions
        self.max_negatives = max_negatives
        self.random_state = random_state
        self.method = method
        self.n_splits = n_splits
        self.mlb_override = mlb_override
        self.verbose = verbose

    def fit(self, X, y, **fit_params):
        """
        Fit underlying estimators. Parallelize fit operation using spark.

        Args:
            X (array-like, shape = [n_samples, n_features]): input data
            y (array-like, shape = [n_samples, ], [n_samples, n_classes]): multi-class targets
            **fit_params (dict of string -> object): parameters passed
                to the ``fit`` method of the estimator
        """
        _check_estimator(self, verbose=self.verbose)

        if (not self.mlb_override and not hasattr(y[0], '__array__')
                and isinstance(y[0], Sequence)
                and not isinstance(y[0], str)):
            self.mlb = MultiLabelBinarizer()
            y = self.mlb.fit_transform(y)

        if isinstance(X, pd.DataFrame):
            X.index = list(range(len(X)))

        self.label_binarizer_ = LabelBinarizer(sparse_output=True)
        self.label_binarizer_.fit(y)
        self.classes_ = self.label_binarizer_.classes_
        self._fit(X, y, **fit_params)
        del self.sc
        if hasattr(self.estimator, "sc"):
            del self.estimator.sc
        return self

    def _fit(self, X, y, **fit_params):
        Y = self.label_binarizer_.transform(y)
        Y = Y.tocsc()
        max_negatives = self.max_negatives
        random_state = self.random_state
        n_splits = self.n_splits
        method = self.method
        estimator = _clone(self.estimator)
        if self.sc is None:
            models_ = Parallel(n_jobs=self.n_jobs)(
                delayed(_fit_binary)(
                    estimator, X, x[1], fit_params,
                    classes=["not %s" % x[0], x[0]],
                    max_negatives=max_negatives,
                    random_state=random_state, method=method)
                for x in list(zip(self.classes_, list(col.toarray().ravel() for col in Y.T))))
        else:
            X = _split_X(X, n_splits, self.sc)
            partitions = _parse_partitions(self.partitions, len(self.classes_))
            estimator = self.sc.broadcast(self.estimator)
            columns = self.sc.parallelize(
                list(zip(self.classes_, list(col.toarray().ravel() for col in Y.T))),
                numSlices=partitions)
            models_ = columns.map(lambda x: _fit_binary(
                estimator, X, x[1], fit_params, classes=["not %s" % x[0], x[0]],
                max_negatives=max_negatives, random_state=random_state, method=method)).collect()
        estimators_ = [x[0] for x in models_]
        classes_ = [x[1] for x in models_]
        self.estimators_ = list([estimators_[classes_.index(x)] for x in self.classes_])
        return self

    def predict_proba(self, X):
        """
        Probability estimates.
        The returned estimates for all classes are ordered by label of classes.

        Args:
            X (array-like, shape = [n_samples, n_features]): input data

        Returns:
            T (array-like, shape = [n_samples, n_classes]): returns the probability
                of the sample for each class in the model, where classes are
                ordered as they are in self.classes_
        """
        probs = []
        for index in range(len(self.estimators_)):
            probs.append(self.estimators_[index].predict_proba(X)[:,1])
        out = np.array([
            [probs[y][index] for y in range(len(self.estimators_))]
            for index in range(len(probs[0]))])
        if self.norm:
            return normalize(out, norm=self.norm)
        else:
            return out

class DistOneVsOneClassifier(OneVsOneClassifier):
    """
    Same as sklearn `OneVsOneClassifier` but with distributed
    training using spark.

    Args:
        estimator (sklearn estimator): An estimator object implementing fit and one of
            decision_function or predict_proba.
        sc (sparkContext): Spark context for spark broadcasting and rdd operations.
        partitions (int or 'auto'): default 'auto'
            Number of partitions to use for parallelization of parameter
            search space. Integer values or None will be used directly for `numSlices`,
            while 'auto' will set `numSlices` to the number required fits.
        verbose (bool): print status messages
        **kwargs: Keyword arguments to be passed to `OneVsOneClassifier`.
    """
    def __init__(self, estimator, sc=None, partitions='auto', verbose=False, **kwargs):
        OneVsOneClassifier.__init__(
            self, estimator, **kwargs)
        self.sc = sc
        self.partitions = partitions
        self.verbose = verbose

    def fit(self, X, y, **fit_params):
        """
        Fit underlying estimators. Parallelize fit operation using spark.

        Args:
            X (array-like, shape = [n_samples, n_features]): intput
                data
            y (array-like, shape = [n_samples]): Multi-class targets.
            **fit_params (dict of string -> object): parameters passed
                to the ``fit`` method of the estimator
        """
        _check_estimator(self, verbose=self.verbose)

        X, y = check_X_y(X, y, accept_sparse=['csr', 'csc'])
        check_classification_targets(y)

        self.classes_ = np.unique(y)
        if len(self.classes_) == 1:
            raise ValueError("OneVsOneClassifier can not be fit when only one"
                             " class is present.")
        n_classes = self.classes_.shape[0]
        class_list = []
        count = -1
        for i in range(n_classes):
            for j in range(i + 1, n_classes):
                count += 1
                class_list.append([count, self.classes_[i], self.classes_[j]])
        estimator = _clone(self.estimator)

        if self.sc is None:
            estimators_indices = list(zip(*(Parallel(n_jobs=self.n_jobs)(
                delayed(_fit_ovo_binary)
                (self.estimator, X, y, self.classes_[i], self.classes_[j], fit_params)
                for i in range(n_classes) for j in range(i + 1, n_classes)))))
            self.estimators_ = [x[0] for x in estimators_indices[0]]
            self.pairwise_indices_ = (
                estimators_indices[1] if self._pairwise else None)
        else:
            estimator = self.sc.broadcast(self.estimator)
            partitions = _parse_partitions(self.partitions, len(class_list))
            estimators_indices = self.sc.parallelize(
                    class_list,
                    numSlices=partitions).groupBy(lambda x: x[0]).map(
                    lambda x: [x[0], _fit_ovo_binary(estimator, X, y, list(x[1])[0][1], list(x[1])[0][2], fit_params)]).collect()

            estimators_indices = [estimators_indices[i][1] for i in np.argsort([x[0] for x in estimators_indices])]
            self.estimators_ = [x[0][0] for x in estimators_indices]
            try:
                self.pairwise_indices_ = (
                    [x[1] for x in estimators_indices] if self._pairwise else None)
            except AttributeError:
                self.pairwise_indices_ = None
        del self.sc
        if hasattr(self.estimator, "sc"):
            del self.estimator.sc
        return self

max_stars_repo_name: ML-Tools/sk-dist

max_issues_repo_name: ML-Tools/sk-dist

max_forks_repo_name: ML-Tools/sk-dist

reason: import numpy,from scipy


text:
import cv2
import mediapipe as mp
import os
import numpy as np
import HandTrackingModule as htm

################################################
#Configurable variables
brushThickness = 15
eraserThickness = 100
################################################

folderPath = "Header Files"
myList = os.listdir(folderPath)
# print(myList)  Output: ['4.png', '1.png', '3.png', '2.png']

overlayList = []
for imgPath in myList:
	image = cv2.imread(f"{folderPath}/{imgPath}")
	overlayList.append(image)
# print(overlayList)

header = overlayList[1]
drawColour = (255, 0, 255)

# cap = cv2.VideoCapture(0)
cap = cv2.VideoCapture(2)
cap.set(3, 1280)
cap.set(4, 720)

detector = htm.HandDetector(detectionConf=0.85)
xp, yp = 0, 0
imageCanvas = np.zeros((720, 1280, 3), np.uint8)

while True:
	# Import Image
	_, image = cap.read()
	image = cv2.flip(image, 1)

	# Find Hand Landmarks
	image = detector.findHands(image)
	landmarkList = detector.findPosition(image, draw=False)

	if len(landmarkList) != 0:
		# print(landmarkList)

		# Tip of  index and middle finger
		x1, y1 = landmarkList[8][1:]
		x2, y2 = landmarkList[12][1:]

		# Check which fingers are up ?
		fingers = detector.fingersUp()
		# print(fingers)

		# Selection mode : 2 fingers are up
		if fingers[1] and fingers[2]:
			# print("Selection mode")
			xp, yp = 0, 0
			if y1 < 125:
				if 250 < x1 < 450:
					header = overlayList[1]
					drawColour = (255, 0, 255) # Purple
				elif 550 < x1 < 750:
					header = overlayList[3]
					drawColour = (255, 0, 0) # Blue
				elif 800 < x1 < 950:
					header = overlayList[2]
					drawColour = (0, 255, 0) # Green
				elif 1050 < x1 < 1200:
					header = overlayList[0]
					drawColour = (0, 0, 0) # Black

			cv2.rectangle(image, (x1, y1-15), (x2, y2+15), drawColour, cv2.FILLED)
		# Drawing mode : Index finger is up
		if fingers[1] and not fingers[2]:
			cv2.circle(image, (x1, y1), 15, drawColour, cv2.FILLED)
			# print("Drawing mode")
			if xp == 0 and yp == 0:
				xp, yp = x1, y1

			if  drawColour == (0, 0, 0):
				cv2.line(image, (xp, yp), (x1, y1), drawColour, eraserThickness)
				cv2.line(imageCanvas, (xp, yp), (x1, y1), drawColour, eraserThickness)

			cv2.line(image, (xp, yp), (x1, y1), drawColour, brushThickness)
			cv2.line(imageCanvas, (xp, yp), (x1, y1), drawColour, brushThickness)
			xp, yp = x1, y1

	imageGray = cv2.cvtColor(imageCanvas, cv2.COLOR_BGR2GRAY)
	_, imageInverse = cv2.threshold(imageGray, 50, 255, cv2.THRESH_BINARY_INV)
	imageInverse = cv2.cvtColor(imageInverse, cv2.COLOR_GRAY2BGR)
	image = cv2.bitwise_and(image, imageInverse)
	image = cv2.bitwise_or(image, imageCanvas)

	image[0:125, 0:1280] = header # Parcing the image as it is a matrix
	# image = cv2.addWeighted(image, 0.5, imageCanvas, 0.5, 0)
	cv2.imshow("Frontend  Canvas", image)
	# cv2.imshow("Backend Canvas", imageCanvas)
	if cv2.waitKey(1) & 0xFF == ord('q'):
		break


text:
#!/usr/bin/env python3

# Copyright (c) Facebook, Inc. and its affiliates.
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

from typing import List, Optional, Union

import numpy as np

from habitat import get_config as get_task_config
from habitat.config import Config as CN

DEFAULT_CONFIG_DIR = "configs/"
CONFIG_FILE_SEPARATOR = ","
# -----------------------------------------------------------------------------
# EXPERIMENT CONFIG
# -----------------------------------------------------------------------------
_C = CN()
_C.MULTIPLY_SCENES = False
_C.BASE_TASK_CONFIG_PATH = "configs/tasks/explore_replica.yaml"
_C.TASK_CONFIG = CN()  # task_config will be stored as a config node
_C.SHARED_DATA = []  # HACKY share stuff
_C.SHARED_SIZES = []  # HACKY share stuff
_C.CMD_TRAILING_OPTS = []  # store command line options as list of strings
_C.TRAINER_NAME = "ppo"
_C.ENV_NAME = "NavRLEnv"
_C.SIMULATOR_GPU_ID = 0
_C.TORCH_GPU_ID = 0
_C.VIDEO_OPTION = ["disk", "tensorboard"]
_C.VIDEO_OPTION_INTERVAL = 10
_C.TENSORBOARD_DIR = "tb"
_C.VIDEO_DIR = "video_dir"
_C.TEST_EPISODE_COUNT = 36
_C.EVAL_CKPT_PATH_DIR = "data/checkpoints"  # path to ckpt or path to ckpts dir
_C.NUM_PROCESSES = 12
_C.SENSORS = ["RGB_SENSOR", "DEPTH_SENSOR"]
_C.CHECKPOINT_FOLDER = "data/checkpoints"
_C.NUM_UPDATES = 5000
_C.HARD_NUM_UPDATES = 1000000
_C.LOG_INTERVAL = 10
_C.LOG_FILE = "train.log"
_C.CHECKPOINT_INTERVAL = 50
_C.COMMIT = ""
_C.EVAL_MODE = False
_C.PRETRAINED_CHECKPOINT_PATH = ""
_C.LIVE_VIEW_ENV = -1  # Index of env to show obs from

# -----------------------------------------------------------------------------
# IMAGE TRANSFORM
# -----------------------------------------------------------------------------
_C.BATCH_INPUT = 2

_C.TRANSFORM = CN()
_C.TRANSFORM.ENABLED = False
_C.TRANSFORM.min_scale = 1.
_C.TRANSFORM.max_scale = 1.2

_C.TRANSFORM_RGB = CN()
_C.TRANSFORM_RGB.ENABLED = False

_C.TRANSFORM_DEPTH = CN()
_C.TRANSFORM_DEPTH.ENABLED = False
_C.TRANSFORM_DEPTH.std_noise = 0.2
_C.TRANSFORM_DEPTH.depth_min_th = 0.04
_C.TRANSFORM_DEPTH.depth_max_th = 0.8
_C.TRANSFORM_DEPTH.noise_interval = 0.03

# -----------------------------------------------------------------------------
# EVAL CONFIG
# -----------------------------------------------------------------------------
_C.EVAL = CN()
# The split to evaluate on
_C.EVAL.SPLIT = "val"
_C.EVAL.USE_CKPT_CONFIG = True
# -----------------------------------------------------------------------------
# REINFORCEMENT LEARNING (RL) ENVIRONMENT CONFIG
# -----------------------------------------------------------------------------
_C.RL = CN()
_C.RL.SUCCESS_REWARD = 10.0
_C.RL.SLACK_REWARD = -0.01
_C.RL.COLLISION_REWARD_ENABLED = False
_C.RL.COLLISION_DISTANCE = 0.
_C.RL.COLLISION_REWARD = -1.

_C.RL.NO_OPERATION = False
_C.RL.DEPTH_BLOCK_VIEW_FACTOR = 0.15
_C.RL.DEPTH_BLOCK_MIN_UNBLOCK = 0.1

# -----------------------------------------------------------------------------
# DETECTOR
_C.DETECTOR = CN()
_C.DETECTOR.model_def = "yolov3/config/yolov3.cfg"
_C.DETECTOR.weights_path = "yolov3/weights/yolov3.weights"
_C.DETECTOR.class_path = "yolov3/data/coco.names"
_C.DETECTOR.iou_thres = 0.5
_C.DETECTOR.conf_thres = 0.2
_C.DETECTOR.img_size = 256
_C.DETECTOR.nms_thres = 0.4
_C.DETECTOR.out_size = 32

# -----------------------------------------------------------------------------
# PEPPER
# -----------------------------------------------------------------------------
_C.PEPPER = CN()
_C.PEPPER.BufferSize = 10
_C.PEPPER.ForwardStep = 0.25
_C.PEPPER.TurnStep = 0.0872
_C.PEPPER.RGBTopic = "/pepper_robot/naoqi_driver/camera/front/image_raw"
_C.PEPPER.DepthTopic = "/pepper_robot/naoqi_driver/camera/depth/image_raw"
_C.PEPPER.MoveTopic = "/move_base_simple/goal"
_C.PEPPER.PoseTopic = "/slam_out_pose"
_C.PEPPER.DisplayImages = False
_C.PEPPER.OdomTopic = "/pepper_robot/naoqi_driver/odom"
_C.PEPPER.GoalTopic = "/clickedpoint"
_C.PEPPER.SonarTopic = "/pepper_robot/naoqi_driver/sonar/front"

_C.PEPPER.EpisodePath = "/home/amiro/workspace/alex/habitat-api/dataset_pepper nou/"

# -----------------------------------------------------------------------------
# PROXIMAL POLICY OPTIMIZATION (PPO)
# -----------------------------------------------------------------------------
_C.RL.PPO = CN()
_C.RL.PPO.actor_critic = CN()
_C.RL.PPO.actor_critic.fixed_distribution = []
_C.RL.PPO.actor_critic.type = "ExploreNavBaselinePolicy"
_C.RL.PPO.actor_critic.num_recurrent_layers = 1
_C.RL.PPO.actor_critic.rnn_type = "GRU"
_C.RL.PPO.actor_critic.aux = []
_C.RL.PPO.actor_critic.map_aux_to_obs = []

_C.RL.PPO.actor_critic.RelativePositionPredictor = CN()
_C.RL.PPO.actor_critic.RelativePositionPredictor.name = "rel_pos"
_C.RL.PPO.actor_critic.RelativePositionPredictor.out_size = 3
_C.RL.PPO.actor_critic.RelativePositionPredictor.loss_coeff = 1.
_C.RL.PPO.actor_critic.RelativePositionPredictor.target = "gps_compass"

_C.RL.PPO.actor_critic.RelativeRegressionStartPositionPredictor = CN()
_C.RL.PPO.actor_critic.RelativeRegressionStartPositionPredictor.name = "rel_start_pos_reg"
_C.RL.PPO.actor_critic.RelativeRegressionStartPositionPredictor.out_size = 3
_C.RL.PPO.actor_critic.RelativeRegressionStartPositionPredictor.loss_coeff = 1.
_C.RL.PPO.actor_critic.RelativeRegressionStartPositionPredictor.target = "gps_compass_start"
_C.RL.PPO.actor_critic.RelativeRegressionStartPositionPredictor.max_value = 15.

_C.RL.PPO.actor_critic.AdversarialDomainAdaptation = CN()
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.name = "adv_domain_adaptation"
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.out_size = 0
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.loss_coeff = 1.
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.env_memory_size = 1000.
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.real_memory_size = 1000.
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.finetune_rl = False
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.train_epochs = 1000
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.batch_size = 64
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.optim_name_d = "Adam"
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.optim_name_v = "Adam"
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.optim_param_d = CN()
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.optim_param_d.lr = 0.0001
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.optim_param_v = CN()
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.optim_param_v.lr = 0.0004
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.feature_loss = "mse_loss"
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.log_freq = 2
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.optim_step_d = 1
_C.RL.PPO.actor_critic.AdversarialDomainAdaptation.optim_step_v = 1

_C.RL.PPO.actor_critic.ActionPrediction = CN()
_C.RL.PPO.actor_critic.ActionPrediction.name = "action"
_C.RL.PPO.actor_critic.ActionPrediction.out_size = 3
_C.RL.PPO.actor_critic.ActionPrediction.loss_coeff = 1.
_C.RL.PPO.actor_critic.ActionPrediction.target = ""

_C.RL.PPO.actor_critic.SonarPredictor = CN()
_C.RL.PPO.actor_critic.SonarPredictor.name = "sonar"
_C.RL.PPO.actor_critic.SonarPredictor.out_size = 1
_C.RL.PPO.actor_critic.SonarPredictor.min_sonar = 0.25
_C.RL.PPO.actor_critic.SonarPredictor.max_sonar = 2.8
_C.RL.PPO.actor_critic.SonarPredictor.loss_coeff = 1.
_C.RL.PPO.actor_critic.SonarPredictor.target = "depth2"

_C.RL.PPO.visual_encoder = "SimpleCNN"
_C.RL.PPO.visual_encoder_dropout = 0.0
_C.RL.PPO.channel_scale = 1
_C.RL.PPO.clip_param = 0.1
_C.RL.PPO.ppo_epoch = 4
_C.RL.PPO.num_mini_batch = 4
_C.RL.PPO.value_loss_coef = 0.5
_C.RL.PPO.entropy_coef = 0.01
_C.RL.PPO.action_loss_coef = 1.0
_C.RL.PPO.lr = 2.5e-4
_C.RL.PPO.eps = 1e-5
_C.RL.PPO.max_grad_norm = 0.5
_C.RL.PPO.num_steps = 128
_C.RL.PPO.hidden_size = 512
_C.RL.PPO.use_gae = True
_C.RL.PPO.use_linear_lr_decay = True
_C.RL.PPO.use_linear_clip_decay = False
_C.RL.PPO.gamma = 0.99
_C.RL.PPO.tau = 0.95
_C.RL.PPO.reward_window_size = 50
# -----------------------------------------------------------------------------
# REACHABILITY NETWORK CONFIG
# -----------------------------------------------------------------------------
_C.RL.REACHABILITY = CN()
_C.RL.REACHABILITY.enabled = False
_C.RL.REACHABILITY.train = False
_C.RL.REACHABILITY.skip_train_ppo_without_rtrain = False
_C.RL.REACHABILITY.only_intrinsic_reward = False
_C.RL.REACHABILITY.experience_buffer_size = 720000
_C.RL.REACHABILITY.num_recurrent_steps = 1
_C.RL.REACHABILITY.batch_size = 64
_C.RL.REACHABILITY.num_train_epochs = 10
_C.RL.REACHABILITY.feature_extractor_size = 512
_C.RL.REACHABILITY.memory_size = 200
_C.RL.REACHABILITY.log_freq = 10
_C.RL.REACHABILITY.grid_resolution = 1

_C.RL.REACHABILITY.optimizer = "Adam"
_C.RL.REACHABILITY.optimizer_args = CN()
_C.RL.REACHABILITY.optimizer_args.lr = 0.00025

# Hyperparams
_C.RL.REACHABILITY.max_action_distance_k = 5
_C.RL.REACHABILITY.negative_sample_multiplier = 5
_C.RL.REACHABILITY.curiosity_bonus_scale_a = 0.030
_C.RL.REACHABILITY.reward_shift_b = 0.5
_C.RL.REACHABILITY.novelty_threshold = 0.5
_C.RL.REACHABILITY.similarity_aggregation = "percentile"
_C.RL.REACHABILITY.similarity_percentile = 90

# -----------------------------------------------------------------------------
# ORBSLAM2 BASELINE
# -----------------------------------------------------------------------------
_C.ORBSLAM2 = CN()
_C.ORBSLAM2.SLAM_VOCAB_PATH = "habitat_baselines/slambased/data/ORBvoc.txt"
_C.ORBSLAM2.SLAM_SETTINGS_PATH = (
    "habitat_baselines/slambased/data/mp3d3_small1k.yaml"
)
_C.ORBSLAM2.MAP_CELL_SIZE = 0.1
_C.ORBSLAM2.MAP_SIZE = 40
_C.ORBSLAM2.CAMERA_HEIGHT = get_task_config().SIMULATOR.DEPTH_SENSOR.POSITION[
    1
]
_C.ORBSLAM2.BETA = 100
_C.ORBSLAM2.H_OBSTACLE_MIN = 0.3 * _C.ORBSLAM2.CAMERA_HEIGHT
_C.ORBSLAM2.H_OBSTACLE_MAX = 1.0 * _C.ORBSLAM2.CAMERA_HEIGHT
_C.ORBSLAM2.D_OBSTACLE_MIN = 0.1
_C.ORBSLAM2.D_OBSTACLE_MAX = 4.0
_C.ORBSLAM2.PREPROCESS_MAP = True
_C.ORBSLAM2.MIN_PTS_IN_OBSTACLE = (
    get_task_config().SIMULATOR.DEPTH_SENSOR.WIDTH / 2.0
)
_C.ORBSLAM2.ANGLE_TH = float(np.deg2rad(15))
_C.ORBSLAM2.DIST_REACHED_TH = 0.15
_C.ORBSLAM2.NEXT_WAYPOINT_TH = 0.5
_C.ORBSLAM2.NUM_ACTIONS = 3
_C.ORBSLAM2.DIST_TO_STOP = 0.05
_C.ORBSLAM2.PLANNER_MAX_STEPS = 500
_C.ORBSLAM2.DEPTH_DENORM = get_task_config().SIMULATOR.DEPTH_SENSOR.MAX_DEPTH

def get_config(
    config_paths: Optional[Union[List[str], str]] = None,
    opts: Optional[list] = None,
) -> CN:
    r"""Create a unified config with default values overwritten by values from
    `config_paths` and overwritten by options from `opts`.
    Args:
        config_paths: List of config paths or string that contains comma
        separated list of config paths.
        opts: Config options (keys, values) in a list (e.g., passed from
        command line into the config. For example, `opts = ['FOO.BAR',
        0.5]`. Argument can be used for parameter sweeping or quick tests.
    """
    config = _C.clone()

    task_config_opts = []

    if config_paths:
        if isinstance(config_paths, str):
            if CONFIG_FILE_SEPARATOR in config_paths:
                config_paths = config_paths.split(CONFIG_FILE_SEPARATOR)
            else:
                config_paths = [config_paths]

        for config_path in config_paths:

            # Read config and extract TASK_CONFIG args to add as opts
            with open(config_path, "r") as f:
                cfg = config.load_cfg(f)

            cfg_clone = cfg.clone()
            for k in list(cfg.keys()):
                if not k.startswith("TASK_CONFIG"):
                    cfg_clone.pop(k)
                else:
                    cfg.pop(k)

            task_config_opts.append(cfg_clone)

            config.merge_from_other_cfg(cfg)

    config.TASK_CONFIG = get_task_config(config.BASE_TASK_CONFIG_PATH)

    for tsk_opts in task_config_opts:
        config.merge_from_other_cfg(tsk_opts)

    if opts:
        config.CMD_TRAILING_OPTS = opts
        config.merge_from_list(opts)

    config.freeze()
    return config

max_stars_repo_name: alexandrugavril/habitat-api

max_issues_repo_name: alexandrugavril/habitat-api

max_forks_repo_name: alexandrugavril/habitat-api


text:
import numpy as np
import h5py as h5
import hepfile

import matplotlib.pylab as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import colors

# Number of layers
nlayers = 3

# Number of events
nevents = 4

# Assign dimensions
z_size = [20, 24, 28]

# Max number of pixels in each layer
# Assume i pixels per ten degrees
maxpix = []
for i in range(0,nlayers):
    #maxpix.append(z_size[i]*(i+1)*36)
    maxpix.append(z_size[i]*(i+1)*18)

# Initialize the data dictionary for a single matrix
data = hepfile.initialize()

#Create groups
hepfile.create_group(data, "pixels", counter="npixels")

# Create datasets within the groups
hepfile.create_dataset(data, ["z","theta","r","value"], group="pixels")

# Create an event from the data container
event = hepfile.create_single_event(data)

for i in range(0,nevents):

    npixtot = 0
    ## Random selection of number of hit pixels in the layer
    for j in range(nlayers):
        npix = np.random.randint(0,maxpix[j])

        # For each point, randomly select a z and theta
        # Repeats are ok in this example
        for i in range(0,npix):
            event['pixels/z'].append(np.random.randint(0,z_size[j]) - z_size[j]/2)
            event['pixels/r'].append(j)
            event['pixels/theta'].append(np.random.random()*2*np.pi)
            event['pixels/value'].append(np.random.randint(1,70))

        # Keep a count of the total number of pixels
        npixtot += npix

    # Counters are the same here because there is always one data point per point
    # They could be different if there was more data collected for each point
    event['pixels/npixels'] = npixtot

    # Add the event to the matrix dictionary
    hepfile.pack(data,event)

    # Clear the event matrix for the next iteration
    hepfile.clear_event(event)

###############
# Write to file
###############
filename = 'cylindrical_example_output.hdf5'
hdfile = hepfile.write_to_file(filename,data,comp_type='gzip',comp_opts=9)

################
# Read from file
################
data,event = hepfile.load(filename)
nevents = data['nentries']

fig = plt.figure(figsize=(10,10))

for i in range(0,nevents):

    hepfile.unpack(event,data,n=i)

    npixels = event['pixels/npixels']
    #npixels = data['pixels/npixels'][i]
    z = event['pixels/z']
    r = event['pixels/r']
    theta = event['pixels/theta']
    d = event['pixels/value']

    x = (r+1)*np.cos(theta)
    y = (r+1)*np.sin(theta)

    #x1 = np.linspace(-1,1,100)
    #y1 = np.linspace(-1,1,100)
    #z1 = np.linspace(0,10,100)

    #zipped = zip(x,y,z,d)
    #sorted(zipped, key=lambda g: g[2])

    ax = fig.add_subplot(2, 2, i+1, projection='3d')
    ax.scatter(x,y,z,c=d, alpha=0.2)
    #ax.scatter(x,y,z,c=r, alpha=0.2)
    #ax.plot_surface(x1,y1,z1)
    #ax.plot_surface(x1,y1,z1)
    #ax.set_zlim(0, 10)

plt.show()

max_stars_repo_name: mattbellis/hepfile

max_issues_repo_name: mattbellis/hepfile

max_forks_repo_name: mattbellis/hepfile


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import _init_paths
import os
import os.path as osp
import cv2
import logging
import argparse
import motmetrics as mm
import numpy as np
import torch
from lib.tracker.tracker import JDETracker
# from lib.tracker.multitracker import JDETracker
# from lib.tracker.mot20tracker import JDETracker
# from lib.tracker.tracker_test import JDETracker
from lib.tracker.GSI import GSInterpolation
from lib.tracker.AFL import AFLink
from lib.tracker.byte_tracker import BYTETracker
from lib.tracking_utils import visualization as vis
from lib.tracking_utils.log import logger
from lib.tracking_utils.timer import Timer
from lib.tracking_utils.evaluation import Evaluator
from lib.datasets.dataset import jde as datasets

from lib.tracking_utils.utils import mkdir_if_missing
import lib.tracker.det_feat_record as det_feat_record
import lib.tracker.refined_track_vis as refined_track_vis
from opts import opts

def write_results(filename, results, data_type):
    if data_type == 'mot':
        save_format = '{frame},{id},{x1},{y1},{w},{h},1,-1,-1,-1\n'
    elif data_type == 'kitti':
        save_format = '{frame} {id} pedestrian 0 0 -10 {x1} {y1} {x2} {y2} -10 -10 -10 -1000 -1000 -1000 -10\n'
    else:
        raise ValueError(data_type)

    with open(filename, 'w') as f:
        for frame_id, tlwhs, track_ids in results:
            if data_type == 'kitti':
                frame_id -= 1
            for tlwh, track_id in zip(tlwhs, track_ids):
                if track_id < 0:
                    continue
                x1, y1, w, h = tlwh
                x2, y2 = x1 + w, y1 + h
                line = save_format.format(frame=frame_id, id=track_id, x1=x1, y1=y1, x2=x2, y2=y2, w=w, h=h)
                f.write(line)
    logger.info('save results to {}'.format(filename))

def eval_seq(opt, dataloader, data_type, result_filename,seq, save_dir=None, show_image=True, frame_rate=30):
    if save_dir:
        mkdir_if_missing(save_dir)
    tracker = JDETracker(opt, frame_rate=frame_rate)
    use_mat = {'MOT17-02-SDP': False, 'MOT17-04-SDP': False, "MOT17-05-SDP": True, "MOT17-09-SDP": False,
               "MOT17-10-SDP": True, "MOT17-11-SDP": True, "MOT17-13-SDP": True,
               'MOT17-01-SDP': False, 'MOT17-03-SDP': False, "MOT17-06-SDP": True, "MOT17-07-SDP": True,
               "MOT17-08-SDP": False, "MOT17-12-SDP": True, "MOT17-14-SDP": True,
               'MOT17-01-FRCNN': False, 'MOT17-03-FRCNN': False, "MOT17-06-FRCNN": True, "MOT17-07-FRCNN": True,
               "MOT17-08-FRCNN": False, "MOT17-12-FRCNN": True, "MOT17-14-FRCNN": True,
               'MOT17-01-DPM': False, 'MOT17-03-DPM': False, "MOT17-06-DPM": True, "MOT17-07-DPM": True,
               "MOT17-08-DPM": False, "MOT17-12-DPM": True, "MOT17-14-DPM": True,
               "MOT20-01": False, "MOT20-02": False, "MOT20-03": False,"MOT20-04": False,
               "MOT20-05": False, "MOT20-06": False, "MOT20-07": False, "MOT20-08": False,
               'MOT16-02': False, 'MOT16-04': False, "MOT16-05": True, "MOT16-09": False,
               "MOT16-10": True, "MOT16-11": True, "MOT16-13": True,
               'MOT16-01': False, 'MOT16-03': False, "MOT16-06": True, "MOT16-07": True,
               "MOT16-08": False, "MOT16-12": True, "MOT16-14": True,
               }
    tracker.use_mat = use_mat[seq]

    tracker.recorder =  det_feat_record.det_feat_recorder(seq,'/home/hust/yly/Dataset/MOT17/','get')

    timer = Timer()
    results = []
    len_all = len(dataloader)
    start_frame = len_all // 2
    frame_id = start_frame
    tracker.start_frame_id = start_frame
    for i, (path, img, img0) in enumerate(dataloader):
        if i < start_frame:
            continue
        if frame_id % 20 == 0:
            logger.info('Processing frame {} ({:.2f} fps)'.format(frame_id, 1. / max(1e-5, timer.average_time)))

        # run tracking
        timer.tic()
        blob = torch.from_numpy(img).cuda().unsqueeze(0)
        online_targets,insert_frame = tracker.update(blob, img0)
        online_tlwhs = []
        online_ids = []
        #online_scores = []
        for t in online_targets:
            tlwh = t[0]
            tid = t[1]
            vertical = tlwh[2] / tlwh[3] > 1.6
            if tlwh[2] * tlwh[3] > opt.min_box_area and not vertical:
                online_tlwhs.append(tlwh)
                online_ids.append(tid)
                #online_scores.append(t.score)
        timer.toc()
        # save results
        results.append((frame_id + 1, online_tlwhs, online_ids))
        for track_id in insert_frame.keys():
            frames = insert_frame[track_id]
            for frame in frames:
                tlwh,id = frame
                results[id-1][1].append(tlwh)
                results[id-1][2].append(track_id)
        #results.append((frame_id + 1, online_tlwhs, online_ids, online_scores))
        if show_image or save_dir is not None:
            online_im = vis.plot_tracking(img0, online_tlwhs, online_ids, frame_id=frame_id,
                                          fps=1. / timer.average_time)
        if show_image:
            cv2.imshow('online_im', online_im)
        if save_dir is not None:
            cv2.imwrite(os.path.join(save_dir, '{:05d}.jpg'.format(frame_id)), online_im)
        frame_id += 1
    # save results
    for result in results:
        for i,id in enumerate(result[2]):
            if id in tracker.id_map.keys():
                id = tracker.id_map[id]
            result[2][i] = id

    write_results(result_filename, results, data_type)

    afl = AFLink(result_filename,result_filename,0.4,(0,30))
    afl.feat = tracker.feat_record
    afl.link()
    GSInterpolation(path_in=result_filename,
                path_out=result_filename,
                interval=20,
                tau=10)

    #write_results_score(result_filename, results, data_type)
    return frame_id, timer.average_time, timer.calls

def main(opt, data_root='/data/MOT16/train', det_root=None, seqs=('MOT16-05',), exp_name='demo',
         save_images=False, save_videos=False, show_image=True):
    logger.setLevel(logging.INFO)
    result_root = os.path.join(data_root, '..', 'results', exp_name)
    mkdir_if_missing(result_root)
    data_type = 'mot'

    # run tracking
    accs = []
    n_frame = 0
    timer_avgs, timer_calls = [], []
    for seq in seqs:
        output_dir = os.path.join(data_root, '..', 'outputs', exp_name, seq) if save_images or save_videos else None
        logger.info('start seq: {}'.format(seq))
        dataloader = datasets.LoadImages(osp.join(data_root, seq, 'img1'), opt.img_size)
        result_filename = os.path.join(result_root, '{}.txt'.format(seq))
        meta_info = open(os.path.join(data_root, seq, 'seqinfo.ini')).read()
        frame_rate = int(meta_info[meta_info.find('frameRate') + 10:meta_info.find('\nseqLength')])
        nf, ta, tc= eval_seq(opt, dataloader, data_type, result_filename,seq,
                              save_dir=output_dir, show_image=show_image, frame_rate=frame_rate)
        n_frame += nf
        timer_avgs.append(ta)
        timer_calls.append(tc)

        # eval
        logger.info('Evaluate seq: {}'.format(seq))
        evaluator = Evaluator(data_root, seq, data_type)
        accs.append(evaluator.eval_file(result_filename))
        events = evaluator.acc.events
        events_csv = os.path.join(result_root,'{}.csv'.format(seq))
        events.to_csv(events_csv)
        print(events_csv)
        if save_videos:
            output_video_path = osp.join(output_dir, '{}.mp4'.format(seq))
            cmd_str = 'ffmpeg -f image2 -i {}/%05d.jpg -c:v copy {}'.format(output_dir, output_video_path)
            os.system(cmd_str)
    timer_avgs = np.asarray(timer_avgs)
    timer_calls = np.asarray(timer_calls)
    all_time = np.dot(timer_avgs, timer_calls)
    avg_time = all_time / np.sum(timer_calls)
    logger.info('Time elapsed: {:.2f} seconds, FPS: {:.2f}'.format(all_time, 1.0 / avg_time))

    # get summary
    metrics = mm.metrics.motchallenge_metrics
    mh = mm.metrics.create()
    summary = Evaluator.get_summary(accs, seqs, metrics)
    strsummary = mm.io.render_summary(
        summary,
        formatters=mh.formatters,
        namemap=mm.io.motchallenge_metric_names
    )
    print(strsummary)
    import json
    Evaluator.save_summary(summary, os.path.join(result_root, 'summary_{}.xlsx'.format(exp_name)))

if __name__ == '__main__':
    torch.cuda.set_device(0)
    opt = opts().init()
    print(opt)

    if  opt.val_mot16:
        seqs_str = '''MOT16-02
                      MOT16-04
                      MOT16-05
                      MOT16-09
                      MOT16-10
                      MOT16-11
                      MOT16-13'''
        data_root = os.path.join('/home/hust/yly/Dataset', 'MOT16/train')
    if opt.test_mot16:
        seqs_str = '''MOT16-01
                      MOT16-03
                      MOT16-06
                      MOT16-07
                      MOT16-08
                      MOT16-12
                      MOT16-14'''
        data_root = os.path.join('/home/hust/yly/Dataset', 'MOT16/test')
    if opt.test_mot15:
        seqs_str = '''ADL-Rundle-1
                      ADL-Rundle-3
                      AVG-TownCentre
                      ETH-Crossing
                      ETH-Jelmoli
                      ETH-Linthescher
                      KITTI-16
                      KITTI-19
                      PETS09-S2L2
                      TUD-Crossing
                      Venice-1'''
        data_root = os.path.join(opt.data_dir, 'MOT15/images/test')
    if opt.test_mot17:
        seqs_str = '''MOT17-01-DPM
                      MOT17-03-DPM
                      MOT17-06-DPM
                      MOT17-07-DPM
                      MOT17-08-DPM
                      MOT17-12-DPM
                      MOT17-14-DPM
                      MOT17-01-FRCNN
                      MOT17-03-FRCNN
                      MOT17-06-FRCNN
                      MOT17-07-FRCNN
                      MOT17-08-FRCNN
                      MOT17-12-FRCNN
                      MOT17-14-FRCNN
                      MOT17-01-SDP
                      MOT17-03-SDP
                      MOT17-06-SDP
                      MOT17-07-SDP
                      MOT17-08-SDP
                      MOT17-12-SDP
                      MOT17-14-SDP'''
        data_root = os.path.join('/home/hust/yly/Dataset', 'MOT17/test')
    if opt.val_mot17:
        seqs_str = '''MOT17-02-SDP
                      MOT17-04-SDP
                      MOT17-05-SDP
                      MOT17-09-SDP
                      MOT17-10-SDP
                      MOT17-11-SDP
                      MOT17-13-SDP'''
        # seqs_str = '''
        #               MOT17-05-SDP
        #               MOT17-10-SDP
        #               MOT17-11-SDP
        #               MOT17-13-SDP
        #             '''
        data_root = os.path.join('/home/hust/yly/Dataset', 'MOT17/train')
    if opt.val_mot15:
        seqs_str = '''Venice-2
                      KITTI-13
                      KITTI-17
                      ETH-Bahnhof
                      ETH-Sunnyday
                      PETS09-S2L1
                      TUD-Campus
                      TUD-Stadtmitte
                      ADL-Rundle-6
                      ADL-Rundle-8
                      ETH-Pedcross2
                      TUD-Stadtmitte'''
        data_root = os.path.join(opt.data_dir, 'MOT15/images/train')
    if opt.val_mot20:
        seqs_str = '''MOT20-01
                      MOT20-02
                      MOT20-03
                      MOT20-05
                      '''
        data_root = os.path.join('/home/hust/yly/Dataset', 'MOT20/train')
    if opt.test_mot20:
        seqs_str = '''MOT20-04
                      MOT20-06
                      MOT20-07
                      MOT20-08
                      '''
        data_root = os.path.join('/home/hust/yly/Dataset', 'MOT20/test')
    seqs = [seq.strip() for seq in seqs_str.split()]

    main(opt,
         data_root=data_root,
         seqs=seqs,
         exp_name='mot17_half_yolov5s',
         show_image=False,
         save_images=False,
         save_videos=False)

max_stars_repo_name: Alan-yly/FairMOT

max_issues_repo_name: Alan-yly/FairMOT

max_forks_repo_name: Alan-yly/FairMOT


text:
from AnimatedWidget import AnimatedWidget
from PyQt5.QtWidgets import QOpenGLWidget

import OpenGL.GL as gl
import OpenGL.GLU as glu

from PyQt5.Qt import Qt
from PyQt5.QtCore import QPoint

import math as mt
from PIL import Image
import os

import numpy as np

from cython_modules.color_policy import multi_iteration_normalize
from cython_modules.cython_parse import getLayerOutline, genCubes
from ColorPolicy import ColorPolicy

class AbstractGLContext(QOpenGLWidget, AnimatedWidget):
    def __init__(self, parent=None):
        super(AbstractGLContext, self).__init__(parent)

        self.lastPos = QPoint()
        self.setFocusPolicy(Qt.StrongFocus)  # needed if keyboard to be active

        self.rotation = [0, 0, 0]  # xyz degrees in xyz axis
        self.position = [0, 0, -50]  # xyz initial
        self.drawing_function = None
        self.function_select = 'fast'
        self.background = [0.5, 0.5, 0.5]
        self.record = False
        self.spacer = 0.2
        self.steps = 1

    def shareData(self, **kwargs):
        super().shareData(**kwargs)
        super().handleOptionalData()
        self.receivedOptions()
        self.i = self.current_state
        print(self.iterations)

    @classmethod
    def normalize_specification(cls, color_vectors, vbo=False):
        """
        normalization procedure
        """
        multi_iteration_normalize(color_vectors)
        background = np.array([0.5, 0.5, 0.5])
        if cls.__name__ == 'CubicGLContext':
            # replace black with background colors
            # NOTE: This is dangerous since dot product can be zero
            color_vectors[~color_vectors.any(axis=2)] = background
        elif cls.__name__ == 'VectorGLContext':
            if vbo:
                # replace black with background colors
                # NOTE: This is dangerous since dot product can be zero
                color_vectors[~color_vectors.any(axis=2)] = background

    def prerendering_calculation(self):
        """
        Some calculations that take place before object gets rendered
        """
        # get vector outline
        self.vectors_list = getLayerOutline(self.file_header)
        self.auto_center()
        # adjust spacing
        self.spacer = self.spacer*self.scale

        xc = int(self.file_header['xnodes'])
        yc = int(self.file_header['ynodes'])
        zc = int(self.file_header['znodes'])
        # change drawing function
        self.color_vectors, self.vectors_list, decimate = \
                    ColorPolicy.standard_procedure(self.vectors_list,
                                                   self.color_vectors,
                                                   self.iterations,
                                                   self.averaging,
                                                   xc, yc, zc,
                                                   self.layer,
                                                   self.vector_set,
                                                   self.decimate,
                                                   disableDot=self.disableDot)
        if decimate is not None:
            # this is arbitrary
            self.spacer *= decimate*3

    def handleOptionalData(self):
        # must handle iterations since these are optional
        try:
            getattr(self, 'iterations')
        except NameError:
            self.iterations = 1

    def auto_center(self):
        """
        auto centers the structure in widget screen
        """
        x_fix = (self.file_header['xnodes'] * self.file_header['xbase'] * 1e9) / 2
        y_fix = (self.file_header['ynodes'] * self.file_header['ybase'] * 1e9) / 2
        z_fix = (self.file_header['znodes'] * self.file_header['zbase'] * 1e9) / 2
        for vec in self.vectors_list:
            vec[0] -= x_fix
            vec[1] -= y_fix
            vec[2] -= z_fix

    def screenshot_manager(self):
        """
        saves the screenshot to the folder specified in screenshot_dir
        """
        # fetch dimensions for highest resolution
        _, _, width, height = gl.glGetIntegerv(gl.GL_VIEWPORT)
        color = gl.glReadPixels(0, 0,
                               width, height,
                               gl.GL_RGB, gl.GL_UNSIGNED_BYTE)
        image = Image.frombytes(mode='RGB', size=(width, height),
                                                    data=color)
        image = image.transpose(Image.FLIP_TOP_BOTTOM)
        try:
            image.save(os.path.join(self.screenshot_dir,
                                        str(self.i).zfill(4) + ".png"))
        except FileNotFoundError:
            # if basic dir not found, create it and save there
            os.mkdir(self.screenshot_dir)
            image.save(os.path.join(self.screenshot_dir,
                                        str(self.i).zfill(4) + ".png"))

    def initial_transformation(self):
        """
        resets the view to the initial one
        :return:
        """
        self.rotation = [0, 0, 0]  # xyz degrees in xyz axis
        self.position = [0, 0 , -50]  # xyz initial

    def transformate(self):
        """
        applies rotation and transformation
        """
        gl.glRotatef(self.rotation[0], 0, 1, 0)  # rotate around y axis
        gl.glRotatef(self.rotation[1], 1, 0, 0)  # rotate around x axis
        gl.glRotatef(self.rotation[2], 0, 0, 1)  # rotate around z axis
        gl.glTranslatef(self.position[0], self.position[1], self.position[2])

    def initializeGL(self):
        """
        Initializes openGL context and scenery
        """

        gl.glClearColor(*self.background, 1)
        gl.glEnable(gl.GL_DEPTH_TEST)

    def resizeGL(self, w, h):
        """
        Resize function, applied when windows change size
        @param w width - use self.width() function
        @param h height - use self.height() function
        """
        gl.glViewport(0, 0, w, h)
        # using Projection mode
        gl.glMatrixMode(gl.GL_PROJECTION)
        gl.glLoadIdentity()
        aspectRatio = w / h
        glu.gluPerspective(85 * self.steps, aspectRatio, 1, 1000)
        gl.glMatrixMode(gl.GL_MODELVIEW)
        gl.glLoadIdentity()

    def paintGL(self):
        """
        Clears the buffer and redraws the scene
        """
        gl.glClear(gl.GL_COLOR_BUFFER_BIT | gl.GL_DEPTH_BUFFER_BIT)
        # Push Matrix onto stack
        gl.glPushMatrix()
        self.transformate()
        self.drawing_function()
        # Pop Matrix off stack
        gl.glPopMatrix()
        self.update()

    def set_i(self, value, trigger=False):
        if trigger:
            self.i += 1
        else:
            self.i = value
        self.i %= self.iterations

    def keyPressEvent(self, event):
        """
        Key mapping:
        R - reset central view
        I - zoom in
        O - zoom out
        Y - start/stop recording
        S - take a screenshot
        """
        key = event.key()
        if key == Qt.Key_R:
            self.initial_transformation()
            self.update()

        elif key == Qt.Key_I:
            self.zoomIn()

        elif key == Qt.Key_O:
            self.zoomOut()

        if key == Qt.Key_Y:
            self.record = not self.record

        if key == Qt.Key_S:
            self.screenshot_manager()

    def zoomIn(self):
        self.steps = 1
        # SMART SCROLL BETA
        self.position[0] -= mt.sin(self.rotation[0] * mt.pi / 180) * \
                            mt.cos(self.rotation[1] * mt.pi / 180) * self.steps
        self.position[1] += mt.cos(self.rotation[0] * mt.pi / 180) * \
                            mt.sin(self.rotation[1] * mt.pi / 180) * self.steps
        self.position[2] += mt.cos(self.rotation[0] * mt.pi / 180) * \
                            mt.cos(self.rotation[1] * mt.pi / 180) * self.steps

    def zoomOut(self):
        self.steps = -1

        self.position[0] -= mt.sin(self.rotation[0] * mt.pi / 180) * \
                            mt.cos(self.rotation[1] * mt.pi / 180) * self.steps
        self.position[1] += mt.cos(self.rotation[0] * mt.pi / 180) * \
                            mt.sin(self.rotation[1] * mt.pi / 180) * self.steps
        self.position[2] += mt.cos(self.rotation[0] * mt.pi / 180) * \
                            mt.cos(self.rotation[1] * mt.pi / 180) * self.steps

    def wheelEvent(self, event):
        """
        Handles basic mouse scroll
        """
        degs = event.angleDelta().y() / 8
        self.steps = degs / 15
        # SMART SCROLL BETA
        self.position[0] -= mt.sin(self.rotation[0] * mt.pi / 180) * \
                            mt.cos(self.rotation[1] * mt.pi / 180) * self.steps
        self.position[1] += mt.cos(self.rotation[0] * mt.pi / 180) * \
                            mt.sin(self.rotation[1] * mt.pi / 180) * self.steps
        self.position[2] += mt.cos(self.rotation[0] * mt.pi / 180) * \
                            mt.cos(self.rotation[1] * mt.pi / 180) * self.steps

        self.update()

    def mouseMoveEvent(self, event):
        """
        Handles basic mouse press
        SHOULD BE MOUSE DRAG RATHER A MOUSE EVENT
        """
        dx = event.x() - self.lastPos.x()
        dy = event.y() - self.lastPos.y()
        self.lastPos = event.pos()
        if event.buttons() & Qt.LeftButton:
            rotation_speed = 0.5
            self.rotation[0] += dx * rotation_speed
            xpos = self.position[0] * mt.cos(dx * rotation_speed * mt.pi / 180) \
                   - self.position[2] * mt.sin(dx * rotation_speed * mt.pi / 180)
            zpos = self.position[0] * mt.sin(dx * rotation_speed * mt.pi / 180) \
                   + self.position[2] * mt.cos(dx * rotation_speed * mt.pi / 180)

            self.position[0] = xpos
            self.position[2] = zpos

        elif event.buttons() & Qt.RightButton:
            if abs(self.rotation[0])%360 < 90:
                self.position[0] += dx * 0.2
            else:
                self.position[0] -= dx * 0.2

            self.position[1] -= dy * 0.2
        self.update()

    @staticmethod
    def get_open_gl_info():
        info = """
            Vendor: {0}
            Renderer: {1}
            OpenGL Version: {2}
            Shader Version: {3}
            """.format(
            gl.glGetString(gl.GL_VENDOR),
            gl.glGetString(gl.GL_RENDERER),
            gl.glGetString(gl.GL_VERSION),
            gl.glGetString(gl.GL_SHADING_LANGUAGE_VERSION))
        return info


text:
def calculate_observation_errors(obs, analysis, firstguess, dim='time', n=730, min_periods=20, center=True, **kwargs):
    """ Calculate Observation Error statistics from first guess and analysis departures

    Args:
        obs (xarray.DataArray): Observations
        analysis (xarray.DataArray): Analysis
        firstguess (DataArray): First Guess
        dim (str): datetime dimension
        n (int): number of average (730 = 2 Years)
        min_periods (int): minimum required values
        center (bool): average alignment
        **kwargs:

    Returns:

    """
    import xarray
    #
    # Estimate obs errors from fg, an dep.
    #
    an_dep = obs - analysis
    an_dep.name = 'an_dep'
    fg_dep = obs - firstguess
    fg_dep.name = 'fg_dep'
    tmp = xarray.merge([an_dep, fg_dep])
    tmp = tmp.rolling(min_periods=min_periods, center=center, **{dim: n}).construct('pieces')
    return obserror(tmp, a='an_dep', b='fg_dep', dim='pieces', attrs=obs.attrs)

def obserror(data, a='t_an_dep', b='t_fg_dep', dim='time', attrs=None, **kwargs):
    import xarray as xr
    import numpy as np
    #
    #
    #
    if not isinstance(data, xr.Dataset):
        raise ValueError()
    #
    # figure out dimensions
    #
    jdims = list(data[a].dims)
    jdims.remove(dim)
    axis = list(data[a].dims).index(dim)
    if attrs is None:
        attrs = data[a].attrs.copy()
    #
    # Calculate product of DataArray's
    #
    tmp = np.sqrt(xr.apply_ufunc(np.nanmean, data[a]*data[b],
                                 input_core_dims=[data[a].dims],
                                 output_core_dims=[jdims],
                                 kwargs={'axis': axis}
                                 )
                  )
    #
    # Metadata
    #
    tmp.attrs.update(attrs)
    tmp.attrs.update({'standard_name': 'obs_error'})
    return tmp

def errorous(x, dim='time', n=730, min_periods=20, center=True, sigma=6, **kwargs):
    """ removes errorous values outside 4 sigma

    Args:
        x:
        dim:
        n:
        min_periods:
        center:

    Returns:

    """
    import xarray as xr
    import numpy as np
    if not isinstance(x, xr.DataArray):
        raise ValueError()

    m = x.rolling(min_periods=min_periods, center=center, **{dim: n}).mean().ffill(dim).bfill(dim)
    s = x.rolling(min_periods=min_periods, center=center, **{dim: n}).std().ffill(dim).bfill(dim)
    # print(m,s)
    x = x.where((~s.isnull()) & (~m.isnull()) & (x < (m + sigma*s)) & (x > (m - sigma*s)), np.nan)
    return x

def obserror_dpd(data, a='t_an_dep', b='t_fg_dep', c='rh_an_dep', d='rh_fg_dep', dim='time', attrs=None, **kwargs):
    import xarray as xr
    import numpy as np
    from .convert import to_dpd
    #
    #
    #
    if not isinstance(data, xr.Dataset):
        raise ValueError()
    #
    # figure out dimensions
    #
    jdims = list(data[a].dims)
    jdims.remove(dim)
    axis = list(data[a].dims).index(dim)
    if attrs is None:
        attrs = data[a].attrs.copy()
    #
    # Calculate DPD dep
    # sqrt( t_err^2 + td_err^2)

    # Error DPD ?
    # dpd = t - td(esat(T)*rh)
    # error prop for esat
    # e_err = 2.57775e6*sqrt((exp((35.004*t-9561.69)/(t-32.19))*t_1**2 )/ (t-32.19)**4)
    # sqrt( t_err^2 +   e_err^2 + (rh_err / abs(rh))^2 + e_err^2 * 216.218^2)
    #
    # Calculate product of DataArray's
    #
    tmp = np.sqrt(xr.apply_ufunc(np.nanmean, data[a]*data[b],
                                 input_core_dims=[data[a].dims],
                                 output_core_dims=[jdims],
                                 kwargs={'axis': axis}
                                 )
                  )
    #
    # Metadata
    #
    tmp.attrs.update(attrs)
    tmp.attrs.update({'standard_name': 'obs_error'})
    return tmp

max_stars_repo_name: MBlaschek/rasotools

max_issues_repo_name: MBlaschek/rasotools

max_forks_repo_name: MBlaschek/rasotools


text:
import csv
import numpy as np
from preprocess_dataset.audio.processors import get_processor
from tensorflow.contrib import predictor

PROCESSOR = 'mfcc'

def load_genres(genres_metadata_fn):
    with open(genres_metadata_fn, 'r') as f:
        reader = csv.DictReader(f)
        items = [
            (int(item['genre_id']), item['title'])
            for item in reader
        ]
        return items

def extract_genres(vector, genres):
    vector = np.array(vector).tolist()[0]
    result = [
        (genre_title, vector[genre_id])
        for genre_id, genre_title in genres
        if len(vector) > genre_id
    ]
    result = sorted(result, key=lambda x: x[1])
    return result

def demonstrate(music_file, model_path, genres_metadata_fn):
    processor = get_processor(PROCESSOR)
    genres = load_genres(genres_metadata_fn)
    features = processor(music_file)
    features = np.reshape(features, (1,) + features.shape + (1,))

    predict_fn = predictor.from_saved_model(model_path)
    # estimator = tf.estimator.Estimator(
    #     model_fn=model_fn,
    #     config=tf.estimator.RunConfig(
    #         save_checkpoints_steps=10,
    #         model_dir=model_path,
    #     )
    # )
    #
    # predictions = estimator.predict(
    #     input_fn=tf.estimator.inputs.numpy_input_fn(
    #         x={'feature': features},
    #         shuffle=False
    #     )
    # )

    predictions = predict_fn({'feature': features})
    result = extract_genres(predictions['genres_top'], genres)
    for genre_title, prob in result:
        print(genre_title, ':', prob)


text:
from numba import njit

@njit
def if_replacement(x1, x2):
    if x1 > x2:
        x1 = 1
    return x1, 1

max_stars_repo_name: amipy/numerous

max_issues_repo_name: amipy/numerous

max_forks_repo_name: amipy/numerous

reason: from numba