text:
# run: tensorboard --logdir=/tmp/keras_drm_vae

# export results:
# python keras_deep_vae.py --train 0 --dset /media/tim/Tim\ 1500\ GB/GriesheimDriveAround/dset_runway_final.h5 --load /media/tim/Tim\ 1500\ GB/2017_ivs_drm/models/vae_weights_complete.h5 --save_pred /media/tim/Tim\ 1500\ GB/2017_ivs_drm/models/pred_vae.h5

# NOTE: this sys stuff is there to prevent the "Using Tensorflow backend" from being constantly displayed
# import sys
# stderr = sys.stderr
# sys.stderr = open('/dev/null', 'w')

from keras.callbacks import TensorBoard, ModelCheckpoint, EarlyStopping
from keras.models import Model
from keras.regularizers import l2, l1
from keras.layers import Input, Dense, Lambda, Flatten, Reshape, Merge
from keras.layers.convolutional import Convolution2D, Deconvolution2D, UpSampling2D
from keras.layers.normalization import BatchNormalization
from keras import backend as K

# sys.stderr = stderr

import time
import argparse
import tensorflow as tf
tf.python.control_flow_ops = tf # bugfix see https://github.com/fchollet/keras/issues/3857
import numpy as np
import random
import h5py

##########################
# Input Parser

parser = argparse.ArgumentParser()
parser.add_argument('--batch_size', type=int, default=16)
parser.add_argument('--nb_epoch',   type=int, default=100)
parser.add_argument('--l2',         type=float, default=0.0001) # weight l2 regularization
parser.add_argument('--verbosity',  type=int, default=1)
parser.add_argument('--train',      type=int, default=1)
parser.add_argument('--act',        type=str, default='relu') # activation
parser.add_argument('--opt',        type=str, default='adadelta') # optimizer
parser.add_argument('--tbdir',      type=str, default='/tmp/keras_drm_vae/')
parser.add_argument('--tblog',      type=str, default='')
parser.add_argument('--dset',       type=str, default='dset_runway.h5')
parser.add_argument('--save',       type=str, default='') # log file
parser.add_argument('--load',       type=str, default='') # optionally specify filepath for initial net to load from h5 file
parser.add_argument('--save_pred',  type=str, default='vae_predictions.h5')

parser.add_argument('--logP_var', type=float, default=0.1)
parser.add_argument('--epsilon_std', type=float, default=0.01)
parser.add_argument('--dim_z', type=int, default=16)
args = parser.parse_args()

if len(args.tblog) == 0:
	args.tblog = args.tbdir + time.strftime("run%Y%m%d-%H%M%S")

if args.verbosity > 0:
	print args

##########################
# Load Data

data = h5py.File(args.dset, 'r')
O_test = data['data/O_test'][:]
T_test = data['data/T_test'][:]
Y_test = data['data/Y_test'][:]
O_train = data['data/O_train'][:]
T_train = data['data/T_train'][:]
Y_train = data['data/Y_train'][:]
data.close()

O_train = np.swapaxes(np.swapaxes(O_train, 0, 3), 1, 2)
T_train = np.swapaxes(np.swapaxes(T_train, 0, 3), 1, 2)
Y_train = np.swapaxes(np.swapaxes(Y_train, 0, 3), 1, 2)
O_test = np.swapaxes(np.swapaxes(O_test, 0, 3), 1, 2)
T_test = np.swapaxes(np.swapaxes(T_test, 0, 3), 1, 2)
Y_test = np.swapaxes(np.swapaxes(Y_test, 0, 3), 1, 2)

n_train = O_train.shape[0] - (O_train.shape[0]%args.batch_size)
n_test = O_test.shape[0] - (O_test.shape[0]%args.batch_size)

O_train = O_train[0:n_train,:,:,:]
T_train = T_train[0:n_train,:,:,:]
Y_train = Y_train[0:n_train,:,:,:]
O_test = O_test[0:n_test,:,:,:]
T_test = T_test[0:n_test,:,:,:]
Y_test = Y_test[0:n_test,:,:,:]

if args.verbosity > 0:
	print "\nSHAPES"
	print "\tO_train shape: ", O_train.shape
	print "\tT_train shape: ", T_train.shape
	print "\tY_train shape: ", Y_train.shape
	print "\tO_test shape:  ", O_test.shape
	print "\tT_test shape:  ", T_test.shape
	print "\tY_test shape:  ", Y_test.shape
	print "\n"

# #########################
# Construct (& Load) Model

##########################
# Object Head
O_in = Input(shape=(O_train.shape[1], O_train.shape[2], O_train.shape[3]), dtype='float32', name='O')                 # [None, 2, 1,  9]
O_conv1 = Convolution2D(16, 1, 1, name='o_conv1', border_mode='same', activation=args.act, W_regularizer=l2(args.l2)) # [None, 2, 1, 16]
O_conv2 = Convolution2D(16, 1, 1, name='o_conv2', border_mode='same', activation=args.act, W_regularizer=l2(args.l2)) # [None, 2, 1, 16]
O_out   = Flatten() # [None, 32]

o_conv1 = O_conv1(O_in)
o_conv2 = O_conv2(o_conv1)
o_out = O_out(o_conv2)

##########################
# Terrain Head
#T_in = Input(shape=(T_train.shape[1],), dtype='float32', name='T') # [None, 1]

T_in = Input(shape=(T_train.shape[1],T_train.shape[2], T_train.shape[3]), dtype='float32', name='T') # [None, 1]
T_conv1 = Convolution2D(2, 3, 3, name='t_conv1', border_mode='same', subsample=(2,2), activation=args.act, W_regularizer=l2(args.l2)) # [None, 64, 64, 1]
T_conv2 = Convolution2D(4, 3, 3, name='t_conv2', border_mode='same', subsample=(2,2), activation=args.act, W_regularizer=l2(args.l2)) # [None, 32, 32, 2]
T_conv3 = Convolution2D(8, 3, 3, name='t_conv3', border_mode='same', subsample=(2,2), activation=args.act, W_regularizer=l2(args.l2)) # [None, 16, 16, 4]
T_conv4 = Convolution2D(16, 3, 3, name='t_conv4', border_mode='same', subsample=(2,2), activation=args.act, W_regularizer=l2(args.l2)) # [None, 8, 8, 8]
T_conv5 = Convolution2D(32, 3, 3, name='t_conv5', border_mode='same', subsample=(2,2), activation=args.act, W_regularizer=l2(args.l2)) # [None, 4, 4, 16]
T_conv6 = Convolution2D(64, 3, 3, name='t_conv6', border_mode='same', activation=args.act, W_regularizer=l2(args.l2)) # [None, 2, 2, 32]
T_out   = Flatten() # [None, 32]

t_conv1 = T_conv1(T_in)
t_conv2 = T_conv2(t_conv1)
t_conv3 = T_conv3(t_conv2)
t_conv4 = T_conv4(t_conv3)
t_conv5 = T_conv5(t_conv4)
t_conv6 = T_conv6(t_conv5)
t_out = T_out(t_conv6)
##########################
# X Head
#X_in = Merge(mode='concat', concat_axis=1)([o_out, T_in]) # [None, 33]
X_in = Merge(mode='concat', concat_axis=1)([o_out, t_out]) # [None, 33]
X_dense1 = Dense(128, name='x_dense1', activation=args.act, W_regularizer=l2(args.l2))
X_bnorm1 = BatchNormalization(mode=2)
X_dense2 = Dense(128, name='x_dense2', activation=args.act, W_regularizer=l2(args.l2))
X_out    = BatchNormalization(mode=2)

x_dense1 = X_dense1(X_in)
x_bnorm1 = X_bnorm1(x_dense1)
x_dense2 = X_dense2(x_bnorm1)
x_out = X_out(x_dense2)

##########################
# Y Head
Y_in = Input(shape=(Y_train.shape[1], Y_train.shape[2], Y_train.shape[3]), dtype='float32', name='Y_in') # [None, 64, 64, 1]
Y_conv1_1 = Convolution2D( 8, 3, 3, name='y_conv1_1', subsample=(2,2), border_mode='same', activation=args.act, W_regularizer=l2(args.l2)) # [None, 32, 32,  8]
Y_conv1_2 = Convolution2D( 8, 3, 3, name='y_conv1_2', subsample=(1,1), border_mode='same', activation=args.act, W_regularizer=l2(args.l2)) # [None, 32, 32,  8]
Y_conv2_1 = Convolution2D(16, 3, 3, name='y_conv2_1', subsample=(2,2), border_mode='same', activation=args.act, W_regularizer=l2(args.l2)) # [None, 16, 16, 16]
Y_conv2_2 = Convolution2D(16, 3, 3, name='y_conv2_2', subsample=(1,1), border_mode='same', activation=args.act, W_regularizer=l2(args.l2)) # [None, 16, 16, 16]
Y_conv3_1 = Convolution2D(32, 3, 3, name='y_conv3_1', subsample=(2,2), border_mode='same', activation=args.act, W_regularizer=l2(args.l2)) # [None,  8,  8, 32]
Y_conv3_2 = Convolution2D(32, 3, 3, name='y_conv3_2', subsample=(1,1), border_mode='same', activation=args.act, W_regularizer=l2(args.l2)) # [None,  8,  8, 32]
Y_conv4_1 = Convolution2D(32, 3, 3, name='y_conv4_1', subsample=(2,2), border_mode='same', activation=args.act, W_regularizer=l2(args.l2)) # [None,  4,  4, 32]
Y_conv4_2 = Convolution2D( 4, 3, 3, name='y_conv4_2', subsample=(1,1), border_mode='same', activation=args.act, W_regularizer=l2(args.l2)) # [None,  4,  4, 4]
Y_out     = Flatten() # [64]

y_conv1_1 = Y_conv1_1(Y_in)
y_conv1_2 = Y_conv1_2(y_conv1_1)
y_conv2_1 = Y_conv2_1(y_conv1_2)
y_conv2_2 = Y_conv2_2(y_conv2_1)
y_conv3_1 = Y_conv3_1(y_conv2_2)
y_conv3_2 = Y_conv3_2(y_conv3_1)
y_conv4_1 = Y_conv4_1(y_conv3_2)
y_conv4_2 = Y_conv4_2(y_conv3_2)
y_out = Y_out(y_conv4_2)

##########################
# Encoder
E_in = Merge(mode='concat', concat_axis=1)
E_dense1 = Dense(32, name='e_dense1', activation='linear', W_regularizer=l2(args.l2))
E_bnorm1 = BatchNormalization(mode=2)
E_dense2 = Dense(32, name='e_dense2', activation='linear', W_regularizer=l2(args.l2))
E_bnorm2 = BatchNormalization(mode=2)
Z_mean = Dense(args.dim_z, name='dense_z_mean', activation='linear', W_regularizer=l2(args.l2))
Z_logvar = Dense(args.dim_z, name='dense_z_logvar', activation='linear', W_regularizer=l2(args.l2))

e_in = E_in([x_out, y_out])
e_dense1 = E_dense1(e_in)
e_bnorm1 = E_bnorm1(e_dense1)
e_dense2 = E_dense2(e_bnorm1)
e_bnorm2 = E_bnorm2(e_dense2)
z_mean = Z_mean(e_bnorm2)
z_logvar = Z_logvar(e_bnorm2)

def sampling(sampling_args):
    z_mean, z_logvar = sampling_args
    epsilon = K.random_normal(shape=(args.batch_size, args.dim_z),
                              mean=0.0, std=args.epsilon_std)
    return z_mean + K.exp(z_logvar) * epsilon

Z = Lambda(sampling, output_shape=(args.dim_z,))
z = Z([z_mean, z_logvar])

# Decoder

D_in = Merge(mode='concat', concat_axis=1)
D_dense1  = Dense(128, name='dense3', activation=args.act, W_regularizer=l2(args.l2))
D_bnorm1  = BatchNormalization(mode=2)
D_dense2  = Dense(256, name='dense4', activation=args.act, W_regularizer=l2(args.l2))
D_bnorm2  = BatchNormalization(mode=2)
D_reshape = Reshape((8, 8, 4)) # [None, 8, 8, 2]
Deco1_1   = Deconvolution2D(8, 3, 3, name='deco1_1', output_shape=(args.batch_size, 16, 16, 8), activation=args.act, border_mode='same', subsample=(2, 2), W_regularizer=l2(args.l2)) # [None, 16, 16, 8]
Deco1_2   = Deconvolution2D(8, 3, 3, name='deco1_2', output_shape=(args.batch_size, 16, 16, 8), activation=args.act, border_mode='same', subsample=(1, 1), W_regularizer=l2(args.l2)) # [None, 16, 16, 8]
Deco2_1   = Deconvolution2D(8, 3, 3, name='deco2_1', output_shape=(args.batch_size, 32, 32, 8), activation=args.act, border_mode='same', subsample=(2, 2), W_regularizer=l2(args.l2)) # [None, 32, 32, 8]
Deco2_2   = Deconvolution2D(8, 3, 3, name='deco2_2', output_shape=(args.batch_size, 32, 32, 8), activation=args.act, border_mode='same', subsample=(1, 1), W_regularizer=l2(args.l2)) # [None, 32, 32, 8]
Deco3_1   = Deconvolution2D(4, 3, 3, name='deco3_1', output_shape=(args.batch_size, 64, 64, 4), activation=args.act, border_mode='same', subsample=(2, 2), W_regularizer=l2(args.l2)) # [None, 64, 64, 4]
Y_out     = Deconvolution2D(1, 3, 3, name='deco3_2',       output_shape=(args.batch_size, 64, 64, 1), activation='linear', border_mode='same', subsample=(1, 1), W_regularizer=l2(args.l2)) # [None, 64, 64, 1]

d_in = D_in([x_out, z])
d_dense1 = D_dense1(d_in)
d_bnorm1 = D_bnorm1(d_dense1)
d_dense2 = D_dense2(d_bnorm1)
d_bnorm2 = D_bnorm2(d_dense2)
d_reshape = D_reshape(d_bnorm2)
deco1_1 = Deco1_1(d_reshape)
deco1_2 = Deco1_2(deco1_1)
deco2_1 = Deco2_1(deco1_2)
deco2_2 = Deco2_2(deco2_1)
deco3_1 = Deco3_1(deco2_2)
y_out = Y_out(deco3_1)

def vae_loss(Y_true, Y_pred):
	y_true = K.flatten(Y_true)
	y_mean = K.flatten(Y_pred)

	logP_loss = 0.5 * K.mean(K.log(args.logP_var) + K.square(y_mean - y_true)/args.logP_var, axis=-1) # negative logl
	kl_loss =  -0.5 * K.mean(1 + z_logvar - K.square(z_mean) - K.exp(z_logvar), axis=-1) # penalizes deviation from normal distr
	return logP_loss + kl_loss

model = Model(input=[O_in, T_in, Y_in], output=y_out)
model.compile(optimizer=args.opt, loss=vae_loss)

if len(args.load) > 0:
	if args.verbosity > 0:
		print 'loading model weights from ', args.load
	model.load_weights(args.load)

if args.verbosity > 0:
	model.summary()

##########################
# Training

if args.train != 0:
	model.fit([O_train, T_train, Y_train], Y_train,
          nb_epoch=args.nb_epoch,
          batch_size=args.batch_size,
          shuffle=True,
          verbose=args.verbosity,
          validation_data=([O_test, T_test, Y_test], Y_test),
          callbacks=[TensorBoard(log_dir=args.tblog),
                     ModelCheckpoint('training_vae.h5', save_best_only=True, save_weights_only=True),
                     EarlyStopping(monitor='val_loss',patience=args.nb_epoch/4)],
          )

##########################
# Save Model

if len(args.save) > 0:
	if args.verbosity > 0:
		print 'saving model weights to ', args.save
	model.save_weights(args.save)

if len(args.save_pred) > 0:

	print("SAVING PREDICTIONS")

	pred_o_conv1 = O_conv1(O_in)
	pred_o_conv2 = O_conv2(pred_o_conv1)
	pred_o_out = O_out(pred_o_conv2)

	pred_t_conv1 = T_conv1(T_in)
	pred_t_conv2 = T_conv2(pred_t_conv1)
	pred_t_conv3 = T_conv3(pred_t_conv2)
	pred_t_conv4 = T_conv4(pred_t_conv3)
	pred_t_conv5 = T_conv5(pred_t_conv4)
	pred_t_conv6 = T_conv6(pred_t_conv5)
	pred_t_out = T_out(t_conv6)

	pred_x_in = Merge(mode='concat', concat_axis=1)([pred_o_out, pred_t_out])
	pred_x_dense1 = X_dense1(pred_x_in)
	pred_x_bnorm1 = X_bnorm1(pred_x_dense1)
	pred_x_dense2 = X_dense2(pred_x_bnorm1)
	pred_x_out = X_out(pred_x_dense2)

	Z_test = np.random.randn(O_test.shape[0],args.dim_z)*args.epsilon_std
	Z_train = np.random.randn(O_train.shape[0],args.dim_z)*args.epsilon_std

	Z_in = Input(shape=(args.dim_z,), dtype='float32', name='pred_z') # drawn from N(O,I)
	pred_d_in = Merge(mode='concat', concat_axis=1)([pred_x_out, Z_in])
	pred_d_dense1 = D_dense1(pred_d_in)
	pred_d_bnorm1 = D_bnorm1(pred_d_dense1)
	pred_d_dense2 = D_dense2(pred_d_bnorm1)
	pred_d_bnorm2 = D_bnorm2(pred_d_dense2)
	pred_d_reshape = D_reshape(pred_d_bnorm2)
	pred_deco1_1 = Deco1_1(pred_d_reshape)
	pred_deco1_2 = Deco1_2(pred_deco1_1)
	pred_deco2_1 = Deco2_1(pred_deco1_2)
	pred_deco2_2 = Deco2_2(pred_deco2_1)
	pred_deco3_1 = Deco3_1(pred_deco2_2)
	pred_y_out = Y_out(pred_deco3_1)

	model = Model(input=[O_in, T_in, Z_in], output=pred_y_out)
	model.compile(optimizer=args.opt, loss=vae_loss)

	if len(args.load) > 0:
		if args.verbosity > 0:
			print 'loading model weights from ', args.load
		model.load_weights(args.load, by_name=True)

	#data = h5py.File(args.dset, 'r')
	#Z_pred = data['data/Z_pred'][:]
	#data.close()
	#Z_pred = np.swapaxes(Z_pred, 0, 1)
	#Z_pred = Z_pred[0:n_test,:]

	#if args.verbosity > 0:
	#		print "\tZ_pred shape:  ", Z_pred.shape

	f = h5py.File(args.save_pred, 'w')
	f.create_dataset("predict/Y_pred", data=model.predict([O_test, T_test, Z_test], batch_size=args.batch_size, verbose=args.verbosity))
	f.create_dataset("predict/Y_pred_train", data=model.predict([O_train, T_train, Z_train], batch_size=args.batch_size, verbose=args.verbosity))
	f.close()


text:
import numpy as np
import healpy as hp

from .. import InterpolatingComponent
from .. import units as u

def test_interpolating(tmp_path):

    nside = 4
    shape = (3, hp.nside2npix(nside))
    hp.write_map(tmp_path / "10.fits", np.zeros(shape, dtype=np.float32))
    hp.write_map(tmp_path / "20.fits", np.ones(shape, dtype=np.float32))

    interp = InterpolatingComponent(
        tmp_path, "uK_RJ", nside, interpolation_kind="linear"
    )

    interpolated_map = interp.get_emission(15 * u.GHz)
    np.testing.assert_allclose(0.5 * np.ones(shape) * u.uK_RJ, interpolated_map)

    # test pick one of the available maps
    interpolated_map = interp.get_emission(20 * u.GHz)
    np.testing.assert_allclose(np.ones(shape) * u.uK_RJ, interpolated_map)

ext: py

max_stars_repo_name: LBJ-Wade/pysm

max_issues_repo_name: LBJ-Wade/pysm

max_forks_repo_name: LBJ-Wade/pysm

reason: import numpy


text:
"""
    TODO:
    1. Model probability in the future.
"""
import logging
import os
from pathlib import Path
from typing import Tuple

import click
import numpy as np
import pandas as pd
import torch
from tqdm import tqdm
from transformers import AutoModelWithHeads, AutoTokenizer

from food_ke.entailment.custom_typing import PathLike
from food_ke.entailment.predict import get_predictions_for_candidates_df

logging.basicConfig(
    format="[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s",
    level=logging.DEBUG,
)

if torch.cuda.is_available():
    device = torch.device("cuda")
else:
    device = torch.device("cpu")

def _compute_hyp_novelty_score(
    foodb_food_id: str, foodb_chemical_id: str, contents_kg: pd.DataFrame
):
    """Helper function to compute novelty score for a hypothesis.

    Args:
        hypothesis (pd.Series): The hypothesis to be scored.
        contents_kg (pd.DataFrame): The content in knowledge graph.

    Returns:
        (int): The novelty score of the hypothesis.

    """
    if foodb_food_id is None or foodb_chemical_id is None:
        # Novel if the food or the chemical is not in the FooDB.
        return 1

    content = contents_kg[
        (contents_kg["food_id"] == foodb_food_id)
        & (contents_kg["chemical_id"] == foodb_chemical_id)
    ]

    if content.empty:
        # Novel if the relation is not in the FooDB.
        return 1

    return 0

def rank_by_novelty(data, contents_kg, aggregation="sum"):
    """Rank the premises based on their hypothesis novelty.

    Args:
        data (pd.DataFrame): The data being ranked.
        contents_kg (pd.DataFrame): The content in knowledge graph.
        scoring (str): The scoring method.

    Returns:
        (list): Novelty scores.

    """
    data = data.copy()
    logging.info(
        f"Computing novelty scores and aggregating by {aggregation}...\n"
    )

    scores = []

    novelty_score = [
        _compute_hyp_novelty_score(
            foodb_food_id, foodb_chemical_id, contents_kg
        )
        for foodb_food_id, foodb_chemical_id in zip(
            data["hypothesis_food_id"],
            data["hypothesis_chemical_id"],
        )
    ]

    data["hypothesis_novelty"] = novelty_score

    premise_novelty = data.groupby("premise")["hypothesis_novelty"].agg(
        aggregation
    )

    data["premise_novelty"] = data.index.map(premise_novelty)

    return data

def rank_by_completeness(data: pd.DataFrame, scoring: str):
    """Rank the data into complete and incomplete dataframes in terms of
    the metadata, (e.g., food parts).

    Args:
        data (pd.DataFrame): The dataframe to be ranked.
        scoring (str): [mean|sum], Completeness scoring method.

    Returns:
        (list): Scores for premises.

    """
    data = data.copy()
    data["hypothesis_completeness"] = np.nan
    data["premise_completeness"] = np.nan
    logging.info(f"Computing completeness scores with {scoring} metric...")

    scores = []

    for premise in tqdm(data["premise"].unique().tolist()):
        try:
            counts_food_part = data["hypothesis_food_part"][
                premise
            ].value_counts()
        except AttributeError:
            # Only 1 hypothesis associated with the premise, so score = 0.
            scores.append(0)
            continue

        logging.info("counts food part:" + str(counts_food_part))

        if scoring == "mean":
            score = 1 - (
                counts_food_part["(whole plant)"] / counts_food_part.sum()
            )
        elif scoring == "sum":
            score = counts_food_part.sum() - counts_food_part["(whole plant)"]
        else:
            raise ValueError(f"Unknown scoring method: {scoring}")
        scores.append(score)

    data["hypothesis_completeness"] = scores

    return data

def rank_by_confidence_and_uncertainty(
    data, model, tokenizer, aggregation="sum", confidence_mode: str = "entails"
) -> Tuple[list, list]:
    data = data.copy()

    preds = get_predictions_for_candidates_df(data, model, tokenizer)
    preds.index = data.index
    data["proba_entails"] = preds["proba_entails"]
    data["proba_not_entails"] = preds["proba_not_entails"]

    if confidence_mode == "entails":
        logging.info("Using proba_entails as confidence value")
        data["hypothesis_confidence"] = preds["proba_entails"]

    elif confidence_mode == "both":
        logging.info(
            "Using max of proba_entails or proba_not_entails as confidence value"
        )
        data["hypotheses_confidence"] = preds[
            ["proba_entails", "proba_not_entails"]
        ].max(axis=1)
    else:
        raise ValueError(
            f"confidence_mode should be either 'entails' or 'both'"
        )

    logging.info(
        "Using min of proba_entails and proba_not_entails as uncertainty"
    )
    data["hypothesis_uncertainty"] = preds[
        ["proba_entails", "proba_not_entails"]
    ].min(axis=1)

    logging.info(
        "Computing premise confidence and uncertainty scores with "
        f"'{aggregation}' metric...\n"
    )

    data["premise_confidence"] = data["premise"].map(
        data.groupby("premise")["hypothesis_confidence"]
        .agg(aggregation)
        .to_dict()
    )
    data["premise_uncertainty"] = data["premise"].map(
        data.groupby("premise")["hypothesis_uncertainty"]
        .agg(aggregation)
        .to_dict()
    )

    return data

def rank_data_unlabelled(
    data_path: PathLike,
    model_path: PathLike,
    adapter_path: PathLike,
    tokenizer_name: PathLike,
    kg_content_path: PathLike,
    rank_novelty: bool = True,
    agg_novelty: str = "sum",
    rank_confidence: bool = True,
    agg_confidence: str = "sum",
    rank_completeness: bool = True,
    agg_completeness: str = "sum",
):
    data = pd.read_csv(data_path)
    data.index = data["premise"]
    kg_content = pd.read_csv(kg_content_path)

    model = AutoModelWithHeads.from_pretrained(model_path)
    model.active_adapters = model.load_adapter(
        adapter_path, model_name="roberta-base"
    )
    model = model.to(device)
    model.eval()
    tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)

    if rank_novelty:
        data = rank_by_novelty(data, kg_content, agg_novelty)
    if rank_confidence:
        data = rank_by_confidence_and_uncertainty(
            data,
            model,
            tokenizer,
            agg_confidence,
            confidence_mode="entails",
        )
    if rank_completeness:
        data = rank_by_completeness(data, agg_completeness)

    return data

@click.command()
@click.option("--data-path", type=str)
@click.option("--output-path", type=str)
@click.option("--model-path", type=str)
@click.option("--adapter-path", type=str)
@click.option("--tokenizer-name", type=str, default="roberta-base")
@click.option(
    "--kg-content-path",
    type=str,
    default=os.path.join(
        f"{os.path.abspath(os.path.dirname(__file__))}",
        "..",
        "..",
        "data",
        "knowledge_bases",
        "foodb",
        "foodb_2020_04_07_csv",
        "FooDB.csv",
    ),
)
@click.option("--score-weights", default=None)
@click.option("--scoring", default=None)
@click.option("--rank-novelty", type=bool, default=True)
@click.option("--agg-novelty", type=bool, default=True)
@click.option("--rank-confidence", type=bool, default=True)
@click.option("--agg-confidence", type=bool, default=True)
@click.option("--rank-completeness", type=bool, default=False)
@click.option("--agg-completeness", type=bool, default=True)
def main(
    data_path: PathLike,
    output_path: PathLike,
    model_path: PathLike,
    adapter_path: PathLike,
    tokenizer_name: str,
    kg_content_path: PathLike,
    score_weights=None,
    scoring=None,
    rank_novelty=True,
    agg_novelty="sum",
    rank_confidence=True,
    agg_confidence="sum",
    rank_completeness=False,
    agg_completeness="sum",
):
    if score_weights is not None or scoring is not None:
        raise NotImplementedError(
            "Score_weights and scoring are used to calculate premise scores which are "
            "not implemented yet."
        )

    data = rank_data_unlabelled(
        data_path,
        model_path,
        adapter_path,
        tokenizer_name,
        kg_content_path,
        rank_novelty,
        agg_novelty,
        rank_confidence,
        agg_confidence,
        rank_completeness,
        agg_completeness,
    )

    if output_path:
        data.to_csv(output_path, index=False)

    return data

if __name__ == "__main__":
    main()

max_stars_repo_name: IBPA/FoodAtlas

max_issues_repo_name: IBPA/FoodAtlas

max_forks_repo_name: IBPA/FoodAtlas


text:
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

from pytorch_lightning.core.lightning import LightningModule

from abc import abstractmethod

from zephyr.losses import getLoss
from zephyr.data_util import convertResultsBOP, saveOutputs
from zephyr.utils import meta2K
from zephyr.utils.icp import icpRefinement
from zephyr.utils.metrics import AddErrorMetric

import csv, os, copy

class BaseModel(LightningModule):
    def __init__(self, args):
        super(BaseModel, self).__init__()
        self.criterion = getLoss(args)

        self.unseen_oids = args.unseen_oids
        self.lr = args.lr
        self.chunk_size = args.chunk_size
        self.n_datasets = len(args.dataset_root)
        self.dataset_name = args.dataset_name

        self.args = args

        self.add01d_metrics = {}
        for dn in self.dataset_name:
            assert type(dn) is str
            self.add01d_metrics[dn] = AddErrorMetric(dn)

    @abstractmethod
    def forward(self, data):
        raise NotImplementedError

    def getRegLoss(self):
        return 0

    def splitChunks(self, data, chunk_size):
        chunks = []
        pp_err = data['pp_err']
        idx_chunks = torch.split(torch.arange(pp_err.shape[0]), chunk_size)
        for idx_chunk in idx_chunks:
            chunk = {}
            for k, v in data.items():
                if type(v) is torch.Tensor and v.nelement() > 1 and \
                        k in ['pp_err', 'transforms'] or k[-2:] == "_x":
                    chunk[k] = v[idx_chunk]
                else:
                    chunk[k] = v
            chunks.append(chunk)
        return chunks

    def train_dataloader(self):
        return None

    def val_dataloader(self):
        return None

    def training_step(self, batch, batch_idx):
        data = batch
        pp_err = data['pp_err']

        if self.chunk_size is not None and pp_err.shape[0] <= self.chunk_size:
            pred_y = self.forward(data)
        else:
            chunks = self.splitChunks(data, self.chunk_size)
            pred_y = []
            for chunk in chunks:
                pred_y_chunk = self.forward(chunk)
                pred_y.append(pred_y_chunk)
            pred_y = torch.cat(pred_y)

        return {"pred_y": pred_y, "pp_err": pp_err}

    def training_step_end(self, outputs):
        pred_y = torch.cat([outputs['pred_y']])
        pp_err = torch.cat([outputs['pp_err']])

        loss = self.criterion(pred_y, pp_err)
        loss += self.getRegLoss()

        gt_idx = torch.argmin(pp_err)
        max_idx = torch.argmax(pred_y)
        gt_err = pp_err[gt_idx]
        max_err = pp_err[max_idx]

        log = {"train_err": max_err, "train_loss": loss}
        return {"loss": loss, 'log': log}

    def training_epoch_end(self, outputs):
        train_epoch_err = 0
        train_epoch_loss = 0
        for o in outputs:
            train_epoch_err += o['train_err']
            train_epoch_loss += o['train_loss']
        train_epoch_err /= len(outputs)
        train_epoch_loss /= len(outputs)

        log = {
            "train_epoch_err": train_epoch_err,
            'train_epoch_loss': train_epoch_loss
        }

        # print("training_epoch_end: train_epoch_err =", train_epoch_err)

        return {
            "train_epoch_err": train_epoch_err,
            "log": log
        }

    def validation_step(self, batch, batch_idx):
        data = batch
        # print("validation_step")
        # for k, v in data.items():
        #     print(k)
        #     print(type(v))
        #     print(v.device)
        # print(data['pp_err'].shape)
        pp_err = data['pp_err']

        # print(transforms.shape)
        # print(pp_err.shape)

        if self.chunk_size is not None and pp_err.shape[0] <= self.chunk_size:
            pred_y = self.forward(data)
        else:
            chunks = self.splitChunks(data, self.chunk_size)
            pred_y = []
            for chunk in chunks:
                pred_y_chunk = self.forward(chunk)
                pred_y.append(pred_y_chunk)
            pred_y = torch.cat(pred_y)

        # if self.args.mask_th is not None:
        #     hypo_mask = data['seg_scores'] > self.args.mask_th
        # else:
        #     hypo_mask = torch.ones_like(pred_y)
        if self.args.use_mask_test:
            hypo_mask = data['mask_scores'] > 0.2
        else:
            hypo_mask = torch.ones_like(pred_y)

        return {
            "pred_y": pred_y, "pp_err": pp_err,
            "transforms": data['transforms'],
            "object_id": data['object_id'][0],
            "scene_id": data['scene_id'][0],
            "im_id": data['im_id'][0],
            "dataset_i": data['dataset_i'][0],
            "hypo_mask": hypo_mask,
            "depth": data['depth'],
            "meta_data": data['meta_data'],
            "uv_original": data['uv_original'],
            "model_points": data['model_points']
        }

    def validation_step_end(self, outputs):
        pred_y = torch.cat([outputs['pred_y']])
        pp_err = torch.cat([outputs['pp_err']])
        transforms = torch.cat([outputs['transforms']])
        hypo_mask = torch.cat([outputs['hypo_mask']])
        uv_original = torch.cat([outputs['uv_original']])

        # assert pp_err[-1] == 0
        assert pp_err[-2] > 0

        loss = self.criterion(pred_y, pp_err)
        loss += self.getRegLoss()

        # # Invalidate the SIFT hypotheses
        # if self.args.sift_only:
        #     pred_y[:self.args.n_ppf_hypos] = pred_y.min()
        #     pred_y[-1:] = pred_y.min()
        #
        # # Invalidate the PPF hypotheses
        # if self.args.ppf_only:
        #     pred_y[self.args.n_ppf_hypos : -1] = pred_y.min()

        # # Invalidate the hypotheses that does not meet the mask test
        # if hypo_mask.sum() > 0 and self.args.use_mask_test:
        #     pred_y[~hypo_mask] = pred_y.min() - 1

        # Purly based on order
        if self.args.no_model:
            pred_y = - torch.arange(len(pred_y), dtype=pred_y.dtype, device=pred_y.device)

        max_idx = torch.argmax(pred_y)
        max_err = pp_err[max_idx]
        mat = transforms[max_idx]

        if len(pred_y) == 1:
            mat_wo_oracle = torch.eye(4, device = transforms.device, dtype=transforms.dtype)
            max_err_wo_oracle = 1
        else:
            max_idx_wo_oracle = torch.argmax(pred_y[:-1])
            max_err_wo_oracle = pp_err[max_idx_wo_oracle]
            mat_wo_oracle = transforms[max_idx_wo_oracle]

            if self.args.icp:
                cam_K = meta2K(outputs['meta_data'])
                depth = to_np(outputs['depth'])
                model_points = to_np(outputs['model_points'])
                # mat, _ = icpRefinement(depth, to_np(uv_original[max_idx]), to_np(mat), cam_K, model_points, inpaint_depth=False, icp_max_dist=0.01)
                mat_wo_oracle, _ = icpRefinement(depth, to_np(uv_original[max_idx_wo_oracle]), to_np(mat_wo_oracle), cam_K, model_points, inpaint_depth=False, icp_max_dist=0.01)

        if len(outputs['object_id'].shape) >= 1:
            object_id = outputs['object_id'][0]
            scene_id = outputs['scene_id'][0]
            im_id = outputs['im_id'][0]
            dataset_i = outputs['dataset_i'][0]
        else:
            object_id = outputs['object_id'].item()
            scene_id = outputs['scene_id'].item()
            im_id = outputs['im_id'].item()
            dataset_i = outputs['dataset_i'].item()

        return {
            "val_loss": loss,
            "val_err": max_err, "val_err_wo_oracle": max_err_wo_oracle,
            "mat": mat, "mat_wo_oracle": mat_wo_oracle,
            "object_id": object_id,
            "scene_id": scene_id,
            "im_id": im_id,
            "dataset_i": dataset_i,
        }

    def validation_epoch_end(self, outputs_all):
        '''For each dataset'''
        log = {}
        seen_outputs_all = []
        unseen_outputs_all = []
        for dataset_i in range(self.n_datasets):
            unseen_oids = self.unseen_oids[dataset_i]
            dataset_name = self.dataset_name[dataset_i]
            outputs = [o for o in outputs_all if o['dataset_i'] == dataset_i]
            seen_outputs = [o for o in outputs if o['object_id'] not in unseen_oids]
            unseen_outputs = [o for o in outputs if o['object_id'] in unseen_oids]
            seen_outputs_all += seen_outputs
            unseen_outputs_all += unseen_outputs

            seen_loss = torch.mean(torch.Tensor([o['val_loss'] for o in seen_outputs]))
            unseen_loss = torch.mean(torch.Tensor([o['val_loss'] for o in unseen_outputs]))
            avg_loss = torch.mean(torch.Tensor([o['val_loss'] for o in outputs]))

            seen_err = torch.mean(torch.Tensor([o['val_err'] for o in seen_outputs]))
            unseen_err = torch.mean(torch.Tensor([o['val_err'] for o in unseen_outputs]))
            avg_err = torch.mean(torch.Tensor([o['val_err'] for o in outputs]))

            seen_err_wo_oracle = torch.mean(torch.Tensor([o['val_err_wo_oracle'] for o in seen_outputs]))
            unseen_err_wo_oracle = torch.mean(torch.Tensor([o['val_err_wo_oracle'] for o in unseen_outputs]))
            avg_err_wo_oracle = torch.mean(torch.Tensor([o['val_err_wo_oracle'] for o in outputs]))

            seen_add01d = self.add01d_metrics[dataset_name](map(lambda x: x['val_err_wo_oracle'], seen_outputs), map(lambda x: x['object_id'], seen_outputs))
            unseen_add01d = self.add01d_metrics[dataset_name](map(lambda x: x['val_err_wo_oracle'], unseen_outputs), map(lambda x: x['object_id'], unseen_outputs))
            avg_add01d = self.add01d_metrics[dataset_name](map(lambda x: x['val_err_wo_oracle'], outputs), map(lambda x: x['object_id'], outputs))

            log.update({
                ("val_err_%s" % dataset_name): avg_err, ("val_loss_%s" % dataset_name): avg_loss,
                ("val_seen_err_%s" % dataset_name): seen_err, ("val_unseen_err_%s" % dataset_name): unseen_err,
                ("val_add01d_%s" % dataset_name): avg_add01d,
                ("val_seen_add01d_%s" % dataset_name): seen_add01d, ("val_unseen_add01d_%s" % dataset_name): unseen_add01d,
            })

        '''Average over all datasets'''
        seen_loss = torch.mean(torch.Tensor([o['val_loss'] for o in seen_outputs_all]))
        unseen_loss = torch.mean(torch.Tensor([o['val_loss'] for o in unseen_outputs_all]))
        avg_loss = torch.mean(torch.Tensor([o['val_loss'] for o in outputs_all]))

        seen_err = torch.mean(torch.Tensor([o['val_err'] for o in seen_outputs_all]))
        unseen_err = torch.mean(torch.Tensor([o['val_err'] for o in unseen_outputs_all]))
        avg_err = torch.mean(torch.Tensor([o['val_err'] for o in outputs_all]))

        seen_err_wo_oracle = torch.mean(torch.Tensor([o['val_err_wo_oracle'] for o in seen_outputs_all]))
        unseen_err_wo_oracle = torch.mean(torch.Tensor([o['val_err_wo_oracle'] for o in unseen_outputs_all]))
        avg_err_wo_oracle = torch.mean(torch.Tensor([o['val_err_wo_oracle'] for o in outputs_all]))

        seen_add01d = self.add01d_metrics[dataset_name](map(lambda x: x['val_err_wo_oracle'], seen_outputs_all), map(lambda x: x['object_id'], seen_outputs_all))
        unseen_add01d = self.add01d_metrics[dataset_name](map(lambda x: x['val_err_wo_oracle'], unseen_outputs_all), map(lambda x: x['object_id'], unseen_outputs_all))
        avg_add01d = self.add01d_metrics[dataset_name](map(lambda x: x['val_err_wo_oracle'], outputs_all), map(lambda x: x['object_id'], outputs_all))

        log.update({"val_err": avg_err, "val_loss": avg_loss, "val_seen_err": seen_err, "val_unseen_err": unseen_err})
        log.update({"val_add01d": avg_add01d, "val_seen_add01d": seen_add01d, "val_unseen_add01d": unseen_add01d})

        to_return = {
            "log": log, "val_epoch_err": avg_err,
            "seen_err": seen_err,
            "unseen_err": unseen_err,
            "avg_err": avg_err,
            "seen_err_wo_oracle": seen_err_wo_oracle,
            "unseen_err_wo_oracle": unseen_err_wo_oracle,
            "avg_err_wo_oracle": avg_err_wo_oracle,
        }

        return to_return

    def test_step(self, batch, batch_idx):
        return self.validation_step(batch, batch_idx)

    def test_step_end(self, outputs):
        return self.validation_step_end(outputs)

    def test_epoch_end(self, outputs):
        '''Save the results'''
        # input_file = self.args.exp_name + "_" + self.args.resume_path.split('/')[-1].split("-")[0] + "_%s.csv" % self.args.data_split
        input_file = self.args.exp_name + "_%s.csv" % self.args.data_split
        save_path = os.path.join("test_logs", input_file)

        for d in ["test_logs", "test_logs/bop_results"]:
            if not os.path.exists(d):
                os.makedirs(d)

        saveOutputs(outputs, save_path)
        # with open(save_path, 'w', newline='') as csvfile:
        #     fieldnames = outputs[0].keys()
        #     writer = csv.DictWriter(csvfile, fieldnames=fieldnames, quoting = csv.QUOTE_NONE)
        #
        #     writer.writeheader()
        #     for i in range(len(outputs)):
        #         o = copy.deepcopy(outputs[i])
        #         # for k, v in o.items():
        #         #     if torch.is_tensor(v):
        #         #         print(k, v.shape)
        #         o = {k: to_np(v) for k, v in o.items()}
        #         o['mat'] = " ".join(["%0.6f"%_ for _ in o['mat'].reshape(-1).tolist()])
        #         o['mat_wo_oracle'] = " ".join(["%0.4f"%_ for _ in o['mat_wo_oracle'].reshape(-1).tolist()])
        #         writer.writerow(o)

        '''Again save the results in BOP format'''
        convertResultsBOP(
            input_root="test_logs",
            output_root=os.path.join("test_logs", "bop_results"),
            input_file=input_file,
            dataset_name=self.args.dataset_name[0],
            subname=None,
            mat_name="mat_wo_oracle"
        )
        convertResultsBOP(
            input_root="test_logs",
            output_root=os.path.join("test_logs", "bop_results"),
            input_file=input_file,
            dataset_name=self.args.dataset_name[0],
            subname="orahypo",
            mat_name="mat"
        )

        results = self.validation_epoch_end(outputs)
        print("w/ oracle hypo: avg %.3f, seen %.3f, unseen %.3f" % (
            results['avg_err'], results['seen_err'], results['unseen_err']
        ))

        print("w/o oracle hypo: avg %.3f, seen %.3f, unseen %.3f" % (
            results['avg_err_wo_oracle'], results['seen_err_wo_oracle'], results['unseen_err_wo_oracle']
        ))
        return results

    def configure_optimizers(self):
        optimizer = torch.optim.Adam(self.parameters(), lr=self.lr)
        scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[30, 80], gamma=0.1)
        return [optimizer], [scheduler]

def to_np(x):
    if type(x) is int:
        return x
    elif torch.is_tensor(x):
        return x.detach().cpu().numpy()
    else:
        return x.detach().data.cpu().numpy()

max_stars_repo_name: r-pad/zephyr

max_issues_repo_name: r-pad/zephyr

max_forks_repo_name: r-pad/zephyr


text:
"""Tests for all FEP-based estimators in ``alchemlyb``.

"""
import pytest

import numpy as np
import pandas as pd

from alchemlyb.parsing import gmx, amber, namd, gomc
from alchemlyb.estimators import MBAR, BAR
import alchemtest.gmx
import alchemtest.amber
import alchemtest.gomc
import alchemtest.namd

def gmx_benzene_coul_u_nk():
    dataset = alchemtest.gmx.load_benzene()

    u_nk = pd.concat([gmx.extract_u_nk(filename, T=300)
                      for filename in dataset['data']['Coulomb']])

    return u_nk

def gmx_benzene_vdw_u_nk():
    dataset = alchemtest.gmx.load_benzene()

    u_nk = pd.concat([gmx.extract_u_nk(filename, T=300)
                      for filename in dataset['data']['VDW']])

    return u_nk

def gmx_expanded_ensemble_case_1():
    dataset = alchemtest.gmx.load_expanded_ensemble_case_1()

    u_nk = pd.concat([gmx.extract_u_nk(filename, T=300)
                      for filename in dataset['data']['AllStates']])

    return u_nk

def gmx_expanded_ensemble_case_2():
    dataset = alchemtest.gmx.load_expanded_ensemble_case_2()

    u_nk = pd.concat([gmx.extract_u_nk(filename, T=300)
                      for filename in dataset['data']['AllStates']])

    return u_nk

def gmx_expanded_ensemble_case_3():
    dataset = alchemtest.gmx.load_expanded_ensemble_case_3()

    u_nk = pd.concat([gmx.extract_u_nk(filename, T=300)
                      for filename in dataset['data']['AllStates']])

    return u_nk

def gmx_water_particle_with_total_energy():
    dataset = alchemtest.gmx.load_water_particle_with_total_energy()

    u_nk = pd.concat([gmx.extract_u_nk(filename, T=300)
                      for filename in dataset['data']['AllStates']])

    return u_nk

def gmx_water_particle_with_potential_energy():
    dataset = alchemtest.gmx.load_water_particle_with_potential_energy()

    u_nk = pd.concat([gmx.extract_u_nk(filename, T=300)
                      for filename in dataset['data']['AllStates']])

    return u_nk

def gmx_water_particle_without_energy():
    dataset = alchemtest.gmx.load_water_particle_without_energy()

    u_nk = pd.concat([gmx.extract_u_nk(filename, T=300)
                      for filename in dataset['data']['AllStates']])

    return u_nk

def amber_bace_example_complex_vdw():
    dataset = alchemtest.amber.load_bace_example()

    u_nk = pd.concat([amber.extract_u_nk(filename, T=300)
                      for filename in dataset['data']['complex']['vdw']])
    return u_nk

def gomc_benzene_u_nk():
    dataset = alchemtest.gomc.load_benzene()

    u_nk = pd.concat([gomc.extract_u_nk(filename, T=298)
                      for filename in dataset['data']])

    return u_nk

def namd_tyr2ala():
    dataset = alchemtest.namd.load_tyr2ala()
    u_nk1 = namd.extract_u_nk(dataset['data']['forward'][0], T=300)
    u_nk2 = namd.extract_u_nk(dataset['data']['backward'][0], T=300)

    # combine dataframes of fwd and rev directions
    u_nk1[u_nk1.isna()] = u_nk2
    u_nk = u_nk1.sort_index(level=u_nk1.index.names[1:])

    return u_nk

class FEPestimatorMixin:
    """Mixin for all FEP Estimator test classes.

    """

    def compare_delta_f(self, X_delta_f):
        est = self.cls().fit(X_delta_f[0])
        delta_f, d_delta_f = self.get_delta_f(est)

        assert X_delta_f[1] == pytest.approx(delta_f, rel=1e-3)
        assert X_delta_f[2] == pytest.approx(d_delta_f, rel=1e-3)

    def get_delta_f(self, est):
        return est.delta_f_.iloc[0, -1], est.d_delta_f_.iloc[0, -1]

class TestMBAR(FEPestimatorMixin):
    """Tests for MBAR.

    """
    cls = MBAR

    @pytest.fixture(scope="class",
                    params=[(gmx_benzene_coul_u_nk, 3.041, 0.02088),
                            (gmx_benzene_vdw_u_nk, -3.007, 0.04519),
                            (gmx_expanded_ensemble_case_1, 75.923, 0.14124),
                            (gmx_expanded_ensemble_case_2, 75.915, 0.14372),
                            (gmx_expanded_ensemble_case_3, 76.173, 0.11345),
                            (gmx_water_particle_with_total_energy, -11.680, 0.083655),
                            (gmx_water_particle_with_potential_energy, -11.675, 0.083589),
                            (gmx_water_particle_without_energy, -11.654, 0.083415),
                            (amber_bace_example_complex_vdw, 2.40200, 0.0618453),
                            (gomc_benzene_u_nk, -0.79994, 0.091579),
                    ])
    def X_delta_f(self, request):
        get_unk, E, dE = request.param
        return get_unk(), E, dE

    def test_mbar(self, X_delta_f):
        self.compare_delta_f(X_delta_f)

class TestBAR(FEPestimatorMixin):
    """Tests for BAR.

    """
    cls = BAR

    @pytest.fixture(scope="class",
                    params = [(gmx_benzene_coul_u_nk, 3.044, 0.01640),
                              (gmx_benzene_vdw_u_nk, -3.033, 0.03438),
                              (gmx_expanded_ensemble_case_1, 75.993, 0.11056),
                              (gmx_expanded_ensemble_case_2, 76.009, 0.11220),
                              (gmx_expanded_ensemble_case_3, 76.219, 0.08886),
                              (gmx_water_particle_with_total_energy, -11.675, 0.065055),
                              (gmx_water_particle_with_potential_energy, -11.724, 0.064964),
                              (gmx_water_particle_without_energy, -11.660, 0.064914),
                              (amber_bace_example_complex_vdw, 2.37846, 0.050899),
                              (namd_tyr2ala, 11.0044, 0.10235),
                              (gomc_benzene_u_nk, -0.87095, 0.071263),
                    ])
    def X_delta_f(self, request):
        get_unk, E, dE = request.param
        return get_unk(), E, dE

    def test_bar(self, X_delta_f):
        self.compare_delta_f(X_delta_f)

    def get_delta_f(self, est):
        ee = 0.0

        for i in range(len(est.d_delta_f_) - 1):
            ee += est.d_delta_f_.values[i][i+1]**2
        return est.delta_f_.iloc[0, -1], ee**0.5


text:
import sys
from numba.core.errors import _MovedModule
sys.modules[__name__] = _MovedModule(locals(), "numba.core.generators")

reason: from numba


text:
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as torch_functional

from autodo.custom_models import BasicBlock
from autodo.model.attention_module import AttentionModule_stage3

IMG_WIDTH = 448
IMG_HEIGHT = 128

class BackBone(nn.Module):
    def __init__(self, block, layers, num_classes=3):
        self.inplanes = 64  # 64
        super(BackBone, self).__init__()
        #        self.simplemodel = reallysimplemodel()
        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
                               bias=False)
        self.bn0 = nn.BatchNorm2d(64)
        self.relu = nn.ReLU(inplace=True)
        self.selu = nn.SELU()
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.layer1 = self._make_layer(block, 64, layers[0])  # 64
        self.layer2 = self._make_layer(block, 256, layers[1], stride=2)  # 128
        self.attention_module3 = AttentionModule_stage3(256, 256, (16, 42))
        self.layer3 = self._make_layer(block, 512, layers[2], stride=2)  # 256
        self.attention_module4 = AttentionModule_stage3(512, 512, (8, 21))
        self.layer4 = self._make_layer(block, 1024, layers[3], stride=2)  # 512

    def _make_layer(self, block, planes, blocks, stride=1):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.inplanes, planes * block.expansion,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(planes * block.expansion),
            )
        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes))
        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)  # 224x224
        x = self.bn0(x)
        x = self.relu(x)
        x = self.maxpool(x)  # 112x112
        x = self.layer1(x)  # 56x56
        x = self.layer2(x)  # 28x28
        x = self.attention_module3(x)
        x = self.layer3(x)  # 14x14
        x = self.attention_module4(x)
        x = self.layer4(x)  # 7x7
        return x

class double_conv(nn.Module):
    """(conv => BN => ReLU) * 2"""

    def __init__(self, in_ch, out_ch):
        super(double_conv, self).__init__()
        self.conv = nn.Sequential(
            nn.Conv2d(in_ch, out_ch, 3, padding=1),
            nn.BatchNorm2d(out_ch),
            nn.ReLU(inplace=True),
            nn.Conv2d(out_ch, out_ch, 3, padding=1),
            nn.BatchNorm2d(out_ch),
            nn.ReLU(inplace=True)
        )

    def forward(self, x):
        x = self.conv(x)
        return x

class up(nn.Module):
    def __init__(self, in_ch, out_ch, bilinear=True):
        super(up, self).__init__()

        #  would be a nice idea if the upsampling could be learned too,
        #  but my machine do not have enough memory to handle all those weights
        if bilinear:
            self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)
        else:
            self.up = nn.ConvTranspose2d(in_ch // 2, in_ch // 2, 2, stride=2)

        self.conv = double_conv(in_ch, out_ch)

    def forward(self, x1, x2=None):
        x1 = self.up(x1)

        # input is CHW
        diffY = x2.size()[2] - x1.size()[2]
        diffX = x2.size()[3] - x1.size()[3]

        x1 = torch_functional.pad(x1, [diffX // 2, diffX - diffX // 2,
                                       diffY // 2, diffY - diffY // 2])

        # for padding issues, see
        # https://github.com/HaiyongJiang/U-Net-Pytorch-Unstructured-Buggy/commit/0e854509c2cea854e247a9c615f175f76fbb2e3a
        # https://github.com/xiaopeng-liao/Pytorch-UNet/commit/8ebac70e633bac59fc22bb5195e513d5832fb3bd

        if x2 is not None:
            x = torch.cat([x2, x1], dim=1)
        else:
            x = x1
        x = self.conv(x)
        return x

class MyUNet(nn.Module):
    """Mixture of previous classes"""

    def __init__(self, n_classes, device):
        super(MyUNet, self).__init__()
        self.device = device
        self.base_model = BackBone(BasicBlock, [3, 4, 6, 3]).to(device)

        self.conv0 = double_conv(6, 64)
        self.conv1 = double_conv(64, 128)
        self.conv2 = double_conv(128, 512)
        self.conv3 = double_conv(512, 1024)

        self.mp = nn.MaxPool2d(2)

        self.up1 = up(1024 + 1024 + 2, 512)
        self.up2 = up(512 + 512, 256)
        self.up3 = up(256 + 128, 128)
        self.up4 = up(128 + 64, 64)
        self.up5 = up(64 + 6, 64)
        self.outc = nn.Conv2d(64, n_classes, 1)

    def get_mesh(self, batch_size, shape_x, shape_y):
        mg_x, mg_y = np.meshgrid(np.linspace(0, 1, shape_y), np.linspace(0, 1, shape_x))
        mg_x = np.tile(mg_x[None, None, :, :], [batch_size, 1, 1, 1]).astype('float32')
        mg_y = np.tile(mg_y[None, None, :, :], [batch_size, 1, 1, 1]).astype('float32')
        mesh = torch.cat([torch.tensor(mg_x).to(self.device), torch.tensor(mg_y).to(self.device)], 1)
        return mesh

    def forward(self, x):
        batch_size = x.shape[0]
        mesh1 = self.get_mesh(batch_size, x.shape[2], x.shape[3])
        x0 = torch.cat([x, mesh1], 1)
        x1 = self.mp(self.conv0(x0))
        x2 = self.mp(self.conv1(x1))
        x3 = self.mp(self.conv2(x2))
        x4 = self.mp(self.conv3(x3))

        x_center = x[:, :3, :, IMG_WIDTH // 8: -IMG_WIDTH // 8]
        feats = self.base_model(x_center)
        bg = torch.zeros([feats.shape[0], feats.shape[1], feats.shape[2], feats.shape[3] // 8]).to(self.device)
        feats = torch.cat([bg, feats, bg], 3)

        # Add positional info
        mesh2 = self.get_mesh(batch_size, feats.shape[2], feats.shape[3])
        feats = torch.cat([feats, mesh2], 1)
        x = self.up1(feats, x4)
        x = self.up2(x, x3)
        x = self.up3(x, x2)
        x = self.up4(x, x1)
        x = self.up5(x, x0)
        x = self.outc(x)
        return x

max_stars_repo_name: MatthewScholefield/autodo

max_issues_repo_name: MatthewScholefield/autodo

max_forks_repo_name: MatthewScholefield/autodo


text:
import abc
import decimal
import collections

import numpy as np
from pandas.compat.numpy import function as nv
import pandas as pd
from pandas.core import nanops
import money
from pandas.api.extensions import ExtensionDtype

from ._accessor import (DelegatedMethod, DelegatedProperty,
                        delegated_method)
from .base import NumPyBackedExtensionArrayMixin
from .parser import _as_money_object
import re

# -----------------------------------------------------------------------------
# Extension Type
# -----------------------------------------------------------------------------

@pd.api.extensions.register_extension_dtype
class MoneyType(ExtensionDtype):
    name = 'money'
    na_value = np.nan
    type = money.XMoney
    kind = 'O'
    default_money_code = None
    _record_type = np.dtype([('va', np.float64), ('cu', 'U3')])
    _record_na_value = (0, '')

    def __init__(self, *args, default_money_code=None, **kwargs):
        self.default_money_code = default_money_code

        super(MoneyType, self).__init__(*args, **kwargs)

    @classmethod
    def construct_from_string(cls, string):
        if string == cls.name:
            return cls()
        else:
            match = re.match(cls.name + r'\[([A-Z]{3})\]', string)

            if match:
                default_money_code = match.group(0)
                return cls(default_money_code=default_money_code)

            raise TypeError("Cannot construct a '{}' from "
                            "'{}'".format(cls, string))

    @classmethod
    def construct_array_type(cls):
        return MoneyArray

# -----------------------------------------------------------------------------
# Extension Container
# -----------------------------------------------------------------------------

class MoneyArray(NumPyBackedExtensionArrayMixin):
    """Holder for Money Amounts.

    MoneyArray is a container for Money Amounts. It satisfies pandas'
    extension array interface, and so can be stored inside
    :class:`pandas.Series` and :class:`pandas.DataFrame`.

    See :ref:`usage` for more.
    """
    __array_priority__ = 1000
    _dtype = MoneyType()
    _itemsize = 20
    ndim = 1
    can_hold_na = True
    default_money_code = None

    def __init__(self, values, default_money_code=None, dtype=None, copy=False):
        from .parser import _to_money_array

        # TODO: copy
        if dtype and dtype != self.dtype:
            raise TypeError("Can only construct MoneyArray with underlying (f64, U3) not {}".format(dtype))

        values, self.default_money_code = _to_money_array(values, default_money_code=default_money_code)  # TODO: avoid potential copy
        # TODO: dtype?
        if copy:
            values = values.copy()
        self.data = values

    def to_decimals(self, money_code=None):
        r"""Create a list of decimals from an ISO4712 code, attempting conversion with XMoney where necessary.

        Parameters
        ----------
        money_code : ISO4712 3-letter currency code

        Returns
        -------
        list of decimals

        Examples
        --------
        >>> arr = MoneyArray([10, 20], 'GBP')
        >>> values = arr.to_decimals('GBP')
        >>> values
        [10, 20]

        See Also
        --------
        to_bytes
        """

        if not money_code:
            money_code = self.default_money_code
            if not money_code:
                codes = {c['cu'] for c in self.data if c['cu']}
                if len(codes) != 1:
                    raise TypeError("Cannot output mixed-currency monies as decimal "
                        "without either a target or default currency")
                money_code = codes[0]

        mask = self.isna()
        same = (self.data['cu'] == money_code) | mask
        decimalize = np.vectorize(decimal.Decimal)
        result = decimalize(self.data['va'])
        for i, ceq in enumerate(same):
            if not ceq:
                result[i] = money.XMoney(*self.data[i]).to(money_code).amount

        return result

    # Operations thanks to pandas.core.arrays.base.numpy_
    def _min(self, ndarray, axis=None, out=None, keepdims=False, skipna=True):
        nv.validate_min((), dict(out=out, keepdims=keepdims))
        return nanops.nanmin(ndarray, axis=axis, skipna=skipna)

    def _max(self, ndarray, axis=None, out=None, keepdims=False, skipna=True):
        nv.validate_max((), dict(out=out, keepdims=keepdims))
        return nanops.nanmax(ndarray, axis=axis, skipna=skipna)

    def _sum(
        self,
        ndarray,
        axis=None,
        dtype=None,
        out=None,
        keepdims=False,
        initial=None,
        skipna=True,
        min_count=0,
    ):
        nv.validate_sum(
            (), dict(dtype=dtype, out=out, keepdims=keepdims, initial=initial)
        )
        return nanops.nansum(
            ndarray, axis=axis, skipna=skipna, min_count=min_count
        )

    def _reduce(self, name, skipna=True, **kwargs):
        """ _reduce is called when min, sum or max is called via the pandas series (column).
        It's stored as an array of floats & the _reduce operation is performed on that.
        """
        currencies = [cu for cu in np.unique(self.data['cu']) if cu]
        totals = {}

        if name == 'mean':
            meth = getattr(self, '_sum', None)
        else:
            meth = getattr(self, '_' + name, None)

        if meth:
            if len(currencies) > 1:
                money_code = self.default_money_code if self.default_money_code else currencies[0]
                for i, currency in enumerate(currencies):
                    totals[currency] = money.XMoney(
                        meth(self.data['va'][self.data['cu'] == currency], skipna=skipna, **kwargs),
                        currency
                    )
                total = meth(
                    np.array([subtotal.to(money_code).amount for subtotal in totals.values()]),
                    skipna=skipna,
                    **kwargs
                )
                if name == 'mean':
                    total = total / len(self.data)
                total = money.XMoney(amount=total, currency=money_code)
            else:
                money_code = currencies[0] if currencies else self.default_money_code
                total = money.XMoney(meth(self.data['va'], skipna=skipna, **kwargs), money_code)

            return total
        else:
            msg = "'{}' does not implement reduction '{}'"
            raise TypeError(msg.format(type(self).__name__, name))

    @classmethod
    def from_bytes(cls, bytestring):
        r"""Create a MoneyArray from a bytestring.

        Parameters
        ----------
        bytestring : bytes
            Note that bytestring is a Python 3-style string of bytes

        Returns
        -------
        MoneyArray

        Examples
        --------
        >>> arr = MoneyArray([10, 20])
        >>> buf = arr.to_bytes()
        >>> buf
        b'\x00\x00\...x00\x02'
        >>> MoneyArray.from_bytes(buf)
        MoneyArray(['10GBP', '10GBP'])

        See Also
        --------
        to_bytes
        """
        data = np.frombuffer(bytestring, dtype=MoneyType._record_type)
        return cls._from_ndarray(data)

    @classmethod
    def _from_ndarray(cls, data, copy=False):
        """Zero-copy construction of an MoneyArray from an ndarray.

        Parameters
        ----------
        data : ndarray
            This should have MoneyType._record_type dtype
        copy : bool, default False
            Whether to copy the data.

        Returns
        -------
        ExtensionArray
        """
        if copy:
            data = data.copy()
        new = MoneyArray([])
        new.data = data
        return new

    # -------------------------------------------------------------------------
    # Properties
    # -------------------------------------------------------------------------
    # With thanks to the pandas docs
    def take(self, indices, allow_fill=False, fill_value=None):
        from pandas.core.algorithms import take

        if allow_fill and fill_value is None:
            fill_value = self.dtype.na_value

        if fill_value is self.dtype.na_value:
            fill_value = self.dtype._record_na_value

        # fill value should always be translated from the scalar
        # type for the array, to the physical storage type for
        # the data, before passing to take.

        indices = np.asarray(indices)
        if allow_fill:
            mask = (indices == -1)
            if not len(self):
                if not (indices == -1).all():
                    msg = "Invalid take for empty array. Must be all -1."
                    raise IndexError(msg)
                else:
                    # all NA take from and empty array
                    result = np.zeros(len(indices), dtype=self.dtype._record_type)
                    result.fill(fill_value)
                    return self._from_ndarray(result)
            if (np.asarray(indices) < -1).any():
                msg = ("Invalid value in 'indices'. Must be all >= -1 "
                       "for 'allow_fill=True'")
                raise ValueError(msg)

        result = take(self.data, indices, allow_fill=False)

        if allow_fill:
            result[mask] = fill_value

        return self._from_sequence(result, dtype=self.dtype, default_money_code=self.default_money_code)

    @classmethod
    def _from_sequence(cls, scalars, dtype=None, copy=False, default_money_code=None):
        return cls(scalars, dtype=dtype, copy=copy, default_money_code=default_money_code)

    @classmethod
    def _from_sequence_of_strings(cls, strings, dtype=None, copy=False, default_money_code=None):
        return cls(strings, dtype=dtype, copy=copy, default_money_code=default_money_code)

    def isna(self):
        return self.data['cu'] == ''

    # -------------------------------------------------------------------------
    # Interfaces
    # -------------------------------------------------------------------------

    def __repr__(self):
        rep = super(MoneyArray, self).__repr__()

        money_code = self.default_money_code
        if money_code:
            class_name = self.__class__.__name__
            rep = rep.replace(class_name, f"{class_name}[{money_code}]")

        return rep

    @staticmethod
    def _box_scalar(scalar):
        if scalar == (0, ''):
            return np.nan
        elif type(scalar) is tuple:
            return money.XMoney(scalar[0], scalar[1])
        return money.XMoney(scalar['va'], scalar['cu'])

    @property
    def _parser(self):
        from .parser import to_money
        return lambda val: to_money(val, default_money_code=self.default_money_code)

    def __setitem__(self, key, value):
        from .parser import to_money

        value = to_money(value, default_money_code=self.default_money_code).data
        self.data[key] = value

    def __iter__(self):
        return iter(self.to_pymoney())

    # ------------------------------------------------------------------------
    # Serializaiton / Export
    # ------------------------------------------------------------------------

    def to_pymoney(self):
        """Convert the array to a list of scalar Money objects.

        Returns
        -------
        addresses : List
            Each element of the list will be a :class:`money.XMoney` or np.nan

        See Also
        --------

        Examples
        ---------
        >>> MoneyArray(['120 EUR', '127 USD']).to_pymoney()
        [XMoney('120', 'EUR'), XMoney('127', 'USD')]
        """
        return [money.XMoney(x['va'], x['cu']) if x['cu'] else np.nan for x in self.data]

    def to_bytes(self):
        r"""Serialize the MoneyArray as a Python bytestring.

        This and :meth:MoneyArray.from_bytes is the fastest way to roundtrip
        serialize and de-serialize a MoneyArray.

        See Also
        --------
        MoneyArray.from_bytes

        Examples
        --------
        >>> arr = MoneyArray('GBP', [10, 20])
        >>> arr.to_bytes()
        b'\x00\x00\...x00\x02'
        """
        return self.data.tobytes()

    def astype(self, dtype, copy=True):
        if isinstance(dtype, MoneyType):
            if copy:
                self = self.copy()
            return self
        return super(MoneyArray, self).astype(dtype)

    # ------------------------------------------------------------------------
    # Ops
    # ------------------------------------------------------------------------

    def __eq__(self, other):
        # Currently, this does not account for exchange, unlike other comparators
        if not isinstance(other, MoneyArray):
            return NotImplemented
        mask = self.isna() | other.isna()
        result = self.data == other.data
        result[mask] = False
        return result

    def __lt__(self, other):
        if not isinstance(other, MoneyArray):
            return NotImplemented
        mask = self.isna() | other.isna()
        same = (self.data['cu'] == other.data['cu']) | mask
        result = (self.data['va'] < other.data['va'])
        for i, ceq in enumerate(same):
            if not ceq:
                result[i] = money.XMoney(*self.data[i]) < money.XMoney(*self.other[i])

        result[mask] = False
        return result

    def __le__(self, other):
        if not isinstance(other, MoneyArray):
            return NotImplemented
        mask = self.isna() | other.isna()
        same = (self.data['cu'] == other.data['cu']) | mask
        result = (self.data['va'] < other.data['va'])
        for i, ceq in enumerate(same):
            if not ceq:
                result[i] = money.XMoney(*self.data[i]) < money.XMoney(*self.other[i])

        result[mask] = False
        return result

    def __gt__(self, other):
        if not isinstance(other, MoneyArray):
            return NotImplemented
        mask = self.isna() | other.isna()
        same = (self.data['cu'] == other.data['cu']) | mask
        result = (self.data['va'] > other.data['va'])
        for i, ceq in enumerate(same):
            if not ceq:
                result[i] = money.XMoney(*self.data[i]) > money.XMoney(*self.other[i])

        result[mask] = False
        return result

    def __ge__(self, other):
        if not isinstance(other, MoneyArray):
            return NotImplemented
        mask = self.isna() | other.isna()
        same = (self.data['cu'] == other.data['cu']) | mask
        result = (self.data['va'] >= other.data['va'])
        for i, ceq in enumerate(same):
            if not ceq:
                result[i] = money.XMoney(*self.data[i]) < money.XMoney(*self.other[i])

        result[mask] = False
        return result

    def equals(self, other):
        if not isinstance(other, MoneyArray):
            raise TypeError("Cannot compare 'MoneyArray' "
                            "to type '{}'".format(type(other)))
        # TODO: missing
        return (self.data == other.data).all()

    _formatting_values = None
    def _formatter(self, boxed=False):
        def fmt(x):
            if isinstance(x, money.XMoney):
                return str(x)
            elif not x:
                return "NA"
        return fmt

    def _values_for_factorize(self):
        return self.astype(object), (0, '')

    def to_currency(self, money_code, shallow=True, in_place=False):
        if shallow:
            if in_place:
                copy = self
            else:
                copy = self.copy()
            copy.default_money_code = money_code
        else:
            mask = self.isna()
            same = (self.data['cu'] == money_code) | mask
            decimalize = np.vectorize(decimal.Decimal)

            result = self.data
            if not in_place:
                result = result.copy()

            for i, ceq in enumerate(same):
                if not ceq:
                    va = money.XMoney(self.data[i]['va'], self.data[i]['cu']) \
                           .to(money_code).amount
                    result[i] = (va, money_code)

            if in_place:
                self.data = result
            copy = self.__class__(
                result,
                default_money_code=money_code,
                dtype=self.dtype
            )

        return copy

# -----------------------------------------------------------------------------
# Accessor
# -----------------------------------------------------------------------------

@pd.api.extensions.register_series_accessor("money")
class MoneyAccessor:

    isna = DelegatedMethod("isna")

    def __init__(self, obj):
        self._validate(obj)
        self._data = obj.values
        self._index = obj.index
        self._name = obj.name

    @staticmethod
    def _validate(obj):
        if not is_money_type(obj):
            raise AttributeError("Cannot use 'money' accessor on objects of "
                                 "dtype '{}'.".format(obj.dtype))

    def to_currency(self, money_code, shallow=True, in_place=True):
        return delegated_method(
            self._data.to_currency,
            self._index,
            self._name,
            money_code,
            shallow,
            in_place
        )

def is_money_type(obj):
    t = getattr(obj, 'dtype', obj)
    try:
        return isinstance(t, MoneyType) or issubclass(t, MoneyType)
    except Exception:
        return False

max_stars_repo_name: albcunha/moneypandas

max_issues_repo_name: albcunha/moneypandas

max_forks_repo_name: albcunha/moneypandas


text:
# -*- coding: utf-8 -*-

"""
This file is designed to find numerical solution the non-linear equation through recursive method and Newton method.

Bibliograhpy: Numerical Method(计算方法) Damei Li(李大美), Wuhan University.
"""
import numpy as np

""" Recursive Method
 f(x) = 0 <=> x = g(x)

Notation:
g = lambda x: x**2-3*x**(1/2)+4 # Here input your recursive function
x0 = 1 # Your initial point
n = 10 # Your iteration times
residual = 0.001 # Your tolerance
"""
def Recursive_Method(g,x0,n,residual):
    x = np.zeros(n+1)
    x[0] = x0
    for index in range(1,n+1):
        x[index] = g(x[index-1])
        difference = x[index] - x[index-1]
        if np.abs(x[index]-x[index-1])<residual:
            print("Iteration %r: x=%r, difference=%r <= Residual=%r" %(index,x[index],difference,residual))
            break
        else:
            print("Iteration %r: x=%r, difference=%r" %(index,x[index],difference))
    print("Terminal: The final result is %r" % x[index])
    return x[index]

""" Test
Recursive_Method(g,x0,n,residual)
g = lambda x: 0.5*(10-x**3)**0.5
Recursive_Method(g,1.5,10,10e-6)
"""

"""
    Newton Method
    Input your function f(x) and f'(x) in x

Notation:
f = lambda x: x**2+3*x+1 # Your function f(x)
f1 = lambda x: 2*x+3 # First order deviation of f(x)
x0 = 1 # The initial point
n = 100 # Iteration time
residual = 10e-6 # Tolerance
"""

def Newton_Recursive(f,f1,x0,n,residual):
    x = np.zeros(n+1)
    x[0] = x0
    for index in range(1,n+1):
        x[index] = x[index-1]-f(x[index-1])/f1(x[index-1])
        difference = x[index] - x[index-1]
        if np.abs(x[index]-x[index-1])<residual:
            print("Iteration %r: x=%r, difference=%r <= Residual=%r" %(index,x[index],difference,residual))
            break
        else:
            print("Iteration %r: x=%r, difference=%r" %(index,x[index],difference))
    print("Terminal: The final result is %r" % x[index])
    return x[index]

""" Test
Newton_Recursive(f,f1,x0,n,residual)
f = lambda x: x**3+4*x**2-10
f1 = lambda x: 3*x**2+8*x
Newton_Recursive(f,f1,1.5,10,10e-6)
"""

max_stars_repo_name: DickLiTQ/NumAnalysis

max_issues_repo_name: DickLiTQ/NumAnalysis

max_forks_repo_name: DickLiTQ/NumAnalysis


text:
from __future__ import division
import functools
from inspect import signature, Parameter
import os
import time
from glob import glob
import tensorflow as tf
import numpy as np
from six.moves import xrange

# from ops import *
# from utils import *

def auto_assign(func):
    # Signature:
    sig = signature(func)
    for name, param in sig.parameters.items():
        if param.kind in (Parameter.VAR_POSITIONAL, Parameter.VAR_KEYWORD):
            raise RuntimeError('Unable to auto assign if *args or **kwargs in signature.')
    # Wrapper:
    @functools.wraps(func)
    def wrapper(self, *args, **kwargs):
        for i, (name, param) in enumerate(sig.parameters.items()):
            # Skip 'self' param:
            if i == 0: continue
            # Search value in args, kwargs or defaults:
            if i - 1 < len(args):
                val = args[i - 1]
            elif name in kwargs:
                val = kwargs[name]
            else:
                val = param.default
            setattr(self, name, val)
        func(self, *args, **kwargs)
    return wrapper

class Model(object):
    @auto_assign
    def __init__(self, sess, config):
        """
        initialize
        """
        print("model initialize")

    def train(self, data):
        print("train", data)

    def test(self, data):
        print("test", data)

    def run(self, train_data, test_data):
        print("run")
        self.train(train_data)
        self.test(test_data)

    def save(self):
        print("save")
        # model_name = "DCGAN.model"
        # model_dir = "%s_%s_%s" % (self.dataset_name, self.batch_size, self.output_size)
        # checkpoint_dir = os.path.join(checkpoint_dir, model_dir)

        # if not os.path.exists(checkpoint_dir):
            # os.makedirs(checkpoint_dir)

        # self.saver.save(self.sess,
                        # os.path.join(checkpoint_dir, model_name),
                        # global_step=step)

    def load(self):
        print(" [*] Reading checkpoints...")

        # model_dir = "%s_%s_%s" % (self.dataset_name, self.batch_size, self.output_size)
        # checkpoint_dir = os.path.join(checkpoint_dir, model_dir)

        # ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
        # if ckpt and ckpt.model_checkpoint_path:
            # ckpt_name = os.path.basename(ckpt.model_checkpoint_path)
            # self.saver.restore(self.sess, os.path.join(checkpoint_dir, ckpt_name))
            # return True
        # else:
            # return False


text:
import pytest
import numpy as np
import tensorflow as tf
from sleap.nn.system import use_cpu_only; use_cpu_only()  # hide GPUs for test

from sleap.nn.data import providers
from sleap.nn.data import instance_centroids
from sleap.nn.data import instance_cropping
from sleap.nn.config import InstanceCroppingConfig

def test_normalize_bboxes():
    bbox = tf.convert_to_tensor([[0, 0, 3, 3]], tf.float32)
    norm_bbox = instance_cropping.normalize_bboxes(bbox, 9, 9)
    np.testing.assert_array_equal(norm_bbox, [[0, 0, 0.375, 0.375]])

    unnorm_bbox = instance_cropping.unnormalize_bboxes(norm_bbox, 9, 9)
    np.testing.assert_array_equal(unnorm_bbox, bbox)

def test_make_centered_bboxes():
    bbox = instance_cropping.make_centered_bboxes(
        tf.convert_to_tensor([[1, 1]], tf.float32), box_height=3, box_width=3)
    np.testing.assert_array_equal(bbox, [[0, 0, 2, 2]])

    bbox = instance_cropping.make_centered_bboxes(
        tf.convert_to_tensor([[2, 2]], tf.float32), box_height=4, box_width=4)
    np.testing.assert_array_equal(bbox, [[0.5, 0.5, 3.5, 3.5]])

def test_crop_bboxes():
    xv = tf.cast(tf.range(4), tf.uint8)
    yv = tf.cast(tf.range(5), tf.uint8)
    XX, YY = tf.meshgrid(xv, yv)
    img = tf.stack([XX, YY], axis=-1)

    centroids = tf.convert_to_tensor([[1, 1]], tf.float32)
    bboxes = instance_cropping.make_centered_bboxes(centroids,
        box_height=3, box_width=3)
    crops = instance_cropping.crop_bboxes(img, bboxes)

    patch_xx = [[0, 1, 2],
                [0, 1, 2],
                [0, 1, 2]]
    patch_yy = [[0, 0, 0],
                [1, 1, 1],
                [2, 2, 2]]
    expected_patch = np.expand_dims(np.stack([patch_xx, patch_yy], axis=-1), axis=0)
    np.testing.assert_array_equal(crops, expected_patch)
    np.testing.assert_array_equal(crops, np.expand_dims(img.numpy()[:3, :3, :], axis=0))
    assert crops.dtype == img.dtype

def test_crop_bboxes_rounding():
    # Test for rounding truncation bug when computing bounding box size for cropping.
    bboxes = instance_cropping.make_centered_bboxes(
        tf.cast([[464.42838, 550.14276]], tf.float32),
        box_height=100, box_width=100
    )
    crops = instance_cropping.crop_bboxes(
        tf.zeros([16, 16, 1], tf.float32),
        bboxes=bboxes
    )
    assert crops.shape == (1, 100, 100, 1)

def test_instance_cropper(min_labels):
    labels_reader = providers.LabelsReader(min_labels)
    instance_centroid_finder = instance_centroids.InstanceCentroidFinder(
        center_on_anchor_part=True, anchor_part_names="A",
        skeletons=labels_reader.labels.skeletons)
    instance_cropper = instance_cropping.InstanceCropper(
        crop_width=160, crop_height=160, keep_full_image=False)

    ds = instance_centroid_finder.transform_dataset(labels_reader.make_dataset())
    ds = instance_cropper.transform_dataset(ds)

    example = next(iter(ds))

    assert example["instance_image"].shape == (160, 160, 1)
    assert example["instance_image"].dtype == tf.uint8

    assert example["bbox"].shape == (4,)
    assert example["bbox"].dtype == tf.float32

    assert example["center_instance"].shape == (2, 2)
    assert example["center_instance"].dtype == tf.float32

    assert example["center_instance_ind"] == 0
    assert example["center_instance_ind"].dtype == tf.int32

    assert example["all_instances"].shape == (2, 2, 2)
    assert example["all_instances"].dtype == tf.float32

    assert example["centroid"].shape == (2,)
    assert example["centroid"].dtype == tf.float32

    assert example["full_image_height"] == 384
    assert example["full_image_height"].dtype == tf.int32

    assert example["full_image_width"] == 384
    assert example["full_image_width"].dtype == tf.int32

    assert example["video_ind"] == 0
    assert example["video_ind"].dtype == tf.int32

    assert example["frame_ind"] == 0
    assert example["frame_ind"].dtype == tf.int64

    np.testing.assert_array_equal(example["scale"], (1.0, 1.0))
    assert example["scale"].dtype == tf.float32

    assert example["skeleton_inds"].shape == (2,)
    assert example["skeleton_inds"].dtype == tf.int32

    assert "image" not in example

def test_instance_cropper_keeping_full_image(min_labels):
    labels_reader = providers.LabelsReader(min_labels)
    instance_centroid_finder = instance_centroids.InstanceCentroidFinder(
        center_on_anchor_part=True, anchor_part_names="A",
        skeletons=labels_reader.labels.skeletons)
    instance_cropper = instance_cropping.InstanceCropper(
        crop_width=160, crop_height=160, keep_full_image=True)

    ds = instance_centroid_finder.transform_dataset(labels_reader.make_dataset())
    ds = instance_cropper.transform_dataset(ds)

    example = next(iter(ds))

    assert example["instance_image"].shape == (160, 160, 1)
    assert example["instance_image"].dtype == tf.uint8

    assert example["full_image_height"] == 384
    assert example["full_image_height"].dtype == tf.int32

    assert example["full_image_width"] == 384
    assert example["full_image_width"].dtype == tf.int32

    assert example["image"].shape == (384, 384, 1)
    assert example["image"].dtype == tf.uint8

def test_instance_cropper_from_config():
    cropper = instance_cropping.InstanceCropper.from_config(
        config=InstanceCroppingConfig(crop_size=16),
        crop_size=None,
    )
    assert cropper.crop_width == 16
    assert cropper.crop_height == 16

    cropper = instance_cropping.InstanceCropper.from_config(
        config=InstanceCroppingConfig(crop_size=16),
        crop_size=24,
    )
    assert cropper.crop_width == 16
    assert cropper.crop_height == 16

    with pytest.raises(ValueError):
        cropper = instance_cropping.InstanceCropper.from_config(
            config=InstanceCroppingConfig(crop_size=None),
            crop_size=None,
        )


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2018/10/17 5:40
# @Author : {ZM7}
# @File : main.py
# @Software: PyCharm

from __future__ import division
import numpy as np
from model import *
from utils import build_graph, Data, split_validation
import pickle
import argparse
import datetime

parser = argparse.ArgumentParser()
parser.add_argument('--dataset', default='sample', help='dataset name: diginetica/yoochoose1_4/yoochoose1_64/sample')
parser.add_argument('--method', type=str, default='ggnn', help='ggnn/gat/gcn')
parser.add_argument('--validation', action='store_true', help='validation')
parser.add_argument('--epoch', type=int, default=30, help='number of epochs to train for')
parser.add_argument('--batchSize', type=int, default=100, help='input batch size')
parser.add_argument('--hiddenSize', type=int, default=100, help='hidden state size')
parser.add_argument('--l2', type=float, default=1e-5, help='l2 penalty')
parser.add_argument('--lr', type=float, default=0.001, help='learning rate')
parser.add_argument('--step', type=int, default=1, help='gnn propogation steps')
parser.add_argument('--nonhybrid', action='store_true', help='global preference')
parser.add_argument('--lr_dc', type=float, default=0.1, help='learning rate decay rate')
parser.add_argument('--lr_dc_step', type=int, default=3, help='the number of steps after which the learning rate decay')
opt = parser.parse_args()
train_data = pickle.load(open('../datasets/' + opt.dataset + '/train.txt', 'rb'))
test_data = pickle.load(open('../datasets/' + opt.dataset + '/test.txt', 'rb'))
# all_train_seq = pickle.load(open('../datasets/' + opt.dataset + '/all_train_seq.txt', 'rb'))
if opt.dataset == 'diginetica':
    n_node = 43098
elif opt.dataset == 'yoochoose1_64' or opt.dataset == 'yoochoose1_4':
    n_node = 37484
else:
    n_node = 310
# g = build_graph(all_train_seq)
train_data = Data(train_data, sub_graph=True, method=opt.method, shuffle=True)
test_data = Data(test_data, sub_graph=True, method=opt.method, shuffle=False)
model = GGNN(hidden_size=opt.hiddenSize, out_size=opt.hiddenSize, batch_size=opt.batchSize, n_node=n_node,
                 lr=opt.lr, l2=opt.l2,  step=opt.step, decay=opt.lr_dc_step * len(train_data.inputs) / opt.batchSize, lr_dc=opt.lr_dc,
                 nonhybrid=opt.nonhybrid)
print(opt)
best_result = [0, 0]
best_epoch = [0, 0]
for epoch in range(opt.epoch):
    print('epoch: ', epoch, '===========================================')
    slices = train_data.generate_batch(model.batch_size)
    fetches = [model.opt, model.loss_train, model.global_step]
    print('start training: ', datetime.datetime.now())
    loss_ = []
    for i, j in zip(slices, np.arange(len(slices))):
        adj_in, adj_out, alias, item, mask, targets = train_data.get_slice(i)
        _, loss, _ = model.run(fetches, targets, item, adj_in, adj_out, alias,  mask)
        loss_.append(loss)
    loss = np.mean(loss_)
    slices = test_data.generate_batch(model.batch_size)
    print('start predicting: ', datetime.datetime.now())
    hit, mrr, test_loss_ = [], [],[]
    for i, j in zip(slices, np.arange(len(slices))):
        adj_in, adj_out, alias, item, mask, targets = test_data.get_slice(i)
        scores, test_loss = model.run([model.score_test, model.loss_test], targets, item, adj_in, adj_out, alias,  mask)
        test_loss_.append(test_loss)
        index = np.argsort(scores, 1)[:, -20:]
        for score, target in zip(index, targets):
            hit.append(np.isin(target - 1, score))
            if len(np.where(score == target - 1)[0]) == 0:
                mrr.append(0)
            else:
                mrr.append(1 / (20-np.where(score == target - 1)[0][0]))
    hit = np.mean(hit)*100
    mrr = np.mean(mrr)*100
    test_loss = np.mean(test_loss_)
    if hit >= best_result[0]:
        best_result[0] = hit
        best_epoch[0] = epoch
    if mrr >= best_result[1]:
        best_result[1] = mrr
        best_epoch[1]=epoch
    print('train_loss:\t%.4f\ttest_loss:\t%4f\tRecall@20:\t%.4f\tMMR@20:\t%.4f\tEpoch:\t%d,\t%d'%
          (loss, test_loss, best_result[0], best_result[1], best_epoch[0], best_epoch[1]))


text:
# Modified from: https://github.com/pliang279/LG-FedAvg/blob/master/models/Nets.py
# credit goes to: Paul Pu Liang

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Python version: 3.6

import torch
from torch import nn
import torch.nn.functional as F
from torchvision import models
import json
import numpy as np
from models.language_utils import get_word_emb_arr

class MLP(nn.Module):
    def __init__(self, dim_in, dim_hidden, dim_out):
        super(MLP, self).__init__()
        self.layer_input = nn.Linear(dim_in, 512)
        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(0)
        self.layer_hidden1 = nn.Linear(512, 256)
        self.layer_hidden2 = nn.Linear(256, 64)
        self.layer_out = nn.Linear(64, dim_out)
        self.softmax = nn.Softmax(dim=1)
        self.weight_keys = [['layer_input.weight', 'layer_input.bias'],
                            ['layer_hidden1.weight', 'layer_hidden1.bias'],
                            ['layer_hidden2.weight', 'layer_hidden2.bias'],
                            ['layer_out.weight', 'layer_out.bias']
                            ]

    def forward(self, x):
        x = x.view(-1, x.shape[1]*x.shape[-2]*x.shape[-1])
        x = self.layer_input(x)
        x = self.relu(x)
        x = self.layer_hidden1(x)
        x = self.relu(x)
        x = self.layer_hidden2(x)
        x = self.relu(x)
        x = self.layer_out(x)
        return self.softmax(x)

class CNNMnist(nn.Module):
    def __init__(self, args):
        super(CNNMnist, self).__init__()
        self.conv1 = nn.Conv2d(1, 64, kernel_size=5)
        self.conv2 = nn.Conv2d(64, 64, kernel_size=5)
        self.conv2_drop = nn.Dropout2d()
        self.fc1 = nn.Linear(1024, 128)
        self.fc2 = nn.Linear(128, 64)
        self.fc3 = nn.Linear(64, args.num_classes)

    def forward(self, x):
        x = F.relu(F.max_pool2d(self.conv1(x), 2))
        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
        x = x.view(-1, x.shape[1]*x.shape[2]*x.shape[3])
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return F.log_softmax(x, dim=1)

class CNNCifar(nn.Module):
    def __init__(self, args):
        super(CNNCifar, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(64, 64, 5)
        self.fc1 = nn.Linear(64 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 64)
        self.fc3 = nn.Linear(64, args.num_classes)
        self.cls = args.num_classes

        self.weight_keys = [['fc1.weight', 'fc1.bias'],
                            ['fc2.weight', 'fc2.bias'],
                            ['fc3.weight', 'fc3.bias'],
                            ['conv2.weight', 'conv2.bias'],
                            ['conv1.weight', 'conv1.bias'],
                            ]

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 64 * 5 * 5)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.drop(self.fc3(x))
        return F.log_softmax(x, dim=1)

class CNNCifar100(nn.Module):
    def __init__(self, args):
        super(CNNCifar100, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.drop = nn.Dropout(0.6)
        self.conv2 = nn.Conv2d(64, 128, 5)
        self.fc1 = nn.Linear(128 * 5 * 5, 256)
        self.fc2 = nn.Linear(256, 128)
        self.fc3 = nn.Linear(128, args.num_classes)
        self.cls = args.num_classes

        self.weight_keys = [['fc1.weight', 'fc1.bias'],
                            ['fc2.weight', 'fc2.bias'],
                            ['fc3.weight', 'fc3.bias'],
                            ['conv2.weight', 'conv2.bias'],
                            ['conv1.weight', 'conv1.bias'],
                            ]

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 128 * 5 * 5)
        x = F.relu(self.fc1(x))
        x = self.drop((F.relu(self.fc2(x))))
        x = self.fc3(x)
        return F.log_softmax(x, dim=1)

class CNN_FEMNIST(nn.Module):
    def __init__(self, args):
        super(CNN_FEMNIST, self).__init__()
        self.conv1 = nn.Conv2d(1, 4, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(4, 12, 5)
        self.fc1 = nn.Linear(12 * 4 * 4, 120)
        self.fc2 = nn.Linear(120, 100)
        self.fc3 = nn.Linear(100, args.num_classes)

        self.weight_keys = [['fc1.weight', 'fc1.bias'],
                            ['fc2.weight', 'fc2.bias'],
                            ['fc3.weight', 'fc3.bias'],
                            ['conv2.weight', 'conv2.bias'],
                            ['conv1.weight', 'conv1.bias'],
                            ]

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 12 * 4 * 4)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return F.log_softmax(x, dim=1)

class RNNSent(nn.Module):
    """
    Container module with an encoder, a recurrent module, and a decoder.
    Modified by: Hongyi Wang from https://github.com/pytorch/examples/blob/master/word_language_model/model.py
    """

    def __init__(self,args, rnn_type, ntoken, ninp, nhid, nlayers, dropout=0.5, tie_weights=False, emb_arr=None):
        super(RNNSent, self).__init__()
        VOCAB_DIR = 'models/embs.json'
        emb, self.indd, vocab = get_word_emb_arr(VOCAB_DIR)
        self.encoder = torch.tensor(emb).to(args.device)

        if rnn_type in ['LSTM', 'GRU']:
            self.rnn = getattr(nn, rnn_type)(ninp, nhid, nlayers)
        else:
            try:
                nonlinearity = {'RNN_TANH': 'tanh', 'RNN_RELU': 'relu'}[rnn_type]
            except KeyError:
                raise ValueError( """An invalid option for `--model` was supplied,
                                 options are ['LSTM', 'GRU', 'RNN_TANH' or 'RNN_RELU']""")
            self.rnn = nn.RNN(ninp, nhid, nlayers, nonlinearity=nonlinearity, dropout=dropout)
        self.fc = nn.Linear(nhid, 10)
        self.decoder = nn.Linear(10, ntoken)

        # "Using the Output Embedding to Improve Language Models" (Press & Wolf 2016)
        # https://arxiv.org/abs/1608.05859
        # and
        # "Tying Word Vectors and Word Classifiers: A Loss Framework for Language Modeling" (Inan et al. 2016)
        # https://arxiv.org/abs/1611.01462
        if tie_weights:
            if nhid != ninp:
                raise ValueError('When using the tied flag, nhid must be equal to emsize')
            self.decoder.weight = self.encoder.weight

        self.drop = nn.Dropout(dropout)
        self.init_weights()
        self.rnn_type = rnn_type
        self.nhid = nhid
        self.nlayers = nlayers
        self.device = args.device

    def init_weights(self):
        initrange = 0.1
        self.fc.bias.data.zero_()
        self.fc.weight.data.uniform_(-initrange, initrange)
        self.decoder.bias.data.zero_()
        self.decoder.weight.data.uniform_(-initrange, initrange)

    def forward(self, input, hidden):
        input = torch.transpose(input, 0,1)
        emb = torch.zeros((25,4,300))
        for i in range(25):
            for j in range(4):
                emb[i,j,:] = self.encoder[input[i,j],:]
        emb = emb.to(self.device)
        emb = emb.view(300,4,25)
        self.rnn.flatten_parameters()
        output, hidden = self.rnn(emb, hidden)
        output = self.drop(F.relu(self.fc(output)))
        decoded = self.decoder(output[-1,:,:])
        return decoded.t(), hidden

    def init_hidden(self, bsz):
        weight = next(self.parameters())
        if self.rnn_type == 'LSTM':
            return (weight.new_zeros(self.nlayers, bsz, self.nhid),
                    weight.new_zeros(self.nlayers, bsz, self.nhid))
        else:
            return weight.new_zeros(self.nlayers, bsz, self.nhid)

max_stars_repo_name: Clej/FedRep

max_issues_repo_name: Clej/FedRep

max_forks_repo_name: Clej/FedRep


text:
import argparse
import csv
import os

import numpy as np

from constants import PROJECT_HOME
from data.preprocess import restore_correlation_from_min_max_normalize
from model.predict import predict_last_row, predict_last_row_using_latest_model

def _main():
    parser = argparse.ArgumentParser(description='Arguments for predicting.')
    parser.add_argument('-m', '--model', type=str, dest='model',
                        help='The name of the model to be trained.', default=None, required=False)
    args = parser.parse_args()
    print('Start predicting.')
    if args.model is not None:
        result = predict_last_row(model_name=args.model)[0]
    else:
        result = predict_last_row_using_latest_model()[0]
    print('Done.')
    result = np.reshape(result, newshape=(19900,))
    print()
    print('Result:')
    print('Before restore:')
    print('Min:', np.min(result), 'Max:', np.max(result))
    result = restore_correlation_from_min_max_normalize(result, does_overwrite=True)
    print('After restore:')
    print('Min:', np.min(result), 'Max:', np.max(result))
    with open(os.path.join(PROJECT_HOME, 'submit.csv'), mode='w', newline='') as f:
        writer = csv.writer(f)
        writer.writerow(['ID', 'value'])
        idx = 0
        for fund1_no in range(199):
            for fund2_no in range(fund1_no+1, 200):
                writer.writerow(['Fund %d-Fund %d' % (fund1_no+1, fund2_no+1), result[idx]])
                idx += 1

if __name__ == '__main__':
    _main()


text:
#!/usr/bin/env python

# Loads a file where each line contains a label, followed by a tab, followed
# by a sequence of words with a binary parse indicated by space-separated parentheses.
#
# Example:
# sentence_label    ( ( word word ) ( ( word word ) word ) )

import collections
import numpy as np
import sys

from spinn import util
from spinn.data.sst.base import convert_unary_binary_bracketed_data

SENTENCE_PAIR_DATA = False

LABEL_MAP = {
    "0": 0,
    "1": 1,
}

def load_data(path, vocabulary=None, seq_length=None, batch_size=32, eval_mode=False, logger=None):

    def keep_fn(label):
        if label == "2":
            return False
        return True

    def convert_fn(label):
        if label == "0" or label == "1":
            return "0"
        elif label == "3" or label == "4":
            return "1"
        else:
            raise ValueError("Bad Label: {}".format(label))

    dataset = convert_unary_binary_bracketed_data(path, keep_fn=keep_fn, convert_fn=convert_fn)
    return dataset, None

if __name__ == "__main__":
    pass

max_stars_repo_name: woojinchung/lms

max_issues_repo_name: woojinchung/lms

max_forks_repo_name: woojinchung/lms


text:
#!/usr/bin/env python
# coding: utf-8

# In[109]:

import networkx as nx
import matplotlib.pyplot as plt
from collections import defaultdict
# import re
# import pylab
import warnings
warnings.filterwarnings("ignore")
#unnamed(1).graphml
DAG = nx.read_graphml("unnamed.graphml")

# In[5]:

# DAG

# In[8]:

# from networkx.drawing.nx_pydot import graphviz_layout
# import pydot
# pos = graphviz_layout(DAG, prog="twopi")
# nx.draw(DAG, pos, with_labels=True)
# plt.show()

# In[17]:

# Этап 1
vertices_sorted = list(reversed(list(nx.lexicographical_topological_sort(DAG))))

# In[258]:

def get_neighbors(G, parent):
    parents = list(G.predecessors(parent))
    children = list(G.neighbors(parent))
    if not isinstance(G, nx.DiGraph) and parent is not None:
        children.remove(parent)
    return children, parents

# In[249]:

def get_max_P_vertex(v, sets):
#     return max(coffman_graham(DAG), key=coffman_graham(DAG).get)
    max_ = ["", 0]
    for i in sets:
        if v[i] > max_[1]:
            max_[0] = i
            max_[1] = v[i]
    return max_[0]

# In[307]:

def get_all_elements_on_layers(L):
    all_layers = set()
    for k, v in L.items():
        all_layers.update(v)
#     for i in range(k-1):
# #         print(L[i])
#         all_layers.update(L[i])
    return all_layers

# In[310]:

def coffman_graham(G, W=3):
    '''
    '''
    λ = {}
    vertices_sorted = list(reversed(list(nx.lexicographical_topological_sort(DAG))))

    V = set(vertices_sorted)

    for i, v in enumerate(vertices_sorted[::-1]):
        λ[v] = i+1

    k = 1 # Номер слоя
    U = set() # Множество изначально распределенных по слоям вершины
    L = defaultdict(list) # Множество вершин i-го слоя

    count = 0
    while U != V:
        v_sub_u = V.difference(U)
        u = get_max_P_vertex(λ, v_sub_u)
#         print(u)

        neighbors = get_neighbors(DAG, u)

        L_k = get_all_elements_on_layers(L)
        L_k = L_k.difference(set(L[k]))

        if len(L[k]) < W and set(neighbors[0]).issubset(L_k):
            L[k].append(u)
        else:
            k += 1
            L[k].append(u)
        U.add(u)
        # count += 1
#         break
    return λ, L

# In[311]:

v, L = coffman_graham(DAG)
print(L)

# In[313]:

##### list(nx.edge_dfs(DAG, "n11",orientation='reverse'))
# get_neighbors(DAG, "n11")

# In[154]:

# get_children(DAG, "n5")
# a = ['n11']
# L = defaultdict(list)
# get_all_elements_on_layers(L)

# In[260]:

# b = set(reversed(list(nx.lexicographical_topological_sort(DAG))))
# b

# In[ ]:

reason: import networkx,from networkx