text:
"""Helper code for creating/updating ``CaseVariantStats`` and related records."""

from django.db import transaction
import numpy as np

from projectroles.plugins import get_backend_api
from var_stats_qc.qc import compute_het_hom_chrx, compute_relatedness, compute_relatedness_many

from .forms import FILTER_FORM_TRANSLATE_EFFECTS
from .models import SmallVariant, CaseVariantStats, ProjectVariantStats

#: Effects to ignore when computing stats.
IGNORE_EFFECTS = (
    "downstream",
    "upstream",
    "intergenic",
    "non_coding_transcript_exon_variant",
    "non_coding_transcript_intron_variant",
)

#: Transition pairs
TRANSITIONS = (("A", "G"), ("G", "A"), ("C", "T"), ("T", "C"))

def _get_dp_bin(dp):
    """Returns lower bin width for the given depth.

    Bin width

    - 0..19: 1bp
    - 20..49: 5bp
    - 50..199: 10bp
    - 200..: = 200
    """
    if dp < 20:
        return dp
    elif dp < 50:
        return (dp // 2) * 2
    elif dp < 200:
        return (dp // 5) * 5
    else:
        return 200

def gather_variant_stats(variant_set):
    """Iterate over ``SmallVariant`` objects of ``variant_set`` and collect various statistics."""
    # TODO: could be refactored into class with multiple smaller, easier-to-read functions
    samples = variant_set.case.get_members_with_samples()
    transitions = {name: 0 for name in samples}
    transversions = {name: 0 for name in samples}
    snvs = {name: 0 for name in samples}
    indels = {name: 0 for name in samples}
    mnvs = {name: 0 for name in samples}
    effect_counts = {
        name: {effect: 0 for effect in FILTER_FORM_TRANSLATE_EFFECTS.values()} for name in samples
    }
    max_indel_size = 10
    indel_sizes = {name: {} for name in samples}
    read_depths = {name: {} for name in samples}
    dps = {name: [] for name in samples}
    hets = {name: 0 for name in samples}
    homs = {name: 0 for name in samples}

    ignore_set = set(IGNORE_EFFECTS)
    for small_var in SmallVariant.objects.filter(
        set_id=variant_set.pk, case_id=variant_set.case.pk
    ):
        if not (set(small_var.ensembl_effect) & ignore_set):
            for sample in samples:
                if small_var.genotype[sample]["gt"].count("1") == 1:
                    hets[sample] += 1
                else:
                    homs[sample] += 1
                dps[sample].append(small_var.genotype[sample]["dp"])
                bin = _get_dp_bin(small_var.genotype[sample]["dp"])
                read_depths[sample].setdefault(bin, 0)
                read_depths[sample][bin] += 1
                for effect in small_var.ensembl_effect:
                    effect_counts[sample][effect] += small_var.genotype[sample]["gt"].count("1")
            if small_var.var_type == "snv":
                for sample in samples:
                    snvs[sample] += small_var.genotype[sample]["gt"].count("1")
                if (small_var.reference, small_var.alternative) in TRANSITIONS:
                    for sample in samples:
                        transitions[sample] += small_var.genotype[sample]["gt"].count("1")
                else:
                    for sample in samples:
                        transversions[sample] += small_var.genotype[sample]["gt"].count("1")
            elif small_var.var_type == "mnvs":
                for sample in samples:
                    mnvs[sample] += small_var.genotype[sample]["gt"].count("1")
            elif small_var.var_type == "indel":
                for sample in samples:
                    count = small_var.genotype[sample]["gt"].count("1")
                    indels[sample] += count
                    delta = len(small_var.reference) - len(small_var.alternative)
                    if delta > max_indel_size:
                        delta = max_indel_size
                    elif delta < -max_indel_size:
                        delta = -max_indel_size
                    indel_sizes[sample].setdefault(delta, 0)
                    indel_sizes[sample][delta] += count

    dp_quantiles = {}
    for sample in samples:
        if dps[sample]:
            arr = np.asarray(dps[sample])
            dp_quantiles[sample] = list(np.percentile(arr, [0, 25, 50, 75, 100]))
        else:
            dp_quantiles[sample] = [0] * 5

    het_ratio = {}
    for sample in samples:
        if hets[sample] + homs[sample]:
            het_ratio[sample] = hets[sample] / (hets[sample] + homs[sample])
        else:
            het_ratio[sample] = 0.0

    return (
        transitions,
        transversions,
        snvs,
        indels,
        mnvs,
        effect_counts,
        indel_sizes,
        read_depths,
        dp_quantiles,
        het_ratio,
    )

def rebuild_case_variant_stats(engine, variant_set, logger=lambda _: None):
    """Rebuild the ``CaseVariantStats`` for the given ``SmallVariantSet`` using the SQL Alchemy ``connection``."""
    # Compute statistics.
    logger("... compute relatedness")
    het, het_shared, ibs0, ibs1, ibs2 = compute_relatedness(engine, SmallVariant, variant_set)
    logger("... compute het hom ratio on chromosome X")
    chrx_het_hom = compute_het_hom_chrx(engine, SmallVariant, variant_set)
    logger("... gather variant stats")
    (
        transitions,
        transversions,
        snvs,
        indels,
        mnvs,
        effect_counts,
        indel_sizes,
        read_depths,
        dp_quantiles,
        het_ratio,
    ) = gather_variant_stats(variant_set)

    # Rebuild the case variant statistics atomically.
    with transaction.atomic():
        # Remove existing record if any.
        logger("... delete old case variant stats if available")
        try:
            variant_set.variant_stats.delete()
            logger("... (deletion successful)")
        except CaseVariantStats.DoesNotExist:
            logger("... (none found)")
            pass  # swallow, nothing to delete

        # Create statistics object.
        logger("... create new case variant stats object")
        stats = CaseVariantStats.objects.create(variant_set=variant_set)
        # Insert basic information.
        for sample in variant_set.case.get_members_with_samples():
            logger("... creating sample variant stats object for sample %s" % sample)
            stats.sample_variant_stats.create(
                sample_name=sample,
                ontarget_transitions=transitions[sample],
                ontarget_transversions=transversions[sample],
                ontarget_snvs=snvs[sample],
                ontarget_indels=indels[sample],
                ontarget_mnvs=mnvs[sample],
                ontarget_effect_counts=effect_counts[sample],
                chrx_het_hom=chrx_het_hom[sample],
                ontarget_indel_sizes=indel_sizes[sample],
                ontarget_dps=read_depths[sample],
                ontarget_dp_quantiles=dp_quantiles[sample],
                het_ratio=het_ratio[sample],
            )
        # Insert relatedness information
        for pair in het_shared.keys():
            logger("... create relatedness object for samples %s and %s" % (pair[0], pair[1]))
            stats.relatedness.create(
                sample1=pair[0],
                sample2=pair[1],
                het_1_2=het_shared[pair],
                het_1=het[pair[0]],
                het_2=het[pair[1]],
                n_ibs0=ibs0[pair],
                n_ibs1=ibs1[pair],
                n_ibs2=ibs2[pair],
            )
        return stats

def rebuild_project_variant_stats(engine, project, user, logger=lambda _: None):
    timeline = get_backend_api("timeline_backend")
    if timeline:
        tl_event = timeline.add_event(
            project=project,
            app_name="variants",
            user=user,
            event_name="project_stats_build",
            description="build project-wide variant statistics",
            status_type="INIT",
        )
    cases = project.case_set.all()
    with transaction.atomic():
        het, het_shared, ibs0, ibs1, ibs2 = compute_relatedness_many(engine, SmallVariant, cases)
    logger("Done computing relatedness, now saving to DB")
    try:
        with transaction.atomic():
            # Remove existing record if any.
            try:
                project.variant_stats.delete()
            except ProjectVariantStats.DoesNotExist:
                pass  # swallow, nothing to delete
            else:
                logger("Done removing old statistics")

            # Create statistics object.
            stats = ProjectVariantStats.objects.create(project=project)
            # Insert relatedness information
            for pair in het_shared.keys():
                stats.relatedness.create(
                    sample1=pair[0],
                    sample2=pair[1],
                    het_1_2=het_shared[pair],
                    het_1=het[pair[0]],
                    het_2=het[pair[1]],
                    n_ibs0=ibs0[pair],
                    n_ibs1=ibs1[pair],
                    n_ibs2=ibs2[pair],
                )
            if timeline:
                tl_event.set_status("OK", "finished storing new project-wide variant statistics")
            return stats
    except Exception as e:
        if timeline:
            tl_event.set_status(
                "FAILED", "could not update project-wide variant statistics: {}".format(e)
            )
        raise

def execute_rebuild_project_variant_stats_job(engine, stats_job):
    """Rebuild the ``ProjectVariantStats`` for the given ``project"""
    stats_job.mark_start()
    try:
        stats = rebuild_project_variant_stats(
            engine, stats_job.project, stats_job.bg_job.user, stats_job.add_log_entry
        )
        stats_job.mark_success()
        return stats
    except Exception as e:
        stats_job.mark_error("problem updating project-wide variant statistics: {}".format(e))
        raise

max_stars_repo_name: brand-fabian/varfish-server

max_issues_repo_name: brand-fabian/varfish-server

max_forks_repo_name: brand-fabian/varfish-server


text:
import scipy.io
from LearnCPDsGivenGraph import LearnCPDsGivenGraph
from ClassifyDataset import ClassifyDataset

mat = scipy.io.loadmat('PA8Data.mat')
G1 = mat['G1']

trainData = mat['trainData']
train_data_labels = trainData[0][0]
trainData_data = train_data_labels['data']
trainData_labels = train_data_labels['labels']
P1, likelihood1 = LearnCPDsGivenGraph(trainData_data, G1, trainData_labels)

testData = mat['testData']
test_data_labels = testData[0][0]
test_data = test_data_labels['data']
test_labels = test_data_labels['labels']

accuracy1 = ClassifyDataset(test_data, test_labels, P1, G1)

ext: py

reason: import scipy


text:
import sys
import time
import random
import pprint
import numpy as np
from math import inf
from typing import Any
from scipy.sparse import construct
from sklearn.model_selection import train_test_split

# TF
import tensorflow as tf
import tensorflow.keras.backend as K
from tensorflow.keras.layers import Input
from tensorflow.keras.models import Model
from tensorflow.keras.utils import Progbar
from tensorflow.keras.optimizers import Adam

# Internal
from keras_frcnn import config, data_generators
from keras_frcnn import losses as loss_funcs
import keras_frcnn.roi_helpers as roi_helpers
from keras_frcnn.simple_parser import get_data

# Limit GPU access if
gpus = tf.config.experimental.list_physical_devices('GPU')
if gpus:
  # Restrict TensorFlow to only allocate 8GB of memory on the first GPU
  try:
    tf.config.experimental.set_virtual_device_configuration(
        gpus[0],
        [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=8*1024)])
    logical_gpus = tf.config.experimental.list_logical_devices('GPU')
    print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
  except RuntimeError as e:
    # Virtual devices must be set before GPUs have been initialized
    print(e)

sys.setrecursionlimit(40000)

# pass the settings from the command line, and persist them in the config object
C = config.Config("./config.json")

# Ensure we have a train path specified
if not C.train_path:
	print(f"Invalid training path specified {C.train_path}")
	raise ValueError

# load model
nn: Any = None
num_features = 1024
if C.network == "vgg":
	from keras_frcnn import vgg
	nn = vgg
	num_features = 512
elif C.network == "resnet":
	from keras_frcnn import resnet
	nn = resnet
else:
	print(f"Not a valid model: {C.network}")
	raise ValueError

all_imgs, classes_count, class_mapping = get_data(C.train_path)
train_imgs, val_imgs = train_test_split(all_imgs, test_size=0.2, random_state=C.seed)

if 'bg' not in classes_count:
	classes_count['bg'] = 0
	class_mapping['bg'] = len(class_mapping)

C.class_mapping = class_mapping

print(class_mapping)

inv_map = {v: k for k, v in class_mapping.items()}

print('Training images per class:')
pprint.pprint(classes_count)
print(f'Num classes (including bg) = {len(classes_count)}')

random.shuffle(train_imgs)

num_imgs = len(train_imgs)

print(f'Num train samples {len(train_imgs)}')
print(f'Num val samples {len(val_imgs)}')

data_gen_train = data_generators.get_anchor_gt(
	train_imgs, classes_count, C,
	nn.get_img_output_length, "tf",
	mode='train'
)
data_gen_val = data_generators.get_anchor_gt(
	val_imgs, classes_count, C,
	nn.get_img_output_length,"tf",
	mode='val'
)

input_shape_img = (None, None, 3)
input_shape_features = (None, None, num_features)

img_input = Input(shape=input_shape_img)
roi_input = Input(shape=(C.num_rois, 4))
feature_map_input = Input(shape=input_shape_features)

# define the base network (resnet here, can be VGG, Inception, etc)
shared_layers = nn.nn_base(img_input, trainable=True)

# define the RPN, built on the base layers
num_anchors = len(C.anchor_box_scales) * len(C.anchor_box_ratios)
rpn = nn.rpn(shared_layers, num_anchors)
model_rpn = Model(img_input, rpn)

# Define the classifiers (we create one with a full feature map to be used during validation)
classifier_train = nn.classifier(shared_layers, roi_input, C.num_rois, nb_classes=len(classes_count), trainable=True)
model_classifier_train = Model([img_input, roi_input], classifier_train)

classifier_val = nn.classifier(feature_map_input, roi_input, C.num_rois, nb_classes=len(classes_count), trainable=True)
model_classifier_val = Model([feature_map_input, roi_input], classifier_val)

# this is a model that holds both the RPN and the classifier, used to load/save weights for the models
model_all = Model([img_input, roi_input], rpn + classifier_train)

try:
	print(f'loading weights from {C.base_net_weights}')
	model_rpn.load_weights(C.base_net_weights, by_name=True)
	model_classifier_train.load_weights(C.base_net_weights, by_name=True)
	model_classifier_val.load_weights(C.base_net_weights, by_name=True)
except:
	print('Could not load pretrained model weights. Weights can be found in the keras application folder \
		https://github.com/fchollet/keras/tree/master/keras/applications')

rpn_optimizer = Adam(learning_rate=1e-5)
class_optimizer = Adam(learning_rate=1e-5)

# RPN losses
rpn_cls_loss = loss_funcs.rpn_loss_cls(num_anchors)
rpn_regr_loss = loss_funcs.rpn_loss_regr(num_anchors)

# Class losses
class_cls_loss = loss_funcs.class_loss_cls
class_regr_loss = loss_funcs.class_loss_regr(len(classes_count)-1)

model_all.compile(optimizer='sgd', loss='mae')

epoch_length = C.epoch_length
num_epochs = C.epochs
iter_num = 0

losses = np.zeros((epoch_length, 5))
rpn_accuracy_for_epoch = []
start_time = time.time()

best_f1 = np.Inf

class_mapping_inv = {v: k for k, v in class_mapping.items()}
print('Starting training')

# METRICS
train_acc_metric = tf.keras.metrics.CategoricalAccuracy()

def train_step(X, Y, img_data):

	Y = [tf.cast(Y[0], 'float32'), tf.cast(Y[1], 'float32')]

	# Train the rpn model
	with tf.GradientTape() as tape:
		P_rpn = model_rpn(X)
		loss_rpn_0 = rpn_cls_loss(Y[0], P_rpn[0])
		loss_rpn_1 = rpn_regr_loss(Y[1], P_rpn[1])

	loss_rpn = [loss_rpn_0, loss_rpn_1]
	grads = tape.gradient([loss_rpn_0, loss_rpn_1], model_rpn.trainable_weights)
	rpn_optimizer.apply_gradients(zip(grads, model_rpn.trainable_weights))

	R = roi_helpers.rpn_to_roi(P_rpn[0].numpy(), P_rpn[1].numpy(), C, "tf", use_regr=True, overlap_thresh=0.7, max_boxes=300)
	# note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format
	X2, Y1, Y2, ious = roi_helpers.calc_iou(R, img_data, C, class_mapping)

	if X2 is None:
		rpn_accuracy_for_epoch.append(0)
		return loss_rpn, [], []

	neg_samples = np.where(Y1[0, :, -1] == 1)
	pos_samples = np.where(Y1[0, :, -1] == 0)

	if len(neg_samples[0]) > 0:
		neg_samples = list(neg_samples[0])
	else:
		neg_samples = []

	if len(pos_samples[0]) > 0:
		pos_samples = list(pos_samples[0])
	else:
		pos_samples = []

	rpn_accuracy_for_epoch.append(len(pos_samples))

	if C.num_rois > 1:
		if len(pos_samples) < C.num_rois//2:
			selected_pos_samples = pos_samples
		else:
			selected_pos_samples = list(np.random.choice(pos_samples, C.num_rois//2, replace=False))
		try:
			selected_neg_samples = list(np.random.choice(neg_samples, C.num_rois - len(selected_pos_samples), replace=False))
		except:
			if len(neg_samples) > 0:
				selected_neg_samples = list(np.random.choice(neg_samples, C.num_rois - len(selected_pos_samples), replace=True))
			else:
				selected_neg_samples = []

		sel_samples = selected_pos_samples + selected_neg_samples
	else:
		# in the extreme case where num_rois = 1, we pick a random pos or neg sample
		selected_pos_samples = pos_samples
		selected_neg_samples = neg_samples
		if np.random.randint(0, 2):
			sel_samples = random.choice(neg_samples)
		else:
			sel_samples = random.choice(pos_samples)

	# Train the classification model
	with tf.GradientTape() as tape:
		P_class = model_classifier_train([X, X2[:, sel_samples, :]])
		loss_class_0 = class_cls_loss(Y1[:, sel_samples, :], P_class[0]) # classifiation loss
		loss_class_1 = class_regr_loss(Y2[:, sel_samples, :], P_class[1]) # regression loss

	# Update classification accuracy metric
	train_acc_metric.update_state(P_class[0], Y1[:, sel_samples, :])

	class_metrics = [train_acc_metric.result()]
	loss_class = [loss_class_0, loss_class_1]

	# Apply to gradients to "training" classifier
	grads = tape.gradient([loss_class_0, loss_class_1], model_classifier_train.trainable_weights)
	class_optimizer.apply_gradients(zip(grads, model_classifier_train.trainable_weights))

	return loss_rpn, loss_class, class_metrics

# Get validation accuracy after each epoch
def computeValidation():

	# Precision and recall from each iteration
	precisions = []
	recalls = []

	for _ in range(10):

		# try:

			start = time.time()
			X, Y, img_data = next(data_gen_val)

			# RPN
			Y1, Y2, F = model_rpn.predict(X)

			R = roi_helpers.rpn_to_roi(Y1, Y2, C, "tf", overlap_thresh=0.7)

			# convert from (x1,y1,x2,y2) to (x,y,w,h)
			R[:, 2] -= R[:, 0]
			R[:, 3] -= R[:, 1]

			# apply the spatial pyramid pooling to the proposed regions
			bboxes: dict = {}
			probs: dict = {}

			for jk in range(R.shape[0]//C.num_rois + 1):
				ROIs = np.expand_dims(R[C.num_rois*jk:C.num_rois*(jk+1), :], axis=0)
				if ROIs.shape[1] == 0:
					break

				if jk == R.shape[0]//C.num_rois:
					#pad R
					curr_shape = ROIs.shape
					target_shape = (curr_shape[0],C.num_rois,curr_shape[2])
					ROIs_padded = np.zeros(target_shape).astype(ROIs.dtype)
					ROIs_padded[:, :curr_shape[1], :] = ROIs
					ROIs_padded[0, curr_shape[1]:, :] = ROIs[0, 0, :]
					ROIs = ROIs_padded

				[P_cls, P_regr] = model_classifier_val.predict([F, ROIs])

				new_boxes, new_probs = roi_helpers.preds_to_boxes(P_cls, P_regr, ROIs, 0.8, inv_map, C)

				for cls_name in new_boxes:

					# Get bounding boxes and probs for this class
					bboxes[cls_name] = bboxes.get(cls_name, []) + new_boxes[cls_name]
					probs[cls_name] = probs.get(cls_name, []) + new_probs[cls_name]

			for key in bboxes:
				bbox = np.array(bboxes[key])

				new_boxes, _ = roi_helpers.non_max_suppression_fast(bbox, np.array(probs[key]), overlap_thresh=0.5)
				bboxes[key] = new_boxes

			# Update classification precision and recall
			if len(bboxes) == 0:
				precisions.append(0.0)
				recalls.append(0.0)

			else:
				precision, recall = roi_helpers.calc_stats(bboxes, img_data, C, class_mapping)
				precisions.append(precision)
				recalls.append(recall)

		# except Exception as e:
		# 	print(f"WARNING: error in validation step -> ", e)

	# Get result and reset metric
	precision = np.mean(precisions)
	recall = np.mean(recalls)
	f1 = 2 * ( (precision * recall) / (precision + recall + 1e-7) )

	return precision, recall, f1

best_loss = inf
# TRAINING LOOP
for epoch_num in range(num_epochs):

	progbar = Progbar(epoch_length)
	print(f'\nEpoch {epoch_num + 1}/{num_epochs}')

	while True:

		# try:

			X, Y, img_data = next(data_gen_train)
			loss_rpn, loss_class, class_metrics = train_step(X, Y, img_data)

			if len(loss_class) == 0:
				continue

			losses[iter_num, 0] = loss_rpn[0]
			losses[iter_num, 1] = loss_rpn[1]

			losses[iter_num, 2] = loss_class[0]
			losses[iter_num, 3] = loss_class[1]

			progbar.update(iter_num+1, [('rpn_cls', losses[iter_num, 0]), ('rpn_regr', losses[iter_num, 1]),
										('detector_cls', losses[iter_num, 2]), ('detector_regr', losses[iter_num, 3])])

			iter_num += 1

			if iter_num == epoch_length:
				# Reset metric at the end of each epoch
				train_acc_metric.reset_states()

				loss_rpn_cls = np.mean(losses[:, 0])
				loss_rpn_regr = np.mean(losses[:, 1])
				loss_class_cls = np.mean(losses[:, 2])
				loss_class_regr = np.mean(losses[:, 3])
				class_acc = np.mean(class_metrics[0])
				val_precision, val_recall, val_f1 = computeValidation()

				mean_overlapping_bboxes = float(sum(rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch)
				rpn_accuracy_for_epoch = []

				if C.verbose:
					print(f'\nMean number of bounding boxes from RPN overlapping ground truth boxes: {mean_overlapping_bboxes}')
					print(f'Training accuracy for bounding boxes: {class_acc}')
					print(f'Validation precision for bounding boxes: {val_precision}')
					print(f'Validation recall for bounding boxes: {val_recall}')
					print(f'Validation f1 for bounding boxes: {val_f1}')
					print(f'Elapsed time: {time.time() - start_time}')

				curr_f1 = val_f1
				iter_num = 0
				start_time = time.time()

				# Use best val f1 for saving model
				if curr_f1 < best_f1:
					best_f1 = curr_f1
					model_all.save_weights(C.model_path)

				break

		# except Exception as e:
		# 	print("WARNING: error in training step -> ", e)
		# 	# model_all.save_weights("model_weights/emergency_model_frcnn.hdf5")
		# 	break

print('Training complete, exiting.')

max_stars_repo_name: dstolbert/keras-frcnn

max_issues_repo_name: dstolbert/keras-frcnn

max_forks_repo_name: dstolbert/keras-frcnn

reason: import numpy,from scipy


text:
# This script will swipe through the output of the data reduction pipeline,
# collect all the relevant information, and will write it to a LSS catalogue
# file.
#
# Current version has few columns and basically rewrites a zcat.fits file and a
# targ.fits file but this will change as the code evolves.
#
# Lado Samushia March, 2017 (colkhis@gmail.com)

# The following information will be written into the LSS catalogue fits file:
# target_id -- unique target_id
# object_type -- this is one out of the following list: (GALAXY, QSO)
# redshift
# ra -- right ascension
# dec -- declination
# redhsift_err -- redshift error
# redshift_warn -- redshift warning: 0 means ???, 4 means ???

from astropy.io import fits
from astropy.table import Table
import argparse
from datetime import date
import numpy as np
import os
from itertools import izip

# Parse arguments
# Two input files, one output file
# Input:
# zcat -- file containing measured redshifts
# targ -- file containing original targeting information
# Output:
# lsscat -- the output LSS catalogue
parser = argparse.ArgumentParser(description='Create a LSS catalogue.')
parser.add_argument('zcat',help='input redshift catalogue')
parser.add_argument('targ',help='input merged target list file')
parser.add_argument('lsscat',help='output LSS catalogue')
args = parser.parse_args()

if os.path.isfile(args.lsscat) == True:
    print("This LSScat file already exists. Can not overwrite.")
    print("Choose an alternative name for the output LSScat file.")
    quit()

print("Extract information from targets fits file")
targ_file = fits.open(args.targ)
targ = targ_file[1].gata

targ_target_id = targ.field('targetid')
ra = targ.field('ra')
dec = targ.field('dec')
# desi_target 1 = LRG, 2 = ELG, 4 = QSO
desi_target = targ.field('desi_target')
depth_r = targ.field('depth_r')
galdepth_r = targ.field('galdepth_r')
Ntarg = np.size(targ_target_id)
# The arrays below will be filled in from the zcat file
z = np.zeros(Ntarg)
z_err = np.zeros(Ntarg)
z_warn = np.zeros(Ntarg)
stype = np.array(['000000' for item in range(Ntarg)])
# Create a dictionary from target_id to position in array
print("Create dictionary")
tid_dict = dict(zip(targ_target_id,range(Ntarg)))

print("Extract information from zcat fits file")
zcat_file = fits.open(args.zcat)
zcat = zcat_file[1].data

zcat_target_id = zcat.field('targetid')
spec_type = zcat.field('spectype')
redshift = zcat.field('z')
redshift_err = zcat.field('zerr')
redshift_warn = zcat.field('zwarn')

print("select redshift and types")
for ID, st, zz, zz_err, zz_warn in izip(zcat_target_id, spec_type, redshift, redshift_err, redshift_warn):
    index = tid_dict[ID]
    stype[index] = st
    z[index] = zz
    z_err[index] = zz_err
    z_warn[index] = zz_warn

# Write information into LSS catalogue fits file
lss = Table([targ_target_id, ra, dec, desi_target, depth_r, galdepth_r, stype, z, z_err, z_warn], names=('targetid','ra','dec','desitarget','depthr','galdepthr','stype','z','z_err','z_warn'))
lss.meta['comment'] = 'DESI Large-Scale Structure Catalogue'
lss.meta['date'] = date.today()
lss.write(args.lsscat, format='fits')

max_stars_repo_name: echaussidon/LSS

max_issues_repo_name: echaussidon/LSS

max_forks_repo_name: echaussidon/LSS

reason: import numpy,from astropy


text:
from IProcess import IProcess, EDataType

from PIL import Image
import dtcwt
import numpy as np
import math
from dtcwt.utils import asfarray

import warnings

class DTCWaveletPic(IProcess):
	def __init__(self, level=1):
		IProcess.__init__(self)

		self.waveletLevel = level
		self.waveletLevelMax = level
		pass

	def getType(self):
		return EDataType.DTCWaveletPic

	def toId(self):
		return str(__class__.__name__)+"_DTWL"+str(self.waveletLevel)

	### Transforms imageData with given WaveletTransform into a DTCWaveletT.
	### Index is used to set the image, to make the transforms from.
	def do(self, imageData):

		warnings.filterwarnings("ignore")

		def warn(*args, **kwargs):
			pass

		warnings.warn = warn

		IProcess.do(self, imageData)
		self.data = [] # Delete previous data.

		processImage = imageData.data[-1].getdata()

		# Check for event length.
		(width, height) = processImage.size

		#print("Image Size: "+str(width)+" and "+str(height))

		if not width % 2 == 0 and not height % 2 == 0:
			processImage = processImage.resize((width - 1, height - 1))
		elif not width % 2 == 0:
			processImage = processImage.resize((width - 1, height))
		elif not height % 2 == 0:
			processImage = processImage.resize((width, height - 1))

		(width, height) = processImage.size

		processImageArr = np.array(processImage, np.uint8).reshape(height, width)

		#print("Image Shape: "+str(processImageArr.shape[0])+" and "+str(processImageArr.shape[1]))

		#coeffs2 = pywt.dwt2(processImage, self.waveletMode)
		transform = dtcwt.Transform2d()

		t = transform.forward(processImageArr, nlevels = self.waveletLevel)
		'''
		print(t.highpasses[1])
		print(t.lowpass)
		print(len(t.highpasses))
		print("|--------------|")

		print("Image Size: "+str(len(processImage)))
		print("HighPass Size: "+str(len(t.highpasses[0][0])))
		print("LowPass Size: "+str(len(t.lowpass[0])))
		print("HighPass Type: "+str(type(t.highpasses[0][0])))
		'''

		for level in reversed(range(self.waveletLevel)):
		    for slice_idx in reversed(range(t.highpasses[level].shape[2])):
		        highpass = np.abs(t.highpasses[level][:,:,slice_idx])

		        # Interpolate to grayscale values.
		        #highpass_interpolated = np.interp(highpass, [np.amin(highpass), np.amax(highpass)], [255, 0])

		        # Generate squared image result from highpass filter.
		        half_size = math.floor(math.sqrt(highpass.size) + 1)
		        slice_img = Image.frombuffer(self.imageMode, (half_size, half_size), highpass)

		        self.data.append(slice_img)

		# STUB: lowpass filter is not a result wich is wished, but ignored in further progressing.
		self.data.append(Image.fromarray(processImageArr))

		return self

reason: import numpy


text:
"""
Testing some fragments of the pipeline
"""

import unittest

import numpy as np

from demuxalot import Demultiplexer
from demuxalot.utils import FeatureLookup
from scipy.special import softmax

def test_feature_lookup(n_combinations=1000, n_samples=100000):
    feat1_comb = np.random.randint(0, 100, n_combinations)
    feat2_comb = np.random.randint(0, 1000, n_combinations)
    feat3_comb = np.random.randint(0, 100_000, n_combinations)

    samples_id = np.random.randint(0, n_combinations, n_samples)
    feat1 = feat1_comb[samples_id]
    feat2 = feat2_comb[samples_id]
    feat3 = feat3_comb[samples_id]

    lookup = FeatureLookup(feat1, feat2, feat3)
    compressed, counts = lookup.compress(feat1, feat2, feat3)
    assert compressed.max() < n_combinations
    assert np.allclose(counts, np.bincount(compressed, minlength=len(counts)))
    feat1_new, feat2_new, feat3_new = lookup.uncompress(compressed)
    assert np.allclose(feat1, feat1_new)
    assert np.allclose(feat2, feat2_new)
    assert np.allclose(feat3, feat3_new)

def test_doublet_penalties():
    for n_genotypes in [2, 3, 10]:
        for doublet_prob in [0., 0.25, 0.5]:
            doublet_logits = Demultiplexer._doublet_penalties(n_genotypes=n_genotypes, doublet_prior=doublet_prob)
            prior_probs = softmax(doublet_logits)

            assert np.allclose(prior_probs[:n_genotypes].sum(), (1 - doublet_prob))

max_stars_repo_name: herophilus/demuxalot

max_issues_repo_name: herophilus/demuxalot

max_forks_repo_name: herophilus/demuxalot


text:
# -*- coding: UTF-8 -*-
################################################################################
#
#   Copyright (c) 2020  Baidu, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#################################################################################
"""
本文件定义指标的相关类
"""

import numpy as np
from paddle.fluid import layers

from ddparser.parser.nets import nn

class Metric(object):
    def __init__(self, eps=1e-8):
        super(Metric, self).__init__()

        self.eps = eps
        self.total = 0.0
        self.correct_arcs = 0.0
        self.correct_rels = 0.0

    def __repr__(self):
        """repr"""
        return "UAS: {:6.2%} LAS: {:6.2%}".format(self.uas, self.las)

    def __call__(self, arc_preds, rel_preds, arc_golds, rel_golds, mask):
        """call"""
        arc_mask = nn.masked_select(arc_preds == arc_golds, mask)
        rel_mask = layers.logical_and(nn.masked_select(rel_preds == rel_golds, mask), arc_mask)
        self.total += len(arc_mask)
        self.correct_arcs += np.sum(arc_mask.numpy()).item()
        self.correct_rels += np.sum(rel_mask.numpy()).item()

    @property
    def score(self):
        """score"""
        return self.las

    @property
    def uas(self):
        """uas"""
        return self.correct_arcs / (self.total + self.eps)

    @property
    def las(self):
        """las"""
        return self.correct_rels / (self.total + self.eps)

    def __lt__(self, other):
        """lt"""
        return self.score < other

    def __le__(self, other):
        """le"""
        return self.score <= other

    def __ge__(self, other):
        """ge"""
        return self.score >= other

    def __gt__(self, other):
        """gt"""
        return self.score > other

max_stars_repo_name: ishine/DDParser

max_issues_repo_name: ishine/DDParser

max_forks_repo_name: ishine/DDParser


text:
#!/usr/bin/env python3

import argparse
import os
import sys
import json
import numpy as np

PROG = os.path.basename(sys.argv[0])

def main():
    parser = argparse.ArgumentParser(
        description='Fix a raman.json file created before 99e7a42a5 (June 14)',
    )
    parser.add_argument('INPUT', nargs='*')
    parser.add_argument(
        '--temperature', type=float,
        help="specify temperature, for double checking that the missing prefactor matches expectation.",
    )
    parser.add_argument(
        '--mistake-no', type=parse_mistake, default=1,
        help="which mistake defines our expectation?"
             " 1: first mistake (completely missing),"
             " 2: second mistake (missing sqrt),"
             " *: any of the above mistakes",
    )
    args = parser.parse_args()

    for path in args.INPUT:
        corrected_loc = path + '.corrected'

        with open(path) as f:
            d = json.load(f)

        frequencies = np.array(d['frequency'])
        correct_averages = np.array(d['average-3d'])
        recorded_tensors = np.array(d['raman-tensor'])
        actual_averages = np.sum(recorded_tensors**2, axis=(1,2)) / 9

        if np.allclose(correct_averages, actual_averages, rtol=1e-10, atol=0):
            continue

        missing_prefactors = correct_averages / actual_averages
        missing_prefactors[frequencies <= 0] = 0

        if args.temperature is not None:
            if check_expected_prefactors(frequencies, temperature=args.temperature, mistake=args.mistake_no, missing_prefactors=missing_prefactors):
                warn(f"{path} has missing prefactors that match expectation")
            else:
                warn(f"{path} has missing prefactors that DO NOT match expectation!!")
                # print(np.hstack([correct_averages[:, None], actual_averages[:, None]]), file=sys.stderr)
                # print(np.hstack([expected_prefactors[:, None], missing_prefactors[:, None]]), file=sys.stderr)

        else:
            warn(f"{path} has missing prefactors")

        warn(f"Writing {corrected_loc}")

        correct_tensors = recorded_tensors * np.sqrt(missing_prefactors[:, None, None])
        d['raman-tensor'] = correct_tensors.tolist()
        with open(corrected_loc, 'w') as f:
            json.dump(d, f)
            print(file=f)

ANY_MISTAKE = object()

def parse_mistake(s):
    if s == '*':
        return ANY_MISTAKE
    elif s in '12':
        return int(s)
    else:
        raise ValueError(f'invalid mistake: {repr(s)}')

def check_expected_prefactors(frequencies, temperature, missing_prefactors, mistake):
    if mistake is ANY_MISTAKE:
        return any(
            check_expected_prefactors(
                frequencies=frequencies,
                temperature=temperature,
                missing_prefactors=missing_prefactors,
                mistake=m,
            ) for m in [1, 2]
        )
    hk = 0.22898852319

    if temperature == 0:
        bose_occupation = 1
    else:
        expm1 = np.expm1(hk * frequencies / temperature)
        bose_occupation = (1.0 + 1.0 / expm1)
    prefactors = bose_occupation / frequencies

    if mistake == 1:
        expected_prefactors = np.where(frequencies <= 0.0, 0.0, prefactors)
    elif mistake == 2:
        expected_prefactors = np.where(frequencies <= 0.0, 0.0, prefactors ** -1)
    else:
        raise ValueError('mistake')

    return np.allclose(expected_prefactors, missing_prefactors, atol=0)

# ------------------------------------------------------

def warn(*args, **kw):
    print(f'{PROG}:', *args, file=sys.stderr, **kw)

def die(*args, code=1):
    warn('Fatal:', *args)
    sys.exit(code)

# ------------------------------------------------------

if __name__ == '__main__':
    main()

max_stars_repo_name: colin-daniels/agnr-ml

max_issues_repo_name: colin-daniels/agnr-ml

max_forks_repo_name: colin-daniels/agnr-ml


text:
import sys
import numpy as np
import math
class VisualTemplate:
    #instance=[]
    def __init__(self):
        self.id=0
        self.data=np.array([])
        self.mean=0
        #self.instance.append(self)

class LocalViewMatch:
    def __init__(self):
        self. VT_MIN_PATCH_NORMAISATION_STD=0.0
        self.VT_PATCH_NORMAISATION=0.0
        self.VT_NORMALISATION=0.0
        self.VT_SHIFT_MATCH=5.0
        self.VT_PANORAMIC=0.0
        self.VT_MATCH_THRESHOLD=0.03
        self.TEMPLATE_X_SIZE=60
        self.TEMPLATE_Y_SIZE=20
        self.IMAGE_VT_X_RANGE_MIN=0
        self.IMAGE_VT_X_RANGE_MAX=-1
        self.IMAGE_VT_Y_RANGE_MIN=0
        self.IMAGE_VT_Y_RANGE_MAX=150
        self.TEMPLATE_SIZE=self.TEMPLATE_X_SIZE*self.TEMPLATE_Y_SIZE
        self.current_mean=0
        #self.templates=np.array([])
        #self.templates.resize(10000)

        #for i in range(10000):
         #   self.templates[i]=x
        #self.templates.resize(10000)
        self.current_view=np.zeros(self.TEMPLATE_SIZE)
        self.current_view.resize(self.TEMPLATE_SIZE)
        self.xxx=0
        self.current_vt=0
        self.prev_vt=0
        self.DBL_MAX=sys.float_info.max
        self.VT_STEP_MATCH=1
        #self.view_rgb=""
        self.vt_match_id=0
        self.grayscale=False
        self.prev_vt=0

        self.vt_relative_rad=0

        self.vt_error=0
        self.VT_NORMALISATION=0.0
        self.VT_MIN_PATCH_NORMALISATION_STD=0.0
        self.VT_PANORAMIC=0

        self.templates=[]
        xxx=VisualTemplate
        xxx.id=0
        xxx.data=[]
        xxx.mean=0.0
        self.templates.append(xxx)
        self.data=np.array([])
        self.VT_SHIFT_MATCH=4
        self.vt_match_id=0
    def on_image(self,view_rgb,greyscale,image_width,image_height):

        if view_rgb==None:
            return
        self.IMAGE_WIDTH=image_width
        self.IMAGE_HEIGHT=image_height

        if self.IMAGE_VT_X_RANGE_MAX==-1:
            self.IMAGE_VT_X_RANGE_MAX=self.IMAGE_WIDTH
        if self.IMAGE_VT_Y_RANGE_MAX==-1:
            self.IMAGE_VT_Y_RANGE_MAX==self.IMAGE_HEIGHT

        self.view_rgb=view_rgb
        self.grayscale=greyscale

        self.convert_view_to_view_template(greyscale)
        self.prev_vt=self.get_current_vt()

        self.compare()
        if self.vt_error<=self.VT_MATCH_THRESHOLD:
            self.set_current_vt(int(self.vt_match_id))
            print "VTM[", self.get_current_vt(),"]","\n"

            sys.stdout.flush()
        else:
            self.vt_relative_rad=0
            self.set_current_vt(self.create_template())
            print "VTN[",self.get_current_vt(),"]","\n"
            sys.stdout.flush()

    def LocalViewMatch(self,x,y):
        self.vt_relative_rad=0
        if x<0:
            x=0
        elif x>self.TEMPLATE_X_SIZE-1:
            x=self.TEMPLATE_X_SIZE-1
        if y<0:
            y=0
        elif y>self.TEMPLATE_Y_SIZE-1:
            y=self.TEMPLATE_Y_SIZE-1

    def convert_view_to_view_template(self,grayscale):
        data_next=0
        sub_range_x=self.IMAGE_VT_X_RANGE_MAX-self.IMAGE_VT_X_RANGE_MIN
        sub_range_y=self.IMAGE_VT_Y_RANGE_MAX-self.IMAGE_VT_Y_RANGE_MIN

        x_block_size=sub_range_x/self.TEMPLATE_X_SIZE
        y_block_size=sub_range_y/self.TEMPLATE_Y_SIZE

        for i in range(len(self.current_view)):
            self.current_view[i]=0
        if self.grayscale==True:
            y_block=self.IMAGE_VT_Y_RANGE_MIN
            for y_block_cout in range(self.TEMPLATE_Y_SIZE):
                x_block=self.IMAGE_VT_X_RANGE_MIN
                for x_block_count in range(self.TEMPLATE_X_SIZE):

                    for x in range(x_block,x_block+x_block_size):
                        for y in range(y_block,y_block+y_block_size):
                            pos=x+y*self.IMAGE_WIDTH
                            self.current_view[data_next]+=float(self.view_rgb[pos])
                    self.current_view[data_next]/=255.0
                    self.current_view[data_next]/=x_block_size*y_block_size
                    data_next+=1
                    x_block+=x_block_size
                y_block+=y_block_size
        else:
            y_block=self.IMAGE_VT_Y_RANGE_MIN
            for y_block_count in range(self.TEMPLATE_Y_SIZE):
                x_block=self.IMAGE_VT_X_RANGE_MIN
                for x_block_count in range(self.TEMPLATE_X_SIZE):
                    for x in range(x_block,x_block+x_block_size):
                        for y in range(y_block,y_block+y_block_size):
                            pos=(x+y*self.IMAGE_WIDTH)*3
                            self.current_view[data_next]+=float(self.view_rgb[pos])+float(self.view_rgb[pos+1])+float(self.view_rgb[pos+2])
                    self.current_view[data_next]/=255.0*3.0
                    self.current_view[data_next]/=x_block_size*y_block_size
                    data_next+=1
                    x_block+=x_block_size
                y_block+=y_block_size
        if self.VT_NORMALISATION>0:
            avg_value=0
            for i in range(len(self.current_view)):
                avg_value+=self.current_view[i]
            avg_value/=len(self.current_view)
            for i in len(self.current_view):
                self.current_view[i]=max(0.0,min(self.current_view[i]*self.VT_NORMALISATION/avg_value,1))

        if self.VT_PATCH_NORMAISATION>0:
            patch_size=self.VT_PATCH_NORMAISATION
            patch_total=(patch_size*2+1)*(patch_size*2+1)

            current_view_copy=np.array([])
            current_view_copy.resize(len(self.current_view))
            for i in range(len(self.current_view)):
                current_view_copy[i]=self.current_view[i]
            for x in range(self.TEMPLATE_X_SIZE):
                for y in range(self.TEMPLATE_Y_SIZE):
                    patch_sum=0
                    for patch_x in range(x-patch_size,x+patch_size):
                        for patch_y in range(y-patch_size,y+patch_size+1):
                            patch_x_clip=patch_x
                            patch_y_clip=patch_y
                            self.clip_view_x_y(patch_x_clip,patch_y_clip)
                            patch_sum+=((current_view_copy[patch_x_clip+patch_y_clip*self.TEMPLATE_X_SIZE]-patch_mean)*\
                                (current_view_copy[patch_x_clip+patch_y_copy*self.TEMPLATE_X_SIZE]-patch_mean))

                    patch_std=math.sqrt(patch_sum/patch_total)
                    if patch_std<self.VT_MIN_PATCH_NORMALISATION_STD:
                        self.current_view[x+y*self.TEMPLATE_X_SIZE]=0.5
                    else:
                        self.current_view[x+y*self.TEMPLATE_X_SIZE]=max(0.0,min(1.0,(((self.current_view_copy[x+y*self.TEMPLATE_X_SIZE]-patch_mean)/patch_std)+3.0)/6.0))

        sum=0
        for i in range(len(self.current_view)):
            sum+=self.current_view[i]
        self.current_mean=sum/len(self.current_view)

    def create_template(self):

        vtt=VisualTemplate
        vtt.data=self.current_view
        vtt.mean=self.current_mean
        vtt.id=len(self.templates)
        self.templates.append(vtt)
        '''
        self.templates.append(VisualTemplate())
        data_ptr=self.current_view

        self.templates[len(self.templates)-1].data=np.array(self.TEMPLATE_SIZE)
        self.templates[len(self.templates)-1].data=data_ptr
        self.templates[len(self.templates)-1].mean=self.current_mean
        '''
        return len(self.templates)-1
    #def compare(self,vt_err,vt_match_id):

    def compare(self):

        if(len(self.templates)==0):
            vt_err=self.DBL_MAX
            self.vt_error=vt_err
            return

        self.data=self.current_view
        mindiff=self.DBL_MAX
        vt_err=self.DBL_MAX
        min_template=0
        vt=VisualTemplate()
        epsilon=0.005
        if (self.VT_PANORAMIC==True):
            for i in range(len(self.templates)):
                vt=self.templates[i]
                if(math.abs(self.current_mean-vt.mean)>self.VT_MATCH_THRESHOLD+epsilon):
                    continue

                offset=0
                while offset<self.TEMPLATE_X_SIZE:
                    cdiff=0
                    template_start_ptr=vt.data[0]+offset
                    column_start_ptr=self.data[0]
                    row_size=self.TEMPLATE_X_SIZE

                    sub_ros_size=self.TEMPLATE_X_SIZE-offset
                    template_row_ptr=offset
                    for column_row_ptr in range(len(self.data)):
                        template_ptr=template_row_ptr
                        for column_ptr in range(column_row_ptr,column_row_ptr+sub_ros_size):
                            cdiff+=math.abs(vt.data[template_ptr]-self.data[column_ptr])
                            template_ptr+=1
                        template_row_ptr+=row_size
                        if cdiff>mindiff:
                            break

                    template_start_ptr=0
                    column_start_ptr=self.TEMPLATE_SIZE-offset
                    row_size=self.TEMPLATE_X_SIZE
                    column_end_ptr=self.TEMPLATE_SIZE
                    sub_row_size=offset

                    template_row_ptr=template_start_ptr
                    column_row_ptr=column_start_ptr
                    while column_row_ptr<column_end_ptr:
                        column_ptr=column_row_ptr
                        template_ptr=template_row_ptr
                        while column_ptr<column_row_ptr+sub_row_size:
                            cdiff+=math.abs(self.data[column_ptr]-vt.data[template_ptr])
                            column_ptr+=1
                            template_ptr+=1
                        column_row_ptr+=row_size
                        template_row_ptr+=row_size
                        if cdiff>mindiff:
                            break

                    if cdiff<mindiff:
                        mindiff=cdiff
                        min_template=vt.id
                        min_offset=offset

                    #for i in range(column_start_ptr,column_end_ptr):
                self.vt_relative_rad=float(min_offset/self.TEMPLATE_X_SIZE*2.0*np.pi)
                if self.vt_relative_rad>np.pi:
                    self.vt_relative_rad=self.vt_relative_rad-2.0*np.pi
                vt_err=mindiff/float(self.TEMPLATE_SIZE)
                self.vt_match_id=min_template

        else:

            for i in range(len(self.templates)-1):
                vt=self.templates[i]

                if (abs(self.current_mean-vt.mean)>self.VT_MATCH_THRESHOLD+epsilon):
                    continue
                offset=0

                while offset<self.VT_SHIFT_MATCH*2+1:
                    cdiff=0
                    template_start_ptr=offset
                    column_start_ptr=self.VT_SHIFT_MATCH
                    row_size=self.TEMPLATE_X_SIZE
                    column_end_ptr=self.TEMPLATE_SIZE-self.VT_SHIFT_MATCH
                    sub_row_size=self.TEMPLATE_X_SIZE-2*self.VT_SHIFT_MATCH

                    column_row_ptr=column_start_ptr
                    template_row_ptr=template_start_ptr

                    while column_row_ptr<column_end_ptr:
                        template_ptr=template_row_ptr
                        for column_ptr in range(column_row_ptr,column_row_ptr+sub_row_size):
                            cdiff+=abs(self.data[column_ptr]-vt.data[template_ptr])
                            template_ptr+=1
                        column_row_ptr+=row_size
                        template_row_ptr+=row_size
                        if cdiff>mindiff:
                            break
                    if cdiff<mindiff:
                        mindiff=cdiff
                        min_template=vt.id
                        min_offset=0
                    offset+=self.VT_STEP_MATCH

            self.vt_relative_rad=0

            vt_err=mindiff/float(self.TEMPLATE_SIZE-2*self.VT_SHIFT_MATCH*self.TEMPLATE_Y_SIZE)
            self.vt_match_id=min_template
            self.vt_error=vt_err
    def clip_view_x_y(self,x,y):
        if x<0:
            x=0
        elif x>self.TEMPLATE_X_SIZE-1:
            x=self.TEMPLATE_X_SIZE-1
        if y<0:
            y=0
        elif y>self.TEMPLATE_Y_SIZE-1:
            y=self.TEMPLATE_Y_SIZE-1

    def get_current_vt(self):
        return self.current_vt
    def get_relative_rad(self):
        return self.vt_relative_rad

    def set_current_vt(self,id):
        if self.current_vt!=id:
            self.prev_vt=self.current_view
        self.current_vt=id

max_stars_repo_name: raouiyounes/ratslamPython

max_issues_repo_name: raouiyounes/ratslamPython

max_forks_repo_name: raouiyounes/ratslamPython


text:
import xarray as xr
import atmPy.clouds.ceilometry as _ceilometry
import pandas as _pd
import numpy as np
# def read_ceilometer_nc(fname):
#     ds = xr.open_dataset(fname)
#     ceil = _ceilometry.Ceilometer()
#     ceil.backscatter = _ceilometry.Backscatter(ds.backscatter.to_pandas())
#     return ceil

def read_ceilometer_nc(fname, timezone = None, keep_xr_dataset = False):

    if type(fname) == str:
        fname = [fname]

    bs = []
    cloudbases = []
    for fn in fname:
        ds = xr.open_dataset(fn)
        bs.append(ds.backscatter.to_pandas())

        fcb = ds.first_cbh.to_pandas()
        scb = ds.second_cbh.to_pandas()
        tcb = ds.third_cbh.to_pandas()
        cloudbases.append(_pd.DataFrame({'First_cloud_base': fcb,
                      'Second_cloud_base': scb,
                      'Third_cloud_base': tcb}))

    bsdf = _pd.concat(bs).sort_index()
    cbdf = _pd.concat(cloudbases).sort_index()

    if not isinstance(timezone, type(None)):
        bsdf.index += np.timedelta64(timezone, 'h')
        cbdf.index += np.timedelta64(timezone, 'h')

    ceil = _ceilometry.Ceilometer()
    ceil.backscatter = bsdf
    ceil.cloudbase = cbdf

    if keep_xr_dataset:
        ceil.xr_dataset = ds
    return ceil

max_stars_repo_name: wblumberg/atm-py

max_issues_repo_name: wblumberg/atm-py

max_forks_repo_name: wblumberg/atm-py


text:
import numpy as np
import sys
import os
import time

syspath = os.path.dirname(os.path.realpath(__file__)) + "/.."
sys.path.insert(0, syspath)

from vbsw_module.algorithms.vbsw import vbsw
from vbsw_module.algorithms.training import training
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_boston

case_name="boston_paper_results"
import datetime
if os.path.isdir(syspath + "/results/" + case_name):
    listdir = os.listdir(syspath + "/results/" + case_name)
    if len(listdir) > 0:
        if not os.path.isdir(syspath + "/results/" + case_name + "/old"):
            os.makedirs(syspath + "/results/" + case_name + "/old")
        today = str(datetime.datetime.today()).replace(" ", "")
        os.makedirs(syspath + "/results/" + case_name + "/old/" + today)
        for file in listdir:
            if file != "old":
                os.system("mv ~/scratch/results/" + case_name + "/" + file + " ~/scratch/results/" + case_name + "/old/" + today)

### Load the data
x, y = load_boston(return_X_y=True)
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
y_train = np.reshape(y_train, (y_train.shape[0], 1))
y_test = np.reshape(y_test, (y_test.shape[0], 1))

### Preprocess the data
max_x = np.max(x_train, axis=0)
x_train /= max_x
x_test /= max_x

### Train a linear model with VBSW
model_vbsw = vbsw(training_set=(x_train, y_train),
                  test_set=(x_test, y_test),
                  ratio=8,
                  N_stat=35,
                  N_seeds=int(sys.argv[-1]),
                  N_layers=0,
                  N_units=0,
                  activation_hidden="",
                  activation_output="linear",
                  batch_size=x_train.shape[0],
                  epochs=50000,
                  optimizer="adam",
                  learning_rate=1e-3,
                  loss_function="mse",
                  test_losses=["mse"],
                  keep_best=True,
                  saving_period=100,
                  case_name=case_name,
                  dataset="boston")

### Train a linear model without VBSW
save_dir = syspath + "/results/" + case_name
res_name = "res_" + str(os.getpid()) + str(time.time())[:10]
save_file = os.path.join(save_dir, res_name)

model = training(training_set=(x_train, y_train),
                 test_set=(x_test, y_test),
                 N_layers=0,
                 N_units=0,
                 activation_hidden="",
                 activation_output="linear",
                 N_seeds=int(sys.argv[-1]),
                 batch_size=x_train.shape[0],
                 epochs=50000,
                 optimizer="adam",
                 learning_rate=1e-3,
                 loss_function="mse",
                 test_losses=["mse"],
                 keep_best=True,
                 saving_period=100,
                 save_file=save_file)

max_stars_repo_name: paulnovello/vbsw

max_issues_repo_name: paulnovello/vbsw

max_forks_repo_name: paulnovello/vbsw


text:
# -*- coding: utf-8 -*-
# This file as well as the whole tsfresh package are licenced under the MIT licence (see the LICENCE.txt)
# Maximilian Christ (maximilianchrist.com), Blue Yonder Gmbh, 2016
"""
This module contains the main function to interact with tsfresh: extract features
"""

from __future__ import absolute_import, division

import warnings
from builtins import str
from multiprocessing import Pool
from functools import partial
from six.moves.queue import Queue
import logging
import pandas as pd
import numpy as np

from tqdm import tqdm

from tsfresh.utilities import dataframe_functions, helper_functions, profiling
from tsfresh.feature_extraction.settings import FeatureExtractionSettings

_logger = logging.getLogger(__name__)

def extract_features(timeseries_container, feature_extraction_settings=None,
                     column_id=None, column_sort=None, column_kind=None, column_value=None,
                     parallelization=None):
    """
    Extract features from

    * a :class:`pandas.DataFrame` containing the different time series

    or

    * a dictionary of :class:`pandas.DataFrame` each containing one type of time series

    In both cases a :class:`pandas.DataFrame` with the calculated features will be returned.

    For a list of all the calculated time series features, please see the
    :class:`~tsfresh.feature_extraction.settings.FeatureExtractionSettings` class,
    which is used to control which features with which parameters are calculated.

    For a detailed explanation of the different parameters and data formats please see :ref:`data-formats-label`.

    Examples
    ========

    >>> from tsfresh.examples import load_robot_execution_failures
    >>> from tsfresh import extract_features
    >>> df, _ = load_robot_execution_failures()
    >>> X = extract_features(df, column_id='id', column_sort='time')

    which would give the same results as described above. In this case, the column_kind is not allowed.
    Except that, the same rules for leaving out the columns apply as above.

    :param timeseries_container: The pandas.DataFrame with the time series to compute the features for, or a
            dictionary of pandas.DataFrames.
    :type timeseries_container: pandas.DataFrame or dict

    :param feature_extraction_settings: settings object that controls which features are calculated
    :type feature_extraction_settings: tsfresh.feature_extraction.settings.FeatureExtractionSettings

    :param column_id: The name of the id column to group by.
    :type column_id: str

    :param column_sort: The name of the sort column.
    :type column_sort: str

    :param column_kind: The name of the column keeping record on the kind of the value.
    :type column_kind: str

    :param column_value: The name for the column keeping the value itself.
    :type column_value: str

    :param parallelization: Either ``'per_sample'`` or ``'per_kind'``   , see
                            :func:`~tsfresh.feature_extraction.extraction._extract_features_parallel_per_sample`,
                            :func:`~tsfresh.feature_extraction.extraction._extract_features_parallel_per_kind` and
                            :ref:`parallelization-label` for details.
    :type parallelization: str

    :return: The (maybe imputed) DataFrame containing extracted features.
    :rtype: pandas.DataFrame
    """
    import logging
    logging.basicConfig()

    # Always use the standardized way of storing the data.
    # See the function normalize_input_to_internal_representation for more information.
    kind_to_df_map, column_id, column_value = \
        dataframe_functions.normalize_input_to_internal_representation(df_or_dict=timeseries_container,
                                                                       column_id=column_id,
                                                                       column_sort=column_sort,
                                                                       column_kind=column_kind,
                                                                       column_value=column_value)

    # Use the standard setting if the user did not supply ones himself.
    if feature_extraction_settings is None:
        feature_extraction_settings = FeatureExtractionSettings()
        for key in kind_to_df_map:
            feature_extraction_settings.set_default_parameters(key)

    # Choose the parallelization according to a rule-of-thumb
    if parallelization is None:
        parallelization = 'per_sample' if (feature_extraction_settings.n_processes / 2) > len(kind_to_df_map) \
            else 'per_kind'

    _logger.info('Parallelizing feature calculation {}'.format(parallelization))

    # If requested, do profiling (advanced feature)
    if feature_extraction_settings.PROFILING:
        profiler = profiling.start_profiling()

    # Calculate the result
    if parallelization == 'per_kind':
        result = _extract_features_per_kind(kind_to_df_map, feature_extraction_settings,
                                            column_id, column_value)
    elif parallelization == 'per_sample':
        result = _extract_features_parallel_per_sample(kind_to_df_map, feature_extraction_settings,
                                                       column_id, column_value)
    elif parallelization == 'no_parallelization':
        result = _extract_features_per_kind(kind_to_df_map, feature_extraction_settings,
                                            column_id, column_value, serial=True)
    else:
        raise ValueError("Argument parallelization must be one of: 'per_kind', 'per_sample'")

    # Turn off profiling if it was turned on
    if feature_extraction_settings.PROFILING:
        profiling.end_profiling(profiler, filename=feature_extraction_settings.PROFILING_FILENAME,
                                sorting=feature_extraction_settings.PROFILING_SORTING)

    return result

def _extract_features_per_kind(kind_to_df_map, settings, column_id, column_value, serial=False):
    """
    Parallelize the feature extraction per kind.

    :param kind_to_df_map: The time series to compute the features for in our internal format
    :type kind_to_df_map: dict of pandas.DataFrame

    :param column_id: The name of the id column to group by.
    :type column_id: str
    :param column_value: The name for the column keeping the value itself.
    :type column_value: str

    :param settings: settings object that controls which features are calculated
    :type settings: tsfresh.feature_extraction.settings.FeatureExtractionSettings

    :param serial: Do not parallelize the extraction. This can be handy if (1) you want to debug something
       (2) you want to profile something or (3) your environment does not support multiprocessing

    :return: The (maybe imputed) DataFrame containing extracted features.
    :rtype: pandas.DataFrame
    """
    partial_extract_features_for_one_time_series = partial(_extract_features_for_one_time_series,
                                                           column_id=column_id,
                                                           column_value=column_value,
                                                           settings=settings)
    pool = Pool(settings.n_processes)

    chunksize = helper_functions.calculate_best_chunksize(kind_to_df_map, settings)

    total_number_of_expected_results = len(kind_to_df_map)

    if serial:
        map_function = map
    else:
        map_function = partial(pool.imap_unordered, chunksize=chunksize)

    extracted_features = tqdm(map_function(partial_extract_features_for_one_time_series, kind_to_df_map.items()),
                              total=total_number_of_expected_results,
                              desc="Feature Extraction", disable=settings.disable_progressbar)

    pool.close()

    # Concatenate all partial results
    result = pd.concat(extracted_features, axis=1, join='outer').astype(np.float64)

    # Impute the result if requested
    if settings.IMPUTE is not None:
        settings.IMPUTE(result)

    pool.join()
    return result

def _extract_features_parallel_per_sample(kind_to_df_map, settings, column_id, column_value):
    """
    Parallelize the feature extraction per kind and per sample.

    As the splitting of the dataframes per kind along column_id is quite costly, we settled for an async map in this
    function. The result objects are temporarily stored in a fifo queue from which they can be retrieved in order
    of submission.

    :param kind_to_df_map: The time series to compute the features for in our internal format
    :type kind_to_df_map: dict of pandas.DataFrame

    :param column_id: The name of the id column to group by.
    :type column_id: str
    :param column_value: The name for the column keeping the value itself.
    :type column_value: str

    :param settings: settings object that controls which features are calculated
    :type settings: tsfresh.feature_extraction.settings.FeatureExtractionSettings

    :return: The (maybe imputed) DataFrame containing extracted features.
    :rtype: pandas.DataFrame
    """
    partial_extract_features_for_one_time_series = partial(_extract_features_for_one_time_series,
                                                           column_id=column_id,
                                                           column_value=column_value,
                                                           settings=settings)
    pool = Pool(settings.n_processes)
    total_number_of_expected_results = 0

    # Submit map jobs per kind per sample
    results_fifo = Queue()

    for kind, df_kind in kind_to_df_map.items():
        df_grouped_by_id = df_kind.groupby(column_id)

        total_number_of_expected_results += len(df_grouped_by_id)

        chunksize = helper_functions.calculate_best_chunksize(df_grouped_by_id, settings)

        results_fifo.put(
            pool.imap_unordered(
                partial_extract_features_for_one_time_series,
                [(kind, df_group) for _, df_group in df_grouped_by_id],
                chunksize=chunksize
            )
        )

    pool.close()

    # Wait for the jobs to complete and concatenate the partial results
    dfs_per_kind = []

    # Do this all with a progress bar
    with tqdm(total=total_number_of_expected_results, desc="Feature Extraction", disable=settings.disable_progressbar) as progress_bar:
        # We need some sort of measure, when a new result is there. So we wrap the
        # map_results into another iterable which updates the progress bar each time
        # a new result is there
        def iterable_with_tqdm_update(queue, progress_bar):
            for element in queue:
                progress_bar.update(1)
                yield element

        result = pd.DataFrame()
        while not results_fifo.empty():
            map_result = results_fifo.get()
            dfs_kind = iterable_with_tqdm_update(map_result, progress_bar)
            df_tmp = pd.concat(dfs_kind, axis=0).astype(np.float64)
            result = pd.concat([result, df_tmp], axis=1).astype(np.float64)

    # Impute the result if requested
    if settings.IMPUTE is not None:
        settings.IMPUTE(result)

    pool.join()
    return result

def _extract_features_for_one_time_series(prefix_and_dataframe, column_id, column_value, settings):
    """
    Extract time series features for a given data frame based on the passed settings.

    This is an internal function, please use the extract_features function.

    The `dataframe` is expected to have at least two columns: column_id and column_value. The data is grouped together
    by their column_id value and the time series features are calculated independently for each of the groups.
    As a result, the function returns a :class:`pandas.DataFrame` with the ids as an index and as many columns as there
    were features calculated.

    To distinguish the features from others, all the calculated columns are given the prefix passed in by column_prefix.

    For example, if you pass in a `dataframe` of shape

        +====+=======+=====+
        | id | value | ... |
        +====+=======+=====+
        | A  | 1     | ... |
        +----+-------+-----+
        | A  | 2     | ... |
        +----+-------+-----+
        | A  | 3     | ... |
        +----+-------+-----+
        | B  | 1     | ... |
        +----+-------+-----+
        | B  | 2     | ... |
        +----+-------+-----+
        | B  | 3     | ... |
        +----+-------+-----+

    with `column_id="id"`, `column_value="value"` and `column_prefix="prefix"` the resulting :class:`pandas.DataFrame`
    will have shape

        +=======+==================+==================+=====+==================+
        | Index | prefix_feature_1 | prefix_feature_2 | ... | prefix_feature_N |
        +=======+==================+==================+=====+==================+
        | A     | ...              | ...              | ... | ...              |
        +-------+------------------+------------------+-----+------------------+
        | B     | ...              | ...              | ... | ...              |
        +-------+------------------+------------------+-----+------------------+

    where N is the number of features that were calculated. Which features are calculated is controlled by the
    passed settings instance (see :class:`~tsfresh.feature_extraction.settings.FeatureExtractionSettings` for a list of
    all possible features to calculate).

    The parameter `dataframe` is not allowed to have any NaN value in it. It is possible to have different numbers
    of values for different ids.

    :param prefix_and_dataframe: Tuple of column_prefix and dataframe
        column_prefix is the string that each extracted feature will be prefixed with (for better separation)
        dataframe with at least the columns column_id and column_value to extract the time
        series features for.
    :type prefix_and_dataframe: (str, DataFrame)
    :param column_id: The name of the column with the ids.
    :param column_value: The name of the column with the values.
    :param settings: The settings to control, which features will be extracted.
    :return: A dataframe with the extracted features as the columns (prefixed with column_prefix) and as many
        rows as their are unique values in the id column.
    """
    column_prefix, dataframe = prefix_and_dataframe
    column_prefix = str(column_prefix)

    # Ensure features are calculated on float64
    dataframe[column_value] = dataframe[column_value].astype(np.float64)

    with warnings.catch_warnings():
        if not settings.show_warnings:
            warnings.simplefilter("ignore")
        else:
            warnings.simplefilter("default")

        if settings.set_default and column_prefix not in settings.kind_to_calculation_settings_mapping:
            settings.set_default_parameters(column_prefix)

        # Calculate the aggregation functions
        column_name_to_aggregate_function = settings.get_aggregate_functions(column_prefix)

        if column_name_to_aggregate_function:
            extracted_features = dataframe.groupby(column_id)[column_value].aggregate(column_name_to_aggregate_function)
        else:
            extracted_features = pd.DataFrame(index=dataframe[column_id].unique())

        # Calculate the apply functions
        apply_functions = settings.get_apply_functions(column_prefix)

        if apply_functions:
            list_of_extracted_feature_dataframes = [extracted_features]
            for apply_function, kwargs in apply_functions:
                current_result = dataframe.groupby(column_id)[column_value].apply(apply_function, **kwargs).unstack()
                if len(current_result) > 0:
                    list_of_extracted_feature_dataframes.append(current_result)

            if len(list_of_extracted_feature_dataframes) > 0:
                extracted_features = pd.concat(list_of_extracted_feature_dataframes, axis=1,
                                               join_axes=[extracted_features.index])

        return extracted_features

max_stars_repo_name: mynameisvinn/tsfresh

max_issues_repo_name: mynameisvinn/tsfresh

max_forks_repo_name: mynameisvinn/tsfresh


text:
import numpy as np

from pyinterpolate.distance.calculate_distances import calc_point_to_point_distance
from pyinterpolate.transform.prepare_kriging_data import prepare_kriging_data
from pyinterpolate.transform.tests import does_variogram_exist

class Krige:
    """
    Class for kriging interpolation of the unknown values at a given location (position). Class takes two arguments
    during the initialization:
    semivariogram_model - semivariogram model,
    known_points - array of known values [x, y, val]

    Available methods:

    - ordinary_kriging - ordinary kriging of unknown point value,
    - simple_kriging - simple kriging of unknown point value.

    Class methods may raise ValueError if estimated value is below 0. You may use try: ... except: ... statement to
        overwrite those values with some constant or NaN or you could use different semivariogram model. Sometimes this
        problem is related to the input data, especially to the clustered groups of points. In this case aggregate
        those clusters and then estimate semivariogram and perform kriging.

    INITLIALIZATION PARAMS:

    :param semivariogram_model: semivariogram model returned by TheoreticalSemivariogram class
    :param known_points: dataset with known values and locations. Each column should represent different dimension and
        the last column represents values example: [[dim_x1, dim_y1, val_1], [dim_x2, dim_y2, val_2]].
    """

    def __init__(self, semivariogram_model, known_points):
        """
        INPUT:

        :param semivariogram_model: (TheoreticalSemivariogram) Theoretical Semivariogram used for data interpolation,
        :param known_points: (numpy array) dataset with known values and locations. Each column should represent
            different dimension and the last column represents values:
            [[dim_x1, dim_y1, val_1], [dim_x2, dim_y2, val_2]]
        """

        self.dataset = known_points

        # Test semivariogram model
        does_variogram_exist(semivariogram_model)
        self.model = semivariogram_model
        self.distances = None

    def ordinary_kriging(self, unknown_location, number_of_neighbours, test_anomalies=True):
        """
        Function predicts value at unknown location with Ordinary Kriging technique.

        INPUT:

        :param unknown_location: (tuple) position of unknown location,
        :param number_of_neighbours: (int) number of the closest locations to the unknown position which should be
            included in the modeling,
        :param test_anomalies: (bool) check if weights are negative.

        OUTPUT:

        :return: predicted, error, estimated mean, weights
            [value_in_unknown_location, error, estimated_mean, weights]
        """

        prepared_data = prepare_kriging_data(unknown_position=unknown_location,
                                             data_array=self.dataset,
                                             number_of_neighbours=number_of_neighbours)
        n = number_of_neighbours
        unknown_distances = prepared_data[:, -1]
        k = self.model.predict(unknown_distances)
        k = k.T
        k_ones = np.ones(1)[0]
        k = np.r_[k, k_ones]

        dists = calc_point_to_point_distance(prepared_data[:, :-2])

        predicted_weights = self.model.predict(dists.ravel())
        predicted = np.array(predicted_weights.reshape(n, n))
        p_ones = np.ones((predicted.shape[0], 1))
        predicted_with_ones_col = np.c_[predicted, p_ones]
        p_ones_row = np.ones((1, predicted_with_ones_col.shape[1]))
        p_ones_row[0][-1] = 0.
        weights = np.r_[predicted_with_ones_col, p_ones_row]

        w = np.linalg.solve(weights, k)
        zhat = prepared_data[:, -2].dot(w[:-1])

        # Test for anomalies
        if test_anomalies:
            if zhat < 0:
                user_input_message = 'Estimated value is below zero and it is: {}. \n'.format(zhat)
                text_error = user_input_message + 'Program is terminated. Try different semivariogram model. ' \
                                                  '(Did you use gaussian model? \
                            If so then try to use other models like linear or exponential) and/or analyze your data \
                            for any clusters which may affect the final estimation'

                raise ValueError(text_error)

        sigmasq = (w.T * k)[0]
        if sigmasq < 0:
            sigma = 0
        else:
            sigma = np.sqrt(sigmasq)
        return zhat, sigma, w[-1], w

    def simple_kriging(self, unknown_location, number_of_neighbours, global_mean, test_anomalies=True):
        """
        Function predicts value at unknown location with Simple Kriging technique.

        INPUT:

        :param unknown_location: (tuple) position of unknown location,
        :param number_of_neighbours: (int) number of the closest locations to the unknown position which should be
            included in the modeling,
        :param global_mean: (float) global mean which should be known before processing,
        :param test_anomalies: (bool) check if weights are negative.

        OUTPUT:

        :return: predicted, error, mean, weights:
            [value_in_unknown_location, error, mean, weights]
        """

        prepared_data = prepare_kriging_data(unknown_position=unknown_location,
                                             data_array=self.dataset,
                                             number_of_neighbours=number_of_neighbours)
        n = number_of_neighbours

        unknown_distances = prepared_data[:, -1]
        k = self.model.predict(unknown_distances)
        k = k.T

        dists = calc_point_to_point_distance(prepared_data[:, :-2])
        predicted_weights = self.model.predict(dists.ravel())
        predicted = np.array(predicted_weights.reshape(n, n))

        w = np.linalg.solve(predicted, k)
        r = prepared_data[:, -2] - global_mean
        zhat = r.dot(w)
        zhat = zhat + global_mean

        # Test for anomalies
        if test_anomalies:
            if zhat < 0:
                user_input_message = 'Estimated value is below zero and it is: {}. \n'.format(zhat)
                text_error = user_input_message + 'Program is terminated. Try different semivariogram model. ' \
                                                  '(Did you use gaussian model? \
                            If so then try to use other models like linear or exponential) and/or analyze your data \
                            for any clusters which may affect the final estimation'

                raise ValueError(text_error)

        sigmasq = (w.T * k)[0]
        if sigmasq < 0:
            sigma = 0
        else:
            sigma = np.sqrt(sigmasq)
        return zhat, sigma, global_mean, w

max_stars_repo_name: NarayanAdithya/pyinterpolate

max_issues_repo_name: NarayanAdithya/pyinterpolate

max_forks_repo_name: NarayanAdithya/pyinterpolate


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import numpy as np
import onnx
from onnx.helper import make_tensor_value_info, make_tensor, make_node
from onnx import TensorProto
from onnx import numpy_helper
from . import Layer
from ..utils import np2tensor

class BatchNormalization(Layer):
    def convert(self):
        super().convert()
        assert len(self.input_full_names) == 1
        config = self.config.copy()

        # ToDo: ListWrapper
        #assert config['axis'] == -1, 'onnx batchnorm is only implemented for channel axis (1)'

        # load weights from keras layer
        scale, b, mean, var = self.layer.get_weights()
        # add initializers to graph for holding w and b
        label_scale  = self.get_label('scale')
        scale_tensor = np2tensor(scale, label_scale)
        label_b      = self.get_label('bias')
        b_tensor     = np2tensor(b, label_b)
        label_mean   = self.get_label('mean')
        mean_tensor  = np2tensor(mean, label_mean)
        label_var    = self.get_label('var')
        var_tensor   = np2tensor(var, label_var)
        self.onnx_initializers.extend([scale_tensor, b_tensor, mean_tensor, var_tensor])

        bn_node = make_node(
            'BatchNormalization',
            inputs   = [self.input_full_names[0], label_scale,
                        label_b, label_mean, label_var],
            outputs  = self.output_full_names,
            epsilon  = config['epsilon'],
            momentum = config['momentum'],
        )

        self.onnx_nodes.extend([bn_node])


text:
import json
import numpy as np
import pandas as pd
from pandas import json_normalize
from enum import Enum

class ManalyzeDetectionReason(Enum):
    UNUSUAL_SECTION_NAME = 1
    W_AND_X = 2
    FEW_IMPORTS = 3
    KNOWN_SECTION_NAME = 4
    BROKEN_RITCH_HEADER = 5
    BROKEN_RESOURCE = 6

    SUMMARY_PACKED_KNOWN = 7

    @staticmethod
    def has_unusual_section_name(msg):
        return ("Unusual section name found:" in msg)

    @staticmethod
    def has_w_and_x_section(msg):
        return ("is both writable and executable." in msg)

    @staticmethod
    def has_few_imports(msg):
        return ("The PE only has" in msg)

    @staticmethod
    def has_known_section_name(msg):
        return ("The PE is packed with" in msg)\
            or ("This PE is packed with" in msg)\
            or ("This PE is a" in msg)

    @staticmethod
    def has_broken_rich_header(msg):
        return ("The RICH header checksum is invalid." in msg)\
            or ("The number of imports reported in the RICH header is inconsistent." in msg)

    @staticmethod
    def has_broken_resource(msg):
        return "The PE's resources are bigger than it is." in msg

    @staticmethod
    def msg_to_enum(msg):
        if ManalyzeDetectionReason.has_unusual_section_name(msg):
            return ManalyzeDetectionReason.UNUSUAL_SECTION_NAME
        # elif ManalyzeDetectionReason.has_w_and_x_section(msg):
        #    return ManalyzeDetectionReason.W_AND_X
        # elif ManalyzeDetectionReason.has_few_imports(msg):
        #     return ManalyzeDetectionReason.FEW_IMPORTS
        elif ManalyzeDetectionReason.has_known_section_name(msg):
            return ManalyzeDetectionReason.KNOWN_SECTION_NAME
        elif ManalyzeDetectionReason.has_broken_resource(msg):
            return ManalyzeDetectionReason.BROKEN_RESOURCE
        elif ManalyzeDetectionReason.has_broken_rich_header(msg):
            return ManalyzeDetectionReason.BROKEN_RITCH_HEADER
        else:
            return None

class PyPackerDetectionReason(Enum):
    TOO_FEW_IMPORTS = 1
    NONSTANDARD_SECTIONNAME = 2
    SECTION_NAME_IS_KNOWN = 3
    BAD_ENTRY_POINT = 4
    @staticmethod
    def has_bad_entry_point(msg):
        return ("Null entry point" in msg)\
            or ("doesn't fall in valid section" in msg)\
            or ("falls in overlapping sections:" in msg)\
            or ("in irregular section(s):" in msg)

    @staticmethod
    def has_lowimport(msg):
        return "Too few imports" in msg

    @staticmethod
    def has_nonstandard_section_name(msg):
        return ("Section name with invalid characters" in msg)\
            or ("non-standard sections:" in msg)\
            or ("sections with invalid names" in msg)

    @staticmethod
    def has_known_section_name(msg):
        return "matches known packer:" in msg

    @staticmethod
    def msg_to_enum(msg):
        if PyPackerDetectionReason.has_bad_entry_point(msg):
            return PyPackerDetectionReason.BAD_ENTRY_POINT
        # elif PyPackerDetectionReason.has_lowimport(msg):
        #     return PyPackerDetectionReason.TOO_FEW_IMPORTS
        elif PyPackerDetectionReason.has_nonstandard_section_name(msg):
            return PyPackerDetectionReason.NONSTANDARD_SECTIONNAME
        elif PyPackerDetectionReason.has_known_section_name(msg):
            return PyPackerDetectionReason.SECTION_NAME_IS_KNOWN
        else:
            return None

def process_manalyze_result(l):
    if isinstance(l["result"], dict):
        result_level = l["result"]["level"]
        result_output = list(l["result"]["plugin_output"].values())
        if "summary" in l["result"].keys():
            result_summary = l["result"]["summary"]
        else:
            result_summary = None
    else:
        result_level = None
        result_output = None
        result_summary = None
    return {
        "name": l["name"],
        "manalyze_result_level": result_level,
        "manalyze_result_output": result_output,
        "manalyze_result_summary": result_summary
    }

def process_manalyze_summary(msg):
    if msg is None:
        return []
    if "possibly packed" in msg:
        return []
    elif "packed with" in msg or "self-extractor" in msg or "installer":
        return [ManalyzeDetectionReason.SUMMARY_PACKED_KNOWN]
    else:
        return []

def process_peid(l):
    return {
        "name": l["name"],
        "peid_packed": l["Packed"] == "yes",
        "peid_PEiD": l["PEiD"]
    }

def process_pypacker(l):
    return {
        "name": l["name"],
        "pypacker_suspicions": l["feature_suspicions"],
        "pypacker_detections": l["feature_detections"]
    }

def load_nested_json(fname: str, process):
    with open(fname, "r") as fin:
        dat = json_normalize(process(json.loads(l)) for l in fin)
    return dat

def process_pypacker_detection_reason(msg):
    return [PyPackerDetectionReason.msg_to_enum(i) for i in msg if PyPackerDetectionReason.msg_to_enum(i)] if msg else []

def process_manalyze_detection_reason(msg):
    return [ManalyzeDetectionReason.msg_to_enum(i) for i in msg if ManalyzeDetectionReason.msg_to_enum(i)] if msg else []

def make_detection_reasons_columns(df):
    df["pypacker_reason"] = df["pypacker_suspicions"].apply(lambda x: process_pypacker_detection_reason(x)) + df["pypacker_detections"].apply(lambda x: process_pypacker_detection_reason(x))
    df["manalyze_reason"] = df["manalyze_result_output"].apply(lambda x: process_manalyze_detection_reason(x)) + df["manalyze_result_summary"].apply(lambda x: process_manalyze_summary(x))

max_stars_repo_name: FFRI/PackerDetectorConsideration

max_issues_repo_name: FFRI/PackerDetectorConsideration

max_forks_repo_name: FFRI/PackerDetectorConsideration