text:
from socialnetwork import SocialNetwork
import argparse
import numpy as np
from tqdm import tqdm
from utils import regret, partial_regret, ind_levels
from collections import defaultdict
from votingrules import VotingRules
from dataset import Dataset
from networks import generate_graphs
import random
from scipy.stats import ttest_ind

# since every graph type has diff. parameter spaces,
# I have created this wrapper that returns a generator
# of parameters, given a graph type name.
def param_generator(graph_type):
    degrees = [4, 8, 12, 16]
    probs = [0.25, 0.5, 0.75]

    if graph_type in ('path', 'scale-free'):
        return (dict() for _ in range(1))

    elif graph_type == 'random':
        return ({'prob': prob} for prob in probs)

    elif graph_type == 'regular':
        return ({'degree': degree} for degree in degrees)

    elif graph_type == 'small-world':
        def param_generator():
            for degree in degrees:
                for prob in probs:
                    yield {'degree': degree, 'prob': prob}

        return param_generator()

    else:
        raise NotImplementedError()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--seed', type=int, default=42, help='Random seed')
    parser.add_argument('--alternatives', type=int, default=4, help='Number of alternatives.')
    parser.add_argument('--voters', type=int, default=100, help='Number of voters.')
    parser.add_argument('--voters_source', type=str, default='random', help='How to generate voters')
    parser.add_argument('--dataset_path', type=str, default='dataset/ED-00004-00000001.soc', help="If using preflib, which dataset?")
    parser.add_argument('--experiments', type=int, default=100, help='Number of experiments.')
    parser.add_argument('--graphs_per_setting', type=int, default=25, help='How many graphs to generate per param settings')
    parser.add_argument('--voter_types', type=int, default=2, help='Number of types for \'types\' source')
    parser.add_argument('--type_gen', type=str, default='half_normal', help='distribution for \'types\' source [half_normal|tshirt]')
    parser.add_argument('--print_graph', action='store_true', help='Print the generated graph')
    parser.add_argument('--print_delegations', action='store_true', help='Print the delegation chains')
    parser.add_argument('--print_preferences', action='store_true', help='Print the preference counts')
    parser.add_argument('--skip_print_winners', action='store_true', help='Skip the printing of the winner counts')
    parser.add_argument('--partial_regret', action='store_true', help='Use also the alternative metric of partial regret.')
    parser.add_argument('--ttest', action='store_true', help='Perform t-test')
    parser.add_argument('--indecisiveness', type=float, nargs='+', default=[0, 0.2, 0.2, 0.2, 1, 1, 1, 1, 1, 1, 1, 1, 1],
                        help="indecisiveness distribution")
    parser.add_argument('--graph_structures', type=str, nargs='+', default=['regular'],
                        help='specify which graph structures you want to use')

    args = parser.parse_args()

    random.seed(args.seed)

    graph_types = args.graph_structures

    paradigms = ['direct', 'proxy', 'liquid']

    # for regret, we need a three level structure: graph type, paradigm and rule.
    regrets = defaultdict(lambda : defaultdict(lambda : defaultdict(lambda : [])))
    winners = defaultdict(lambda : defaultdict(lambda : defaultdict(lambda : defaultdict(lambda : 0))))
    if args.partial_regret:
        partial_regrets = defaultdict(lambda : defaultdict(lambda : defaultdict(lambda : [])))

    # this is used for the progress bar. First, we need to compute the total number
    # of graph types. Since diff. graph types have different settings, let's compute
    # their number explicitly
    COUNT_GRAPH_SETTINGS = 0
    for graph_type in graph_types:
        for param in param_generator(graph_type):
            COUNT_GRAPH_SETTINGS += 1

    # Now, total number of steps:
    TOT_EXPERIMENTS = COUNT_GRAPH_SETTINGS * args.graphs_per_setting * args.experiments * len(paradigms) * len(VotingRules.rules)

    # possible_indecision_levels = ind_levels(args.alternatives)
    possible_indecision_levels = args.indecisiveness

    # progress bar
    with tqdm(total=TOT_EXPERIMENTS, leave = False) as pbar:
        if args.voters_source == 'random':
            data = Dataset(source='random', rand_params=[args.alternatives, args.voters])
            true_preferences, true_counts = data.preferences, data.counts
        elif args.voters_source == 'preflib':
            data = Dataset(source=args.dataset_path)
            true_preferences, true_counts = data.preferences, data.counts
        elif args.voters_source == 'types':
            data = Dataset(source='type_random', rand_params=[args.alternatives, args.voters, args.voter_types],
                           type_generation=args.type_gen)
            true_preferences, true_counts = data.preferences, data.counts
        else:
            raise NotImplementedError('Unknown voter source')

        # for all types of graphs
        for graph_type in graph_types:

            # for all parameters settings of this graph
            for params in param_generator(graph_type):

                # generate some graphs with this parameters
                graph_generator = generate_graphs(num_voters=data.count_voters(), \
                    num_graphs=args.graphs_per_setting, gtype=graph_type, seed = args.seed, params = params)

                # for every graph
                for graph in graph_generator:

                    # get the corresponding SN
                    SN = SocialNetwork(strategy = 'dataset_and_nx_graph', possible_indecision_levels = possible_indecision_levels, \
                        graph = graph, dataset = data, print_graph = args.print_graph)

                    # and compare it under every paradigm
                    for paradigm in paradigms:

                        # for more than one experiment
                        for _ in range(args.experiments):
                            # get the preferences
                            SN_preferences, SN_counts = SN.get_preferences(paradigm,\
                                print_delegations = args.print_delegations, print_preferences = args.print_preferences)

                            # and get the winner for every rule
                            for rule in VotingRules.rules:
                                # this corresponds to random tie breaking
                                winner = VotingRules.elect(rule, SN_preferences, SN_counts, tiebreaking = lambda winners : random.choice(list(winners)))

                                regrets[graph_type][paradigm][rule].append(regret(winner, true_preferences, true_counts))
                                if args.partial_regret:
                                    partial_regrets[graph_type][paradigm][rule].append(partial_regret(winner, SN.id2voter.values()))

                                winners[graph_type][paradigm][rule][winner] += 1

                                # update the progress bar
                                pbar.update(1)

    # print result
    def print_results(data, name = 'regret', print_winners = True):
        # by default, we say it is not passed
        t_tests = defaultdict(lambda : defaultdict(lambda : defaultdict(lambda : defaultdict(lambda : 'FAILED'))))
        for graph_type in graph_types:
            for rule in VotingRules.rules:
                for paradigm in paradigms:
                    # data
                    regs = data[graph_type][paradigm][rule]
                    print(f'avg {name} {graph_type}, {rule}, {paradigm}: {np.mean(regs):.4f} (+- {np.std(regs):.4f})')
                    if not args.skip_print_winners and print_winners:
                        print(', '.join([f'{w} won {c} times' for w, c in sorted(dict(winners[graph_type][paradigm][rule]).items())]))
                    # t test
                    for other in paradigms:
                        if other != paradigm:
                            data1 = data[graph_type][paradigm][rule]
                            data2 = data[graph_type][other][rule]
                            stat, p = ttest_ind(data1, data2)
                            if p <= 0.05:
                                t_tests[graph_type][paradigm][other][rule] = 'PASSED'
                print("#######")

        if args.ttest:
            print("######### T-TESTS ###########")
            for graph_type in graph_types:
                for rule in VotingRules.rules:
                    for paradigm in paradigms:
                        # basically, exclude one paradigm and look at the other two
                        # this is a quick way to generate all the combinations without repetition
                        others_two = list(set(paradigms) - {paradigm})
                        print(f"{graph_type}, {rule}, {others_two[0]}/{others_two[1]}: {t_tests[graph_type][others_two[0]][others_two[1]][rule]}")
                print('##')

        print("*********")

    print_results(regrets)
    if args.partial_regret:
        print_results(partial_regrets, name = 'partial regret', print_winners = False)

reason: import numpy,from scipy


text:
"""
Functions to evaluate a model over the omniglot continual learning benchmark.

This code builds on the experiments proposed in [1] and goes further to compute continual learning metrics for
mini-ImageNet as the mean accuracy over all classes as they are added incrementally.

References:
    [1] Javed, K. and White, M., 2019. Meta-learning representations for continual learning.
        In Advances in Neural Information Processing Systems (pp. 1820-1830).
        https://proceedings.neurips.cc/paper/2019/hash/f4dd765c12f2ef67f98f3558c282a9cd-Abstract.html
"""
import copy
from typing import Tuple, Union

import numpy as np

from fed_recon.data.omniglot.omniglot import Omniglot, IMAGE_RESIZE_HW
from fed_recon.models.embedding_distribution_network.model import DistributionNetworkCL
from fed_recon.models.protonet.model import PrototypicalNetworkCL

def eval_omniglot_cil(
    model: Union[PrototypicalNetworkCL, DistributionNetworkCL],
    root_dir: str,
    n_classes_per_task: int = 1,
    k: int = 15,
    test_shots_per_class: int = 5,
    runs: int = 10,
    n_meta_test_classes: int = 600,
    image_resize_hw: Tuple[int, int] = IMAGE_RESIZE_HW,
    device: str = "cuda",
):
    """
        Evaluate a model on the continual learning omniglot benchmark from [1] in which all 600 meta-test classes are seen in
    procession. At each new class, the model is evaluated on the accuracy of distinguishing the current class and
    all previously seen classes.

    :param model: A nn.Module with an `evaluate` method which will learn the current training class(es) and
        compute the accuracy on the union of all previously seen classes and the current set of classes
    :param root_dir: Directory containing the omniglot data.
    :param n_classes_per_task: how many classes to add to the model at each time step
    :param k: Number of training examples per class.
    :param test_shots_per_class: how many examples to evaluate the model on per class
    :param runs: How many independent runs.
    :param n_meta_test_classes: total number of test classes to learn
    :param image_resize_hw: dims of the input images
    :param device: whether to put tensors on the cpu or cuda
    :return: Meta-test test-set accuracy for each class increment, for each run.
    """
    dataset = Omniglot(
        root_dir=root_dir,
        n_classes_per_task=n_meta_test_classes,
        k_shots_per_class=k,
        test_shots_per_class=test_shots_per_class,
        meta_split="test",
    )

    all_accs = []
    all_n_classes = []
    for i in range(runs):
        all_data = dataset.sample_meta_batch(
            batch_size=1,
            n_classes_per_task=n_meta_test_classes,
            k_shots_per_class=k,
            test_shots_per_class=test_shots_per_class,
            return_n_k_along_same_axis=False,
            rotate_classes=False,
        )  # -> ([b, n, k, channels, rows, cols], [b, n, k])
        model_i = copy.deepcopy(model)
        print(f"Evaluating {k}-shot class-incremental learning run {i} of {runs}")
        for j in range(0, n_meta_test_classes, n_classes_per_task):
            train_images, train_labels = all_data["train"]
            train_images = train_images[:, j : j + n_classes_per_task, :, :, :, :]
            train_labels = train_labels[:, j : j + n_classes_per_task]
            train_images = train_images.reshape(
                [1, n_classes_per_task * k, 1, *image_resize_hw]
            )
            train_labels = train_labels.reshape([1, n_classes_per_task * k])

            test_images, test_labels = all_data["test"]
            test_images = test_images[:, 0 : j + n_classes_per_task, :, :, :, :]
            test_labels = test_labels[:, 0 : j + n_classes_per_task]
            test_images = test_images.reshape(
                [
                    1,
                    (j + 1) * n_classes_per_task * test_shots_per_class,
                    1,
                    *image_resize_hw,
                ]
            )
            test_labels = test_labels.reshape(
                [1, (j + 1) * n_classes_per_task * test_shots_per_class]
            )

            # FIXME: cuda implementation not working
            train_images.to(device=device)
            train_labels.to(device=device)
            test_images.to(device=device)
            test_labels.to(device=device)

            acc = model_i.evaluate(
                train_inputs=train_images,
                train_labels=train_labels,
                test_inputs=test_images,
                test_labels=test_labels,
                n_train_classes=n_classes_per_task,
                k_shots=k,
            )
            print(
                f"Class-incremental test-set accuracy over {j + 1} meta-test tasks: {np.round(acc.item(), 2)}"
            )
            all_accs.append(acc.item())
            all_n_classes.append(j + 1)

    print(
        f"Final continual learning, class-incremental accuracy averaged across tasks: {np.mean(all_accs)}"
    )

    return all_accs, all_n_classes

ext: py

reason: import numpy


text:
import PINN_Survey.benchmarking.benchmark as benchmark
import PINN_Survey.problems.klein_gordon.v1 as klein_gordon
from PINN_Survey.problems.klein_gordon.data.load import load_klein_gordon_bounds
import numpy as np
import os

problem_desc = {
    "equation": "Klein Gordon",
    "description_file": ""
}

optimizer_desc = {
    "name": "L-BFGS",
}

def klein_gordon_arch_comparison_v1(
        log_file="logs/klein_gordon_arch_comparison_v1.json",
        n_trials=20,
        n_df=10000,
        width=20,
        depth=8):

    path = os.path.dirname(os.path.abspath(__file__))
    file_path = f"{path}/{log_file}"

    X_true, U_true, X_bounds, U_bounds, [x, t, u] = load_klein_gordon_bounds()

    X = np.vstack(X_bounds)
    U = np.vstack(U_bounds)

    idx = np.random.choice(list(range(X_true.shape[0])), size=n_df)
    X_df = X_true[idx, :]

    lower_bound = np.min(X_true, axis=0)
    upper_bound = np.max(X_true, axis=0)

    layers_base = [2] + ([width]*depth) + [1]

    layers_approx = [2] + ([width]*2) + [1]
    layers_mesh = [2] + ([width]*(depth-2))

    alpha = -1
    beta = 0
    gamma = 1
    k = 3

    model_base = klein_gordon.Klein_Gordon(
        lower_bound, upper_bound, layers_base, alpha, beta, gamma, k)

    benchmark_klein_gordon = benchmark.Benchmark(
        problem_desc, model_base, [X, U, X_df, X_true, U_true], optimizer_desc)

    print("Beginning Base")
    benchmark.log_benchmark(
        benchmark_klein_gordon, n_trials, file_path)

    model_softmesh = klein_gordon.Klein_Gordon_Soft_Mesh(
        lower_bound, upper_bound, layers_approx, layers_mesh, alpha, beta, gamma, k)

    benchmark_klein_gordon_softmesh = benchmark.Benchmark(
        problem_desc, model_softmesh, [X, U, X_df, X_true, U_true], optimizer_desc)

    print("Beginning Soft mesh")
    benchmark.log_benchmark(
        benchmark_klein_gordon_softmesh, n_trials, file_path)

    model_domain_transformer = klein_gordon.Klein_Gordon_Domain_Transformer(
        lower_bound, upper_bound, width, depth-2, alpha, beta, gamma, k)

    print("Beginning Domain Transformer")
    benchmark_klein_gordon_domain_transformer = benchmark.Benchmark(problem_desc, model_domain_transformer, [
        X, U, X_df, X_true, U_true], optimizer_desc)

    benchmark.log_benchmark(
        benchmark_klein_gordon_domain_transformer, n_trials, file_path)

def klein_gordon_sphere_mesh_v1(
        log_file="logs/kelin_gordon_arch_comparison_v1.json",
        n_trials=20,
        n_df=10000,
        layers_mesh=[2, 20, 20, 20, 20],
        layers_approx=[2, 20, 20, 1]):

    path = os.path.dirname(os.path.abspath(__file__))
    file_path = f"{path}/{log_file}"

    X_true, U_true, X_bounds, U_bounds, [x, t, u] = load_klein_gordon_bounds()

    X = np.vstack(X_bounds)
    U = np.vstack(U_bounds)

    idx = np.random.choice(list(range(X_true.shape[0])), size=n_df)
    X_df = X_true[idx, :]

    alpha = -1
    beta = 0
    gamma = 1
    k = 3

    lower_bound = np.min(X_true, axis=0)
    upper_bound = np.max(X_true, axis=0)

    model_sphere_mesh = klein_gordon.Klein_Gordon_Sphere_Mesh(
        lower_bound, upper_bound, layers_approx, layers_mesh, alpha, beta, gamma, k)

    benchmark_klein_gordon_sphere_mesh = benchmark.Benchmark(
        problem_desc, model_sphere_mesh, [X, U, X_df, X_true, U_true], optimizer_desc)

    benchmark.log_benchmark(
        benchmark_klein_gordon_sphere_mesh, n_trials, file_path)

if __name__ == "__main__":
    width = 20
    for depth in [4, 5, 6, 7, 8, 9, 10]:
        layers_approx = [2] + ([width]*2) + [1]
        layers_mesh = [2] + ([width]*(depth-2))
        klein_gordon_sphere_mesh_v1(
            n_trials=25,
            layers_approx=layers_approx,
            layers_mesh=layers_mesh,
            log_file="logs/klein_gordon_arch_comparison_v1.json")


text:
import numpy as np
import matplotlib.pyplot as plt
import cv2
from Finding_Lane import find_lane_pixels

def generate_polynomials(binary_warped, ym_per_pix, xm_per_pix):
    '''
    Generates the second order polynomial
    '''

    # Find our lane pixels
    leftx, lefty, rightx, righty, out_img = find_lane_pixels(binary_warped)
    # Fit a second order polynomial
    left_fit_cr = np.polyfit(lefty*ym_per_pix, leftx*xm_per_pix, 2)
    right_fit_cr = np.polyfit(righty*ym_per_pix, rightx*xm_per_pix, 2)

    return left_fit_cr, right_fit_cr

def measure_curvature_real(binary_warped_img, ym_per_pix, xm_per_pix):
    '''
    Calculates the curvature of polynomial functions in meters.
    '''

    # get the second order polynomia
    left_fit_cr, right_fit_cr = generate_polynomials(binary_warped_img, ym_per_pix, xm_per_pix)

    # The maximum y-value => the bottom of image
    y_eval = np.max(binary_warped_img.shape[0])*ym_per_pix

    x_left_cr = left_fit_cr[0]*y_eval**2 + left_fit_cr[1]*y_eval + left_fit_cr[2]
    x_right_cr = right_fit_cr[0]*y_eval**2 + right_fit_cr[1]*y_eval + right_fit_cr[2]
    x_max = binary_warped_img.shape[1]*xm_per_pix
    veh_position = x_max/2 - (x_left_cr + x_right_cr)/2 # if in left pos < 0, if in right pos > 0

    # Radius of curvature
    if (0 != left_fit_cr[0]):
        left_curverad = pow(1 + (2*y_eval*left_fit_cr[0] + left_fit_cr[1])**2, 3/2)/abs(2*left_fit_cr[0])
    else:
        left_curverad = 0

    if (0 != right_fit_cr[0]):
        right_curverad = pow(1 + (2*y_eval*right_fit_cr[0] + right_fit_cr[1])**2, 3/2)/abs(2*right_fit_cr[0])
    else:
        right_curverad = 0

    return left_curverad, right_curverad, veh_position

max_stars_repo_name: BofeiCHEN/CarND-Advanced-Lane-Lines

max_issues_repo_name: BofeiCHEN/CarND-Advanced-Lane-Lines

max_forks_repo_name: BofeiCHEN/CarND-Advanced-Lane-Lines


text:
from sklearn.metrics import precision_score, recall_score, accuracy_score
from sklearn.model_selection import cross_val_score
import numpy as np
import classification as clf

def class_precision_recall(y_true, y_pred):
    """
    Calculate precision and recall for each class
    :param y_true: a list of true labels
    :param y_pred: a list of predicted labels
    :return: an object with the precision and recall for each class
    """
    return {
        'healthy': {
            'precision': precision_score(y_true, y_pred, pos_label=1),
            'recall': recall_score(y_true, y_pred, pos_label=1)
        },
        'unhealthy': {
            'precision': precision_score(y_true, y_pred, pos_label=0),
            'recall': recall_score(y_true, y_pred, pos_label=0)
        }
    }

def evaluate(model, img_dir, test_x, test_y, desc_dict):
    """
    Evaluate a classifier using a test bee image dataset and a descriptor dictionary
    :param model: a previously fit model for beehive health classification
    :param img_dir: a directory where image files are located
    :param test_x: a list of bee image files for testing
    :param test_y: a list of booleans indicating if the hive is healthy for a bee image
    :param desc_dict: a list of descriptors representing a dictionary of visual words
    :return: an object with the precision and recall for each class
    """
    histograms = clf.histograms_by_image(img_dir, test_x, desc_dict)
    pred_y = model.predict(histograms)
    return class_precision_recall(test_y, pred_y)

def cross_val(model, img_dir, train_x, train_y, desc_dict):
    """
    Evaluate a model using 10-fold cross validation
    :param model: the model to evaluate
    :param img_dir: the directory to look for the image files
    :param train_x: a list of image files
    :param train_y: a list of booleans indicating if the hive is healthy for a bee image
    :param desc_dict: a list of descriptors representing a dictionary of visual words
    :return: the average of cross validation scores for this model
    """
    histograms = clf.histograms_by_image(img_dir, train_x, desc_dict)
    return np.mean(cross_val_score(model, histograms, train_y, cv=10))

def accuracy(model, img_dir, test_x, test_y, desc_dict):
    """
    Return the accuracy score for the current model in the given test dataset
    :param model: a previously fit model for beehive health classification
    :param img_dir: a directory where image files are located
    :param test_x: a list of bee image files for testing
    :param test_y: a list of booleans indicating if the hive is healthy for a bee image
    :param desc_dict: a list of descriptors representing a dictionary of visual words
    :return: the accuracy of the model
    """
    return accuracy_score(test_y, model.predict(clf.histograms_by_image(img_dir, test_x, desc_dict)))

def recall(model, img_dir, test_x, test_y, desc_dict):
    """
    Return the recall score for the current model in the given test dataset
    :param model: a previously fit model for beehive health classification
    :param img_dir: a directory where image files are located
    :param test_x: a list of bee image files for testing
    :param test_y: a list of booleans indicating if the hive is healthy for a bee image
    :param desc_dict: a list of descriptors representing a dictionary of visual words
    :return: the recall of the model
    """
    return recall_score(test_y, model.predict(clf.histograms_by_image(img_dir, test_x, desc_dict)))


text:
# -*- coding: utf-8 -*-

# =============================================================================
# Here we will be testing with mixtures of student-t, 1d
# =============================================================================
import sys
sys.path.insert(0,"../../src")
import math
import functools

import torch
import numpy as np
from scipy.special import gamma
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

from src.variational_boosting_bmc import VariationalBoosting
from src import vb_utils
from src import sampling

dim=3
folder_name = "ex1b_d%i"%dim
#%%
np.random.seed(100)
torch.manual_seed(100)
ninit = 5

distribdata = np.load("%s/distribdata.npz"%folder_name,allow_pickle=True)
data = np.load("%s/tracking.npz"%folder_name,allow_pickle=True)
means = data["means"]
covs = data["covs"]

elbo = data["elbo"]
steps = data["steps"]
fig1,ax1 = plt.subplots()
color = 'tab:blue'
ax1.plot(steps,np.log10(means),color=color,marker='.',linestyle=' ')
ax1.set_xlabel("iteration")
ax1.set_ylabel(r"$\log_{10}||\mu - \mu_0||_2$",color=color)
ax1.set_ylim(bottom=-2.5,top=1.0)
ax1.tick_params(axis='y', labelcolor=color)
ax2 = ax1.twinx()
color = 'tab:red'
ax2.plot(steps,np.log10(covs/np.linalg.norm(data["true_cov"])),color=color,marker='x',linestyle=' ')
ax2.set_ylabel(r"$\log_{10}||\Sigma - \Sigma_0||_F/||\Sigma_0||_F$",color=color)
ax2.set_ylim(bottom=-4.0,top=1.0)
ax2.tick_params(axis='y', labelcolor=color)
fig1.savefig("../../tex/figs/%s_meancov"%folder_name)
#%%
def gskl(mu1,mu2,cov1,cov2):
    d = len(mu1)
    term1 = -2*d
    term2 = np.trace(np.linalg.inv(cov1)@cov2 + np.linalg.inv(cov2)@cov1)
    dmu = (mu1-mu2).reshape(1,-1)
    term3 = dmu@(np.linalg.inv(cov1)+np.linalg.inv(cov2))@dmu.reshape(-1,1)
    return 0.25*(term1+term2+term3)
distrib = torch.load("%s/vbdistrib"%folder_name)
mu1 = distrib.mean().numpy()
cov1 = distrib.cov().numpy()
mu2 = data["true_mean"]
cov2 = data["true_cov"]
print(gskl(mu1,mu2,cov1,cov2))

max_stars_repo_name: DFNaiff/Dissertation

max_issues_repo_name: DFNaiff/Dissertation

max_forks_repo_name: DFNaiff/Dissertation


text:
# Created by Marcus Pappik

import numpy as np
import pandas as pd
from hics.scored_slices import ScoredSlices

class AbstractResultStorage:

    def update_relevancies(self, new_relevancies: pd.DataFrame):
        raise NotImplementedError()

    def update_redundancies(self, new_redundancies: pd.DataFrame):
        raise NotImplementedError()

    def update_bivariate_redundancies(self, new_redundancies: pd.DataFrame, new_weights: pd.DataFrame):
        raise NotImplementedError()

    def update_slices(self, new_slices: dict()):
        raise NotImplementedError()

    def get_bivariate_redundancies(self):
        raise NotImplementedError()

    def get_redundancies(self):
        raise NotImplementedError()

    def get_relevancies(self):
        raise NotImplementedError()

    def get_slices(self):
        raise NotImplementedError()

class DefaultResultStorage(AbstractResultStorage):

    def __init__(self, features: list()):
        self.relevancies = pd.DataFrame(columns=['relevancy', 'iteration'])
        self.redundancies = pd.DataFrame(columns=['redundancy', 'iteration'])

        redundancy_dict = {
            'redundancy': pd.DataFrame(data=0, columns=features, index=features),
            'weight': pd.DataFrame(data=0, columns=features, index=features)
        }
        self.bivariate_redundancies = pd.Panel(redundancy_dict)

        self.slices = {}

    def update_relevancies(self, new_relevancies: pd.DataFrame):
        self.relevancies = new_relevancies

    def update_redundancies(self, new_redundancies: pd.DataFrame):
        """Updates redundancies by averaging it with existing values
        """
        current_redundancies = self.redundancies

        new_index = [index for index in new_redundancies.index
                     if index not in current_redundancies.index]

        redundancy_apppend = pd.DataFrame(data=0, index=new_index,
                                          columns=current_redundancies.columns)
        current_redundancies = current_redundancies.append(redundancy_apppend)

        current_redundancies.loc[new_redundancies.index, 'redundancy'] = \
            (current_redundancies.iteration / (current_redundancies.iteration + new_redundancies.iteration)) \
            * current_redundancies.redundancy \
            + \
            (new_redundancies.iteration / (current_redundancies.iteration + new_redundancies.iteration)) \
            * new_redundancies.redundancy
        current_redundancies.loc[new_redundancies.index, 'iteration'] += new_redundancies.iteration

        self.redundancies = current_redundancies

    def update_bivariate_redundancies(self, new_redundancies: pd.DataFrame, new_weights: pd.DataFrame):
        current_weights = new_weights.loc[new_weights.index, new_weights.columns]
        current_redundancies = new_redundancies.loc[new_redundancies.index, new_redundancies.columns]

        current_redundancies[current_weights < 1] = np.inf

        current_redundancies = np.minimum(new_redundancies, current_redundancies)
        current_weights += new_weights

        current_redundancies[current_weights < 1] = 0

        self.bivariate_redundancies.redundancy = current_redundancies
        self.bivariate_redundancies.weight = current_weights

    def update_slices(self, new_slices):
        self.slices = new_slices

    def get_bivariate_redundancies(self):
        return self.bivariate_redundancies.redundancy, self.bivariate_redundancies.weight

    def get_redundancies(self):
        return self.redundancies

    def get_relevancies(self):
        return self.relevancies

    def get_slices(self):
        return self.slices

max_stars_repo_name: KDD-OpenSource/wRaR

max_issues_repo_name: KDD-OpenSource/wRaR

max_forks_repo_name: KDD-OpenSource/wRaR


text:
import numpy as np
# from visualize_cluster_proposals import load_cluster_detections,load_gt,show_detection

def ICM(boxes,iouthreshold,topN):
    """
    """
    boxes_num=len(boxes[:,0])
    # if there are no boxes, return an empty list
    if len(boxes) == 0:
        return []

    while boxes_num>topN:

        boxes=NMM(boxes, iouthreshold)
        boxes_num_after_merge=len(boxes[:,0])
        if boxes_num==boxes_num_after_merge:
            boxes_num_min=np.minimum(boxes_num_after_merge,topN)
            boxes=boxes[:boxes_num_min]
            break
        else:
            boxes_num=boxes_num_after_merge
    return boxes

def NMM(boxes,iouthreshold):
    """
    """
    # if there are no boxes, return an empty list
    if len(boxes) == 0:
        return []
    # sort the boxes according to score
    scores=boxes[:,4]
    idxs=np.argsort(-scores)
    boxes=boxes[idxs,:]
    # if the bounding boxes integers, convert them to floats --
    # this is important since we'll be doing a bunch of divisions
    if boxes.dtype.kind == "i":
        boxes = boxes.astype("float")

    # initialize the list of picked indexes
    pick = []

    # grab the coordinates of the bounding boxes
    x1 = boxes[:, 0]
    y1 = boxes[:, 1]
    x2 = boxes[:, 2]
    y2 = boxes[:, 3]

    # compute the area of the bounding boxes and sort the bounding
    # boxes by the bottom-right y-coordinate of the bounding box
    area = (x2 - x1 + 1) * (y2 - y1 + 1)
    idxs = range(len(area))  # np.argsort(y2)
    idxs.reverse()
    # keep looping while some indexes still remain in the indexes
    # list

    while len(idxs) > 0:
        # grab the last index in the indexes list and add the
        # index value to the list of picked indexes
        last = len(idxs) - 1
        i = idxs[last]
        pick.append(i)

        # find the largest (x, y) coordinates for the start of
        # the bounding box and the smallest (x, y) coordinates
        # for the end of the bounding box
        xx1 = np.maximum(x1[i], x1[idxs[:last]])
        yy1 = np.maximum(y1[i], y1[idxs[:last]])
        xx2 = np.minimum(x2[i], x2[idxs[:last]])
        yy2 = np.minimum(y2[i], y2[idxs[:last]])

        # compute the width and height of the bounding box
        w = np.maximum(0, xx2 - xx1 + 1)
        h = np.maximum(0, yy2 - yy1 + 1)

        # compute the ratio of overlap
        overlap = (w * h) / area[idxs[:last]]

        # merge all boxes with overlap larger than threshold
        delete_idxs = np.concatenate(([last],
                                       np.where(overlap > iouthreshold)[0]))
        idxs=np.array(idxs)
        xx1 = np.amin(x1[idxs[delete_idxs]])
        yy1 = np.amin(y1[idxs[delete_idxs]])
        xx2 = np.amax(x2[idxs[delete_idxs]])
        yy2 = np.amax(y2[idxs[delete_idxs]])

        boxes[i,:4]=np.array([xx1,yy1,xx2,yy2])

        # delete all indexes from the index list that have
        idxs=np.delete(idxs,delete_idxs)

    # return only the bounding boxes that were picked using the
    # integer data type

    return boxes[pick].astype("int")
if __name__ == "__main__":

    cluster_proposal_file = '/media/fanyang/C/code/detector/Detectron-Cascade-RCNN' \
                            '/test/coco_2014_val/generalized_rcnn/detections.pkl'
    gt_file = '/media/fanyang/C/code/detector/' \
              'Detectron-Cascade-RCNN/detectron/datasets/' \
              'data/coco/annotations/instances_val2014.json'
    img_path = '/media/fanyang/C/code/detector/' \
               'Detectron-Cascade-RCNN/detectron/datasets/data/coco/coco_val2014'
    cluster_detection = load_cluster_detections(cluster_proposal_file)
    gt = load_gt(gt_file)
    show_detection(gt,cluster_detection,img_path)


text:
# -*- coding: utf-8 -*-
        """
        qfx2_xxx = qfx2_dist_
        %timeit np.vstack([row.take(sortx) for sortx, row in zip(qfx2_sortx, qfx2_xxx)])
        %timeit np.vstack((row.take(sortx) for sortx, row in zip(qfx2_sortx, qfx2_xxx)))
        %timeit np.vstack([row[sortx] for sortx, row in zip(qfx2_sortx, qfx2_xxx)])
        %timeit np.vstack((row[sortx] for sortx, row in zip(qfx2_sortx, qfx2_xxx)))

        import utool as ut
        #ut.rrrr()
        #np.set_printoptions(threshold=10000000)
        setup = ut.unindent(
        '''
        import numpy as np
        from numpy import array, float32, int32
        qfx2_sortx = %s
        qfx2_xxx = %s
        ''' % (np.array_repr(qfx2_sortx), np.array_repr(qfx2_xxx)))
        stmt1 = 'np.vstack([row.take(sortx) for sortx, row in zip(qfx2_sortx, qfx2_xxx)])'
        stmt2 = 'np.vstack([row[sortx] for sortx, row in zip(qfx2_sortx, qfx2_xxx)])'
        stmt3 = 'np.vstack((row[sortx] for sortx, row in zip(qfx2_sortx, qfx2_xxx)))'
        iterations = 100
        verbose = True
        stmt_list = [stmt1, stmt2, stmt3]
        (passed, time_list, result_list) = ut.timeit_compare(stmt_list, setup=setup, iterations=iterations, verbose=verbose)
        """

reason: import numpy,from numpy


text:
import numpy as np

from imlib.image import binning

def test_get_bins():
    image_size = (1000, 20, 125, 725)
    bin_sizes = (500, 2, 25, 100)

    dim0_bins = np.array((0, 500))
    dim1_bins = np.array((0, 2, 4, 6, 8, 10, 12, 14, 16, 18))
    dim2_bins = np.array((0, 25, 50, 75, 100))
    dim3_bins = np.array((0, 100, 200, 300, 400, 500, 600, 700))
    bins = binning.get_bins(image_size, bin_sizes)

    assert (dim0_bins == bins[0]).all()
    assert (dim1_bins == bins[1]).all()
    assert (dim2_bins == bins[2]).all()
    assert (dim3_bins == bins[3]).all()

max_stars_repo_name: noisysky/imlib

max_issues_repo_name: noisysky/imlib

max_forks_repo_name: noisysky/imlib


text:
import os
import json
import math
from tqdm import tqdm

import numpy as np

import torch
import torchnet as tnt

from protonets.utils import filter_opt, merge_dict
import protonets.utils.data as data_utils
import protonets.utils.model as model_utils
import protonets.utils.log as log_utils

from protonets.data.sg_base import detect_separator

import seaborn as sns

def get_options_from_json( jsonfile):

    with open(jsonfile) as json_file:
        opt = json.load(json_file)

    return opt

def test(model_path, model_opt, test_episodes=100):

    opt = model_opt

    model = torch.load(model_path)
    model.eval()

    # Postprocess arguments
    model_opt['model.x_dim'] = map(int, model_opt['model.x_dim'].split(','))
    model_opt['log.fields'] = model_opt['log.fields'].split(',')

    # construct data
    data_opt = { 'data.' + k: v for k,v in filter_opt(model_opt, 'data').items() }

    episode_fields = {
        'data.test_way': 'data.way',
        'data.test_shot': 'data.shot',
        'data.test_query': 'data.query',
        'data.test_episodes': 'data.train_episodes'
    }

    for k,v in episode_fields.items():
        if opt[k] != 0:
            data_opt[k] = opt[k]
        elif model_opt[k] != 0:
            data_opt[k] = model_opt[k]
        else:
            data_opt[k] = model_opt[v]

    data_opt['data.test_episodes']=test_episodes
    model_opt['data.test_episodes']=test_episodes
    # print("Evaluating {:d}-way, {:d}-shot with {:d} query examples/class over {:d} episodes".format(
    # data_opt['data.test_way'], data_opt['data.test_shot'],
    # data_opt['data.test_query'], data_opt['data.test_episodes']))

    config_name = "TEST_SG_80_10_10_C{:d}_K{:d}_Q{:d}_over_{:d}_episodes".format(
    data_opt['data.test_way'], data_opt['data.test_shot'],
    data_opt['data.test_query'], data_opt['data.test_episodes'])

    torch.manual_seed(1234)
    if data_opt['data.cuda']:
        torch.cuda.manual_seed(1234)

    data = data_utils.load(model_opt, ['test'])

    if data_opt['data.cuda']:
        model.cuda()

    writer, project = log_utils.get_summary_writer(model_opt, "") #no need of timestamp
    #print(project)

    model.eval()

    test_loss=np.zeros((len(data['test'])))
    test_acc=np.zeros((len(data['test'])))
    for i, sample in enumerate(tqdm(data['test'], desc = "test")):
        #print(f"Evaluation: sample size: {sample.shape}")
        _, output = model.loss(sample)
        test_loss[i]=output['loss']
        test_acc[i]=output['acc']
        writer.add_scalar('Loss/test', output['loss'], i)
        writer.add_scalar('Accuracy/test', output['acc'], i)

    print("SHAPE test: ",test_acc.shape)
    print(config_name)
    print(f"Test results over {test_episodes} episodes: ")
    print(f"test loss: {test_loss.mean():.6f} +/- {(1.96*test_loss.std())/np.sqrt(len(test_loss)):.6f},  min loss: {test_loss.min()}, max loss: {test_loss.max()}" )
    print(f"test accuracy: {test_acc.mean():.6f} +/- {(1.96*test_acc.std())/np.sqrt(len(test_acc)):.6f}, min acc: {test_acc.min()}, max acc: {test_acc.max()}" )

    return test_loss, test_acc, config_name

def test_all_models(test_episodes=1000):
    results_dir="results"

    configurations = [os.path.join(results_dir,f) for f in os.listdir(results_dir) if os.path.isdir(os.path.join(results_dir,f))]

    for config in configurations[:]:
        model_names = [os.path.join(config,f) for f in os.listdir(config) if os.path.join(results_dir,f).endswith(".pt")]
        print(model_names)
        for model_path in model_names:
            print(model_path)
            time_stamp = model_path.split("best_model_")[1].split(".pt")[0]
            print(time_stamp)
            json_file=os.path.join(config,f"opt_{time_stamp}.json")
            print(json_file)

            opt = get_options_from_json(json_file)
            test_loss, test_acc, config_name =  test(model_path, opt, test_episodes=test_episodes)
            path_separator = detect_separator(config)
            model_name = model_path.split(path_separator)[-1]
            np.savez(os.path.join("test_results",f"{config_name}__{model_name[:-3]}.npz"), test_loss=test_loss, test_acc=test_acc)

if __name__ == '__main__':

    test_all_models(test_episodes=1000)


text:
# coding: utf-8

# # 20 newsgroup text classification with BERT finetuning
#
# In this script, we'll use a pre-trained BERT
# (https://arxiv.org/abs/1810.04805) model for text classification
# using TensorFlow 2 / Keras and HuggingFace's Transformers
# (https://github.com/huggingface/transformers). This notebook is
# based on "Predicting Movie Review Sentiment with BERT on TF Hub"
# (https://github.com/google-research/bert/blob/master/predicting_movie_reviews_with_bert_on_tf_hub.ipynb)
# by Google and "BERT Fine-Tuning Tutorial with PyTorch"
# (https://mccormickml.com/2019/07/22/BERT-fine-tuning/) by Chris
# McCormick.

# **Note that using a GPU with this script is highly recommended.**
#
# First, the needed imports.

import tensorflow as tf
from tensorflow.keras.utils import plot_model
from tensorflow.keras.callbacks import TensorBoard

from transformers import BertTokenizer, TFBertForSequenceClassification
from transformers import __version__ as transformers_version

from distutils.version import LooseVersion as LV

from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix

import io, sys, os, datetime

from zipfile import ZipFile
import numpy as np

print('Using TensorFlow version:', tf.__version__,
      'Keras version:', tf.keras.__version__,
      'Transformers version:', transformers_version, flush=True)
assert(LV(tf.__version__) >= LV("2.3.0"))

if len(tf.config.list_physical_devices('GPU')):
    from tensorflow.python.client import device_lib
    for d in device_lib.list_local_devices():
        if d.device_type == 'GPU':
            print('GPU:', d.physical_device_desc)
else:
    print('No GPU, using CPU instead.')

if 'DATADIR' in os.environ:
    DATADIR = os.environ['DATADIR']
else:
    DATADIR = "/scratch/project_2005299/data/"
print('Using DATADIR', DATADIR)

# ## 20 Newsgroups data set
#
# Next we'll load the 20 Newsgroups
# (http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.html)
# data set.
#
# The dataset contains 20000 messages collected from 20 different
# Usenet newsgroups (1000 messages from each group):
#
# alt.atheism           | soc.religion.christian   | comp.windows.x     | sci.crypt
# talk.politics.guns    | comp.sys.ibm.pc.hardware | rec.autos          | sci.electronics
# talk.politics.mideast | comp.graphics            | rec.motorcycles    | sci.space
# talk.politics.misc    | comp.os.ms-windows.misc  | rec.sport.baseball | sci.med
# talk.religion.misc    | comp.sys.mac.hardware    | rec.sport.hockey   | misc.forsale

TEXT_DATA_ZIP = os.path.join(DATADIR, "20_newsgroup.zip")
zf = ZipFile(TEXT_DATA_ZIP, 'r')

print('Processing text dataset from', TEXT_DATA_ZIP, flush=True)

texts = []  # list of text samples
labels_index = {}  # dictionary mapping label name to numeric id
labels = []  # list of label ids
for fullname in sorted(zf.namelist()):
    parts = fullname.split('/')
    dirname = parts[1]
    fname = parts[2] if len(parts) > 2 else None
    zinfo = zf.getinfo(fullname)
    if zinfo.is_dir() and len(dirname) > 0:
        label_id = len(labels_index)
        labels_index[dirname] = label_id
        print(' ', dirname, label_id)
    elif fname is not None and fname.isdigit():
        with zf.open(fullname) as f:
            t = f.read().decode('latin-1')
            i = t.find('\n\n')  # skip header
            if 0 < i:
                t = t[i:]
            texts.append(t)
        labels.append(label_id)

labels = np.array(labels)
print('Found %s texts.' % len(texts))

# Split the data into a training set and a test set using
# scikit-learn's train_test_split().

TEST_SET = 4000

(texts_train, texts_test,
 labels_train, labels_test) = train_test_split(texts, labels,
                                               test_size=TEST_SET,
                                               shuffle=True, random_state=42)

(texts_train, texts_valid,
 labels_train, labels_valid) = train_test_split(texts_train, labels_train,
                                                shuffle=False,
                                                test_size=0.1)

print('Length of training texts:', len(texts_train),
      'labels:', len(labels_train))
print('Length of validation texts:', len(texts_valid),
      'labels:', len(labels_valid))
print('Length of test texts:', len(texts_test), 'labels:', len(labels_test))

# ## BERT
#
# Next we specify the pre-trained BERT model we are going to use. The
# model "bert-base-uncased" is the lowercased "base" model (12-layer,
# 768-hidden, 12-heads, 110M parameters).
#
# ### Tokenization
#
# We load the used vocabulary from the BERT model, and use the BERT
# tokenizer to convert the sentences into tokens that match the data
# the BERT model was trained on.

BERTMODEL='bert-base-uncased'
CACHE_DIR=os.path.join(DATADIR, 'transformers-cache')

tokenizer = BertTokenizer.from_pretrained(BERTMODEL,
                                          do_lower_case=True,
                                          cache_dir=CACHE_DIR)

# Next we tokenize all datasets. We set the maximum sequence lengths
# for our training and test sentences as MAX_LEN_TRAIN and
# MAX_LEN_TEST. The maximum length supported by the used BERT model is
# 512 tokens.

MAX_LEN_TRAIN, MAX_LEN_TEST = 128, 512

data_train = tokenizer(texts_train, padding=True, truncation=True,
                       return_tensors="tf", max_length=MAX_LEN_TRAIN)
data_valid = tokenizer(texts_valid, padding=True, truncation=True,
                       return_tensors="tf", max_length=MAX_LEN_TRAIN)
data_test = tokenizer(texts_test, padding=True, truncation=True,
                      return_tensors="tf", max_length=MAX_LEN_TEST)

print("Truncated tokenized first training message: As token ids:")
print(data_train["input_ids"][0])
print("Converted back to tokens:")
print(tokenizer.decode(data_train["input_ids"][0]))

# ### TF Datasets
#
# Let's now define our TF Datasets for training, validation, and
# test data. A batch size of 16 or 32 is often recommended for
# fine-tuning BERT on a specific task.

BATCH_SIZE = 32

dataset_train = tf.data.Dataset.from_tensor_slices((data_train.data,
                                                    labels_train))
dataset_train = dataset_train.shuffle(len(dataset_train)).batch(BATCH_SIZE)
dataset_valid = tf.data.Dataset.from_tensor_slices((data_valid.data,
                                                    labels_valid))
dataset_valid = dataset_valid.batch(BATCH_SIZE)
dataset_test = tf.data.Dataset.from_tensor_slices((data_test.data,
                                                   labels_test))
dataset_test = dataset_test.batch(BATCH_SIZE)

# ### Model initialization
#
# We now load a pretrained BERT model with a single linear
# classification layer added on top.

model = TFBertForSequenceClassification.from_pretrained(BERTMODEL,
                                                        cache_dir=CACHE_DIR,
                                                        num_labels=20)

# We use Adam as the optimizer, categorical crossentropy as loss, and
# then compile the model.
#
# LR is the learning rate for the Adam optimizer (2e-5 to 5e-5
# recommended for BERT finetuning).

LR = 2e-5

optimizer = tf.keras.optimizers.Adam(learning_rate=LR, epsilon=1e-08,
                                     clipnorm=1.0)
loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')

model.compile(optimizer=optimizer, loss=loss, metrics=[metric])

print(model.summary())

# ## Learning

logdir = os.path.join(os.getcwd(), "logs",
                      "20ng-bert-"+datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))
print('TensorBoard log directory:', logdir)
os.makedirs(logdir)
callbacks = [TensorBoard(log_dir=logdir)]

# For fine-tuning BERT on a specific task, 2-4 epochs is often
# recommended.

EPOCHS = 4

history = model.fit(dataset_train, validation_data=dataset_valid,
                    epochs=EPOCHS, verbose=2, callbacks=callbacks)

# ## Inference
#
# For a better measure of the quality of the model, let's see the
# model accuracy for the test messages.

test_scores = model.evaluate(dataset_test, verbose=2)
print("Test set %s: %.2f%%" % (model.metrics_names[1], test_scores[1]*100))

max_stars_repo_name: arcada-uas/intro-to-dl

max_issues_repo_name: arcada-uas/intro-to-dl

max_forks_repo_name: arcada-uas/intro-to-dl


text:
"""input_element_int_test.py

Author: Kangwon Lee (kangwonlee)
Date: 22 Oct 2017

Modified:
* 29 Dec 2017, RMM - updated file name and added header
"""

import numpy as np
from control import dcgain, ss, tf

class TestTfInputIntElement:
    """input_element_int_test

    Unit tests contributed as part of PR gh-158, "SISO tf() may not work
    with numpy arrays with numpy.int elements
    """

    def test_tf_den_with_numpy_int_element(self):
        num = 1
        den = np.convolve([1, 2, 1], [1, 1, 1])

        sys = tf(num, den)

        np.testing.assert_array_max_ulp(1., dcgain(sys))

    def test_tf_num_with_numpy_int_element(self):
        num = np.convolve([1], [1, 1])
        den = np.convolve([1, 2, 1], [1, 1, 1])

        sys = tf(num, den)

        np.testing.assert_array_max_ulp(1., dcgain(sys))

    # currently these pass
    def test_tf_input_with_int_element(self):
        num = 1
        den = np.convolve([1.0, 2, 1], [1, 1, 1])

        sys = tf(num, den)

        np.testing.assert_array_max_ulp(1., dcgain(sys))

    def test_ss_input_with_int_element(self):
        a = np.array([[0, 1],
                      [-1, -2]], dtype=int)
        b = np.array([[0],
                       [1]], dtype=int)
        c = np.array([[0, 1]], dtype=int)
        d = np.array([[1]], dtype=int)

        sys = ss(a, b, c, d)
        sys2 = tf(sys)
        np.testing.assert_array_max_ulp(dcgain(sys), dcgain(sys2))

    def test_ss_input_with_0int_dcgain(self):
        a = np.array([[0, 1],
                      [-1, -2]], dtype=int)
        b = np.array([[0],
                       [1]], dtype=int)
        c = np.array([[0, 1]], dtype=int)
        d = 0
        sys = ss(a, b, c, d)
        np.testing.assert_allclose(dcgain(sys), 0,
                                   atol=np.finfo(np.float).epsneg)

max_stars_repo_name: stelselim/python-control

max_issues_repo_name: stelselim/python-control

max_forks_repo_name: stelselim/python-control


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

from pathlib import Path
from typing import List, NamedTuple, Optional, Union

import dask.array as da
import numpy as np
import xarray as xr

###############################################################################

# IO Types
PathLike = Union[str, Path]
ArrayLike = Union[np.ndarray, da.Array]
MetaArrayLike = Union[ArrayLike, xr.DataArray]
ImageLike = Union[
    PathLike, ArrayLike, MetaArrayLike, List[MetaArrayLike], List[PathLike]
]

# Image Utility Types
class PhysicalPixelSizes(NamedTuple):
    Z: Optional[float]
    Y: Optional[float]
    X: Optional[float]

max_stars_repo_name: griffinfujioka/aicsimageio

max_issues_repo_name: griffinfujioka/aicsimageio

max_forks_repo_name: griffinfujioka/aicsimageio


text:
import functools
import cv2
from collections import defaultdict
import torch
import torch.nn.functional as F
import matplotlib.pyplot as plt
import numpy as np
from tensorboardX import SummaryWriter
from .comm import is_main_process

# ref:
# * https://github.com/jytime/Mask_RCNN_Pytorch/blob/master/visualize.py
# * https://github.com/chengsq/pytorch-lighthead/blob/master/lib/model/utils/net_utils.py

DEBUG = True

def need_main_process(func):
    def wrapper(*args, **kwargs):
        if is_main_process():
            if DEBUG:
                func(*args, **kwargs)
            else:
                try:
                    func(*args, **kwargs)
                    # print("visualization: ", args, kwargs)
                except:
                    print("Meet error when perform visualization: ", args, kwargs)

    return wrapper

def to_grayscale(image):
    """
    image: tensor, chw
    """
    image = image.float().abs().mean(dim=0)
    return image

def alpha_blend(image, mask, alpha=0.5, color=(0,255,0)):
    """
    image: tensor, 3*h*w
    mask: h*w
    """
    h, w = mask.shape[-2:]
    mask = mask.view(1, h, w)

    H, W = image.shape[-2:]
    mask = F.interpolate(mask[None], size=(H, W), mode='nearest')[0]  # 1*H*W
    mask = (mask > 0.5).float()

    new_image = torch.zeros_like(image)
    for i in range(image.shape[0]):
        new_image[i, :, :] = torch.where(
            mask == 1,
            image[i, :, :],
            image[i, :, :] * (1 - alpha) + alpha * color[i]
        )

    return new_image

def has_substrings(s, subs):
    """
    s: str
    subs: list[str]
    """
    for sub in subs:
        if sub not in s:
            return False
    return True

def normalize(feature, scale=255.0):
    """
    feature: tensor
    """
    m0, m1 = feature.min(), feature.max()
    return (feature - m0) / (m1 - m0 + 1e-4) * scale

def draw_boxes(feature, boxes, color, thickness=1):
    """
    add borders to a feature map
    feature: numpy array, hwc
    box: [[x0,y0,x1,y1], ...]
    """
    for i, box in enumerate(boxes):
        x0, y0, x1, y1 = box
        feature = cv2.rectangle(feature, (x0,y0), (x1,y1), color, thickness)
    return feature

def plot_boxes(image, boxes, scores, labels, classes=None, linestyle='solid',
               text_pos='tl', alpha=1, filename=None):
    """
    plot boxes on image
    boxes: array, xyxy
    """
    assert text_pos in ['tl', 'br']

    fig = plt.figure()
    if image is not None:
        plt.imshow(image)

    colors = plt.cm.hsv(np.linspace(0, 1, len(boxes) + 1)).tolist()
    currentAxis = plt.gca()

    for i in range(len(boxes)):
        pt = boxes[i][:4]
        label_name = classes[labels[i]] if len(labels) > 0 else 'C'
        if len(scores):
            display_txt = '%s:%s' % (label_name, round(scores[i], 4))
        else:
            display_txt = '%s' % label_name
        coords = (pt[0], pt[1]), pt[2] - pt[0] + 1, pt[3] - pt[1] + 1
        color = colors[i]
        currentAxis.add_patch(
            plt.Rectangle(*coords,
                          fill=False,
                          edgecolor=color,
                          linewidth=2,
                          linestyle=linestyle,
                          alpha=alpha)
        )
        if text_pos == 'tl':
            pos = pt[:2]
        else:
            pos = pt[2:]
        currentAxis.text(*pos, display_txt, bbox={'facecolor': color, 'alpha': 0.5})

    if filename is not None:
        plt.savefig(filename)
        plt.close()

    return fig

class SummaryWriterX(SummaryWriter):
    """
    see issue: https://github.com/tensorflow/tensorflow/issues/9512
    """
    def __init__(self, logdir, env, show_period, plot_period, classes, iteration=0):
        super(SummaryWriterX, self).__init__(logdir=logdir)
        self.classes = classes
        self.vis_env = env
        self.show_period = show_period
        self.plot_period = plot_period
        self.iteration = iteration
        self.topk_pred_boxes = 100
        self.inner_curves = defaultdict(dict)
        self.batch_idx = 0  # show which image in this batch
        self.event_list = []

    @need_main_process
    def update_iteration(self, iteration):
        self.iteration = iteration

    @need_main_process
    def update_curve_values(self, group, key, value):
        if self.is_plot():
            self.inner_curves[group].update({key: value})

    def is_show(self):
        return self.iteration % self.show_period == 0

    def is_plot(self):
        return self.iteration % self.plot_period == 0

    @need_main_process
    def vis_inner_curves(self):
        """
        no period checking here
        """
        for main_tag, data in self.inner_curves.items():
            self.vis_curves(data, main_tag)

    @need_main_process
    def vis_curves(self, data_dict, main_tag):
        """
        may call this func use different periods, thus no period checking here
        """
        for k, v in data_dict.items():
            if isinstance(v, torch.Tensor):
                data_dict[k] = v.item()  # scalar
            else:
                data_dict[k] = v
        # print(loss_dict)
        main_tag = self.vis_env + '/' + main_tag
        self.add_scalars(main_tag, data_dict, self.iteration)

    @need_main_process
    def vis_feat_hist(self, model, main_tag, filter='conv'):
        if not self.is_show():
            return

        main_tag = self.vis_env + '/' + main_tag
        with torch.no_grad():
            for i, (name, param) in enumerate(model.named_parameters()):
                if filter in name:
                    self.add_histogram(main_tag + '/' + name, param, 0)

    @need_main_process
    def vis_image(self, image, main_tag, targets=None):
        """
        show boxes on an image
        image: c*h*w
        boxes: BoxList
        """
        if not self.is_show():
            return

        main_tag = self.vis_env + '/' + main_tag
        image = image.cpu().data.numpy().transpose(1, 2, 0).astype(np.uint8)  # HWC
        if targets is None:
            fig = plt.figure()
            plt.imshow(image)
        else:
            boxes_ = targets.bbox.cpu().data.numpy().astype(int)
            if targets.has_field('labels'):
                labels_ = targets.get_field('labels').cpu().data.numpy()
            else:
                labels_ = []
            if targets.has_field('scores'):
                scores_ = targets.get_field('scores').cpu().data.numpy()
            else:
                scores_ = []

            if len(scores_):
                id_sort = np.argsort(scores_)[::-1]
                id_sort = id_sort[:self.topk_pred_boxes].tolist()
                scores_ = scores_[id_sort]
                if len(labels_):
                    labels_ = labels_[id_sort]
                boxes_ = boxes_[id_sort]

            boxes_ = boxes_.tolist()
            fig = plot_boxes(
                image,  # need HWC
                boxes_, scores_,
                labels_,
                classes=self.classes,
                linestyle='solid',
                text_pos='tl',
                alpha=1
            )
        self.add_figure(main_tag, fig, self.iteration)

    @need_main_process
    def vis_image_points(self, image, points, main_tag, color='r'):
        """
        show an image with some points plotted on it
        points: n*2, xy
        """
        if not self.is_show():
            return

        main_tag = self.vis_env + '/' + main_tag
        image = image.cpu().data.numpy().transpose(1, 2, 0).astype(np.uint8)  # HWC
        points = points.cpu().data.numpy()
        fig = plt.figure()
        plt.imshow(image)
        if len(points):
            plt.scatter(x=points[:,0], y=points[:,1], c=color, s=40, alpha=0.5)
        self.add_figure(main_tag, fig, self.iteration)

    @need_main_process
    def vis_feature_map(self, feature_map, main_tag):
        """
        visualize a feature map
        feature_map: c*h*w
        """
        if not self.is_show():
            return

        main_tag = self.vis_env + '/' + main_tag
        feature_map = normalize(to_grayscale(feature_map.data)).cpu().numpy().astype(np.uint8)  # hw
        feature_map = cv2.applyColorMap(feature_map, cv2.COLORMAP_JET)[:, :, ::-1]  # hwc

        fig = plt.figure()
        plt.imshow(feature_map)
        self.add_figure(main_tag, fig, self.iteration)
        # print(self.iteration, main_tag)

    @need_main_process
    def vis_mon_feature_map(self, feature_map, ranges, main_tag):
        """
        visualize a montage feature map
        feature_map: c*h*w
        """
        if not self.is_show():
            return

        if ranges is None:
            self.vis_feature_map(feature_map, main_tag)
            return

        main_tag = self.vis_env + '/' + main_tag
        # print(feature_map.shape)
        feature_map = to_grayscale(feature_map.data).cpu().data
        if isinstance(ranges, torch.Tensor):
            ranges = ranges.cpu().data.numpy()
        for i, rg in enumerate(ranges):
            x0, y0, x1, y1 = rg
            feature_map[y0:y1, x0:x1] = normalize(feature_map[y0:y1, x0:x1])
        feature_map = feature_map.numpy().astype(np.uint8)
        feature_map = cv2.applyColorMap(feature_map, cv2.COLORMAP_JET)  # hwc
        feature_map = draw_boxes(feature_map, ranges, (255, 255, 255), thickness=1)
        feature_map = feature_map[:, :, ::-1]  # to RGB

        fig = plt.figure()
        plt.imshow(feature_map)
        self.add_figure(main_tag, fig, self.iteration)
        # print(self.iteration, main_tag)

    @need_main_process
    def add_event(self, event, *args, **kwargs):
        if not self.is_show():
            return
        self.event_list.append(functools.partial(event, *args, **kwargs))

    @need_main_process
    def digest_events(self):
        for event in self.event_list:
            event()
        self.event_list = []


text:
import matplotlib.pyplot as plt
import numpy as np
import csv
import os

class DataLoder:

    def load_data(self, train_path, test_path, label_path):
        # Please make sure to change this function to load your train/validation/test data.
        # train_data = np.array([plt.imread('./data/test_images/0.jpg'), plt.imread('./data/test_images/1.jpg'),
        #               plt.imread('./data/test_images/2.jpg'), plt.imread('./data/test_images/3.jpg')])

        trainPath = train_path
        #trainPath = "./100_3dimage/"

        trainDir = os.listdir(trainPath)
        train_data = np.array(plt.imread(trainPath + trainDir[0])).reshape(1, 224, 224, 3)
        classData = {}

        with open(label_path, mode='r', encoding="UTF8") as f :
            data = csv.reader(f)
            for row in data :
                classData[row[0] + ".jpg"] = row[1]

        # with open('./trainSet.csv', mode='r', encoding="UTF8") as f :
        #     data = csv.reader(f)
        #     for row in data :
        #         classData[row[0].replace(".vir", ".jpg").replace('\ufeff','')] = row[1]

        classLabel = [int(classData[trainDir[0]])]
        i = 0
        for fil in trainDir[1:] :
            i += 1
            tempArry = np.array(plt.imread(trainPath + fil)).reshape(1, 224, 224, 3)
            train_data = np.append(train_data, tempArry, axis=0)
            classLabel.append(int(classData[fil]))
            if i % 100 == 0 :
                print(i, "files now loaded!")

        print(train_data.shape)

        # 인풋 데이터
        X_train = train_data
        # 트레인 데이터 라벨
        y_train = np.array(classLabel)
        print("train label")

        valPath = test_path

        valDir = os.listdir(valPath)
        val_data = np.array(plt.imread(valPath + valDir[0])).reshape(1, 224, 224, 3)

        valClassLabel = [int(classData[valDir[0]])]
        i = 0
        for fil in valDir[1:] :
            i += 1
            tempArry = np.array(plt.imread(valPath + fil)).reshape(1, 224, 224, 3)
            val_data = np.append(val_data, tempArry, axis=0)
            valClassLabel.append(int(classData[fil]))
            if i % 100 == 0 :
                print(i, "files now loaded!")

        print(val_data.shape)

        X_val = val_data
        y_val = np.array(valClassLabel)

        return X_train.astype('float32'), X_val.astype('float32'), y_train,  y_val


text:
import numpy as np
from bsds500 import BSDS
from datetime import datetime
import matplotlib.pyplot as plt

def main():
    now = datetime.now()
    Xtrain, ytrain, _ = BSDS.load(which='train')
    print Xtrain.shape, ytrain.shape, len(_)
    Xtest, ytest, _ = BSDS.load(which='test')
    print Xtest.shape, ytest.shape, len(_)
    Xval, yval, _ = BSDS.load(which='val')
    print Xval.shape, yval.shape, len(_)
    print 'loading all data took ', (datetime.now() - now)

    print 'displaying a random sample from training set'
    idx = np.random.randint(0, Xtrain.shape[0])
    plt.subplot(1,2,1)
    plt.imshow(Xtrain[idx,...].astype(np.uint8))
    plt.axis('off')

    plt.subplot(1,2,2)
    plt.imshow(ytrain[idx,...], cmap='gray')
    plt.axis('off')

    plt.show()
    print 'displaying a random sample from testing set'
    idx = np.random.randint(0, Xtest.shape[0])
    plt.subplot(1,2,1)
    plt.imshow(Xtest[idx,...].astype(np.uint8))
    plt.axis('off')

    plt.subplot(1,2,2)
    plt.imshow(ytest[idx,...], cmap='gray')
    plt.axis('off')

    plt.show()
    print 'displaying a random sample from validation set'
    idx = np.random.randint(0, Xval.shape[0])
    plt.subplot(1,2,1)
    plt.imshow(Xval[idx,...].astype(np.uint8))
    plt.axis('off')

    plt.subplot(1,2,2)
    plt.imshow(yval[idx,...], cmap='gray')
    plt.axis('off')

    plt.show()

    Xtrain, ytrain, _ = BSDS.list_load(which='train')
    print len(Xtrain), len(ytrain), len(_)
    Xtest, ytest, _ = BSDS.list_load(which='test')
    print len(Xtest), len(ytest), len(_)
    Xval, yval, _ = BSDS.list_load(which='val')
    print len(Xval), len(yval), len(_)

    print 'displaying a random sample from training set'
    idx = np.random.randint(0, len(Xtrain))
    plt.subplot(1,2,1)
    plt.imshow(Xtrain[idx].astype(np.uint8))
    plt.axis('off')

    plt.subplot(1,2,2)
    plt.imshow(ytrain[idx], cmap='gray')
    plt.axis('off')

    plt.show()
    print 'displaying a random sample from testing set'
    idx = np.random.randint(0, len(Xtest))
    plt.subplot(1,2,1)
    plt.imshow(Xtest[idx].astype(np.uint8))
    plt.axis('off')

    plt.subplot(1,2,2)
    plt.imshow(ytest[idx], cmap='gray')
    plt.axis('off')

    plt.show()
    print 'displaying a random sample from validation set'
    idx = np.random.randint(0, len(Xval))
    plt.subplot(1,2,1)
    plt.imshow(Xval[idx].astype(np.uint8))
    plt.axis('off')

    plt.subplot(1,2,2)
    plt.imshow(yval[idx], cmap='gray')
    plt.axis('off')

    plt.show()

if __name__ == '__main__':
    main()

max_stars_repo_name: erfannoury/SuperEdge

max_issues_repo_name: erfannoury/SuperEdge

max_forks_repo_name: erfannoury/SuperEdge


text:
import pandas as pd
import statsmodels.formula.api as smf
import statsmodels.api as sm
import statsmodels.genmod.families.links as L

from model.Banner import Banner
from model.AfterCorData import AfterCorData
from service.InputController import InputController
from service.DistributionService import DistributionService

class ModelProperties:
    def __init__(self):
        self.inputController = InputController()
        self.afterCorData = AfterCorData()
        self.distributionService = DistributionService()
        self.modelType = ''
        self.familyDistribution = ''
        self.stringModel = ''
        self.resModel = ''

    def setInputController(self, inputController):
        self.inputController = inputController

    def setAfterCorData(self, afterCorData):
        self.afterCorData = afterCorData

    def setDistributionService(self, distributionService):
        self.distributionService = distributionService

    def setModelType(self):
        self.inputController.setModellingMethod()
        self.modelType = self.inputController.modellingMethod

    def setStringModel(self):
        stringModel = self.afterCorData.modelTargetData.name + " ~ "
        for _ in self.afterCorData.newData.columns:
            if _ == self.afterCorData.newData.columns[-1]:
                stringModel += 'newData' \
                               + "['" + _ + "']"
            else:
                stringModel += 'newData' \
                               + "['" + _ + "']" + " + "

        if len(self.afterCorData.corBetweenData) != 0:
            stringModel += " + "

        for item in self.afterCorData.corBetweenData:
            for _ in item:
                if _ == item[-1]:
                    stringModel += 'newData' \
                                   + "['" + _ + "']"
                else:
                    stringModel += 'newData' \
                                   + "['" + _ + "']" + ":"

            if item == self.afterCorData.corBetweenData[-1]:
                stringModel += ''
            else:
                stringModel += " + "

        Banner.show(self.stringModel)

        self.stringModel = stringModel

    def setFamilyDistribution(self):
        comparator = list(self.distributionService.selectedDistribution.keys())[0]
        if comparator == 'Normal':
            self.familyDistribution = sm.families.Gaussian(
                link=L.identity
            )
        if comparator == 'Poisson':
            self.familyDistribution = sm.families.Poisson(
                link=L.log
            )
        if comparator == 'Exp':
            self.familyDistribution = sm.families.Gaussian(
                link=L.inverse_power
            )
        if comparator == 'Gamma':
            self.familyDistribution = sm.families.Gamma(
                link=L.inverse_power
            )
        if comparator == 'Binomial':
            self.familyDistribution = sm.families.Binomial(
                link=L.logit
            )

    def setResModel(self):
        newData = self.afterCorData.newData
        if self.modelType == 'lm'.casefold():
            self.resModel = smf.ols(
                self.stringModel,
                data=pd.concat([
                    self.afterCorData.newData,
                    self.afterCorData.modelTargetData
                ],
                    axis=1
                )
            ).fit()

        if self.modelType == 'glm'.casefold():
            self.resModel = smf.glm(
                self.stringModel,
                data=pd.concat([
                    self.afterCorData.newData,
                    self.afterCorData.modelTargetData
                ],
                    axis=1
                ),
                family=self.familyDistribution
            ).fit()

        Banner.show("# Model Summary")
        print(self.resModel.summary())

max_stars_repo_name: namnhGru/auto-modelling-ds-vnu

max_issues_repo_name: namnhGru/auto-modelling-ds-vnu

max_forks_repo_name: namnhGru/auto-modelling-ds-vnu

reason: import statsmodels


text:
from sklearn.decomposition import PCA
import numpy as np
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report

NUMERAL_SUFFIXES = ['-th', '-st', '-nd', '-rd', '-th']

def make_crossvalidation_data(split,x,y):
	train_data = []
	train_data_classes = []
	test_data = []
	test_data_classes = []

	for train_index in split[0]:
		train_data.append(x[train_index])
		train_data_classes.append(y[train_index])
	for test_index in split[1]:
		test_data.append(x[test_index])
		test_data_classes.append(y[test_index])

	return train_data, train_data_classes, test_data, test_data_classes

def numeral_str(numeral):
	num = abs(numeral)
	while num >= 10:
		num /= 10
	num = int(num)
	if num > 4:
		return str(numeral) + NUMERAL_SUFFIXES[4]
	else:
		return str(numeral) + NUMERAL_SUFFIXES[num]

def make_report(test_data_classes,results,digits=4):
	score = accuracy_score(test_data_classes,results)
	report = classification_report(test_data_classes,results,digits=4,output_dict=False)
	report_dict = classification_report(test_data_classes,results,output_dict=True)
	cm = confusion_matrix(test_data_classes,results)

	res = {}
	res['results'] = results
	res['accuracy_score'] = score
	res['classification_report'] = report
	res['classification_report_dict'] = report_dict
	res['confusion_matrix'] = cm

	return res

def dots_to_underscores(s):
	return s.replace(".", "_")

max_stars_repo_name: jbrowarczyk/jb-masters-thesis

max_issues_repo_name: jbrowarczyk/jb-masters-thesis

max_forks_repo_name: jbrowarczyk/jb-masters-thesis


text:
# -*- coding: utf-8 -*-
"""
Red John has committed another murder. But this time, he doesn't leave a red smiley behind. What he leaves behind is a
puzzle for Patrick Jane to solve. He also texts Teresa Lisbon that if Patrick is successful, he will turn himself in.
The puzzle begins as follows.

There is a wall of size 4xN in the victim's house. The victim also has an infinite supply of bricks of size 4x1 and 1x4
in her house. There is a hidden safe which can only be opened by a particular configuration of bricks in the wall. In
every configuration, the wall has to be completely covered using the bricks. There is a phone number written on a note
in the safe which is of utmost importance in the murder case. Gale Bertram wants to know the total number of ways in
which the bricks can be arranged on the wall so that a new configuration arises every time. He calls it M. Since Red
John is back after a long time, he has also gained a masters degree in Mathematics from a reputed university. So, he
wants Patrick to calculate the number of prime numbers (say P) up to M (i.e. <= M). If Patrick calculates P, Teresa
should call Red John on the phone number from the safe and he will surrender if Patrick tells him the correct answer.
Otherwise, Teresa will get another murder call after a week.

You are required to help Patrick correctly solve the puzzle.
"""
import math

__author__ = 'Danyang'

class Solution(object):
    def solve(self, cipher):
        """
        F(n) = F(n-4)+F(n-1)

        F(0) = 0
        F(1..3) = 1
        F(4) = 2

        :param cipher: the cipher
        """
        return self.sum_prime(self.get_configs(cipher))

    def get_configs(self, N):
        if N < 4:
            return 1

        F = [0 for _ in xrange(N + 1)]
        for i in xrange(1, 4):
            F[i] = 1
        F[4] = 2

        for i in xrange(5, N + 1):
            F[i] = F[i - 4] + F[i - 1]

        return F[N]

    def prime(self, n):
        """
        is_prime[2*j::j] = 0
        Sieve of Eratosthenes
        """
        is_prime = [1 for _ in xrange(n + 1)]
        for i in xrange(2):
            is_prime[i] = 0

        n_max = int(math.sqrt(len(is_prime)))
        for i in xrange(2, n_max + 1):
            for j in xrange(2 * i, len(is_prime), i):
                is_prime[j] = 0

        return sum(is_prime)

    def sum_prime(self, n):
        """
        HackerRank does not support numpy
        Sieve of Eratosthenes
        """
        import numpy as np

        is_prime = np.ones((n + 1,), dtype=bool)
        is_prime[:2] = 0
        N_max = int(np.sqrt(len(is_prime)))  # just to speed up
        for j in xrange(2, N_max):
            is_prime[2 * j::j] = False  # not start from j, since don't clean j itself
        return np.sum(is_prime)

if __name__ == "__main__":
    import sys

    f = open("1.in", "r")
    # f = sys.stdin
    solution = Solution()
    testcases = int(f.readline().strip())

    for t in xrange(testcases):
        # construct cipher
        cipher = int(f.readline().strip())

        # solve
        s = "%s\n" % (solution.solve(cipher))
        print s,

max_stars_repo_path: Red John is Back.py

max_stars_repo_name: Shaikh-Nabeel/HackerRankAlgorithms

max_issues_repo_path: Red John is Back.py

max_issues_repo_name: Shaikh-Nabeel/HackerRankAlgorithms

max_forks_repo_path: Red John is Back.py

max_forks_repo_name: Shaikh-Nabeel/HackerRankAlgorithms


text:
import numpy as np
import cv2

import pydicom
from pydicom.pixel_data_handlers.util import apply_voi_lut

__all__ = [
    "get_clahe",
    "get_uint8_rgb"
]

def _read_dicom_image(
    path,
    voi_lut=True,
    fix_monochrome=True,
    do_norm=True):

    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way
    dicom = pydicom.read_file(path)

    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to
    # "human-friendly" view
    if voi_lut:
        data = apply_voi_lut(dicom.pixel_array, dicom)
    else:
        data = dicom.pixel_array

    # depending on this value, X-ray may look inverted - fix that:
    if fix_monochrome and dicom.PhotometricInterpretation == "MONOCHROME1":
        data = np.max(data) - data

    if do_norm:
        data = data - np.min(data)
        data = (data / np.max(data)) if np.max(data) > 0 else data

    return data.astype(np.float32)

def get_clahe():
    return cv2.createCLAHE(clipLimit=2.0, tileGridSize=(10,10))

def _calc_image_features(image, clahe=None):
    # from https://www.kaggle.com/socom20/effdet-v2
    img_uint = (255 * image).astype(np.uint8)
    clahe = clahe or get_clahe()

    try:
        img_equ  = cv2.equalizeHist(img_uint) if np.max(img_uint) > 0 else img_uint
        img_clahe = clahe.apply(img_uint) if np.max(img_uint) > 0 else img_uint

        img_ret = np.concatenate(
            [
                image[:,:,None],
                img_clahe[:,:,None].astype(np.float32)  / 255,
                img_equ[:,:,None].astype(np.float32)  / 255,
                ],
            axis=-1)
    except:
        print('exception')
        img_ret = np.concatenate(
            [
                image[:,:,None],
                image[:,:,None],
                image[:,:,None],
                ],
            axis=-1)

    return img_ret

def get_uint8_rgb(dicom_path):
    """
    Reads dicom from path and returns rgb uint8 array
    where R: min-max normalized, G: CLAHE, B: histogram equalized.
    Image size remains original.
    """
    dcm = _read_dicom_image(dicom_path)
    feats = _calc_image_features(dcm)
    return (feats*255).astype(np.uint8)


text:
"""
Pedersen commitment
"""

from petlib.ec import EcGroup
import numpy as np

class PublicKey:
    """Simple public key for Pedersen's commitment scheme"""

    def __init__(self, group, n):
        """Create a public key for the Pedersen commitment scheme.

        Create a public key for a Pedersen commitment scheme in group `group` for n
        elements. We set the bases by hashing integers to points on the curve.

        Example:
            >>> G = EcGroup()
            >>> pk = PublicKey(G, 2)
        """

        self.group = group
        self.order = self.group.order()
        self.n = n
        self.generators = [
            self.group.hash_to_point(str(i).encode()) for i in range(n + 1)
        ]
        self.generators = np.array(self.generators)

    def commit(self, values, randomizer=None):
        """Commit to a list of values

        Returns two values: the Commitment and the randomizer used to create
        it. The randomizer can also be passed in as the optional parameter.

        Example:
            >>> G = EcGroup()
            >>> pk = PublicKey(G, 2)
            >>> com, rand = pk.commit([10, 20])
        """

        if len(values) != self.n:
            raise RuntimeError(
                "Incorrect length of input {0} expected {1}".format(len(values), self.n)
            )

        if randomizer is None:
            randomizer = self.group.order().random()

        powers = np.array(values + [randomizer])
        commitment = Commitment(np.sum(powers * self.generators))
        return commitment, randomizer

    def commit_reduced(self, values, reduced_n, randomizer=None):
        """Commit to a list of values with a reduced number of generators

        Returns two values as in the method above 'commit'
        """

        generators = self.generators[: reduced_n + 1]

        if len(values) != reduced_n:
            raise RuntimeError(
                "Incorrect length of input {} expected {}".format(
                    len(values), reduced_n
                )
            )

        if randomizer is None:
            randomizer = self.group.order().random()

        powers = np.array(values + [randomizer])
        commitment = Commitment(np.sum(powers * generators))
        return commitment, randomizer

    def export(self):
        # TODO: fix export to be robust
        export = bytes([0x00, 0xFF])
        for gen in self.generators:
            export += gen.export()
        return export

class Commitment:
    """A Pedersen commitment"""

    def __init__(self, commitment):
        self.commitment = commitment

    def __mul__(self, other):
        """Multiply two Pedersen commitments

        The commitment scheme is additively homomorphic. Multiplying two
        commitments gives a commitment to the pointwise sum of the original
        values.

        Example:
            >>> G = EcGroup()
            >>> pk = PublicKey(G, 2)
            >>> com1, rand1 = pk.commit([10, 20])
            >>> com2, rand2 = pk.commit([13, 19])
            >>> comsum = com1 * com2
            >>> com, rand = pk.commit([23, 39], randomizer=rand1 + rand2)
            >>> com == comsum
            True
        """

        return Commitment(self.commitment + other.commitment)

    def __pow__(self, exponent):
        """Raise Pedersen commitment to the power of a constant

        The commitment scheme is additively homomorphic. Raising a commitment
        to a constant power multiplies the committed vector by that constant.

        Example:
            >>> G = EcGroup()
            >>> pk = PublicKey(G, 2)
            >>> com1, rand1 = pk.commit([10, 20])
            >>> commul = com1 ** 10
            >>> com, rand = pk.commit([100, 200], randomizer=10 * rand1)
            >>> com == commul
            True
        """
        return Commitment(exponent * self.commitment)

    def __eq__(self, other):
        return self.commitment == other.commitment

    def export(self):
        return self.commitment.export()

if __name__ == "__main__":
    import doctest

    doctest.testmod()

max_stars_repo_name: spring-epfl/voteagain

max_issues_repo_name: spring-epfl/voteagain

max_forks_repo_name: spring-epfl/voteagain


text:
import numpy as np
import math

def distance(p1, p2):
    """
    Eucludean distance between two 2D points
        p1: Point()
        p2: Point()
    """
    ssq = (p1[0]-p2[0])**2 + (p1[1]-p2[1])**2
    return np.sqrt(ssq)

def rotate(x, y, beta):
    """
    Rotate vector(x,y) by beta radians counterclockwise
    https://matthew-brett.github.io/teaching/rotation_2d.html
    """
    x2 = math.cos(beta)*x - math.sin(beta)*y
    y2 = math.sin(beta)*x + math.cos(beta)*y
    return (x2, y2)

def get_chisquare(ci):
    """
    Upper-tail critical values of chi-square distribution
    with 2 degrees of freedom
    """
    if ci == 0.90:
        return 4.605
    elif ci == 0.95:
        return 5.991
    elif ci == 0.975:
        return 7.378
    else:
        return 5.991

def update_sigma(old_sigma, s_major, s_minor, ci=0.975):
    """
    Get a new covariance matrix from
        old_sigma: old covariance matrix
        s_major: updated semi-major axis
        s_minor: updated semi-minor axis
    """
    # update the eigenvalues
    l1 = (s_major**2)/get_chisquare(ci)
    l2 = (s_minor**2)/get_chisquare(ci)

    # Recover P matrix from the eigenvectors
    P = np.linalg.eig(old_sigma)[1]
    # Diagonal matrix of the new eigenvectors
    D = np.diag((l1, l2))
    # Updated covariance matrix
    cov = np.matmul(P, D)
    cov = np.matmul(cov, np.linalg.inv(P))
    return cov


text:
"""
GP_flowsheet project

                                Gaussian process

Author   : Antonio del Rio Chanona
Edited by: Edgar Ivan Sanchez Medina
Email: sanchez@mpi-magdeburg.mpg.de
Date: April 2020

Notes on editions (edited lines have the following at the end '#!!!'):

    1. The nugget term was added to the covariance matrix to prevent not
        positive definite matrix.
    2. A security section was added in the negative_loglikelihood function to
        ensure there are not nan or inf in K. This problem is
        encountered sometimes during the hyperparameters' optimization. In case
        any nan or inf is present, 10e80 is return as the final value.
    3. A security section was added before the cholesky decomposition to
        ensure there are not negative eigenvalues in K. This problem is
        encountered sometimes during the hyperparameters' optimization. In case
        any negative eigenvalue is present, 10e80 is return as the final value.
    4. Bounds were included during the hyperparameter optimization to prevent
        not positive definite Matrix.
    5. The function 'calc_cov_sample_casadi' was added to replace this calculation
        whenever the function 'GP_inference_casadi' is called.
    6. Introduce the Matern 3/2 and 5/2 kernel.

"""
import numpy as np
from scipy.spatial.distance import cdist
import sobol_seq
from scipy.optimize import minimize
from casadi import *

class GP:

    ###########################
    # --- initializing GP --- #
    ###########################
    def __init__(self, X, Y, kernel, multi_hyper):

        # GP variable definitions
        self.X, self.Y, self.kernel = X, Y, kernel
        self.n_point, self.nx_dim   = X.shape[0], X.shape[1]
        self.ny_dim                 = Y.shape[1]
        self.multi_hyper            = multi_hyper

        # normalize data
        self.X_mean, self.X_std     = np.mean(X, axis=0), np.std(X, axis=0)
        self.Y_mean, self.Y_std     = np.mean(Y, axis=0), np.std(Y, axis=0)
        self.X_norm, self.Y_norm    = (X-self.X_mean)/self.X_std, (Y-self.Y_mean)/self.Y_std

        # determine hyperparameters
        self.hypopt, self.invKopt   = self.determine_hyperparameters()

    #############################
    # --- Covariance Matrix --- #
    #############################

    def Cov_mat(self, kernel, X_norm, W, sf2):
        '''
        Calculates the covariance matrix of a dataset Xnorm
        --- decription ---
        '''
        nugget     = np.eye(X_norm.shape[0])* 1.1e-6                        #!!!
        if kernel == 'RBF':
            dist       = cdist(X_norm, X_norm, 'seuclidean', V=W)**2
            cov_matrix = sf2*np.exp(-0.5*dist) + nugget                     #!!!
            # Note: cdist =>  sqrt(sum(u_i-v_i)^2/V[x_i])
        elif kernel =='Matern_32':
            dist = cdist(X_norm, X_norm, 'seuclidean', V=W)
            t_1 = 1 + 3**0.5*dist
            t_2 = np.exp(-3**0.5*dist)
            cov_matrix = sf2 * t_1 * t_2 + nugget
        elif kernel =='Matern_52':
            dist = cdist(X_norm, X_norm, 'seuclidean', V=W)
            t_1  = 1 + 5**0.5* dist + 5/3*dist**2
            t_2  = np.exp(-5**2*dist)
            cov_matrix = sf2 * t_1 * t_2 + nugget
        else:
            print('ERROR no kernel with name ', kernel)

        return cov_matrix

    ################################
    # --- Covariance of sample --- #
    ################################

    def calc_cov_sample(self,xnorm,Xnorm,ell,sf2, kernel):
        '''
        Calculates the covariance of a single sample xnorm against the dataset Xnorm
        --- decription ---
        '''
        # internal parameters
        nx_dim = self.nx_dim

        if kernel == 'RBF':
            dist = cdist(Xnorm, xnorm.reshape(1,nx_dim), 'seuclidean', V=ell)**2
            cov_matrix = sf2 * np.exp(-.5*dist)
        elif kernel =='Matern_32':
            dist = cdist(Xnorm, xnorm.reshape(1,nx_dim), 'seuclidean', V=ell)
            t_1 = 1 + 3**0.5*dist
            t_2 = np.exp(-3**0.5*dist)
            cov_matrix = sf2 * t_1 * t_2
        elif kernel == 'Matern_52':
            dist = cdist(Xnorm, xnorm.reshape(1,nx_dim), 'seuclidean', V=ell)
            t_1  = 1 + 5**0.5* dist + 5/3*dist**2
            t_2  = np.exp(-5**2*dist)
            cov_matrix = sf2 * t_1 * t_2
        else:
            print('ERROR no kernel with name ', kernel)

        return cov_matrix

    def calc_cov_sample_casadi(self, xnorm, Xnorm, ell, sf2, kernel): #!!!
        '''
        Calculates covariance of single sample against data set using CASADI notation
        '''
        if kernel == 'RBF':
            dist = sum2((Xnorm - repmat(xnorm, len(Xnorm)))**2 /
                    repmat(transpose(ell), len(Xnorm)))
            cov_matrix = sf2 * exp(-.5*dist)
        elif kernel =='Matern_32':
            dist = sqrt(sum2((Xnorm - repmat(xnorm, len(Xnorm)))**2 /
                    repmat(transpose(ell), len(Xnorm))))
            t_1 = 1 + 3**0.5*dist
            t_2 = exp(-3**0.5*dist)
            cov_matrix = sf2 * t_1 * t_2
        elif kernel == 'Matern_52':
            dist = sqrt(sum2((Xnorm - repmat(xnorm, len(Xnorm)))**2 /
                    repmat(transpose(ell), len(Xnorm))))
            t_1  = 1 + 5**0.5* dist + 5/3*dist**2
            t_2  = exp(-5**2*dist)
            cov_matrix = sf2 * t_1 * t_2
        else:
            print('ERROR no kernel with name ', kernel)

        return cov_matrix

    ###################################
    # --- negative log likelihood --- #
    ###################################

    def negative_loglikelihood(self, hyper, X, Y):
        '''
        --- decription ---
        '''
        # internal parameters
        n_point, nx_dim = self.n_point, self.nx_dim
        kernel          = self.kernel

        W               = np.exp(2*hyper[:nx_dim])   # W <=> 1/lambda
        sf2             = np.exp(2*hyper[nx_dim])    # variance of the signal
        sn2             = np.exp(2*hyper[nx_dim+1])  # variance of noise

        K       = self.Cov_mat(kernel, X, W, sf2)  # (nxn) covariance matrix (noise free)
        K       = K + (sn2 + 1e-8)*np.eye(n_point) # (nxn) covariance matrix
        K       = (K + K.T)*0.5                    # ensure K is simetric

        #print(np.linalg.eig(K)[0])                 # Print eigenvalues of K
        # Ensure there is no inf or nan in K                                #!!!
        if np.any(np.isnan(K)) or np.any(np.isinf(K)):
            return 10e80
        # Ensure K has no negative eigenvalues                              #!!!
        if np.any(np.linalg.eig(K)[0] < 0):
            return 10e80
        L       = np.linalg.cholesky(K)            # do a cholesky decomposition
        logdetK = 2 * np.sum(np.log(np.diag(L)))   # calculate the log of the determinant of K the 2* is due to the fact that L^2 = K
        invLY   = np.linalg.solve(L,Y)             # obtain L^{-1}*Y
        alpha   = np.linalg.solve(L.T,invLY)       # obtain (L.T L)^{-1}*Y = K^{-1}*Y
        NLL     = np.dot(Y.T,alpha) + logdetK      # construct the NLL

        return NLL

    ############################################################
    # --- Minimizing the NLL (hyperparameter optimization) --- #
    ############################################################

    def determine_hyperparameters(self):
        '''
        --- decription ---
        Notice we construct one GP for each output
        '''
        # internal parameters
        X_norm, Y_norm  = self.X_norm, self.Y_norm
        nx_dim, n_point = self.nx_dim, self.n_point
        kernel, ny_dim  = self.kernel, self.ny_dim
        Cov_mat         = self.Cov_mat

        lb               = np.array([-3.]*(nx_dim+1) + [-8.])  # lb on parameters (this is inside the exponential)
        ub               = np.array([3.]*(nx_dim+1) + [ 4.])   # lb on parameters (this is inside the exponential)
        bounds           = np.hstack((lb.reshape(nx_dim+2,1),
                                      ub.reshape(nx_dim+2,1)))

        #bounds           = [[1, 3], [1, 3], [1, 3]]

        multi_start      = self.multi_hyper                   # multistart on hyperparameter optimization
        multi_startvec   = sobol_seq.i4_sobol_generate(nx_dim + 2,multi_start)

        options  = {'disp':False,'maxiter':10000}             # solver options
        hypopt   = np.zeros((nx_dim+2, ny_dim))               # hyperparams w's + sf2+ sn2 (one for each GP i.e. output var)
        localsol = [0.]*multi_start                           # values for multistart
        localval = np.zeros((multi_start))                    # variables for multistart

        invKopt = []
        # --- loop over outputs (GPs) --- #
        for i in range(ny_dim):
            # --- multistart loop --- #
            for j in range(multi_start):
                #print('multi_start hyper parameter optimization iteration = ',j,'  input = ',i)
                hyp_init    = lb + (ub-lb)*multi_startvec[j,:]
                # --- hyper-parameter optimization --- #
                res = minimize(self.negative_loglikelihood,hyp_init,args=(X_norm,Y_norm[:,i])\
                               ,method='SLSQP',options=options,tol=1e-12, bounds=bounds) #!!!
                #print(' Initial guess for hyperparameters: ',hyp_init )
                #print(' Bounds for hyper parameters: ', bounds)
                #print(' Optimal hyperparameters: ', res.x)
                #print(' Optimal Objective function value: ', res.fun )
                localsol[j] = res.x
                localval[j] = res.fun

            # --- choosing best solution --- #
            minindex    = np.argmin(localval)
            hypopt[:,i] = localsol[minindex]
            ellopt      = np.exp(2.*hypopt[:nx_dim,i])
            sf2opt      = np.exp(2.*hypopt[nx_dim,i])
            sn2opt      = np.exp(2.*hypopt[nx_dim+1,i]) + 1e-8

            # --- constructing optimal K --- #
            Kopt        = Cov_mat(kernel, X_norm, ellopt, sf2opt) + sn2opt*np.eye(n_point)
            # --- inverting K --- #
            invKopt     += [np.linalg.solve(Kopt,np.eye(n_point))]

        return hypopt, invKopt

    ########################
    # --- GP inference --- #
    ########################

    def GP_inference_np(self, x):
        '''
        --- decription ---
        '''
        nx_dim                   = self.nx_dim
        kernel, ny_dim           = self.kernel, self.ny_dim
        hypopt, Cov_mat          = self.hypopt, self.Cov_mat
        stdX, stdY, meanX, meanY = self.X_std, self.Y_std, self.X_mean, self.Y_mean
        calc_cov_sample          = self.calc_cov_sample
        invKsample               = self.invKopt
        Xsample, Ysample         = self.X_norm, self.Y_norm
        # Sigma_w                = self.Sigma_w (if input noise)

        xnorm = (x - meanX)/stdX
        mean  = np.zeros(ny_dim)
        var   = np.zeros(ny_dim)
        # --- Loop over each output (GP) --- #
        for i in range(ny_dim):
            invK           = invKsample[i]
            hyper          = hypopt[:,i]
            ellopt, sf2opt = np.exp(2*hyper[:nx_dim]), np.exp(2*hyper[nx_dim])
            # --- determine covariance of each output --- #
            k       = calc_cov_sample(xnorm,Xsample,ellopt,sf2opt,kernel)
            mean[i] = np.matmul(np.matmul(k.T,invK),Ysample[:,i])
            var[i]  = sf2opt - np.matmul(np.matmul(k.T,invK),k)
            #var[i] = sf2opt + Sigma_w[i,i]/stdY[i]**2 - np.matmul(np.matmul(k.T,invK),k) (if input noise)

        # --- compute un-normalized mean --- #
        mean_sample = mean*stdY + meanY
        var_sample  = var*stdY**2

        return mean_sample, var_sample

    def GP_inference_casadi(self, x): #!!!
        '''
        --- decription ---
        '''
        nx_dim                   = self.nx_dim
        kernel, ny_dim           = self.kernel, self.ny_dim
        hypopt, Cov_mat          = self.hypopt, self.Cov_mat
        stdX, stdY, meanX, meanY = self.X_std.reshape(1,-1), self.Y_std, self.X_mean.reshape(1,-1), self.Y_mean
        calc_cov_sample          = self.calc_cov_sample_casadi
        invKsample               = self.invKopt
        Xsample, Ysample         = self.X_norm, self.Y_norm
        # Sigma_w                = self.Sigma_w (if input noise)

        xnorm = (x - meanX)/stdX
        # --- Loop over each output (GP) --- #
        for i in range(ny_dim):
            if ny_dim != 1:
                raise Exception ('Current implementation only supports a single-output GP')
            invK           = invKsample[i]
            hyper          = hypopt[:,i]
            ellopt, sf2opt = np.exp(2*hyper[:nx_dim]), np.exp(2*hyper[nx_dim])
            # --- determine covariance of each output --- #
            k       = calc_cov_sample(xnorm, Xsample, ellopt, sf2opt,kernel)
            mean    = (k.T @ invK) @ Ysample[:,i].reshape(-1,1)
            var     = sf2opt - (k.T @ invK) @ k
            #var[i] = sf2opt + Sigma_w[i,i]/stdY[i]**2 - np.matmul(np.matmul(k.T,invK),k) (if input noise)

        # --- compute un-normalized mean --- #
        mean_sample = mean*stdY + meanY
        var_sample  = var*stdY**2

        return mean_sample, var_sample