text:
import numpy as np

class StirlingEngine:
    """
    Numerically efficient engine for computing and storing computed Stirling numbers.

    CONSTRUCTOR PARAMETERS
    - Nmax: largest integer n for which s(n,m) will need to be computed

    PRIVATE ATTRIBUTES
    - s_memo_, slog_memo_: running tables of previously computed values
    """

    def __init__(self, Nmax):
        self.s_memo_ = np.full((Nmax, Nmax), np.nan)
        self.slog_memo_ = np.full((Nmax, Nmax), np.nan)

    def stirling(self, n, m):
        """
        Computes an unsigned Stirling number of the first kind.
        Uses dynamic programming to store previously computed s(n,m) values,
        as this is a repeatedly-called recursive algorithm.
        """

        assert n < self.s_memo_.shape[0] and m < self.s_memo_.shape[0]
        # If this has already been computed, return stored value
        if not np.isnan(self.s_memo_[n, m]):
            return self.s_memo_[n, m]
        else:
            return_val = np.nan

            # Base cases
            if (n == 0 and m == 0) or (n == 1 and m == 1):
                return_val = 1
            elif (n > 0 and m == 0) or m > n:
                return_val = 0
            # Recursion relation
            else:
                return_val = self.stirling(n-1, m-1) + (n-1)*self.stirling(n-1, m)

            self.s_memo_[n, m] = return_val
            return return_val

    def stirlog(self, n, m):
        """
        Computes the natural logarithm of an unsigned Stirling number,
        using the same dynamic programming approach as above.
        If s(n,m) = 0, this gets returned as -inf (np.exp(-inf) == 0.0)

        This is the preferred function, as stirling() can encounter overflow errors.
        """

        assert n < self.slog_memo_.shape[0] and m < self.slog_memo_.shape[0]
        # If this has already been computed, return stored value
        if not np.isnan(self.slog_memo_[n, m]):
            return self.slog_memo_[n, m]
        else:
            return_val = np.nan

            # Base cases
            if (n == 0 and m == 0) or (n == 1 and m == 1):
                return_val = 0
            elif (n > 0 and m == 0) or m > n:
                return_val = -np.inf
            # Recursion relation
            else:
                log_s1, log_s2 = self.stirlog(n-1, m-1), self.stirlog(n-1, m)
                # If s1 == 0 (log_s1 == -inf), just return (n-1)*log_s2
                # By definition, must have s2 > s1, so only need to check s1
                if np.isfinite(log_s1):
                    val = (n-1) * np.exp(log_s2 - log_s1)
                    # If there is overflow/underflow in `val`, approximate log(1+x) = log(x)
                    if np.isfinite(val):
                        return_val = log_s1 + np.log1p(val)
                    else:
                        return_val = log_s2 + np.log(n-1)
                else:
                    return_val = log_s2 + np.log(n-1)

            self.slog_memo_[n, m] = return_val
            return return_val

max_stars_repo_name: quinnfrank/HDP-py

max_issues_repo_name: quinnfrank/HDP-py

max_forks_repo_name: quinnfrank/HDP-py


text:
# Copyright (c) 2019 Microsoft Corporation.
# Copyright (c) 2018 Ricky Tian Qi Chen
# Licensed under the MIT license.

import math
import os
import torch
from torch.utils.data import DataLoader
from torch.autograd import Variable

import lib.utils as utils
from metric_helpers.loader import load_model
import vae_quant

import png
import numpy as np
import random

from elbo_decomposition import elbo_decomposition
from plot_latent_vs_true import plot_vs_gt_shapes, plot_vs_gt_faces, plot_vs_gt_cars3d, plot_vs_gt_ICA  # noqa: F401

import disentanglement_metrics

def evaluate(args, outputdir, vae, dataset, prefix=''):
    if os.path.exists(os.path.join(outputdir, prefix+'combined_data.pth')):
        return

    # Report statistics
    vae.eval()
    dataset_loader = DataLoader(dataset, batch_size=1000, num_workers=0, shuffle=True)

    if not os.path.exists(outputdir):
        os.makedirs(outputdir)

    logpx, dependence, information, dimwise_kl, analytical_cond_kl, elbo_marginal_entropies, elbo_joint_entropy = \
        elbo_decomposition(vae, dataset_loader)
    torch.save({
        'args': args,
        'logpx': logpx,
        'dependence': dependence,
        'information': information,
        'dimwise_kl': dimwise_kl,
        'analytical_cond_kl': analytical_cond_kl,
        'marginal_entropies': elbo_marginal_entropies,
        'joint_entropy': elbo_joint_entropy
    }, os.path.join(outputdir, prefix+'elbo_decomposition.pth'))

    metric, metric_marginal_entropies, metric_cond_entropies = eval('disentanglement_metrics.mutual_info_metric_' + args.dataset)(vae, dataset_loader.dataset, eval_subspaces=False)
    torch.save({
        'args': args,
        'metric': metric,
        'marginal_entropies': metric_marginal_entropies,
        'cond_entropies': metric_cond_entropies,
    }, os.path.join(outputdir, prefix+'disentanglement_metric.pth'))
    print('logpx: {:.2f}'.format(logpx))
    print('MIG: {:.2f}'.format(metric))

    eval('plot_vs_gt_' + args.dataset)(vae, dataset_loader.dataset, os.path.join(outputdir, prefix+'gt_vs_latent.png'))

    torch.save({
        'args': args,
        'logpx': logpx,
        'dependence': dependence,
        'information': information,
        'dimwise_kl': dimwise_kl,
        'analytical_cond_kl': analytical_cond_kl,
        'elbo_marginal_entropies': elbo_marginal_entropies,
        'elbo_joint_entropy': elbo_joint_entropy,
        'metric': metric,
        'metric_marginal_entropies': metric_marginal_entropies,
        'metric_cond_entropies': metric_cond_entropies,
    }, os.path.join(outputdir, prefix+'combined_data.pth'))

    if args.dist == 'lpnested':
        eval('plot_vs_gt_' + args.dataset)(vae, dataset_loader.dataset, os.path.join(outputdir, 'gt_vs_grouped_latent.png'), eval_subspaces=True)

        metric_subspaces, metric_marginal_entropies_subspaces, metric_cond_entropies_subspaces = eval('disentanglement_metrics.mutual_info_metric_' + args.dataset)(vae, dataset_loader.dataset, eval_subspaces=True)
        torch.save({
            'args': args,
            'metric': metric_subspaces,
            'marginal_entropies': metric_marginal_entropies_subspaces,
            'cond_entropies': metric_cond_entropies_subspaces,
        }, os.path.join(outputdir, 'disentanglement_metric_subspaces.pth'))
        print('MIG grouped by subspaces: {:.2f}'.format(metric_subspaces))

        torch.save({
            'args': args,
            'logpx': logpx,
            'dependence': dependence,
            'information': information,
            'dimwise_kl': dimwise_kl,
            'analytical_cond_kl': analytical_cond_kl,
            'elbo_marginal_entropies': elbo_marginal_entropies,
            'elbo_joint_entropy': elbo_joint_entropy,
            'metric': metric,
            'metric_marginal_entropies': metric_marginal_entropies,
            'metric_cond_entropies': metric_cond_entropies,
            'metric_subspaces': metric_subspaces,
            'metric_marginal_entropies_subspaces': metric_marginal_entropies_subspaces,
            'metric_cond_entropies_subspaces': metric_cond_entropies_subspaces
        }, os.path.join(outputdir, 'combined_data.pth'))

def process_subdirectories(args, outputdir, use_cuda=True):
    loader = vae_quant.setup_data_loaders(args.dataset, batch_size=16, use_cuda=use_cuda)
    loader = DataLoader(dataset=loader.dataset, batch_size=16, num_workers=0, shuffle=True, pin_memory=True, worker_init_fn=np.random.seed(0))
    imginput = iter(loader)
    xs = imginput.next()
    for dirpath, dirnames, filenames in os.walk(args.directory):
        for i,dir in enumerate(dirnames):
            checkpt = os.path.join(args.directory,dir,'checkpt-0000.pth')
            print(checkpt)
            vae, cpargs = load_model(checkpt, use_cuda=use_cuda)
            if vae is None:
                continue
            if cpargs.dataset!=args.dataset:
                print("error: model was trained on different dataset: {}".format(cpargs.dataset))
                exit(1)
            if loader==None:
                loader = vae_quant.setup_data_loaders(cpargs.dataset, batch_size=16, use_cuda=use_cuda)

            this_outputdir = os.path.join(outputdir, dir)
            evaluate(cpargs, this_outputdir, vae, loader.dataset, prefix='')

# python vis_traversal.py --directory ../paramsearch-celeba-10-eu2-mseloss/ --gpu -1 --dataset celeba --save ../test-celeba-images-mse2/

if __name__ == '__main__':
    # for deterministic results
    torch.manual_seed(0)
    torch.cuda.manual_seed(0)
    np.random.seed(0)
    random.seed(0)
    torch.backends.cudnn.deterministic=True

    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--checkpt', type=str, default='')
    parser.add_argument('--directory', type=str, default='')
    parser.add_argument('--gpu', type=int, default=0)
    parser.add_argument('--save', type=str, default='.')
    parser.add_argument('--prefix', type=str, default='')
    parser.add_argument('-d', '--dataset', default='shapes', type=str, help='dataset name',
        choices=['shapes', 'faces', 'celeba', 'cars3d', '3dchairs', 'ICA'])
    args = parser.parse_args()

    # create output directory if it does not exist
    outputdir = args.save
    if not os.path.exists(outputdir):
        os.makedirs(outputdir)

    if args.gpu != -1:
        torch.cuda.set_device(args.gpu)
        use_cuda = True
    else:
        use_cuda = False

    if args.checkpt!='':
        vae, cpargs = load_model(args.checkpt, use_cuda=use_cuda)
    elif args.directory!='':
        process_subdirectories(args, outputdir, use_cuda=use_cuda)
        exit(0)
    else:
        print('error: either a checkpoint file or a directory containing experiments must be specified!')
        exit(1)

    # dataset loader
    loader = vae_quant.setup_data_loaders(args.dataset, batch_size=16, use_cuda=use_cuda)

    evaluate(cpargs, outputdir, vae, loader.dataset)

ext: py

max_forks_repo_name: microsoft/isa-vae

reason: import numpy


text:
import os
import sys
import pickle
sys.path.append(os.path.abspath(os.path.dirname(__file__) + '/' + '../..'))

import matplotlib as mpl

mpl.use('pgf')
pgf_with_latex = {
    "pgf.texsystem": "xelatex",
    "pgf.rcfonts": False,
    "text.usetex": True,
    "font.family": "Times New Roman",
    "pgf.preamble": [
        r"\usepackage{fontspec}",
        r"\setmainfont{Times New Roman}",
        r"\usepackage{unicode-math}",
        r"\setmathfont{xits-math.otf}"
        ]
    }
mpl.rcParams.update(pgf_with_latex)

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import matplotlib.patches as mpatches
import statsmodels.api as sm
import statsmodels.formula.api as smf
import pystan
import scipy

from python.tools import (
    clean_folder,
    estimate_R
)

################
## Parameters ##
################

output_folder = './estimate_R/output/example_filter_smoother'
input_folder = './estimate_R/input/example_filter_smoother'
days_infectious = 7
num_iter = 10000 # MCMC iterations
num_warmup = 2000 # MCMC warmup period
num_chains = 3 # MCMC chains
mean_mu0    = 0.35 # Prior for initial state
std_dev_mu0 = 0.50
sig_levels = [5, 35] # Significance levels for credible intervals
rng = np.random.RandomState(20200504) # For setting seed in Stan's estimation
np.random.seed(19801980) # For reproducible Bayesian filtering

# Implied parameters
gamma = 1 / float(days_infectious)

###############
## Load data ##
###############

clean_folder(output_folder)

# Load data for China
df = pd.read_csv('./estimate_R/input/estimate_R_STAN/dataset.csv')
mask = df['Country/Region'] == 'China'
df = df.loc[mask, ]
df.reset_index(inplace = True)
del df['index']
df['Date'] = pd.to_datetime(df['Date'])

###################
## Get estimates ##
###################

# Frequentist estimates
mod = sm.tsa.UnobservedComponents(df['gr_infected_{}'.format(days_infectious)].values, 'local level')
res_freq = mod.fit(disp = False)
df_freq = pd.DataFrame()
df_freq['Date'] = df['Date']
df_freq['R_filtered'] = 1 + gamma ** (-1) * res_freq.filtered_state[0]
df_freq['R_smoothed'] = 1 + gamma ** (-1) * res_freq.smoothed_state[0]
df_freq.to_csv('{}/frequentist_estimates.csv'.format(output_folder), index = False)

# Bayesian estimates

# Load STAN models
stm_missing = pickle.load(open('./estimate_R/input/estimate_R_STAN/model_missing.pkl', 'rb'))
stm_no_missing = pickle.load(open('./estimate_R/input/estimate_R_STAN/model_no_missing.pkl', 'rb'))
res_Bayes = estimate_R(y = df['gr_infected_{}'.format(days_infectious)].values,
                       gamma = 1 / days_infectious,
                       stm_missing = stm_missing,
                       stm_no_missing = stm_no_missing,
                       num_iter = num_iter,
                       num_chains = num_chains,
                       num_warmup = num_warmup,
                       rng = rng,
                       sig_levels = sig_levels,
                       full_output = True)

# Collect results
df_smoother_Bayesian = pd.DataFrame()
df_smoother_Bayesian['Date'] = df['Date']
df_smoother_Bayesian['R'] = res_Bayes['R']
df_smoother_Bayesian['lb_65'] = res_Bayes['lb_65']
df_smoother_Bayesian['ub_65'] = res_Bayes['ub_65']
df_smoother_Bayesian.to_csv('{}/bayesian_smoother.csv'.format(output_folder), index = False)

# Get Bayesian filtered states
fit_res = res_Bayes['stan_fit'].extract(permuted = True)
dist_var_irregular = fit_res['precision_irregular'] ** (-1) # Draws from the estimated distribution from Stan
dist_signal_to_noise = fit_res['signal_to_noise'] # Draws from the estimated distribution from Stan

# Monte Carlo loop -- take random draws from the distributions
# of variances for the irregular and level components
# and run the Kalman filter
res_filter_Bayesian = []
for mm in range(num_iter):
    # Pick a random draw for the variances of the
    # irregular and level components from the Bayesian estimates
    var_irregular = np.random.choice(dist_var_irregular, 1)[0]
    var_level = var_irregular * np.random.choice(dist_signal_to_noise, 1)[0]

    # Run Kalman filter using these variances
    mod = sm.tsa.UnobservedComponents(df['gr_infected_{}'.format(days_infectious)].values, 'local level')
    mod.initialize_known(np.array([mean_mu0]), np.array([[std_dev_mu0]]))
    res_temp = mod.smooth(params = np.array([var_irregular, var_level]))

    df_temp = pd.DataFrame()
    df_temp['R'] = 1 + (1 / gamma) * res_temp.filtered_state[0]
    df_temp['R_se'] = (1 / gamma) * res_temp.filtered_state_cov[0][0] ** 0.5

    alpha = [0.05, 0.35]
    names = ['95', '65']
    for aa, name in zip(alpha, names):
        t_crit = scipy.stats.norm.ppf(1 - aa / 2)
        df_temp['ub_{}'.format(name)] = df_temp['R'] + t_crit * df_temp['R_se']
        df_temp['lb_{}'.format(name)] = np.maximum(df_temp['R'] - t_crit * df_temp['R_se'], 0.0)
    df_temp['MC_id'] = mm
    df_temp['Date'] = df['Date']
    res_filter_Bayesian.append(df_temp)
df_filter_Bayesian = pd.concat(res_filter_Bayesian)
df_filter_Bayesian = df_filter_Bayesian.groupby('Date').mean()[['R', 'lb_65', 'ub_65']].reset_index()
df_filter_Bayesian.to_csv('{}/bayesian_filter.csv'.format(output_folder), index = False)

###############
## Get graph ##
###############

fig, ax = plt.subplots(figsize = (5.0, 4.0))
plt.plot(df_smoother_Bayesian['Date'], df_smoother_Bayesian['R'], '-r', linewidth = 1.5, label = 'Bayesian Smoother')
plt.plot(df_freq['Date'], df_freq['R_smoothed'], '--k', label = 'Classical Smoother')
plt.fill_between(df_smoother_Bayesian['Date'],
                 df_smoother_Bayesian['lb_65'],
                 df_smoother_Bayesian['ub_65'],
                 color = 'r', alpha = 0.15)
plt.plot(df_filter_Bayesian['Date'], df_filter_Bayesian['R'], '-b',
         linewidth = 1.5, label = 'Bayesian Filter')
plt.fill_between(df_filter_Bayesian['Date'],
                 df_filter_Bayesian['lb_65'],
                 df_filter_Bayesian['ub_65'],
                 color = 'b', alpha = 0.15)
plt.plot(df_freq['Date'], df_freq['R_filtered'], ':k', label = 'Classical Filter')
plt.legend(frameon = False, fontsize = 12)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b-%d'))
plt.gca().xaxis.set_major_locator(mdates.DayLocator(interval = 14))
plt.axhline(1, linestyle = '--', color = 'k', alpha = 0.25)
plt.axhline(0, linestyle = '--', color = 'k', alpha = 0.25)
plt.ylabel('Effective Repr. Number ($\\mathcal{R}$)', fontsize = 12)
fig.savefig("{}/filter_smoother.png".format(output_folder), bbox_inches = 'tight', dpi = 600)
fig.savefig("{}/filter_smoother.pgf".format(output_folder), bbox_inches = 'tight', dpi = 600)

reason: import numpy,import scipy,import statsmodels


text:
import numpy as np
import json
import os
import logging
import click
from skimage import io

from deepfigures import settings

# Get all the directories in the folder
DPI = 100

logger = logging.getLogger(__name__)

def visualize_box(box, image, thick = 3):
    """
    Drawing the rectangular bounding boxes on the images of pdf pages
        @PARAM:
            box: list of 4 int, the four coordinates of the bounding boxes
            image: numpy array, the 3D numpy array representing images
            thick: int, the thickness of the bounding boxes depicted, DEFAULT = 3 pixels
        @RETURN:
            I_new: the images with bounding boxes on them
    """
    height = image.shape[0]
    width = image.shape[1]
    I_new = np.copy(image)
    t = max(box[0]-thick,0)
    b = min(box[2]+thick,height)
    l = max(box[1]-thick,0)
    r = min(box[3]+thick,width)
    for h in range(t,box[0]):
        I_new[h, l:r, 0] = 0
        I_new[h, l:r, 1] = 255
        I_new[h, l:r, 2] = 0
    for h in range(box[0],box[2]):
        I_new[h, l:box[1], 0] = 0
        I_new[h, l:box[1], 1] = 255
        I_new[h, l:box[1], 2] = 0
        I_new[h, box[3]:r, 0] = 0
        I_new[h, box[3]:r, 1] = 255
        I_new[h, box[3]:r, 2] = 0
    for h in range(box[2],b):
        I_new[h, l:r, 0] = 0
        I_new[h, l:r, 1] = 255
        I_new[h, l:r, 2] = 0
    return I_new

def depict_boxes(dir, dpi, images, boxes, Error_Margin, thick = 3):
    """
    Drawing the rectangular bounding boxes on the images of pdf pages
        @PARAM:
            dir: the directory where pdf is saved
            dpi: the dpi in the setting
            images: list of numpy arrays, the filenames of the images of the pdf pages, in PNG
            boxes: list of dictionaries, the coordinates of bounding boxes
            thick: int, the thickness of the bounding boxes depicted, DEFAULT = 3 pixels
        @RETURN:
            Void
    """
    SAVE_PATH = os.path.join(dir,"images_with_boxes_dpi_" + str(dpi))  # Directory to save the images with boxes
    IMAGE_NUM = len(images)
    PDF_NAME = next(x for x in os.listdir(dir) if x[-4:] == ".pdf").replace(".pdf", "").strip(" ")                                      # Total number of pdf pages

    try:
        os.makedirs(SAVE_PATH)
    except OSError: # stops if figures have already been extracted
        if len(os.listdir(SAVE_PATH)) > IMAGE_NUM:
            return

    for page_num in range(IMAGE_NUM):
        image_new = images[page_num]
        num_images = len(boxes[page_num])
        if  num_images > 0:
            for box_num in range(num_images):
                # Reset cropped each time so we start fresh with the full page
                cropped = image_new
                box_idx = boxes[page_num][box_num]
                try:
                    box = [int(box_idx["y1"]) - Error_Margin, int(box_idx["x1"]) - Error_Margin,
                        int(box_idx["y2"]) + Error_Margin, int(box_idx["x2"]) + Error_Margin]                          # Get box boundary coordinates
                    image_new = visualize_box(box = box, image = image_new)
                    cropped = image_new[box[0]:box[2], box[1]:box[3]]
                    # We are going to save each image on every page
                    io.imsave(os.path.join(SAVE_PATH, PDF_NAME + "Page" + str(page_num + 1) + '-' + str(box_num + 1) + ".png"), cropped)  # Depict bounding boxes
                except IndexError:
                    box = [int(box_idx["y1"]), int(box_idx["x1"]),
                    int(box_idx["y2"]), int(box_idx["x2"])]                          # Get box boundary coordinates
                    image_new = visualize_box(box = box, image = image_new)
                    cropped = image_new[box[0]:box[2], box[1]:box[3]]
                    # We are going to save each image on every page
                    io.imsave(os.path.join(SAVE_PATH, PDF_NAME + "Page" + str(page_num + 1) + '-' + str(box_num + 1) + ".png"), cropped)  # Depict bounding boxes

@click.command(
    context_settings={
        'help_option_names': ['-h', '--help']
    })

@click.option(
    '--Error_Margin', default = 10,
    help='Making bounding box with Error_Margin # pixels larger on each side. Default is 10.')

@click.argument(
    'pdf_directory',
    type=click.Path(
        exists=True,
        file_okay=False,
        dir_okay=True,
        resolve_path=True))

def boundbox_overlay(pdf_directory, error_margin):
    'Draws bounding boxes over figures in pdfs in output directory'

    dirs = os.listdir(pdf_directory)

    Failed_Files = 0
    Processed_Files = 0
    Processed_Images = 0
    try:
        # Remove done file if from a previous run
        os.remove(os.path.join(os.getcwd(), pdf_directory, "Done_Extracting_Figures.txt"))
    except FileNotFoundError:
        pass
    # Loop through all directories
    for dir in dirs:
        dir = os.path.join(os.getcwd(), pdf_directory, dir) # Adding path to dir
        if (not os.path.isdir(dir)):
            continue # Then skip
        try:
            pdf_name = next(x for x in os.listdir(dir) if x[-4:] == ".pdf").replace(".pdf", "").strip(" ")  # Get pdf name by removing extra white space and .pdf
            if (os.path.exists(dir + "/Image_Extracted.txt")):
                print("Skipping processed file")
                continue # Skip if already processed
            print('processing file')
            with open(os.path.join(dir, pdf_name + "deepfigures-results.json")) as f:       # Load JSON file containing bounding
                output = json.load(f)
            boxes = output["raw_detected_boxes"]      # Get bounding boxes for all pages

            IMAGE_PATH = os.path.join(dir, pdf_name + ".pdf-images", "ghostscript", "dpi" + str(DPI))
            image_names = sorted(os.listdir(IMAGE_PATH))
            image_names.remove("_SUCCESS")                 # Get image names for all pages
            images = []

            for name in image_names:
                images.append(np.array(io.imread(os.path.join(IMAGE_PATH, name))))  # Get images as numpy arrays
                depict_boxes(dir=dir, dpi=DPI, images=images, boxes=boxes, Error_Margin=error_margin)
                Processed_Images += 1
            Processed_Files += 1
            # File that contains the number of processed files
            with open(os.path.join(os.getcwd(), pdf_directory, "Processed_Files.txt"), 'a+') as f:
                f.write("File Success #:" + str(Processed_Files) + ', File name:' + pdf_name + ', Number of images:' + str(Processed_Images) + "\n")
            open(os.path.join(dir, "Image_Extracted.txt"), "w+")
        except FileNotFoundError:
            Failed_Files += 1 # Add 1 to Failed file count
            print("Skipped file because it had issues will processing in deepfigures.")
            with open(os.path.join(os.getcwd(), pdf_directory, "Failed_Files.txt"), 'a+') as f:
                f.write("File Failed #:" + str(Failed_Files) + ", File name:" + pdf_name + "\n")
            open(os.path.join(dir, "Image_Extracted.txt"), "w+")
            continue
        except NotADirectoryError:
            continue

    # Create done file to indicate the boundbox finished
    f = open(os.path.join(os.getcwd(), pdf_directory, "Done_Extracting_Figures.txt"), 'a+')
    f.close()
    print("Done extracting figures!")

if __name__ == '__main__':
    boundbox_overlay()


text:
from __future__ import division
from numpy import log, mean, std

class Random:
    """
    Defines custom random number generator
    """

    def __init__(self):

        self.ia = 16807
        self.im = 2147483647
        self.am = (1.0 / self.im)
        self.iq = 127773
        self.ir = 2836
        self.idum = 12458
        self.mask = 123459876

    def set_seed(self, seed_val=1):
        self.idum = long(seed_val)

    def rand0(self, idnum=None):
        """
        Uniform distribution
        """

        if idnum is not None:
            self.set_seed(idnum)

        self.idum = long(self.idum ^ self.mask)
        k = long(self.idum / self.iq)
        self.idum = long(self.ia * (self.idum - k * self.iq) - self.ir * k)
        if self.idum < 0:
            self.idum += self.im
        ans = self.am * self.idum
        return ans

    def uniform(self, lo=0, hi=1):
        return lo + (hi - lo) * self.rand0()

    def int(self, lo=0, hi=100):
        return int(self.uniform(lo, hi))

    def exponential(self, lam=1/3000, idnum=None):

        if idnum is not None:
            self.set_seed(idnum)

        dummy = 0
        while dummy == 0:
            dummy = self.rand0(idnum)

        return -log(dummy) / lam

    def bounded_pareto(self, a=1.1, k=332, p=10e10, idnum=None):

        if idnum is not None:
            self.set_seed(idnum)

        dummy = 0
        while dummy == 0:
            dummy = self.rand0(idnum)

        return ((p**a+dummy*k**a-dummy*p**a)/((p*k)**a))**-(1/a)

if __name__ == "__main__":
    from pdb import set_trace
    rand = Random()
    rand.set_seed(seed_val=1729)
    a = [rand.bounded_pareto() for _ in xrange(100000)]
    set_trace()

reason: from numpy


text:
import numpy

def jenks(data, num_breaks):
    """
    Calculate Jenks natural breaks.

    Adapted from http://danieljlewis.org/files/2010/06/Jenks.pdf
    Credit: Daniel Lewis

    Arguments:
    data -- Array of values to classify.
    num_breaks -- Number of breaks to perform.
    """

    data = numpy.ma.compressed(data)
    if len(data) > 1000:
        data.sort()
        ls = numpy.linspace(0, len(data)-1, 1000)
        ls = [int(round(x)) for x in ls]
        data_list = data[ls]
    else:
        data_list = data

    data_list.sort()

    mat1 = []
    for i in range(0, len(data_list) + 1):
        temp = []
        for j in range(0, num_breaks + 1):
            temp.append(0)
        mat1.append(temp)

    mat2 = []
    for i in range(0, len(data_list) + 1):
        temp = []
        for j in range(0, num_breaks + 1):
            temp.append(0)
        mat2.append(temp)

    for i in range(1, num_breaks + 1):
        mat1[1][i] = 1
        mat2[1][i] = 0
        for j in range(2, len(data_list) + 1):
            mat2[j][i] = float('inf')

    v = 0.0
    for l in range(2, len(data_list) + 1):
        s1 = 0.0
        s2 = 0.0
        w = 0.0
        for m in range(1, l + 1):
            i3 = l - m + 1

            val = float(data_list[i3-1])

            s2 += val * val
            s1 += val

            w += 1
            v = s2 - (s1 * s1) / w
            i4 = i3 - 1

            if i4 != 0:
                for j in range(2, num_breaks + 1):
                    if mat2[l][j] >= (v + mat2[i4][j - 1]):
                        mat1[l][j] = i3
                        mat2[l][j] = v + mat2[i4][j - 1]

        mat1[l][1] = 1
        mat2[l][1] = v

    k = len(data_list)
    kclass = []
    for i in range(0, num_breaks + 1):
        kclass.append(0)

    kclass[num_breaks] = float(data_list[len(data_list) - 1])

    count_num = num_breaks
    while count_num >= 2:
        id = int((mat1[k][count_num]) - 2)

        kclass[count_num - 1] = data_list[id]
        k = int((mat1[k][count_num] - 1))
        count_num -= 1

    return [float(x) for x in kclass][1:]

def quantile(data, num_breaks):
    """
    Calculate quantile breaks.

    Arguments:
    data -- Array of values to classify.
    num_breaks -- Number of breaks to perform.
    """

    def scipy_mquantiles(a, prob=list([.25,.5,.75]), alphap=.4, betap=.4, axis=None, limit=()):
        """ function copied from scipy 0.13.3::scipy.stats.mstats.mquantiles """

        def _quantiles1D(data,m,p):
            x = numpy.sort(data.compressed())
            n = len(x)
            if n == 0:
                return numpy.ma.array(numpy.empty(len(p), dtype=float), mask=True)
            elif n == 1:
                return numpy.ma.array(numpy.resize(x, p.shape), mask=numpy.ma.nomask)
            aleph = (n*p + m)
            k = numpy.floor(aleph.clip(1, n-1)).astype(int)
            gamma = (aleph-k).clip(0,1)
            return (1.-gamma)*x[(k-1).tolist()] + gamma*x[k.tolist()]

        # Initialization & checks ---------
        data = numpy.ma.array(a, copy=False)
        if data.ndim > 2:
            raise TypeError("Array should be 2D at most !")
        #
        if limit:
            condition = (limit[0] < data) & (data < limit[1])
            data[~condition.filled(True)] = numpy.ma.masked
        #
        p =  numpy.array(prob, copy=False, ndmin=1)
        m = alphap + p*(1.-alphap-betap)
        # Computes quantiles along axis (or globally)
        if (axis is None):
            return _quantiles1D(data, m, p)
        return numpy.ma.apply_along_axis(_quantiles1D, axis, data, m, p)

    return scipy_mquantiles(data, numpy.linspace(1.0 / num_breaks, 1, num_breaks))

def equal(data, num_breaks):
    """
    Calculate equal interval breaks.

    Arguments:
    data -- Array of values to classify.
    num_breaks -- Number of breaks to perform.
    """

    step = (numpy.amax(data) - numpy.amin(data)) / num_breaks
    return numpy.linspace(numpy.amin(data) + step, numpy.amax(data), num_breaks)

max_stars_repo_name: consbio/ncdjango

max_issues_repo_name: consbio/ncdjango

max_forks_repo_name: consbio/ncdjango

reason: import numpy,from scipy


text:
import numpy as np
from nnfs.initializers import zeros, he_normal

class Parameter:
    def __init__(self, initial_value):
        self.shape = initial_value.shape
        self.value = initial_value
        self.grad = np.zeros(initial_value.shape)

class Layer:
    def get_parameters(self):
        return []

    def get_loss(self):
        return 0.0

class Linear(Layer):
    def __init__(self, n_inputs, n_neurons,
                 weights_inititalizer=he_normal,
                 bias_initializer=zeros,
                 weights_regularizer=None,
                 bias_regularizer=None):
        self.weights = Parameter(weights_inititalizer(n_inputs, n_neurons))
        self.biases = Parameter(bias_initializer(1, n_neurons))
        self.weights_regularizer = weights_regularizer
        self.bias_regularizer = bias_regularizer

    def forward(self, inputs):
        self._cached_inputs = inputs
        return np.dot(inputs, self.weights.value) + self.biases.value

    def backward(self, grad_out):
        grad_in = np.dot(grad_out, self.weights.value.T)
        self.weights.grad = np.dot(self._cached_inputs.T, grad_out)
        if self.weights_regularizer:
            self.weights.grad += self.weights_regularizer.get_grad(self.weights.value)
        self.biases.grad = np.sum(grad_out, axis=0)
        if self.bias_regularizer:
            self.biases.grad += self.bias_regularizer.get_grad(self.biases.value)
        return grad_in

    def get_parameters(self):
        return [self.weights, self.biases]

    def get_loss(self):
        loss = 0.0
        if self.weights_regularizer:
            loss += self.weights_regularizer(self.weights.value)
        if self.bias_regularizer:
            loss += self.bias_regularizer(self.biases.value)
        return loss

class Sigmoid(Layer):
    def forward(self, inputs):
        self._cached_inputs = inputs
        return 1.0 / (1.0 + np.exp(-inputs))

    def backward(self, grad_out):
        sig = 1.0 / (1.0 + np.exp(-self._cached_inputs))
        return grad_out * sig * (1.0 - sig)

class ReLU(Layer):
    def forward(self, inputs):
        self._cached_inputs = inputs
        return np.maximum(0.0, inputs)

    def backward(self, grad_out):
        grad_in = np.where(self._cached_inputs < 0.0, 0.0, 1.0)
        return grad_out * grad_in

class Softmax(Layer):
    def forward(self, inputs):
        maxs = np.max(inputs, axis=1).reshape((-1, 1))
        exps = np.exp(inputs - maxs)
        self._cached_outputs = exps / np.sum(exps, axis=1).reshape((-1, 1))
        return self._cached_outputs

    def backward(self, grad_out):
        a = np.empty((grad_out.shape[0], 1))
        for i in range(a.shape[0]):
            a[i, :] = np.dot(grad_out[i, :], self._cached_outputs[i, :])
        return self._cached_outputs * (grad_out - a)

max_stars_repo_name: tblut/NNFS

max_issues_repo_name: tblut/NNFS

max_forks_repo_name: tblut/NNFS


text:
### This file is originally from: [mlcommons repo](https://github.com/mlcommons/training/tree/9947bdf21ee3f2488fa4b362eec2ce7deb2ec4dd/single_stage_detector/ssd/mlperf_logger.py)
# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch
import numpy as np
import os
from mlperf_logging import mllog
from mlperf_logging.mllog import constants as mllog_const
mllogger = mllog.get_mllogger()
mllog.config(
    filename=(os.getenv("COMPLIANCE_FILE") or "mlperf_compliance.log"),
    root_dir=os.path.normpath(os.path.dirname(os.path.realpath(__file__))))

def ssd_print(*args, sync=True, **kwargs):
    use_cuda = os.getenv('USE_CUDA')
    if sync and use_cuda=='True':
        barrier()
    if get_rank() == 0:
        kwargs['stack_offset'] = 2
        mllogger.event(*args, **kwargs)

def barrier():
    """
    Works as a temporary distributed barrier, currently pytorch
    doesn't implement barrier for NCCL backend.
    Calls all_reduce on dummy tensor and synchronizes with GPU.
    """
    if torch.distributed.is_initialized():
        torch.distributed.all_reduce(torch.cuda.FloatTensor(1))
        torch.cuda.synchronize()

def get_rank():
    """
    Gets distributed rank or returns zero if distributed is not initialized.
    """
    if torch.distributed.is_initialized():
        rank = torch.distributed.get_rank()
    else:
        rank = os.getenv('RANK', os.getenv('LOCAL_RANK', 0))
    return rank

def broadcast_seeds(seed, device):
    if torch.distributed.is_initialized():
        seeds_tensor = torch.LongTensor([seed]).to(device)
        torch.distributed.broadcast(seeds_tensor, 0)
        seed = seeds_tensor.item()
    return seed

max_stars_repo_name: Pandinosaurus/models-intelai

max_issues_repo_name: Pandinosaurus/models-intelai

max_forks_repo_name: Pandinosaurus/models-intelai


text:
# Currently the reference source for this code in this form is the
# 'Curt' branch here:
#
#     https://github.com/clolsonus/GMS-Feature-Matcher/tree/Curt
#
# This version has been modified to better match the naming
# conventions of the original C++ version of the algorithm.  It also
# has some structure copying and float->int conversion fixes so the
# algorithm runs without errors and exactly matches the output of the
# C++ version.
#
# Modifications should be made in the 'Curt' branch at the above link
# and copied here, if 'here' isn't there.
#

import copy
import math
from enum import Enum

import numpy as np
import sys

if sys.version_info[0] < 3:
    print("gms_matcher.py: Must be using Python 3")
    # quit()

THRESHOLD_FACTOR = 6

ROTATION_PATTERNS = [
    [1, 2, 3,
     4, 5, 6,
     7, 8, 9],

    [4, 1, 2,
     7, 5, 3,
     8, 9, 6],

    [7, 4, 1,
     8, 5, 2,
     9, 6, 3],

    [8, 7, 4,
     9, 5, 1,
     6, 3, 2],

    [9, 8, 7,
     6, 5, 4,
     3, 2, 1],

    [6, 9, 8,
     3, 5, 7,
     2, 1, 4],

    [3, 6, 9,
     2, 5, 8,
     1, 4, 7],

    [2, 3, 6,
     1, 5, 9,
     4, 7, 8]]

# 5 level scales
mScaleRatios = [1.0, 1.0 / 2, 1.0 / math.sqrt(2.0), math.sqrt(2.0), 2.0]

class DrawingType(Enum):
    ONLY_LINES = 1
    LINES_AND_POINTS = 2

class Size:
    def __init__(self, width, height):
        self.width = width
        self.height = height

class GmsMatcher:
    def __init__(self, vuv1, size1, vuv2, size2, vDMatches):
        # Input initialize
        self.mvP1 = self.NormalizePoints(vuv1, size1)
        self.mvP2 = self.NormalizePoints(vuv2, size2)
        self.mNumberMatches = len(vDMatches)
        self.mvMatches = self.ConvertMatches(vDMatches)

        # Grid Initialize
        self.mGridSizeLeft = Size(20, 20)
        self.mGridNumberLeft = int(self.mGridSizeLeft.width * self.mGridSizeLeft.height)

        # Initialize the neighbor of left grid
        self.mGridNeighborLeft = np.zeros((self.mGridNumberLeft, 9))
        self.InitializeNeighbors(self.mGridNeighborLeft, self.mGridSizeLeft)

        self.mGridSizeRight = copy.copy(self.mGridSizeLeft)

    # Normalize Key points to range (0-1)
    def NormalizePoints(self, uv_list, size):
        npts = []
        for uv in uv_list:
            npts.append((uv[0] / size.width,
                         uv[1] / size.height))
        return npts

    # Convert OpenCV match to list of tuples
    def ConvertMatches(self, vDMatches):
        vMatches = []
        for match in vDMatches:
            vMatches.append((match.queryIdx, match.trainIdx))
        return vMatches

    def InitializeNeighbors(self, neighbor, grid_size):
        for i in range(neighbor.shape[0]):
            neighbor[i] = self.get_nb9(i, grid_size)

    def get_nb9(self, idx, grid_size):
        nb9 = [-1 for _ in range(9)]
        idx_x = idx % grid_size.width
        idx_y = idx // grid_size.width

        for yi in range(-1, 2):
            for xi in range(-1, 2):
                idx_xx = idx_x + xi
                idx_yy = idx_y + yi

                if idx_xx < 0 or idx_xx >= grid_size.width or idx_yy < 0 or idx_yy >= grid_size.height:
                    continue
                nb9[xi + 4 + yi * 3] = idx_xx + idx_yy * grid_size.width

        return nb9

    def GetInlierMask(self, with_scale, with_rotation):
        max_inlier = 0

        if not with_scale and not with_rotation:
            self.SetScale(0)
            max_inlier = self.run(1)
            return self.mvbInlierMask, max_inlier
        elif with_scale and with_rotation:
            vb_inliers = []
            for scale in range(5):
                self.SetScale(scale)
                for RotationType in range(1, 9):
                    num_inlier = self.run(RotationType)
                    print('    ', scale, RotationType, num_inlier)
                    if num_inlier > max_inlier:
                        vb_inliers = self.mvbInlierMask
                        max_inlier = num_inlier

            if vb_inliers != []:
                return vb_inliers, max_inlier
            else:
                return self.mvbInlierMask, max_inlier
        elif with_rotation and not with_scale:
            self.SetScale(0)
            vb_inliers = []
            for RotationType in range(1, 9):
                num_inlier = self.run(RotationType)
                print('    ', RotationType, num_inlier)
                if num_inlier > max_inlier:
                    vb_inliers = self.mvbInlierMask
                    max_inlier = num_inlier

            if vb_inliers != []:
                return vb_inliers, max_inlier
            else:
                return self.mvbInlierMask, max_inlier
        else:
            vb_inliers = []
            for scale in range(5):
                self.SetScale(scale)
                num_inlier = self.run(1)
                if num_inlier > max_inlier:
                    vb_inliers = self.mvbInlierMask
                    max_inlier = num_inlier

            if vb_inliers != []:
                return vb_inliers, max_inlier
            else:
                return self.mvbInlierMask, max_inlier

    def SetScale(self, Scale):
        # Set Scale
        self.mGridSizeRight.width = int(self.mGridSizeLeft.width * mScaleRatios[Scale])
        self.mGridSizeRight.height = int(self.mGridSizeLeft.height * mScaleRatios[Scale])
        self.mGridNumberRight = int(self.mGridSizeRight.width * self.mGridSizeRight.height)

        # Initialize the neighbour of right grid
        self.mGridNeighborRight = np.zeros((int(self.mGridNumberRight), 9))
        self.InitializeNeighbors(self.mGridNeighborRight, self.mGridSizeRight)

    def run(self, RotationType):
        self.mvbInlierMask = [False for _ in range(self.mNumberMatches)]

        # Initialize motion statistics
        self.mMotionStatistics = np.zeros((int(self.mGridNumberLeft), int(self.mGridNumberRight)))
        self.mvMatchPairs = [[0, 0] for _ in range(self.mNumberMatches)]

        for GridType in range(1, 5):
            self.mMotionStatistics = np.zeros((int(self.mGridNumberLeft), int(self.mGridNumberRight)))
            self.mCellPairs = [-1 for _ in range(self.mGridNumberLeft)]
            self.mNumberPointsInPerCellLeft = [0 for _ in range(self.mGridNumberLeft)]

            self.AssignMatchPairs(GridType)
            self.VerifyCellPairs(RotationType)

            # Mark inliers
            for i in range(self.mNumberMatches):
                if self.mCellPairs[int(self.mvMatchPairs[i][0])] == self.mvMatchPairs[i][1]:
                    self.mvbInlierMask[i] = True

        return sum(self.mvbInlierMask)

    def AssignMatchPairs(self, grid_type):
        for i in range(self.mNumberMatches):
            lp = self.mvP1[self.mvMatches[i][0]]
            rp = self.mvP2[self.mvMatches[i][1]]
            lgidx = self.mvMatchPairs[i][0] = self.GetGridIndexLeft(lp, grid_type)

            if grid_type == 1:
                rgidx = self.mvMatchPairs[i][1] = self.GetGridIndexRight(rp)
            else:
                rgidx = self.mvMatchPairs[i][1]

            if lgidx < 0 or rgidx < 0:
                continue
            self.mMotionStatistics[int(lgidx)][int(rgidx)] += 1
            self.mNumberPointsInPerCellLeft[int(lgidx)] += 1

    def GetGridIndexLeft(self, pt, type):
        if type == 1:
            x = pt[0] * self.mGridSizeLeft.width
            y = pt[1] * self.mGridSizeLeft.height
        elif type == 2:
            x = pt[0] * self.mGridSizeLeft.width + 0.5
            y = pt[1] * self.mGridSizeLeft.height
        elif type == 3:
            x = pt[0] * self.mGridSizeLeft.width
            y = pt[1] * self.mGridSizeLeft.height + 0.5
        elif type == 4:
            x = pt[0] * self.mGridSizeLeft.width + 0.5
            y = pt[1] * self.mGridSizeLeft.height + 0.5
        x = int(math.floor(x))
        y = int(math.floor(y))

        if x >= self.mGridSizeLeft.width or y >= self.mGridSizeLeft.height:
            return -1
        return x + y * self.mGridSizeLeft.width

    def GetGridIndexRight(self, pt):
        x = int(math.floor(pt[0] * self.mGridSizeRight.width))
        y = int(math.floor(pt[1] * self.mGridSizeRight.height))
        return x + y * self.mGridSizeRight.width

    def VerifyCellPairs(self, RotationType):
        CurrentRP = ROTATION_PATTERNS[RotationType - 1]

        for i in range(self.mGridNumberLeft):
            if sum(self.mMotionStatistics[i]) == 0:
                self.mCellPairs[i] = -1
                continue
            max_number = 0
            for j in range(int(self.mGridNumberRight)):
                value = self.mMotionStatistics[i]
                if value[j] > max_number:
                    self.mCellPairs[i] = j
                    max_number = value[j]

            idx_grid_rt = self.mCellPairs[i]
            nb9_lt = self.mGridNeighborLeft[i]
            nb9_rt = self.mGridNeighborRight[idx_grid_rt]
            score = 0
            thresh = 0
            numpair = 0

            for j in range(9):
                ll = nb9_lt[j]
                rr = nb9_rt[CurrentRP[j] - 1]
                if ll == -1 or rr == -1:
                    continue

                score += self.mMotionStatistics[int(ll), int(rr)]
                thresh += self.mNumberPointsInPerCellLeft[int(ll)]
                numpair += 1

            thresh = THRESHOLD_FACTOR * math.sqrt(thresh/numpair)

            if score < thresh:
                self.mCellPairs[i] = -2

max_stars_repo_name: lpenuelac/ImageAnalysis

max_issues_repo_name: lpenuelac/ImageAnalysis

max_forks_repo_name: lpenuelac/ImageAnalysis


text:
import torch
import model_dispatcher
import config
import os
import numpy as np
from PIL import Image
import math

model = model_dispatcher.MODELS[config.MODEL_NAME]

model.load_state_dict(torch.load(config.MODEL_SAVE))
model.eval()

image = os.path.join(config.TRAIN_PATH_X, '924_0.jpg')

image = Image.open(image).convert('L')

print(image.size)

w, h = image.size

factor = (32.0/h)
width = int(math.ceil(w*factor))

image = np.array(image.resize((width*4, 32)))[... , np.newaxis]

print(image.shape)

image = np.transpose(image, (2, 0, 1)).astype(np.float32)

image = torch.tensor(image[np.newaxis, ...], dtype=torch.float)

y_hat = model(image).squeeze(0).detach().numpy()

print(y_hat.shape)

print(np.argmax(y_hat, -1).squeeze())

print(config.VOCAB.decoder(np.argmax(y_hat, -1).squeeze()))


text:
# Copyright The PyTorch Lightning team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import namedtuple
from functools import partial

import numpy as np
import pytest
import torch
from skimage.metrics import peak_signal_noise_ratio

from tests.helpers import seed_all
from tests.helpers.testers import BATCH_SIZE, NUM_BATCHES, MetricTester
from torchmetrics.functional import psnr
from torchmetrics.image import PSNR

seed_all(42)

Input = namedtuple("Input", ["preds", "target"])

_input_size = (NUM_BATCHES, BATCH_SIZE, 32, 32)
_inputs = [
    Input(
        preds=torch.randint(n_cls_pred, _input_size, dtype=torch.float),
        target=torch.randint(n_cls_target, _input_size, dtype=torch.float),
    )
    for n_cls_pred, n_cls_target in [(10, 10), (5, 10), (10, 5)]
]

def _to_sk_peak_signal_noise_ratio_inputs(value, dim):
    value = value.numpy()
    batches = value[None] if value.ndim == len(_input_size) - 1 else value

    if dim is None:
        return [batches]

    num_dims = np.size(dim)
    if not num_dims:
        return batches

    inputs = []
    for batch in batches:
        batch = np.moveaxis(batch, dim, np.arange(-num_dims, 0))
        psnr_input_shape = batch.shape[-num_dims:]
        inputs.extend(batch.reshape(-1, *psnr_input_shape))
    return inputs

def _sk_psnr(preds, target, data_range, reduction, dim):
    sk_preds_lists = _to_sk_peak_signal_noise_ratio_inputs(preds, dim=dim)
    sk_target_lists = _to_sk_peak_signal_noise_ratio_inputs(target, dim=dim)
    np_reduce_map = {"elementwise_mean": np.mean, "none": np.array, "sum": np.sum}
    return np_reduce_map[reduction](
        [
            peak_signal_noise_ratio(sk_target, sk_preds, data_range=data_range)
            for sk_target, sk_preds in zip(sk_target_lists, sk_preds_lists)
        ]
    )

def _base_e_sk_psnr(preds, target, data_range, reduction, dim):
    return _sk_psnr(preds, target, data_range, reduction, dim) * np.log(10)

@pytest.mark.parametrize(
    "preds, target, data_range, reduction, dim",
    [
        (_inputs[0].preds, _inputs[0].target, 10, "elementwise_mean", None),
        (_inputs[1].preds, _inputs[1].target, 10, "elementwise_mean", None),
        (_inputs[2].preds, _inputs[2].target, 5, "elementwise_mean", None),
        (_inputs[2].preds, _inputs[2].target, 5, "elementwise_mean", 1),
        (_inputs[2].preds, _inputs[2].target, 5, "elementwise_mean", (1, 2)),
        (_inputs[2].preds, _inputs[2].target, 5, "sum", (1, 2)),
    ],
)
@pytest.mark.parametrize(
    "base, sk_metric",
    [
        (10.0, _sk_psnr),
        (2.718281828459045, _base_e_sk_psnr),
    ],
)
class TestPSNR(MetricTester):
    @pytest.mark.parametrize("ddp", [True, False])
    @pytest.mark.parametrize("dist_sync_on_step", [True, False])
    def test_psnr(self, preds, target, data_range, base, reduction, dim, sk_metric, ddp, dist_sync_on_step):
        _args = {"data_range": data_range, "base": base, "reduction": reduction, "dim": dim}
        self.run_class_metric_test(
            ddp,
            preds,
            target,
            PSNR,
            partial(sk_metric, data_range=data_range, reduction=reduction, dim=dim),
            metric_args=_args,
            dist_sync_on_step=dist_sync_on_step,
        )

    def test_psnr_functional(self, preds, target, sk_metric, data_range, base, reduction, dim):
        _args = {"data_range": data_range, "base": base, "reduction": reduction, "dim": dim}
        self.run_functional_metric_test(
            preds,
            target,
            psnr,
            partial(sk_metric, data_range=data_range, reduction=reduction, dim=dim),
            metric_args=_args,
        )

    # PSNR half + cpu does not work due to missing support in torch.log
    @pytest.mark.xfail(reason="PSNR metric does not support cpu + half precision")
    def test_psnr_half_cpu(self, preds, target, data_range, reduction, dim, base, sk_metric):
        self.run_precision_test_cpu(
            preds, target, PSNR, psnr, {"data_range": data_range, "base": base, "reduction": reduction, "dim": dim}
        )

    @pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires cuda")
    def test_psnr_half_gpu(self, preds, target, data_range, reduction, dim, base, sk_metric):
        self.run_precision_test_gpu(
            preds, target, PSNR, psnr, {"data_range": data_range, "base": base, "reduction": reduction, "dim": dim}
        )

@pytest.mark.parametrize("reduction", ["none", "sum"])
def test_reduction_for_dim_none(reduction):
    match = f"The `reduction={reduction}` will not have any effect when `dim` is None."
    with pytest.warns(UserWarning, match=match):
        PSNR(reduction=reduction, dim=None)

    with pytest.warns(UserWarning, match=match):
        psnr(_inputs[0].preds, _inputs[0].target, reduction=reduction, dim=None)

def test_missing_data_range():
    with pytest.raises(ValueError):
        PSNR(data_range=None, dim=0)

    with pytest.raises(ValueError):
        psnr(_inputs[0].preds, _inputs[0].target, data_range=None, dim=0)

max_stars_repo_name: bibinwils/metrics

max_issues_repo_name: bibinwils/metrics

max_forks_repo_name: bibinwils/metrics


text:
'''
Project: Predicting movie genres from movie posters
Course: COMPSCI 682 Neural Networks: A Modern Introduction

File: data_manage.py
Description: Manages each movie as an object with its distinct features.
Author: Nirman Dave
Adapted from: Vleminckx, Benoit. dnn-movie-posters. (2018). Github repository. https://github.com/benckx/dnn-movie-posters.
'''

import io
import os.path
import numpy as np
import pandas as pd
from PIL import Image
import urllib.request

images_dir = 'data/images/'
parsed_movies = []

class Movie:
	'''
	The object movie stores all information about a particular movie that is provided
	by the dataset. Along with various functions that make this object ML ready.
	'''

	movie_id = 0
	title = ''
	year = 0
	genres = []
	poster_url = ''

	def poster_exists(self):
	    return os.path.isfile(self.poster_file_path())

	def download_poster(self):
		try:
			response = urllib.request.urlopen(self.poster_url)
			data = response.read()
			file = open(self.poster_file_path(), 'wb')
			file.write(bytearray(data))
			file.close()
			return data
		except:
			print('error downloading poster')

	def poster_file_path(self, size=100):
		return images_dir + str(size) + '/' + self.poster_file_name()

	def poster_file_name(self):
		return str(self.movie_id) + '.jpg'

	def img_to_rgb(self, poster_size):
		'''
		Takes an image of 3 channels (RGB) can converts it to a row of pixels.
		'''
		path = images_dir + str(poster_size) + '/' + str(self.movie_id) + '.jpg'
		data = open(path, 'rb').read()
		img = Image.open(io.BytesIO(data))
		rgb_img = img.convert('RGB')
		pixels = []
		for i in range(img.size[0]):
			row =[]
			for j in range(img.size[1]):
				r, g, b = rgb_img.getpixel((i, j))
				pixel = [r/255, g/255, b/255]
				row.append(pixel)
			pixels.append(row)
		return pixels

	def genres_to_vector(self, genres):
		'''
		Takes all the genres and converts them into multi-hot encoded vector.
		'''
		if len(genres) == 1:
			has_genre = self.has_genre(genres[0])
			return [int(has_genre), int(not has_genre)]
		else:
			vector = []
			if self.has_any_genre(genres):
				for genre in genres:
					vector.append(int(self.has_genre(genre)))
			return vector

	def has_genre(self, genre):
		return genre in self.genres

	def has_any_genre(self, genres):
		return len(set(self.genres).intersection(genres)) > 0

	def is_valid(self) -> bool:
		'''
		Checks if movie exists in a dataset.
		'''
		return self.poster_url.startswith('https://') \
			and 1900 <= self.year <= 2018 \
			and len(self.title) > 1 \
			and len(self.genres) > 1

	def __str__(self):
		return str(self.title) + '(' + str(self.year) + ')'

def download_posters(min_year=0):
	'''
	Downloads movie posters for a given min_year, if already downloaded
	it is ignored.
	'''
	for movie in list_movies():
		print(str(movie))
		if movie.year >= min_year:
			if movie.poster_exists() == False:
				movie.download_poster()
				if movie.poster_exists() == True:
					print('downloaded')
				else:
					print('cannot download')
			else:
				print('already downloaded')
		else:
			print('movie is too old to download')

def list_movies(year=None, genres=None):
	'''
	Lists all movies within a given year and genre resulting a list of
	Movie objects.
	'''
	if len(parsed_movies) == 0:
		data = pd.read_csv('data/MovieGenre.csv', encoding='ISO-8859-1')
		for i, r in data.iterrows():
			movie = parse_row(r)
			if movie.is_valid():
				parsed_movies.append(movie)

		parsed_movies.sort(key=lambda m: m.movie_id)

	result = parsed_movies

	if year != None:
		result = [movie for movie in result if movie.year == year]

	if genres != None:
		result = [movie for movie in result if movie.has_any_genre(genres)]

	return result

def parse_row(row):
	'''
	Parses a row in the dataset CSV file and converts that into a Movie object.
	'''
	movie = Movie()
	movie.movie_id = int(row['imdbId'])
	movie.title = row['Title'][:-7]
	year = row['Title'][-5:-1]
	if year.isdigit() and len(year)==4:
		movie.year = int(year)

	url = str(row['Poster'])
	if len(url) > 0:
		movie.poster_url = url.replace('"', '')

	genre_str = str(row['Genre'])
	if len(genre_str) > 0:
		movie.genres = genre_str.split('|')

	return movie

def search_movie(movie_id=None, title=None):
	'''
	Search a movie by its movie id or by title.
	'''
	movies = list_movies()
	for movie in movies:
		if movie_id is not None and movie.movie_id == movie_id:
			return movie
		if title is not None and movie.title == title:
			return movie

def list_movies_external(year=None, genres=None):
	'''
	Using a different datasheet for external testing. Hence a different function to
	avoid double name errors.
	'''
	if len(parsed_movies) == 0:
		data = pd.read_csv('data/MovieGenre_external.csv', encoding='ISO-8859-1')
		for i, r in data.iterrows():
			movie = parse_row(r)
			# if movie.is_valid():
			parsed_movies.append(movie)

		parsed_movies.sort(key=lambda m: m.movie_id)

	result = parsed_movies

	if year != None:
		result = [movie for movie in result if movie.year == year]

	if genres != None:
		result = [movie for movie in result if movie.has_any_genre(genres)]

	return result

def search_movie_external(movie_id=None, title=None):
	'''
	Searches movie by movie id or title from the external dataset.
	'''
	movies = list_movies_external()
	for movie in movies:
		if movie_id is not None and movie.movie_id == movie_id:
			return movie
		if title is not None and movie.title == title:
			return movie

max_stars_repo_name: nddave/Genre-Prediction

max_issues_repo_name: nddave/Genre-Prediction

max_forks_repo_name: nddave/Genre-Prediction


text:
from skimage import io
from skimage import morphology
from skimage import draw
import os
import numpy as np

# contando o numero de elementos presentes na matrix 3D
# a entrada ImageStackData eh um ndarray
def CountElements(ImageStackData):

    print('\nCounting elements in the stack...')
    print('Elements number in initial matrix is ', ImageStackData.max())

    CountL1=[]

    for k in list(range(1,ImageStackData.max()+1)):
        sumC=list(ImageStackData.flatten()).count(k)
        CountL1.append([k,sumC])

    return(CountL1)

# A funcao abaixo seleciona somente elementos com determinado valor de área
# a entrada VolumeCountList eh uma lista
def FindVolElements(VolumeCountList,minVolume,maxVolume):

    ElementsList=[]
    for x in list(range(len(VolumeCountList))):
        if minVolume <= VolumeCountList[x][1] <= maxVolume:
            ElementsList.append(VolumeCountList[x][0])

    return(ElementsList)

# Encontrando os elementos de maior volume nas matrizes
# a entrada VolumeCountList eh uma lista
def FindLargestElements(VolumeCountList,NumberofElements=1,ElementsList=[]):

    print('\nNumber of elements in the input are', VolumeCountList)

    if len(VolumeCountList) == 0:
        NullList=[0]
        return(NullList)

    else:
        LargestElement=VolumeCountList[0]
        index=0
        for x in list(range(len(VolumeCountList)-1)):
            if VolumeCountList[x+1][1] > LargestElement[1]:
                LargestElement=VolumeCountList[x+1]
                index=x+1

        ElementsList.append(LargestElement[0])

        #print('Os maiores elementos são', ElementsList)
        del(VolumeCountList[index])

        if len(ElementsList) < NumberofElements:
            return(FindLargestElements(VolumeCountList,NumberofElements,ElementsList))

        else:
            print('\nLargest elements in the matrix are', ElementsList)
            return(ElementsList)

# Selecionando nas matrizes somente os elementos de maior volume
# a entrada ImgStackData eh um ndarray e ElementsList uma lista
def ElementsSelectionbyRange(ImgStackData,ElementsList): # o parametro ElementsList indica quais elementos ficarao na matriz

    try:
        ImgStackData[0,0,0]
        for z in list(range(len(ImgStackData))):
            for y in list(range(len(ImgStackData[z]))):
                for x in list(range(len(ImgStackData[z][y]))):
                    if ImgStackData[z][y][x] not in ElementsList:
                        ImgStackData[z][y][x] = 0

        print('\nThe dimension of the output stack is ', ImgStackData.shape)
        print('\nThe largest value in the matrix of the final image is ', ImgStackData.max())

    except IndexError:
        for y in list(range(len(ImgStackData))):
            for x in list(range(len(ImgStackData[y]))):
                if ImgStackData[y][x] not in ElementsList:
                    ImgStackData[y][x] = 0

        print('\nThe dimension of the output stack is ', ImgStackData.shape)
        print('\nThe largest value in the matrix of the final image is ', ImgStackData.max())

    return(ImgStackData)

# Selecionando nas matrizes somente os elementos de maior volume
# A entrada ImgStackData eh um ndarray e LargestElementsList eh uma lista
# o parametro LargestElementsList indica quantos dos maiores elementos ficarao na matrix
def ElementsSelectionLarge(ImgStackData,LargestElementsList):

    try:
        ImgStackData[0,0,0]
        for z in list(range(len(ImgStackData))):
            for y in list(range(len(ImgStackData[z]))):
                for x in list(range(len(ImgStackData[z][y]))):
                    if ImgStackData[z][y][x] not in LargestElementsList:
                        ImgStackData[z][y][x] = 0

        print('\nThe dimension of the output stack is ', ImgStackData.shape)
        print('\nThe largest value in the matrix of the final image is ', ImgStackData.max())

    except IndexError:
        for y in list(range(len(ImgStackData))):
            for x in list(range(len(ImgStackData[y]))):
                if ImgStackData[y][x] not in LargestElementsList:
                    ImgStackData[y][x] = 0

        print('\nThe dimension of the output stack is ', ImgStackData.shape)
        print('\nThe largest value in the matrix of the final image is ', ImgStackData.max())

    return(ImgStackData)

# Filtrando os elementos que estão nos edges da matrix
# A entrada ImgStackData eh um labelled ndarray
def ElementsSelectionOutEdge(ImgStackData):

    LabelledImgStack=morphology.label(ImgStackData, connectivity=1)
    MaxElement=LabelledImgStack.max()

    try:
        ImgStackData[0,0,0]
        for z in list(range(len(LabelledImgStack))):
            for y in list(range(len(LabelledImgStack[z]))):
                for x in list(range(len(LabelledImgStack[z,y]))):
                    for elemN in list(np.arange(1, 1+MaxElement)):
                        if LabelledImgStack[z, y, x] == elemN:
                            try:
                                LabelledImgStack[z-1, y, x]
                                LabelledImgStack[z+1, y, x]
                                LabelledImgStack[z, y-1, x]
                                LabelledImgStack[z, y+1, x]
                                LabelledImgStack[z, y, x-1]
                                LabelledImgStack[z, y, x+1]

                            except IndexError:
                                LabelledImgStack=np.where(LabelledImgStack == elemN, 0, LabelledImgStack)
                                continue
    except IndexError:
            for y in list(range(len(LabelledImgStack))):
                for x in list(range(len(LabelledImgStack[y]))):
                    if LabelledImgStack[y, x] == elemN:
                        for elemN in list(np.arange(1, 1+MaxElement)):
                            try:
                                LabelledImgStack[y-1, x]
                                LabelledImgStack[y+1, x]
                                LabelledImgStack[y, x-1]
                                LabelledImgStack[y, x+1]

                            except IndexError:
                                LabelledImgStack=np.where(LabelledImgStack == elemN, 0, LabelledImgStack)

    return(LabelledImgStack)

# Função para importar imagens e filtrar elementos por range de volume
def SelectionbyRange(diretorio,
                     stackNumber,
                     FirstSlice,LastSlice,
                     Xmin,Xmax,Ymin,Ymax,
                     minVolume,maxVolume,
                     ImportstackRootName='/Raw',
                     ImageBaseName = 'Slice',
                     importFormat='.png',
                     ExportDir='/Processed',
                     AreaBorderSize=5,

                     CropVolMode=False,
                     ReferenceStackToCrop='/Binarize/bac/ImageJ/t1',
                     RefminArea=1,RefmaxArea=2000000,
                     MAXElementsToCrop=10,

                     ColonyTimeCheckMode=False,
                     ReferenceImgToCheckAttachment='/Binarize/bac/ImageJ/t1',
                     ReferenceImgToCheckDetachment='/Binarize/bac/ImageJ/t2',
                     RefminAreaCheck=1,RefmaxAreaCheck=2000000,
                     MAXElementsToCheck=10,
                     MaxTimePoint=15):

    SliceRange=list(range(FirstSlice,LastSlice+1))
    Imglist1=[]

    print('\nImporting images... \n')

    #criando a projeção do stack
    Zprojection = np.zeros((Ymax,Xmax))

    for x in SliceRange:
        if x < 10:
            a=io.imread(diretorio + ImportstackRootName + '/t' + str(stackNumber) + '/' + ImageBaseName + '00' + str(x) + importFormat)
            b=a[Ymin:Ymax,Xmin:Xmax]

        elif 10 <= x < 100:
            a=io.imread(diretorio + ImportstackRootName + '/t' + str(stackNumber) + '/' + ImageBaseName + '0' + str(x) + importFormat)
            b=a[Ymin:Ymax,Xmin:Xmax]

        elif 100 <= x < 1000:
            a=io.imread(diretorio + ImportstackRootName + '/t' + str(stackNumber) + '/' + ImageBaseName + str(x) + importFormat)
            b=a[Ymin:Ymax,Xmin:Xmax]

        Imglist1.append(b)
        Zprojection = Zprojection + b

    ImgArray3D=np.array(Imglist1)
    Zprojection_img = np.where(Zprojection > 0, 255, 0)
    io.imsave(diretorio + ImportstackRootName + '/t' + str(stackNumber) + '/ZProjection' + importFormat, np.array(Zprojection_img, dtype='uint8'), check_contrast=False)

    print('Stack has ', len(ImgArray3D),' slices')
    print('The images have ', len(ImgArray3D[0]),' lines')
    print('The images have ', len(ImgArray3D[0][0]),' columns')
    print('The largest value in the stack is ', ImgArray3D.max())

    try:
        print('The image has ',len(ImgArray3D[0][0][0]),' canais')

    except TypeError:
        print('The image has only 1 channel.')

    ImgArray3DNorm = np.where(ImgArray3D > 0, 1, 0)
    #print('The largest value in the normalized stack is ', ImgArray3DNorm.max())

    print('\nIdentifying the elements... \n')

    LabelledImgs=morphology.label(ImgArray3DNorm,connectivity=1)

    #print('Type morphImgs',type(LabelledImgs))

    if CropVolMode == False and ColonyTimeCheckMode == False:

        print('\nInitiating the elements selection for the whole stack... \n')

        SelP1=CountElements(LabelledImgs)
        SelP2=FindVolElements(SelP1,minVolume,maxVolume)
        SelP3=ElementsSelectionbyRange(LabelledImgs,SelP2)

        del(SelP1)
        del(SelP2)

        for imgN in list(range(len(SelP3))):

            if not os.path.exists(diretorio + ExportDir + '/t' + str(stackNumber)):
                os.makedirs(diretorio + ExportDir + '/t' + str(stackNumber))

            abc1=np.where(SelP3[imgN] > 0, 255, 0)
            io.imsave(diretorio + ExportDir + '/t' + str(stackNumber) + '/' + ImageBaseName + str(SliceRange[imgN]) + importFormat, np.array(abc1, dtype='uint8'), check_contrast=False)

    elif CropVolMode == True:
        print('\nInitiating the Region Crop Module for the stack ', str(stackNumber), ' ... \n')

        #--------------------------------------------------------------------------------------
        #checking if there is a ZProjection
        if not os.path.exists(diretorio + ReferenceStackToCrop + '/ZProjection' + importFormat):

            Zprojection = np.zeros((Ymax,Xmax))

            for x in SliceRange:
                if x < 10:
                    a=io.imread(diretorio + ReferenceStackToCrop + '/' + ImageBaseName + '00' + str(x) + importFormat)
                    b=a[Ymin:Ymax,Xmin:Xmax]

                elif 10 <= x < 100:
                    a=io.imread(diretorio + ReferenceStackToCrop + '/' + ImageBaseName + '0' + str(x) + importFormat)
                    b=a[Ymin:Ymax,Xmin:Xmax]

                elif 100 <= x < 1000:
                    a=io.imread(diretorio + ReferenceStackToCrop + '/' + ImageBaseName + str(x) + importFormat)
                    b=a[Ymin:Ymax,Xmin:Xmax]

                Zprojection = Zprojection + b

            refImage = np.where(Zprojection > 0, 255, 0)
            io.imsave(diretorio + ReferenceStackToCrop + '/ZProjection' + importFormat, np.array(refImage, dtype='uint8'), check_contrast=False)
            refImageNorm = np.where(Zprojection > 0, 1, 0)

        else:
            refImage = io.imread(diretorio + ReferenceStackToCrop + '/ZProjection' + importFormat)
            refImageNorm = np.where(refImage > 0, 1, 0)

        #--------------------------------------------------------------------------------------

        RefimgLab=morphology.label(refImageNorm,connectivity=1)

        RefP1=CountElements(RefimgLab)
        RefP2=FindVolElements(RefP1,RefminArea,RefmaxArea)

        if len(RefP2) > MAXElementsToCrop:

            print('\n \nElements to be cropped ', len(RefP2), '\n \n ')
            print('\n \nAborting due to the excess of elements to be cropped\n \n ')

        else:
            RefP3=ElementsSelectionbyRange(RefimgLab,RefP2)

            if not os.path.exists(diretorio + '/Images'):
                os.makedirs(diretorio + '/Images')

            RefP4 = np.where(RefP3 > 0, 255, 0)
            io.imsave(diretorio + ImportstackRootName + '/t' + str(stackNumber) + '/ZProjectionFiltered' + importFormat, RefP4, check_contrast=False)

            print('\n \nIt will be  ', len(RefP2), 'to be cropped\n \n ')

            del(RefP1)

            indexToSave=1
            for elem in list(range(len(RefP2))):
                print('\nSearching for element ', RefP2[elem], '\n')
                XPosList=[]
                YPosList=[]
                for YPos in list(range(len(RefP3))):
                    for XPos in list(range(len(RefP3[YPos]))):
                        if RefP3[YPos,XPos] == RefP2[elem]:
                            YPosList.append(YPos)
                            XPosList.append(XPos)

                Ymin=min(YPosList)
                Ymax=max(YPosList)
                Xmin=min(XPosList)
                Xmax=max(XPosList)

                del(XPosList)
                del(YPosList)

                if (Ymin-AreaBorderSize) <= 0 or (Ymax+AreaBorderSize) >= len(RefP3) or (Xmin-AreaBorderSize) <= 0 or (Xmax+AreaBorderSize) >= len(RefP3[0]):

                    print('Indexes out of the matrix. Excluding element  ', RefP2[elem])
                    pass

                else:
                    try:

                        print('\nCropping Region ', indexToSave)

                        for imgN in list(range(len(LabelledImgs))):
                            aa1F=LabelledImgs[imgN][Ymin-AreaBorderSize:Ymax+AreaBorderSize,Xmin-AreaBorderSize:Xmax+AreaBorderSize]
                            bb1F=np.where(aa1F > 0, 255, 0)

                            if not os.path.exists(diretorio + ExportDir + '/t' + str(stackNumber) + '/CropRegions/Region' + str(indexToSave)):
                                os.makedirs(diretorio + ExportDir + '/t' + str(stackNumber) + '/CropRegions/Region' + str(indexToSave))

                            io.imsave(diretorio + ExportDir + '/t' + str(stackNumber) + '/CropRegions/Region' + str(indexToSave) + '/' + ImageBaseName + str(SliceRange[imgN]) + importFormat, np.array(bb1F, dtype='uint8'), check_contrast=False)

                            if not os.path.exists(diretorio + '/Images'):
                                os.makedirs(diretorio + '/Images')

                            try:
                                RefImgwithRegOverlay=io.imread(diretorio + '/Images/ZProjectionCropped' + importFormat)

                            except FileNotFoundError:
                                RefImgwithRegOverlay = np.zeros((len(ImgArray3D[0]), len(ImgArray3D[0][0])), dtype='uint8')

                            rectangle=np.array(((Ymin-AreaBorderSize,Xmin-AreaBorderSize),(Ymax+AreaBorderSize,Xmin-AreaBorderSize),(Ymax+AreaBorderSize,Xmax+AreaBorderSize),(Ymin-AreaBorderSize,Xmax+AreaBorderSize),(Ymin-AreaBorderSize,Xmin-AreaBorderSize)))
                            rr, cc = draw.polygon(rectangle[:,0], rectangle[:,1])
                            RefImgwithRegOverlay[rr, cc] = 255

                            io.imsave(diretorio + '/Images/ZProjectionCropped' + importFormat, np.array(RefImgwithRegOverlay, dtype='uint8'), check_contrast=False)

                        indexToSave = indexToSave+1

                    except PermissionError or OSError or IndexError:
                        pass

    elif ColonyTimeCheckMode == True:
        print('\nInitiating the Time Check Mode module for the stack ', str(stackNumber), ' ... \n')

        print('Analyzing elements for the attachment map...')
        refImageCheckAT=io.imread(diretorio + ReferenceImgToCheckAttachment + '/ZProjection' + importFormat)

        refImageNormCheckAT=np.where(refImageCheckAT > 0, 1, 0)
        RefimgLabCheckAT=morphology.label(refImageNormCheckAT, connectivity=1)
        RefP1b=CountElements(RefimgLabCheckAT)
        RefP2b=FindVolElements(RefP1b,RefminAreaCheck,RefmaxAreaCheck)

        print('Analyzing elements for the detachment map...')
        refImageCheckDeT=io.imread(diretorio + ReferenceImgToCheckDetachment + '/ZProjection' + importFormat)

        refImageNormCheckDeT=np.where(refImageCheckDeT > 0, 1, 0)
        RefimgLabCheckDeT=morphology.label(refImageNormCheckDeT, connectivity=1)
        RefP1c=CountElements(RefimgLabCheckDeT)
        RefP2c=FindVolElements(RefP1c,RefminAreaCheck,RefmaxAreaCheck)

        if len(RefP2b) > MAXElementsToCheck or len(RefP2c) > MAXElementsToCheck:

            print('\n \nElements to be checked for attachment', len(RefP2b), '\n \n ')
            print('\n \nElements to be checked for dettachment', len(RefP2c), '\n \n ')
            print('\n \nAborting due to the excess of elements to be cropped\n \n ')

        else:
            RefP3b=ElementsSelectionbyRange(RefimgLabCheckAT, RefP2b)
            RefP3c=ElementsSelectionbyRange(RefimgLabCheckDeT, RefP2c)

            if not os.path.exists(diretorio + '/Images'):
                os.makedirs(diretorio + '/Images')

            RefP4b = np.where(RefP3b > 0, 255, 0)
            RefP4c = np.where(RefP3c > 0, 255, 0)
            io.imsave(diretorio + ReferenceImgToCheckAttachment + '/ZProjectionFiltered_AT' + importFormat, np.array(RefP4b, dtype='uint8'), check_contrast=False)
            io.imsave(diretorio + ReferenceImgToCheckDetachment + '/ZProjectionFiltered_DeT' + importFormat, np.array(RefP4c, dtype='uint8'), check_contrast=False)

            print('\n \nThere will be  ', len(RefP2b), ' elements to check for the attachment analysis\n \n ')
            print('\n \nThere will be  ', len(RefP2c), ' elements to check for the detachment analysis\n \n ')

            del(RefP1b)
            del(RefP1c)

            #Attachment Analysis
            for elem in list(range(len(RefP2b))):
                print('\nSearching for element ', RefP2b[elem], '\n')
                XPosList=[]
                YPosList=[]
                for YPos in list(range(len(RefP3b))):
                    for XPos in list(range(len(RefP3b[YPos]))):
                        if RefP3b[YPos,XPos] == RefP2b[elem]:
                            YPosList.append(YPos)
                            XPosList.append(XPos)

                Ymin=min(YPosList)
                Ymax=max(YPosList)
                Xmin=min(XPosList)
                Xmax=max(XPosList)

                del(XPosList)
                del(YPosList)

                if (Ymin-AreaBorderSize) <= 0 or (Ymax+AreaBorderSize) >= len(RefP3b) or (Xmin-AreaBorderSize) <= 0 or (Xmax+AreaBorderSize) >= len(RefP3b[0]):

                    print('Indixes out of the matrix. Excluding element ', RefP2b[elem])
                    pass

                else:
                    if LabelledImgs[:,Ymin-AreaBorderSize:Ymax+AreaBorderSize,Xmin-AreaBorderSize:Xmax+AreaBorderSize].sum() > 0:

                        try:
                            RefImgwithRegOverlayb=io.imread(diretorio + '/Images/TimeAnalysis_AT' + importFormat)

                        except FileNotFoundError:
                            RefImgwithRegOverlayb = np.zeros((len(ImgArray3D[0]), len(ImgArray3D[0][0])), dtype=np.uint8)

                        rectangleb=np.array(((Ymin-AreaBorderSize,Xmin-AreaBorderSize),(Ymax+AreaBorderSize,Xmin-AreaBorderSize),(Ymax+AreaBorderSize,Xmax+AreaBorderSize),(Ymin-AreaBorderSize,Xmax+AreaBorderSize),(Ymin-AreaBorderSize,Xmin-AreaBorderSize)))
                        rr, cc = draw.polygon(rectangleb[:,0], rectangleb[:,1])

                        try:
                            if RefImgwithRegOverlayb[Ymin-1:Ymax+1,Xmin-1:Xmax+1].sum() == 0:
                                RefImgwithRegOverlayb[rr, cc] = int((stackNumber/MaxTimePoint) * 255)
                                io.imsave(diretorio + '/Images/TimeAnalysis_AT' + importFormat, np.array(RefImgwithRegOverlayb, dtype='uint8'), check_contrast=False)

                        except IndexError:
                            pass

                    else:
                        pass

            #Detachment Analysis
            for elem in list(range(len(RefP2c))):
                print('\nSearching for element', RefP2c[elem], '\n')
                XPosList=[]
                YPosList=[]
                for YPos in list(range(len(RefP3c))):
                    for XPos in list(range(len(RefP3c[YPos]))):
                        if RefP3c[YPos,XPos] == RefP2c[elem]:
                            YPosList.append(YPos)
                            XPosList.append(XPos)

                Ymin=min(YPosList)
                Ymax=max(YPosList)
                Xmin=min(XPosList)
                Xmax=max(XPosList)

                del(XPosList)
                del(YPosList)

                if (Ymin-AreaBorderSize) <= 0 or (Ymax+AreaBorderSize) >= len(RefP3c) or (Xmin-AreaBorderSize) <= 0 or (Xmax+AreaBorderSize) >= len(RefP3c[0]):

                    print('Indexes out of the matrix. Excluding element ', RefP2c[elem])
                    pass

                else:
                    if LabelledImgs[:,Ymin-AreaBorderSize:Ymax+AreaBorderSize,Xmin-AreaBorderSize:Xmax+AreaBorderSize].sum() == 0:

                        try:
                            RefImgwithRegOverlayb=io.imread(diretorio + '/Images/TimeAnalysis_DeT' + importFormat)

                        except FileNotFoundError:
                            RefImgwithRegOverlayb = np.zeros((len(ImgArray3D[0]), len(ImgArray3D[0][0])), dtype=np.uint8)

                        rectangleb=np.array(((Ymin-AreaBorderSize,Xmin-AreaBorderSize),(Ymax+AreaBorderSize,Xmin-AreaBorderSize),(Ymax+AreaBorderSize,Xmax+AreaBorderSize),(Ymin-AreaBorderSize,Xmax+AreaBorderSize),(Ymin-AreaBorderSize,Xmin-AreaBorderSize)))
                        rr, cc = draw.polygon(rectangleb[:,0], rectangleb[:,1])

                        try:
                            if RefImgwithRegOverlayb[Ymin-1:Ymax+1,Xmin-1:Xmax+1].sum() == 0:
                                RefImgwithRegOverlayb[rr, cc] = int((stackNumber/MaxTimePoint) * 255)
                                io.imsave(diretorio + '/Images/TimeAnalysis_DeT' + importFormat, np.array(RefImgwithRegOverlayb, dtype='uint8'), check_contrast=False)

                        except IndexError:
                            pass

                    else:
                        pass

# Função para importar imagens e filtrar maiores elementos escolhidos
def SelectionInnerElements(diretorio,stackNumber,
                           FirstSlice,LastSlice,
                           ImportstackRootName='/Raw',
                           ImageBaseName='Slice',
                           importFormat='.png',
                           ExportDir='/Processed',
                           RegionAnalysis=False):

    SliceRange=list(range(FirstSlice,LastSlice+1))

    if RegionAnalysis == False:

        Imglist1=[]
        for x in SliceRange:
            a=io.imread(diretorio + ImportstackRootName + '/t' + str(stackNumber) + '/' + ImageBaseName + str(x) + importFormat)

            Imglist1.append(a)

        ImgArray3D=np.array(Imglist1)

        print('\nThe stack has ',len(ImgArray3D),' slices')
        print('\nThe images have ',len(ImgArray3D[0]),' lines')
        print('\nThe images have',len(ImgArray3D[0][0]),' columns')
        print('\nThe largest value in the stack is ', ImgArray3D.max())

        try:
            print('\nThe images have',len(ImgArray3D[0][0][0]),' channels')

        except TypeError:
            print('\nThe image has just 1 channel')

        ImgArray3DNorm = np.where(ImgArray3D > 0, 1, 0)
        print('\nThe largest value in the normalized stack is ', ImgArray3DNorm.max())

        #print('Type morphImgs',type(LabelledImgs))

        SelP3=ElementsSelectionOutEdge(ImgArray3DNorm)

        #print('SelP3',type(SelP3))

        for imgN in list(range(len(SelP3))):

            if not os.path.exists(diretorio + ExportDir + '/t' + str(stackNumber)):
                os.makedirs(diretorio + ExportDir + '/t' + str(stackNumber))

            abc1=np.where(SelP3[imgN] > 0, 255, 0)
            io.imsave(diretorio + ExportDir + '/t' + str(stackNumber) + '/' + ImageBaseName + str(SliceRange[imgN]) + importFormat, np.array(abc1, dtype='uint8'), check_contrast=False)

    if RegionAnalysis == True:

        RegionNumber = len(os.listdir(diretorio + '/ExportedData/Cropped/t1/CropRegions'))

        RegionList=list(range(1, RegionNumber+1))

        for region in RegionList:

            print('\nCropping Region ', region)

            Imglist1=[]
            for x in SliceRange:
                a=io.imread(diretorio + '/ExportedData/Cropped/t' + str(stackNumber) + '/CropRegions/Region' + str(region) + '/' + ImageBaseName + str(x) + importFormat)

                Imglist1.append(a)

            ImgArray3D=np.array(Imglist1)

            print('\nThe stack has',len(ImgArray3D),' slices')
            print('\nThe images have',len(ImgArray3D[0]),' lines')
            print('\nThe images have',len(ImgArray3D[0][0]),' columns')
            print('\nThe largest value in the stack is ', ImgArray3D.max())

            try:
                print('\nThe images have ',len(ImgArray3D[0][0][0]),' channel')

            except TypeError:
                print('\nThe image has only 1 channel')

            ImgArray3DNorm = np.where(ImgArray3D > 0, 1, 0)
            print('\nThe largest value in the normalized stack is ', ImgArray3DNorm.max())

            #print('Type morphImgs',type(LabelledImgs))

            SelP3=ElementsSelectionOutEdge(ImgArray3DNorm)

            #print('SelP3',type(SelP3))

            for imgN in list(range(len(SelP3))):

                if not os.path.exists(diretorio + ExportDir + '/t' + str(stackNumber) + '/CropRegions/Region' + str(region)):
                    os.makedirs(diretorio + ExportDir + '/t' + str(stackNumber) + '/CropRegions/Region' + str(region))

                abc1=np.where(SelP3[imgN] > 0, 255, 0)
                io.imsave(diretorio + ExportDir + '/t' + str(stackNumber) + '/CropRegions/Region' + str(region) + '/' + ImageBaseName + str(SliceRange[imgN]) + importFormat, np.array(abc1, dtype='uint8'), check_contrast=False)

max_stars_repo_name: amaurijp/BioSPA

max_issues_repo_name: amaurijp/BioSPA

max_forks_repo_name: amaurijp/BioSPA


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on 14/06/18

"""

from RecSysFramework.Recommender import Recommender
from RecSysFramework.Utils import check_matrix
from RecSysFramework.Recommender.DataIO import DataIO

import numpy as np
import scipy.sparse as sps

from sklearn.neighbors import NearestNeighbors
from numpy.linalg import lstsq

def tr(x, y):
    if sps.isspmatrix(x) or sps.isspmatrix(y):
        return x.multiply(y).sum()
    else:
        return np.multiply(x, y).sum()

class LCE(Recommender):

    """
    Local Collective Embeddings
    "Item Cold-Start Recommendations: Learning Local Collective Embeddings"
    Martin Saveski et al.

    """

    RECOMMENDER_NAME = "LCE"

    def __init__(self, URM_train, ICM):

        super(LCE, self).__init__(URM_train)

        # CSR is faster during evaluation
        self.n_users, self.n_items = self.URM_train.shape
        self.ICM = check_matrix(ICM, 'csr')
        self.n_features = self.ICM.shape[1]

    def construct_A(self, X, k, binary=False):

        """

        Constructs an adjacency matrix based on the nearest neighbor graph

        Parameters:
        X: data matrix where every point is a row

        k: number of nearest neighbors to be included in the graph

        binary: boolean, whether to include boolean values or the cosine
           similarity.

        """

        nbrs = NearestNeighbors(n_neighbors=1 + k, n_jobs=-1).fit(X)
        if binary:
            return nbrs.kneighbors_graph(X)
        else:
            return nbrs.kneighbors_graph(X, mode='distance')

    def fit(self, k=500, alpha=0.5, beta=0.05, lamb=0.5,
            epsilon=0.001, maxiter=500, rnd_seed=42, verbose=False):

        """

        Adapted from matlab implementation at:
        https://github.com/msaveski/LCE

        Parameters:

        k: number of topics. Controls the complexity of the model.

        alpha in [0, 1] controls the importance of each factorization.
         Setting alpha = 0.5 gives equal importance to both factorizations, while
         values of alpha >0.5 (or alpha < 0.5) give more importance to the
         factorization of Xs (or Xu).
         Default: 0.5

        beta: Controls the influence of the Laplacian regularization.

        lambda: hyper-paramter controlling the Thikonov Regularization.
         Enforces smooth solutions and avoids over fitting.
         Default: 0.5

        epsilon: Threshold controlling the number of iterations.
         If the objective function decreases less then epsilon from one iteration
         to another, the optimization procedure is stopped.

        maxiter: Maximum number of iterations.

        verbose: True|False. If set to True prints the value of the objective function
         in each iteration.

        """

        super(LCE, self).fit()

        np.random.seed(rnd_seed)

        Xs = self.ICM.copy()
        Xu = self.URM_train.transpose(copy=True)

        A = self.construct_A(Xs, 1, True)
        self._print("Matrix A contructed".format(self.RECOMMENDER_NAME))

        n = Xs.shape[0]
        v1 = Xs.shape[1]
        v2 = Xu.shape[1]

        W = np.random.uniform(0., 1., (n, k))
        Hs = np.random.uniform(0., 1., (k, v1))
        Hu = np.random.uniform(0., 1., (k, v2))

        D = sps.diags(A.sum(axis=0).A.flatten(), shape=A.shape)

        gamma = 1 - alpha
        trXstXs = tr(Xs, Xs)
        trXutXu = tr(Xu, Xu)

        Wt = W.transpose().conjugate()
        WtW = np.dot(Wt, W)
        WtXs = sps.csr_matrix.dot(Wt, Xs)
        WtXu = sps.csr_matrix.dot(Wt, Xu)
        WtWHs = np.dot(WtW, Hs)
        WtWHu = np.dot(WtW, Hu)
        DW = D.dot(W)
        AW = A.dot(W)

        itNum = 1
        ObjHistory = []
        delta = 2 * epsilon

        self._print("Preprocessing completed, starting iterations".format(self.RECOMMENDER_NAME))

        while delta > epsilon and itNum <= maxiter:

            Hs = np.multiply(Hs, np.divide(alpha * WtXs, np.maximum(alpha * WtWHs + lamb * Hs, 1e-10)))
            Hu = np.multiply(Hu, np.divide(gamma * WtXu, np.maximum(gamma * WtWHu + lamb * Hu, 1e-10)))

            W = np.multiply(W, np.divide(alpha * Xs.dot(Hs.transpose().conjugate()) +
                                         gamma * Xu.dot(Hu.transpose().conjugate()) + beta * AW,
                        np.maximum(alpha * np.dot(W, np.dot(Hs, Hs.transpose().conjugate())) +
                                   gamma * np.dot(W, np.dot(Hu, Hu.transpose().conjugate())) + beta * DW + lamb * W,
                                   1e-10)))

            Wt = W.transpose().conjugate()
            WtW = np.dot(Wt, W)
            WtXs = sps.csr_matrix.dot(Wt, Xs)
            WtXu = sps.csr_matrix.dot(Wt, Xu)
            WtWHs = np.dot(WtW, Hs)
            WtWHu = np.dot(WtW, Hu)
            DW = D.dot(W)
            AW = A.dot(W)

            tr1 = alpha * (trXstXs - 2 * tr(Hs, WtXs) + tr(Hs, WtWHs))
            tr2 = gamma * (trXutXu - 2 * tr(Hu, WtXu) + tr(Hu, WtWHu))
            tr3 = beta * (tr(W, DW) - tr(W, AW))
            tr4 = lamb * (np.trace(WtW) + tr(Hs, Hs) + tr(Hu, Hu))
            Obj = tr1 + tr2 + tr3 + tr4
            ObjHistory.append(Obj)

            if itNum > 1:
                delta = np.abs(ObjHistory[-1] - ObjHistory[-2])
                if verbose:
                    self._print('Iteration: {} \t Objective: {:.5f} \t Delta: {:.5f}'
                                .format(self.RECOMMENDER_NAME, itNum, Obj, delta))
            elif verbose:
                self._print('Iteration: {} \t Objective: {:.5f}'
                            .format(self.RECOMMENDER_NAME, itNum, Obj))

            itNum += 1

        self.W = W
        self.Hs = Hs
        self.Hu = Hu

        self.Ws_test = []
        batch_size = 1000
        for i in range(int(self.n_items / batch_size) + 1):
            if verbose:
                self._print("Calculating Ws test ({}/{})"
                            .format(self.RECOMMENDER_NAME, i, int(self.n_items / batch_size)))
            start = i * batch_size
            end = min(self.n_items, (1+i) * batch_size)
            self.Ws_test.append(lstsq(Hs.T, Xs[start:end, :].toarray().T, rcond=None)[0].T)

        self.Ws_test = np.vstack(self.Ws_test)

    def compute_item_score(self, user_id_array, items_to_compute=None):

        assert self.Hu.shape[1] > user_id_array.max(),\
                "PureSVD: Cold users not allowed. Users in trained model are {}, " \
                "requested prediction for users up to {}".format(self.Hu.shape[1], user_id_array.max())

        if items_to_compute is not None:
            item_scores = self.Ws_test.dot(self.Hu[:, user_id_array])
        else:
            item_scores = self.Ws_test.dot(self.Hu[:, user_id_array])

        return item_scores.T

    def _get_dict_to_save(self):

        return {
            "Ws_test": self.Ws_test,
            "Hs": self.Hs,
            "Hu": self.Hu,
        }

max_stars_repo_name: damicoedoardo/NNMF

max_issues_repo_name: damicoedoardo/NNMF

max_forks_repo_name: damicoedoardo/NNMF

reason: import numpy,from numpy,import scipy