text:
"""Utility functions for testing."""
# pylint: disable=protected-access
import os
import tempfile

import pysam
from numpy import random

import pybedtools
from pybedtools import create_interval_from_list

import iCount

BASES = ['A', 'C', 'G', 'T']

def get_temp_file_name(tmp_dir=None, extension=''):
    """Return an availiable name for temporary file."""
    tmp_name = next(tempfile._get_candidate_names())
    if not tmp_dir:
        tmp_dir = tempfile._get_default_tempdir()
    if extension:
        tmp_name += '.' + extension
    return os.path.join(tmp_dir, tmp_name)

def get_temp_dir(tmp_dir=None):
    """Return a temporary directory."""
    return tempfile.mkdtemp()

def make_file_from_list(data, bedtool=True, extension='', tmp_dir=None, tfile=None, sort=False):
    """Return path to file with the content from `data` (list of lists)."""
    if tfile is None:
        tfile = get_temp_file_name(extension=extension, tmp_dir=tmp_dir)
    if bedtool:
        tmp_bt = pybedtools.BedTool(
            pybedtools.create_interval_from_list(line) for line in data).saveas()
        if sort:
            tmp_bt.sort().saveas(tfile)
        else:
            tmp_bt.saveas(tfile)
    else:
        with open(tfile, 'wt') as file_:
            for list_ in data:
                file_.write('\t'.join(map(str, list_)) + '\n')
    return os.path.abspath(tfile)

def make_list_from_file(fname, fields_separator=None):
    """Read file to list of lists."""
    data = []
    with iCount.files.gz_open(fname, 'rt') as file_:
        for line in file_:
            data.append(line.strip().split(fields_separator))
    return data

def list_to_intervals(data):
    """Transform list of lists to list of pybedtools.Intervals."""
    return [create_interval_from_list(list_) for list_ in data]

def intervals_to_list(data):
    """Transform list of pybedtools.Intervals to list of lists."""
    return [interval.fields for interval in data]

def reverse_strand(data):
    """
    Reverse the strand of every element in data.

    Elements can be pybedtools.Intervals or lists with same content as
    interval.fields.
    """
    rstrands = ['-' if i[6] == '+' else '+' for i in data]
    if isinstance(data[0], pybedtools.Interval):
        return [create_interval_from_list(
            data[i][:6] + [rstrands[i]] + data[i][7:]) for i in range(len(data))]
    else:
        return [
            data[i][:6] + [rstrands[i]] + data[i][7:] for i in range(len(data))]

def make_sequence(size, include_n=False, rnd_seed=42):
    """Make random DNA segment of length `size`."""
    random.seed(rnd_seed)  # pylint:disable=no-member
    if include_n:
        bases = ['A', 'C', 'G', 'T', 'N']
    else:
        bases = ['A', 'C', 'G', 'T']
    return ''.join(random.choice(bases, size))  # pylint: disable=no-member

def make_quality_scores(size, min_chr=33, max_chr=74, rnd_seed=42):
    """Make random DNA segment of length `size`."""
    random.seed(rnd_seed)  # pylint:disable=no-member
    scores = [chr(i) for i in range(min_chr, max_chr + 1)]
    return ''.join(random.choice(scores, size))  # pylint: disable=no-member

def make_aligned_segment(data, rnd_seed=None):
    """
    Return pysam.AlignedSegment() object with the data in `data`.

    Each pysam.AlignedSegment element has 11 mandatory tab-separated
    fields. Additionaly there is TAGS field for additional info::

        QNAME FLAG RNAME POS MAPQ CIGAR RNEXT PNEXT TLEN SEQ QUAL TAGS

    Since only some fields are required for our purpuses, the input
    parameter `data` should be a tuple with the following content:

    (qname, flag, refname, pos, mapq, cigar, tags)

        * qname - querry name
        * flag - bitwise flag, detailed description in [1]
        * refname - Index of the reference_sequence in header
        * pos - starting position of read in reference sequence
        * mapq - mapping quality
        * cigar - CIGAR values, detailed description in [1]
        * tags - additional information

    Flag is a bitwise value. For detailed explanation, see [1], but for our
    needs, we only need values 4 (is_unmapped) and 16 (reverse_strand).

    Columns RNEXT and RNAME are left undefined, since we are only interested
    in sigle-end reads. TLEN is also set to undefined (value 0). SEQ and
    QUAL are randomly generated, their size is determined by cigar value.

    Example of input parameter `data`:

        data = ('name123', 20, 0, 30, 20, [(0, 90), (2, 5), (0, 5)], {'NH': 5})

    Read more about SAM file specifications here:
    [1] https://samtools.github.io/hts-specs/SAMv1.pdf

    Parameters
    ----------
    data : tuple
        Input data for AlignedSegment.

    Returns
    -------
    pysam.AlignedSegment
        AlignedSegment, filled with given data.

    """
    # pylint: disable=no-member
    segment = pysam.AlignedSegment()
    segment.query_name = data[0]
    segment.flag = data[1]
    segment.reference_id = data[2]
    segment.reference_start = data[3]
    segment.mapping_quality = data[4]
    segment.cigar = data[5]

    segment.next_reference_id = 0
    segment.next_reference_start = 0
    segment.template_length = 0

    length = sum([n2 for (n1, n2) in segment.cigar if n1 in [0, 1, 4, 7, 8]])
    segment.query_sequence = make_sequence(size=length, include_n=True, rnd_seed=rnd_seed)
    segment.query_qualities = pysam.qualitystring_to_array(
        make_quality_scores(size=length, rnd_seed=rnd_seed))

    segment.tags = data[6].items()

    return segment

def make_bam_file(data, rnd_seed=None):
    """
    Make a BAM file and fill it with content of `data`.

    Each BAM file consists of header and allignment entries. Main content of the
    header are the reference sequence definitions (chromosomes). Allignment
    entries (segments) are records of how each read is mapped to a reference.

    The structure of input parameter `data` should be the following:

        data = {
            # define all the chromosome names and their lengths:
            'chromosomes': [('chr1', 3000), ('chr2', 2000)],
            'segments': [
                # Each segment should be a 7-tuple - for detailed description
                # see `make_aligned_segment` function docstring
                # (qname, flag, refname, pos, mapq, cigar, tags)
                ('name1', 4, 0, 100, 20, [(0, 100)], {'NH': 2}),
                ...
                ('namex', 4, 0, 100, 20, [(0, 100)], {'NH': 2})]}

    Links for further read:
    https://samtools.github.io/hts-specs/SAMv1.pdf
    http://pysam.readthedocs.io/en/stable/usage.html#creating-bam-cram-sam-files-from-scratch

    Parameters
    ----------
    data : dict
        Input data for BAM file

    Returns
    -------
    str
        Absoulte path to bamfile.

    """
    random.seed(rnd_seed)  # pylint:disable=no-member
    fname = get_temp_file_name(extension='bam')

    # Make header:
    chromosomes = [dict(
        [('SN', chrom), ('LN', size)]) for chrom, size in data['chromosomes']]
    header = {'HD': {'VN': '1.0'}, 'SQ': chromosomes}

    random_seeds = random.randint(10**5, size=len(data['segments']))  # pylint:disable=no-member
    with pysam.AlignmentFile(fname, "wb", header=header) as outf:  # pylint: disable=no-member
        for segment_data, rseed in zip(data['segments'], random_seeds):
            outf.write(make_aligned_segment(segment_data, rnd_seed=rseed))

    pysam.index(fname)  # pylint:disable=no-member

    return os.path.abspath(fname)

def make_fasta_file(sequences=None, headers=None, out_file=None, num_sequences=10, seq_len=80,
                    rnd_seed=None):
    """Make artificial FASTA file."""
    random.seed(rnd_seed)  # pylint:disable=no-member
    if sequences is None and headers is None:
        headers = ['{}'.format(i + 1) for i in range(num_sequences)]
        random_seeds = random.randint(10**5, size=num_sequences)  # pylint:disable=no-member
        sequences = [make_sequence(seq_len, rnd_seed=rnd) for rnd in random_seeds]
    elif sequences is None:
        random_seeds = random.randint(10**5, size=len(headers))  # pylint:disable=no-member
        sequences = [make_sequence(seq_len, rnd_seed=rnd) for rnd in random_seeds]
    elif headers is None:
        headers = ['{}'.format(i + 1) for i in range(len(sequences))]

    if out_file is None:
        out_file = get_temp_file_name(extension='fasta')
    with open(out_file, 'wt') as ofile:
        for header, seq in zip(headers, sequences):
            ofile.write('>' + header + '\n')
            ofile.write(seq + '\n')

    return os.path.abspath(out_file)

def make_fastq_file(genome=None, barcodes=None, adapter='', out_file=None,
                    num_sequences=10, seq_len=80, rnd_seed=None):
    """
    Make artificial FASTQ file.

    TODO: refactor and write more descriptive doscrting.
    """
    random.seed(rnd_seed)  # pylint:disable=no-member
    if barcodes is None:
        barcodes = ['NNN']

    if genome is not None:
        genome_data = iCount.files.fasta.read_fasta(genome)
        num_sequences = len(genome_data)
        seq_len = len(genome_data[0])

    seq_len_reduced = seq_len - (len(barcodes[0]) if barcodes else 0) - len(adapter)

    num_barcodes = len(barcodes)

    def make_fastq_entry(ofile, rseed):  # pylint: disable=missing-docstring
        description = 'artificial header {}'.format(random.random())

        # Select random barcode and transform 'N'-s to nucleotids:
        pre_barcode = barcodes[random.randint(0, num_barcodes)]  # pylint: disable=no-member
        barcode = ''
        for letter in pre_barcode:
            if letter == 'N':
                # Randomly select one of bases:
                barcode += BASES[random.randint(0, 4)]  # pylint: disable=no-member
            else:
                barcode += letter

        if genome:
            int1 = random.randint(0, high=len(genome_data))  # pylint: disable=no-member
            int2 = random.randint(0, high=len(genome_data[int1][1]))  # pylint: disable=no-member
            random_piece = genome_data[int1][1][int2:int2 + seq_len_reduced]
            seq = barcode + random_piece + adapter
        else:
            seq = barcode + make_sequence(seq_len_reduced, rnd_seed=rseed) + adapter
        quality_scores = make_quality_scores(len(seq), rnd_seed=rseed)

        ofile.write('@' + description + '\n')
        ofile.write(seq + '\n')
        ofile.write('+' + '\n')
        ofile.write(quality_scores + '\n')

    if out_file is None:
        out_file = get_temp_file_name()

    random_seeds = random.randint(10**5, size=num_sequences)  # pylint:disable=no-member
    with open(out_file, 'wt') as ofile:
        for rseed in random_seeds:
            make_fastq_entry(ofile, rseed)

    return os.path.abspath(out_file)

def attrs(gid='.', tid=None, exn=None, gbio=None, tbio=None, iid=None, bio=None):
    """Make gtf attributes (to shorten line)."""
    attrs_ = ['gene_id "{}"'.format(gid)]
    if tid:  # transcript_id
        attrs_.append('transcript_id "{}"'.format(tid))
    if exn:  # exon number
        attrs_.append('exon_number "{}"'.format(exn))
    if gbio:  # gene biotype
        attrs_.append('gene_type "{}"'.format(gbio))
    if tbio:  # transcript biotype
        attrs_.append('transcript_type "{}"'.format(tbio))
    if iid:  # intergenic ID number
        attrs_.append('ID "{}"'.format(iid))
    if bio:  # intergenic ID number
        attrs_.append('biotype "{}"'.format(bio))
    return '; '.join(sorted(attrs_)) + ';'

max_stars_repo_name: genialis/iCount

max_issues_repo_name: genialis/iCount

max_forks_repo_name: ulelab/iCount

reason: from numpy


text:
from scipy.integrate import quad
import numpy as np
import logging

def test_age():
    from isochrones.priors import age_prior
    age_prior.test_integral()
    age_prior.test_sampling()

def test_distance():
    from isochrones.priors import distance_prior
    distance_prior.test_integral()
    distance_prior.test_sampling()

def test_AV():
    from isochrones.priors import AV_prior
    AV_prior.test_integral()
    AV_prior.test_sampling()

def test_q():
    from isochrones.priors import q_prior
    q_prior.test_integral()
    q_prior.test_sampling()

def test_salpeter():
    from isochrones.priors import salpeter_prior
    salpeter_prior.test_integral()
    salpeter_prior.test_sampling()

def test_feh():
    from isochrones.priors import feh_prior
    feh_prior.test_integral()
    feh_prior.test_sampling()

ext: py

max_stars_repo_name: joelczinn/isochrones

max_issues_repo_name: joelczinn/isochrones

max_forks_repo_name: joelczinn/isochrones

reason: import numpy,from scipy


text:
import json
import os
import random
from typing import Dict

import numpy as np
import torch
from sklearn.metrics import f1_score
from torch.utils.data import Dataset
from transformers.data.metrics import simple_accuracy

from ..model import ModelMode, task_model
from ..model.model_args import ModelArguments
from ..task.task_args import TaskArguments, parse_tasks_args
from ..train.trainer import Trainer
from ..train.trainer_utils import PredictionOutput, EvalPrediction, InputExample
from ..train.training_args import TrainingArguments
from ..data import DataArguments, TaskDataset, DataProcessor
from ..transformersx_base import aiutils, log, join_path

def set_seed(seed: int):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    # ^^ safe to call this function even if cuda is not available

##TODO: need to refactor the TaskModel for various cases
##(1) train a pretrained model from scratch
##(2) finetuning a task based on a pretrained Model
##(3) resume a task from the training interruption
## Currently, if need to resume the training, you must reset the model_path to the checkpoint
class TaskModel:
    def __init__(self, modelArgs: ModelArguments, model_class=None):
        self._model_args = modelArgs
        self._user_model_class = model_class
        self._init()

    def _init(self):
        self.task_model, model_cache_dir = self._build_task_model(self._model_args, self._user_model_class)
        log.info("Loading the task model from {}, {}".format(model_cache_dir, str(self.task_model)))

        self.config, self.tokenizer, self.model = self.task_model.load(
            num_labels=self._model_args.num_labels,
            unit_test=self._model_args.unit_test,
            cache_dir=model_cache_dir
        )

        log.info(
            "Loaded task model, config: {}, tokenizer: {}, "
            "model: {}".format(self.config.to_json_string(False),
                               type(self.tokenizer),
                               type(self.model))
        )

        self._freeze_parameters()

    def _freeze_parameters(self):
        # Currently, freezing the main paramerter is supported only
        self._freeze_weights_main()
        # self._freeze_parameters_others()

        if hasattr(self.model, 'num_parameters'):
            log.info("num params:" + str(self.model.num_parameters()))
            log.info("num trainable params:" + str(self.model.num_parameters(only_trainable=True)))

        if hasattr(self.model, 'named_parameters'):
            log.info("Model Parameters Details: ")
            for name, param in self.model.named_parameters():
                log.info("{}:{}".format(name, param.size()))

    def _freeze_weights_main(self):
        if self.model is None or not self._model_args.freeze_main:
            return

        main_parameters = eval("self.model." + self.task_model.main_parameter)
        if hasattr(main_parameters, "parameters"):
            for param in main_parameters.parameters():
                param.requires_grad = False

    def _freeze_weights_others(self):
        if self.model is None:
            return

        if self._model_args.freeze_parameter and hasattr(self.model, self._model_args.freeze_parameter):
            freeze_model = eval("self.model." + self._model_args.freeze_parameter)
            if hasattr(freeze_model, "parameters"):
                for param in freeze_model.parameters():
                    param.requires_grad = False

    def _build_task_model(self, modelArgs, model_class=None):
        mode_cache_dir = modelArgs.model_pretrained_dir if not modelArgs.not_use_pretrained else modelArgs.model_finetuned_dir

        if not os.path.exists(join_path(mode_cache_dir, modelArgs.model_name)):
            mode_cache_dir = modelArgs.model_pretrained_dir if modelArgs.not_use_pretrained else modelArgs.model_finetuned_dir

        if not os.path.exists(join_path(mode_cache_dir, modelArgs.model_name)):
            raise ValueError(
                "Can not find mode {} in {} or {}".format(modelArgs.model_name, modelArgs.model_finetuned_dir,
                                                          modelArgs.model_pretrained_dir)
            )

        t_model = task_model(model_path=modelArgs.model_name,
                             model_task_type=modelArgs.model_task_type,
                             language=modelArgs.language,
                             framework=modelArgs.framework)

        # use the custom class to replace the model Class
        if model_class is not None:
            t_model.model_class = model_class

        log.info("Built task model: {}".format(str(t_model)))

        return t_model, mode_cache_dir

    def _acc_and_f1(self, preds, labels):
        acc = simple_accuracy(preds, labels)
        f1 = f1_score(y_true=labels, y_pred=preds, average="weighted")
        return {
            "acc": acc,
            "f1": f1,
            "acc_and_f1": (acc + f1) / 2,
        }

    def predict(self, *input):
        return self.model(*input)

    def compute_metrics(self, p: EvalPrediction) -> Dict:
        if self._model_args.model_mode == ModelMode.classification:
            preds = np.argmax(p.predictions, axis=1)
        elif self._model_args.model_mode == ModelMode.regression:
            preds = np.squeeze(p.predictions)
        return self._acc_and_f1(preds, p.label_ids)

class TaskData:
    def __init__(self, dataArgs: DataArguments,
                 tokenizer, dataProcessor: DataProcessor, local_rank):
        self._data_args = dataArgs
        self._tokenizer = tokenizer
        self._processor = dataProcessor
        self._local_rank = local_rank
        self._train_data, self._eval_data = None, None

    def train_data(self):
        if not self._train_data and self._processor:
            self._train_data = TaskDataset(self._data_args,
                                           tokenizer=self._tokenizer,
                                           processor=self._processor,
                                           local_rank=self._local_rank)
        return self._train_data

    def eval_data(self):
        if not self._eval_data and self._processor:
            self._eval_data = TaskDataset(self._data_args,
                                          tokenizer=self._tokenizer,
                                          processor=self._processor,
                                          local_rank=self._local_rank,
                                          evaluate=True)
        return self._eval_data

class TaskTrainer:
    def __init__(self, trainingArgs: TrainingArguments,
                 taskModel: TaskModel, taskData: TaskData, compute_metrics=None):
        self._training_args = trainingArgs
        self._taskModel = taskModel
        self._taskData = taskData
        self._compute_metrics = compute_metrics if compute_metrics else taskModel.compute_metrics
        self._trainer = self._create_trainer()

    def _check_train_args(self):
        if (
                self._training_args.do_train
                and not self._training_args.overwrite_output_dir
                and os.path.exists(self._training_args.output_dir)
                and os.listdir(self._training_args.output_dir)
        ):
            raise ValueError(
                f"Output directory ({self._training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
            )

    def _create_trainer(self):
        return Trainer(
            model=self._taskModel.model,
            args=self._training_args,
            train_dataset=self._taskData.train_data() if self._training_args.do_train and self._taskData else None,
            eval_dataset=self._taskData.eval_data() if self._training_args.do_eval and self._taskData else None,
            compute_metrics=self._compute_metrics,
        )

    def train(self):
        if self._training_args.do_train:
            # Here, the model_path is for the trainer to load the optimizer and scheduler states
            self._trainer.train(
                model_path=self._taskModel.task_model.model_path
            )
            self._trainer.save_model()
            # For convenience, we also re-save the tokenizer to the same directory,
            # so that you can share your model easily on huggingface.co/models =)
            if self._trainer.is_world_master():
                self._taskModel.tokenizer.save_pretrained(self._training_args.output_dir)

        return self

    def eval(self):
        # Evaluation
        results = {}
        if self._training_args.do_eval and self._training_args.local_rank in [-1, 0]:
            log.info("*** Evaluate ***")

            eval_datasets = [self._taskData.eval_data()]

            for eval_dataset in eval_datasets:
                result = self._trainer.evaluate(eval_dataset=eval_dataset)

                output_eval_file = join_path(
                    self._training_args.output_dir, f"eval_results_{eval_dataset.args.task_name}.txt"
                )
                with open(output_eval_file, "w") as writer:
                    log.info("***** Eval results {} *****".format(eval_dataset.args.task_name))
                    for key, value in result.items():
                        log.info("  %s = %s", key, value)
                        writer.write("%s = %s\n" % (key, value))

                results.update(result)
        return results

    def predict(self, test_dataset: Dataset = None) -> PredictionOutput:
        return self._trainer.predict(test_dataset=self._taskData.eval_data() if not test_dataset else test_dataset)

class TransformerTask:
    def __init__(self, task_name, task_args: TaskArguments,
                 dataProcessor=None, model_class=None, compute_metric=None):
        log.info("Create a TransformerTask with arguments: ")
        log.info(json.dumps(task_args, default=lambda obj: obj.__dict__, indent=True))

        self.task_args = task_args
        self.task_args.validate_and_set_task_name(task_name)
        self._training_args = task_args.training_args

        self._taskModel = TaskModel(task_args.model_args, model_class)
        self._taskData = TaskData(task_args.data_args, self._taskModel.tokenizer, dataProcessor,
                                  task_args.training_args.local_rank) if dataProcessor else None
        self._taskTrainer = TaskTrainer(task_args.training_args, self._taskModel,
                                        self._taskData, compute_metric)

    def _log_task_start(self):
        log.warning(
            "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
            self._training_args.local_rank,
            self._training_args.device,
            self._training_args.n_gpu,
            bool(self._training_args.local_rank != -1),
            self._training_args.fp16,
        )
        log.info("Training/evaluation parameters %s", self._training_args)

    def train(self):
        if not self._training_args.do_train:
            log.warn("do_train argument is set for False")
            return
        self._log_task_start()

        set_seed(self._training_args.seed)
        results = self._taskTrainer.train().eval()
        return results

    def eval(self):
        self._log_task_start()
        set_seed(self._training_args.seed)
        results = self._taskTrainer.eval()
        return results

    def single_predict(self, example: InputExample):
        data_processor = DataProcessor()
        data_processor.get_dev_examples = lambda: [example]
        data_processor.get_labels = lambda: []
        return self.predict(data_processor)

    def predict(self, data_processor=None) -> PredictionOutput:
        dataset = TaskDataset(self.task_args.data_args,
                              tokenizer=self._taskModel.tokenizer,
                              processor=data_processor,
                              evaluate=True) if data_processor else None

        return self._taskTrainer.predict(dataset)

class DefaultTask(TransformerTask):
    def __init__(self, dataProcessorClass, model_class=None, compute_metric=None):
        super().__init__("default", parse_tasks_args(), dataProcessorClass, model_class, compute_metric)

def get_transformers_model(models_dir, model_name):
    modelArgs: ModelArguments = ModelArguments()
    modelArgs.model_name = model_name
    modelArgs.model_finetuned_dir = models_dir
    modelArgs.not_use_pretrained = True
    return TaskModel(modelArgs)

max_stars_repo_name: aicanhelp/ai-transformers

max_issues_repo_name: aicanhelp/ai-transformers

max_forks_repo_name: aicanhelp/ai-transformers

reason: import numpy


text:
#!/usr/bin/env python3

# Copyright 2018-2021 VMware, Inc., Microsoft Inc., Carnegie Mellon University, ETH Zurich, and University of Washington
# SPDX-License-Identifier: BSD-2-Clause

import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import sys
import operator
import bisect
from parser import Experiment
from PlotHelper import *
from TimeSeries import *

output_filename = "compare.png"

def plot_perf_compare(experiments):
    plotHelper = PlotHelper(4, scale=2)

    try: plotThroughput(plotHelper.nextAxis(depth=2), experiments)
    except: raise

    try: plotGrandUnifiedMemory(plotHelper.nextAxis(depth=2), experiments)
    except: raise

#    try: plotSlowIos(plotHelper.nextAxis(depth=2), experiments)
#    except: pass
#
#    try: plotCpuTime(plotHelper.nextAxis(depth=2), experiments)
#    except: raise
#
#    try: plotProcIoBytes(plotHelper.nextAxis(depth=2), experiments)
#    except: raise

#    try: plotIoLatencyCdf(plotHelper.nextAxis(depth=2), experiments)
#    except: raise

#    try: plotRocksIo(plotHelper.nextAxis(depth=2), experiments)
#    except: raise

    plotHelper.save(output_filename)

experiments = []
for arg in sys.argv[1:]:
    nick,fn = arg.split("=")
    if nick=="output":
        output_filename = fn
    else:
        try:
            exp = Experiment(fn, nick)
            #exp.sortedOpns = exp.sortedOpns[:-5]    # hack: truncate teardown tail of completed exp where memory all goes to 0
            experiments.append(exp)
        except (ValueError,FileNotFoundError):
            print("Can't parse %s; skipping" % nick)
plot_perf_compare(experiments)

max_stars_repo_name: scottviteri/verified-betrfs

max_issues_repo_name: scottviteri/verified-betrfs

max_forks_repo_name: scottviteri/verified-betrfs


text:
from src.utils import utils
from src.configurations import classifier_config as config
from src.configurations import embeddings_config as embeddings_config
from src.dataset.dataset import *
from src.evaluation.evaluators import RetrievalEvaluator
from src.models.sentence_encoder import SentenceTransformerWrapper
from src.utils.metrics import RetrievalAccuracyMeter, cos_sim
import numpy as np
import torch
from torch import nn
from torch.nn import functional as F

MODEL_NAMES = {
    "mbert_avg": "mbert_sense_features_avg",
    "mbert_cls": "mbert_sense_cls_features",
    "baseline": "mbert_no_sense_cls",
    "siamese_contrastive_cls": "training_siamese_paws_sense_online_contrastive_cls_pooling_5epochs"
}

MODEL_NAME = MODEL_NAMES["baseline"]
PRETRAINED_PATH = f"../training/trained_models/{MODEL_NAME}"

metrics = {"validation": [RetrievalAccuracyMeter]}

model = MBERTClassifier(
    strategy="cls",
    train_model = False,
    use_sense_embeddings=False
)

model.load_pretrained(PRETRAINED_PATH)

model.to(config.DEVICE)

model.eval()

language_pairs = ['eng-jpn', 'eng-deu', 'eng-fra', 'eng-spa', 'eng-kor']
dev_paths = [f'../data/tatoeba/parallel-sentences/Tatoeba-{pair}-dev.tsv.gz' for pair in language_pairs]

langs = {
    "jp": 0,
    "de": 1,
    "fr": 2,
    "es": 3,
    "ko": 4
}

lang = langs["jp"]

valid_dataset = ParallelDataset.build_dataset([dev_paths[lang]])

valid_data_loader = SmartParaphraseDataloader.build_batches(valid_dataset, 16, mode="tatoeba")

metrics = {"validation": [RetrievalAccuracyMeter]}

print(f"Lang: {language_pairs[lang]}")

evaluator = RetrievalEvaluator(
    model = model,
    data_loader=valid_data_loader,
    device=config.DEVICE,
    metrics=metrics,
    fp16=True,
    verbose=True,
    return_predictions = False

)

evaluator.evaluate()


text:
from copy import copy
import numpy as np
from ctapipe.io import read_table
from ctapipe.instrument import SubarrayDescription
from ctapipe.visualization import CameraDisplay
import matplotlib.pyplot as plt

path = 'data-structures/lst_events.dl1.hdf5'
subarray = SubarrayDescription.from_hdf(path)

event_index = 1

images1 = read_table(path, '/dl1/event/telescope/images/tel_003')
images2 = read_table(path, '/dl1/event/telescope/images/tel_002')

fig, axs = plt.subplots(2, 2, constrained_layout=True)

cmap_image = copy(plt.get_cmap('inferno'))
cmap_image.set_bad('k')
cmap_time = copy(plt.get_cmap('RdBu_r'))
cmap_time.set_bad('lightgray')

cam = subarray.tel[1].camera.geometry

displays = [
    CameraDisplay(cam, show_frame=False, ax=axs[0, 0], cmap=cmap_image),
    CameraDisplay(cam, show_frame=False, ax=axs[0, 1], cmap=cmap_time),
    CameraDisplay(cam, show_frame=False, ax=axs[1, 0], cmap=cmap_image),
    CameraDisplay(cam, show_frame=False, ax=axs[1, 1], cmap=cmap_time),
]

displays[0].image = images1['image'][event_index]
peak_time1 = images1['peak_time'][event_index]
peak_time1[~images1['image_mask'][event_index]] = np.nan
displays[1].image = peak_time1

displays[2].image = images2['image'][event_index]
peak_time2 = images2['peak_time'][event_index]
peak_time2[~images2['image_mask'][event_index]] = np.nan
displays[3].image = peak_time2

displays[0].norm = 'log'
displays[2].norm = 'log'

for ax in axs.flat:
    ax.set_axis_off()
    ax.set_title('')

for display in displays:
    display.add_colorbar()
    display.pixels.set_rasterized(True)

# displays[1].set_limits_minmax(10, 20)
axs[0, 0].set_title('Number of Photons')
axs[0, 1].set_title('Arrival Time in ns')

fig.savefig('build/data-structures/lst_images.pdf')


text:
# -*- mode: python; coding: utf-8 -*-
# Copyright (c) 2018 Radio Astronomy Software Group
# Licensed under the 2-clause BSD License

"""Tests for uvbase object.

"""
from astropy.time import Time
import pytest
import numpy as np

from pyuvdata.uvbase import UVBase
from pyuvdata.uvbase import _warning
from pyuvdata import parameter as uvp

ref_latlonalt = (-26.7 * np.pi / 180.0, 116.7 * np.pi / 180.0, 377.8)
ref_xyz = (-2562123.42683, 5094215.40141, -2848728.58869)

class UVTest(UVBase):
    def __init__(self):
        """ UVBase test object. """
        # add some test UVParameters to the class

        self._int1 = uvp.UVParameter(
            "int1", description="integer value", expected_type=int, value=3
        )

        self._int2 = uvp.UVParameter(
            "int2", description="integer value", expected_type=int, value=5
        )

        self._float1 = uvp.UVParameter(
            "float1", description="float value", expected_type=np.float, value=18.2
        )

        self._string1 = uvp.UVParameter(
            "string1", description="string value", form="str", value="test"
        )

        self._string2 = uvp.UVParameter(
            "string2", description="string value", expected_type="str", value="test"
        )

        self._floatarr = uvp.UVParameter(
            "floatarr",
            description="float array",
            form=("int1", "int2"),
            expected_type=np.float,
            value=np.random.rand(self._int1.value, self._int2.value),
        )

        self._floatarr2 = uvp.UVParameter(
            "floatarr2",
            description="float array",
            form=4,
            expected_type=np.float,
            value=np.random.rand(4),
        )

        self._strlist = uvp.UVParameter(
            "strlist",
            description="string list",
            form=("int1",),
            expected_type=str,
            value=["s" + str(i) for i in np.arange(self._int1.value)],
        )

        self._intlist = uvp.UVParameter(
            "intlist",
            description="integer list",
            form=("int1",),
            expected_type=int,
            value=list(range(self._int1.value)),
        )

        self._angle = uvp.AngleParameter(
            "angle", description="angle", expected_type=np.float, value=np.pi / 4.0
        )

        self._location = uvp.LocationParameter(
            "location", description="location", value=np.array(ref_xyz)
        )

        self._time = uvp.UVParameter(
            "time",
            description="astropy Time object",
            value=Time("2015-03-01 00:00:00", scale="utc"),
            expected_type=Time,
        )

        self._optional_int1 = uvp.UVParameter(
            "optional_int1",
            description="optional integer value",
            expected_type=int,
            value=3,
            required=False,
        )

        self._optional_int2 = uvp.UVParameter(
            "optional_int2",
            description="optional integer value",
            expected_type=int,
            value=5,
            required=False,
        )

        self._unset_int1 = uvp.UVParameter(
            "unset_int1",
            description="An unset parameter.",
            expected_type=int,
            value=None,
            required=False,
        )

        super(UVTest, self).__init__()

def test_equality():
    """Basic equality test."""
    test_obj = UVTest()
    assert test_obj == test_obj

def test_equality_nocheckextra():
    """Test equality if optional params are different and check_extra is false."""
    test_obj = UVTest()
    test_obj2 = test_obj.copy()
    test_obj2.optional_int1 = 4
    assert test_obj.__eq__(test_obj2, check_extra=False)

def test_inequality_extra():
    """Basic equality test."""
    test_obj = UVTest()
    test_obj2 = test_obj.copy()
    test_obj2.optional_int1 = 4
    assert test_obj != test_obj2

def test_inequality_different_extras():
    """Basic equality test."""
    test_obj = UVTest()
    test_obj2 = test_obj.copy()
    test_obj2._optional_int3 = uvp.UVParameter(
        "optional_int3",
        description="optional integer value",
        expected_type=int,
        value=7,
        required=False,
    )
    assert test_obj != test_obj2

def test_inequality():
    """Check that inequality is handled correctly."""
    test_obj = UVTest()
    test_obj2 = test_obj.copy()
    test_obj2.float1 = 13
    assert test_obj != test_obj2

def test_class_inequality():
    """Test equality error for different classes."""
    test_obj = UVTest()
    assert test_obj != test_obj._floatarr

def test_check():
    """Test simple check function."""
    test_obj = UVTest()
    assert test_obj.check()

def test_check_required():
    """Test simple check function."""
    test_obj = UVTest()
    assert test_obj.check(check_extra=False)

def test_string_type_check():
    """Test check function with wrong type (string)."""
    test_obj = UVTest()
    test_obj.string1 = 1
    pytest.raises(ValueError, test_obj.check)

def test_string_form_check():
    """Test check function with wrong type (string)."""
    test_obj = UVTest()
    test_obj.string2 = 1
    pytest.raises(ValueError, test_obj.check)

def test_single_value_check():
    """Test check function with wrong type."""
    test_obj = UVTest()
    test_obj.int1 = np.float(test_obj.int1)
    pytest.raises(ValueError, test_obj.check)

def test_check_array_type():
    """Test check function with wrong array type."""
    test_obj = UVTest()
    test_obj.floatarr = test_obj.floatarr + 1j * test_obj.floatarr
    pytest.raises(ValueError, test_obj.check)

def test_check_array_shape():
    """Test check function with wrong array dimensions."""
    test_obj = UVTest()
    test_obj.floatarr = np.array([4, 5, 6], dtype=np.float)
    pytest.raises(ValueError, test_obj.check)

def test_list_dims():
    """Test check function with wrong list dimensions."""
    test_obj = UVTest()
    test_obj.strlist = ["s" + str(i) for i in np.arange(test_obj.int2)]
    pytest.raises(ValueError, test_obj.check)

def test_list_type():
    """Test check function with wrong list type."""
    test_obj = UVTest()
    test_obj.intlist = ["s" + str(i) for i in np.arange(test_obj.int1)]
    pytest.raises(ValueError, test_obj.check)

    test_obj.intlist = list(np.arange(test_obj.int1))
    test_obj.intlist[1] = "test"
    pytest.raises(ValueError, test_obj.check)

def test_angle():
    test_obj = UVTest()
    test_obj2 = test_obj.copy()
    test_obj2.angle_degrees = 45.0
    assert test_obj == test_obj2

def test_angle_none():
    test_obj = UVTest()
    test_obj2 = test_obj.copy()
    test_obj.angle = None
    test_obj2.angle_degrees = None
    assert test_obj == test_obj2

def test_location():
    test_obj = UVTest()
    test_obj2 = test_obj.copy()
    test_obj2.location_lat_lon_alt = ref_latlonalt
    assert test_obj == test_obj2

def test_location_degree():
    test_obj = UVTest()
    test_obj2 = test_obj.copy()
    test_obj2.location_lat_lon_alt_degrees = (
        np.rad2deg(ref_latlonalt[0]),
        np.rad2deg(ref_latlonalt[1]),
        ref_latlonalt[2],
    )
    assert test_obj == test_obj2

def test_location_none():
    test_obj = UVTest()
    test_obj2 = test_obj.copy()
    test_obj.location = None
    test_obj2.location_lat_lon_alt = None
    assert test_obj == test_obj2

def test_location_degree_none():
    test_obj = UVTest()
    test_obj2 = test_obj.copy()
    test_obj.location = None
    test_obj2.location_lat_lon_alt_degrees = None
    assert test_obj == test_obj2

def test_warning():
    output = _warning("hello world")
    assert output, "hello world\n"

def test_check_ignore_unset():
    test_obj = UVTest()
    test_obj._unset_int1.required = True
    pytest.raises(ValueError, test_obj.check)
    assert test_obj.check(ignore_requirements=True)

max_stars_repo_name: no-lex/pyuvdata

max_issues_repo_name: no-lex/pyuvdata

max_forks_repo_name: no-lex/pyuvdata

reason: import numpy,from astropy


text:
# Copyright 2020 The Magenta Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Generate from trained model from scratch or condition on a partial score."""
import itertools as it
import os
import re
import time

from magenta.models.coconet import lib_graph
from magenta.models.coconet import lib_logging
from magenta.models.coconet import lib_mask
from magenta.models.coconet import lib_pianoroll
from magenta.models.coconet import lib_sampling
from magenta.models.coconet import lib_tfsampling
from magenta.models.coconet import lib_util
import numpy as np
import pretty_midi
import tensorflow.compat.v1 as tf  # noqa

FLAGS = tf.app.flags.FLAGS
flags = tf.app.flags
flags.DEFINE_integer("gen_batch_size", 3,
                     "Num of samples to generate in a batch.")
flags.DEFINE_string("strategy", None,
                    "Use complete_midi when using midi.")
flags.DEFINE_float("temperature", 0.99, "Softmax temperature")
flags.DEFINE_integer("piece_length", 32, "Num of time steps in generated piece")
flags.DEFINE_string("generation_output_dir", None,
                    "Output directory for storing the generated Midi.")
flags.DEFINE_string("prime_midi_melody_fpath", None,
                    "Path to midi melody to be harmonized.")
flags.DEFINE_string("checkpoint", None, "Path to checkpoint file")
flags.DEFINE_bool("midi_io", False, "Run in midi in and midi out mode."
                  "Does not write any midi or logs to disk.")
flags.DEFINE_bool("tfsample", True, "Run sampling in Tensorflow graph.")

def main(unused_argv):
    if FLAGS.checkpoint is None or not FLAGS.checkpoint:
        raise ValueError(
            "Need to provide a path to checkpoint directory.")

    if FLAGS.tfsample:
        generator = TFGenerator(FLAGS.checkpoint)
    else:
        wmodel = instantiate_model(FLAGS.checkpoint)
        generator = Generator(wmodel, FLAGS.strategy)
    midi_outs = generator.run_generation(
        gen_batch_size=FLAGS.gen_batch_size, piece_length=FLAGS.piece_length)

    # Creates a folder for storing the process of the sampling.
    label = "sample_%s_%s_%s_T%g_l%i_%.2fmin" % (lib_util.timestamp(),
                                                 FLAGS.strategy,
                                                 generator.hparams.architecture,
                                                 FLAGS.temperature,
                                                 FLAGS.piece_length,
                                                 generator.time_taken)
    basepath = os.path.join(FLAGS.generation_output_dir, label)
    tf.logging.info("basepath: %s", basepath)
    tf.gfile.MakeDirs(basepath)

    # Saves the results as midi or returns as midi out.
    midi_path = os.path.join(basepath, "midi")
    tf.gfile.MakeDirs(midi_path)
    tf.logging.info("Made directory %s", midi_path)
    save_midis(midi_outs, midi_path, label)

    result_npy_save_path = os.path.join(basepath, "generated_result.npy")
    tf.logging.info("Writing final result to %s", result_npy_save_path)
    with tf.gfile.Open(result_npy_save_path, "w") as p:
        np.save(p, generator.pianorolls)

    if FLAGS.tfsample:
        tf.logging.info("Done")
        return

    # Stores all the (intermediate) steps.
    intermediate_steps_path = os.path.join(basepath, "intermediate_steps.npz")
    with lib_util.timing("writing_out_sample_npz"):
        tf.logging.info("Writing intermediate steps to %s", intermediate_steps_path)
        generator.logger.dump(intermediate_steps_path)

    # Save the prime as midi and npy if in harmonization mode.
    # First, checks the stored npz for the first (context) and last step.
    tf.logging.info("Reading to check %s", intermediate_steps_path)
    with tf.gfile.Open(intermediate_steps_path, "r") as p:
        foo = np.load(p)
        for key in foo.keys():
            if re.match(r"0_root/.*?_strategy/.*?_context/0_pianorolls", key):
                context_rolls = foo[key]
                context_fpath = os.path.join(basepath, "context.npy")
                tf.logging.info("Writing context to %s", context_fpath)
                with lib_util.atomic_file(context_fpath) as context_p:
                    np.save(context_p, context_rolls)
                if "harm" in FLAGS.strategy:
                    # Only synthesize the one prime if in Midi-melody-prime mode.
                    primes = context_rolls
                    if "Melody" in FLAGS.strategy:
                        primes = [context_rolls[0]]
                    prime_midi_outs = get_midi_from_pianorolls(primes, generator.decoder)
                    save_midis(prime_midi_outs, midi_path, label + "_prime")
                break
    tf.logging.info("Done")

class Generator(object):
    """Instantiates model and generates according to strategy and midi input."""

    def __init__(self, wmodel, strategy_name="complete_midi"):
        """Initializes Generator with a wrapped model and strategy name.

        Args:
          wmodel: A lib_tfutil.WrappedModel loaded from a model checkpoint.
          strategy_name: A string specifying the key of the default strategy.
        """
        self.wmodel = wmodel
        self.hparams = self.wmodel.hparams
        self.decoder = lib_pianoroll.get_pianoroll_encoder_decoder(self.hparams)
        self.logger = lib_logging.Logger()
        # Instantiates generation strategy.
        self.strategy_name = strategy_name
        self.strategy = BaseStrategy.make(self.strategy_name, self.wmodel,
                                          self.logger, self.decoder)
        self._pianorolls = None
        self._time_taken = None

    def run_generation(self,
                       midi_in=None,
                       pianorolls_in=None,
                       gen_batch_size=3,
                       piece_length=16,
                       new_strategy=None):
        """Generates, conditions on midi_in if given, returns midi.

        Args:
          midi_in: An optional PrettyMIDI instance containing notes to be
              conditioned on.
          pianorolls_in: An optional numpy.ndarray encoding the notes to be
              conditioned on as pianorolls.
          gen_batch_size: An integer specifying the number of outputs to generate.
          piece_length: An integer specifying the desired number of time steps to
              generate for the output, where a time step corresponds to the
              shortest duration supported by the model.
          new_strategy: new_strategy: A string specifying the key of the strategy
              to use. If not set, the most recently set strategy is used. If a
              strategy was never specified, then the default strategy that was
              instantiated during initialization is used.

        Returns:
          A list of PrettyMIDI instances, with length gen_batch_size.
        """
        if new_strategy is not None:
            self.strategy_name = new_strategy
            self.strategy = BaseStrategy.make(self.strategy_name, self.wmodel,
                                              self.logger, self.decoder)
        # Update the length of piece to be generated.
        self.hparams.crop_piece_len = piece_length
        shape = [gen_batch_size] + self.hparams.pianoroll_shape
        tf.logging.info("Tentative shape of pianorolls to be generated: %r", shape)

        # Generates.
        start_time = time.time()

        if midi_in is not None and "midi" in self.strategy_name.lower():
            pianorolls = self.strategy((shape, midi_in))
        elif "complete_manual" == self.strategy_name.lower():
            pianorolls = self.strategy(pianorolls_in)
        else:
            pianorolls = self.strategy(shape)
        self._pianorolls = pianorolls
        self._time_taken = (time.time() - start_time) / 60.0

        # Logs final step
        self.logger.log(pianorolls=pianorolls)

        midi_outs = get_midi_from_pianorolls(pianorolls, self.decoder)
        return midi_outs

    @property
    def pianorolls(self):
        return self._pianorolls

    @property
    def time_taken(self):
        return self._time_taken

class TFGenerator(object):
    """Instantiates model and generates according to strategy and midi input."""

    def __init__(self, checkpoint_path):
        """Initializes Generator with a wrapped model and strategy name.

        Args:
          checkpoint_path: A string that gives the full path to the folder that
              holds the checkpoint.
        """
        self.sampler = lib_tfsampling.CoconetSampleGraph(checkpoint_path)
        self.hparams = self.sampler.hparams
        self.endecoder = lib_pianoroll.get_pianoroll_encoder_decoder(self.hparams)

        self._time_taken = None
        self._pianorolls = None

    def run_generation(self,
                       midi_in=None,
                       pianorolls_in=None,
                       masks=None,
                       gen_batch_size=3,
                       piece_length=16,
                       sample_steps=0,
                       current_step=0,
                       total_gibbs_steps=0,
                       temperature=0.99):
        """Generates, conditions on midi_in if given, returns midi.

        Args:
          midi_in: An optional PrettyMIDI instance containing notes to be
              conditioned on.
          pianorolls_in: An optional numpy.ndarray encoding the notes to be
              conditioned on as pianorolls.
          masks: a 4D numpy array of the same shape as pianorolls, with 1s
              indicating mask out.  If is None, then the masks will be where have 1s
              where there are no notes, indicating to the model they should be
              filled in.
          gen_batch_size: An integer specifying the number of outputs to generate.
          piece_length: An integer specifying the desired number of time steps to
              generate for the output, where a time step corresponds to the
              shortest duration supported by the model.
          See the CoconetSampleGraph class in lib_tfsample.py for more detail.
          sample_steps: an integer indicating the number of steps to sample in this
              call.  If set to 0, then it defaults to total_gibbs_steps.
          current_step: an integer indicating how many steps might have already
              sampled before.
          total_gibbs_steps: an integer indicating the total number of steps that
              a complete sampling procedure would take.
          temperature: a float indicating the temperature for sampling from softmax.

        Returns:
          A list of PrettyMIDI instances, with length gen_batch_size.
        """
        # Update the length of piece to be generated.
        self.hparams.crop_piece_len = piece_length
        shape = [gen_batch_size] + self.hparams.pianoroll_shape
        tf.logging.info("Tentative shape of pianorolls to be generated: %r", shape)

        # Generates.
        if midi_in is not None:
            pianorolls_in = self.endecoder.encode_midi_to_pianoroll(midi_in, shape)
        elif pianorolls_in is None:
            pianorolls_in = np.zeros(shape, dtype=np.float32)
        results = self.sampler.run(
            pianorolls_in, masks, sample_steps=sample_steps,
            current_step=current_step, total_gibbs_steps=total_gibbs_steps,
            temperature=temperature)
        self._pianorolls = results["pianorolls"]
        self._time_taken = results["time_taken"]
        tf.logging.info("output pianorolls shape: %r", self.pianorolls.shape)
        midi_outs = get_midi_from_pianorolls(self.pianorolls, self.endecoder)
        return midi_outs

    @property
    def pianorolls(self):
        return self._pianorolls

    @property
    def time_taken(self):
        return self._time_taken

def get_midi_from_pianorolls(rolls, decoder):
    midi_datas = []
    for pianoroll in rolls:
        tf.logging.info("pianoroll shape: %r", pianoroll.shape)
        midi_data = decoder.decode_to_midi(pianoroll)
        midi_datas.append(midi_data)
    return midi_datas

def save_midis(midi_datas, midi_path, label=""):
    for i, midi_data in enumerate(midi_datas):
        midi_fpath = os.path.join(midi_path, "%s_%i.midi" % (label, i))
        tf.logging.info("Writing midi to %s", midi_fpath)
        with lib_util.atomic_file(midi_fpath) as p:
            midi_data.write(p)
    return midi_fpath

def instantiate_model(checkpoint, instantiate_sess=True):
    wmodel = lib_graph.load_checkpoint(
        checkpoint, instantiate_sess=instantiate_sess)
    return wmodel

###################
# ## Strategies ###
###################
# Commonly used compositions of samplers, user-selectable through FLAGS.strategy

class BaseStrategy(lib_util.Factory):
    """Base class for setting up generation strategies."""

    def __init__(self, wmodel, logger, decoder):
        self.wmodel = wmodel
        self.logger = logger
        self.decoder = decoder

    def __call__(self, shape):
        label = "%s_strategy" % self.key
        with lib_util.timing(label):
            with self.logger.section(label):
                return self.run(shape)

    def blank_slate(self, shape):
        return (np.zeros(shape, dtype=np.float32), np.ones(shape, dtype=np.float32))

    # convenience function to avoid passing the same arguments over and over
    def make_sampler(self, key, **kwargs):
        kwargs.update(wmodel=self.wmodel, logger=self.logger)
        return lib_sampling.BaseSampler.make(key, **kwargs)

# pylint:disable=missing-docstring
class HarmonizeMidiMelodyStrategy(BaseStrategy):
    """Harmonizes a midi melody (fname given by FLAGS.prime_midi_melody_fpath)."""
    key = "harmonize_midi_melody"

    def load_midi_melody(self, midi=None):
        if midi is None:
            midi = pretty_midi.PrettyMIDI(FLAGS.prime_midi_melody_fpath)
        return self.decoder.encode_midi_melody_to_pianoroll(midi)

    def make_pianoroll_from_melody_roll(self, mroll, requested_shape):
        # mroll shape: time, pitch
        # requested_shape: batch, time, pitch, instrument
        bb, tt, pp, ii = requested_shape
        tf.logging.info("requested_shape: %r", requested_shape)
        assert mroll.ndim == 2
        assert mroll.shape[1] == 128
        hparams = self.wmodel.hparams
        assert pp == hparams.num_pitches, "%r != %r" % (pp, hparams.num_pitches)
        if tt != mroll.shape[0]:
            tf.logging.info("WARNING: requested tt %r != prime tt %r" % (
                tt, mroll.shape[0]))
        rolls = np.zeros((bb, mroll.shape[0], pp, ii), dtype=np.float32)
        rolls[:, :, :, 0] = mroll[None, :, hparams.min_pitch:hparams.max_pitch + 1]
        tf.logging.info("resulting shape: %r", rolls.shape)
        return rolls

    def run(self, tuple_in):
        shape, midi_in = tuple_in
        mroll = self.load_midi_melody(midi_in)
        pianorolls = self.make_pianoroll_from_melody_roll(mroll, shape)
        masks = lib_sampling.HarmonizationMasker()(pianorolls.shape)
        gibbs = self.make_sampler(
            "gibbs",
            masker=lib_sampling.BernoulliMasker(),
            sampler=self.make_sampler("independent", temperature=FLAGS.temperature),
            schedule=lib_sampling.YaoSchedule())

        with self.logger.section("context"):
            context = np.array([
                lib_mask.apply_mask(pianoroll, mask)
                for pianoroll, mask in zip(pianorolls, masks)
            ])
            self.logger.log(pianorolls=context, masks=masks, predictions=context)
        pianorolls = gibbs(pianorolls, masks)

        return pianorolls

class ScratchUpsamplingStrategy(BaseStrategy):
    key = "scratch_upsampling"

    def run(self, shape):
        # start with an empty pianoroll of length 1, then repeatedly upsample
        initial_shape = list(shape)
        desired_length = shape[1]
        initial_shape[1] = 1
        initial_shape = tuple(shape)

        pianorolls, masks = self.blank_slate(initial_shape)

        sampler = self.make_sampler(
            "upsampling",
            desired_length=desired_length,
            sampler=self.make_sampler(
                "gibbs",
                masker=lib_sampling.BernoulliMasker(),
                sampler=self.make_sampler(
                    "independent", temperature=FLAGS.temperature),
                schedule=lib_sampling.YaoSchedule()))

        return sampler(pianorolls, masks)

class BachUpsamplingStrategy(BaseStrategy):
    key = "bach_upsampling"

    def run(self, shape):
        # optionally start with bach samples
        init_sampler = self.make_sampler("bach", temperature=FLAGS.temperature)
        pianorolls, masks = self.blank_slate(shape)
        pianorolls = init_sampler(pianorolls, masks)
        desired_length = 4 * shape[1]
        sampler = self.make_sampler(
            "upsampling",
            desired_length=desired_length,
            sampler=self.make_sampler(
                "gibbs",
                masker=lib_sampling.BernoulliMasker(),
                sampler=self.make_sampler(
                    "independent", temperature=FLAGS.temperature),
                schedule=lib_sampling.YaoSchedule()))
        return sampler(pianorolls, masks)

class RevoiceStrategy(BaseStrategy):
    key = "revoice"

    def run(self, shape):
        init_sampler = self.make_sampler("bach", temperature=FLAGS.temperature)
        pianorolls, masks = self.blank_slate(shape)
        pianorolls = init_sampler(pianorolls, masks)

        sampler = self.make_sampler(
            "gibbs",
            masker=lib_sampling.BernoulliMasker(),
            sampler=self.make_sampler("independent", temperature=FLAGS.temperature),
            schedule=lib_sampling.YaoSchedule())

        for i in range(shape[-1]):
            masks = lib_sampling.InstrumentMasker(instrument=i)(shape)
            with self.logger.section("context"):
                context = np.array([
                    lib_mask.apply_mask(pianoroll, mask)
                    for pianoroll, mask in zip(pianorolls, masks)
                ])
                self.logger.log(pianorolls=context, masks=masks, predictions=context)
            pianorolls = sampler(pianorolls, masks)

        return pianorolls

class HarmonizationStrategy(BaseStrategy):
    key = "harmonization"

    def run(self, shape):
        init_sampler = self.make_sampler("bach", temperature=FLAGS.temperature)
        pianorolls, masks = self.blank_slate(shape)
        pianorolls = init_sampler(pianorolls, masks)

        masks = lib_sampling.HarmonizationMasker()(shape)

        gibbs = self.make_sampler(
            "gibbs",
            masker=lib_sampling.BernoulliMasker(),
            sampler=self.make_sampler("independent", temperature=FLAGS.temperature),
            schedule=lib_sampling.YaoSchedule())

        with self.logger.section("context"):
            context = np.array([
                lib_mask.apply_mask(pianoroll, mask)
                for pianoroll, mask in zip(pianorolls, masks)
            ])
            self.logger.log(pianorolls=context, masks=masks, predictions=context)
        pianorolls = gibbs(pianorolls, masks)
        with self.logger.section("result"):
            self.logger.log(
                pianorolls=pianorolls, masks=masks, predictions=pianorolls)

        return pianorolls

class TransitionStrategy(BaseStrategy):
    key = "transition"

    def run(self, shape):
        init_sampler = lib_sampling.BachSampler(
            wmodel=self.wmodel, temperature=FLAGS.temperature)
        pianorolls, masks = self.blank_slate(shape)
        pianorolls = init_sampler(pianorolls, masks)

        masks = lib_sampling.TransitionMasker()(shape)
        gibbs = self.make_sampler(
            "gibbs",
            masker=lib_sampling.BernoulliMasker(),
            sampler=self.make_sampler("independent", temperature=FLAGS.temperature),
            schedule=lib_sampling.YaoSchedule())

        with self.logger.section("context"):
            context = np.array([
                lib_mask.apply_mask(pianoroll, mask)
                for pianoroll, mask in zip(pianorolls, masks)
            ])
            self.logger.log(pianorolls=context, masks=masks, predictions=context)
        pianorolls = gibbs(pianorolls, masks)
        return pianorolls

class ChronologicalStrategy(BaseStrategy):
    key = "chronological"

    def run(self, shape):
        sampler = self.make_sampler(
            "ancestral",
            temperature=FLAGS.temperature,
            selector=lib_sampling.ChronologicalSelector())
        pianorolls, masks = self.blank_slate(shape)
        pianorolls = sampler(pianorolls, masks)
        return pianorolls

class OrderlessStrategy(BaseStrategy):
    key = "orderless"

    def run(self, shape):
        sampler = self.make_sampler(
            "ancestral",
            temperature=FLAGS.temperature,
            selector=lib_sampling.OrderlessSelector())
        pianorolls, masks = self.blank_slate(shape)
        pianorolls = sampler(pianorolls, masks)
        return pianorolls

class IgibbsStrategy(BaseStrategy):
    key = "igibbs"

    def run(self, shape):
        pianorolls, masks = self.blank_slate(shape)
        sampler = self.make_sampler(
            "gibbs",
            masker=lib_sampling.BernoulliMasker(),
            sampler=self.make_sampler("independent", temperature=FLAGS.temperature),
            schedule=lib_sampling.YaoSchedule())
        pianorolls = sampler(pianorolls, masks)
        return pianorolls

class AgibbsStrategy(BaseStrategy):
    key = "agibbs"

    def run(self, shape):
        pianorolls, masks = self.blank_slate(shape)
        sampler = self.make_sampler(
            "gibbs",
            masker=lib_sampling.BernoulliMasker(),
            sampler=self.make_sampler(
                "ancestral",
                selector=lib_sampling.OrderlessSelector(),
                temperature=FLAGS.temperature),
            schedule=lib_sampling.YaoSchedule())
        pianorolls = sampler(pianorolls, masks)
        return pianorolls

class CompleteManualStrategy(BaseStrategy):
    key = "complete_manual"

    def run(self, pianorolls):
        # fill in the silences
        masks = lib_sampling.CompletionMasker()(pianorolls)
        gibbs = self.make_sampler(
            "gibbs",
            masker=lib_sampling.BernoulliMasker(),
            sampler=self.make_sampler("independent", temperature=FLAGS.temperature),
            schedule=lib_sampling.YaoSchedule())

        with self.logger.section("context"):
            context = np.array([
                lib_mask.apply_mask(pianoroll, mask)
                for pianoroll, mask in zip(pianorolls, masks)
            ])
            self.logger.log(pianorolls=context, masks=masks, predictions=context)
        pianorolls = gibbs(pianorolls, masks)
        with self.logger.section("result"):
            self.logger.log(
                pianorolls=pianorolls, masks=masks, predictions=pianorolls)
        return pianorolls

class CompleteMidiStrategy(BaseStrategy):
    key = "complete_midi"

    def run(self, tuple_in):
        shape, midi_in = tuple_in
        pianorolls = self.decoder.encode_midi_to_pianoroll(midi_in, shape)
        # fill in the silences
        masks = lib_sampling.CompletionMasker()(pianorolls)
        gibbs = self.make_sampler(
            "gibbs",
            masker=lib_sampling.BernoulliMasker(),
            sampler=self.make_sampler("independent", temperature=FLAGS.temperature),
            schedule=lib_sampling.YaoSchedule())

        with self.logger.section("context"):
            context = np.array([
                lib_mask.apply_mask(pianoroll, mask)
                for pianoroll, mask in zip(pianorolls, masks)
            ])
            self.logger.log(pianorolls=context, masks=masks, predictions=context)
        pianorolls = gibbs(pianorolls, masks)
        with self.logger.section("result"):
            self.logger.log(
                pianorolls=pianorolls, masks=masks, predictions=pianorolls)
        return pianorolls

# pylint:enable=missing-docstring

# ok something else entirely.
def parse_art_to_pianoroll(art, tt=None):
    """Parse ascii art for pianoroll."""
    assert tt is not None
    ii = 4
    # TODO(annahuang): Properties of the model/data_tools, not of the ascii art.
    pmin, pmax = 36, 81
    pp = pmax - pmin + 1

    pianoroll = np.zeros((tt, pp, ii), dtype=np.float32)

    lines = art.strip().splitlines()
    klasses = "cCdDefFgGaAb"
    klass = None
    octave = None
    cycle = None
    for li, line in enumerate(lines):
        match = re.match(r"^\s*(?P<class>[a-gA-G])?(?P<octave>[0-9]|10)?\s*\|"
                         r"(?P<grid>[SATB +-]*)\|\s*$", line)
        if not match:
            if cycle is not None:
                print("ignoring unmatched line", li, repr(line))
            continue

        if cycle is None:
            # set up cycle through pitches and octaves
            print(match.groupdict())
            assert match.group("class") and match.group("class") in klasses
            assert match.group("octave")
            klass = match.group("class")
            octave = int(match.group("octave"))
            cycle = reversed(list(it.product(list(range(octave + 1)), klasses)))
            cycle = list(cycle)
            print(cycle)
            cycle = it.dropwhile(lambda ok: ok[1] != match.group("class"), cycle)  # pylint: disable=cell-var-from-loop
            o, k = next(cycle)
            assert k == klass
            assert o == octave
            cycle = list(cycle)
            print(cycle)
            cycle = iter(cycle)
        else:
            octave, klass = next(cycle)
            if match.group("class"):
                assert klass == match.group("class")
            if match.group("octave"):
                assert octave == int(match.group("octave"))

        pitch = octave * len(klasses) + klasses.index(klass)
        print(klass, octave, pitch, "\t", line)

        p = pitch - pmin
        for t, c in enumerate(match.group("grid")):
            if c in "+- ":
                continue
            i = "SATB".index(c)
            pianoroll[t, p, i] = 1.

    return pianoroll

if __name__ == "__main__":
    tf.app.run()

max_stars_repo_name: hologerry/magenta

max_issues_repo_name: hologerry/magenta

max_forks_repo_name: hologerry/magenta


text:
"""Adapted from:
    @longcw faster_rcnn_pytorch: https://github.com/longcw/faster_rcnn_pytorch
    @rbgirshick py-faster-rcnn https://github.com/rbgirshick/py-faster-rcnn
    Licensed under The MIT License [see LICENSE for details]
"""

from __future__ import print_function
import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
from torch.autograd import Variable

import sys
sys.path.append(".")

from data import *
import torch.utils.data as data

import sys
import os
import time
import numpy as np
import pickle
import cv2
from log import log
from evaluation.mean_average_precision import MetricBuilder

if sys.version_info[0] == 2:
    import xml.etree.cElementTree as ET
else:
    import xml.etree.ElementTree as ET

args_cuda = True

def str2bool(v):
    return v.lower() in ("yes", "true", "t", "1")

if torch.cuda.is_available():
    if args_cuda:
        torch.set_default_tensor_type('torch.cuda.FloatTensor')
    if not args_cuda:
        print("WARNING: It looks like you have a CUDA device, but aren't using \
              CUDA.  Run with --cuda for optimal eval speed.")
        torch.set_default_tensor_type('torch.FloatTensor')
else:
    torch.set_default_tensor_type('torch.FloatTensor')

class Timer(object):
    """A simple timer."""
    def __init__(self):
        self.total_time = 0.
        self.calls = 0
        self.start_time = 0.
        self.diff = 0.
        self.average_time = 0.

    def tic(self):
        # using time.time instead of time.clock because time time.clock
        # does not normalize for multithreading
        self.start_time = time.time()

    def toc(self, average=True):
        self.diff = time.time() - self.start_time
        self.total_time += self.diff
        self.calls += 1
        self.average_time = self.total_time / self.calls
        if average:
            return self.average_time
        else:
            return self.diff

def parse_rec(filename):
    """ Parse a PASCAL VOC xml file """
    tree = ET.parse(filename)
    objects = []
    for obj in tree.findall('object'):
        obj_struct = {}
        obj_struct['name'] = obj.find('name').text
        obj_struct['pose'] = obj.find('pose').text
        obj_struct['truncated'] = int(obj.find('truncated').text)
        obj_struct['difficult'] = 0
        bbox = obj.find('bndbox')
        obj_struct['bbox'] = [int(bbox.find('xmin').text) - 1,
                              int(bbox.find('ymin').text) - 1,
                              int(bbox.find('xmax').text) - 1,
                              int(bbox.find('ymax').text) - 1]
        objects.append(obj_struct)

    return objects

def get_output_dir(name, phase):
    """Return the directory where experimental artifacts are placed.
    If the directory does not exist, it is created.
    A canonical path is built using the name from an imdb and a network
    (if not None).
    """
    filedir = os.path.join(name, phase)
    if not os.path.exists(filedir):
        os.makedirs(filedir)
    return filedir

def get_voc_results_file_template(image_set, cls):
    # VOCdevkit/VOC2007/results/det_test_aeroplane.txt
    filename = 'det_' + image_set + '_%s.txt' % (cls)
    filedir = os.path.join(devkit_path, 'results')
    if not os.path.exists(filedir):
        os.makedirs(filedir)
    path = os.path.join(filedir, filename)
    return path

def write_voc_results_file(all_boxes, dataset):
    for cls_ind, cls in enumerate(labelmap):
        print('Writing {:s} VOC results file'.format(cls))
        filename = get_voc_results_file_template(set_type, cls)
        with open(filename, 'wt') as f:
            for im_ind, index in enumerate(dataset.ids):
                dets = all_boxes[cls_ind+1][im_ind]
                if dets == []:
                    continue
                # the VOCdevkit expects 1-based indices
                for k in range(dets.shape[0]):
                    f.write('{:s} {:.3f} {:.1f} {:.1f} {:.1f} {:.1f}\n'.
                            format(index[1], dets[k, -1],
                                   dets[k, 0] + 1, dets[k, 1] + 1,
                                   dets[k, 2] + 1, dets[k, 3] + 1))

def do_python_eval(output_dir='output', use_12=False):
    cachedir = os.path.join(devkit_path, 'annotations_cache')
    aps = []
    # The PASCAL VOC metric changed in 2010
    use_12_metric = use_12
    print('VOC12 metric? ' + ('Yes' if use_12_metric else 'No'))
    if not os.path.isdir(output_dir):
        os.mkdir(output_dir)
    for i, cls in enumerate(labelmap):
        filename = get_voc_results_file_template(set_type, cls)
        metrics, map = voc_eval(
           filename, annopath, imgsetpath.format(set_type), cls, cachedir,
           ovthresh=0.75, use_12_metric=use_12_metric)
        aps += [map]
        print('AP for {} = {:.4f}'.format(cls, map))
        # with open(os.path.join(output_dir, cls + '_pr.pkl'), 'wb') as f:
        #     pickle.dump({'rec': rec, 'prec': prec, 'ap': ap}, f)
        log.l.info('AP for {} = {:.4f}'.format(cls, map))
    log.l.info('Mean AP = {:.4f}'.format(np.mean(aps)))
    print('Mean AP = {:.4f}'.format(np.mean(aps)))
    print('~~~~~~~~')
    print('Results:')
    for ap in aps:
        print('{:.4f}'.format(ap))
    print('{:.4f}'.format(np.mean(aps)))
    print('~~~~~~~~')
    print('')
    print('--------------------------------------------------------------')
    print('Results computed with the **unofficial** Python eval code.')
    print('refer to https://github.com/bes-dev/mean_average_precision')
    print('--------------------------------------------------------------')

def voc_eval(detpath,
             annopath,
             imagesetfile,
             classname,
             cachedir,
             ovthresh=0.75,
             use_12_metric=False):
    """rec, prec, ap = voc_eval(detpath,
                           annopath,
                           imagesetfile,
                           classname,
                           [use_12_metric])
Top level function that does the PASCAL VOC evaluation.
detpath: Path to detections
   detpath.format(classname) should produce the detection results file.
annopath: Path to annotations
   annopath.format(imagename) should be the xml annotations file.
imagesetfile: Text file containing the list of images, one image per line.
classname: Category name (duh)
cachedir: Directory for caching the annotations
[ovthresh]: Overlap threshold (default = 0.75)
[use_12_metric]: Whether to use VOC12's all points AP computation
   (default True)
"""
# assumes detections are in detpath.format(classname)
# assumes annotations are in annopath.format(imagename)
# assumes imagesetfile is a text file with each line an image name
# cachedir caches the annotations in a pickle file
# first load gt
    if not os.path.isdir(cachedir):
        os.mkdir(cachedir)
    cachefile = os.path.join(cachedir, 'annots.pkl')
    # read list of images
    with open(imagesetfile, 'r') as f:
        lines = f.readlines()
    imagenames = [x.strip() for x in lines]
    if not os.path.isfile(cachefile):
        # load annots
        recs = {}
        for i, imagename in enumerate(imagenames):
            recs[imagename] = parse_rec(annopath % (imagename))
            if i % 100 == 0:
                print('Reading annotation for {:d}/{:d}'.format(
                   i + 1, len(imagenames)))
        # save
        print('Saving cached annotations to {:s}'.format(cachefile))
        with open(cachefile, 'wb') as f:
            pickle.dump(recs, f)
    else:
        # load
        with open(cachefile, 'rb') as f:
            recs = pickle.load(f)

    # create metric_fn
    metric_fn = MetricBuilder.build_evaluation_metric("map_2d", async_mode=True, num_classes=1)

    # read dets
    detfile = detpath.format(classname)
    with open(detfile, 'r') as f:
        lines = f.readlines()
    if any(lines) == 1:
        # exchange dets as key-value dic
        preds_dic = {}
        splitlines = [x.strip().split(' ') for x in lines]
        for x in splitlines:
            if x[0] in preds_dic.keys():
                preds_dic[x[0]] = np.vstack((preds_dic[x[0]], np.append(np.array([float(z) for z in x[2:]]), [int(0), float(x[1])])))
            else:
                preds_dic[x[0]] = np.append(np.array([float(z) for z in x[2:]]), [int(0), float(x[1])])
                preds_dic[x[0]] = np.expand_dims(preds_dic[x[0]], axis=0)

        # extract gt objects for this class
        for imagename in imagenames:
            R = [obj for obj in recs[imagename] if obj['name'] == classname]
            GT_bbox = np.array([x['bbox'] for x in R])
            GT_append = np.zeros((GT_bbox.shape[0], 3), dtype=np.int)
            GTs = np.hstack((GT_bbox, GT_append))
            if imagename not in preds_dic.keys():
                preds = np.array([[]])
            else:
                preds = preds_dic[imagename]
            metric_fn.add(preds, GTs)

        metrics = metric_fn.value(iou_thresholds=np.arange(0.5, 1.0, 0.05), recall_thresholds=np.arange(0., 1.01, 0.01), mpolicy='soft')
        map = metrics["mAP"]
        # metrics = metric_fn.value(iou_thresholds=ovthresh)
        # ap = metrics[ovthresh][0]['ap']
        # rec = metrics[ovthresh][0]['recall']
        # prec = metrics[ovthresh][0]['precision']
    else:
        metrics = {}
        map = -1.

    return metrics, map

def test_net(save_folder, obj_type, voc_root, set_type_, labelmap_, net, cuda, dataset, transform, top_k,
             im_size=300, thresh=0.05):

    if not os.path.exists(save_folder):
        os.mkdir(save_folder)

    global annopath
    global imgpath
    global imgsetpath
    global devkit_path
    global set_type
    global labelmap
    annopath = os.path.join(voc_root, 'Annotations', '%s.xml')
    imgpath = os.path.join(voc_root, 'JPEGImages', '%s.jpg')
    imgsetpath = os.path.join(voc_root, 'ImageSets', 'Main', '{:s}.txt')
    devkit_path = voc_root
    set_type = set_type_
    labelmap = labelmap_

    num_images = len(dataset)
    # all detections are collected into:
    #    all_boxes[cls][image] = N x 5 array of detections in
    #    (x1, y1, x2, y2, score)
    all_boxes = [[[] for _ in range(num_images)]
                 for _ in range(len(labelmap)+1)]

    # timers
    _t = {'im_detect': Timer(), 'misc': Timer()}
    output_dir = get_output_dir(save_folder + '/ssd' + str(im_size) + '_' + obj_type, set_type)
    det_file = os.path.join(output_dir, 'detections.pkl')

    total_time = 0
    for i in range(num_images):
        im, gt, h, w = dataset.pull_item(i)
        x = Variable(im.unsqueeze(0))
        if args_cuda:
            x = x.cuda()
        _t['im_detect'].tic()
        if obj_type in ['two_stage_end2end', 'two_stage_end2end_with_border', 'TITS_Neuro']:
            detections = net(x, []).data
        else:
            detections = net(x).data
        detect_time = _t['im_detect'].toc(average=False)

        # skip j = 0, because it's the background class
        for j in range(1, detections.size(1)):
            dets = detections[0, j, :]
            if obj_type in ['VOC', 'COCO', 'car', 'carplate', 'car_carplate', 'two_branch']:
                mask = dets[:, 0].gt(0.).expand(5, dets.size(0)).t()
                dets = torch.masked_select(dets, mask).view(-1, 5)
            elif obj_type == 'car_carplate_offset':
                mask = dets[:, 0].gt(0.).expand(10, dets.size(0)).t()
                dets = torch.masked_select(dets, mask).view(-1, 10)
            elif obj_type in ['carplate_four_corners','carplate_four_corners_with_border',
                            'carplate_only_four_corners', 'carplate_only_four_corners_with_border',
                            'two_stage_end2end', 'two_stage_end2end_with_border',
                            'TITS_Neuro']:
                mask = dets[:, 0].gt(0.).expand(13, dets.size(0)).t()
                dets = torch.masked_select(dets, mask).view(-1, 13)
            else:
                assert False, "wrong object type"
            if dets.size(0) == 0:
                continue
            boxes = dets[:, 1:5]
            boxes[:, 0] *= w
            boxes[:, 2] *= w
            boxes[:, 1] *= h
            boxes[:, 3] *= h
            scores = dets[:, 0].cpu().numpy()
            cls_dets = np.hstack((boxes.cpu().numpy(),
                                  scores[:, np.newaxis])).astype(np.float32,
                                                                 copy=False)
            all_boxes[j][i] = cls_dets

        print('im_detect: {:d}/{:d} {:.3f}s'.format(i + 1,
                                                    num_images, detect_time))
        if i > 0:
            total_time += detect_time
    print("average time: " + str(total_time / (num_images-1) * 1000) + ' ms')

    # with open(det_file, 'wb') as f:
    #     pickle.dump(all_boxes, f, pickle.HIGHEST_PROTOCOL)

    print('Evaluating detections')
    evaluate_detections(all_boxes, output_dir, dataset)

def evaluate_detections(box_list, output_dir, dataset):
    write_voc_results_file(box_list, dataset)
    do_python_eval(output_dir)


text:
#! /bin/python

import os
import sys
import json

import luigi
import numpy as np
import nifty.tools as nt

import cluster_tools.utils.volume_utils as vu
import cluster_tools.utils.function_utils as fu
from cluster_tools.cluster_tasks import SlurmTask, LocalTask, LSFTask

#
# Node Label Tasks
#

class BlockStatisticsBase(luigi.Task):
    """ BlockStatistics base class
    """

    task_name = 'block_statistics'
    src_file = os.path.abspath(__file__)

    path = luigi.Parameter()
    key = luigi.Parameter()

    def clean_up_for_retry(self, block_list):
        super().clean_up_for_retry(block_list)
        # TODO remove any output of failed blocks because it might be corrupted

    def run_impl(self):
        # get the global config and init configs
        shebang, block_shape, roi_begin, roi_end = self.global_config_values()
        self.init(shebang)

        # load the task config
        config = self.get_task_config()

        # update the config with input and graph paths and keys
        # as well as block shape
        config.update({'path': self.path,
                       'key': self.key,
                       'tmp_folder': self.tmp_folder,
                       'block_shape': block_shape})

        with vu.file_reader(self.path, 'r') as f:
            shape = f[self.key].shape

        if self.n_retries == 0:
            block_list = vu.blocks_in_volume(shape, block_shape,
                                             roi_begin, roi_end)
        else:
            block_list = self.block_list
            self.clean_up_for_retry(block_list)
        n_jobs = min(len(block_list), self.max_jobs)

        # prime and run the jobs
        self.prepare_jobs(n_jobs, block_list, config)
        self.submit_jobs(n_jobs)

        # wait till jobs finish and check for job success
        self.wait_for_jobs()
        self.check_jobs(n_jobs)

class BlockStatisticsLocal(BlockStatisticsBase, LocalTask):
    """ BlockStatistics on local machine
    """
    pass

class BlockStatisticsSlurm(BlockStatisticsBase, SlurmTask):
    """ BlockStatistics on slurm cluster
    """
    pass

class BlockStatisticsLSF(BlockStatisticsBase, LSFTask):
    """ BlockStatistics on lsf cluster
    """
    pass

#
# Implementation
#

def merge_stats(stat_list):
    sizes = np.array([stat['size'] for stat in stat_list])
    size = np.sum(sizes)

    means = np.array([stat['mean'] for stat in stat_list])
    mean = np.sum(means * sizes) / size

    variances = np.array([stat['variance'] for stat in stat_list])
    var = (sizes * (variances + (means - mean) ** 2)).sum() / size

    max_ = np.max([stat['max'] for stat in stat_list])
    min_ = np.min([stat['min'] for stat in stat_list])

    stats = {
        'mean': mean,
        'max': max_,
        'min': min_,
        'std': np.sqrt(var),
        'variance': var,
        'size': size
    }
    return stats

def _compute_block_stats(block_id, blocking, ds):
    block = blocking.getBlock(block_id)
    bb = vu.block_to_bb(block)

    data = ds[bb]
    var = float(np.var(data))
    stats = {
        'mean': float(np.mean(data)),
        'max': float(np.max(data)),
        'min': float(np.min(data)),
        'std': float(np.sqrt(var)),
        'variance': var,
        'size': float(data.size)
    }
    return stats

def block_statistics(job_id, config_path):

    fu.log("start processing job %i" % job_id)
    fu.log("reading config from %s" % config_path)

    # get the config
    with open(config_path) as f:
        config = json.load(f)

    path = config['path']
    key = config['key']
    tmp_folder = config['tmp_folder']

    block_shape = config['block_shape']
    block_list = config['block_list']

    with vu.file_reader(path, 'r') as f:
        shape = f[key].shape

    blocking = nt.blocking([0, 0, 0],
                           list(shape),
                           list(block_shape))

    with vu.file_reader(path, 'r') as f_in:
        ds = f_in[key]
        block_stats = [_compute_block_stats(block_id, blocking, ds)
                       for block_id in block_list]

    save_path = os.path.join(tmp_folder, 'block_statistics_job%i.json' % job_id)
    job_stats = merge_stats(block_stats)
    with open(save_path, 'w') as f:
        json.dump(job_stats, f)

    fu.log_job_success(job_id)

if __name__ == '__main__':
    path = sys.argv[1]
    assert os.path.exists(path), path
    job_id = int(os.path.split(path)[1].split('.')[0].split('_')[-1])
    block_statistics(job_id, path)


text:
import re
import numpy as nm
import os.path as osp

try:
  import pyfits as pf
except ImportError as e:
  # try astropy then
  from astropy.io import fits as pf

def scarray(li,scal=False):
  if len(li)==1 and scal:
    return li[0]
  else:
    if type(li[0])!=str:
      return nm.array(li)
    return li

def read_array(fname,dirname):
  fname = lookupfile(fname,dirname)

  try:
    pfits = pf.open(fname)
    ii=0
    while pfits[ii].data is None:
      ii+=1
    return pfits[ii].data
  except Exception:
    return nm.loadtxt(fname)

class transformme:
  def __init__(self,tfunc,pf,isar=False):
    self.tfunc = tfunc
    self.pf = pf
    self.df = None
    self.isar = isar
    self.scal=False

  def __getattr__(self,val):
    val = val.replace("_DOT_",".")
    val = val.replace("_dot_",".")
    if self.isar and val+".file" in self.pf.pf:
      vl = read_array(self.pf.pf[val+".file"],self.pf.localdir).flat[:]
      self.pf._access_list += [val+".file"]
    else:
      try:
        vl = self.pf.pf[val]
        self.pf._access_list += [val]
      except Exception as e:
        if self.df is None:
          raise e
        else:
          vl = self.df
          self.pf._access_list += [val+".file"]

    if self.isar:
      if isinstance(vl,str):
        vvl = vl.split()
      else:
        vvl = vl
      return scarray([self.tfunc(v) for v in vvl],self.scal)
    return self.tfunc(vl)

  def __call__(self,**kk):
    if "default" in kk:
      self.df = kk["default"]
    if kk.get("scalarize",False):
      self.scal = True
    return self

def getnextline(txtit):
  l = next(txtit)
  l = re.split("#|!",l)[0]
  if len(l.strip())==0:
    return getnextline(txtit)
  return l

def cleantxtit(txtit):
  while(1) :
    yield getnextline(txtit)

def parse_text(txtit):
  ctxtit = cleantxtit(txtit)
  cont = ""
  pf = {}
  for l in ctxtit:
    k = cont
    if not cont:
      k,v = re.findall("(?<!#)((?:\w|\.)+)\s*=\s*(.+)",l)[0]
      v = v.strip()
    else:
      v = l.strip()
      v = pf[k]+ " "+ v
    cont = ""
    if v[-1]=="&": #continuation
        v = v[:-1]
        cont = k
    pf[k] = v
  return pf

def lookupfile(fi,dirs):
  dirs = list(dirs) + ["."]
  if osp.exists(fi):
    return osp.realpath(fi)
  for d in dirs:
    if osp.exists(osp.join(dd,fi)):
      return osp.realpath(osp.join(dd,fi))
  raise IOError("cannot find %s in any of the following directories %s"%(fi,dirs))

class miniparse(object):
  def _parse(self,txtit):
    cont = ""
    for l in txtit:

      # include
      mtch = re.search("\s*#\s*include\s*[\"|\'](.+?)[\"|\']",l)
      if mtch:
        rfn = lookupfile(mtch.group(1),self.localdir)
        self.localdir += [osp.dirname(rfn)]
        self._parse(open(rfn))

      #undef
      mtch = re.search("\s*#\s*undef\s*(.+)",l)
      if mtch:
        name = mtch.group(1).strip()
        if name in self.pf:
          del(self.pf[name])

      # deal with comments
      l = re.split("#|!",l)[0]
      if len(l.strip())==0:
        continue

      k = cont
      if not cont:
        k,v = re.findall("(?<!#)((?:\w|\.)+)\s*=\s*(.+)",l)[0]
        v = v.strip()
      else:
        v = l.strip()
        v = self.pf[k]+ " "+ v

      cont = ""
      if v[-1]=="&": #continuation
        v = v[:-1]
        cont = k

      self.pf[k] = v

  def __init__(self, pfn,**kk):
    self.pf = {}

    self.localdir = [osp.dirname(osp.abspath(pfn or "."))]

    if pfn!=None:
      print("read parameter file %s"%pfn)
      self._parse(open(pfn))
      #pff =open(pfn)
      #txt = "\n".join([to.split("#")[0] for to in pff])+"\n"
      #pf = dict(re.findall("(?<!#)((?:\w|\.)+)\s*=\s*(.+?)\n",txt))
      #pf = parse_text(pff)
      #self.pf.update(pf)

    self.pf.update(kk)
    self._access_list = []

  def keys(self,prefix=""):
    return [k for k in list(self.pf.keys()) if k[:len(prefix)]==prefix]

  def __repr__(self):
    rr = []
    print(self._access_list)
    for v in self._access_list:
      print(v) #ICICICICI
      rr += ["%s = %s"%(v,getattr(self,v))]
    return "\n".join(rr)

  def __contains__(self,val):
    res = val in self.pf or val+".file" in self.pf
    if res:
      self._access_list += [val]
    return res

  @property
  def bool(self):
    return transformme(lambda val:str(val).lower() in ("t","1","true"),self)

  @property
  def bool_array(self):
    return transformme(lambda val:str(val).lower() in ("t","1","true"),self,True)

  @property
  def int(self):
    return transformme(int,self)

  @property
  def int_array(self):
    return transformme(int,self,True)

  @property
  def float(self):
    return transformme(float,self)

  @property
  def float_array(self):
    return transformme(float,self,True)

  @property
  def str(self):
    return transformme(str,self)

  @property
  def str_array(self):
    return transformme(str,self,True)

  def __getattr__(self,val):
    res = getattr(self.str,val)
    return res

def fromargv():
  import sys
  argv = sys.argv

  if len(argv)!=2:
    print("usage: %s parfile\n"%(argv[0]))
    sys.exit(-1)

  pfn = argv[1]
  pf = miniparse(pfn)
  return pf


text:
# -*- coding: utf-8 -*-

"""
@date: 2020/9/18 上午9:15
@file: tests.py
@author: zj
@description:
"""

import os
import numpy as np
import torch

from zcls.config import cfg
from zcls.data.build import build_data
from zcls.model.recognizers.build import build_recognizer
from zcls.engine.inference import do_evaluation
from zcls.util.collect_env import collect_env_info
from zcls.util import logging
from zcls.util.distributed import get_device, get_local_rank
from zcls.util.parser import parse_args, load_config
from zcls.util.misc import launch_job
from zcls.util.distributed import synchronize, init_distributed_training

logger = logging.get_logger(__name__)

def test(cfg):
    # Set up environment.
    init_distributed_training(cfg)
    local_rank_id = get_local_rank()

    # Set random seed from configs.
    np.random.seed(cfg.RNG_SEED + 10 * local_rank_id)
    torch.manual_seed(cfg.RNG_SEED + 10 * local_rank_id)
    torch.backends.cudnn.deterministic = False
    torch.backends.cudnn.benchmark = True

    logging.setup_logging(cfg.OUTPUT_DIR)

    device = get_device(local_rank=local_rank_id)
    model = build_recognizer(cfg, device=device)

    test_data_loader = build_data(cfg, is_train=False, rank_id=local_rank_id)

    synchronize()
    do_evaluation(cfg, model, test_data_loader, device)

def main():
    args = parse_args()
    load_config(args, cfg)

    if not os.path.exists(cfg.OUTPUT_DIR):
        os.makedirs(cfg.OUTPUT_DIR)

    logging.setup_logging(cfg.OUTPUT_DIR)
    logger.info(args)

    logger.info("Environment info:\n" + collect_env_info())
    logger.info("Loaded configuration file {}".format(args.config_file))
    if args.config_file:
        with open(args.config_file, "r") as cf:
            config_str = "\n" + cf.read()
            logger.info(config_str)
    logger.info("Running with config:\n{}".format(cfg))

    launch_job(cfg=cfg, init_method=cfg.INIT_METHOD, func=test)

if __name__ == '__main__':
    main()

max_stars_repo_name: ZJCV/ZCls

max_issues_repo_name: zjykzj/MPDataset

max_forks_repo_name: zjykzj/MPDataset


text:
# MIT License
#
# Copyright (c) 2019 Tuomas Halvari, Juha Harviainen, Juha Mylläri, Antti Röyskö, Juuso Silvennoinen
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import numpy as np

from dpemu.nodes import Array
from dpemu.filters.text import MissingArea
from dpemu import radius_generators

data = np.array(["Lorem ipsum dolor sit amet,\n" +
                 "consectetur adipisci elit,\n" +
                 "sed eiusmod tempor incidunt\n" +
                 "ut labore et dolore magna aliqua.\n" +
                 "Ut enim ad minim veniam,\n" +
                 "quis nostrud exercitation ullamco\n" +
                 "laboris nisi ut aliquid ex ea commodi consequat. \n" +
                 "Quis aute iure reprehenderit in voluptate\n" +
                 "velit esse cillum dolore eu fugiat nulla pariatur.\n" +
                 "Excepteur sint obcaecat cupiditat non proident,\n" +
                 "sunt in culpa qui officia\n" +
                 "deserunt mollit anim id est laborum.",
                 "Hello\n" +  # the next string starts here
                 "Hello\n" +
                 "Hello\n" +
                 "Hello\n" +
                 "Hello"])

root_node = Array()
root_node.addfilter(MissingArea("p", "radius_gen", "value"))

params = {}
params['value'] = " "

params['p'] = .03
params['radius_gen'] = radius_generators.GaussianRadiusGenerator(1, 1)

out = root_node.generate_error(data, params)

for elem in out:
    print(elem, end="\n\n")

params['p'] = .04
params['radius_gen'] = radius_generators.ProbabilityArrayRadiusGenerator([0, 0.6, 0.4])

out = root_node.generate_error(data, params)
for elem in out:
    print(elem, end="\n\n")

max_forks_repo_name: thalvari/dpEmu-AutoML


text:
import random
import time
import numpy as np

def create_random_walk(positions, T):
    for i in range(len(positions)):
        x = 0
        for _ in range(T):
            if random.random() < 0.5:
                x += 1
            else:
                x -= 1
        positions[i] = x

def create_random_walk_with_numpy_array(positions, T):
    positions[:] = 0
    for _ in range(T):
        positions += (
            np.random.randint(0,2,size=np.size(positions)) * 2 - 1
            )

def benchmark_create_random_walk(n, T, repeats=100):
    times = []
    for _ in range(repeats):
        positions = [0 for _ in range(n)]
        start_time = time.perf_counter_ns()
        create_random_walk(positions, T)
        end_time = time.perf_counter_ns()
        times.append(end_time - start_time)
    return times

def benchmark_create_random_walk_with_numpy(n, T, repeats=100):
    times = []
    for _ in range(repeats):
        positions = np.zeros(n)
        start_time = time.perf_counter_ns()
        create_random_walk_with_numpy_array(positions, T)
        end_time = time.perf_counter_ns()
        times.append(end_time - start_time)
    return times

from multiprocessing import Pool
def walk1D(T):
    x = 0
    for _ in range(T):
        x += 1 if random.random() < 0.5 else -1
    return x

def run_walk_in_parallel(N, T, num_processes):
    positions = [0 for _ in range(N)]
    with Pool(num_processes) as pool:
        positions = pool.map(walk1D, [T for _ in range(N)])
    return None

def benchmark_1D_walk_in_parallel(n, T, repeats=30):
    times = []
    for _ in range(repeats):
        start_time = time.perf_counter_ns()
        run_walk_in_parallel(n, T, 32)
        end_time = time.perf_counter_ns()
        times.append(end_time - start_time)
    return times

if __name__=="__main__":
    N = 1024
    T = 100

    # Raw Python
    times = benchmark_create_random_walk(N, T, repeats=15)
    min_time = min(times)
    print(f"The minimum time (raw python) for N={N}, T={T} is {min_time/1000} micro seconds.")

    # Numpy arrays
    times = benchmark_create_random_walk_with_numpy(N, T, repeats=15)
    min_time = min(times)
    print(f"The minimum time (numpy) for N={N}, T={T} is {min_time/1000} micro seconds.")

    # Raw python in parallel
    N=1024*1024
    times = benchmark_1D_walk_in_parallel(N, T, repeats=15)
    min_time = min(times)
    print(f"The minimum time (1D parallel) for N={N}, T={T} is {min_time/1000} micro seconds.")

max_stars_repo_name: JamieMair/intro-to-julia-talk

max_issues_repo_name: JamieMair/intro-to-julia-talk

max_forks_repo_name: JamieMair/intro-to-julia-talk