text:
"""
Extract mentions.

This file takes in a jsonlines file with sentences
and extract aliases and spans using a pre-computed alias table.
"""
import argparse
import logging
import multiprocessing
import os
import string
import time
from collections import defaultdict

import jsonlines
import nltk
import numpy as np
import spacy
import spacy_stanza
import stanza
from spacy.cli.download import download as spacy_download
from tqdm import tqdm

from bootleg.symbols.constants import ANCHOR_KEY
from bootleg.symbols.entity_symbols import EntitySymbols
from bootleg.utils.classes.nested_vocab_tries import VocabularyTrie
from bootleg.utils.utils import get_lnrm

logger = logging.getLogger(__name__)

nlp = None
ALL_STOPWORDS = {}
PUNC = string.punctuation
KEEP_POS = {"PROPN", "NOUN"}  # ADJ, VERB, ADV, SYM
PLURAL = {}
PUNC_TO_NONE_TABLE = str.maketrans(
    dict.fromkeys(PUNC)
)  # OR {key: None for key in string.punctuation}

def init(lang, spacy_model=None, use_stanza=False):
    """Initialize language constructs
    Args:
        lang: the language code to use to locate language model
        spacy_model: if spacy 3.X supports the requested language out of the box. Then provide the name of the model to download and use
        use_stanza: if spacy does not support the requested language out of the box set this to True (and ignore the spacy_model argument)
    """
    global nlp, ALL_STOPWORDS, PLURAL
    if use_stanza:
        try:
            nlp = spacy_stanza.load_pipeline(lang, disable=["parser", "ner"])
        except OSError:
            logger.warning(
                "Spacy-stanza model not found. Downloading and installing..."
            )
            try:
                stanza.download(lang)
                nlp = spacy_stanza.load_pipeline(lang, disable=["parser", "ner"])
            except OSError:
                nlp = None
    else:
        if not spacy_model:
            raise Exception('If use_stanza==False you must provide name of spacy model to load')
        try:
            nlp = spacy.load(spacy_model, disable=["parser", "ner"])
        except OSError:
            logger.warning(
                "Spacy models en_core_web_sm not found.  Downloading and installing."
            )
            try:
                spacy_download(spacy_model)
                nlp = spacy.load(spacy_model, disable=["parser", "ner"])
            except OSError:
                nlp = None

    # We want this to pass gracefully in the case Readthedocs is trying to build.
    # This will fail later on if a user is actually trying to run Bootleg without mention extraction
    if nlp is not None:
        ALL_STOPWORDS = nlp.Defaults.stop_words
    else:
        ALL_STOPWORDS = {}
    if lang == 'en':
        PLURAL = {"s", "'s"}

def parse_args():
    """Generate args."""
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--in_file", type=str, required=True, help="File to extract mentions from"
    )
    parser.add_argument(
        "--out_file",
        type=str,
        required=True,
        help="File to write extracted mentions to",
    )
    parser.add_argument(
        "--entity_db_dir", type=str, required=True, help="Path to entity db"
    )
    parser.add_argument("--min_alias_len", type=int, default=1)
    parser.add_argument("--max_alias_len", type=int, default=6)
    parser.add_argument("--num_workers", type=int, default=8)
    parser.add_argument("--num_chunks", type=int, default=8)
    parser.add_argument("--verbose", action="store_true")
    return parser.parse_args()

def create_out_line(sent_obj, final_aliases, final_spans):
    """Create JSON output line.

    Args:
        sent_obj: input sentence JSON
        final_aliases: list of final aliases
        final_spans: list of final spans

    Returns: JSON object
    """
    sent_obj["aliases"] = final_aliases
    sent_obj["spans"] = final_spans
    # we don't know the true QID (or even if there is one) at this stage
    # we assign to the most popular candidate for now so models w/o NIL can also evaluate this data
    sent_obj["qids"] = ["Q-1"] * len(final_aliases)
    # global alias2qids
    # sent_obj["qids"] = [alias2qids[alias][0] for alias in final_aliases]
    sent_obj[ANCHOR_KEY] = [True] * len(final_aliases)
    return sent_obj

def get_new_to_old_dict(split_sentence):
    """Return a mapped dictionary from new tokenized words with Spacy to old.

    (Spacy sometimes splits words with - and other punc).

    Args:
        split_sentence: list of words in sentence

    Returns: Dict of new word id -> old word id
    """
    old_w = 0
    new_w = 0
    new_to_old = defaultdict(list)
    while old_w < len(split_sentence):
        old_word = split_sentence[old_w]
        tokenized_word = nlp(old_word)
        new_w_ids = list(range(new_w, new_w + len(tokenized_word)))
        for i in new_w_ids:
            new_to_old[i] = old_w
        new_w = new_w + len(tokenized_word)
        old_w += 1
    new_to_old[new_w] = old_w
    new_to_old = dict(new_to_old)
    return new_to_old

def find_aliases_in_sentence_tag(
    sentence, all_aliases, min_alias_len=1, max_alias_len=6
):
    """Extract function.

    Args:
        sentence: text
        all_aliases: VocabTrie of all aliases in our save
        min_alias_len: minimum length (in words) of an alias
        max_alias_len: maximum length (in words) of an alias

    Returns: list of aliases, list of span offsets
    """
    used_aliases = []
    # Remove multiple spaces and replace with single - tokenization eats multiple spaces but
    # ngrams doesn't which can cause parse issues
    sentence = " ".join(sentence.strip().split())

    doc = nlp(sentence)
    split_sent = sentence.split()
    new_to_old_span = get_new_to_old_dict(split_sent)
    # find largest aliases first
    for n in range(max_alias_len, min_alias_len - 1, -1):
        grams = nltk.ngrams(doc, n)
        j_st = -1
        j_end = n - 1
        for gram_words in grams:
            j_st += 1
            j_end += 1
            if j_st not in new_to_old_span or j_end not in new_to_old_span:
                print("BAD")
                print(sentence)
                return [], []
            j_st_adjusted = new_to_old_span[j_st]
            j_end_adjusted = new_to_old_span[j_end]
            # Check if nlp has split the word and we are looking at a subword mention - which we don't want
            is_subword = j_st_adjusted == j_end_adjusted
            if j_st > 0:
                is_subword = is_subword | (j_st_adjusted == new_to_old_span[j_st - 1])
            # j_end is exclusive and should be a new word from the previous j_end-1
            is_subword = is_subword | (j_end_adjusted == new_to_old_span[j_end - 1])
            if is_subword:
                continue
            # Assert we are a full word
            assert (
                j_st_adjusted != j_end_adjusted
            ), f"Something went wrong getting mentions for {sentence}"
            # If single word and not in a POS we care about, skip
            if len(gram_words) == 1 and gram_words[0].pos_ not in KEEP_POS:
                continue
            # If multiple word and not any word in a POS we care about, skip
            if len(gram_words) > 1 and not any(g.pos_ in KEEP_POS for g in gram_words):
                continue
            # print("@", gram_words, [g.pos_ for g in gram_words])
            # If we are part of a proper noun, make sure there isn't another part of the proper noun to the
            # left or right - this means we didn't have the entire name in our alias and we should skip
            if len(gram_words) == 1 and gram_words[0].pos_ == "PROPN":
                if j_st > 0 and doc[j_st - 1].pos_ == "PROPN":
                    continue
                # End spans are exclusive so no +1
                if j_end < len(doc) and doc[j_end].pos_ == "PROPN":
                    continue
            # print("3", j_st, gram_words, [g.pos_ for g in gram_words])
            # We don't want punctuation words to be used at the beginning/end unless it's capitalized
            # or first word of sentence
            if (
                gram_words[-1].text in PLURAL
                or gram_words[0].text in PLURAL
                or (
                    gram_words[0].text.lower() in ALL_STOPWORDS
                    and (not gram_words[0].text[0].isupper() or j_st == 0)
                )
            ):
                continue
            # If the word starts with punctuation and there is a space in between, also continue; keep
            # if punctuation is part of the word boundary
            # print("4", j_st, gram_words, [g.pos_ for g in gram_words])
            if (
                gram_words[0].text in PUNC
                and (
                    j_st + 1 >= len(doc)
                    or new_to_old_span[j_st] != new_to_old_span[j_st + 1]
                )
            ) or (
                gram_words[-1].text in PUNC
                and (
                    j_end - 2 < 0
                    or new_to_old_span[j_end - 1] != new_to_old_span[j_end - 2]
                )
            ):
                continue
            joined_gram = " ".join(split_sent[j_st_adjusted:j_end_adjusted])
            # If 's in alias, make sure we remove the space and try that alias, too
            joined_gram_merged_plural = joined_gram.replace(" 's", "'s")
            # If PUNC in alias, make sure we remove the space and try that alias, too
            joined_gram_merged_nopunc = joined_gram_merged_plural.translate(PUNC_TO_NONE_TABLE)
            gram_attempt = get_lnrm(joined_gram, strip=True, lower=True)
            gram_attempt_merged_plural = get_lnrm(
                joined_gram_merged_plural, strip=True, lower=True
            )
            gram_attempt_merged_nopunc = get_lnrm(
                joined_gram_merged_nopunc, strip=True, lower=True
            )
            # Remove numbers
            if (
                gram_attempt.isnumeric()
                or joined_gram_merged_plural.isnumeric()
                or gram_attempt_merged_nopunc.isnumeric()
            ):
                continue
            final_gram = None
            # print("4", gram_attempt, [g.pos_ for g in gram_words])
            if gram_attempt in all_aliases:
                final_gram = gram_attempt
            elif gram_attempt_merged_plural in all_aliases:
                final_gram = gram_attempt_merged_plural
            elif gram_attempt_merged_nopunc in all_aliases:
                final_gram = gram_attempt_merged_nopunc
                # print("5", final_gram, [g.pos_ for g in gram_words])
            # print("FINAL GRAM", final_gram)
            if final_gram is not None:
                keep = True
                # We start from the largest n-grams and go down in size. This prevents us from adding an alias that
                # is a subset of another. For example: "Tell me about the mother on how I met you mother" will find
                # "the mother" as alias and "mother". We want to only take "the mother" and not "mother" as it's
                # likely more descriptive of the real entity.
                for u_al in used_aliases:
                    u_j_st = u_al[1]
                    u_j_end = u_al[2]
                    if j_st_adjusted < u_j_end and j_end_adjusted > u_j_st:
                        keep = False
                        break
                if not keep:
                    continue
                used_aliases.append(tuple([final_gram, j_st_adjusted, j_end_adjusted]))
    # sort based on span order
    aliases_for_sorting = sorted(used_aliases, key=lambda elem: [elem[1], elem[2]])
    used_aliases = [a[0] for a in aliases_for_sorting]
    spans = [[a[1], a[2]] for a in aliases_for_sorting]
    assert all([sp[1] <= len(doc) for sp in spans]), f"{spans} {sentence}"
    return used_aliases, spans

def get_num_lines(input_src):
    """Count number of lines in file.

    Args:
        input_src: input file

    Returns: number of lines
    """
    # get number of lines
    num_lines = 0
    with open(input_src, "r", encoding="utf-8") as in_file:
        try:
            for line in in_file:
                num_lines += 1
        except Exception as e:
            logger.error("ERROR READING IN TRAINING DATA")
            logger.error(e)
            return []
    return num_lines

def chunk_text_data(input_src, chunk_files, chunk_size, num_lines):
    """Chunk text input file into chunk_size chunks.

    Args:
        input_src: input file
        chunk_files: list of chunk file names
        chunk_size: chunk size in number of lines
        num_lines: total number of lines
    """
    logger.debug(f"Reading in {input_src}")
    start = time.time()
    # write out chunks as text data
    chunk_id = 0
    num_lines_in_chunk = 0
    # keep track of what files are written
    out_file = open(chunk_files[chunk_id], "w", encoding="utf-8")
    with open(input_src, "r", encoding="utf-8") as in_file:
        for i, line in enumerate(in_file):
            out_file.write(line)
            num_lines_in_chunk += 1
            # move on to new chunk when it hits chunk size
            if num_lines_in_chunk == chunk_size:
                chunk_id += 1
                # reset number of lines in chunk and open new file if not at end
                num_lines_in_chunk = 0
                out_file.close()
                if i < (num_lines - 1):
                    out_file = open(chunk_files[chunk_id], "w", encoding="utf-8")
    out_file.close()
    logger.debug(f"Wrote out data chunks in {round(time.time() - start, 2)}s")

def subprocess(args):
    """
    Extract mentions single process.

    Args:
        args: subprocess args
    """
    in_file = args["in_file"]
    out_file = args["out_file"]
    min_alias_len = args["min_alias_len"]
    max_alias_len = args["max_alias_len"]
    verbose = args["verbose"]
    all_aliases = VocabularyTrie(load_dir=args["all_aliases_trie_f"])
    num_lines = sum(1 for _ in open(in_file))
    with jsonlines.open(in_file) as f_in, jsonlines.open(out_file, "w") as f_out:
        for line in tqdm(
            f_in, total=num_lines, disable=not verbose, desc="Processing data"
        ):
            found_aliases, found_spans = find_aliases_in_sentence_tag(
                line["sentence"], all_aliases, min_alias_len, max_alias_len
            )
            f_out.write(create_out_line(line, found_aliases, found_spans))

def merge_files(chunk_outfiles, out_filepath):
    """Merge output files.

    Args:
        chunk_outfiles: list of chunk files
        out_filepath: final output file path
    """
    sent_idx_unq = 0
    with jsonlines.open(out_filepath, "w") as f_out:
        for file in chunk_outfiles:
            with jsonlines.open(file) as f_in:
                for line in f_in:
                    if "sent_idx_unq" not in line:
                        line["sent_idx_unq"] = sent_idx_unq
                    f_out.write(line)
                    sent_idx_unq += 1

def extract_mentions(
    in_filepath,
    out_filepath,
    entity_db_dir,
    min_alias_len=1,
    max_alias_len=6,
    num_workers=8,
    num_chunks=None,
    verbose=False,
):
    """Extract mentions from file.

    Args:
        in_filepath: input file
        out_filepath: output file
        entity_db_dir: path to entity db
        min_alias_len: minimum alias length (in words)
        max_alias_len: maximum alias length (in words)
        num_workers: number of multiprocessing workers
        num_chunks: number of subchunks to feed to workers
        verbose: verbose boolean
    """
    assert os.path.exists(in_filepath), f"{in_filepath} does not exist"
    entity_symbols: EntitySymbols = EntitySymbols.load_from_cache(entity_db_dir)
    all_aliases_trie: VocabularyTrie = entity_symbols.get_all_alias_vocabtrie()
    if num_chunks is None:
        num_chunks = num_workers
    start_time = time.time()
    # multiprocessing
    if num_workers > 1:
        prep_dir = os.path.join(os.path.dirname(out_filepath), "prep")
        os.makedirs(prep_dir, exist_ok=True)

        all_aliases_trie_f = os.path.join(prep_dir, "mention_extract_alias.marisa")
        all_aliases_trie.dump(all_aliases_trie_f)

        # chunk file for multiprocessing
        num_lines = get_num_lines(in_filepath)
        num_processes = min(num_workers, int(multiprocessing.cpu_count()))
        num_chunks = min(num_lines, num_chunks)
        logger.debug(f"Using {num_processes} workers...")
        chunk_size = int(np.ceil(num_lines / num_chunks))
        chunk_file_path = os.path.join(prep_dir, "data_chunk")
        chunk_infiles = [
            f"{chunk_file_path}_{chunk_id}_in.jsonl" for chunk_id in range(num_chunks)
        ]
        chunk_text_data(in_filepath, chunk_infiles, chunk_size, num_lines)
        logger.debug("Calling subprocess...")
        # call subprocesses on chunks
        pool = multiprocessing.Pool(processes=num_processes)
        chunk_outfiles = [
            f"{chunk_file_path}_{chunk_id}_out.jsonl" for chunk_id in range(num_chunks)
        ]
        subprocess_args = [
            {
                "in_file": chunk_infiles[i],
                "out_file": chunk_outfiles[i],
                "min_alias_len": min_alias_len,
                "max_alias_len": max_alias_len,
                "all_aliases_trie_f": all_aliases_trie_f,
                "verbose": verbose,
            }
            for i in range(num_chunks)
        ]
        pool.map(subprocess, subprocess_args)
        pool.close()
        pool.join()
        logger.debug("Merging files...")
        # write all chunks back in single file
        merge_files(chunk_outfiles, out_filepath)
        logger.debug("Removing temporary files...")
        # clean up and remove chunked files
        for file in chunk_infiles:
            try:
                os.remove(file)
            except PermissionError:
                pass
        for file in chunk_outfiles:
            try:
                os.remove(file)
            except PermissionError:
                pass
        try:
            os.remove(all_aliases_trie_f)
        except PermissionError:
            pass

    # single process
    else:
        logger.debug("Using 1 worker...")
        with jsonlines.open(in_filepath, "r") as in_file, jsonlines.open(
            out_filepath, "w"
        ) as out_file:
            sent_idx_unq = 0
            for line in in_file:
                found_aliases, found_spans = find_aliases_in_sentence_tag(
                    line["sentence"], all_aliases_trie, min_alias_len, max_alias_len
                )
                new_line = create_out_line(line, found_aliases, found_spans)
                if "sent_idx_unq" not in new_line:
                    new_line["sent_idx_unq"] = sent_idx_unq
                    sent_idx_unq += 1
                out_file.write(new_line)

    logger.debug(
        f"Finished in {time.time() - start_time} seconds. Wrote out to {out_filepath}"
    )

def main():
    """Run."""
    args = parse_args()
    in_file = args.in_file
    out_file = args.out_file
    init('en', 'en_core_web_sm')
    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
    print(args)
    extract_mentions(
        in_file,
        out_file,
        entity_db_dir=args.entity_db_dir,
        min_alias_len=args.min_alias_len,
        max_alias_len=args.max_alias_len,
        num_workers=args.num_workers,
        num_chunks=args.num_chunks,
        verbose=args.verbose,
    )

if __name__ == "__main__":
    main()

max_stars_repo_name: rubmz/bootleg

max_issues_repo_name: rubmz/bootleg

max_forks_repo_name: rubmz/bootleg


text:
"""
This file contains a function that can be used to generate a Hess plot.

:requires: NumPy

:version: 0.1

:author: Sami-Matias Niemi
:contact: niemi@stsci.edu
"""
import numpy as N

def hess_plot(xdata, ydata, weight,
              xmin, xmax, nxbins,
              ymin, ymax, nybins,
              pmax=1.0, pmin=0.1):
    """
    This function can be used to calculate a hess plot
    i.e. the conditional probability 2D histogram.

    :param xdata: 1D NumPy array containing data in x dim.
    :param ydata: 1D NumPy array containing data in y dim.
    :param weight: 1D NumPy array containing weight for
                   each data point
    :param xmin: a minimum x value to be considered
    :param xmax: a maximum x value to be considered
    :param nxbins: the number of bins in x dim.
    :param nybins: the number of bins in y dim.
    :param pmax: 1.0
    :param pmin: 0.1

    :return: 2D array, min, max
    """
    #xbins
    xbin = N.linspace(xmin, xmax, nxbins + 1)
    dx = xbin[1] - xbin[0]
    xmids = xbin[:-1] + dx / 2.
    #ybins
    ybin = N.linspace(ymin, ymax, nybins + 1)
    dy = ybin[1] - ybin[0]
    ymids = ybin[:-1] + dy / 2.

    yhist = N.zeros((nybins, nxbins))
    yhist_num = N.zeros((nybins, nxbins))

    #number of h in each xbin
    nh = N.zeros(nxbins)

    for i, x in enumerate(xbin[:-1]):
        xmask = (xdata >= x) & (xdata < x + dx)
        if len(xdata[xmask]) > 0:
            nh[i] = N.sum(weight[xmask])
            for j, y in enumerate(ybin[:-1]):
                ymask = (xdata >= x) & (xdata < x + dx) & (ydata >= y) & (ydata < y + dy)
                if len(ydata[ymask]) > 0:
                    yhist[j, i] = N.sum(weight[ymask])
                    yhist_num[j, i] = len(ydata[ymask])

    #conditional distributions in Mh bins
    for i in range(nxbins):
        if nh[i] > 0.0:
            yhist[:, i] /= nh[i]

    s = N.log10(pmax) - N.log10(yhist)
    smax = N.log10(pmax) - N.log10(pmin)
    smin = N.log10(pmax)

    s[yhist <= 0.0] = 2.0 * smax
    s[yhist_num <= 1] = 2.0 * smax

    return s, smin, smax

def hess_plot_old(xdata, ydata, weight,
                  xmin, xmax, nxbins,
                  ymin, ymax, nybins,
                  pmax=1.0, pmin=0.1):
    """
    :note: obsolete, do not use this version!
    """
    dx = (xmax - xmin) / float(nxbins)
    mbin = xmin + (N.arange(nxbins)) * dx + dx / 2.

    dy = (ymax - ymin) / float(nybins)
    ybin = ymin + (N.arange(nybins)) * dy + dy / 2.

    yhist = N.zeros((nybins, nxbins))

    nogas = N.zeros(nxbins)

    yhist_num = N.zeros((nybins, nxbins))
    #number of h in each bin
    nh = N.zeros(nxbins)

    #this should be rewritten...
    for i in range(len(xdata)):
        imbin = int(N.floor((xdata[i] - xmin) / dx))
        if imbin >= 0 and imbin < nxbins:
            nh[imbin] = nh[imbin] + weight[i]
            iy = int(N.floor((ydata[i] - ymin) / dy))
            if iy >= 0 and iy < nybins:
                yhist[iy, imbin] = yhist[iy, imbin] + weight[i]
                yhist_num[iy, imbin] = yhist_num[iy, imbin] + 1

    #conditional distributions in Mh bins
    for i in range(nxbins):
        if nh[i] > 0.0:
            yhist[:, i] /= nh[i]

    s = N.log10(pmax) - N.log10(yhist)
    smax = N.log10(pmax) - N.log10(pmin)
    smin = N.log10(pmax)

    s[yhist <= 0.0] = 2.0 * smax
    s[yhist_num <= 1] = 2.0 * smax

    return s, smin, smax

if __name__ == '__main__':
    """
    This is just to test that the new Hess and old Hess
    function results agree
    """
    import db.sqlite as sq
    import os
    #find the home directory, because the output is to dropbox
    #and my user name is not always the same, this hack is required.
    hm = os.getenv('HOME')
    #constants
    path = hm + '/Dropbox/Research/Herschel/runs/reds_zero/'
    db = 'sams.db'

    bulge = 0.4;
    pmin = 0.05;
    pmax = 1.0
    xbin1 = 15;
    ybin1 = 15
    xbin2 = 15;
    ybin2 = 15
    xmin1 = 7.9;
    xmax1 = 11.7
    ymin = 0.1;
    ymax = 10

    query1 = """select galprop.mstar, galprop.r_disk, Pow10(galprop.mbulge - galprop.mstar)
                from FIR, galprop where
                FIR.z >= 2.0 and
                FIR.z < 4.0 and
                FIR.gal_id = galprop.gal_id and
                FIR.halo_id = galprop.halo_id and
                galprop.tmerge > 0.5 and
                FIR.spire250_obs > 1e-15 and
                FIR.spire250_obs < 1e6
                """
    #get data
    data = N.array(sq.get_data_sqlitePowerTen(path, db, query1))
    #slice the data
    disks = data[:, 2] <= bulge
    xd1 = data[:, 0][disks]
    yd1 = data[:, 1][disks]
    #old algorithm
    so1, somin1, somax1 = hess_plot_old(xd1, yd1, N.ones(len(xd1)),
                                        xmin1, xmax1, xbin1,
                                        ymin, ymax, ybin1,
                                        pmax=pmax, pmin=pmin)

    #new algorithm
    sn1, snmin1, snmax1 = hess_plot(xd1, yd1, N.ones(len(xd1)),
                                    xmin1, xmax1, xbin1,
                                    ymin, ymax, ybin1,
                                    pmax=pmax, pmin=pmin)

    print somin1 == snmin1
    print
    print somax1 == snmax1
    print
    print sn1
    print
    print N.round(so1, 4) == N.round(sn1, 4)
    print
    print N.round(so1 - sn1, 4)

ext: py

max_stars_repo_name: sniemi/SamPy

max_issues_repo_name: sniemi/SamPy

max_forks_repo_name: sniemi/SamPy

reason: import numpy


text:
from numpy import add
import models
import yfinance
from fastapi import FastAPI, Request, Depends, BackgroundTasks
from fastapi.templating import Jinja2Templates
from sqlalchemy.orm import Session
from database import SessionLocal, engine
from pydantic import BaseModel
from models import Stock

fastapi = FastAPI

models.Base.metadata.create_all(bind=engine)

templates = Jinja2Templates(directory="templates")

app = fastapi()

class StockRequest(BaseModel):
    symbol: str

def get_db():
    try:
        db = SessionLocal()
        yield db
    finally:
        db.close()

@app.get("/")
def home(request: Request, forward_pe = None, dividend_yield = None, ma50 = None, ma200 = None, db: Session = Depends(get_db)):
    """
    displays the home page
    """
    stocks = db.query(Stock)

    print(stocks)

    if forward_pe:
        stocks = stocks.filter(Stock.forward_pe < forward_pe)

    if dividend_yield:
        stocks = stocks.filter(Stock.dividend_yield > dividend_yield)

    if ma50:
        stocks = stocks.filter(Stock.price > Stock.ma50)

    if ma200:
        stocks = stocks.filter(Stock.price > Stock.ma200)

    return templates.TemplateResponse("home.html", {
        "request": request,
        "stocks": stocks,
        "dividend_yield": dividend_yield,
        "ma50": ma50,
        "ma200": ma200,
    })

def fetch_stock_data(id: int):
    db = SessionLocal()
    stock = db.query(Stock).filter(Stock.id ==id).first()

    stock_data = yfinance.Ticker(stock.symbol)

    stock.ma200 = stock_data.info['twoHundredDayAverage']
    stock.m50 = stock_data.info['fiftyDayAverage']
    stock.price = stock_data.info['previousClose']
    stock.forward_pe = stock_data.info['forwardPE']
    stock.forward_eps = stock_data.info['forwardEps']
    if stock_data.info['dividendYield'] is not None:
        stock.dividend_yield = stock_data.info['dividendYield']

    db.add(stock)
    db.commit()

@app.post("/stock")
async def create_stock(stock_request: StockRequest, background_tasks: BackgroundTasks, db: Session = Depends(get_db)):
    """
    create the stock and stores it on the db
    """
    stock = Stock()
    stock.symbol = stock_request.symbol

    db.add(stock)
    db.commit()

    background_tasks.add_task(fetch_stock_data, stock.id)

    return{
        "code": "sucess",
        "message": "stock created"
    }

max_stars_repo_path: Stock Screener FASTapi/main.py

max_stars_repo_name: marxmyllyanno/krakentest

max_issues_repo_path: Stock Screener FASTapi/main.py

max_issues_repo_name: marxmyllyanno/krakentest

max_forks_repo_path: Stock Screener FASTapi/main.py

max_forks_repo_name: marxmyllyanno/krakentest

reason: from numpy


text:
# -*- coding: utf-8 -*-
# @Time : 2021/4/13 15:36
# @Author : aurorazeng
# @File : extract_test_patches.py
# @license: (C) Copyright 2021-2026, aurorazeng; No reprobaiction without permission.

import os
import numpy as np
import cv2

tumor = [1, 19, 20]
lymph = [3, 10, 11, 14]
stroma = [2]
necrosis = [4]
others = [5, 6, 7, 8, 9, 12, 13, 15, 16, 17, 18, 21]

def convertMultiLabels(mask):
    new_mask = np.zeros((mask.shape[0], mask.shape[1]), dtype='uint8')
    for i in tumor:
        new_mask[mask == i] = 1
    # 2 - stroma - blue
    for i in stroma:
        new_mask[mask == i] = 2
    # 3 - lymphocytic_infiltrate - green
    for i in lymph:
        new_mask[mask == i] = 3
    # 4 - necrosis_or_debris - cyan
    for i in necrosis:
        new_mask[mask == i] = 4
    # 5 - others - yellow
    for i in others:
        new_mask[mask == i] = 5
    # 0: outside_roi - white
    new_mask[mask == 0] = 0
    return new_mask

if __name__ == "__main__":

    source = r'C:\Users\aurorazeng\Desktop\need_product\data\test'
    image_path = os.path.join(source, 'images')
    mask_path = os.path.join(source, 'masks')

    patch_size = 768
    overlap = 0.5
    # stride = int(patch_size * (1 - overlap))
    stride = 512
    target_mpp = 0.46

    isTrain = True

    patch_path = os.path.join(source, 'patch_' + str(target_mpp))
    patch_image = os.path.join(patch_path, 'images')
    patch_mask = os.path.join(patch_path, 'masks')
    patch_visual = os.path.join(patch_path, 'visual')
    patch_label = os.path.join(patch_path, 'labels')

    if not os.path.isdir(patch_path):
        os.makedirs(patch_path)
    if not os.path.isdir(patch_image):
        os.makedirs(patch_image)
    if not os.path.isdir(patch_mask):
        os.makedirs(patch_mask)
    if not os.path.isdir(patch_visual):
        os.makedirs(patch_visual)
    if not os.path.isdir(patch_label):
        os.makedirs(patch_label)

    imageList = os.listdir(image_path)
    #    for name in imageList[0].split(' '):
    for name in imageList:
        name_noext, _ = os.path.splitext(name)
        print(name_noext)
        # read file
        image = cv2.imread(os.path.join(image_path, name_noext + '.png'))
        mask = cv2.imread(os.path.join(mask_path, name_noext + '.png'), 0)

        # resize file
        mpp = float(name_noext.split('MPP-')[1])
        scale = round(mpp / target_mpp, 4)
        resize_height = int(np.ceil(image.shape[0] * scale / 32) * 32)
        resize_width = int(np.ceil(image.shape[1] * scale / 32) * 32)

        ds_image = cv2.resize(image, (resize_width, resize_height))
        ds_mask = cv2.resize(mask, (resize_width, resize_height),
                             interpolation=cv2.INTER_NEAREST)
        ds_mask = convertMultiLabels(ds_mask)

        # get max_x and max_y and padding zero to image and mask
        max_y = int((np.ceil(resize_height / stride) - 1) * stride) + patch_size + 256
        max_x = int((np.ceil(resize_width / stride) - 1) * stride) + patch_size + 256

        image = np.zeros((max_y, max_x, 3), dtype='uint8')
        # print(image.shape)
        # print(ds_image.shape)
        image[128:ds_image.shape[0] + 128, 128:ds_image.shape[1] + 128, :] = ds_image
        #        plt.imshow(image)

        label = np.zeros((max_y, max_x), dtype='uint8')
        label[128:ds_mask.shape[0] + 128, 128:ds_mask.shape[1] + 128] = ds_mask

        y_list = range(0, max_y, stride)
        x_list = range(0, max_x, stride)
        n = 0
        for y in y_list:
            for x in x_list:
                patch = image[y:y + patch_size, x:x + patch_size, :]
                label = mask[y:y + patch_size, x:x + patch_size]

                save_name = name_noext.split('xmin')[0]
                out_fn = '{}{:03d}_{:d}_{}{}'.format(save_name, n,
                                                     patch_size, '{:.3f}'.format(target_mpp), '.png')

                cv2.imwrite(os.path.join(patch_image, out_fn), patch)
                cv2.imwrite(os.path.join(patch_label, out_fn), label)
                n += 1


text:
import os,sys
import numpy as np
import cntk as C
from cntk import load_model, combine
import cntk.io.transforms as xforms
from cntk.logging import graph
from cntk.logging.graph import get_node_outputs
import shutil
import csv
import pathlib2
import glob

class CNTKFeatureExtraction(object):
    def __init__(self):
        pass

    def _download_model(self, model_file_name, model_url , model_download_path):
        try:
            from urllib.request import urlretrieve
        except ImportError:
            from urllib import urlretrieve

        pathlib2.Path(model_download_path).mkdir(parents=True, exist_ok=True)
        filename = os.path.join(model_download_path, model_file_name)

        if not os.path.exists(filename):
            print('Downloading model from ' + model_url + ', may take a while...')
            urlretrieve(model_url, filename)
            print('Saved model as ' + model_file_name)
        else:
            print('CNTK model already available at ' + filename)
        return model_file_name

    def create_mb_source(self, image_height, image_width, num_channels, map_file):
        transforms = [xforms.scale(width=image_width, height=image_height, channels=num_channels, interpolations='linear')]
        return C.io.MinibatchSource(
            C.io.ImageDeserializer(map_file, C.io.StreamDefs(
                features=C.io.StreamDef(field='image', transforms=transforms),
                labels=C.io.StreamDef(field='label', shape=4096))),
            randomize=False)

    def eval_and_write(self, model_file, node_name, minibatch_source , map_file , features_folder_path):
        # load model and pick desired node as output
        loaded_model  = load_model(model_file)
        node_in_graph = loaded_model.find_by_name(node_name)
        output_nodes  = combine([node_in_graph.owner])

        # evaluate model and get desired node output
        print("Evaluating model for output node %s" % node_name)
        features_si = minibatch_source['features']

        with open(map_file) as f:
            reader = csv.reader(f, delimiter="\t")
            d = list(reader)

        pathlib2.Path(features_folder_path).mkdir(parents=True, exist_ok=True)
        for val in d:
            output_file = os.path.splitext(val[1])[0] + '.txt'
            output_file = os.path.join(features_folder_path , os.path.basename(output_file))

            with open(output_file, 'wb') as results_file:
                mb = minibatch_source.next_minibatch(1)
                output = output_nodes.eval(mb[features_si])
                # write results to file
                out_values = output[0].flatten()
                np.savetxt(results_file, out_values, newline = "\n")

    def _create_map_file(self, image_path , images_txt_save_path):
        filename = image_path

        if os.path.exists(filename):
            os.chdir(filename)
            text_file = open(os.path.join(images_txt_save_path , "cnn_images.txt"), "w")

            types = ('*.jpeg', '*.jpg' , '*.JPEG' , '*.png') # the tuple of file types
            files_grabbed = []
            for files in types:
                files_grabbed.extend(glob.glob(files))

            for file in files_grabbed:
                text_file.write("0\t" + os.path.join(filename , file) + "\t0" + "\n")
            text_file.close()
            print("images.txt created at location : " + images_txt_save_path)
            os.chdir(images_txt_save_path)
            return os.path.join(images_txt_save_path , "cnn_images.txt")
        else:
            print("No Folder exist of name \"" +  image_path + "\" Please create and put images into it")
            exit(0)

    def extract_feature(self, model_download_path , images_path, main_dir, pretrained_model , extract_from_layer):
        model_file = self._download_model("ResNet18_ImageNet_CNTK.model", "https://www.cntk.ai/Models/CNTK_Pretrained/ResNet18_ImageNet_CNTK.model" , model_download_path)

        # define location of model and data and check existence
        model_file  = os.path.join(model_download_path , model_file)
        images_txt_save_path = os.path.join(main_dir , "features_extraction")  # TODO Really bad idea.. What if folder change?
        map_file = self._create_map_file(images_path, images_txt_save_path)
        if not (os.path.exists(model_file)):
            print("Cannot locate model")

        if not (os.path.exists(map_file)):
            print("Cannot locate mapped file for images")

        # create minibatch source
        image_height = 224
        image_width  = 224
        num_channels = 3
        minibatch_source = self.create_mb_source(image_height, image_width, num_channels, map_file)

        # use this to print all node names of the model (and knowledge of the model to pick the correct one)
        node_outputs = get_node_outputs(load_model(model_file))
        for out in node_outputs: print("{0} {1}".format(out.name, out.shape))

        # evaluate model and write out the desired layer output
        features_folder_path = os.path.join(main_dir , "dataset", "features_etd1a" , pretrained_model , extract_from_layer)
        self.eval_and_write(model_file, extract_from_layer, minibatch_source , map_file , features_folder_path)

if __name__ == "__main__":
    main_dir = "C:\\Users\\Sabs\\Desktop\\img-search-cnn\webapp\\"
    obj = CNTKFeatureExtraction()

    obj.extract_feature( "C:\\Users\\Sabs\\Desktop\\img-search-cnn\webapp\\feature_extraction\\models\\", "D:\\Deep Learning DATA\\DLImages\\images_TRY\\" ,main_dir , "" , "z" )


text:
import torch.nn as nn
import numpy as np
import pytest

from test.utils import convert_and_test

class LayerSoftmax(nn.Module):
    """
    Test for nn.layers based types
    """
    def __init__(self, dim):
        super(LayerSoftmax, self).__init__()
        self.dim = dim
        self.softmax = nn.Softmax(dim=dim)

    def forward(self, x):
        x = self.softmax(x)
        return x

class FSoftmax(nn.Module):
    """
    Test for nn.functional types
    """
    def __init__(self, dim):
        super(FSoftmax, self).__init__()
        self.dim = dim

    def forward(self, x):
        from torch.nn import functional as F
        return F.softmax(x, self.dim)

@pytest.mark.parametrize('change_ordering', [True, False])
@pytest.mark.parametrize('dim', [0, 1, 2, 3])
def test_layer_softmax(change_ordering, dim):
    model = LayerSoftmax(dim)
    model.eval()
    input_np = np.random.uniform(0, 1, (1, 3, 224, 224))
    error = convert_and_test(model, input_np, verbose=False, change_ordering=change_ordering)

@pytest.mark.parametrize('change_ordering', [True, False])
@pytest.mark.parametrize('dim', [0, 1, 2, 3])
def test_f_softmax(change_ordering, dim):
    model = FSoftmax(dim)
    model.eval()
    input_np = np.random.uniform(0, 1, (1, 3, 224, 224))
    error = convert_and_test(model, input_np, verbose=False, change_ordering=change_ordering)


text:
# -*- coding: utf-8 -*-
"""
Created on Mon July 9 22:20:12 2018

@author: Adam
"""
import os
import sqlite3
import numpy as np
import pandas as pd
from datetime import datetime
from emonitor.core import TABLE, DATA_DIRE
from emonitor.tools import db_path, db_init, db_check, db_describe, db_insert
from emonitor.data import EmonitorData
from emonitor import history

# constants
COLUMNS = ('A', 'B', 'C')
TCOL = 'TIMESTAMP'
NAME = '__pytest__.db'
DB = db_path(NAME)
if os.path.isfile(DB):
    os.remove(DB)
CONN = sqlite3.connect(DB)
DATA = [('2016-12-09 09:08:13', 1, 34.8, 3),
        ('2018-12-10 09:08:13', 2, 12, 3),
        ('2018-12-10 09:10:13', 3, 6.7, 3)]

def test_datadire_exists():
    assert os.path.exists(DATA_DIRE)

def test_new_db():
    db_init(CONN, TABLE, COLUMNS)
    db_check(CONN, TABLE, COLUMNS)

def test_desc():
    DESC = ("[(0, 'TIMESTAMP', 'timestamp', 1, 'CURRENT_TIMESTAMP', 0),"
            " (1, 'A', 'DOUBLE', 0, 'NULL', 0),"
            " (2, 'B', 'DOUBLE', 0, 'NULL', 0),"
            " (3, 'C', 'DOUBLE', 0, 'NULL', 0)]")
    assert str(db_describe(CONN, TABLE)) == DESC

def test_insert():
    cols = (TCOL,) + COLUMNS
    for d in DATA:
        db_insert(CONN, TABLE, cols, d)

def test_history():
    start = datetime(2015, 12, 9, 9, 8, 13)
    end = datetime(2018, 12, 11, 9, 8, 13)
    df = history(CONN, start, end)
    vals = np.array([row[1:] for row in DATA])
    assert np.array_equal(df.values, vals)

def test_clean():
    CONN.close()
    os.remove(DB)

def test_emonitordata():
    name = NAME
    data = EmonitorData(DATA_DIRE)
    data.create(name, columns=[1, 2, 3], quiet=True)
    fils = data.show()
    assert isinstance(fils, list)
    assert name in fils
    data.destroy(name, force=True)
    fils = data.show()
    assert name not in fils


text:
# pangadfs/pangadfs/misc.py
# -*- coding: utf-8 -*-
# Copyright (C) 2020 Eric Truett
# Licensed under the MIT License

from typing import Dict, Iterable, Tuple
import numpy as np

def diversity(population: np.ndarray) -> np.ndarray:
    """Calculates diversity of lineups

    Args:
        population (np.ndarray): the population

    Returns:
        np.ndarray: is square, shape len(population) x len(population)

    """
    uniques = np.unique(population)
    a = (population[..., None] == uniques).sum(1)
    return np.einsum('ij,kj->ik', a, a)

def exposure(population: np.ndarray = None) -> Dict[int, int]:
    """Returns dict of index: count of individuals

    Args:
        population (np.ndarray): the population

    Returns:
        Dict[int, int]: key is index, value is count of lineup

    Examples:
        >>> fittest_population = population[np.where(fitness > np.percentile(fitness, 97))]
        >>> exposure = population_exposure(fittest_population)
        >>> top_exposure = np.argpartition(np.array(list(exposure.values())), -10)[-10:]
        >>> print([round(i, 3) for i in sorted(top_exposure / len(fittest_population), reverse=True)])

    """
    flat = population.flatten
    return dict(zip(flat, np.bincount(flat)[flat]))

def multidimensional_shifting(elements: Iterable,
                              num_samples: int,
                              sample_size: int,
                              probs: Iterable) -> np.ndarray:
    """Based on https://medium.com/ibm-watson/incredibly-fast-random-sampling-in-python-baf154bd836a

    Args:
        elements (iterable): iterable to sample from, typically a dataframe index
        num_samples (int): the number of rows (e.g. initial population size)
        sample_size (int): the number of columns (e.g. team size)
        probs (iterable): is same size as elements

    Returns:
        ndarray: of shape (num_samples, sample_size)

    """
    replicated_probabilities = np.tile(probs, (num_samples, 1))
    random_shifts = np.random.random(replicated_probabilities.shape)
    random_shifts /= random_shifts.sum(axis=1)[:, np.newaxis]
    shifted_probabilities = random_shifts - replicated_probabilities
    samples = np.argpartition(shifted_probabilities, sample_size, axis=1)[:, :sample_size]
    return elements.to_numpy()[samples]

def parents(population: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
    """Evenly splits population

    Args:
        population (np.ndarray): the population to crossover. Shape is n_individuals x n_chromosomes.

    Returns:
        Tuple[np.ndarray, np.ndarray]: population split into two equal-size arrays

    """
    fathers, mothers = np.array_split(population, 2)
    size = min(len(fathers), len(mothers))
    return fathers[:size], mothers[:size]

max_stars_repo_name: sansbacon/pangadfs

max_issues_repo_name: sansbacon/pangadfs

max_forks_repo_name: sansbacon/pangadfs


text:
#! /usr/bin/env python3

import numpy as np
import os
import sys
import random
def loadSpectrum(classLabelReal,EDSData,CodePath):
    #sess = tfclf.sess
    if classLabelReal==0:
        phase = 0
        filename = CodePath + 'ResultsAndData' + os.path.sep + 'EDSSpectra' + os.path.sep + EDSData.Folder +  os.path.sep + 'Phase_1' + os.path.sep + 'EDSvalid1.npy'
        vec = np.load(filename)
        randNum = random.randint(0, EDSData.NumSpectra-1);
        r = vec[randNum, :]
        s = np.random.poisson(EDSData.Noiselambda, len(r))
    else:
        phase = classLabelReal
        filename = CodePath + 'ResultsAndData' + os.path.sep + 'EDSSpectra' + os.path.sep + EDSData.Folder +  os.path.sep + 'Phase_' + str(phase) + os.path.sep + 'EDSvalid' +str(classLabelReal) + '.npy'
        vec = np.load(filename)
        randNum = random.randint(0, EDSData.NumSpectra-1);
        r = vec[randNum, :]
        s = r + np.random.poisson(EDSData.Noiselambda, len(r))
    return s,classLabelReal

#def loadSpectrum(classLabelReal,EDSData,CodePath):
#    randFloat = random.random();
#    randNum = random.randint(0, EDSData.NumSpectra-1);
#    if classLabelReal==255: #or classLabelReal==1:
#        phase = 1
#        filename = CodePath + 'ResultsAndData' + os.path.sep + 'EDSSpectra' + os.path.sep + EDSData.Folder +  os.path.sep + 'Phase_' + str(phase) + os.path.sep + 'EDSvalid1.npy'
#    else:
#        phase = 2
#        filename = CodePath + 'ResultsAndData' + os.path.sep + 'EDSSpectra' + os.path.sep + EDSData.Folder +  os.path.sep + 'Phase_' + str(phase) + os.path.sep + 'EDSvalid2.npy'
#    vec = np.load(filename)
#    r = vec[randNum, :]
#    if randFloat>EDSData.ErrorSpectrumProb:
#        if EDSData.NoiseType == 'P':
#            s = r + np.random.poisson(EDSData.Noiselambda, len(r))
#    else:
#        s = np.random.poisson(EDSData.Noiselambda, len(r))
#        phase=0.5
##    print(phase)
#    return s,phase

#global classifySpectrum
def classifySpectrum(s,CodePath,EDSData, tfclf):
#    global sess
#    global W_conv1, W_conv2, W_fc1, W_fc2, W_fc3, W_fco, b_conv1, b_conv2, b_fc1, b_fc2, b_fc3, b_fco
#    global x_image, x, y_, h_conv1, h_conv2, h_pool1, h_pool2, size_hp, h_flat, h_fc1, h_fc2, h_fc3, keep_prob, h_fc1_drop, y_conv

#    filename = CodePath + 'ResultsAndData' + os.path.sep + 'EDSSpectra' + os.path.sep + 'NdFeB' +  os.path.sep + 'tfClfTrain' + os.path.sep
#    filename = CodePath + 'ResultsAndData' + os.path.sep + 'EDSSpectra' + os.path.sep + 'NdFeB' +  os.path.sep
#    sys.path.append(filename)
#    from EDStfClfLoader import tfClassify
#    classLabel = tfClassify(s)

    sess = tfclf.sess
    W_conv1 = tfclf.W_conv1
    W_conv2 = tfclf.W_conv2
    W_fc1 = tfclf.W_fc1
    W_fc2 = tfclf.W_fc2
    W_fc3 = tfclf.W_fc3
    W_fco = tfclf.W_fco
#    b_conv1 = tfclf.b_conv1
#    b_conv2 = tfclf.b_conv2
    x_image = tfclf.x_image
    x = tfclf.x
    y_ = tfclf.y_
    h_conv1 = tfclf.h_conv1
    h_conv2 = tfclf.h_conv2
    h_pool1 = tfclf.h_pool1
    h_pool2 = tfclf.h_pool2
    size_hp = tfclf.size_hp
    h_flat = tfclf.h_flat
    h_fc1 = tfclf.h_fc1
    h_fc2 = tfclf.h_fc2
    h_fc3 = tfclf.h_fc3
    keep_prob = tfclf.keep_prob
    h_fc1_drop = tfclf.h_fc1_drop
    y_conv = tfclf.y_conv

    classLabel = np.argmax(sess.run(y_conv, feed_dict={x: s.reshape((1,2040)), y_: np.zeros((1,2)), keep_prob: 1.0}))

#    import _pickle as cPickle
#    import cPickle
#    with open(filename + 'Classifier.pkl', 'rb') as fid:
#        clf = cPickle.load(fid)
#    classLabel = clf.predict(s)
#    print(classLabel)
    return classLabel+1

def regressSpectrum(s,CodePath,EDSData):
#    filename = CodePath + 'ResultsAndData' + os.path.sep + 'EDSSpectra' + os.path.sep + 'NdFeB' +  os.path.sep + 'tfRegTrain' + os.path.sep
    filename = CodePath + 'ResultsAndData' + os.path.sep + 'EDSSpectra' + os.path.sep + EDSData.Folder +  os.path.sep + 'skRegTrain' + os.path.sep
    #filename = '/home/yzhang/Research-ANL/1_Projects/SLADS-Python-v4-EDS_ClfReg_20170407/training_nn/skRegTrain/'
#    sys.path.append(filename)
#    from EDStfRegLoader import tfRegress
#    regressValue = tfRegress(s)

#    import _pickle as cPickle
    import cPickle
    with open(filename + 'Regressor.pkl', 'rb') as fid:
        reg = cPickle.load(fid)
    regressValue = reg.predict(s)
    return regressValue

max_stars_repo_name: cphatak/SLADS-Net

max_issues_repo_name: cphatak/SLADS-Net

max_forks_repo_name: cphatak/SLADS-Net


text:
# -*- coding: utf-8 -*-
import numpy as np
from numpy import testing

from sktime.classification.hybrid._catch22_forest_classifier import (
    Catch22ForestClassifier,
)
from sktime.datasets import load_gunpoint, load_basic_motions

def test_catch22_forest_classifier_on_gunpoint():
    # load gunpoint data
    X_train, y_train = load_gunpoint(split="train", return_X_y=True)
    X_test, y_test = load_gunpoint(split="test", return_X_y=True)
    indices = np.random.RandomState(0).permutation(10)

    # train c22f
    c22f = Catch22ForestClassifier(random_state=0)
    c22f.fit(X_train.iloc[indices], y_train[indices])

    # assert probabilities are the same
    probas = c22f.predict_proba(X_test.iloc[indices])
    testing.assert_array_equal(probas, catch22_forest_classifier_gunpoint_probas)

def test_catch22_forest_classifier_on_basic_motions():
    # load basic motions data
    X_train, y_train = load_basic_motions(split="train", return_X_y=True)
    X_test, y_test = load_basic_motions(split="test", return_X_y=True)
    indices = np.random.RandomState(0).permutation(20)

    # train c22f
    c22f = Catch22ForestClassifier(random_state=0)
    c22f.fit(X_train.iloc[indices], y_train[indices])

    # assert probabilities are the same
    probas = c22f.predict_proba(X_test.iloc[indices])
    testing.assert_array_equal(probas, catch22_forest_classifier_basic_motions_probas)

catch22_forest_classifier_gunpoint_probas = np.array(
    [
        [
            0.075,
            0.925,
        ],
        [
            0.45,
            0.55,
        ],
        [
            0.35,
            0.65,
        ],
        [
            0.16,
            0.84,
        ],
        [
            0.11,
            0.89,
        ],
        [
            0.855,
            0.145,
        ],
        [
            0.08,
            0.92,
        ],
        [
            0.7,
            0.3,
        ],
        [
            0.65,
            0.35,
        ],
        [
            0.085,
            0.915,
        ],
    ]
)
catch22_forest_classifier_basic_motions_probas = np.array(
    [
        [
            1.0,
            0.0,
        ],
        [
            0.135,
            0.865,
        ],
        [
            0.845,
            0.155,
        ],
        [
            0.005,
            0.995,
        ],
        [
            0.99,
            0.01,
        ],
        [
            1.0,
            0.0,
        ],
        [
            0.185,
            0.815,
        ],
        [
            0.99,
            0.01,
        ],
        [
            0.015,
            0.985,
        ],
        [
            0.045,
            0.955,
        ],
        [
            0.015,
            0.985,
        ],
        [
            0.985,
            0.015,
        ],
        [
            0.01,
            0.99,
        ],
        [
            0.01,
            0.99,
        ],
        [
            0.875,
            0.125,
        ],
        [
            0.995,
            0.005,
        ],
        [
            0.055,
            0.945,
        ],
        [
            0.24,
            0.76,
        ],
        [
            0.88,
            0.12,
        ],
        [
            1.0,
            0.0,
        ],
    ]
)

# def print_array(array):
#     print('[')
#     for sub_array in array:
#         print('[')
#         for value in sub_array:
#             print(value.astype(str), end='')
#             print(', ')
#         print('],')
#     print(']')
#
# if __name__ == "__main__":
#     X_train, y_train = load_gunpoint(split="train", return_X_y=True)
#     X_test, y_test = load_gunpoint(split="test", return_X_y=True)
#     indices = np.random.RandomState(0).permutation(10)
#
#     c22f_u = Catch22ForestClassifier(random_state=0)
#
#     c22f_u.fit(X_train.iloc[indices], y_train[indices])
#     probas = c22f_u.predict_proba(X_test.iloc[indices])
#     print_array(probas)
#
#     X_train, y_train = load_basic_motions(split="train", return_X_y=True)
#     X_test, y_test = load_basic_motions(split="test", return_X_y=True)
#     indices = np.random.RandomState(0).permutation(20)
#
#     c22f_m = Catch22ForestClassifier(random_state=0)
#
#     c22f_m.fit(X_train.iloc[indices], y_train[indices])
#     probas = c22f_m.predict_proba(X_test.iloc[indices])
#     print_array(probas)

max_stars_repo_name: ltoniazzi/sktime

max_issues_repo_name: ltoniazzi/sktime

max_forks_repo_name: ltoniazzi/sktime

reason: import numpy,from numpy


text:
'''
    R-matrix model
    Analyzing 3He(alpha, gamma) data
        * capture
        * scattering (SONIK)
'''

import numpy as np
from scipy import stats
from brick.azr import AZR

import constants as const

input_filename = __name__ + '.azr'
azr = AZR(input_filename)
azr.ext_capture_file = 'output/intEC.dat'
azr.root_directory = '/tmp/'

output_filenames = [
    'AZUREOut_aa=1_R=1.out',
    'AZUREOut_aa=1_R=2.out',
    'AZUREOut_aa=1_R=3.out',
    'AZUREOut_aa=1_TOTAL_CAPTURE.out'
]

# Capture Data
# We will apply normalization factors to the capture data. That data is returned
# in a single column, so we need to know what order they are in and how many
# points are in each set.
capture_data_files = [
    'Seattle_XS.dat',
    'Weizmann_XS.dat',
    'LUNA_XS.dat',
    'ERNA_XS.dat',
    'ND_XS.dat',
    'ATOMKI_XS.dat',
]
num_pts_capture = [
    np.size(np.loadtxt('data/'+f)[:, 0]) for f in capture_data_files
]
num_pts_total_capture = sum(num_pts_capture)

# Scattering Data
scatter_data_files = [
    'sonik_inflated_239.dat',
    'sonik_inflated_291.dat',
    'sonik_inflated_432.dat',
    'sonik_inflated_586.dat',
    'sonik_inflated_711.dat',
    'sonik_inflated_873_1.dat',
    'sonik_inflated_873_2.dat',
    'sonik_inflated_1196.dat',
    'sonik_inflated_1441.dat',
    'sonik_inflated_1820.dat'
]
num_pts_scatter = [np.size(np.loadtxt('data/'+f)[:, 0]) for f in scatter_data_files]
num_pts_total_scatter = sum(num_pts_scatter)

# Center-of-Mass (COM) Data
xs1_data = np.loadtxt('output/'+output_filenames[0]) # scattering data
xs2_data = np.loadtxt('output/'+output_filenames[1]) # capture XS to GS
xs3_data = np.loadtxt('output/'+output_filenames[2]) # capture XS to ES
xs_data = np.loadtxt('output/'+output_filenames[3]) # total capture

# COM Branching Ratio Data
# James sent me this setup. He used a modified version of AZURE2 that computes
# the branching ratio. So the data values in xs2_data and xs3_data both
# correspond to the measured data. No need to divide xs3 by xs2.
branching_ratio = xs2_data[:, 5]
branching_ratio_err = xs2_data[:, 6]

## COM Scattering
def rutherford(energy, angle):
    '''
    Rutherford differential cross section
    '''
    return (4*const.ALPHA*const.HBARC / (4*energy*np.sin(angle/2*np.pi/180)**2))**2 / 100

energies_scatter = xs1_data[:, 0]
angles_scatter = xs1_data[:, 2]
scatter_rutherford = np.array(
    [rutherford(ei, ai) for (ei, ai) in zip(energies_scatter, angles_scatter)]
)
scatter = xs1_data[:, 5] / scatter_rutherford
# FIXED RELATIVE EXTRINSIC UNCERTAINTY ADDED IN "sonik_inflated_*.dat" FILES
scatter_err = xs1_data[:, 6] / scatter_rutherford

# All of the energies (COM).
x = np.hstack((xs2_data[:, 0], xs_data[:, 0], xs1_data[:, 0]))
angles = np.hstack((xs2_data[:, 2], xs_data[:, 2], xs1_data[:, 2]))
# All of the observables: branching ratios, total capture S factor, differential
# cross sections.
y = np.hstack((branching_ratio, xs_data[:, 7], scatter))
# All of the associated uncertainties reported with the data.
dy = np.hstack((branching_ratio_err, xs_data[:, 8], scatter_err))

nbr = xs2_data.shape[0]
nxs = xs_data.shape[0]

assert num_pts_total_scatter == xs1_data.shape[0], f'''
Number of scattering ({num_pts_total_scatter}, {xs1_data.shape[0]}) data points is inconsistent.
'''

nrpar = len(azr.parameters)
nf_capture = len(capture_data_files)
nf_scatter = len(scatter_data_files)
ndim = nrpar + nf_capture + nf_scatter

def map_uncertainty(theta, ns):
    return np.hstack([theta_i*np.ones(n) for (theta_i, n) in zip(theta, ns)])

# Calculate the branching ratios, total capture cross sections, and scattering
# differential cross sections at point theta.
def calculate(theta):
    paneru, capture_gs, capture_es, capture_tot = azr.predict(theta)
    bratio = capture_es.xs_com_fit/capture_gs.xs_com_fit
    S_tot = capture_tot.sf_com_fit
    scatter_dxs = paneru.xs_com_fit / scatter_rutherford
    return np.hstack((bratio, S_tot, scatter_dxs))

def calculate_norm(theta):
    '''
    Apply normalization factors to theory predictions (total capture and
    scattering).
    '''
    fj_capture = map_uncertainty(
        theta[nrpar:nrpar+nf_capture], num_pts_capture
    )
    fj_scatter = map_uncertainty(
        theta[nrpar+nf_capture:nrpar+nf_capture+nf_scatter], num_pts_scatter
    )

    mu = azr.predict(theta)
    paneru, capture_gs, capture_es, capture_tot = mu

    bratio = capture_es.xs_com_fit/capture_gs.xs_com_fit
    S_tot = capture_tot.sf_com_fit
    scatter_dxs = paneru.xs_com_fit /scatter_rutherford
    data_norm = np.hstack((fj_capture*S_tot, fj_scatter*scatter_dxs))
    return np.hstack((bratio, data_norm))

# starting position distributions
p0_dist = [stats.norm(sp, np.abs(sp)/100) for sp in
        azr.config.get_input_values()]

max_stars_repo_name: odell/r-matrix-emulation

max_issues_repo_name: odell/r-matrix-emulation

max_forks_repo_name: odell/r-matrix-emulation

reason: import numpy,from scipy


text:
import math
import numpy as np
import matplotlib.pyplot as plt

from var import *

from layers import *
from conv import *

#########################

class Block1(Layer):
    def __init__(self, input_size, filter_size, stride, params, weights):
        self.layer_id = Layer.layer_id
        Layer.layer_id += 1

        self.input_size = input_size
        self.f1, self.f2 = filter_size
        self.s = stride

        input_size1 = input_size
        input_size2 = (input_size1[0] // stride, input_size1[1] // stride, input_size1[2])

        self.conv1 = Conv(input_size1, (3,3,self.f1,self.f2), 1, stride, pad1=1, pad2=1, params=params, weights=weights)
        self.conv2 = Conv(input_size2, (3,3,self.f2,self.f2), 1, 1,      pad1=1, pad2=1, params=params, weights=weights, relu_flag=False)

    def init(self, params):
        self.params = params.copy()
        self.conv1.init(self.params)
        self.conv2.init(self.params)

    def set_profile_adc(self, counts):
        self.conv1.set_profile_adc(counts)
        self.conv2.set_profile_adc(counts)

    def profile_adc(self, x):
        y1, r1 = self.conv1.profile_adc(x)
        y2, r2 = self.conv2.profile_adc(y1)
        y3 = relu(x + y2)

        result = {}
        result.update(r1)
        result.update(r2)
        return y3, result

    def forward(self, x, profile=False):
        y1, r1 = self.conv1.forward(x, profile=profile)
        y2, r2 = self.conv2.forward(y1, profile=profile)
        y3 = relu(x + y2)

        result = []
        result.extend(r1)
        result.extend(r2)
        return y3, result

    def set_block_alloc(self, block_alloc):
        pass

    def set_layer_alloc(self, layer_alloc):
        pass

    def weights(self):
        return [self.conv1, self.conv2]

#############

class Block2(Layer):
    def __init__(self, input_size, filter_size, stride, params, weights):
        self.layer_id = Layer.layer_id
        Layer.layer_id += 1

        self.input_size = input_size
        self.f1, self.f2 = filter_size
        self.s = stride

        assert (self.f1 == input_size[2])
        input_size1 = input_size
        input_size2 = (input_size1[0] // stride, input_size1[1] // stride, self.f2)

        self.conv1 = Conv(input_size1, (3,3,self.f1,self.f2), 1, stride, pad1=1, pad2=1, params=params, weights=weights)
        self.conv2 = Conv(input_size2, (3,3,self.f2,self.f2), 1, 1,      pad1=1, pad2=1, params=params, weights=weights, relu_flag=False)
        self.conv3 = Conv(input_size1, (1,1,self.f1,self.f2), 1, stride, pad1=0, pad2=0, params=params, weights=weights, relu_flag=False)

    def init(self, params):
        self.params = params.copy()
        self.conv1.init(self.params)
        self.conv2.init(self.params)
        self.conv3.init(self.params)

    def set_profile_adc(self, counts):
        self.conv1.set_profile_adc(counts)
        self.conv2.set_profile_adc(counts)
        self.conv3.set_profile_adc(counts)

    def profile_adc(self, x):
        y1, r1 = self.conv1.profile_adc(x)
        y2, r2 = self.conv2.profile_adc(y1)
        y3, r3 = self.conv3.profile_adc(x)
        y4 = relu(y2 + y3)

        result = {}
        result.update(r1)
        result.update(r2)
        result.update(r3)
        return y4, result

    def forward(self, x, profile=False):
        y1, r1 = self.conv1.forward(x, profile=profile)
        y2, r2 = self.conv2.forward(y1, profile=profile)
        y3, r3 = self.conv3.forward(x, profile=profile)
        y4 = relu(y2 + y3)

        result = []
        result.extend(r1)
        result.extend(r2)
        result.extend(r3)
        return y4, result

    def set_block_alloc(self, block_alloc):
        pass

    def set_layer_alloc(self, layer_alloc):
        pass

    def weights(self):
        return [self.conv1, self.conv2, self.conv3]

#############


text:
#!/usr/bin/env python
# by TR

from numpy.random import seed, random
from scipy.signal import correlate
from sito import read
from sito.xcorr import xcorrf, acorrf, xcorrt, acorrt, xcorr_obspy#, use_fftw3
import numpy as np
import os
import unittest
from matplotlib.mlab import psd

class TestCase(unittest.TestCase):
    def setUp(self):
        self.stream = read(os.path.join(os.path.dirname(__file__), 'data', 'PKD_1997_246.mseed'))
        self.stream_down = self.stream.copy()
        self.stream_down.downsample2(1)
        N = self.N = 1001
        self.data1 = np.sin(np.arange(N) / 100.)
        self.data1_dem = self.data1.copy()
        self.data1_dem[:] = self.data1_dem[:] - np.mean(self.data1_dem)
        self.data2 = np.e ** (-(np.arange(N) - 500) ** 2 / 100.) - np.e ** (-(np.arange(N) - 50) ** 2 / 100.) + 5 * np.e ** (-(np.arange(N) - 950) ** 2 / 100.)
        self.data2_dem = self.data2.copy()
        self.data2_dem[:] = self.data2_dem[:] - np.mean(self.data2_dem)

    def test_xcorr(self):
        cor1 = correlate(self.data1, self.data2, 'same')
        cor2 = xcorrf(self.data1.copy(), self.data2.copy(), self.N // 2, demean=False, normalize=False)
#        from pylab import plot, show, subplot
#        subplot(211)
#        plot(data1)
#        plot(data2)
#        subplot(212)
#        plot(cor1)
#        plot(cor2)
#        show()
        np.testing.assert_array_almost_equal(cor1, cor2)

    def test_acorr(self):
        cor1 = xcorrf(self.data1, self.data1, self.N // 4, oneside=True)
        cor2 = acorrf(self.data1, self.N // 4)
#        from pylab import plot, show
#        plot(cor1)
#        plot(cor2)
#        show()
        np.testing.assert_array_almost_equal(cor1, cor2, 3)
        self.assertTrue((cor2[0] - 1) ** 2 < 1e-5)

    def test_xcorr_realdata(self):
        stream = self.stream.copy()
        data = stream.select(component='Z')[0].data
        shift = 100
        cor1 = xcorrf(data, data[shift:-shift], shift)
        cor2 = xcorrt(data, data[shift:-shift], shift)
#        from pylab import plot, show
#        plot(cor1)
#        plot(cor2)
#        show()
        np.testing.assert_array_almost_equal(cor1, cor2)

    def test_time_norm(self):
        stream = self.stream_down.copy()
        stream = stream.select(component='Z')
        stream.timeNorm('1bit')
        self.assertTrue(np.sum(np.abs(stream[0].data)) == stream[0].stats.npts)

    def test_spectral_whitening(self):
        stream = self.stream.copy()
        stream.downsample2(1)
        stream = stream.select(component='Z')
        stream.filter2(0.005, 0.2)
        nfft = stream[0].stats.npts
        pxx1, freqs1 = psd(stream[0].data, Fs=stream[0].stats.sampling_rate, NFFT=nfft,
                           scale_by_freq=True)
        stream0 = stream.copy()
        stream2 = stream.copy()
        stream.spectralWhitening(0.001)
        stream.filter2(0.005, 0.2)
#        use_fftw3()
#        stream2.spectralWhitening(0.001)
#        use_fftw3(False)
        stream2.filter2(0.005, 0.2)

        pxx2, freqs2 = psd(stream[0].data, Fs=stream[0].stats.sampling_rate, NFFT=nfft,
                           scale_by_freq=True)
        stream0, pxx1, pxx2, freqs1, freqs2
#        from pylab import plot, show, subplot, legend
#        subplot(311)
#        plot(stream0[0].data / max(stream0[0].data) / 20)
#        plot(stream[0].data)
#        plot(stream2[0].data)
#        ax = subplot(312)
##        ax.set_xscale('log')
##        ax.set_yscale('log')
#        plot(freqs1, pxx1)
#        ax2 = subplot(313, sharex=ax)
##        ax.set_xscale('log')
##        ax2.set_yscale('log')
#        plot(freqs2, pxx2)
#        show()

#        np.testing.assert_array_almost_equal(stream[0].data, stream2[0].data)

########## these tests can be remove later

    def test_xcorr_obspy(self):
        N = 1001
        data1 = np.sin(np.arange(N) / 100.)
        data2 = np.e ** (-(np.arange(N) - 500) ** 2 / 100.) - np.e ** (-(np.arange(N) - 50) ** 2 / 100.) + 5 * np.e ** (-(np.arange(N) - 950) ** 2 / 100.)
        data1[:] = data1[:] - np.mean(data1)
        data2[:] = data2[:] - np.mean(data2)
        cor1 = correlate(data1, data2, 'same')
        cor2 = xcorr_obspy(data1, data2, N // 2)
        #cor3 = xcorr_obspy(data1, data1, N)
        cor1 *= max(cor2) / max(cor1)
#        from pylab import plot, show, subplot
#        subplot(211)
#        plot(data1)
#        plot(data2)
#        subplot(212)
#        plot(cor1)
#        plot(cor2)
#        plot(cor3)
#        show()
#        np.testing.assert_array_almost_equal(cor1, cor2,1)

    def test_xcorr_xcorrt(self):
        N = 1001
        data1 = np.sin(np.arange(N // 2 + 1) / 100.)
        data2 = np.e ** (-(np.arange(N) - 500) ** 2 / 100.) - np.e ** (-(np.arange(N) - 50) ** 2 / 100.) + 5 * np.e ** (-(np.arange(N) - 950) ** 2 / 100.)
        cor1 = correlate(data1 - np.mean(data1), data2 - np.mean(data2), 'full')
        cor2 = xcorrt(data2, data1, 750, window=N, ndat2d=N // 2 + 1)[::-1]
        cor3 = xcorrt(data1, data2, 750, window=N, ndat1d=N // 2 + 1) #@UnusedVariable
        cor3b = xcorrt(data1, data2, 750, window=0) #@UnusedVariable
        cor3c = xcorrt(data2, data1, 750, window=0)[::-1] #@UnusedVariable
        cor1 *= max(cor2) / max(cor1)

        cor4 = correlate(data2, data1, 'full')
        cor5 = xcorrt(data2, data1, 750, demean=False)
        cor5b = xcorrt(data2, data1, 750, shift_zero= -100, demean=False)
        cor5c = xcorrt(data2, data1, 750, shift_zero=100, demean=False)
        cor4 *= max(cor5) / max(cor4)

        cor7 = correlate(data1, data2, 'full')
        cor8 = xcorrt(data1, data2, 750, demean=False, normalize=False)
        cor10 = xcorrt(data1, data2, 750, oneside=True, demean=False, normalize=False) #@UnusedVariable

#        from pylab import plot, show, subplot, legend
#        subplot(411)
#        plot(data1)
#        plot(data2)
#        subplot(412)
#        plot(cor1, label='scipy.signal all demeaned and normalized')
#        plot(cor2, label='xcorrt ndat1 > ndat2 ndatxd')
#        plot(cor3, label='xcorrt ndat1 < ndat2 ndatxd')
#        plot(cor3b, label='xcorrt ndat1 > ndat2 window = 0')
#        plot(cor3c, label='xcorrt ndat1 < ndat2 window = 0')
#        legend()
#        subplot(413)
#        plot(cor4, label='scipy.signal all normalized')
#        plot(cor5, label='xcorrt')
#        plot(cor5b, label='xcorrt shifted -100')
#        plot(cor5c, label='xcorrt shifted 100')
#        legend()
#        subplot(414)
#        plot(cor7, label='scipy.signal')
#        plot(cor8, label='xcorrt')
#        plot(cor10, label='xcorrt oneside=True')
#        legend()
#        show()
        np.testing.assert_array_almost_equal(cor1, cor2)
        np.testing.assert_array_almost_equal(cor4, cor5)
        np.testing.assert_array_almost_equal(cor7, cor8)
        np.testing.assert_array_almost_equal(cor5[200:300], cor5b[100:200])
        np.testing.assert_array_almost_equal(cor5[200:300], cor5c[300:400])

    def test_xcorr_acorrt(self):
        data1 = np.sin(np.arange(1001) * 2 * np.pi / 500.)
        cor1 = xcorrt(data1, data1, 1000)
        cor2 = acorrt(data1, 1000, oneside=False, clipdata=False)
#        from pylab import plot, show, subplot, legend
#        subplot(211)
#        plot(data1)
#        subplot(212)
#        plot(cor1, label='xcorrt')
#        plot(cor2, label='acorr clip=False')
#        legend()
#        show()
        np.testing.assert_array_almost_equal(cor1, cor2)

    def test_xcorr_xcorrf(self):
        N = 1001
        data1 = np.sin(np.arange(N // 2 + 1) / 100.)
        data2 = np.e ** (-(np.arange(N) - 500) ** 2 / 100.) - np.e ** (-(np.arange(N) - 50) ** 2 / 100.) + 5 * np.e ** (-(np.arange(N) - 950) ** 2 / 100.)
        cor1 = xcorrt(data1, data2, 750)
        cor2 = xcorrf(data1, data2, 750)
        cor3 = xcorrf(data2, data1, 750)[::-1]
        cor4 = xcorrt(data1, data2, 750, demean=False, normalize=False)
        cor5 = xcorrf(data1, data2, 750, demean=False, normalize=False)
        cor6 = xcorrt(data1, data2, 750, shift_zero=100)
        cor7 = xcorrf(data1, data2, 750, shift_zero=100)
        cor8 = xcorrt(data1, data2, 750, window=200)
        cor9 = xcorrf(data1, data2, 750, window=200)
#        from pylab import plot, show, subplot, legend
#        subplot(411)
#        plot(data1)
#        plot(data2)
#        subplot(412)
#        plot(cor1, label='xcorrt')
#        plot(cor2, label='xcorrf ndat1 > ndat2')
#        plot(cor3, label='xcorrf ndat2 > ndat1')
#        legend()
#        subplot(413)
#        plot(cor4, label='xcorrt all not demeaned not normalized')
#        plot(cor5, label='xcorrf')
#        legend()
#        subplot(414)
#        plot(cor6, label='xcorrt shift')
#        plot(cor7, label='xcorrf shift')
#        plot(cor8, label='xcorrt window')
#        plot(cor9, label='xcorrf window')
#        legend()
#        show()
        np.testing.assert_array_almost_equal(cor1, cor2)
        np.testing.assert_array_almost_equal(cor1, cor3)
        np.testing.assert_array_almost_equal(cor4, cor5, 5)
        np.testing.assert_array_almost_equal(cor6, cor7)
        np.testing.assert_array_almost_equal(cor8, cor9)

    def test_xcorr_acorrf(self):
        data1 = np.sin(np.arange(1001) * 2.1 * np.pi / 500.)
        cor1 = acorrt(data1, 1024, oneside=True, clipdata=False)
        cor2 = acorrf(data1, 1024, oneside=True, clipdata=False)
#        from pylab import plot, show, subplot, legend
#        subplot(211)
#        plot(data1)
#        subplot(212)
#        plot(cor1, label='acorrt')
#        plot(cor2, label='acorrf')
#        legend()
#        show()
        print (np.sum((cor1 - cor2) ** 2) / len(cor1)) ** 0.5
        self.assertTrue((np.sum((cor1 - cor2) ** 2) / len(cor1)) ** 0.5 < 0.1)
        #np.testing.assert_array_almost_equal(cor1, cor2, 4)

    def test_xcorr_all_again(self):
        seed(42)
        data1 = random(1200) * np.sin(np.arange(1200)) * np.arange(1200) - 0.5
        data2 = random(1000) * np.sin(np.arange(1000) * 0.4) * 500 - 0.5
        data1 -= np.mean(data1[100:1100])
        data2 -= np.mean(data2)
        data2_pad = np.hstack((np.zeros(100), data2, np.zeros(100)))
        cor1a = correlate(data1, data2, 'valid')
        cor1a2 = xcorr_obspy(data1, data2_pad, 100)
        cor1b = xcorrt(data1, data2, 100)
        cor1c = xcorrf(data1, data2, 100)
        cor1a *= max(cor1b) / max(cor1a)
        cor1a2 *= max(cor1b) / max(cor1a2)
        cor3 = xcorrf(data2, data1, 100)[::-1] #@UnusedVariable
        cor4 = xcorrt(data1, data2, 100, demean=False, normalize=False)
        cor5 = xcorrf(data1, data2, 100, demean=False, normalize=False)
        val = max (cor4)
        cor4 /= val
        cor5 /= val
        cor6 = xcorrt(data1, data2, 100, shift_zero=100)
        cor7 = xcorrf(data1, data2, 100, shift_zero=100)
        cor8 = xcorrt(data1, data2, 500, window=200)
        cor9 = xcorrf(data1, data2, 500, window=200)

#        from pylab import plot, show, subplot, legend
#        subplot(411)
#        plot(data1)
#        plot(data2)
#        subplot(412)
#        plot(cor1a, label='convolve')
#        plot(cor1b, label='xcorrt')
#        plot(cor1c, label='xcorrf')
#        plot(cor1a2, label='xcorr_obspy')
#        plot(cor6, label='xcorrt shift')
#        plot(cor7, label='xcorrf shift')
#        legend()
#        subplot(413)
#        plot(cor4, label='xcorrt all not demeaned not normalized')
#        plot(cor5, label='xcorrf')
#        legend()
#        subplot(414)
#        plot(cor8, label='xcorrt window')
#        plot(cor9, label='xcorrf window')
#        legend()
#        show()
        np.testing.assert_array_almost_equal(cor1a, cor1b)
        np.testing.assert_array_almost_equal(cor1a, cor1a2)
        np.testing.assert_array_almost_equal(cor1b, cor1c)
        np.testing.assert_array_almost_equal(cor1c[20:30], cor7[120:130])
        np.testing.assert_array_almost_equal(cor4, cor5)
        np.testing.assert_array_almost_equal(cor6, cor7)
        np.testing.assert_array_almost_equal(cor8, cor9)

def suite():
    return unittest.makeSuite(TestCase, 'test')

if __name__ == '__main__':
    unittest.main(defaultTest='suite')

max_stars_repo_name: trichter/sito

max_issues_repo_name: trichter/sito

max_forks_repo_name: trichter/sito

reason: import numpy,from numpy,from scipy


text:
# Copyright 2018 The Cornac Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

import random

import numpy as np
import torch

"""Firstly, we define a helper function to generate\sample training ordinal triplets:
   Step 1:
   given rated item i, randomly choose item j and check whether rating of j is lower than i,
   if not randomly sample another item.
   each row of the sampled data in the following form:
        [userId itemId_i itemId_j]
   for each user u, he/she prefers item i over item j.
"""

def sample_triplet(X, batch_size):
    sampled_data = np.zeros((batch_size, 3), dtype=np.int)

    count = 0
    while count < batch_size:
        u = random.randint(0, X.shape[0] - 1)
        u_row = X.getrow(u)
        _, u_nz = u_row.nonzero()
        min_rating = u_row[:, u_nz].todense().min()

        i = u_nz[random.randint(0, len(u_nz) - 1)]
        ratingi = u_row[:, i]

        if ratingi > min_rating:
            j = u_nz[random.randint(0, len(u_nz) - 1)]

            while u_row[:, j] >= ratingi:
                j = u_nz[random.randint(0, len(u_nz) - 1)]

            sampled_data[count, :] = [u, i, j]
            count += 1

    print("Done sampling")
    return sampled_data

def coe(X, k, lamda=0.05, n_epochs=150, learning_rate=0.001, batch_size=1000, init_params=None):
    # Data = Dataset(data)

    # Initial user factors
    if init_params['U'] is None:
        U = torch.randn(X.shape[0], k, requires_grad=True, device="cuda")
    else:
        U = init_params['U']
        U = torch.from_numpy(U)

    # Initial item factors
    if init_params['V'] is None:
        V = torch.randn(X.shape[1], k, requires_grad=True, device="cuda")
    else:
        V = init_params['V']
        V = torch.from_numpy(V)

    optimizer = torch.optim.Adam([U, V], lr=learning_rate)
    for epoch in range(n_epochs):
        #        num_steps = int(Data.data.shape[0]/batch_size)

        #       for i in range(1, num_steps + 1):
        #          batch_c,_ = Data.next_batch(batch_size)
        sampled_batch = sample_triplet(X, batch_size)

        regU = U[sampled_batch[:, 0], :]
        regI = V[sampled_batch[:, 1], :]
        regJ = V[sampled_batch[:, 2], :]

        regU_unq = U[np.unique(sampled_batch[:, 0]), :]
        regI_unq = V[np.unique(sampled_batch[:, 1:]), :]

        Scorei = torch.norm(regU - regI, dim=1)
        Scorej = torch.norm(regU - regJ, dim=1)

        loss = lamda * (regU_unq.norm().pow(2) + regI_unq.norm().pow(2)) - torch.log(
            torch.sigmoid(Scorej - Scorei)).sum()
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        print('epoch:', epoch, 'loss:', loss)

    U = U.data.cpu().numpy()
    V = V.data.cpu().numpy()

    res = {'U': U, 'V': V}

    return res


text:
import random
from collections import namedtuple

# Data
import cv2
import numpy as np
from path import Path

Sample = namedtuple('Sample', 'generated_text, file_path')
Batch = namedtuple('Batch', 'imgs, generated_texts, batch_size')

class LoadIAM:
    def __init__(self, data_dir: Path, batch_size: int,
                 data_split: float=0.95):
        assert(data_dir.exists())

        self.data_augmentation = False
        self.cur_index = 0
        self.batch_size = batch_size
        self.samples = []

        f = open(data_dir / 'gt/words.txt')
        chars = set()
        # filter bad images
        bad_images = ['a01-117-05-02', 'r06-022-03-05']
        for line in f:
            if not line or line[0] == '#':
                continue

            line_split = line.strip().split(' ')
            assert len(line_split) >= 9

            file_name_split = line_split[0].split('-')
            file_name_subdirectory1 = file_name_split[0]
            file_name_subdirectory2 = f'{file_name_split[0]}-{file_name_split[1]}'
            file_base_name = line_split[0] + '.png'
            file_name = data_dir / 'img' / file_name_subdirectory1 / file_name_subdirectory2 / file_base_name

            if line_split[0] in bad_images:
                print('Ignore image', file_name)
                continue

            # generated text
            generated_text = ' '.join(line_split[8:])
            chars = chars.union(set(list(generated_text)))

            # add sample to list
            self.samples.append(Sample(generated_text, file_name))

        # split train and validate sets, start with 10/90
        split_index = int(data_split * len(self.samples))
        self.train_samples = self.samples[:split_index]
        self.validation_samples = self.samples[split_index:]

        # put words to lists
        self.train_words = [x.generated_text for x in self.train_samples]
        self.validation_words = [x.generated_text for x in self.validation_samples]

        self.train_set()

        self.char_list = sorted(list(chars))

    def train_set(self):
        self.data_augmentation = True
        self.current_index = 0
        random.shuffle(self.train_samples)
        self.samples = self.train_samples
        self.current_set = 'train'

    def has_next(self):
        if self.current_set == 'train':
            return self.current_index + self.batch_size <= len(self.samples)
        else:
            return self.current_index < len(self.samples)

    def get_image(self, i):
        image = cv2.imread(self.samples[i].file_path, cv2.IMREAD_GRAYSCALE)
        return image

    def get_next(self):
        batch_range = range(self.current_index, min(self.current_index + self.batch_size, len(self.samples)))

        imgs = [self.get_image(i) for i in batch_range]
        gt_texts = [self.samples[i].generated_text for i in batch_range]

        self.current_index += self.batch_size
        return Batch(imgs, gt_texts, len(imgs))

    def get_it_info(self):
        if self.current_set == 'train':
            n_batches = int(np.floor(len(self.samples) / self.batch_size))
        else:
            n_batches = int(np.ceil(len(self.samples) / self.batch_size))
        current_batch = self.current_index // self.batch_size + 1
        return current_batch, n_batches

    def switch_to_validate(self):
        self.data_augmentation = False
        self.current_index = 0
        self.samples = self.validation_samples
        self.current_set = 'val'

max_stars_repo_name: dpayba/Handwritten-Plagiarism-Scanner

max_issues_repo_name: dpayba/Handwritten-Plagiarism-Scanner

max_forks_repo_name: dpayba/Handwritten-Plagiarism-Scanner