text:
import os
import hashlib
import pathlib
import yaml

def get_exp_name():
    """Return current experiment folder, such as exp0."""
    _full_path = str(pathlib.Path().absolute())
    _paths = _full_path.split('/')
    assert _paths[-1] == "src", f"Project structure has been changed. utils.get_exp_folder() should be changed accordingly.\n{_paths}"
    assert len(_paths) > 2, "Why path is so short?"
    _folder_name = _paths[-2]
    return _folder_name

def get_output_data_folder(init=False):
    # Create output folder is not exist yet.
    _path = pathlib.Path("../../../output_data")
    if not _path.is_dir():
        print("Starting a new project? Congratulations! \n\nCreating output data path for the first time.")
        print(f"mkdir {_path.resolve()}")
        _path.mkdir()
    output_data_folder = _path / get_exp_name()
    output_data_folder.mkdir(exist_ok=True)
    _subs = ["tensorboard", "plots", "models", "saved_images", "videos", "checkpoints", "tmp", "bodies", "database"]
    for _sub in _subs:
        (output_data_folder / _sub).mkdir(exist_ok=True)

    if init:
        # Create a symlink to output_data
        _sym_link = pathlib.Path("output_data")
        if _sym_link.is_symlink():
            _sym_link.unlink()
        _sym_link.symlink_to(output_data_folder, target_is_directory=True)

    return output_data_folder

def get_input_data_folder():
    _path = pathlib.Path("../input_data")
    assert _path.exists()
    return _path

def get_current_folder():
    return pathlib.Path()

def check_exp_folder():
    """Make sure .exp_folder contains the right folder name"""
    _exp_folder = pathlib.Path(".exp_folder")

    _folder = get_exp_name()

    if _exp_folder.exists():
        _str = _exp_folder.read_text()
        if _folder == _str:
            return
    _exp_folder.write_text(_folder)
    return

def build_model_filename(args):
    filename = "model-"
    filename += args.train_bodies_str.replace(",", "-")
    if args.with_bodyinfo:
        filename += "-body"
    if args.vec_normalize:
        filename += "-vnorm"
    if args.stack_frames>1:
        filename += f"-stack{args.stack_frames}"
    if args.threshold_threshold!=0:
        filename += f"-thr{args.threshold_threshold}"
    if len(args.initialize_weights_from) > 0:
        filename += f"-initw"

    if args.topology_wrapper=="same":
        if args.realign_method!="":
            filename += f"-realign{args.realign_method}"
    elif args.topology_wrapper=="diff":
        if args.wrapper_case!="":
            filename += f"-case{args.wrapper_case}"
    elif args.topology_wrapper=="MutantWrapper":
        filename += "-MutantWrapper"
    elif args.topology_wrapper=="CustomAlignWrapper":
        str2hash = args.custom_alignment
        md5_string = hashlib.md5(str2hash.encode()).hexdigest()
        filename += f"-CustomAlignWrapper-md{md5_string}"
    else:
        pass

    if args.pns:
        filename += "-pns"
        if args.pns_init:
            filename += "-pns_init"

    # if args.misalign_obs:
    #     filename += f"-mis"
    # if args.random_align_obs:
    #     filename += f"-ra"
    # if args.preserve_header:
    #     filename += f"-ph"
    # if args.random_even_same_body:
    #     filename += f"-resb"
    # if args.preserve_feet_contact:
    #     filename += f"-pfc"

    filename += f"-sd{args.seed}"
    return filename

def mean_and_error(_data):
    """A helper for creating error bar"""
    import numpy as np
    _data = np.array(_data)
    _two_sigma = 2*np.std(_data)
    _mean = np.mean(_data)
    print(f"{_mean:.0f} +- {_two_sigma:.0f}")
    return _mean, _two_sigma

def linux_fullscreen():
    """A helper for entering Full Screen mode in Linux.
    Faking a mouse click and a key press (Ctrl+F11).
    """
    from pymouse import PyMouse
    from pykeyboard import PyKeyboard
    m = PyMouse()
    k = PyKeyboard()
    x_dim, y_dim = m.screen_size()
    m.click(int(x_dim/3), int(y_dim/2), 1)
    k.press_key(k.control_key)
    k.tap_key(k.function_keys[11])
    k.release_key(k.control_key)

def load_hyperparameters(conf_name="MyWalkerEnv"):
    import torch.nn as nn
    with (get_input_data_folder() / "hyperparameters.yml").open() as f:
        hp = yaml.load(f, Loader=yaml.SafeLoader)
    hyperparams = hp[conf_name]
    hyperparams["policy_kwargs"] = eval(hyperparams["policy_kwargs"])
    return hyperparams

def shell_header(exp_name="Unknown", description="Unknown"):
    return f"""#!/bin/sh
set -x
python 0.init.py
EXP_FOLDER=$(cat .exp_folder)
# ========================================

exp_name={exp_name}

description={description}

"""

def shell_tail():
    return """
# ========================================
# log
echo "================" >> ~/gpfs2/experiments.log
date >> ~/gpfs2/experiments.log
pwd >> ~/gpfs2/experiments.log
echo $0 >> ~/gpfs2/experiments.log
echo $exp_name >> ~/gpfs2/experiments.log
echo $description >> ~/gpfs2/experiments.log
squeue -O "JobID,Partition,Name,Nodelist,TimeUsed,UserName,StartTime,Schednodes,Command" --user=sliu1 -n $exp_name | head -n 10 >> ~/gpfs2/experiments.log
echo "================" >> ~/gpfs2/experiments.log

"""

class Log:
    def __init__(self, path="") -> None:
        self.msgs = []
        self.log_path = path
    def record(self, msg):
        self.msgs.append(msg)
    def dump(self):
        if self.log_path=="":
            print("\n".join(self.msgs))
        else:
            with open(self.log_path, "a") as file:
                file.writelines(self.msgs)
        self.msgs = []

max_stars_repo_name: liusida/thesis-bodies

max_issues_repo_name: liusida/thesis-bodies

max_forks_repo_name: liusida/thesis-bodies


text:
"""
.. _tut_stats_cluster_sensor_1samp_tfr:

===============================================================
Non-parametric 1 sample cluster statistic on single trial power
===============================================================

This script shows how to estimate significant clusters
in time-frequency power estimates. It uses a non-parametric
statistical procedure based on permutations and cluster
level statistics.

The procedure consists in:

  - extracting epochs
  - compute single trial power estimates
  - baseline line correct the power estimates (power ratios)
  - compute stats to see if ratio deviates from 1.

"""
# Authors: Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
#
# License: BSD (3-clause)

import numpy as np
import matplotlib.pyplot as plt

import mne
from mne import io
from mne.time_frequency import single_trial_power
from mne.stats import permutation_cluster_1samp_test
from mne.datasets import sample

print(__doc__)

###############################################################################
# Set parameters
# --------------
data_path = sample.data_path()
raw_fname = data_path + '/MEG/sample/sample_audvis_raw.fif'
event_id = 1
tmin = -0.3
tmax = 0.6

# Setup for reading the raw data
raw = io.read_raw_fif(raw_fname)
events = mne.find_events(raw, stim_channel='STI 014')

include = []
raw.info['bads'] += ['MEG 2443', 'EEG 053']  # bads + 2 more

# picks MEG gradiometers
picks = mne.pick_types(raw.info, meg='grad', eeg=False, eog=True,
                       stim=False, include=include, exclude='bads')

# Load condition 1
event_id = 1
epochs = mne.Epochs(raw, events, event_id, tmin, tmax, picks=picks,
                    baseline=(None, 0), reject=dict(grad=4000e-13, eog=150e-6))
data = epochs.get_data()  # as 3D matrix
data *= 1e13  # change unit to fT / cm
# Time vector
times = 1e3 * epochs.times  # change unit to ms

# Take only one channel
ch_name = raw.info['ch_names'][97]
data = data[:, 97:98, :]

evoked_data = np.mean(data, 0)

# data -= evoked_data[None,:,:] # remove evoked component
# evoked_data = np.mean(data, 0)

# Factor to down-sample the temporal dimension of the PSD computed by
# single_trial_power.  Decimation occurs after frequency decomposition and can
# be used to reduce memory usage (and possibly computational time of downstream
# operations such as nonparametric statistics) if you don't need high
# spectrotemporal resolution.
decim = 5
frequencies = np.arange(8, 40, 2)  # define frequencies of interest
sfreq = raw.info['sfreq']  # sampling in Hz
epochs_power = single_trial_power(data, sfreq=sfreq, frequencies=frequencies,
                                  n_cycles=4, n_jobs=1,
                                  baseline=(-100, 0), times=times,
                                  baseline_mode='ratio', decim=decim)

# Crop in time to keep only what is between 0 and 400 ms
time_mask = (times > 0) & (times < 400)
evoked_data = evoked_data[:, time_mask]
times = times[time_mask]

# The time vector reflects the original time points, not the decimated time
# points returned by single trial power. Be sure to decimate the time mask
# appropriately.
epochs_power = epochs_power[..., time_mask[::decim]]

epochs_power = epochs_power[:, 0, :, :]
epochs_power = np.log10(epochs_power)  # take log of ratio
# under the null hypothesis epochs_power should be now be 0

###############################################################################
# Compute statistic
# -----------------
threshold = 2.5
T_obs, clusters, cluster_p_values, H0 = \
    permutation_cluster_1samp_test(epochs_power, n_permutations=100,
                                   threshold=threshold, tail=0)

###############################################################################
# View time-frequency plots
# -------------------------
plt.clf()
plt.subplots_adjust(0.12, 0.08, 0.96, 0.94, 0.2, 0.43)
plt.subplot(2, 1, 1)
plt.plot(times, evoked_data.T)
plt.title('Evoked response (%s)' % ch_name)
plt.xlabel('time (ms)')
plt.ylabel('Magnetic Field (fT/cm)')
plt.xlim(times[0], times[-1])
plt.ylim(-100, 250)

plt.subplot(2, 1, 2)

# Create new stats image with only significant clusters
T_obs_plot = np.nan * np.ones_like(T_obs)
for c, p_val in zip(clusters, cluster_p_values):
    if p_val <= 0.05:
        T_obs_plot[c] = T_obs[c]

vmax = np.max(np.abs(T_obs))
vmin = -vmax
plt.imshow(T_obs, cmap=plt.cm.gray,
           extent=[times[0], times[-1], frequencies[0], frequencies[-1]],
           aspect='auto', origin='lower', vmin=vmin, vmax=vmax)
plt.imshow(T_obs_plot, cmap=plt.cm.RdBu_r,
           extent=[times[0], times[-1], frequencies[0], frequencies[-1]],
           aspect='auto', origin='lower', vmin=vmin, vmax=vmax)
plt.colorbar()
plt.xlabel('time (ms)')
plt.ylabel('Frequency (Hz)')
plt.title('Induced power (%s)' % ch_name)
plt.show()

ext: py

reason: import numpy


text:
# Standard modules
from __future__ import print_function, division
import os
import pickle
import argparse
import pandas as pd
import numpy as np

# Pytorch for data set
import torch
from torch.utils.data import Dataset, DataLoader

# RDkit
from rdkit import Chem

# Molecules dataset definition
from molecules_dataset import *

def write_dataset(ds, out_dir_name, split='random', seed=42, element_list=None):

    datatypes = ['float64', 'float32', 'float16', 'int64', 'int32', 'int16', 'int8', 'uint8', 'bool']

    # Create the directory for this data set
    try: os.mkdir(out_dir_name)
    except FileExistsError: pass

    # Define indices to split the data set
    if split == 'random':
        ind_te, ind_va, ind_tr = ds.split_randomly(train_split=0.8,vali_split=0.1,test_split=0.1,random_seed=seed)
        print('Training: %i molecules. Validation: %i molecules. Test: %i molecules.'%(len(ind_tr),len(ind_va),len(ind_te)))
    elif split == 'stdev':
        ind_te, ind_va, ind_tr = ds.split_by_list(train_list = 'datasets_raw/aqsoldb/split-by-stdev/training.txt',
                                                  vali_list = 'datasets_raw/aqsoldb/split-by-stdev/validation.txt',
                                                  test_list = 'datasets_raw/aqsoldb/split-by-stdev/test.txt',
                                                  random_seed = seed)

    # Save the indices for the split
    np.savetxt(out_dir_name+'/indices_test.dat', ind_te, fmt='%i')
    np.savetxt(out_dir_name+'/indices_vali.dat', ind_va, fmt='%i')
    np.savetxt(out_dir_name+'/indices_train.dat',ind_tr, fmt='%i')

    # Save the SMILES strings for the split
    np.savetxt(out_dir_name+'/smiles_test.txt', np.array(ds.smiles, dtype=str)[ind_te], fmt='%s')
    np.savetxt(out_dir_name+'/smiles_vali.txt', np.array(ds.smiles, dtype=str)[ind_va], fmt='%s')
    np.savetxt(out_dir_name+'/smiles_train.txt',np.array(ds.smiles, dtype=str)[ind_tr], fmt='%s')

    # Save the names for the split
    np.savetxt(out_dir_name+'/names_test.txt', np.array(ds.lnum, dtype=str)[ind_te], fmt='%s')
    np.savetxt(out_dir_name+'/names_vali.txt', np.array(ds.lnum, dtype=str)[ind_va], fmt='%s')
    np.savetxt(out_dir_name+'/names_train.txt',np.array(ds.lnum, dtype=str)[ind_tr], fmt='%s')

    # Save the data sets as compressed numpy files
    test_file_name  = out_dir_name+'/test.npz'
    vali_file_name  = out_dir_name+'/valid.npz'
    train_file_name = out_dir_name+'/train.npz'
    if len(ind_te) > 0: ds.write_compressed(test_file_name, indices=ind_te, datatypes=datatypes, element_list=element_list, write_bonds=True)
    if len(ind_va) > 0: ds.write_compressed(vali_file_name, indices=ind_va, datatypes=datatypes, element_list=element_list, write_bonds=True)
    if len(ind_tr) > 0: ds.write_compressed(train_file_name,indices=ind_tr, datatypes=datatypes, element_list=element_list, write_bonds=True)

    return

############
# - MAIN - #

if __name__ == "__main__":

    # Parse arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('--no_rare_elements', action='store_true',
                        help="Excludes molecules with rare elements" )
    parser.add_argument('--rename_rare_elements', action='store_true',
                        help="Renames rare elements (sets their atomic number to 0)" )
    parser.add_argument('--max_36_at', action='store_true',
                        help="Excludes molecules with more than 36 atoms" )
    parser.add_argument('--only_with_sd', action='store_true',
                        help="Uses the std. dev. and excludes data points with std. dev of zero" )
    parser.add_argument('--split', type=str, choices=['random','stdev'], default='random',
                        help="Split method." )
    parser.add_argument('--seed', type=int, default=42,
                        help="Random seed." )
    args = parser.parse_args()

    dir_name = 'datasets_processed/aqsoldb'

    elements = None
    element_list = None

    if args.only_with_sd:
        data_file = 'datasets_raw/aqsoldb/curated-solubility-dataset-with-SD.csv'
        dir_name += '_withsd'
        sd_names = ['SD']
    else:
        data_file = 'datasets_raw/aqsoldb/curated-solubility-dataset.csv'
        sd_names = None

    if args.no_rare_elements:
        elements = ['C','O','N','Cl','S','F','P','Br','Na']
        dir_name += '_norareelements'

    if args.rename_rare_elements:
        element_list = ['C','H','O','N','Cl','S','F','P','Br','Na','I','K']
        dir_name += '_renamerareelements'

    if args.max_36_at:
        maxnumat = 36
        dir_name += '_max36at'
    else:
        maxnumat = None

    dir_name += '_'+args.split+'-split'

    # Create the data set
    ds = MoleculesDataset(data_file, ['Solubility'], name = 'AqSolDB',
                          alt_labels = ['logS'], sd_names = sd_names, id_name = 'Name',
                          num_conf = 1, order_atoms = False, shuffle_atoms = False, max_num_at = maxnumat,
                          add_h = False, elements = elements, badlist = None)
    print('Created dataset with %i molecules.'%len(ds.smiles))

    # Create the directory for this data set
    try: os.makedirs(dir_name)
    except FileExistsError: pass

    # Dump the entire dataset
    ds_file = open(dir_name+'/dataset.pkl', 'wb')
    pickle.dump(ds,ds_file)

    # Load the dataset
    ds_file = open(dir_name+'/dataset.pkl', 'rb')
    ds = pickle.load(ds_file)

    # Write the dataset in NPZ format
    write_dataset(ds, dir_name, split=args.split, seed=args.seed, element_list=element_list)


text:
"""Functions relating to pairs of regions."""

import numpy as np
import pandas as pd
import scipy.spatial as spa

def build_get_neighbors(criteria, cutoff):
    """Build a neighbor function to check if two subunits are neighboring."""
    def get_neighbors_param(df0, df1):
        return get_neighbors(df0, df1, criteria, cutoff)
    return get_neighbors_param

def get_neighbors(df0, df1, criteria, cutoff):
    """Find neighbors according to provided criteria and cutoff."""
    if criteria == 'heavy':
        return _get_heavy_neighbors(df0, df1, cutoff)
    elif criteria == 'ca':
        return _get_ca_neighbors(df0, df1, cutoff)
    else:
        raise RuntimeError("Unrecognized criteria {:}".format(criteria))

def _get_ca_neighbors(df0, df1, cutoff):
    """Get neighbors for alpha-carbon based distance."""
    ca0 = df0[df0['atom_name'] == 'CA']
    ca1 = df1[df1['atom_name'] == 'CA']

    dist = spa.distance.cdist(ca0[['x', 'y', 'z']], ca1[['x', 'y', 'z']])
    pairs = np.array(np.where(dist < cutoff)).T
    if len(pairs) == 0:
        return [], []
    res0 = ca0.iloc[pairs[:, 0]][['pdb_name', 'model', 'chain', 'residue']]
    res1 = ca1.iloc[pairs[:, 1]][['pdb_name', 'model', 'chain', 'residue']]
    res0 = res0.reset_index(drop=True)
    res1 = res1.reset_index(drop=True)
    return res0, res1

def _get_heavy_neighbors(df0, df1, cutoff):
    """Get neighbors for heavy atom based distance."""
    heavy0 = df0[df0['element'] != 'H']
    heavy1 = df1[df1['element'] != 'H']

    dist = spa.distance.cdist(heavy0[['x', 'y', 'z']], heavy1[['x', 'y', 'z']])
    pairs = np.array(np.where(dist < cutoff)).T
    if len(pairs) == 0:
        return [], []
    # We use the found pairs to find unique pairings of residues.
    res0 = heavy0.iloc[pairs[:, 0]][['pdb_name', 'model', 'chain', 'residue']]
    res1 = heavy1.iloc[pairs[:, 1]][['pdb_name', 'model', 'chain', 'residue']]
    res0 = res0.reset_index(drop=True)
    res1 = res1.reset_index(drop=True)
    # We concatenate so that we can find unique _pairs_.
    res = pd.concat((res0, res1), axis=1)
    res = res.drop_duplicates()
    # Split back out now that we have found duplicates.
    res0 = res.iloc[:, range(4)]
    res1 = res.iloc[:, range(4, 8)]
    res0 = res0.reset_index(drop=True)
    res1 = res1.reset_index(drop=True)
    return res0, res1

reason: import numpy,import scipy


text:
import sys
import math
import os, os.path
import re
import itertools as it
import numpy as np
import networkx as nx
import sklearn
import sklearn.metrics
import scipy
import scipy.sparse as sp
from scipy.stats import gamma
from scipy.sparse.linalg import norm

class FactorLibException(Exception):
  pass

# TODO copied from ppi.irefindex
def transform_nearest_neighbors(G, k=10, attr='weight', attr_type='similarity'):
  """
  Remove edges in G. For each node, keep at most 10 of its neighbors. Kept neighbors have the
  highest 'weight' edge attribute. Returns a copy of G and does not modify G.

  Parameters
  ----------
  G : nx.Graph

  k : int
    default 10

  attr : str
    default 'weight'

  attr_type : str
    'similarity': nearest neighbors are those that are most similar: keep neighbors with highest edge weight
    'distance': nearest neighbers are those that are least distant from each other: keep neighbors with least edge weight

  Returns
  -----
  H : nx.Graph
    k-nearest neighbor graph
    note that a vertex v may have more than k edges incident to it if its k'>k neighbor has v in its k-nearest neighbors
  """
  attr_types = ['similarity', 'distance']
  if(attr_type not in attr_types):
    raise ArgumentError("attr_type: {} should be one of {}".format(attr_type, ",".join(attr_types)))
  H = nx.Graph() # TODO assumes undirected
  for node in G.nodes_iter():
    inc_edges = G.edges([node])
    edge_weight_pairs = []
    for edge in inc_edges:
      weight = G[edge[0]][edge[1]][attr]
      pair = (edge, weight)
      edge_weight_pairs.append(pair)
    if(attr_type == 'similarity'):
      edge_weight_pairs = sorted(edge_weight_pairs, key=lambda x: x[1], reverse=True)
    else:
      edge_weight_pairs = sorted(edge_weight_pairs, key=lambda x: x[1])
    for pair in edge_weight_pairs[:k]:
      edge = pair[0]
      weight = pair[1]
      H.add_edge(edge[0], edge[1], {'weight': weight})
  return H

def relabel_nodes(G, node_attribute):
  """
  Wrapper around nx.relabel_nodes for scripts: relabel nodes using the <node_attribute>
  """
  if node_attribute is not None:
    mapping = {}
    for node in G.nodes():
      node_attrs = G.node[node]
      if node_attribute in node_attrs:
        mapping[node] = node_attrs[node_attribute]
    G = nx.relabel_nodes(G, mapping)
  return G

def rbf_similarity(stdev, x, y):
  return math.exp(-(np.abs(x - y) / stdev) ** 2)

def combine_graphs(Gs, weights):
  """
  Combine graphs into a single graph by constructing a weighted sum of their edge weights

  G_1 := (V_1, E_1)
  G_2 := (V_2, E_2)
  where E_i is a set of tuples (u,v,w)

  G := (V_1 \cup V_2, E)
  TODO
  """
  def get_edge_weight(G, u, v):
    # assumes edge is already known to be present in G
    weight = G[u][v].get('weight')
    if weight is None:
      weight = 1
    return weight

  total_weight = 0.0
  for weight in weights:
    total_weight += weight

  G_rv = nx.Graph()
  for i, G in enumerate(Gs):
    graph_weight = weights[i]
    for edge in G.edges_iter():
      weight = get_edge_weight(G, *edge)

      if G_rv.has_edge(*edge):
        weight_rv = G_rv[edge[0]][edge[1]]['weight']
        G_rv[edge[0]][edge[1]]['weight'] = weight_rv + graph_weight * weight
      else:
        G_rv.add_edge(edge[0], edge[1], {'weight': graph_weight * weight})

  return G_rv

def diffusion(M, adj, alpha=0.7, tol=10e-6):  # TODO equation, M, alpha
    """Network propagation iterative process

    Iterative algorithm for apply propagation using random walk on a network:
        Initialize::
            X1 = M

        Repeat::
            X2 = alpha * X1.A + (1-alpha) * M
            X1 = X2

        Until::
            norm(X2-X1) < tol

        Where::
            A : degree-normalized adjacency matrix

    Parameters
    ----------
    M : sparse matrix
        Data matrix to be diffused.

    adj : sparse matrix
        Adjacency matrice.

    alpha : float, default: 0.7
        Diffusion/propagation factor with 0 <= alpha <= 1.
        For alpha = 0 : no diffusion.
        For alpha = 1 :

    tol : float, default: 10e-6
        Convergence threshold.

    Returns
    -------
    X2 : sparse matrix
        Smoothed matrix.

    Notes
    -----
    Copied from the stratipy Python library
    """
    n = adj.shape[0]
    adj = adj+sp.eye(n)

    d = sp.dia_matrix((np.array(adj.sum(axis=0))**-1, [0]), shape=(n,  n))
    A = adj.dot(d)

    X1 = M
    X2 = alpha * X1.dot(A) + (1-alpha) * M
    i = 0
    while norm(X2-X1) > tol:
        X1 = X2
        X2 = alpha * X1.dot(A) + (1-alpha) * M
        i += 1
    return X2

def parse_ws_delim(fh):
  return parse_nodelist(fh)

# TODO move to prmf_runner.py
def parse_pathway_obj(fp):
  """
  Extract mapping of latent factor to pathway fp
  """
  rv = {}
  regexp = re.compile(r'(\d+)\s*->\s*(.+)')
  with open(fp) as fh:
    for line in fh:
      line = line.rstrip()
      match_data = regexp.match(line)
      if match_data is not None:
        rv[int(match_data.group(1))] = match_data.group(2)
  return rv

# TODO move to prmf_runner.py
def parse_init(fp):
  """
  Extract filepaths used to initialize nmf_pathway.py as reported by its output stored in <fp>
  """
  rv = []
  with open(fp, 'r') as fh:
    for line in fh:
      line = line.rstrip()
      rv.append(line)
  return rv

def prepare_nodelist(lists):
  """
  Combine lists of nodes into a single master list for use by nmf_pathway and other functions that
  accept a nodelist as input
  """
  set_all = set()
  for ll in lists:
    for item in ll:
      set_all.add(item)
  rv = sorted(set_all)
  return rv

def parse_gene_lists(nodelist, gene_list_fps):
  """
  Parse gene lists into a sparse scipy array

  Parameters
  ----------
  nodelist : list of str
    gene identifiers that define the assignment of array indexes to genes

  gene_lists : list of

  Returns
  -------
  mat : sparse csc matrix
  """
  # parse gene lists
  gene_lists = []
  for gene_path in gene_list_fps:
    with open(gene_path) as fh:
      gene_lists.append(parse_ws_delim(fh))

  # verify gene lists present in ppi_db
  def get_row_vec_for_gene_list(gene_list):
    row_vec, missing = embed_ids(nodelist, gene_list)
    sys.stderr.write("missing {}/{} node identifiers: {}\n".format(len(missing), len(gene_list), ", ".join(missing)))
    return row_vec
  row_vecs = map(get_row_vec_for_gene_list, gene_lists)

  mat = sp.vstack(row_vecs)
  return mat

def parse_nodelist(fh):
  """
  Return a list of node identifiers which maps the list index to the identifier
  """
  rv = []
  for line in fh:
    line = line.rstrip()
    words = line.split()
    for word in words:
      rv.append(word)
  return rv

def parse_pathways(graphml_fps):
  Gs = list(map(lambda x: nx.read_graphml(x), graphml_fps))
  return Gs

def parse_pathways_dir(pathways_dir):
  """
  Parse .graphml files in <pathways_dir>

  Returns
  -------
  Gs : list of nx.DiGraph
    parsed pathways in <pathways_dir>
  """
  graphmls = []
  for fname in os.listdir(pathways_dir):
    basename, ext = os.path.splitext(fname)
    if ext == ".graphml":
      graphmls.append(os.path.join(pathways_dir, fname))
  return parse_pathways(graphmls)

def parse_seedlist(fp, filetype=None, node_attr='name'):
  """
  Parse the file with filepath <fp> as a seed list: a list of node identifier strings.

  Parameters
  ----------
  fp : str
    a file containing gene identifiers

  filetype : str, None
    the file type, informs parsing
    if None, determined by the extension of fp
  """
  if filetype is None:
    fp_no_ext, ext = os.path.splitext(fp)
    if ext == '.graphml':
      filetype = 'graphml'
    elif ext == '.txt':
      filetype = 'txt'
  if filetype is None:
    raise FactorLibException("Cannot determine filetype")

  seedlist = None
  if filetype == 'graphml':
    G = nx.read_graphml(fp)
    G = relabel_nodes(G, node_attr)
    seedlist = G.nodes()
  elif filetype == 'txt':
    seedlist = []
    with open(fp, 'r') as fh:
      for line in fh:
        line = line.rstrip()
        seedlist.append(line)
  return seedlist

def embed_ids(all_ids, ids):
  """
  Construct a binary vector of length len(<all_ids>) with a 1 in the positions
  associated with each id in <ids>
  """
  row_vec = np.zeros(len(all_ids))
  missing = []
  for id in ids:
    try:
      index = all_ids.index(id)
      row_vec[index] = 1
    except ValueError:
      missing.append(id)

  row_vec_sparse = sp.csc_matrix(row_vec)
  return (row_vec_sparse, missing)

def embed_arr(all_col_names, some_col_names, arr):
  """
  Embed <arr> with len(some_col_names) number of columns in a larger array with len(all_col_names)
  number of columns. The larger array is constructed and returned.
  """
  m,n = arr.shape
  if len(some_col_names) != n:
    raise FactorLibException('some_col_names != #columns of arr: {} != {}'.format(some_col_names, n))

  n2 = len(all_col_names)
  rv = np.zeros((m,n2))

  all_name_to_ind = {}
  for i,name in enumerate(all_col_names):
    all_name_to_ind[name] = i

  for i in range(m):
    for j, name in enumerate(some_col_names):
      j2 = all_name_to_ind[name]
      rv[i,j2] = arr[i,j]

  return rv

def get_adj_mat(G):
  """Represent ppi network as adjacency matrix

  Parameters
  ----------
  G : networkx graph
    ppi network, see get_ppi()

  Returns
  -------
  adj : square sparse scipy matrix
    (i,j) has a 1 if there is an interaction reported by irefindex

  ids : list
    same length as adj, ith index contains irefindex unique identifier for gene whose interactions are
    reported in the ith row of adj
  """
  ids = G.nodes()
  adj = nx.to_scipy_sparse_matrix(G, nodelist=ids, dtype=bool)

  return adj, ids

def weighted_union(Gs, weight_attr='weight'):
  """
  Construct a weighted graph that is the union of graphs <Gs>.
  For each edge in a network in <Gs>, use <weight_attr> as the edge weight or 1 if that attribute is not present.
  Each edge in G_rv has its <weight_attr> set to the average weight of <Gs> that the edge appears in.
  That is, networks without the edge do not pull down the average toward 0.

  Parameters
  ----------
  Gs : iterable of nx.Graph

  weight_attr : str
    name of edge attribute to use as weight

  Returns
  -------
  G_rv : nx.Graph
  """
  G_rv = nx.Graph()
  edge_to_count = {}

  def get_edge_weight(G, edge):
    weight = G[edge[0]][edge[1]].get(weight_attr)
    if weight is None:
      weight = 1
    return weight

  for G in Gs:
    for edge in G.edges_iter():
      weight = get_edge_weight(G, edge)
      prev_weight = 0
      if G_rv.has_edge(*edge):
        prev_weight = get_edge_weight(G_rv, edge)
      weight_sum = prev_weight + weight
      G_rv.add_edge(edge[0], edge[1], {weight_attr: weight_sum})
      if edge in edge_to_count:
        edge_to_count[edge] += 1
      else:
        edge_to_count[edge] = 1

    for node in G.nodes_iter():
      if node not in G_rv:
        G_rv.add_node(node)

  for edge, count in edge_to_count.items():
    weight = get_edge_weight(G_rv, edge)
    avg_weight = float(weight) / count
    G_rv[edge[0]][edge[1]][weight_attr] = avg_weight

  return G_rv

def mat_to_bipartite(mat):
  G = nx.Graph()
  row_node_ids = []
  col_node_ids = []
  n_rows, n_cols = mat.shape
  for i in range(n_rows):
    node_i = "r{}".format(i)
    row_node_ids.append(node_i)
    G.add_node(node_i, bipartite=0)

  for j in range(n_cols):
    node_j = "c{}".format(j)
    col_node_ids.append(node_j)
    G.add_node(node_j, bipartite=1)

  for i in range(n_rows):
    for j in range(n_cols):
      G.add_edge("r{}".format(i), "c{}".format(j), {'weight': mat[i,j]})
  return G, row_node_ids, col_node_ids

def to_unit_vector(vec):
  rv = None
  norm = np.linalg.norm(vec)
  if(norm == 0):
    rv = vec
  else:
    rv = vec / norm
  return rv

def nodelists_to_mat(sub_lists, uni_list):
  """Represent gene nodelist as a matrix with shape (len(uni_list), len(sub_lists))

  Parameters
  ----------
  sub_lists : list of str

  uni_list : list of str

  Returns
  -------
  mat : np.array
    matrix where each column is associated with a nodelist in <sub_lists> and has a 1 in each
    row where that gene is in the node list

  TODO
  ----
  For a single pathway, this representation has n_genes_in_pathway "units of signal" while
  the sampled signal has units <= n_genes_in_pathway (because a subset is sampled)
  """
  node_to_index = {}
  for i, node in enumerate(uni_list):
    node_to_index[node] = i

  n_pathways = len(sub_lists)
  mat = np.zeros((len(uni_list), n_pathways))
  for pathway_ind in range(len(sub_lists)):
    pathway = sub_lists[pathway_ind]
    for node in pathway:
      index = node_to_index[node]
      if index is None:
        sys.stderr.write("Unrecognized gene ID: {}\n".format(node))
      else:
        mat[index,pathway_ind] = 1
  return mat

def normalize_num_pathways(W_mat, pathways_mat):
  """
  Prior to matching latent factors to pathways, this function MAY be invoked to transform
  W_mat or pathways_mat (whichever is smaller) to make it so the number of latent factors
  is the same as the number of pathways by adding a binary vector of all zeros (the empty pathway).

  Parameters
  ----------
  W_mat : np.array
    array with latent factors as columns and genes as rows with gene loadings into latent factors in cells

  pathways_mat : np.array
    array with pathways as columns and genes as rows with a 1 or 0 in each cell indicating whether that
    gene is present in the pathway or not

  Returns
  -------
  W_mat : np.array
    See description

  pathways_mat : np.array
    See description
  """
  # reshape 1D vectors if needed
  shp = W_mat.shape
  if(len(shp) == 1):
    W_mat = W_mat.reshape((shp[0], 1))
  shp = pathways_mat.shape
  if(len(shp) == 1):
    pathways_mat = pathways_mat.reshape((shp[0], 1))

  n_genes, n_latent_factors = W_mat.shape
  shp = pathways_mat.shape
  n_genes2, n_pathways = pathways_mat.shape
  if(n_genes != n_genes2):
    raise ValueError("{} != {}: number of genes identified by W_mat is not the same as those identified by pathways_mat".format(n_genes, n_genes2))

  arrs = [W_mat, pathways_mat]
  small_arr = -1
  large_arr = -1
  diff = 0
  if(n_latent_factors < n_pathways):
    small_arr = 0
    large_arr = 1
    diff = n_pathways - n_latent_factors
  elif(n_pathways < n_latent_factors):
    small_arr = 1
    large_arr = 0
    diff = n_latent_factors - n_pathways
  # else do nothing

  cols = np.zeros((n_genes, diff))
  arrs[small_arr] = np.concatenate((arrs[small_arr], cols), axis=1)

  return tuple(arrs)

def match(W_mat, pathways_mat):
  return match_pr(W_mat, pathways_mat)

def match_pr(W_mat, pathways_mat):
  n_genes, n_factors = W_mat.shape
  n_genes2, n_pathways = pathways_mat.shape
  if(n_genes != n_genes2):
    raise ValueError("n_genes != n_genes2 : {} != {}".format(n_genes, n_genes2))

  # prepare max weight matching by AUC
  aucs = np.zeros((n_factors, n_pathways))
  for i in range(n_factors):
    factor_vec = W_mat[:,i]
    for j in range(n_pathways):
      pathway_vec = pathways_mat[:,j]

      y_score = factor_vec
      y_true = pathway_vec
      auc = sklearn.metrics.average_precision_score(y_true, y_score)
      aucs[i,j] = auc

  # use networkx for max weight matching
  G, factor_node_ids, pathway_node_ids = mat_to_bipartite(aucs)
  matching = nx.max_weight_matching(G)

  # I don't like the return value from networkx, reorganize data
  rv = transform_matching(G, matching, factor_node_ids)
  return rv

def match_dist(W_mat, pathways_mat):
  """
  Match latent factors to pathways

  Parameters
  ----------
  W_mat : np.array

  pathways_mat : np.array

  Returns
  -------
  rv : list of tpl
    each tpl is of the form (<latent_factor_id>, <pathway_id>, <distance>)
    where W_mat latent factors are identified with names "r0", "r1", etc. (for "row" of the
    distance matrix constructed in this function) associated with each latent factor in W_mat;
    and pathways are identified with names "c0", "c1", etc. (for "column" of the distance
    matrix)

  TODO
  ----
  other type of transformation of distance to similarity? RBF?
  implement version that uses hypergeometic p-values as distance measure?
  """
  n_genes, n_factors = W_mat.shape
  n_genes2, n_pathways = pathways_mat.shape
  if(n_genes != n_genes2):
    raise ValueError("n_genes != n_genes2 : {} != {}".format(n_genes, n_genes2))
  #if(n_factors != n_pathways):
  #  raise ValueError("n_factors != n_pathways: {} != {}".format(n_factors, n_pathways))

  dists = np.zeros((n_factors, n_pathways))
  max_unit_vec_dist = math.sqrt(2)
  for i in range(n_factors):
    factor_vec = W_mat[:,i]
    # scale distance by size of latent factor and pathway
    factor_vec_unit = to_unit_vector(factor_vec)
    for j in range(n_pathways):
      pathway_vec = pathways_mat[:,j]
      pathway_vec_unit = to_unit_vector(pathway_vec)
      dists[i,j] = np.linalg.norm(factor_vec_unit - pathway_vec_unit)
  dists_inv = 1 - (dists / max_unit_vec_dist)

  # use networkx for max weight matching
  G, factor_node_ids, pathway_node_ids = mat_to_bipartite(dists_inv)
  matching = nx.max_weight_matching(G)

  # I don't like the return value from networkx, reorganize data
  rv = transform_matching(G, matching, factor_node_ids)
  return rv

def transform_matching(G, matching, factor_node_ids):
  """
  Transform the return value of a matching from NetworkX into a different data structure

  Returns
  -------
  rv : list of (factor_id, pathway_id, weight)
  """
  rv = []
  for factor_id in factor_node_ids:
    # not all nodes used in matching necessarily
    if factor_id in matching:
      pathway_id = matching[factor_id]
      weight = G[factor_id][pathway_id]['weight']
      tpl = (factor_id, pathway_id, weight)
      rv.append(tpl)
    else:
      # TODO temporary: why are they not included?
      # usually not included due to different numbers of latent factors and pathways
      weights = []
      for neighbor in G.neighbors(factor_id):
        weights.append(G[factor_id][neighbor]['weight'])
      sys.stderr.write("Factor {} not included in matching; weights: ".format(factor_id) + " ; ".join(map(str, weights)) + "\n")
  return rv

def sample(G, n, t=3, alpha_portion=0.9, seed=None, node_order=None):
  """
  Sample <n> random values according to a distribution specified by G.
  Each value is a len(node_order) size vector of positive reals.

  If a node in G has no parents, its value is drawn according to Gamma(t^alpha_portion, t^(1-alpha_portion)).
  t is a reparameterization so that t is a location parameter.
  The default alpha_portion is chosen to be high so that the distribution is less uniform and more peaked around that location.

  Nodes which are present in <node_order> but not <G> are also have values drawn according to the aforementioned distribution.

  A node Y which has parents X_1, ..., X_k is specified by Y = \sum_k Gamma(X_k^alpha_portion, X_k^(1-alpha_portion))
  Y also has a Gamma distribution but it is not easily expressed. For Y = \sum_k Gamma(alpha_k, beta),
  Y ~ Gamma(\sum_k alpha_k, beta). The Y here is different but close.
  """
  loc = 0
  # TODO assumes connected
  if not G.is_directed():
    raise FactorLibException('G must be directed')
  if seed is not None:
    np.random.seed(seed)
  if node_order is None:
    node_order = G.nodes()

  m = len(node_order)
  rv = np.zeros((m,n))
  node_to_ind = {}
  for i, node in enumerate(node_order):
    node_to_ind[node] = i

  roots = [n for n,d in G.in_degree().items() if d==0]
  for j in range(n):
    for root in roots:
      root_ind = node_to_ind[root]
      alpha = math.pow(t, alpha_portion)
      beta = math.pow(t, 1 - alpha_portion)
      rv[root_ind,j] = gamma.rvs(alpha, loc, beta)
      for source, target in nx.dfs_edges(G, root):
        source_ind = node_to_ind[source]
        target_ind = node_to_ind[target]
        alpha = math.pow(rv[source_ind,j], alpha_portion)
        beta = math.pow(rv[source_ind,j], 1 - alpha_portion)
        rv[target_ind,j] += gamma.rvs(alpha, loc, beta)

  other_nodes = set(node_order) - set(G.nodes())
  for j in range(n):
    for node in other_nodes:
      node_ind = node_to_ind[node]
      alpha = math.pow(t, alpha_portion)
      beta = math.pow(t, 1 - alpha_portion)
      rv[node_ind, j] = gamma.rvs(alpha, loc, beta)

  return rv

def compute_man_reg(w, L):
  return (L.dot(w)).dot(w)

def parse_achilles(fp):
  """
  Parse Achilles data from http://portals.broadinstitute.org/achilles/datasets/18/download

  Parameters
  ----------
  fp : str
    filepath to 'gene_dependency' file mentioned in Achilles README
  """
  data = None
  row_to_cell_line = []
  gene_names = None

  with open(fp) as fh:
    for line in fh:
      line = line.rstrip()
      gene_names = line.split(',')[1:]
      break

    for line in fh:
      ind = line.index(',')
      cell_line = line[0:ind]
      row_to_cell_line.append(cell_line)

  data = np.genfromtxt(fp, delimiter=",", skip_header=1)
  data = data[:, 1:]

  gene_names = achilles_to_hugo(gene_names)

  return data, row_to_cell_line, gene_names

def achilles_to_hugo(gene_names):
  """
  Transform gene identifiers from "HUGO (Entrez)" to "HUGO"
  """
  regexp = re.compile(r'([\w-]+)\s\(\w+\)')
  rv = []
  for i, gene_name in enumerate(gene_names):
    match_data = regexp.match(gene_name)
    if match_data is not None:
      rv.append(match_data.group(1))
    else:
      raise FactorLibException("Gene name {} at position {} does not match the specified format".format(gene_name, i))
  return rv

def filter_vec_by_graph(G, vec, nodelist):
  """
  Return a subset of vec and nodelist corresponding to the nodes defined in G
  """
  inds = []
  nodelist_sub = []
  for i, node in enumerate(nodelist):
    if node in G:
      inds.append(i)
      nodelist_sub.append(node)
  vec_sub = vec[inds]
  return vec_sub, nodelist_sub

def measure_cv_performance(gene_by_latent_train, data_test):
  """
  Measure NMF model performance on held out data. Performance is evaluated based on the model's ability
  to reconstuct held out samples.
    \hat{u} := arg min_{u} || x - uV^T || s.t. u >= 0
    \hat{x} := \hat{u} V^T
    error = || x - \hat{x} ||
    normalized_error = error / ||x||

  Parameters
  ----------
  gene_by_latent_train : np.array
    V in the above equations

  data_test : np.array
    X in the above equations

  Returns
  -------
  error : np.array
    normalized error for each sample
  """
  m_samples, n_genes = data_test.shape
  error = np.zeros(m_samples,)

  # TODO multi-sample version of nnls?
  for m in range(m_samples):
    u_hat, err = scipy.optimize.nnls(gene_by_latent_train, data_test[m,:])
    error[m] = err / np.linalg.norm(data_test[m,:])

  return error

max_stars_repo_name: gitter-lab/prmf

max_issues_repo_name: gitter-lab/prmf

max_forks_repo_name: gitter-lab/prmf

reason: import numpy,import scipy,from scipy,import networkx,from networkx


text:
#!/usr/bin/python
import numpy as np
import scipy.sparse
import pickle
import xgboost as xgb

### simple example
# load file from text file, also binary buffer generated by xgboost
dtrain = xgb.DMatrix('../data/agaricus.txt.train')
dtest = xgb.DMatrix('../data/agaricus.txt.test')

# specify parameters via map, definition are same as c++ version
param = {'max_depth':2, 'eta':1, 'silent':1, 'objective':'binary:logistic' }

# specify validations set to watch performance
watchlist  = [(dtest,'eval'), (dtrain,'train')]
num_round = 2
bst = xgb.train(param, dtrain, num_round, watchlist)

# this is prediction
preds = bst.predict(dtest)
labels = dtest.get_label()
print ('error=%f' % ( sum(1 for i in range(len(preds)) if int(preds[i]>0.5)!=labels[i]) /float(len(preds))))
bst.save_model('0001.model')
# dump model
bst.dump_model('dump.raw.txt')
# dump model with feature map
bst.dump_model('dump.nice.txt','../data/featmap.txt')

# save dmatrix into binary buffer
dtest.save_binary('dtest.buffer')
# save model
bst.save_model('xgb.model')
# load model and data in
bst2 = xgb.Booster(model_file='xgb.model')
dtest2 = xgb.DMatrix('dtest.buffer')
preds2 = bst2.predict(dtest2)
# assert they are the same
assert np.sum(np.abs(preds2-preds)) == 0

# alternatively, you can pickle the booster
pks = pickle.dumps(bst2)
# load model and data in
bst3 = pickle.loads(pks)
preds3 = bst3.predict(dtest2)
# assert they are the same
assert np.sum(np.abs(preds3-preds)) == 0

###
# build dmatrix from scipy.sparse
print ('start running example of build DMatrix from scipy.sparse CSR Matrix')
labels = []
row = []; col = []; dat = []
i = 0
for l in open('../data/agaricus.txt.train'):
    arr = l.split()
    labels.append(int(arr[0]))
    for it in arr[1:]:
        k,v = it.split(':')
        row.append(i); col.append(int(k)); dat.append(float(v))
    i += 1
csr = scipy.sparse.csr_matrix((dat, (row,col)))
dtrain = xgb.DMatrix(csr, label = labels)
watchlist  = [(dtest,'eval'), (dtrain,'train')]
bst = xgb.train(param, dtrain, num_round, watchlist)

print ('start running example of build DMatrix from scipy.sparse CSC Matrix')
# we can also construct from csc matrix
csc = scipy.sparse.csc_matrix((dat, (row,col)))
dtrain = xgb.DMatrix(csc, label=labels)
watchlist  = [(dtest,'eval'), (dtrain,'train')]
bst = xgb.train(param, dtrain, num_round, watchlist)

print ('start running example of build DMatrix from numpy array')
# NOTE: npymat is numpy array, we will convert it into scipy.sparse.csr_matrix in internal implementation
# then convert to DMatrix
npymat = csr.todense()
dtrain = xgb.DMatrix(npymat, label = labels)
watchlist  = [(dtest,'eval'), (dtrain,'train')]
bst = xgb.train(param, dtrain, num_round, watchlist)

max_stars_repo_name: cnheider/xgboost

max_issues_repo_name: cnheider/xgboost

max_forks_repo_name: cnheider/xgboost

reason: import numpy,from numpy,import scipy,from scipy


text:
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for roi_generator.py."""

# Import libraries
import numpy as np
import tensorflow as tf

from official.vision.modeling.layers import roi_generator

class MultilevelProposeRoisTest(tf.test.TestCase):

  def test_multilevel_propose_rois_single_level(self):
    rpn_boxes_np = np.array(
        [[[[0, 0, 10, 10], [0.01, 0.01, 9.9, 9.9]],
          [[5, 5, 10, 10], [2, 2, 8, 8]]],
         [[[2, 2, 4, 4], [3, 3, 6, 6]],
          [[3.1, 3.1, 6.1, 6.1], [1, 1, 8, 8]]]])
    rpn_boxes = {
        '2': tf.constant(rpn_boxes_np, dtype=tf.float32)
    }
    rpn_scores_np = np.array(
        [[[[0.6], [0.9]], [[0.2], [0.3]]], [[[0.1], [0.8]], [[0.3], [0.5]]]])
    rpn_scores = {
        '2': tf.constant(rpn_scores_np, dtype=tf.float32)
    }
    anchor_boxes_np = np.array(
        [[[[0, 0, 10, 10], [0.01, 0.01, 9.9, 9.9]],
          [[5, 5, 10, 10], [2, 2, 8, 8]]],
         [[[2, 2, 4, 4], [3, 3, 6, 6]],
          [[3.1, 3.1, 6.1, 6.1], [1, 1, 8, 8]]]])
    anchor_boxes = {
        '2': tf.constant(anchor_boxes_np, dtype=tf.float32)
    }
    image_shape = tf.constant([[20, 20], [20, 20]], dtype=tf.int32)

    selected_rois_np = np.array(
        [[[0.01, 0.01, 9.9, 9.9], [2, 2, 8, 8], [5, 5, 10, 10], [0, 0, 0, 0]],
         [[3, 3, 6, 6], [1, 1, 8, 8], [2, 2, 4, 4], [0, 0, 0, 0]]])
    selected_roi_scores_np = np.array(
        [[0.9, 0.3, 0.2, 0], [0.8, 0.5, 0.1, 0]])

    # Runs on TPU.
    strategy = tf.distribute.TPUStrategy()
    with strategy.scope():
      selected_rois_tpu, selected_roi_scores_tpu = (
          roi_generator._multilevel_propose_rois(
              rpn_boxes,
              rpn_scores,
              anchor_boxes=anchor_boxes,
              image_shape=image_shape,
              pre_nms_top_k=4,
              pre_nms_score_threshold=0.0,
              pre_nms_min_size_threshold=0.0,
              nms_iou_threshold=0.5,
              num_proposals=4,
              use_batched_nms=False,
              decode_boxes=False,
              clip_boxes=False,
              apply_sigmoid_to_score=False))

    # Runs on CPU.
    selected_rois_cpu, selected_roi_scores_cpu = (
        roi_generator._multilevel_propose_rois(
            rpn_boxes,
            rpn_scores,
            anchor_boxes=anchor_boxes,
            image_shape=image_shape,
            pre_nms_top_k=4,
            pre_nms_score_threshold=0.0,
            pre_nms_min_size_threshold=0.0,
            nms_iou_threshold=0.5,
            num_proposals=4,
            use_batched_nms=False,
            decode_boxes=False,
            clip_boxes=False,
            apply_sigmoid_to_score=False))

    self.assertNDArrayNear(
        selected_rois_tpu.numpy(), selected_rois_cpu.numpy(), 1e-5)
    self.assertNDArrayNear(
        selected_roi_scores_tpu.numpy(), selected_roi_scores_cpu.numpy(), 1e-5)

    self.assertNDArrayNear(
        selected_rois_tpu.numpy(), selected_rois_np, 1e-5)
    self.assertNDArrayNear(
        selected_roi_scores_tpu.numpy(), selected_roi_scores_np, 1e-5)

  def test_multilevel_propose_rois_two_levels(self):
    rpn_boxes_1_np = np.array(
        [[[[0, 0, 10, 10], [0.01, 0.01, 9.99, 9.99]],
          [[5, 5, 10, 10], [2, 2, 8, 8]]],
         [[[2, 2, 2.5, 2.5], [3, 3, 6, 6]],
          [[3.1, 3.1, 6.1, 6.1], [1, 1, 8, 8]]]])
    rpn_boxes_2_np = np.array(
        [[[[0, 0, 10.01, 10.01]]], [[[2, 2, 4.5, 4.5]]]])
    rpn_boxes = {
        '2': tf.constant(rpn_boxes_1_np, dtype=tf.float32),
        '3': tf.constant(rpn_boxes_2_np, dtype=tf.float32),
    }
    rpn_scores_1_np = np.array(
        [[[[0.6], [0.9]], [[0.2], [0.3]]], [[[0.1], [0.8]], [[0.3], [0.5]]]])
    rpn_scores_2_np = np.array([[[[0.95]]], [[[0.99]]]])
    rpn_scores = {
        '2': tf.constant(rpn_scores_1_np, dtype=tf.float32),
        '3': tf.constant(rpn_scores_2_np, dtype=tf.float32),
    }
    anchor_boxes_1_np = np.array(
        [[[[0, 0, 10, 10], [0.01, 0.01, 9.99, 9.99]],
          [[5, 5, 10, 10], [2, 2, 8, 8]]],
         [[[2, 2, 2.5, 2.5], [3, 3, 6, 6]],
          [[3.1, 3.1, 6.1, 6.1], [1, 1, 8, 8]]]])
    anchor_boxes_2_np = np.array(
        [[[[0, 0, 10.01, 10.01]]], [[[2, 2, 4.5, 4.5]]]])
    anchor_boxes = {
        '2': tf.constant(anchor_boxes_1_np, dtype=tf.float32),
        '3': tf.constant(anchor_boxes_2_np, dtype=tf.float32),
    }
    image_shape = tf.constant([[20, 20], [20, 20]], dtype=tf.int32)

    selected_rois_np = np.array(
        [[[0, 0, 10.01, 10.01], [0.01, 0.01, 9.99, 9.99]],
         [[2, 2, 4.5, 4.5], [3, 3, 6, 6]]])
    selected_roi_scores_np = np.array([[0.95, 0.9], [0.99, 0.8]])

    # Runs on TPU.
    strategy = tf.distribute.TPUStrategy()
    with strategy.scope():
      selected_rois_tpu, selected_roi_scores_tpu = (
          roi_generator._multilevel_propose_rois(
              rpn_boxes,
              rpn_scores,
              anchor_boxes=anchor_boxes,
              image_shape=image_shape,
              pre_nms_top_k=4,
              pre_nms_score_threshold=0.0,
              pre_nms_min_size_threshold=0.0,
              nms_iou_threshold=0.5,
              num_proposals=2,
              use_batched_nms=False,
              decode_boxes=False,
              clip_boxes=False,
              apply_sigmoid_to_score=False))

    # Runs on CPU.
    selected_rois_cpu, selected_roi_scores_cpu = (
        roi_generator._multilevel_propose_rois(
            rpn_boxes,
            rpn_scores,
            anchor_boxes=anchor_boxes,
            image_shape=image_shape,
            pre_nms_top_k=4,
            pre_nms_score_threshold=0.0,
            pre_nms_min_size_threshold=0.0,
            nms_iou_threshold=0.5,
            num_proposals=2,
            use_batched_nms=False,
            decode_boxes=False,
            clip_boxes=False,
            apply_sigmoid_to_score=False))

    self.assertNDArrayNear(
        selected_rois_tpu.numpy(), selected_rois_cpu.numpy(), 1e-5)
    self.assertNDArrayNear(
        selected_roi_scores_tpu.numpy(), selected_roi_scores_cpu.numpy(), 1e-5)

    self.assertNDArrayNear(
        selected_rois_tpu.numpy(), selected_rois_np, 1e-5)
    self.assertNDArrayNear(
        selected_roi_scores_tpu.numpy(), selected_roi_scores_np, 1e-5)

class MultilevelROIGeneratorTest(tf.test.TestCase):

  def test_serialize_deserialize(self):
    kwargs = dict(
        pre_nms_top_k=2000,
        pre_nms_score_threshold=0.0,
        pre_nms_min_size_threshold=0.0,
        nms_iou_threshold=0.7,
        num_proposals=1000,
        test_pre_nms_top_k=1000,
        test_pre_nms_score_threshold=0.0,
        test_pre_nms_min_size_threshold=0.0,
        test_nms_iou_threshold=0.7,
        test_num_proposals=1000,
        use_batched_nms=False,
    )
    generator = roi_generator.MultilevelROIGenerator(**kwargs)

    expected_config = dict(kwargs)
    self.assertEqual(generator.get_config(), expected_config)

    new_generator = roi_generator.MultilevelROIGenerator.from_config(
        generator.get_config())

    self.assertAllEqual(generator.get_config(), new_generator.get_config())

if __name__ == '__main__':
  tf.test.main()

max_stars_repo_name: arash-mehrzadi/models

max_issues_repo_name: arash-mehrzadi/models

max_forks_repo_name: arash-mehrzadi/models


text:
import functools
from numbers import Number

import numpy as np
import torch

HANDLED_FUNCTIONS = {}

# def implements(torch_function):
#     """Register a torch function override for ScalarTensor"""
#
#     @functools.wraps(torch_function)
#     def decorator(func):
#         HANDLED_FUNCTIONS[torch_function] = func
#         return func
#
#     return decorator
#
#
# def ensure_tensor(data):
#     if isinstance(data, MultiVector):
#         return data.tensor()
#     return torch.as_tensor(data)

def comb(n, k):
    """\
    Returns /n\\
            \\k/

    comb(n, k) --> PyInt
    """

    def fact(n):
        if n == 0:
            return 1
        return np.multiply.reduce(range(1, n + 1))

    return int(fact(n) / (fact(k) * fact(n - k)))

def elements(dims, firstIdx=0):
    """Return a list of tuples representing all 2**dims of blades
    in a dims-dimensional GA.

    elements(dims, firstIdx=0) --> bladeTupList
    """

    indcs = list(range(firstIdx, firstIdx + dims))

    blades = [()]

    for k in range(1, dims + 1):
        # k = grade

        if k == 1:
            for i in indcs:
                blades.append((i,))
            continue

        curBladeX = indcs[:k]

        for i in range(comb(dims, k)):
            if curBladeX[-1] < firstIdx + dims - 1:
                # increment last index
                blades.append(tuple(curBladeX))
                curBladeX[-1] = curBladeX[-1] + 1

            else:
                marker = -2
                tmp = curBladeX[:]  # copy
                tmp.reverse()

                # locate where the steady increase begins
                for j in range(k - 1):
                    if tmp[j] - tmp[j + 1] == 1:
                        marker = marker - 1
                    else:
                        break

                if marker < -k:
                    blades.append(tuple(curBladeX))
                    continue

                # replace
                blades.append(tuple(curBladeX))
                curBladeX[marker:] = list(range(
                    curBladeX[marker] + 1, curBladeX[marker] + 1 - marker))

    return blades

class MultiVector:
    '''
    for now, this just supports Cl(p, q=0)
    '''

    def __init__(self, dim=4, data=None, names=None, device='cpu'):
        if data is None:
            self.device = torch.device(device)
            self._data = torch.zeros(2 ** dim, dtype=torch.float32, device=self.device)
        else:
            self.device = data.device
            self._data = data
        self._dim = dim

        # sig = [+1] * dim
        bladeTupleList = elements(dim, firstIdx=1)
        self.bladeTupleList = list(map(tuple, bladeTupleList))
        self.gradeList = list(map(len, self.bladeTupleList))
        # create names
        if names is None or isinstance(names, str):
            if isinstance(names, str):
                e = names
            else:
                e = 'e'
            self.names = []
            for i in range(self.gaDims):
                if self.gradeList[i] >= 1:
                    self.names.append(e + ''.join(map(str, self.bladeTupleList[i])))
                else:
                    self.names.append('')
        elif len(names) == self.gaDims:
            self.names = names
        else:
            raise ValueError(
                "names list of length %i needs to be of length %i" %
                (len(names), self.gaDims))

    @property
    def dim(self):
        return self._dim

    @property
    def len(self):
        return len(self._data)

    @property
    def gaDims(self):
        return 2 ** self.dim

    @property
    def value(self):
        return self._data

    def __getitem__(self, i):
        return self._data[i]

    def __setitem__(self, i, value):
        self._data[i] = value

    def __delitem__(self, i):
        self._data[i] = 0.

    def __iter__(self):
        for i in range(self.gaDims):
            if self._data[i]:
                yield i

    def __pos__(self):
        x = MultiVector(self.dim, device=self.device)
        x._data = self._data.clone()
        return x

    def __neg__(self):
        x = MultiVector(self.dim, device=self.device)
        x._data = - self._data
        return x

    def __eq__(self, other):
        return torch.eq(self._data, other._data)

    def __len__(self):
        """Return the number of nonzero terms in this multivector."""
        # return len(self._data) - self._data.count(0.0)
        try:
            return len(self._data.nonzero(as_tuple=True)[0])
        except IndexError:
            return 0

    def __add__(self, other):
        if isinstance(other, (Number, torch.Tensor)):
            x = +self
            x[0] += other
            return x
        elif isinstance(other, MultiVector):
            # "mv have the same gaDims"
            if self.gaDims == other.gaDims:
                x = MultiVector(self.dim, device=self.device)
                x._data = self._data + other._data
                return x
            else:
                m = max(self.dim, other.dim)
                x = MultiVector(m)
                for i in range(self.gaDims):
                    x._data[i] = self._data[i]
                for i in range(other.gaDims):
                    x._data[i] += other._data[i]
                return x
        else:
            raise NotImplemented

    def __radd__(self, other):
        return self + other

    def __sub__(self, other):
        return self + (-other)

    def __rsub__(self, other):
        return - self + other

    def __mul__(self, other):
        if isinstance(other, (Number, torch.Tensor)):
            x = MultiVector(self.dim, device=self.device)
            x._data = self._data * other
            return x
        elif isinstance(other, MultiVector):
            if self.dim == other.dim:
                x = MultiVector(self.dim, device=self.device)
                for i in range(self.gaDims):
                    for j in range(other.gaDims):
                        k, s = MultiVector._blade_combine(i, j)
                        x._data[k] += self._data[i] * other._data[j] * s
                return x
            else:
                m = max(self.dim, other.dim)
                x = MultiVector(m, device=self.device)
                for i in range(self.len):
                    if self._data[i]:
                        for j in range(other.len):
                            if other._data[j]:
                                k, s = MultiVector._blade_combine(i, j)
                                x._data[k] += self._data[i] * other._data[j] * s
                return x
        else:
            raise NotImplemented

    def __rmul__(self, other):
        if isinstance(other, (Number, torch.Tensor)):
            return self * other
        elif isinstance(other, MultiVector):
            return other * self
        else:
            raise NotImplemented

    def __and__(self, other):
        '''
        Find the outer (wedge) product of the two multivectors, or of a multivector
        and a number.
        '''
        x = self * other - other * self
        x._data = x._data / 2
        return x

    def __matmul__(self, other):
        '''
        Find the inner (dot) product of the two multivectors, or of a multivector
        and a number.
        '''
        x = self * other + other * self
        x._data = x._data / 2
        return x

    def __or__(self, other):
        '''
        Find the meet (vee) of the two multivectors, or of a multivector
        and a number.
        '''
        return self.dual() * other

    def __abs__(self):
        '''
        Find the norm of the multivector
        '''
        x = 0.0
        for i in self:
            a = self._data[i]
            x += a * a
        return torch.sqrt(x)

    def __invert__(self):
        x = MultiVector(self.dim, device=self.device)
        for i in range(self.len):
            v = self._data[i]
            if v:
                r = MultiVector._rank(i) % 4
                if r == 2 or r == 3:
                    x[i] = -v
                else:
                    x[i] = v
        return x

    def rank(self):
        r = - torch.Tensor(float('Inf'))
        for i in range(len(self._data)):
            r = max(r, MultiVector._rank(i))
        return r

    def cross_product(self, other):
        return (self & other).dual()

    # def left_inv(self):
    #     try:
    #         x = ~self
    #         s = 1.0 / float(self * x)
    #     except TypeError:
    #         return NotImplemented
    #     x._data = x._data * s
    #     return x
    #
    # def right_inv(self):
    #     try:
    #         x = ~self
    #         s = 1.0 / float(x * self)
    #     except TypeError:
    #         return NotImplemented
    #     x._data = x._data * s
    #     return x

    def __truediv__(self, other):
        '''
        Divide the multivectors, if possible.  Or divide the multivector by the scalar.
        '''
        if isinstance(other, (Number, torch.Tensor)):
            if isinstance(other, torch.Tensor):
                assert len(other.shape) == 0, "the tensor must be scalar"
            x = MultiVector(self.dim, device=self.device)
            x._data = self._data / other
            return x
        if isinstance(other, MultiVector):
            # TODO: still have some bugs! Not use this!
            raise NotImplementedError("Not Implement the mv / mv")

    def dual(self):
        """
        Return the dual of the multivector.
        """
        x = self.I
        return self * x * x * x

    @property
    def I(self):
        """
        Return the standard pseudoscalar of the algebra this multivector is in.
        """
        x = MultiVector(self.dim, device=self.device)
        x._data[-1] = 1.0
        return x

    @staticmethod
    def _rank(a):
        return bin(a).count('1')

    @staticmethod
    def _blade_combine(a, b):
        if a == 0:
            return b, 1
        if b == 0:
            return a, 1
        c = a ^ b
        s = 1
        p = max(a, b)
        d = MultiVector._rank(a)
        e = 1
        while e <= p:
            if e & a:
                d -= 1
            if (d & 1) and (e & b):
                s = -s
            e *= 2
        return c, s

    def __str__(self):
        result = ""
        for i in self:
            coef = self[i].item()
            if result:
                result += ' + '
            result += str(coef)
            bit = 1
            k = 1
            first = True
            while bit <= i:
                if i & bit:
                    if first:
                        result += '*['
                        first = False
                    else:
                        result += ','
                    result += 'e%d' % k
                k += 1
                bit *= 2
            if not first:
                result += ']'
        if not result:
            return '0'
        return result

    def __repr__(self):
        return self.__str__()

    def set_values(self, values, index):
        self._data = torch.zeros(self.gaDims, dtype=torch.float32, device=self.device)
        self._data.index_add_(0, index, values)

if __name__ == '__main__':
    a = MultiVector(4)
    a.set_values(torch.tensor([1., 1, 2, 3, 4]), torch.tensor([0, 1, 2, 3, 4]))
    b = MultiVector(4)
    b.set_values(torch.tensor([1., -1]), torch.tensor([1, 2]))
    c = MultiVector(4)
    c.set_values(torch.ones(16, dtype=torch.float32), torch.arange(16))
    print(a + b)
    print(a - b)
    print(a * b)
    print(a / 2)


text:
import numpy as np
import pandas as pd
import pytask

from src.algorithm import draw_candidate_matchings
from src.algorithm import find_best_matching
from src.algorithm import update_matchings_history
from src.config import BLD
from src.config import SRC
from src.read_and_write import read_config
from src.read_and_write import read_matchings_history
from src.read_and_write import read_names
from src.read_and_write import write_file

def format_matching_as_str(matching, names):
    """Format matching in human readable string.

    Args:
        matching (list): Matching in list form. (BETTER EXPLAINATION)
        names (pd.DataFrame): names df, see func ``read_names``

    Returns:
        text (str): The formatted text as string.

    """
    names = names.set_index("id").copy()
    texts = [", ".join(names["name"].loc[group].values) for group in matching]

    text = ""
    for k, text_ in enumerate(texts):
        text += f"Group {k}: {text_}\n"

    return text

def get_participants(names):
    """Extract current participants from names data frame.

    Args:
        names (pd.DataFrame): names.csv file converted to pd.DataFrame

    Returns:
        participants (pd.Series): Series containing ids of individuals that will join.

    """
    participants = names.query("joins == 1")["id"]
    return participants

def add_new_individuals(matchings_history, names):
    """Add new individuals to matchings_history data frame.

    Args:
        matchings_history (pd.DataFrame): Square df containing group information. Index
            and column is given by the 'id' column in src/data/names.csv.
        names (pd.DataFrame): names.csv file converted to pd.DataFrame

    Returns:
        updated (pd.DataFrame): Like matchings_history but with new individuals.

    """
    matchings_history_id = matchings_history.index.values
    names_id = names["id"].values

    new_ids = np.setdiff1d(names_id, matchings_history_id)

    n_new = len(new_ids)
    n_old = len(matchings_history)

    if n_new == 0:
        updated = matchings_history.copy()
    else:
        to_append = pd.DataFrame(np.zeros(n_new, n_old), dtype=int, index=new_ids)
        updated = pd.concat((matchings_history, to_append), axis=0)
        updated = pd.concat((updated, to_append), axis=1)

    return updated

@pytask.mark.build
@pytask.mark.depends_on(SRC / "data" / "matchings_history.csv")
@pytask.mark.produces(
    [
        BLD / "matchings_history.csv",
        BLD / "matching.txt",
    ]
)
def task_create_matchings(depends_on, produces):  # noqa: D103
    config = read_config()
    names = read_names()
    matchings_history = read_matchings_history()

    matchings_history = add_new_individuals(matchings_history, names)
    participants = get_participants(names)

    candidates = draw_candidate_matchings(
        participants, config["min_size"], config["n_candidates"], config["initial_seed"]
    )
    best_matching = find_best_matching(candidates, matchings_history)

    updated_history = update_matchings_history(matchings_history, best_matching)
    updated_history.to_csv(produces[0])

    text = format_matching_as_str(best_matching, names)
    write_file(text, produces[1])


text:
from __future__ import absolute_import
import os
import json
import torch
import argparse
import torchvision
import numpy as np
import torch.nn as nn
from DataLoader import CDATA
from main_model_nn import MemN2NDialog
from torch.autograd import Variable as Var
from data_utils import load_candidates, load_dialog_task, vectorize_candidates

dtype = torch.FloatTensor
if torch.cuda.device_count() > 0:
    dtype = torch.cuda.FloatTensor

class chatBot(object):
    def __init__(self, data_dir, model_dir, task_id, isInteractive=True, OOV=False,
                 memory_size=50, random_state=None, batch_size=32, learning_rate=0.001, epsilon=1e-8,
                 max_grad_norm=40.0, evaluation_interval=10, hops=3, epochs=200, embedding_size=20, save_model=10,
                 checkpoint_path='./models', optim='adam', momentum=0.9, decay=0, gamma=0.1, step=30):
        self.data_dir = data_dir
        self.task_id = task_id
        self.model_dir = model_dir
        self.isInteractive = isInteractive
        self.OOV = OOV
        self.memory_size = memory_size
        self.random_state = random_state
        self.batch_size = batch_size
        self.learning_rate = learning_rate
        self.epsilon = epsilon
        self.max_grad_norm = max_grad_norm
        self.evaluation_interval = evaluation_interval
        self.hops = hops
        self.epochs = epochs
        self.embedding_size = embedding_size
        self.save_model = save_model
        self.checkpoint_path = checkpoint_path
        self.optim = optim
        self.momentum = momentum
        self.decay = decay
        self.gamma = gamma
        self.step = step

        self.train_dataset = CDATA(data_dir=self.data_dir, task_id=self.task_id, memory_size=self.memory_size,
                                   train=0, batch_size=self.batch_size, nn=False)  # 0->train, 1->validate, 2->test
        self.model = MemN2NDialog(batch_size=self.batch_size, vocab_size=self.train_dataset.getParam('vocab_size'),
                                  candidate_size=self.train_dataset.getParam('candidate_sentence_size'), sentence_size=self.train_dataset.getParam('sentence_size'),
                                  candidates_vec=self.train_dataset.getParam('candidates_vec'), embedding_size=self.embedding_size, hops=self.hops,
                                  learning_rate=self.learning_rate, max_grad_norm=self.max_grad_norm, task_id=self.task_id)
        if torch.cuda.is_available():
            self.model = self.model.cuda()

    def train(self):
        trainS, trainQ, trainA = self.train_dataset.getData()
        assert len(trainS) == len(trainQ) and len(trainQ) == len(trainA)
        n_train = len(trainS)
        batches = zip(range(0, n_train - self.batch_size, self.batch_size),
                      range(self.batch_size, n_train, self.batch_size))
        batches = [(start, end) for start, end in batches]

        if self.optim == 'sgd':
            optimizer = torch.optim.SGD(self.model.parameters(), lr=self.learning_rate, momentum=self.momentum)
            print("SGD optimizer")
        elif self.optim == 'rms':
            optimizer = torch.optim.RMSprop(self.model.parameters(), lr=self.learning_rate, momentum=self.momentum)
            print("RMSprop optimizer")
        else:
            optimizer = torch.optim.Adam(self.model.parameters(), lr=self.learning_rate)
            print("Adam optimizer")

        scheduler = None
        if self.decay:
            scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=self.step, gamma=self.gamma)
            print("Decay learning rate initialized")

        for epoch in range(self.epochs):
            print('epoch :', epoch)
            if self.decay:
                scheduler.step()
            np.random.shuffle(batches)
            running_loss = 0.0
            for start, end in batches:
                s = trainS[start:end]
                q = trainQ[start:end]
                a = trainA[start:end]
                optimizer.zero_grad()
                running_loss += self.model.batch_train(s, q, a)
                optimizer.step()
            print('loss = ',running_loss / n_train)
            #-----------------------Save model after every nth epoch-----------------------------------
            if epoch % self.save_model == 0:
                print("Saving models")
                if not os.path.exists(self.checkpoint_path):
                    os.makedirs(self.checkpoint_path)
                model_name = os.path.join(self.checkpoint_path, str(self.task_id) + '.pkl')
                torch.save(self.model.state_dict(), model_name)
                self.test(0)
                self.test(1)
                self.test(2)
            #------------------------------------------------------------------------------------------

    def test(self,data_type):
        # 0->train, 1->validate, 2->test
        print("----------------------------------------------------------------------")
        print("STARTED TESTING: ", data_type)
        dataset = CDATA(data_dir=self.data_dir, task_id=self.task_id, memory_size=self.memory_size,
                        train=data_type, batch_size=self.batch_size)  # 0->train, 1->validate, 2->test
        testS, testQ, testA = dataset.getData()
        assert len(testS) == len(testQ) and len(testQ) == len(testA)
        n_test = len(testS)
        batches = zip(range(0, n_test - self.batch_size, self.batch_size),
                      range(self.batch_size, n_test, self.batch_size))
        batches = [(start, end) for start, end in batches]

        fname = os.path.join(self.checkpoint_path, str(self.task_id) + '.pkl')
        self.model.load_state_dict(torch.load(fname))

        loss_cur = 0.0
        np.random.shuffle(batches)
        running_loss = 0.0
        cnt = 0
        accuracy = 0.0
        for start, end in batches:
            s = trainS[start:end]
            q = trainQ[start:end]
            a = trainA[start:end]
            tmp1, _ += self.model.test(s, q, a)
            accuracy += tmp1
            cnt += 1

        acc, loss = self.model.test(testS, testQ, testA)
        print('Accuracy = ', accuracy/cnt)
        print("----------------------------------------------------------------------")

    def build_vocab(self, data, candidates):
        vocab = reduce(lambda x, y: x | y, (set(
            list(chain.from_iterable(s)) + q) for s, q, a in data))
        vocab |= reduce(lambda x, y: x | y, (set(candidate)
                                             for candidate in candidates))
        vocab = sorted(vocab)
        self.word_idx = dict((c, i + 1) for i, c in enumerate(vocab))
        max_story_size = max(map(len, (s for s, _, _ in data)))
        mean_story_size = int(np.mean([len(s) for s, _, _ in data]))
        self.sentence_size = max(
            map(len, chain.from_iterable(s for s, _, _ in data)))
        self.candidate_sentence_size = max(map(len, candidates))
        query_size = max(map(len, (q for _, q, _ in data)))
        self.memory_size = min(self.memory_size, max_story_size)
        self.vocab_size = len(self.word_idx) + 1  # +1 for nil word
        self.sentence_size = max(
            query_size, self.sentence_size)  # for the position
        # params
        print("vocab size:", self.vocab_size)
        print("Longest sentence length", self.sentence_size)
        print("Longest candidate sentence length",
              self.candidate_sentence_size)
        print("Longest story length", max_story_size)
        print("Average story length", mean_story_size)

def main(params):
    model_dir = "task" + str(params['task_id']) + "_" + params['model_dir']
    if not os.path.exists(model_dir):
        os.makedirs(model_dir)
    chatbot = chatBot(data_dir=params['data_dir'], model_dir=model_dir, task_id=params['task_id'], isInteractive=params['interactive'], OOV=params['OOV'], memory_size=params['memory_size'], random_state=params['random_state'], batch_size=params['batch_size'],
                      learning_rate=params['learning_rate'], epsilon=params['epsilon'], max_grad_norm=params['max_grad_norm'], evaluation_interval=params['evaluation_interval'], hops=params['hops'], epochs=params['epochs'], embedding_size=params['embedding_size'],
                      save_model=params['save_model'], checkpoint_path=params['checkpoint_path'], optim=params['optim'], momentum=params['momentum'],
                      decay=params['decay'], gamma=params['gamma'], step=params['step'])
    if params['train']:
        chatbot.train()
    else:
        chatbot.test(0)
        chatbot.test(1)
        chatbot.test(2)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--learning_rate', default=0.001, type=float,
                        help='Learning rate for Optimizer')
    parser.add_argument('--epsilon', default=1e-8, type=float,
                        help='Epsilon value for Adam Optimizer')
    parser.add_argument('--max_grad_norm', default=40.0, type=float,
                        help='Clip gradients to this norm')
    parser.add_argument('--evaluation_interval', default=10, type=int,
                        help='Evaluate and print results every x epochs')
    parser.add_argument('--batch_size', default=32, type=int, help='Batch size for training')
    parser.add_argument('--hops', default=3, type=int, help='Number of hops in the Memory Network')
    parser.add_argument('--epochs', default=200, type=int, help='Number of epochs to train for')
    parser.add_argument('--embedding_size', default=20, type=int,
                        help='Embedding size for embedding matrices')
    parser.add_argument('--memory_size', default=50, type=int, help='Maximum size of memory')
    parser.add_argument('--task_id', default=6, type=int, help='bAbI task id, 1 <= id <= 6')
    parser.add_argument('--random_state', default=None, help='Random state')
    parser.add_argument('--data_dir', default='data/dialog-bAbI-tasks/',
                        help='Directory containing bAbI tasks')
    parser.add_argument('--model_dir', default='model/',
                        help='Directory containing memn2n model checkpoints')
    parser.add_argument('--train', default=1, type=int, help='Train if True, test if False')
    parser.add_argument('--interactive', default=0, type=int, help='if True, interactive')
    parser.add_argument('--OOV', default=0, type=int, help='if True, use OOV test set')
    parser.add_argument('--print_params', default=1, type=int,
                        help='pass False to turn off printing input parameters')
    parser.add_argument('--save_model', default=1, type=int,
                        help='Save model after every x epochs')
    parser.add_argument('--checkpoint_path', default='./models',
                        help='Path to the directory to save models')
    parser.add_argument('--optim', default='adam',
                        help='Optimizer to be used')
    parser.add_argument('--momentum', default=0.9, type=int,
                        help='momentum for RMSprop/SGD')
    parser.add_argument('--decay', default=0, type=int,
                        help='1 for learning rate decay')
    parser.add_argument('--gamma', default=0.1, type=float,
                        help='factor for decay of learning rate')
    parser.add_argument('--step', default=30, type=int,
                        help='declearse learning rate by gamma after every x steps')

    args = parser.parse_args()
    params = vars(args)  # convert to ordinary dict
    if params['print_params']:
        print('parsed input parameters:')
        print(json.dumps(params, indent=2))
main(params)


text:
import sys
import os
import pickle
import math
import numpy as np
import matplotlib.pyplot as plt
from datetime import datetime

VED_min = 155
VED_max = 180
file_info = '_RI2.26-1.26_density1.8_calib_factor225-sig_precip_72hrs_pre_arrival'
variable_of_interest = 'coat_th' #coat_th

start_date = datetime.strptime('2012/04/01 00:00', '%Y/%m/%d %H:%M')
end_date = datetime.strptime('2012/05/31 00:00', '%Y/%m/%d %H:%M')

os.chdir('C:/Users/Sarah Hanna/Documents/Data/WHI long term record/coatings/')
file = open('coating thicknesses by air mass for 153.37nm to 181.12nm-density1.8_RI2.26-1.26_calib_factor225-sig_precip_72hrs_pre_arrival.binpickl', 'r')
coating_data = pickle.load(file)
file.close()

coating_info = {}
coating_info_sig_rain = {}
lists = []
for air_mass_name, air_mass_info in coating_data.iteritems():
	print air_mass_name

	if air_mass_name in ['cluster_1']:
		air_mass = 'Bering'
	if air_mass_name in ['cluster_2']:
		air_mass = 'N. Coastal/Continental'
	if air_mass_name in ['cluster_3']:
		air_mass = 'N. Pacific'
	if air_mass_name in ['cluster_4','cluster_6']:
		air_mass = 'S. Pacific'
	if air_mass_name in ['cluster_5']:
		air_mass = 'W. Pacific/Asia'
	if air_mass_name in ['cluster_GBPS']:
		air_mass = '>= 24hrs in GBPS'
	if air_mass_name in ['fresh']:
		air_mass = 'Fresh Emissions'

	coatings=[]
	for row in air_mass_info:
		rBC_VED = row[0]
		coat_th = row[1]
		date = row[2]
		sig_rain_str = row[3]
		if sig_rain_str == 'True':
			sig_rain = True
		if sig_rain_str == 'False':
			sig_rain = False

		if  (VED_min <= rBC_VED <VED_max) and (start_date <= date <= end_date):

			Dshell_Dcore = (rBC_VED+2*coat_th)/rBC_VED
			if variable_of_interest == 'coat_th':
				data_type = 'Coating Thickness (nm)'
				var_of_interest =coat_th
			if variable_of_interest == 'Dshell_Dcore':
				data_type = 'Dshell_Dcore'
				var_of_interest =Dshell_Dcore

			if sig_rain == False:
				if air_mass in coating_info:
					coating_info[air_mass].append(var_of_interest)
				else:
					coating_info[air_mass] = [var_of_interest]

			if sig_rain == True:
				if air_mass in coating_info_sig_rain:
					coating_info_sig_rain[air_mass].append(var_of_interest)
				else:
					coating_info_sig_rain[air_mass] = [var_of_interest]

#####plotting
stats = []
for key,data in coating_info.iteritems():
	air_mass = key
	air_mass_coating_info = data
	print air_mass, np.median(air_mass_coating_info), np.mean(air_mass_coating_info), len(air_mass_coating_info)
	stats.append([air_mass, np.median(air_mass_coating_info), np.mean(air_mass_coating_info), np.percentile(air_mass_coating_info, 10),np.percentile(air_mass_coating_info, 90), len(air_mass_coating_info)])
	print '\n'

stats_precip = []
for key,data in coating_info_sig_rain.iteritems():
	air_mass = key
	air_mass_coating_info = data
	print air_mass, np.median(air_mass_coating_info), np.mean(air_mass_coating_info), len(air_mass_coating_info)
	stats_precip.append([air_mass, np.median(air_mass_coating_info), np.mean(air_mass_coating_info), np.percentile(air_mass_coating_info, 10),np.percentile(air_mass_coating_info, 90), len(air_mass_coating_info)])
	print '\n'

#save stats to file
os.chdir('C:/Users/Sarah Hanna/Documents/Data/WHI long term record/coatings/')
file = open('coating (' + data_type + ') histos by air mass - '+ str(VED_min) + '-' +str(VED_max) + 'nm rBC cores_' + file_info +'-no precip.txt', 'w')
file.write('air_mass' + '\t' +  'median'  + '\t'+ 'mean' + '\t' + '10th_percentile' + '\t' + '90th_percentile' + '\t' +'number_of_particles' +'\n')
for row in stats:
	line = '\t'.join(str(x) for x in row)
	file.write(line + '\n')
file.close()

#save stats_precip to file
os.chdir('C:/Users/Sarah Hanna/Documents/Data/WHI long term record/coatings/')
file = open('coating (' + data_type + ') histos by air mass - '+ str(VED_min) + '-' +str(VED_max) + 'nm rBC cores_' + file_info +'-precip.txt', 'w')
file.write('air_mass' + '\t' +  'median'  + '\t'+ 'mean' + '\t' + '10th_percentile' + '\t' + '90th_percentile' + '\t' +'number_of_particles' +'\n')
for row in stats_precip:
	line = '\t'.join(str(x) for x in row)
	file.write(line + '\n')
file.close()

###plotting

fig, axes = plt.subplots(7,1, figsize=(6, 12), facecolor='w', edgecolor='k')
fig.subplots_adjust(hspace = 0., wspace=0.08)

axs = axes.ravel()

bin_number = 40
y_ulim = 0.01

distrs_to_plot = ['Bering','N. Coastal/Continental','N. Pacific','S. Pacific','W. Pacific/Asia','>= 24hrs in GBPS','Fresh Emissions']

i = 0
for distr_to_plot in distrs_to_plot:
	try:
		axs[i].hist(coating_info[distr_to_plot], bins=bin_number,histtype='step', normed = 1, color='red')
		axs[i].hist(coating_info_sig_rain[distr_to_plot], bins=bin_number,histtype='step', normed = 1, color='blue')
		#axs[i].axvline(np.median(coating_info[distr_to_plot]), color='r', linestyle='--')
	except:
		axs[i].hist([0,0], bins=1,histtype='step', normed = 1)

	axs[i].text(0.6, 0.8,distr_to_plot, transform=axs[i].transAxes)
	axs[i].axes.get_yaxis().set_visible(False)
	if var_of_interest == Dshell_Dcore:
		axs[i].set_xlim(0.5,2.5)
	if var_of_interest == coat_th:
		axs[i].set_xlim(-40,150)
	#if i == 6:
	#	axs[i].set_xlabel(data_type)
	i+=1

plt.savefig('coating (' + data_type + ') histos by air mass - '+ str(VED_min) + '-' +str(VED_max) + 'nm rBC cores_' + file_info +'.png', bbox_inches='tight')

plt.show()

##box-plots
fig = plt.figure( figsize=(10, 6))
ax = fig.add_subplot(111)

distrs_to_plot = ['Bering','N. Coastal/Continental','N. Pacific','S. Pacific','W. Pacific/Asia','>= 24hrs in GBPS','Fresh Emissions']

no_sig_rain_boxplt_data = []
sig_rain_boxplt_data = []

for distr_to_plot in distrs_to_plot:
	no_sig_rain_boxplt_data.append(coating_info[distr_to_plot])
	sig_rain_boxplt_data.append(coating_info_sig_rain[distr_to_plot])

x=np.array([1,2,3,4,5,6,7])
no_sig_rain = ax.boxplot(no_sig_rain_boxplt_data, whis=[10,90],sym='',positions=x+0.15, widths = 0.2)
sig_rain = ax.boxplot(sig_rain_boxplt_data, whis=[10,90],sym='',positions=x-0.15, widths = 0.2)

#legend
hSR, = ax.plot([1,1],'b-')
hNR, = ax.plot([1,1],'r-')
plt.legend((hSR, hNR),('Sig Precip 72hrs pre-arrival', 'No Sig Precip 72hrs pre-arrival'))
hSR.set_visible(False)
hNR.set_visible(False)

plt.setp(no_sig_rain['boxes'], color='r')
plt.setp(no_sig_rain['whiskers'], color='r',linestyle='-')
plt.setp(no_sig_rain['caps'], color='r')
plt.setp(no_sig_rain['medians'], color='r')

plt.setp(sig_rain['boxes'], color='b')
plt.setp(sig_rain['whiskers'], color='b',linestyle='-')
plt.setp(sig_rain['caps'], color='b')
plt.setp(sig_rain['medians'], color='b')

ax.set_xlim(0,8)

plt.xticks([1,2,3,4,5,6,7], ['Bering','N. Coastal\n/Continental','N. Pacific','S. Pacific','W. Pacific/Asia','>= 24hrs in GBPS','Fresh Emissions'],rotation=60)

ax.yaxis.grid(True, linestyle='-', which='major', color='grey', alpha=1)

if var_of_interest == Dshell_Dcore:
	ax.set_ylabel('Dp/Dc')
if var_of_interest == coat_th:
	ax.set_ylabel('Coating Thickness (nm)')
if i == 6:
	axs[i].set_xlabel(data_type)

plt.savefig('coating (' + data_type + ') boxplots by air mass - '+ str(VED_min) + '-' +str(VED_max) + 'nm rBC cores_' + file_info +'.png', bbox_inches='tight')

plt.show()


text:
"""
Convert to and from Roman numerals.

This file is adapted from the roman package that is available on PyPI,
which has the following copyright/license information:

Copyright (c) 2001 Mark Pilgrim

This program is free software; you can redistribute it and/or modify
it under the terms of the Python 2.1.1 license, available at
https://www.python.org/download/releases/2.1.1/license/

"""
__all__ = [
    "to_roman",
    "from_roman",
    "is_roman_numeral",
]

import numpy as np
import re

from numbers import Integral
from typing import Union

from plasmapy.utils.exceptions import InvalidRomanNumeralError, OutOfRangeError

# Define digit mapping
_romanNumeralMap = (
    ("M", 1000),
    ("CM", 900),
    ("D", 500),
    ("CD", 400),
    ("C", 100),
    ("XC", 90),
    ("L", 50),
    ("XL", 40),
    ("X", 10),
    ("IX", 9),
    ("V", 5),
    ("IV", 4),
    ("I", 1),
)

# Define pattern to detect valid Roman numerals
_romanNumeralPattern = re.compile(
    """
    ^                   # beginning of string
    M{0,4}              # thousands - 0 to 4 M's
    (CM|CD|D?C{0,3})    # hundreds - 900 (CM), 400 (CD), 0-300 (0 to 3 C's),
                        #            or 500-800 (D, followed by 0 to 3 C's)
    (XC|XL|L?X{0,3})    # tens - 90 (XC), 40 (XL), 0-30 (0 to 3 X's),
                        #        or 50-80 (L, followed by 0 to 3 X's)
    (IX|IV|V?I{0,3})    # ones - 9 (IX), 4 (IV), 0-3 (0 to 3 I's),
                        #        or 5-8 (V, followed by 0 to 3 I's)
    $                   # end of string
    """,
    re.VERBOSE,
)

def to_roman(n: Union[Integral, np.integer]) -> str:
    """
    Convert an integer to a Roman numeral.

    Parameters
    ----------
    n : int or `~numpy.integer`
        The integer to be converted to a Roman numeral that must be
        between 1 and 4999, inclusive.

    Returns
    -------
    result : str
        The number in Roman numeral notation.

    Raises
    ------
    TypeError
        If the input is not an integer.

    OutOfRangeError
        If the number is not between 1 and 4999, inclusive.

    See Also
    --------
    from_roman

    Examples
    --------
    >>> to_roman(5)
    'V'
    >>> to_roman(2525)
    'MMDXXV'

    """
    if not isinstance(n, (Integral, np.integer)):
        raise TypeError(f"{n} cannot be converted to a Roman numeral.")
    if not (0 < n < 5000):
        raise OutOfRangeError("Number is out of range (need 0 < n < 5000)")

    result = ""
    for numeral, integer in _romanNumeralMap:
        while n >= integer:
            result += numeral
            n -= integer
    return result

def from_roman(s: str) -> Integral:
    """
    Convert a Roman numeral to an integer.

    Parameters
    ----------
    s : str
        A Roman numeral.

    Returns
    -------
    result : int
        The integer corresponding to the Roman numeral.

    Raises
    ------
    TypeError
        The argument is not a `str`.

    InvalidRomanNumeralError
        The argument is not a valid Roman numeral.

    See Also
    --------
    to_roman

    Examples
    --------
    >>> from_roman('V')
    5
    >>> from_roman('MMMMCCCLXVII')
    4367

    """
    if not isinstance(s, str):
        raise TypeError("The argument to from_roman must be a string.")
    if not _romanNumeralPattern.search(s):
        raise InvalidRomanNumeralError("Invalid Roman numeral: %s" % s)

    result = 0
    index = 0
    for numeral, integer in _romanNumeralMap:
        while s[index : index + len(numeral)] == numeral:
            result += integer
            index += len(numeral)
    return result

def is_roman_numeral(s: str) -> bool:
    """
    Check whether or not a string is a valid Roman numeral.

    Parameters
    ----------
    s : str
        The possible Roman numeral.

    Returns
    -------
    result : bool
        `True` if the `str` input is a valid Roman numeral, and `False`
        if it is not.

    Raises
    ------
    TypeError
        If the argument is not a `str`.

    See Also
    --------
    to_roman

    from_roman

    Examples
    --------
    >>> is_roman_numeral("CXVII")
    True
    >>> is_roman_numeral("42")
    False

    """
    if not isinstance(s, str):
        raise TypeError("Only strings may be tested ")
    result = bool(_romanNumeralPattern.match(s))
    return result

max_stars_repo_name: kjmontes/PlasmaPy

max_issues_repo_name: kjmontes/PlasmaPy

max_forks_repo_name: kjmontes/PlasmaPy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
sys.path.append(sys.argv[1])

import unittest
import os
import shutil

import numpy as np
from PIL import Image, ImageChops, ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True

from bother_utils.heightmap import *
from bother_utils.srtm import *

EXAMPLES_DIR = 'examples'
TEST_DIR = 'test_data'

test_coords = {
    'mallorca': (39.195468, 2.272535, 39.998336, 3.584351),
    'ireland': (51.296094, -11.017116, 55.596810, -4.403352),
    'constance': (47.321556, 8.830936, 47.980224, 9.860904),
    'titicaca': (-16.970092, -70.246331, -15.032316, -68.334710),
    'alps': (46.408240, 9.555657, 46.975534, 10.378602)
}

# No options provided
MALLORCA_TIF = os.path.join(EXAMPLES_DIR, 'mallorca.tif')
MALLORCA_PNG = os.path.join(EXAMPLES_DIR, 'mallorca.png')

class BotherTestCase(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        print('SETTING UP CLASS')
        if not os.path.exists(TEST_DIR):
            os.mkdir(TEST_DIR)
        print(os.listdir())

    @classmethod
    def tearDownClass(cls):
        print('TEARING DOWN CLASS')
        shutil.rmtree(TEST_DIR)

    def _assert_images_equal(self, im1, im2):
        np.testing.assert_array_equal(
            np.array(im1.getdata()),
            np.array(im2.getdata())
        )

    def _assert_image_files_equal(self, fpath1, fpath2):
        with Image.open(fpath1) as im1:
            with Image.open(fpath2) as im2:
                self._assert_images_equal(im1, im2)

    def test_1_srtm_tif(self):
        """Test the basic downloading of TIF files."""

        for eg in test_coords:
            bottom, left, top, right = test_coords[eg]
            tif_file = os.path.join(TEST_DIR, f'{eg}.tif')
            eg_tif_file = os.path.join(EXAMPLES_DIR, f'{eg}.tif')
            create_tif_file(left, bottom, right, top, os.path.abspath(tif_file))
            self._assert_image_files_equal(tif_file, eg_tif_file)

    def test_2_resize(self):
        """Test basic cropping and scaling."""

        with open(os.path.join(TEST_DIR, 'mallorca.tif'), 'rb') as f:
            memfile = MemoryFile(f)
            with memfile.open() as src:
                data = src.read(1)
                shape = data.shape
            with memfile.open() as src:
                data = src.read(1)
            im = to_png(memfile, True, 255)
        im = crop_image(im, 1574, 787, 'c')
        im1 = scale_image(im, 2048, 1024)
        with Image.open(os.path.join(EXAMPLES_DIR, 'mallorca_no_processing_resized.png')) as im2:
            self._assert_images_equal(im1, im2)

    def test_3_downsample_lakes_raise_low_flat(self):
        """Test data resampling, lake detection, raising low pixels and flattening."""

        with open(os.path.join(TEST_DIR, 'ireland.tif'), 'rb') as f:
            memfile = MemoryFile(f)
            memfile = resample(memfile, 0.5)
            memfile = set_lakes_to_elev(memfile, 100)
            memfile = raise_low_pixels(memfile)
            im1 = to_png(memfile)
        with Image.open(os.path.join(EXAMPLES_DIR, 'ireland_resampled_lakes_low_raised.png')) as im2:
            self._assert_images_equal(im1, im2)

        with open(os.path.join(TEST_DIR, 'ireland.tif'), 'rb') as f:
            memfile = MemoryFile(f)
            memfile = resample(memfile, 0.5)
            memfile = set_lakes_to_elev(memfile, 100)
            memfile = raise_low_pixels(memfile, max_brightness=170)
            im1 = to_png(memfile, False, 170)
        with Image.open(os.path.join(EXAMPLES_DIR, 'ireland_resampled_lakes_low_raised_flat.png')) as im2:
            self._assert_images_equal(im1, im2)

    def test_4_lakes_all_above_sea_epsg_upsample(self):
        """Test a map where everything is above sea level (including one map where there are lakes).
        Also tests reprojection and upsampling of data.
        """

        with open(os.path.join(TEST_DIR, 'titicaca.tif'), 'rb') as f:
            memfile = MemoryFile(f)
            memfile = reproject_raster(memfile, dst_crs='EPSG:3857')
            memfile = set_lakes_to_elev(memfile, min_lake_size=80)
            im1 = to_png(memfile)
        with Image.open(os.path.join(EXAMPLES_DIR, 'titicaca_lakes_3857.png')) as im2:
            self._assert_images_equal(im1, im2)

        with open(os.path.join(TEST_DIR, 'alps.tif'), 'rb') as f:
            memfile = MemoryFile(f)
            im1 = to_png(memfile)
        with Image.open(os.path.join(EXAMPLES_DIR, 'alps.png')) as im2:
            self._assert_images_equal(im1, im2)

if __name__ == '__main__':
    unittest.main(argv=['hi'])

max_stars_repo_name: DFEvans/bother

max_issues_repo_name: DFEvans/bother

max_forks_repo_name: DFEvans/bother


text:
# Copyright 2017-2020 Lawrence Livermore National Security, LLC and other
# CallFlow Project Developers. See the top-level LICENSE file for details.
#
# SPDX-License-Identifier: MIT

# Library imports
import numpy as np
import networkx as nx
from ast import literal_eval as make_list

# CallFlow imports
import callflow

LOGGER = callflow.get_logger(__name__)

class Filter:
    def __init__(
        self, gf=None, mode="single", filter_by="time (inc)", filter_perc="10"
    ):
        self.gf = gf
        self.filter_perc = filter_perc

        self.set_max_min_times()

        if filter_by == "time (inc)":
            self.gf.df = self.df_by_time_inc()
            self.gf.nxg = self.graph_by_time_inc()
        elif filter_by == "time":
            self.gf.df = self.df_by_time()
            self.gf.nxg = self.graph_by_time()

    def set_max_min_times(self):
        self.max_time_inc_list = np.array([])
        self.min_time_inc_list = np.array([])
        self.max_time_exc_list = np.array([])
        self.min_time_exc_list = np.array([])

        self.max_time_inc_list = np.hstack(
            [self.max_time_inc_list, self.gf.df["time (inc)"].max()]
        )
        self.min_time_inc_list = np.hstack(
            [self.min_time_inc_list, self.gf.df["time (inc)"].min()]
        )
        self.max_time_exc_list = np.hstack(
            [self.max_time_exc_list, self.gf.df["time"].max()]
        )
        self.min_time_exc_list = np.hstack(
            [self.min_time_exc_list, self.gf.df["time"].min()]
        )

        LOGGER.info(f"Min. time (inc): {self.min_time_inc_list}")
        LOGGER.info(f"Max. time (inc): {self.max_time_inc_list}")
        LOGGER.info(f"Min. time (exc): {self.min_time_exc_list}")
        LOGGER.info(f"Max. time (exc): {self.max_time_exc_list}")

        self.max_time_inc = np.max(self.max_time_inc_list)
        self.min_time_inc = np.min(self.min_time_inc_list)
        self.max_time_exc = np.max(self.max_time_exc_list)
        self.min_time_exc = np.min(self.min_time_exc_list)

    def df_by_time_inc(self):
        LOGGER.debug(f"[Filter] By Inclusive time : {self.filter_perc}")
        df = self.gf.df.loc[
            (self.gf.df["time (inc)"] > self.filter_perc * 0.01 * self.max_time_inc)
        ]
        filter_call_sites = df["name"].unique()
        return df[df["name"].isin(filter_call_sites)]

    def df_by_time(self, perc):
        LOGGER.debug(f"[Filter] By Exclusive time : {self.filter_perc}")
        df = self.gf.df.loc[self.gf.df["time"] > self.filter_perc]
        filter_call_sites = df["name"].unique()
        return df[df["name"].isin(filter_call_sites)]

    def graph_by_time_inc(self):
        callsites = self.gf.df["name"].unique()

        ret = nx.DiGraph()
        for edge in self.gf.nxg.edges():
            # If source is present in the callsites list
            if edge[0] in callsites and edge[1] in callsites:
                ret.add_edge(edge[0], edge[1])
            else:
                LOGGER.debug(f"Removing the edge: {edge}")

        return ret

    def findPaths(self, g, u, n, excludeSet=None):
        if excludeSet is None:
            excludeSet = set([u])
        else:
            excludeSet.add(u)
        if n == 0:
            return [[u]]

        paths = [
            [].append(path)
            for neighbor in g.neighbors(u)
            if neighbor not in excludeSet
            for path in self.findPaths(g, neighbor, n - 1, excludeSet)
        ]
        excludeSet.remove(u)
        return paths

    def graph_by_time(self, df, g):
        callsites = df["name"].unique()

        ret = nx.DiGraph()

        for callsite in callsites:
            path = df.loc[df["name"] == callsite]["path"].tolist()[0]
            path = make_list(path)
            ret.add_path(path)

        return ret

max_stars_repo_name: jarusified/CallFlow

max_issues_repo_name: jarusified/CallFlow

max_forks_repo_name: jarusified/CallFlow

reason: import numpy,import networkx