text:
import numpy as np
from . import preprocessing
import tensorflow as tf
from tensorflow.python.util import nest
import os,sys
import numpy as np
import horovod.tensorflow as hvd

class DataLoader:

    def __init__(self, config):
        self.config = config

        filename_pattern = os.path.join(config['data_url'], '%s-*')
        filenames_train = sorted(tf.gfile.Glob(filename_pattern % 'train'))
        self.num_training_samples = get_num_records(filenames_train)
        self.config['num_training_samples'] = self.num_training_samples

        filename_pattern = os.path.join(config['data_url'], '%s-*')
        filenames_val = sorted(tf.gfile.Glob(filename_pattern % 'validation'))
        self.num_evaluating_samples = get_num_records(filenames_val)
        self.config['num_evaluating_samples'] = self.num_evaluating_samples

        print( 'total num_training_sampels: %d' %  self.num_training_samples )
        print( 'total num_evaluating_sampels: %d' %  self.num_evaluating_samples )

        self.training_samples_per_rank = self.num_training_samples

    def get_train_input_fn(self):
        take_count = self.training_samples_per_rank
        batch_size = self.config['batch_size']
        shard = self.config['shard']

        return make_dataset(self.config, take_count, batch_size,
                   training=True, shard=shard)

    def get_eval_input_fn(self):
        take_count = self.num_evaluating_samples
        batch_size = self.config['batch_size']
        shard = self.config['shard']

        return make_dataset(self.config, take_count, batch_size,
                 training=False, shard=shard)

#-------------------------------- Funcs -----------------------------------
def get_num_records(filenames):
    def count_records(tf_record_filename):
        count = 0
        for _ in tf.python_io.tf_record_iterator(tf_record_filename):
            count += 1
        return count

    nfile = len(filenames)
    return (count_records(filenames[0]) * (nfile - 1) +
            count_records(filenames[-1]))

def _parse_example_proto(example_serialized):
    feature_map = {
        'image/encoded': tf.FixedLenFeature([], dtype=tf.string,
                                            default_value=''),
        'image/class/label': tf.FixedLenFeature([], dtype=tf.int64, default_value=-1),
        'image/class/text': tf.FixedLenFeature([], dtype=tf.string,
                                               default_value=''),
    }
    sparse_float32 = tf.VarLenFeature(dtype=tf.float32)
    # Sparse features in Example proto.
    feature_map.update(
        {k: sparse_float32 for k in ['image/object/bbox/xmin',
                                     'image/object/bbox/ymin',
                                     'image/object/bbox/xmax',
                                     'image/object/bbox/ymax']})

    features = tf.parse_single_example(example_serialized, feature_map)
    label = tf.cast(features['image/class/label'], dtype=tf.int32)

    xmin = tf.expand_dims(features['image/object/bbox/xmin'].values, 0)
    ymin = tf.expand_dims(features['image/object/bbox/ymin'].values, 0)
    xmax = tf.expand_dims(features['image/object/bbox/xmax'].values, 0)
    ymax = tf.expand_dims(features['image/object/bbox/ymax'].values, 0)

    # Note that we impose an ordering of (y, x) just to make life difficult.
    bbox = tf.concat([ymin, xmin, ymax, xmax], 0)

    # Force the variable number of bounding boxes into the shape
    # [1, num_boxes, coords].
    bbox = tf.expand_dims(bbox, 0)
    bbox = tf.transpose(bbox, [0, 2, 1])

    return features['image/encoded'], label, bbox

# since the preprocessing is done here, we add config file
def parse_record(raw_record, is_training, cfg):
    image_buffer, label, bbox = _parse_example_proto(raw_record)

    image = preprocessing.parse_and_preprocess_image_record(image_buffer, bbox, training=is_training)

    # label-1 for VGG in slimImageet dataset
    return image, label-1

def make_dataset(config, take_count, batch_size,
                 training=False, shard=False):

    shuffle_buffer_size = 10000
    num_readers = 10

    rank_size = hvd.size()
    rank_id = hvd.local_rank()

    if training:
        filename_pattern = os.path.join(config['data_url'], '%s-*')
        filenames = sorted(tf.gfile.Glob(filename_pattern % 'train'))
    else:
        filename_pattern = os.path.join(config['data_url'], '%s-*')
        filenames = sorted(tf.gfile.Glob(filename_pattern % 'validation'))

    ds = tf.data.Dataset.from_tensor_slices(filenames)

    if shard:
        ds = ds.shard(rank_size, rank_id)

    if not training:
        ds = ds.take(take_count)  # make sure all ranks have the same amount

    if training:
        ds = ds.shuffle(1000, seed=7*(1+rank_id))

    ds = ds.interleave(tf.data.TFRecordDataset, cycle_length=num_readers, block_length=1)
    counter = tf.data.Dataset.range(sys.maxsize)
    ds = tf.data.Dataset.zip((ds, counter))

    if training:
        ds = ds.apply(tf.data.experimental.shuffle_and_repeat(shuffle_buffer_size, seed=5*(1+rank_id)))

    ds = ds.map(lambda image, counter: parse_record(image, training, config), num_parallel_calls=14)

    ds = ds.batch(batch_size, drop_remainder=True)
    return ds

max_stars_repo_name: Huawei-Ascend/modelzoo

max_issues_repo_name: Huawei-Ascend/modelzoo

max_forks_repo_name: Huawei-Ascend/modelzoo


text:
"""Test a model and generate submission CSV.

Usage:
    > python test.py --split SPLIT --load_path PATH --name NAME
    where
    > SPLIT is either "dev" or "test"
    > PATH is a path to a checkpoint (e.g., save/train/model-01/best.pth.tar)
    > NAME is a name to identify the test run

Author:
    Chris Chute (chute@stanford.edu)
"""

import csv
from multiprocessing.sharedctypes import Value
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data as data
import util
import numpy as np
import pickle

from args import get_test_args
from collections import OrderedDict
from json import dumps
from models import BiDAF, SCR, SAN
from os.path import join
from tensorboardX import SummaryWriter
from tqdm import tqdm
from ujson import load as json_load
from util import collate_fn, SQuAD

def main(args):
    # Set up logging
    args.save_dir = util.get_save_dir(args.save_dir, args.name, training=False)
    log = util.get_logger(args.save_dir, args.name)
    log.info(f'Args: {dumps(vars(args), indent=4, sort_keys=True)}')
    device, gpu_ids = util.get_available_devices()
    args.batch_size *= max(1, len(gpu_ids))

    # Get embeddings
    log.info('Loading embeddings...')
    word_vectors = util.torch_from_json(args.word_emb_file)

    # Get model
    log.info('Building model...')
    if args.model == "scr":
        model = SCR(word_vectors=word_vectors,
                    hidden_size=args.hidden_size,
                    num_candidates=util.NUM_CANDIDATES,
                    only_dcr=args.only_dcr).to(device)
        cand_model = SAN(word_vectors=word_vectors,
                         hidden_size=args.cand_hidden_size,
                         T=args.cand_time_steps).to(device)
    else:
        model = SAN(word_vectors=word_vectors,
                    hidden_size=args.cand_hidden_size,
                    T=args.cand_time_steps).to(device)
    model = nn.DataParallel(model, gpu_ids)
    if args.load_model_path:
        log.info(f'Loading checkpoint from {args.load_model_path}...')
        model, step = util.load_model(model, args.load_model_path, gpu_ids)
    else:
        step = 0

    if args.load_cand_model_path:
        cand_model = nn.DataParallel(cand_model, gpu_ids)
        log.info(f'Loading candidate model checkpoint from {args.load_cand_model_path}...')
        cand_model, cand_step = util.load_model(cand_model, args.load_cand_model_path, gpu_ids)
    else:
        cand_step = 0
    model = model.to(device)
    model.eval()

    # Get data loader
    log.info('Building dataset...')
    record_file = vars(args)[f'{args.split}_record_file']
    data_aug_file = vars(args)[f'{args.split}_data_aug_file']
    dataset = SQuAD(record_file, data_aug_file, args.use_squad_v2)
    data_loader = data.DataLoader(dataset,
                                  batch_size=args.batch_size,
                                  shuffle=False,
                                  num_workers=args.num_workers,
                                  collate_fn=collate_fn)

    # Evaluate
    log.info(f'Evaluating on {args.split} split...')
    nll_meter = util.AverageMeter()
    pred_dict = {}  # Predictions for TensorBoard
    sub_dict = {}  # Predictions for submission
    eval_file = vars(args)[f'{args.split}_eval_file']
    k_oracle_file = args.K_pickle
    k_oracles = np.array([1, 2, 3, 5, 10, 20, 50, 100])
    k_oracle_data = np.zeros(k_oracles.shape)
    if args.disposition:
        cand_errors = 0
        dcr_errors = 0
    cnt = 0
    with open(eval_file, 'r') as fh:
        gold_dict = json_load(fh)
    with torch.no_grad(), \
            tqdm(total=len(dataset)) as progress_bar:
        for cw_idxs, cc_idxs, qw_idxs, qc_idxs, y1, y2, pos_idxs, ner_idxs, bem_idxs, ids in data_loader:
            # Setup for forward
            cw_idxs = cw_idxs.to(device)
            qw_idxs = qw_idxs.to(device)
            pos_idxs = pos_idxs.to(device)
            ner_idxs = ner_idxs.to(device)
            bem_idxs = bem_idxs.to(device)
            batch_size = cw_idxs.size(0)
            _, p_len = cw_idxs.size()

            # Forward
            y1, y2 = y1.to(device), y2.to(device)
            if args.disposition:
                candidates, candidate_scores, _ = util.generate_candidates(cand_model, cw_idxs, qw_idxs, pos_idxs,
                                                                           ner_idxs, bem_idxs,
                                                                           (y1, y2), util.NUM_CANDIDATES,
                                                                           device, train=False)

                logprob_chunks = model(cw_idxs, qw_idxs, pos_idxs, ner_idxs, bem_idxs, candidates, candidate_scores)
                scr_ans = torch.argmax(logprob_chunks, dim=1)
                c_len = cw_idxs.size()[1]
                c_mask = torch.zeros_like(cw_idxs) != cw_idxs

                for i in range(batch_size):
                    answer_chunk = torch.Tensor([y1[i], y2[i]])

                    found_y = torch.logical_and(candidates[i, :, 0] == answer_chunk[0],
                                                candidates[i, :, 1] == answer_chunk[1]).nonzero()
                    if len(found_y) > 0:
                        # in K-oracle, we are completely correct if one of our candidates is correct
                        idx_correct = found_y[0][0]
                        if scr_ans[i] == idx_correct:
                            print("+", end="")
                        else:
                            dcr_errors += 1
                            print("=", end="")
                    else:
                        cand_errors += 1
                        print("-", end="")

                log_p1, log_p2 = util.convert_probs(logprob_chunks, candidates, c_len, c_mask, device)
            elif args.model == "scr":
                candidates, candidate_scores, _ = util.generate_candidates(cand_model, cw_idxs, qw_idxs, pos_idxs,
                                                                           ner_idxs, bem_idxs,
                                                                           (y1, y2), util.NUM_CANDIDATES,
                                                                           device, train=False)

                logprob_chunks = model(cw_idxs, qw_idxs, pos_idxs, ner_idxs, bem_idxs, candidates, candidate_scores)
                c_len = cw_idxs.size()[1]
                c_mask = torch.zeros_like(cw_idxs) != cw_idxs

                log_p1, log_p2 = util.convert_probs(logprob_chunks, candidates, c_len, c_mask, device)
            elif args.K_oracle != 0:
                if args.K_oracle < 0:
                    candidates, _, _ = util.generate_candidates(model, cw_idxs, qw_idxs, pos_idxs, ner_idxs, bem_idxs,
                                                                (y1, y2), 100, device, train=False)
                else:
                    candidates, _, _ = util.generate_candidates(model, cw_idxs, qw_idxs, pos_idxs, ner_idxs, bem_idxs,
                                                                (y1, y2), args.K_oracle, device, train=False)
                some_log_p1, some_log_p2 = model(cw_idxs, qw_idxs, pos_idxs, ner_idxs, bem_idxs)
                log_p1, log_p2 = torch.zeros(batch_size, p_len), torch.zeros(batch_size, p_len)
                log_p1 = log_p1.to(device)
                log_p2 = log_p2.to(device)
                rat = 0.0
                if args.K_oracle < 0:
                    correct = np.zeros(k_oracles.shape)
                for i in range(batch_size):
                    answer_chunk = torch.Tensor([y1[i], y2[i]])

                    found_y = torch.logical_and(candidates[i, :, 0] == answer_chunk[0],
                                                candidates[i, :, 1] == answer_chunk[1]).nonzero()
                    if len(found_y) > 0:
                        # in K-oracle, we are completely correct if one of our candidates is correct
                        idx_correct = found_y[0][0]
                        log_p1[i, candidates[i, idx_correct, 0]] = 1
                        log_p1[i] = torch.log_softmax(log_p1[i], dim=0)
                        log_p2[i, candidates[i, idx_correct, 1]] = 1
                        log_p2[i] = torch.log_softmax(log_p2[i], dim=0)
                        if args.K_oracle < 0:
                            for i, k in enumerate(k_oracles):
                                if idx_correct < k:
                                    correct[i] += 1
                        rat += 1
                        print("+", end="")
                    else:
                        log_p1[i], log_p2[i] = some_log_p1[i], some_log_p2[
                            i]  # otherwise we are just our normal function
                        print("-", end="")
                if args.K_oracle != 0:
                    log.info(rat / batch_size)
                    cnt += batch_size
                    k_oracle_data += correct
            else:
                log_p1, log_p2 = model(cw_idxs, qw_idxs, pos_idxs, ner_idxs, bem_idxs)
            y1, y2 = y1.to(device), y2.to(device)
            loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2)
            nll_meter.update(loss.item(), batch_size)

            # Get F1 and EM scores
            p1, p2 = log_p1.exp(), log_p2.exp()
            starts, ends = util.discretize(p1, p2, args.max_ans_len, args.use_squad_v2)

            # Log info
            progress_bar.update(batch_size)
            if args.split != 'test':
                # No labels for the test set, so NLL would be invalid
                progress_bar.set_postfix(NLL=nll_meter.avg)

            idx2pred, uuid2pred = util.convert_tokens(gold_dict,
                                                      ids.tolist(),
                                                      starts.tolist(),
                                                      ends.tolist(),
                                                      args.use_squad_v2)
            pred_dict.update(idx2pred)
            sub_dict.update(uuid2pred)

    if args.K_oracle != 0:
        log.info("Dumping results to", k_oracle_file)
        pickle.dump((k_oracles, k_oracle_data / cnt), open(k_oracle_file, "wb"))
    if args.disposition:
        log.info("Dumping dispositions to", args.disp_output)
        pickle.dump((cand_errors, dcr_errors), open(args.disp_output, "wb"))

    # Log results (except for test set, since it does not come with labels)
    if args.split != 'test':
        results = util.eval_dicts(gold_dict, pred_dict, args.use_squad_v2,
                                  args.save_dir + '/q_breakdown.json', args.save_dir + '/a_len_breakdown.json')
        results_list = [('NLL', nll_meter.avg),
                        ('F1', results['F1']),
                        ('EM', results['EM'])]
        if args.use_squad_v2:
            results_list.append(('AvNA', results['AvNA']))
        results = OrderedDict(results_list)

        # Log to console
        results_str = ', '.join(f'{k}: {v:05.2f}' for k, v in results.items())
        log.info(f'{args.split.title()} {results_str}')

        # Log to TensorBoard
        tbx = SummaryWriter(args.save_dir)
        util.visualize(tbx,
                       pred_dict=pred_dict,
                       eval_path=eval_file,
                       step=0,
                       split=args.split,
                       num_visuals=args.num_visuals)

    # Write submission file
    sub_path = join(args.save_dir, args.split + '_' + args.sub_file)
    log.info(f'Writing submission file to {sub_path}...')
    with open(sub_path, 'w', newline='', encoding='utf-8') as csv_fh:
        csv_writer = csv.writer(csv_fh, delimiter=',')
        csv_writer.writerow(['Id', 'Predicted'])
        for uuid in sorted(sub_dict):
            csv_writer.writerow([uuid, sub_dict[uuid]])

if __name__ == '__main__':
    main(get_test_args())

ext: py

max_stars_repo_name: alexdloia/squad

max_issues_repo_name: alexdloia/squad

max_forks_repo_name: alexdloia/squad

reason: import numpy


text:
# test_efficiency.py - unit tests for the efficiency module
#
# Copyright 2015-2017 NetworkX developers.
#
# This file is part of NetworkX.
#
# NetworkX is distributed under a BSD license; see LICENSE.txt for more
# information.
"""Unit tests for the :mod:`networkx.algorithms.efficiency` module."""

from __future__ import division
from nose.tools import assert_equal
import networkx as nx

class TestEfficiency:

    def __init__(self):
        # G1 is a disconnected graph
        self.G1 = nx.Graph()
        self.G1.add_nodes_from([1, 2, 3])
        # G2 is a cycle graph
        self.G2 = nx.cycle_graph(4)
        # G3 is the triangle graph with one additional edge
        self.G3 = nx.lollipop_graph(3, 1)

    def test_efficiency_disconnected_nodes(self):
        """
        When nodes are disconnected, efficiency is 0
        """
        assert_equal(nx.efficiency(self.G1, 1, 2), 0)

    def test_local_efficiency_disconnected_graph(self):
        """
        In a disconnected graph the efficiency is 0
        """
        assert_equal(nx.local_efficiency(self.G1), 0)

    def test_efficiency(self):
        assert_equal(nx.efficiency(self.G2, 0, 1), 1)
        assert_equal(nx.efficiency(self.G2, 0, 2), 1 / 2)

    def test_global_efficiency(self):
        assert_equal(nx.global_efficiency(self.G2), 5 / 6)

    def test_global_efficiency_complete_graph(self):
        """
        Tests that the average global efficiency of the complete graph is one.
        """
        for n in range(2, 10):
            G = nx.complete_graph(n)
            assert_equal(nx.global_efficiency(G), 1)

    def test_local_efficiency_complete_graph(self):
        """
        Test that the local efficiency for a complete graph should be one.
        """
        for n in range(2, 10):
            G = nx.complete_graph(n)
            assert_equal(nx.local_efficiency(G), 1)

    def test_using_ego_graph(self):
        """
        Test that the ego graph is used when computing local efficiency.
        For more information, see GitHub issue #2233.
        """
        assert_equal(nx.local_efficiency(self.G3), 23 / 24)

max_stars_repo_name: alubbock/networkx

max_issues_repo_name: aganezov/networkx

max_forks_repo_name: aganezov/networkx

reason: import networkx


text:
# coding=utf-8
# function q = fastguidedfilter(I, p, r, eps, s) - MATLAB
# %    GUIDEDFILTER O(1) time implementation of guided filter.
# I:   guidance Image
# p:   filtering input image
# r:   window redius
# eps: normalize parameter
# s:   sampling fraction try s = r/4 to s=r
import os
import cv2
import numpy as np
from numpy.matlib import repmat
from scipy.misc import imresize

ra = 16
eps = 2
s = 4

def boxFilter(imSrc, r):
    hei, wid = imSrc.shape
    imDst = np.zeros([hei, wid])
    imCum = np.cumsum(imSrc, axis=0)
    imDst[0:r+1, :] = imCum[r:2*r+1, :]
    imDst[r+1:hei-r, :] = imCum[2*r+1:hei, :] - imCum[0:hei-2*r-1, :]
    imDst[hei-r:hei, :] = repmat(imCum[hei-1, :], r, 1) - imCum[hei-2*r-1:hei-r-1, :]
    imCum = np.cumsum(imDst, axis=1)
    imDst[:, 0:r+1] = imCum[:, r:2*r+1]
    imDst[:, r+1:wid-r] = imCum[:, 2*r+1:wid] - imCum[:, 0:wid-2*r-1]
    repmat(imCum[:,wid-1], r, 1).shape = (imCum.shape[0], r)
    y = np.transpose(repmat(imCum[:,wid-1], r, 1))
    imDst[:, wid-r:wid] = y - imCum[:, wid-2*r-1:wid-r-1]
    return imDst

def guideFilter(I, ra=16, eps=2, s=4):
    I_sub = imresize(I, 1/s, 'nearest')
    I_sub = I_sub/255
    p_sub = imresize(I, 1/s, 'nearest')
    p_sub = p_sub/255
    r_sub = int(ra / s)
    hei, wid = I_sub.shape
    m = np.ones([hei, wid])
    N = boxFilter(m, r_sub)
    mean_I = boxFilter(I_sub, r_sub) / N
    mean_p = boxFilter(p_sub, r_sub) / N
    mean_Ip = boxFilter(I_sub * p_sub, r_sub) / N
    # this is the covariance of (I, p) in each local patch.
    cov_Ip = mean_Ip - mean_I * mean_p
    mean_II = boxFilter(I_sub * I_sub, r_sub) / N
    var_I = mean_II - mean_I * mean_I
    a = cov_Ip / (var_I + eps)
    b = mean_p - a * mean_I
    mean_a = boxFilter(a, r_sub) / N
    mean_b = boxFilter(b, r_sub) / N
    mean_a = cv2.resize(mean_a, (I.shape[1], I.shape[0]), interpolation=cv2.INTER_LINEAR)
    mean_b = cv2.resize(mean_b, (I.shape[1], I.shape[0]), interpolation=cv2.INTER_LINEAR)
    I = I / 255
    q = mean_a * I + mean_b
    return q

if __name__ == '__main__':
    image_dir   = './demo_image/'
    rainy_image = cv2.imread(os.path.join(image_dir, 'rainy_image.jpg'))
    gt_image    = cv2.imread(os.path.join(image_dir, 'ground_truth.jpg'))
    b, g, r     = cv2.split(rainy_image)
    h, w, c     = rainy_image.shape
    base_layer  = np.zeros([h, w, c])
    base_layer[:, :, 0] = guideFilter(b, ra, eps, s)
    base_layer[:, :, 1] = guideFilter(g, ra, eps, s)
    base_layer[:, :, 2] = guideFilter(r, ra, eps, s)
    rainy_image  = rainy_image / 255
    detail_layer = rainy_image - base_layer
    cv2.imshow('rainy_image', rainy_image)
    cv2.imshow('base_layer',  base_layer)
    cv2.imshow('detail_layer', detail_layer)
    cv2.waitKey(3)
    cv2.imwrite(image_dir + 'base_layer.jpg',   base_layer*255)
    cv2.imwrite(image_dir + 'detail_layer.jpg', detail_layer*255)
    # for the GT image - the same.
    # b_, g_, r_ = cv2.split(gt_image)
    # base_layer_ = np.zeros([h, w, c])
    # base_layer_[:, :, 0] = guideFilter(b_, ra, eps, s)
    # base_layer_[:, :, 1] = guideFilter(g_, ra, eps, s)
    # base_layer_[:, :, 2] = guideFilter(r_, ra, eps, s)
    # gt_image = gt_image / 255
    # detail_layer_ = gt_image - base_layer_
    # cv2.imshow('gt_image', gt_image)
    # cv2.imshow('base_layer_',  base_layer_)
    # cv2.imshow('detail_layer_', detail_layer_)
    # cv2.waitKey(0)

max_stars_repo_name: bravotty/GuidedFilter

max_issues_repo_name: bravotty/GuidedFilter

max_forks_repo_name: bravotty/GuidedFilter

reason: import numpy,from numpy,from scipy


text:
import argparse
import os
import sys

import cv2
import matplotlib.pyplot as plt
import numpy as np
from tensorflow.contrib.keras import applications
from tensorflow.contrib.keras import backend as K
from tensorflow.contrib.keras import preprocessing

import cnn_classification.keras.dogs_cats_dataset as dataset
import cnn_classification.keras.utils as utils

def show_top_predictions(preds, top=3):
    decoded_preds = applications.vgg16.decode_predictions(preds, top=top)[0]
    for i, (_, clazz, prob) in enumerate(decoded_preds):
        print('{}. {:20s}: {:.4f}'.format(i + 1, clazz, prob))

def get_top_prediction(preds):
    decoded_preds = applications.vgg16.decode_predictions(preds, top=1)[0]
    clazz = decoded_preds[0][1]
    prob = decoded_preds[0][2]
    return clazz, prob

def normalize_heatmap(heatmap):
    heatmap = np.maximum(heatmap, 0)
    heatmap /= np.max(heatmap)
    return heatmap

def superimpose_image_with_heatmap(dest_img_path, src_img_path, heatmap, strength=0.4):
    img = cv2.imread(src_img_path)

    heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))
    heatmap = np.uint8(255 * heatmap)
    heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
    superimposed_img = heatmap * strength + img

    cv2.imwrite(dest_img_path, superimposed_img)

def gradCAM(model, dog_x, class_idx):
    model_class_output = model.output[:, class_idx]
    print('model_class_output shape', model_class_output.shape)
    last_conv_layer = model.get_layer('block5_conv3')

    grads_list = K.gradients(model_class_output, last_conv_layer.output)
    grads = grads_list[0]
    # grads shape: (?, 14, 14, 512)

    pooled_grads = K.mean(grads, axis=(0, 1, 2))
    # pooled_grads shape: (512,)

    iterate = K.function(inputs=[model.input],
                         outputs=[pooled_grads, last_conv_layer.output[0]])
    pooled_grads_value, conv_layer_output_value = iterate([dog_x])
    # conv_layer_output_value shape: (14, 14, 512)

    for i in range(last_conv_layer.filters):
        conv_layer_output_value[:, :, i] *= pooled_grads_value[i]

    heatmap = np.mean(conv_layer_output_value, axis=-1)
    # heatmap shape: (14, 14)
    heatmap = normalize_heatmap(heatmap)
    return heatmap

def run(i, model, train_dogs_dir, tag):
    img_filename = utils.listdir(train_dogs_dir, recursive=True)[i]
    img_path = os.path.join(train_dogs_dir, img_filename)
    img_dog = preprocessing.image.load_img(img_path, target_size=(224, 224))

    os.makedirs('tmp/output', exist_ok=True)
    processed_img_path = 'tmp/output/processed-{}-{}.png'.format(tag, i)
    img_dog.save(processed_img_path)

    dog_x = preprocessing.image.img_to_array(img_dog)
    dog_x = np.expand_dims(dog_x, axis=0)
    dog_x = applications.vgg16.preprocess_input(dog_x)

    preds = model.predict(dog_x)
    show_top_predictions(preds, top=3)
    clazz, prob = get_top_prediction(preds)

    class_idx = np.argmax(preds[0])

    heatmap = gradCAM(model, dog_x, class_idx)

    if FLAGS.show_intermediate_heatmap:
        plt.matshow(heatmap)
        plt.show()

    superimposed_output_path = 'tmp/output/superimposed-{}-{}-{}-{}.png'.format(tag, i, clazz, int(round(prob * 100)))
    superimpose_image_with_heatmap(superimposed_output_path, img_path, heatmap, strength=0.4)

def main(_):
    model = applications.VGG16(weights='imagenet')

    # load images for visualization
    train_dir, _, _ = dataset.prepare(train_size=2 * FLAGS.n_per_class, valid_size=0, test_size=0)
    train_dogs_dir = os.path.join(train_dir, 'dogs')
    train_cats_dir = os.path.join(train_dir, 'cats')

    for i in range(FLAGS.n_per_class):
        run(i, model, train_dogs_dir, 'dog')
        run(i, model, train_cats_dir, 'cat')

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--n_per_class', type=int, default=10,
                        help='The number of animals per class')
    parser.add_argument('--show_intermediate_heatmap', type=bool, default=False,
                        help='Show intermediate heatmap with matplotlib')
    FLAGS, unparsed = parser.parse_known_args()
    main([sys.argv[0]] + unparsed)

max_stars_repo_name: hitlic/machine-learning-examples

max_issues_repo_name: hitlic/machine-learning-examples

max_forks_repo_name: hitlic/machine-learning-examples


text:
import numpy as np
from savevar import savevar

class arrivalTimes:
    pass

def binFile2Data(fname, storePickle=False):
    """
    Read a binary FCS data file, containing a list of U64 numbers that
    represent the photon counts per microtime bin of about 1 µs.

    ==========  ===============================================================
    Input       Meaning
    ----------  ---------------------------------------------------------------
    fname       Binary file name
    ==========  ===============================================================

    ==========  ===============================================================
    Output      Meaning
    ----------  ---------------------------------------------------------------
    countArray  N x 26 array with for each row the number of photon counts
                for each channel and the sum of all channels per microbin
    ==========  ===============================================================

    """

    from readBinFile import readBinFile
    from u64ToCounts import u64ToCounts
    import numpy as np
    import os
    from savevar import savevar

    readLength = 8 * 65536

    # Get file size [in bytes]
    fsize = os.path.getsize(fname)

    # Empty array to store photon counts
    countArray = np.zeros((int(fsize/2/8), 26), dtype='H')

    startPos = 0  # position in the .bin file to start reading
    row = 0  # row of countArray to write to

    # Go through the file, read a chunk of data, write to the array, and repeat
    while startPos < fsize:
        print(startPos/fsize*100)
        # Read part of the binary file
        data = readBinFile(fname, startPos, readLength)
        startPos += readLength

        # Calculate photon counts
        for i in range(len(data)):
            counts = u64ToCounts(data[i])
            # add counts to array
            countArray[int(np.floor(row/2))][:] = np.add(
                    countArray[int(np.floor(row/2))][:],
                    counts)
            row += 1

    print(100)

    if storePickle:
        print("Storing .pickle file")
        savevar(countArray, fname[0:-4] + "_data")
        print(".pickle file stored")

    return countArray

def binFile2DataAle(fname, storePickle=False):
    """
    Read a binary data file from Ale's FPGA, containing a list of U32 numbers
    that represent photon arrival times

    ==========  ===============================================================
    Input       Meaning
    ----------  ---------------------------------------------------------------
    fname       Binary file name
    ==========  ===============================================================

    ==========  ===============================================================
    Output      Meaning
    ----------  ---------------------------------------------------------------
    data        Class with 32 fields, each one containing a list of the arrival
                times of the photons in that channel. With the 5x5 SPAD array
                detector, only the 25 first fields are used, the others are
                empty. The arrival times are in units of 12.5 ns, i.e.
                multiply each list by 12.5 to get the arrival times in ns.
    ==========  ===============================================================

    """
    # Open .bin file
    print("Opening .bin file.")
    with open(fname, mode='rb') as file:
        rawdata = list(file.read())
    print("Binfile opened.")

    # Number of bytes per line
    NB = 4

    # Number of addresses
    NA = 8

    # number of data points
    N = len(rawdata) / NB  # 4 bytes per 32 bit number

    # convert list into list of lists with each sublist containing 4 numbers
    # rawdata2 = [rawdata[i:i+NB] for i in range(0, len(rawdata), NB)]

    # reverse order of the bytes in each list item
    # rawdata = [list(reversed(rawdata[i])) for i in range(0, len(rawdata))]

    # Create vector data with the macro arrival times
    data = arrivalTimes()
    for i in range(NA):
        for j in range(NB):
            setattr(data, "pixel" + str(i*NB + j), [])

    # go through data set
    oldTimeArr = np.zeros(NA)  # keep track of time
    k = np.zeros(NA)  # keep track of number of macrotimes of each address
    for i in range(round(N)):
        if np.mod(i, 10000) == 0:
            print("{0:.3f} %".format(100 * i/N))
        start = 4 * i
        newRawDataPoint = rawdata[start:start+NB]
        newRawDataPoint = list(reversed(newRawDataPoint))
        address = getAddress(newRawDataPoint)
        time = getTime(newRawDataPoint)
        ch = getChannel(newRawDataPoint)
        # print("time: " + str(time) + " - oldTime: " + str(oldTime))
        # print(ch)
        if time < oldTimeArr[address]:
            k[address] += 1
        oldTimeArr[address] = time
        time = int(k[address] * 4096 + time)
        if ch > 0 and address >= 0:
            pixel = address * 4 + ch - 1
            getattr(data, "pixel" + str(pixel)).append(time)

    if storePickle:
        print("Storing .pickle file")
        savevar(data, fname[0:-4] + "_data")
        print(".pickle file stored")

    return(data)

def getAddress(data):
    address = data[0]
    address = list('{0:08b}'.format(int(address)))
    address = address[0:3]
    address = int("".join(str(x) for x in address), 2)
    return address

def getTime(data):
    byte0 = data[3]
    byte0 = list('{0:08b}'.format(int(byte0)))
    byte1 = data[2]
    byte1 = list('{0:08b}'.format(int(byte1)))
    byte1 = byte1[4:8]
    time = byte1 + byte0
    time = int("".join(str(x) for x in time), 2)
    return time

def getChannel(data):
    byte2 = data[1]
    byte2 = list('{0:08b}'.format(int(byte2)))
    byte3 = data[0]
    byte3 = list('{0:08b}'.format(int(byte3)))
    ch1 = byte2[7]
    ch2 = byte2[6]
    ch3 = byte3[7]
    ch4 = byte3[6]
    channels = [ch1, ch2, ch3, ch4]
    channels = list(map(int, channels))
    if np.sum(channels) > 1 or np.sum(channels) == 0:
        return -1
    return channels.index(1) + 1

max_stars_repo_name: VicidominiLab/spad-ffs

max_issues_repo_name: VicidominiLab/spad-ffs

max_forks_repo_name: VicidominiLab/spad-ffs


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Script to plot result of tensile tests, different forces.
#

import sys
import numpy as np
import matplotlib.pyplot as plt
import csv

# parse all data from result.csv
data = {}
with open('build_release/result.csv') as csvfile:
  reader = csv.reader(csvfile, delimiter=',')

  for scenario_name,strain,stress in reader:
    strain = float(strain)
    stress = float(stress)

    # if value is invalid
    if abs(strain) > 4 or abs(stress) > 1000:
      continue

    if scenario_name not in data:
      data[scenario_name] = {"strain": [], "stress": []}
    data[scenario_name]["strain"].append(strain)
    data[scenario_name]["stress"].append(stress)

# loop over scenario names
for scenario_name in data.keys():

  plt.plot(data[scenario_name]["strain"], data[scenario_name]["stress"], 'o-', label=scenario_name)

plt.ylabel('stress [$N/cm^2$]')
plt.xlabel('strain [-]')
plt.grid(which='major')
plt.legend(loc='best')
plt.show()

max_stars_repo_name: maierbn/opendihu

max_issues_repo_name: maierbn/opendihu

max_forks_repo_name: maierbn/opendihu


text:
# -*- coding: utf-8 -*-

import random

import numpy as np

import torch
import torch.nn as nn
from torch.nn import functional as F
from torch.autograd import Variable

# LSTM and FFNN for popularity

class wordLSTM(nn.Module):
    def __init__(self, emb_dim, hid_dim, n_layers, dropout, pad_idx, batch_size, weights_matrix):
        super(wordLSTM, self).__init__()

        self.emb_dim = emb_dim
        self.hid_dim = hid_dim
        self.n_layers = n_layers
        self.dropout = dropout
        self.batch_size = batch_size
        padding_idx = pad_idx

        # Setting each layer of the NN
        num_embeddings, embedding_dim = weights_matrix.shape
        self.embedding = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx)
        self.embedding.load_state_dict({'weight': weights_matrix})
        # Set requires_grad for embedding as False due to lack of memory
        self.embedding.weight.requires_grad = False
        self.lstm = nn.LSTM(self.emb_dim, self.hid_dim, self.n_layers, dropout=dropout, bidirectional=True, batch_first=True)
        #self.hidden2tag = nn.Linear( hid_dim*2, 1)
        self.dropout = nn.Dropout(dropout)

    def init_hidden(self):
        return (torch.zeros(self.n_layers*2, self.batch_size, self.hid_dim),
            torch.zeros(self.n_layers*2, self.batch_size, self.hid_dim))

    # sequence_idx = [batch_size, sequence_length]
    def forward(self, sequence_idx):
        #self.hidden = self.init_hidden(sequence_idx.shape[1])
        #sequence_idx.t() # [sequence_length, batch_size]
        embedded = self.dropout(self.embedding(sequence_idx)) # embedded = [sequence_length, batch_size, embedding_size]
        outputs, (hidden, cell) = self.lstm(embedded)
        # outputs = [batch_size, sequence_length, hidden_dimension]
        # hidden = [n_layers, batch_size, hidden_dimension]
        # hidden[-1] = [batch_size, sequence_length]
        lstm_output = outputs[:, -1, :]
        #lstm_output = torch.cat( [lstm_output, rank], 1)
        #tag_space = self.hidden2tag(lstm_output)
        # tag_space = [n_layers, sequence_length, output_dim]
        #tag_scores = F.log_softmax(tag_space, dim=1)
        #tag_space = tag_space.squeeze()
        return lstm_output

class FFNN(nn.Module):
    def __init__(self, hid_dim):
        super(FFNN, self).__init__()
        self.hid_dim = hid_dim * 2

        self.fc1 = torch.nn.Linear(self.hid_dim + 1 , self.hid_dim + 1)
        self.relu = torch.nn.ReLU()
        self.hidden2tag = nn.Linear( self.hid_dim + 1, 1 )

    def forward(self, lstm_output, rank):
        data_input = torch.cat( [lstm_output, rank], 1 )
        hidden = self.fc1(data_input)
        relu = self.relu(hidden)
        tag_space = self.hidden2tag(relu)
        tag_space = tag_space.squeeze()
        return tag_space

class Optic(nn.Module):
    def __init__(self, lstm, ffnn, device):
        super(Hel, self).__init__()
        self.lstm = lstm
        self.ffnn = ffnn
        self.device = device

    def forward(self, sequence_idx, rank):
        lstm_output = self.lstm(sequence_idx)
        output = self.ffnn(lstm_output, rank)
        return output.to(self.device)

max_stars_repo_name: ItaloLopes/optic

max_issues_repo_name: ItaloLopes/optic

max_forks_repo_name: ItaloLopes/optic


text:
import unittest

import lasagne
import theano.tensor as T

class TestLasagne(unittest.TestCase):
    def test_network_definition(self):
        input_var = T.tensor4('X')

        network = lasagne.layers.InputLayer((None, 3, 32, 32), input_var)
        network = lasagne.layers.Conv2DLayer(network, 64, (3, 3))

        params = lasagne.layers.get_all_params(network, trainable=True)

        self.assertEqual(2, len(params))

reason: import theano


text:
#!/usr/bin/env python
import math
import numpy as np
import pandas as pd
import random
import string
from scipy.stats import zipf
from itertools import chain
import json

class contentCatalogue():
    def __init__(self, size=1000):
        '''
        Assigns the size and constructs an empty list of contents. Constructs
        an empty list for popularities of contents (probabilities). Constructs an
        empty content matrix as a list.

        Input: contents: list, popularity: list, contentMatrix: list, size: int
        '''
        self.size = size
        self.contents = list()
        self.popularity = list()
        self.contentMatrix = list()

    def characteristics(self):
        '''
        Output: returns specs of content catalogue
        '''
        return 'Content Catalogue Size: {}\nContent Catalogue Popularity:\n{}\nContent Catalogue:\n{}\nContent Catalogue Relations:\n{}'.format(self.size, self.popularity, self.contents, self.contentMatrix)

    def randomSingleContentGenerator(self, stringLength=8):
        """Generate a random string of letters and digits """
        lettersAndDigits = string.ascii_letters + string.digits
        return ''.join(random.choice(lettersAndDigits) for i in range(stringLength))

    def randomMultipleContentsGenerator(self):
        """Generate a list of random strings of letters and digits """
        contents = list()
        for i in range(0, self.size):
            contents.append(self.randomSingleContentGenerator())

        assert(len(contents) == self.size)
        return contents

    def getNrandomElements(self, list, N):
        '''
        Returns random elements from a list

        Input: list and num of items to be returned
        Output: list of N random items
        '''
        return random.sample(list, N)

    def getContents(self):
        '''
        Output: returns contents as a list
        '''
        return self.contents

    def getContentsLength(self):
        '''
        Output: returns contents size
        '''
        return len(self.contents)

    def zipf_pmf(self, lib_size, expn):
        '''
        Returns a probability mass function (list of probabilities summing to 1.0) for the Zipf distribution with the given size "lib_size" and exponent "expn"

        Input: size of content catalogue, exponent
        Output: list of probabilities that sum up to 1
        '''
        K = lib_size
        p0 = np.zeros([K])
        for i in range(K):
            p0[i] = ((i+1)**(-expn))/np.sum(np.power(range(1, K+1), -expn))
        return p0

    def setContentsPopularity(self, distribution='zipf', a=0.78):
        '''
        Sets the popularity of contents given a distribution (zipf by default) .

        Input: distribution, exponent
        Output: vector of probabilities that correspond to the content catalogue
        '''
        if distribution == 'zipf':
            prob = self.zipf_pmf(self.getContentsLength(), a)
            return prob
        else:
            raise Exception('Distribution \'' + distribution +
                            '\' not implemented yet')

    def initializeContentCatalogue(self, contents):
        '''
        Set the content catalogue for the first time in a list format

        Input: strings in a list
        '''
        self.contents = contents
        self.popularity = self.setContentsPopularity()

    def symmetrize(self, a):
        '''
        Forces symmetricity in a content matrix

        Input: a matrix
        Output: a symmetric matrix provided from the original
        '''
        return np.tril(a) + np.tril(a, -1).T

    def createRandomContentMatrixBinary(self, symmetric=True, numOfRelations=10, outputForm='dataframe'):
        '''
        TODO: Fix commentary in this piece of code
        '''
        numOfContents = self.getContentsLength()
        contentMatrix = np.zeros((numOfContents, numOfContents))
        idx = np.random.rand(numOfContents, numOfContents).argsort(1)[
            :, :numOfRelations]
        contentMatrix[np.arange(numOfContents)[:, None], idx] = 1
        print(contentMatrix)
        # if symmetric:
        #     contentMatrix = self.symmetrize(contentMatrix)
        # print(contentMatrix)

        # for row in contentMatrix:
        #     if(len(row) != numOfRelations):
        #         print(row)
        # # print(contentMatrix)
        # for i in range(numOfContents):
        #     for j in range(numOfContents):
        #         if i == j and contentMatrix[i][j] == 1:
        #             indexesOfZeros = np.argwhere(
        #                 contentMatrix[i] == 0).tolist()
        #             contentMatrix[i][j] = 0
        # for i in range(numOfContents):
        #     for j in range(numOfContents):
        #         # print(i, j)
        #         indexesOfOnesCurrentNodeRow = np.argwhere(
        #             contentMatrix[i] == 1).tolist()
        #         # print(len(indexesOfOnesCurrentNodeRow))
        #         while len(indexesOfOnesCurrentNodeRow) < numOfRelations:
        #             randomChoiceOfIndex = random.choice(
        #                 indexesOfOnesCurrentNodeRow)[0]
        #             indexesOfOnesRelatedNodesRow = np.argwhere(
        #                 contentMatrix[randomChoiceOfIndex] == 1).tolist()
        #             if len(indexesOfOnesRelatedNodesRow) < numOfRelations:
        #                 contentMatrix[i][randomChoiceOfIndex] = 1
        #                 contentMatrix[randomChoiceOfIndex][i] = 1

        # assert symmetricity
        # assert(np.allclose(contentMatrix, contentMatrix.T))
        self.contentMatrix = contentMatrix

        # Return in a specific format (list or df)
        if outputForm == 'dataframe':
            names = [_ for _ in self.getContents()]
            df = pd.DataFrame(contentMatrix, index=names, columns=names)
            return df
        return contentMatrix

    def loadContentMatrix_JSON(self, url):
        '''
        Loads an item based NxN content matrix (IDs in rows/columns).

        Input: url of content matrix
        '''
        out = None
        with open(url, 'r') as f:
            out = json.load(f)
        self.contentMatrix = np.array(out)

    def loadContentMatrix_CSV(self, url):
        '''
        Loads an item based NxN content matrix (IDs in rows/columns).
        Also initializes the content catalogue with the given column names

        Input: url of content matrix as a CSV file
        '''
        data = pd.read_csv(url, delimiter='\t')
        self.initializeContentCatalogue(list(data.columns)[1:])
        data = [list(x[1:]) for x in data.to_numpy()]
        self.contentMatrix = np.array(data)

    def relatedContents(self, id):
        '''
        Returns all non zero relations to a given ID
        Relations are extracted from a content matrix (cm)

        Input: id
        Output: related contents list
        '''
        # extract all relations from content matrix
        candidateRelated = self.contentMatrix[self.contents.index(id)]
        # print(len(candidateRelated))
        # extract all non zero relations from the above list - just indexes
        indexesOfPositiveRelations = np.argwhere(candidateRelated == 1)
        # print(len(indexesOfPositiveRelations))

        # make the above indexes a single list, for easier reference
        indexesOfPositiveRelations = list(
            chain.from_iterable(indexesOfPositiveRelations))
        # dereference the indexes => acquire a list of related contents
        related = [self.contents[i] for i in indexesOfPositiveRelations]
        toReturn = []
        # Return also the relation weight for each related content
        for rel in related:
            toReturn.append(
                (rel, candidateRelated[self.contents.index(rel)]))

        # Return items sorted in descending relevance (most relevant item in first position)
        return sorted(toReturn, key=lambda x: x[1], reverse=True)

def main():
    N = 10  # number of relations
    W = 5
    MP = 10000
    r = contentCatalogue(size=10000)
    r.initializeContentCatalogue(r.randomMultipleContentsGenerator())
    # Set numOfRelations equal to the number of relations you want each content to have with all others
    r.createRandomContentMatrixBinary(symmetric=False, numOfRelations=N)

    # Get content catalogue
    names = [_ for _ in r.getContents()]
    df = pd.DataFrame(r.contentMatrix, index=names, columns=names)
    df.to_csv(r'JointCachingRecommendations\Results\contentCatalogue.csv', sep='\t')

    # Get top most popular contents
    mostPopularContents = r.getContents()[0:MP]
    # print(r.relatedContents(mostPopularContents[0]))
    with open(r'JointCachingRecommendations\Results\mostPopular.json', 'w') as f:
        json.dump(mostPopularContents, f, indent=4)

    # Get depth 1 related
    depth1 = {}
    for popular in mostPopularContents:
        depth1[popular] = [x[0] for x in r.relatedContents(popular)[0:W]]
        # print(len(depth1[popular]))
    with open(r'JointCachingRecommendations\Results\dataSet_depth1_width50.json', 'w') as f:
        json.dump(depth1, f, indent=4)

    # Get depth 2 related
    depth1values = list(
        set([item for sublist in depth1.values() for item in sublist]))
    depth2 = {}
    for item in depth1values:
        depth2[item] = [x[0] for x in r.relatedContents(item)[0:W]]
    with open(r'JointCachingRecommendations\Results\dataSet_depth2_width50.json', 'w') as f:
        json.dump(depth2, f, indent=4)

if __name__ == "__main__":
    main()

max_stars_repo_name: kastanakis/CABaRet

max_issues_repo_name: kastanakis/CABaRet

max_forks_repo_name: kastanakis/CABaRet

reason: import numpy,from scipy


text:
import os
import librosa
import numpy as np
import matplotlib.pyplot as plt
from configuration import get_config
from utils import keyword_spot

config = get_config()   # get arguments from parser

# downloaded dataset path
audio_path= r'F:\DataSet\train_test'          # utterance dataset
clean_path = r'F:\DataSet\clean_testset_wav'  # clean dataset
noisy_path = r'F:\DataSet\noisy_testset_wav'  # noisy dataset

def extract_noise():
    """ Extract noise and save the spectrogram (as numpy array in config.noise_path)
        Need: paired clean and noisy data set
    """
    print("start noise extraction!")
    os.makedirs(config.noise_path, exist_ok=True)              # make folder to save noise file
    total = len(os.listdir(clean_path))                        # total length of audio files
    batch_frames = config.speaker_number * config.utterance_number * config.tdsv_frame     # TD-SV frame number of each batch
    stacked_noise = []
    stacked_len = 0
    k = 0
    for i, path in enumerate(os.listdir(clean_path)):
        clean, sr = librosa.core.load(os.path.join(clean_path, path), sr=8000)  # load clean audio
        noisy, _ = librosa.core.load(os.path.join(noisy_path, path), sr=sr)     # load noisy audio
        noise = clean - noisy       # get noise audio by subtract clean voice from the noisy audio
        S = librosa.core.stft(y=noise, n_fft=config.nfft,
                              win_length=int(config.window * sr), hop_length=int(config.hop * sr))   # perform STFT
        stacked_noise.append(S)
        stacked_len += S.shape[1]

        if i % 100 == 0:
            print("%d processing..." % i)

        if stacked_len < batch_frames:   # if noise frames is short than batch frames, then continue to stack the noise
            continue

        stacked_noise = np.concatenate(stacked_noise, axis=1)[:,:batch_frames]          # concat noise and slice
        np.save(os.path.join(config.noise_path, "noise_%d.npy" % k), stacked_noise)   # save spectrogram as numpy file
        print(" %dth file saved" % k, stacked_noise.shape)
        stacked_noise = []     # reset list
        stacked_len = 0
        k += 1

    print("noise extraction is end! %d noise files" % k)

def save_spectrogram_tdsv():
    """ Select text specific utterance and perform STFT with the audio file.
        Audio spectrogram files are divided as train set and test set and saved as numpy file.
        Need : utterance data set (VTCK)
    """
    print("start text dependent utterance selection")
    os.makedirs(config.train_path, exist_ok=True)   # make folder to save train file
    os.makedirs(config.test_path, exist_ok=True)    # make folder to save test file

    utterances_spec = []
    for folder in os.listdir(audio_path):
        spk_dir = os.path.join(audio_path, folder)
        spk_utts = os.listdir(spk_dir)
        spk_utts.sort()
        utter_path= os.path.join(spk_dir, spk_utts[0])
        if os.path.splitext(os.path.basename(utter_path))[0][-3:] != '001':  # if the text utterance doesn't exist pass
            print(os.path.basename(utter_path)[:4], "001 file doesn't exist")
            continue

        utter, sr = librosa.core.load(utter_path, config.sr)               # load the utterance audio
        utter_trim, index = librosa.effects.trim(utter, top_db=14)         # trim the beginning and end blank
        if utter_trim.shape[0]/sr <= config.hop*(config.tdsv_frame+2):     # if trimmed file is too short, then pass
            print(os.path.basename(utter_path), "voice trim fail")
            continue

        S = librosa.core.stft(y=utter_trim, n_fft=config.nfft,
                              win_length=int(config.window * sr), hop_length=int(config.hop * sr))  # perform STFT
        S = keyword_spot(S)          # keyword spot (for now, just slice last 80 frames which contains "Call Stella")
        utterances_spec.append(S)    # make spectrograms list

    utterances_spec = np.array(utterances_spec)  # list to numpy array
    np.random.shuffle(utterances_spec)           # shuffle spectrogram (by person)
    total_num = utterances_spec.shape[0]
    train_num = (total_num//10)*9                # split total data 90% train and 10% test
    print("selection is end")
    print("total utterances number : %d"%total_num, ", shape : ", utterances_spec.shape)
    print("train : %d, test : %d"%(train_num, total_num- train_num))
    np.save(os.path.join(config.train_path, "train.npy"), utterances_spec[:train_num])  # save spectrogram as numpy file
    np.save(os.path.join(config.test_path, "test.npy"), utterances_spec[train_num:])

def save_spectrogram_tisv():
    """ Full preprocess of text independent utterance. The log-mel-spectrogram is saved as numpy file.
        Each partial utterance is splitted by voice detection using DB
        and the first and the last 180 frames from each partial utterance are saved.
        Need : utterance data set (VTCK)
    """
    print("start text independent utterance feature extraction")
    os.makedirs(config.train_path, exist_ok=True)   # make folder to save train file
    os.makedirs(config.test_path, exist_ok=True)    # make folder to save test file

    # utter_min_len = 0
    utter_min_len = (config.tisv_frame * config.hop + config.window) * config.sr    # lower bound of utterance length
    total_speaker_num = len(os.listdir(audio_path))
    # train_speaker_num= (total_speaker_num//10)*9            # split total data 90% train and 10% test
    train_speaker_num= total_speaker_num
    print("total speaker number : %d"%total_speaker_num)
    print("train : %d, test : %d"%(train_speaker_num, total_speaker_num-train_speaker_num))
    for i, folder in enumerate(os.listdir(audio_path)):
        speaker_path = os.path.join(audio_path, folder)     # path of each speaker
        print("%dth speaker processing..."%i)
        utterances_spec = []
        k=0
        for utter_name in os.listdir(speaker_path):
            utter_path = os.path.join(speaker_path, utter_name)         # path of each utterance
            utter, sr = librosa.core.load(utter_path, config.sr)        # load utterance audio
            intervals = librosa.effects.split(utter, top_db=20)         # voice activity detection
            for interval in intervals:
                if (interval[1]-interval[0]) >= utter_min_len:           # If partial utterance is sufficient long,
                    utter_part = utter[interval[0]:interval[1]]         # save first and last 180 frames of spectrogram.
                    S = librosa.core.stft(y=utter_part, n_fft=config.nfft,
                                          win_length=int(config.window * sr), hop_length=int(config.hop * sr))
                    S = np.abs(S) ** 2
                    mel_basis = librosa.filters.mel(sr=config.sr, n_fft=config.nfft, n_mels=40)
                    S = np.log10(np.dot(mel_basis, S) + 1e-6)           # log mel spectrogram of utterances

                    if (interval[1]-interval[0]) > utter_min_len:
                      utterances_spec.append(S[:, :config.tisv_frame])    # first 180 frames of partial utterance
                      utterances_spec.append(S[:, -config.tisv_frame:])   # last 180 frames of partial utterance
                    else:
                      utterances_spec.append(S[:, :config.tisv_frame])    # first 180 frames of partial utterance

        utterances_spec = np.array(utterances_spec)
        print(utterances_spec.shape)
        if i<train_speaker_num:      # save spectrogram as numpy file
            np.save(os.path.join(config.train_path, "speaker%d.npy"%i), utterances_spec)
        else:
            np.save(os.path.join(config.test_path, "speaker%d.npy"%(i-train_speaker_num)), utterances_spec)

if __name__ == "__main__":
    # extract_noise()
    if config.tdsv:
        save_spectrogram_tdsv()
    else:
        save_spectrogram_tisv()


text:
# -*- coding: utf-8 -*-
"""
Created on Sat Sep 27 13:29:51 2014

Demonstration for the samplerate module that is wrapper around the
Secret Rabbit Code from Erik de Castro Lopo (http://www.mega-nerd.com/SRC/)
offering high quality real-time resampling.
"""

import numpy as np
import matplotlib.pyplot as plt
import scikits.samplerate as SRC
#from scikits.samplerate import resample

fs = 44100.
fr = 48000.
# Signal to resample
sins = np.sin(2*np.pi*1000/fs*np.arange(0, fs*2))
# Ideal resampled signal
idsin = np.sin(2*np.pi*1000/fr*np.arange(0, fr*2))
#'zero_order_hold', 'linear', 'sinc_fastest', _medium and_best
conv1 = SRC.resample(sins, fr/fs, 'linear')
conv3 = SRC.resample(sins, fr/fs, 'sinc_best')

err1 = conv1[fr:fr+2000] - idsin[fr:fr+2000]
err3 = conv3[fr:fr+2000] - idsin[fr:fr+2000]

plt.subplot(3, 1, 1)
plt.plot(idsin[fs:fs+2000])
plt.title('Resampler residual quality comparison')

plt.subplot(3, 1, 2)
plt.plot(err1)
plt.ylabel('Err. Linear')

plt.subplot(3, 1, 3)
plt.plot(err3)
plt.ylabel('Err. Sinc')
#plt.savefig('example1.png', dpi = 100)
plt.show()


text:
#
# This file is part of the profilerTools suite (see
# https://github.com/mssm-labmmol/profiler).
#
# Copyright (c) 2020 mssm-labmmol
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from .opts import (optOpts, randOpts, minimOpts, lastMinimOpts,
                   vbgaOpts, llsOpts, dihrestrOpts, geomcheckOpts,
                   cmdlineOpts)
from .randomizer import (RandomizerFactory, LimiterDecorator,
                         SignReverserDecorator)
from sys import stderr
from .configuration import ensemble
import numpy as np
from functools import partial
import itertools

def getline(stream, comm=';'):
    buff = ''
    while buff == '':
        buff = stream.readline()
        if buff == '':
            return buff
        if buff.strip() == '':
            return "END"
        buff = buff.strip()
        buff = buff.split(comm)[0]
    return buff

def getblock(stream, block_cont, comm=';'):
    block_cont.clear()
    _buff = ''
    first = True
    while (True):
        _buff = getline(stream, comm)
        if (_buff != ''):
            if (_buff == 'END'):
                break
            block_cont += [
                fld for fld in _buff.split() if (fld != '[') and (fld != ']')
            ]
            # check if all arguments are numbers
            offset = 3 if first else 0
            for x in _buff.split()[offset:]:
                try:
                    float(x)
                except ValueError:
                    raise ValueError(
                        "%s is not a numeric option for block %s." %
                        (x, block_cont))
        else:
            return False
        first = False
    return True

def read_PARAMETEROPTIMIZATION(blockdict):
    blockname = 'parameter_optimization'
    blocklist = blockdict[blockname]
    optOpts.nTors = int(blocklist.pop(0))
    optOpts.nLJ = int(blocklist.pop(0))
    #optOpts.dihType = {1: 'standard', 2: 'ryckaert'}[int(blocklist.pop(0))]
    optOpts.dihType = 'standard'
    optOpts.isFourier = {1: False, 2: True}[int(blocklist.pop(0))]
    randOpts.mslots = True
    optOpts.kMask = []
    optOpts.phiMask = []
    optOpts.optTors = []
    optOpts.LJMask  = []
    npars = 0
    for j in range(optOpts.nTors):
        currKMask = [0, 0, 0, 0, 0, 0]
        currPhiMask = [0, 0, 0, 0, 0, 0]
        optTorsMask = [0, 0, 0, 0, 0, 0]
        for i in range(6):
            nt = int(blocklist.pop(0))
            if (nt == 0):
                pass
            elif ((i > 3) and optOpts.isFourier):
                print("Warning: Setting k_%d = 0 for Fourier functional form." % (i + 1))
                pass
            elif (nt == 1):
                currKMask[i] = 1
                optTorsMask[i] = 1
                npars += 1
            elif (nt == 2):
                currKMask[i] = 1
                currPhiMask[i] = 1
                optTorsMask[i] = 1
                npars += 2
            else:
                raise Exception()
        optOpts.kMask.append(currKMask)
        optOpts.phiMask.append(currPhiMask)
        optOpts.optTors.append(optTorsMask)

    if optOpts.nTors != len(optOpts.kMask):
        raise ValueError(
            f"Input file: Number of dihedral types {len(optOpts.kMask)} do not match specification {optOpts.nTors}.")

    optOpts.optTors = np.array(optOpts.optTors, dtype=np.uint8)

    for j in range(optOpts.nLJ):
        cs6switch = int(blocklist.pop(0))
        cs12switch = int(blocklist.pop(0))
        if cs6switch == 1:
            npars += 1
        if cs12switch == 1:
            npars += 1
        optOpts.LJMask.append((cs6switch, cs12switch))

    if optOpts.nLJ != len(optOpts.LJMask):
        raise ValueError(
            "Input file: Number of LJ types do not match specification.")

    if (len(optOpts.LJMask) == 0):
        optOpts.LJMask = None
    if (len(optOpts.kMask) == 0):
        optOpts.kMask = None
    if (len(optOpts.phiMask) == 0):
        optOpts.phiMask = None

    optOpts.wTemp = float(blocklist.pop(0))

    return npars

def read_PARAMETERRANDOMIZATION(blockdict):
    blockname = 'parameter_randomization'
    if blockname in blockdict.keys():
        blocklist = blockdict[blockname]
        randOpts.torsPinv = int(blocklist.pop(0))
        if randOpts.torsPinv == 0:
            raise Exception(
                "The value of PINV must be non-zero to allow not"
                " taking the phase constants into account; "
                "read the documentation for more details."
            )
        randOpts.torsDist = int(blocklist.pop(0))
        randOpts.torsMin = float(blocklist.pop(0))
        randOpts.torsMax = float(blocklist.pop(0))
        randOpts.torsMean = float(blocklist.pop(0))
        randOpts.torsStddev = float(blocklist.pop(0))
        randOpts.cs6Dist = int(blocklist.pop(0))
        randOpts.cs6Min = float(blocklist.pop(0))
        randOpts.cs6Max = float(blocklist.pop(0))
        randOpts.cs6Mean = float(blocklist.pop(0))
        randOpts.cs6Stddev = float(blocklist.pop(0))
        randOpts.cs12Dist = int(blocklist.pop(0))
        randOpts.cs12Min = float(blocklist.pop(0))
        randOpts.cs12Max = float(blocklist.pop(0))
        randOpts.cs12Mean = float(blocklist.pop(0))
        randOpts.cs12Stddev = float(blocklist.pop(0))

def read_SEED(blockdict):
    blockname = 'seed'
    blocklist = blockdict[blockname]
    randOpts.seed = int(blocklist.pop(0))

def read_EVOLUTIONARYSTRAT(blockdict):
    blockname = 'evolutionary_strat'
    if blockname in blockdict.keys():
        blocklist = blockdict[blockname]
        strategy_switch = int(blocklist.pop(0))
        if strategy_switch == 1:
            vbgaOpts.strategy = 'CMA-ES'
        elif strategy_switch == 2:
            vbgaOpts.strategy = 'GA'
        else:
            raise ValueError
        vbgaOpts.popSize = int(blocklist.pop(0))
        if (vbgaOpts.popSize <= 0):
            raise Exception("Population size must be a positive number.")
        vbgaOpts.nGens = int(blocklist.pop(0))

def read_LLSSC(blockdict, number_of_pars):
    blockname = 'lls_sc'
    if blockname in blockdict.keys():
        blocklist = blockdict[blockname]
        llsOpts.max_cycles = int(blocklist.pop(0))
        llsOpts.max_dpar = float(blocklist.pop(0))
        llsOpts.reg_switch = int(blocklist.pop(0))
        if (llsOpts.reg_switch != 0):
            llsOpts.reg_center = []
            for i in range(number_of_pars):
                llsOpts.reg_center.append(float(blocklist.pop(0)))
            llsOpts.reg_center = np.array(llsOpts.reg_center)
            llsOpts.lamb = float(blocklist.pop(0))
            if (llsOpts.lamb <= 0):
                llsOpts.lamb = None

def read_MINIMIZATION(blockdict):
    blockname = 'minimization'
    blocklist = blockdict[blockname]
    minimOpts.minimType = int(blocklist.pop(0))
    minimOpts.dx0 = float(blocklist.pop(0))
    minimOpts.dxm = float(blocklist.pop(0))
    minimOpts.maxSteps = int(blocklist.pop(0))
    minimOpts.dele = float(blocklist.pop(0))
    if (minimOpts.minimType != 0) and (minimOpts.maxSteps == 0):
        raise RuntimeError(
            "Minimization algorithm is non-zero but the number of steps is 0!")
    if (minimOpts.minimType == 0):
        minimOpts.minimType = 1
        minimOpts.maxSteps = 0
    lastMinimOpts.minimType = int(blocklist.pop(0))
    lastMinimOpts.dx0 = float(blocklist.pop(0))
    lastMinimOpts.dxm = float(blocklist.pop(0))
    lastMinimOpts.maxSteps = int(blocklist.pop(0))
    lastMinimOpts.dele = float(blocklist.pop(0))
    if (lastMinimOpts.minimType != 0) and (lastMinimOpts.maxSteps == 0):
        raise RuntimeError(
            "Minimization algorithm is non-zero but the number of steps is 0!")
    if (lastMinimOpts.minimType == 0):
        lastMinimOpts.minimType = 1
        lastMinimOpts.maxSteps = 0

def read_TORSIONALSCAN(blockdict):
    blockname = 'torsional_scan'
    blocklist = blockdict[blockname]
    dihrestrOpts.origin = int(blocklist.pop(0))
    dihrestrOpts.ntypes = int(blocklist.pop(0))
    if dihrestrOpts.ntypes < 1:
        raise RuntimeError("You need at least one dihedral-restraint type.")
    for type_ in range(dihrestrOpts.ntypes):
        dihrestrOpts.start.append(float(blocklist.pop(0)))
        dihrestrOpts.step.append(float(blocklist.pop(0)))
        dihrestrOpts.last.append(float(blocklist.pop(0)))
        dihrestrOpts.k.append(float(blocklist.pop(0)))
        dihrestrOpts.nPoints.append(
            int(
                int(dihrestrOpts.last[-1] - dihrestrOpts.start[-1]) /
                dihrestrOpts.step[-1]) + 1)

def read_GEOMETRYCHECK(blockdict):
    blockname = 'geometry_check'
    blocklist = blockdict[blockname]
    geomcheckOpts.lvlBond = int(blocklist.pop(0))
    geomcheckOpts.tolBond = float(blocklist.pop(0))
    geomcheckOpts.lvlAng = int(blocklist.pop(0))
    geomcheckOpts.tolAng = float(blocklist.pop(0))
    geomcheckOpts.lvlRestr = int(blocklist.pop(0))
    geomcheckOpts.tolRestr = float(blocklist.pop(0))
    geomcheckOpts.lvlImpr = int(blocklist.pop(0))
    geomcheckOpts.tolImpr = float(blocklist.pop(0))

# returns a dictionary with the contents of the blocks with their names as keys
def input2dict(fn):
    fp = open(fn, 'r')
    out_dict = {}
    block_cont = []
    while (True):
        read_code = getblock(fp, block_cont)
        try:
            out_dict[block_cont[0]] = block_cont[1:]
        except IndexError:
            pass
        if not (read_code):
            break
    fp.close()
    return out_dict

def readinput(fn):
    outdict = input2dict(fn)
    npars = read_PARAMETEROPTIMIZATION(outdict)
    read_EVOLUTIONARYSTRAT(outdict)
    read_PARAMETERRANDOMIZATION(outdict)
    read_LLSSC(outdict, npars)
    read_MINIMIZATION(outdict)
    read_TORSIONALSCAN(outdict)
    read_SEED(outdict)

def preprocess_args(argv,
                    file_flag='-f'):  # argv should not contain the script name
    first = True
    found_file_flag = False
    if (argv == []):
        print("Argument list is empty!", file=stderr)
        exit()
    for i, arg in enumerate(argv):
        if (arg == file_flag):
            found_file_flag = True
            if not (first):
                print("If present,",
                      file_flag,
                      "<file> must be the only option.",
                      file=stderr)
                exit()
        else:
            if (found_file_flag):
                if (i != len(argv) - 1):
                    print("If present,",
                          file_flag,
                          "<file> must be the only option.",
                          file=stderr)
                    exit()
                # in this case all arguments passed to arparser are in the argfile
                args = []
                readargfile(arg, args)
                return args
        first = False
    return argv

def readargfile(fn, block_cont, comm='#'):
    block_cont.clear()
    _buff = ''
    stream = open(fn, 'r')
    while (True):
        _buff = getline(stream, comm)
        if (_buff != ''):
            block_cont += _buff.split()
        else:
            break
    stream.close()

def argparse2opts(args):
    cmdlineOpts.nProcs = args.nProcs
    cmdlineOpts.trajFiles = args.coords
    cmdlineOpts.stpFiles = args.pars
    cmdlineOpts.weiFiles = args.wei if args.wei is not None else []
    cmdlineOpts.refFiles = args.ref
    cmdlineOpts.outPrefix = args.out
    cmdlineOpts.inputFile = args.input
    cmdlineOpts.debugEmm = args.emm
    cmdlineOpts.dihspecFiles = args.dih_spec
    # consistency check
    nref = len(args.ref)
    ncoords = len(args.coords)
    npars = len(args.pars)
    nwei = len(args.wei) if args.wei is not None else ncoords
    if (nref == ncoords) and (ncoords == npars) and (npars == nwei):
        optOpts.nSystems = nref
    else:
        raise RuntimeError(
            "Number of reference files, coordinate files, parameter files and "
            "weight files (if supplied) do not match.")

# ====================================================================
# Factories/other utils for initializing objects based on input
# parameters.
# ====================================================================

def MaskLooper(loop_func, LJMasks=None, kMasks=None, phiMasks=None):
    if not (len(kMasks) == len(phiMasks)):
        raise ValueError()
    type_count = 0
    out = []
    types = []
    if (LJMasks is not None):
        for C6Switch, C12Switch in LJMasks:
            if C6Switch != 0:
                out.append(loop_func('c6'))
                types.append(type_count)
            if C12Switch != 0:
                out.append(loop_func('c12'))
                types.append(type_count)
            type_count += 1
    type_count_save = type_count
    if (kMasks is not None):
        for sArr in kMasks:
            for m, s in enumerate(sArr):
                if s != 0:
                    out.append(loop_func('k', m + 1))
                    types.append(type_count)
            type_count += 1
    type_count = type_count_save
    if (phiMasks is not None):
        for sArr in phiMasks:
            for m, s in enumerate(sArr):
                if s != 0:
                    out.append(loop_func('phi', m + 1))
                    types.append(type_count)
            type_count += 1
    return out, types

class IndexListCreator:
    def __init__(self, optType, optAtoms, optPairs, optDihs):
        self.optType = optType
        self.optAtoms = optAtoms
        self.optPairs = optPairs
        self.optDihs = optDihs
        self.lenLJ = self._getLenLJ()

    def _getAtomOrPair(self, i):
        if (self.optType == 'atom'):
            return self.optAtoms[i]
        elif (self.optType == 'pair'):
            return self.optPairs[i]
        else:
            raise ValueError

    def _getLenLJ(self):
        if (self.optType == 'atom'):
            return len(self.optAtoms)
        elif (self.optType == 'pair'):
            return len(self.optPairs)
        else:
            return 0

    def _getDih(self, i):
        return self.optDihs[i]

    def get(self, idx):
        """Returns the indexes of the DOFs for the optimized parameter(s) of
        global type `idx`.

        For optimized dihedrals, the output indexes /do not/ take into account
        the different ordering in the MM calculator. Use
        :method:`~profilerTools.energy_force_vectorized.MMCalculator.idx_to_OptIdx`
        to convert to the internal index used in the MMCalculator.
        """
        if idx >= self.lenLJ:
            return self._getDih(idx - self.lenLJ)
        else:
            return self._getAtomOrPair(idx)

class Global_Type_IndexConverter:
    """This class is responsible for the conversion between global parameter
    indexes and type indexes. Global index distinguishes between parameters
    within the same type, e.g. cs6 and cs12. The parameters are ordered with the
    following priorities:

    (1) If p_i is Lennard-Jones and p_j is torsional, p_i < p_j.
    (2) If type_index(p_i) < type_index(p_j), p_i < p_j.
    (3) If p_i is cs6 and p_j is cs12, p_i < p_j.
    (4) If p_i is force constant and p_j is phase, p_i < p_j.
    (5) If p_i and p_j are both force constant or phase and m(p_i) < m(p_j),
        then p_i < p_j.

    Type indexes do not distinguish between parameters of the same type, and
    start at zero for each type of parameter (i.e., type indexes for torsional
    parameters start at 0, as well as those for LJ parameters).
    """

    def __init__(self, nTors=None, nLJ=None,
                 kMask=None, phiMask=None, LJMask=None):

        if (nTors is None):
            nTors = optOpts.nTors

        if (nLJ is None):
            nLJ = optOpts.nLJ

        if (kMask is None):
            kMask = optOpts.kMask

        if (phiMask is None):
            phiMask = optOpts.phiMask

        if (LJMask is None):
            LJMask = optOpts.LJMask

        self._global2type_tors = dict()
        self._global2type_lj   = dict()
        self._global2string    = dict()

        p = 0
        for t in range(nLJ):
            for i, pt in enumerate(LJMask[t]):
                if pt != 0:
                    self._global2type_lj[p] = t
                    if (i == 0):
                        self._global2string[p] = "cs6"
                    elif (i == 1):
                        self._global2string[p] = "cs12"
                    p += 1

        for t in range(nTors):
            for i, pt in enumerate(kMask[t]):
                if pt != 0:
                    self._global2type_tors[p] = nLJ + t
                    self._global2string[p] = f"k_{i+1}"
                    p += 1
            for i, pt in enumerate(phiMask[t]):
                if pt != 0:
                    self._global2type_tors[p] = nLJ + t
                    self._global2string[p] = f"phi_{i+1}"
                    p += 1

    def global_to_type(self, idx):
        """Converts global `idx` to type `idx`. Returns a tuple with the
        converted index and the type of parameter (e.g., 'k_3',
        'phi_2', 'cs6').
        """
        if idx in self._global2type_tors.keys():
            return self._global2type_tors[idx], self._global2string[idx]
        else:
            return self._global2type_lj[idx], self._global2string[idx]

    def global_is_torsional(self, idx):
        """Returns `True` if the parameter with global index `idx` is
        torsional and `False` otherwise.
        """
        if idx in self._global2type_tors.keys():
            return True
        else:
            return False

    def type_is_torsional(self, idx):
        """Returns `True` if the parameter with type index `idx` is
        torsional and `False` otherwise.
        """
        if idx in self._global2type_tors.values():
            return True
        else:
            return False

    def type_to_global(self, idx):
        """Converts type index to a list of global indexes."""
        out = list()

        if self.type_is_torsional(idx):
            search_dict = self._global2type_tors
        else:
            search_dict = self._global2type_lj

        for k, v in search_dict.items():
            if (v == idx):
                out.append(k)

        return out

    def get_k_phi_pairs(self):
        """Returns a list of tuples with the indexes of all corresponding 'k',
        'phi' pairs."""
        def _check(elem, tuplelist):
            for x, y in tuplelist:
                if (elem == x) or (elem == y):
                    return True
            return False

        out = []
        for p_i, t_i in self._global2type_tors.items():
            pair_i = None
            pair_j = None
            name_i = self._global2string[p_i]
            if _check(p_i, out):
                # This means p_i is already listed.
                continue
            for p_j, t_j in self._global2type_tors.items():
                if p_j > p_i:
                    name_j = self._global2string[p_j]
                    if (t_i == t_j):
                        if (name_i.replace('k', 'phi') == name_j):
                            pair_i = p_i
                            pair_j = p_j
                            break
                        elif (name_i.replace('phi', 'k') == name_j):
                            pair_i = p_j
                            pair_j = p_i
                            break
            if name_i.startswith('k'):
                pair_i = p_i
            elif name_i.startswith('phi'):
                pair_j = p_i
            else:
                raise ValueError
            out.append((pair_i, pair_j))
        return out

def RandomizerSwitch2Type(sw):
    switch2type = {
        1: 'uniform',
        2: 'lognormal',
        3: 'gaussian',
        4: 'uniform_dim',
    }
    return switch2type[sw]

def RandomizerSwitch2Parameters(sw, min_, max_, mean, stdev):
    switch2pars = {
        1: {
            'min': min_,
            'max': max_
        },
        2: {
            'mean': mean,
            'stdev': stdev
        },
        3: {
            'mean': mean,
            'stdev': stdev
        },
        4: {
            'min': min_,
            'max': max_
        },
    }
    return switch2pars[sw]

def _InitRandomizerCore(typestr, m=None):
    if (typestr == 'c6'):
        return LimiterDecorator(
            RandomizerFactory(
                RandomizerSwitch2Type(randOpts.cs6Dist),
                **RandomizerSwitch2Parameters(randOpts.cs6Dist,
                                              randOpts.cs6Min, randOpts.cs6Max,
                                              randOpts.cs6Mean,
                                              randOpts.cs6Stddev)),
            randOpts.cs6Min, randOpts.cs6Max)
    elif (typestr == 'c12'):
        return LimiterDecorator(
            RandomizerFactory(
                RandomizerSwitch2Type(randOpts.cs12Dist),
                **RandomizerSwitch2Parameters(randOpts.cs12Dist,
                                              randOpts.cs12Min,
                                              randOpts.cs12Max,
                                              randOpts.cs12Mean,
                                              randOpts.cs12Stddev)),
            randOpts.cs12Min, randOpts.cs12Max)
    elif (typestr == 'k'):
        return SignReverserDecorator(
            LimiterDecorator(
                RandomizerFactory(
                    RandomizerSwitch2Type(randOpts.torsDist),
                    **RandomizerSwitch2Parameters(randOpts.torsDist,
                                                  randOpts.torsMin,
                                                  randOpts.torsMax,
                                                  randOpts.torsMean,
                                                  randOpts.torsStddev)),
                randOpts.torsMin, randOpts.torsMax), randOpts.torsPinv)
    elif (typestr == 'phi'):
        return RandomizerFactory('uniform', low=-180.00, up=+180.00)

def InitRandomizers():
    LJMasks = optOpts.LJMask
    kMasks = optOpts.kMask
    phiMasks = optOpts.phiMask

    rds = MaskLooper(_InitRandomizerCore, LJMasks, kMasks, phiMasks)[0]
    return rds

# ================ Dihedral-Restraint Reference Values ===============

def genRefDihedrals_Systematic(stpData, scanFirst, scanStep, scanLast):
    _refPhi = []
    for type_, reflist in enumerate(stpData['refdihedrals']):
        for d in reflist:
            scanValues = [
                scanFirst[type_] + i * scanStep[type_] for i in range(
                    int((scanLast[type_] - scanFirst[type_]) /
                        scanStep[type_]) + 1)
            ]
            _refPhi.append(scanValues)

    out = np.array(list(itertools.product(*_refPhi))).T
    if out.ndim == 1:
        out = out.reshape(1, -1)
    return out

def genRefDihedrals_Trajectory(stpData, trajFile):
    ens = ensemble()
    ens.readFromTrajectory(trajFile)
    refPhi = []
    for type_, reflist in enumerate(stpData['refdihedrals']):
        for d in reflist:
            zeroIndexed = [ax - 1 for ax in stpData['propers'][0][d][:4]]
            phis = []
            for conf in ens:
                phis.append(conf.getDihedral(*zeroIndexed))
            refPhi.append(phis)
    refPhi = np.array(refPhi)
    if refPhi.ndim == 1:
        refPhi = refPhi.reshape(1, -1)
    return refPhi

def genRefDihedrals_ExternalFile(dihedralSpecFile):
    out = (np.loadtxt(dihedralSpecFile)).T
    if out.ndim == 1:
        out = out.reshape(1, -1)
    return out

def genRefDihedrals(system):
    if (dihrestrOpts.origin == 1):
        return genRefDihedrals_Systematic(optOpts.stpData[system],
                                          dihrestrOpts.start,
                                          dihrestrOpts.step, dihrestrOpts.last)
    elif (dihrestrOpts.origin == 2):
        return genRefDihedrals_Trajectory(optOpts.stpData[system],
                                          cmdlineOpts.trajFiles[system])
    elif (dihrestrOpts.origin == 3):
        return genRefDihedrals_ExternalFile(cmdlineOpts.dihspecFiles[system])
    else:
        raise ValueError

max_stars_repo_name: mssm-labmmol/profiler

max_issues_repo_name: mssm-labmmol/profiler

max_forks_repo_name: mssm-labmmol/profiler


text:
import os
import torch
import torch.nn as nn
import torch.nn.functional as F
import snip
import numpy as np

# Apply feature pruning methods
def apply_zenprune(args, nets, data_loader):
    print('[*] Zen-Prune starts.')
    for net in nets:
        # net.eval()
        net.train()
        net.zero_grad()
        for layer in net.modules():
            snip.add_mask_ones(layer)
            # if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear) or isinstance(layer, nn.ConvTranspose2d):
            #     nn.init.normal_(layer.weight)

    model = nets[0]
    data_iter = iter(data_loader)
    imagesize = data_iter.next()[0].shape

    if args.iter_prune:
        num_iter = 100
    else:
        num_iter = 1

    n_x = 10
    n_eta = 10
    eta = 0.01

    for i in range(num_iter):
        # Zero out gradients for weight_mask so to start a new round of iterative pruning
        model.zero_grad()
        for layer in model.modules():
            if isinstance(layer, (nn.Conv2d, nn.Linear)):
                layer.weight_mask.grad = None
            if isinstance(layer, nn.BatchNorm2d):
                layer.running_mean.fill_(0)
                layer.running_var.fill_(1)

        for _ in range(n_x):
            # initialize weights drawn from Normal distribution N(0,1)
            # with torch.no_grad():
            #     for module in model.modules():
            #         if hasattr(module,'weight') and isinstance(module,(nn.Linear, nn.Conv2d, nn.ConvTranspose2d)):
            #             module.weight.data = torch.randn(module.weight.size(),device=module.weight.device)

            # Taking expectation w.r.t eta
            for _ in range(n_eta):
                input = torch.empty(imagesize)
                nn.init.normal_(input)
                noise = torch.empty(imagesize)
                nn.init.normal_(noise)
                input = input.cuda()
                noise = noise.cuda()
                output = model(input)
                output_perturb = model(input+0.01*noise)
                zen_score = torch.norm(output-output_perturb)
                zen_score.backward()

        # snip.prune_net_increaseloss(model, args.sparse_lvl**((i+1)/num_iter))
        # snip.net_iterative_prune(model, args.sparse_lvl**((i+1)/num_iter))
        snip.net_prune_grasp(model, args.sparse_lvl**((i+1)/num_iter))
        if i % 5 ==0:
            print('Prune ' + str(i) + ' iterations.')
            print('Zen-score is {}'.format(zen_score.item()))
    snip.deactivate_mask_update(model)
    for module in model.modules():
        if isinstance(module, (nn.Conv2d, nn.Linear, nn.ConvTranspose2d)):
            weight_check = module.weight_mask
            print(((weight_check!=0).float().sum()/weight_check.numel()))

    print('-'*20)

    # # remove hooks
    # for handle in handles:
    #     handle.remove()

    # zero out gradients of weights
    for net in nets:
        net.zero_grad()
        net.train()
        # net.reinit()

## Zen-Score Transfer
def apply_zentransfer(args, nets, data_loader):
    mask_update_freq = 5
    num_iter = 100

    print('[*] Zen-Transfer starts.')
    for net in nets:
        # net.eval()
        net.train()
        net.zero_grad()
        for layer in net.modules():
            snip.add_mask_rand(layer, args.sparse_lvl, modify_weight=False, structured=False, requires_grad = False)
            # if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear) or isinstance(layer, nn.ConvTranspose2d):
            #     nn.init.normal_(layer.weight)

    model = nets[0]
    data_iter = iter(data_loader)
    imagesize = data_iter.next()[0].shape
    optim = torch.optim.Adam(model.parameters(), 0.1)

    n_eta = 5
    eta = 0.01

    for i in range(num_iter):
        # Zero out gradients for weight so to start a new round of iterative pruning
        model.zero_grad()

        # Taking expectation w.r.t eta
        for _ in range(n_eta):
            input = torch.empty(imagesize)
            nn.init.normal_(input)
            noise = torch.empty(imagesize)
            nn.init.normal_(noise)
            input = input.cuda()
            noise = noise.cuda()
            output = model(input)
            output_perturb = model(input+0.01*noise)
            zen_score = torch.norm(output-output_perturb)
            (-zen_score).backward()

        optim.step()
        if i % 5 == 0:
            print('Prune ' + str(i) + ' iterations.')
            print('Zen-score is {}'.format(zen_score.item()))

        if i % mask_update_freq == 0:
            snip.net_prune_magnitude(model, args.sparse_lvl, modify_weight = False)
            # Reinitialize BatchNorm layers statistics
            # for layer in model.modules():
            #     if isinstance(layer, (nn.Conv2d, nn.Linear)):
            #         layer.weight_mask.grad = None
            #     if isinstance(layer, nn.BatchNorm2d):
            #         layer.running_mean.fill_(0)
            #         layer.running_var.fill_(1)

    for module in model.modules():
        if isinstance(module, (nn.Conv2d, nn.Linear, nn.ConvTranspose2d)):
            mask_check = module.weight_mask
            weight_check = module.weight
            print('Mask sparsity: {}'.format((mask_check!=0).float().sum()/mask_check.numel()))
            print('Weight sparsity: {}'.format((weight_check!=0).float().sum()/weight_check.numel()))

    print('-'*20)

    # # remove hooks
    # for handle in handles:
    #     handle.remove()

    # zero out gradients of weights
    for net in nets:
        net.zero_grad()
        net.train()
        # net.reinit()

def apply_cont_zenprune(args, nets, data_loader):

    print('[*] Continuous Zen-Prune starts.')

    # A list of parameters that we want to optimize
    mask_params = []
    # Layers that we want to prune
    candidate_layers = []
    # Total number of parameters that we want to prune
    total_param_num = 0

    for net in nets:
        # net.eval()
        net.train()
        net.zero_grad()
        for layer in net.modules():
            # Add continuous relaxed mask as parameters, initialized with gaussian
            # The forward method of layers when cont_relax is True is that
            # using weight: self.weight * torch.sigmoid(self.weight_mask)
            if isinstance(layer, (nn.Conv2d, nn.Linear, nn.ConvTranspose2d)):
                snip.add_mask_cont(layer, cont_relax=True)
                current_weight_mask = layer.weight_mask
                assert isinstance(current_weight_mask, nn.Parameter)
                candidate_layers.append(layer)
                mask_params.append(current_weight_mask)
                total_param_num += current_weight_mask.nelement()

    # Calculating number of parameters that will remain:
    left_params_num = int(total_param_num * args.sparse_lvl)

    model = nets[0]
    data_iter = iter(data_loader)
    imagesize = data_iter.next()[0].shape

    # n_x = 1
    n_eta = 5
    eta = 0.01

    mask_optim = torch.optim.Adam(mask_params, 0.1)
    num_optim = 10

    # We use Adam optimizer to optimize num_optim iterations
    for i in range(num_optim):
        # Zero out gradients for weight_mask and parameters
        model.zero_grad()

        # Taking expectation w.r.t eta
        for _ in range(n_eta):

            # Calculating Zen-scores with batchnorm set to train mode and not
            # initialize parameters with i.i.d Gaussian
            input = torch.empty(imagesize)
            nn.init.normal_(input)
            noise = torch.empty(imagesize)
            nn.init.normal_(noise)
            input = input.cuda()
            noise = noise.cuda()
            output = model(input)
            output_perturb = model(input+0.01*noise)
            zen_score = torch.norm(output-output_perturb)

            # We want to maximize Zen score with a constraint that torch.sigmoid(weight_mask)
            # is roughly remaining number of params
            current_param_num = 0
            for layer in candidate_layers:
                current_param_num += (torch.sigmoid(layer.weight_mask)).sum()
            loss = -zen_score + 0.1 * (current_param_num - left_params_num)**2
            loss.backward()

        # Update weight_mask
        mask_optim.step()

        if i % 5 ==0:
            print('Prune ' + str(i) + ' iterations.')
            print('Zen-score is {}'.format(zen_score.item()))

    print('[*] Final Zen-score (Continuous): {}'.format(zen_score))

    # First choose top total_param_num * args.sparse_lvl parameters
    all_weight_mask = {}
    for layer in candidate_layers:
        all_weight_mask[layer] = layer.weight_mask
    all_weight_mask_flattened = torch.cat([torch.flatten(a) for a in all_weight_mask.values()])
    normalizer = torch.sum(all_weight_mask_flattened)
    all_weight_mask_flattened /= normalizer

    weight_mask_topk, _ = torch.topk(all_weight_mask_flattened, left_params_num)
    threshold = weight_mask_topk[-1]

    # Modify weight mask of each layer
    with torch.no_grad():
        for layer in candidate_layers:
            temp_mask = (layer.weight_mask >= threshold).float()
            print('sparsity level: {}'.format(temp_mask.sum()/temp_mask.nelement()))
            # Unregister weight_mask from parameters and replace it with tensors
            delatrr(layer, 'weight_mask')
            layer.weight_mask = temp_mask
            # Modify forward method to mask forward
            snip.modify_mask_forward(layer, cont_relax = False)

    print('-'*20)

    for net in nets:
        net.zero_grad()
        net.train()

def apply_nsprune(args, nets, data_loader, num_classes, samples_per_class = 10, GAP=True):
    print('Using GAP is {}'.format(GAP))

    for net in nets:
        net.eval()
        # net.train()
        net.zero_grad()
        for layer in net.modules():
            snip.add_mask_ones(layer)

    model = nets[0]
    data_iter = iter(data_loader)
    imagesize = data_iter.next()[0].shape

    if args.iter_prune:
        num_iter = 100
    else:
        num_iter = 1

    n_x = 10
    n_eta = 10
    eta = 0.01
    for i in range(num_iter):
        # Taking expectaion w.r.t x
        model.zero_grad()
        for layer in model.modules():
            if isinstance(layer, (nn.Conv2d, nn.Linear)):
                layer.weight_mask.grad = None

        ns_tracking = 0
        for _ in range(n_x):
            try:
                (input, target) = snip.GraSP_fetch_data(data_iter, num_classes, samples_per_class)
            except:
                data_iter = iter(data_loader)
                (input, target) = snip.GraSP_fetch_data(data_iter, num_classes, samples_per_class)
            input = input.cuda()
            norm_x = torch.norm(input)
            # Taking expectation w.r.t  eta
            for _ in range(n_eta):

                # noise = torch.ones(input.size())*torch.randn(1,)*eta*norm_x
                noise = torch.randn(input.size())*eta*norm_x
                input_perturb = input + noise.cuda()
                if GAP:
                    output = model.GAP(input)
                    output_perturb = model.GAP(input_perturb)
                else:
                    output = model(input)
                    output_perturb = model(input_perturb)
                perturbation = torch.norm(output-output_perturb)/norm_x
                perturbation.backward()
                ns_tracking += perturbation.item()

        snip.net_iterative_prune_wolinear(model, args.sparse_lvl**((i+1)/num_iter))
        # snip.prune_net_decreaseloss(model, args.sparse_lvl**((i+1)/num_iter), True)
        if i % 10 ==0:
            print('Prune ' + str(i) + ' iterations, noise sensitivity:{}'.format(ns_tracking))

    for module in model.modules():
        if isinstance(module, (nn.Conv2d, nn.Linear, nn.ConvTranspose2d)):
            weight_check = module.weight
            print(((weight_check!=0).float().sum()/weight_check.numel()))

    print('-'*20)

    for net in nets:
        net.zero_grad()
        net.train()

def apply_SAP(args, nets, data_loader, criterion, num_classes, samples_per_class = 10):
    print('[*] Currently using SAP pruning.')
    for net in nets:
        # net.eval()
        net.train()
        net.zero_grad()
        for layer in net.modules():
            snip.add_mask_ones(layer)

    model = nets[0]
    data_iter = iter(data_loader)
    imagesize = data_iter.next()[0].shape

    if args.iter_prune:
        num_iter = 100
    else:
        num_iter = 1

    for layer in model.modules():
        if isinstance(layer,(nn.Conv2d, nn.Linear)):
            layer.base_weight = layer.weight.detach()

    n_x = 10
    n_eta = 10
    eta = 0.01

    for i in range(num_iter):
        # Taking expectaion w.r.t x
        model.zero_grad()
        for layer in model.modules():
            if isinstance(layer,(nn.Conv2d, nn.Linear)):
                layer.weight_mask.grad = None

        for _ in range(n_x):
            try:
                (input, target) = snip.GraSP_fetch_data(data_iter, num_classes, samples_per_class)
            except:
                data_iter = iter(data_loader)
                (input, target) = snip.GraSP_fetch_data(data_iter, num_classes, samples_per_class)
            target_var = target.cuda()
            input_var = input.cuda()
            # Taking expectation w.r.t  eta
            for _ in range(n_eta):
                with torch.no_grad():
                    for layer in model.modules():
                        if isinstance(layer,(nn.Conv2d, nn.Linear)):
                            layer.weight.data = layer.base_weight + eta*torch.randn(layer.weight.size(),device = layer.weight.device)
                # compute output
                output = model(input_var)
                loss = criterion(output, target_var)
                loss.backward()
            ##################################################
            # This part is for adversarial perturbations
            ##################################################
            # output = model(input_var)
            # loss = criterion(output, target_var)
            # loss.backward()

            # with torch.no_grad():
            #     for layer in model.modules():
            #         if isinstance(layer,(nn.Conv2d, nn.Linear)):
            #             layer.weight_mask.grad = None
            #             layer.weight.data = layer.base_weight + eta*layer.weight.grad/torch.norm(eta*layer.weight.grad)
            # # compute output
            # output = model(input_var)
            # loss = criterion(output, target_var)
            # loss.backward()

        with torch.no_grad():
            for layer in model.modules():
                if isinstance(layer,(nn.Conv2d, nn.Linear)):
                    layer.weight.data = layer.base_weight
        snip.net_iterative_prune(model, args.sparse_lvl**((i+1)/num_iter))
        # snip.prune_net_decreaseloss(model, args.sparse_lvl**((i+1)/num_iter), True)
        if i % 10 ==0:
            print('Prune ' + str(i) + ' iterations')

    for module in model.modules():
        if isinstance(module, (nn.Conv2d, nn.Linear, nn.ConvTranspose2d)):
            weight_check = module.weight
            print(((weight_check!=0).float().sum()/weight_check.numel()))

    print('-'*20)

    for net in nets:
        net.zero_grad()
        net.train()
# def apply_zenprune(args, nets, data_loader, num_classes, samples_per_class = 10):
#     for net in nets:
#         net.eval()
#         net.zero_grad()
#         for layer in net.modules():
#             snip.add_mask_ones(layer)

#     model = nets[0]
#     data_iter = iter(data_loader)
#     imagesize = data_iter.next()[0].shape

#     if args.iter_prune:
#         num_iter = 100
#     else:
#         num_iter = 1

#     for i in range(num_iter):
#         for _ in range(10):
#             # input = torch.empty(imagesize)
#             try:
#                 (input, target) = snip.GraSP_fetch_data(data_iter, num_classes, samples_per_class)
#             except:
#                 data_iter = iter(data_loader)
#                 (input, target) = snip.GraSP_fetch_data(data_iter, num_classes, samples_per_class)
#             # nn.init.normal_(input)
#             input = input.cuda()
#             output = model.GAP(input)
#             GAP_norm = torch.norm(output)
#             GAP_norm.backward()
#         print('The total GAP norm is:{}'.format(GAP_norm))
#         snip.net_iterative_prune_wolinear(model, args.sparse_lvl**((i+1)/num_iter))
#         if i % 10 ==0:
#             print('Prune ' + str(i) + ' iterations.')

#     for module in model.modules():
#         if isinstance(module, (nn.Conv2d, nn.Linear, nn.ConvTranspose2d)):
#             weight_check = module.weight
#             print(((weight_check!=0).float().sum()/weight_check.numel()))

#     print('-'*20)

#     # # remove hooks
#     # for handle in handles:
#     #     handle.remove()

#     # zero out gradients of weights
#     for net in nets:
#         net.zero_grad()
#         net.train()