text:
"""
Visualize detection results.
"""

import argparse
import cPickle as pickle
import json
import logging
from matplotlib import pyplot as plt
import numpy as np
import os
import pprint
import sys
import tensorflow as tf
import tensorflow.contrib.slim as slim
import time

from config import parse_config_file
from detect import input_nodes, filter_proposals, convert_proposals
import model

def detect_visualize(tfrecords, bbox_priors, checkpoint_path, cfg):

  logger = logging.getLogger()
  logger.setLevel(logging.DEBUG)

  graph = tf.Graph()

  # Force all Variables to reside on the CPU.
  with graph.as_default():

    batched_images, batched_offsets, batched_dims, batched_is_flipped, batched_bbox_restrictions, batched_max_to_keep, batched_heights_widths, batched_image_ids = input_nodes(
      tfrecords=tfrecords,
      num_epochs=1,
      batch_size=cfg.BATCH_SIZE,
      num_threads=cfg.NUM_INPUT_THREADS,
      capacity = cfg.QUEUE_CAPACITY,
      cfg=cfg
    )

    batch_norm_params = {
      # Decay for the batch_norm moving averages.
      'decay': cfg.BATCHNORM_MOVING_AVERAGE_DECAY,
      # epsilon to prevent 0s in variance.
      'epsilon': 0.001,
      'variables_collections' : [tf.GraphKeys.MOVING_AVERAGE_VARIABLES],
      'is_training' : False
    }
    with slim.arg_scope([slim.conv2d],
                        activation_fn=tf.nn.relu,
                        normalizer_fn=slim.batch_norm,
                        normalizer_params=batch_norm_params,
                        weights_regularizer=slim.l2_regularizer(0.00004),
                        biases_regularizer=slim.l2_regularizer(0.00004)):

      locations, confidences, inception_vars = model.build(
        inputs = batched_images,
        num_bboxes_per_cell = cfg.NUM_BBOXES_PER_CELL,
        reuse=False,
        scope=''
      )

    ema = tf.train.ExponentialMovingAverage(
      decay=cfg.MOVING_AVERAGE_DECAY
    )
    shadow_vars = {
      ema.average_name(var) : var
      for var in slim.get_model_variables()
    }

    # Restore the parameters
    saver = tf.train.Saver(shadow_vars, reshape=True)

    fetches = [locations, confidences, batched_offsets, batched_dims, batched_is_flipped, batched_bbox_restrictions, batched_max_to_keep, batched_heights_widths, batched_image_ids, batched_images]

    coord = tf.train.Coordinator()

    sess_config = tf.ConfigProto(
      log_device_placement=False,
      #device_filters = device_filters,
      allow_soft_placement = True,
      gpu_options = tf.GPUOptions(
          per_process_gpu_memory_fraction=cfg.SESSION_CONFIG.PER_PROCESS_GPU_MEMORY_FRACTION
      )
    )
    sess = tf.Session(graph=graph, config=sess_config)

    # Little utility to convert the float images to uint8
    image_to_convert = tf.placeholder(tf.float32)
    convert_image_to_uint8 = tf.image.convert_image_dtype(tf.add(tf.div(image_to_convert, 2.0), 0.5), tf.uint8)

    detection_results = []
    with sess.as_default():

      tf.initialize_all_variables().run()
      tf.initialize_local_variables().run()
      threads = tf.train.start_queue_runners(sess=sess, coord=coord)

      try:

        if tf.gfile.IsDirectory(checkpoint_path):
          checkpoint_path = tf.train.latest_checkpoint(checkpoint_path)

        if checkpoint_path is None:
          print "ERROR: No checkpoint file found."
          return

        # Restores from checkpoint
        saver.restore(sess, checkpoint_path)
        # Assuming model_checkpoint_path looks something like:
        #   /my-favorite-path/cifar10_train/model.ckpt-0,
        # extract global_step from it.
        global_step = int(checkpoint_path.split('/')[-1].split('-')[-1])
        print "Found model for global step: %d" % (global_step,)

        print_str = ', '.join([
          'Step: %d',
          'Time/image (ms): %.1f'
        ])

        plt.ion()
        original_image = None
        current_image_id = None

        step = 0
        done = False
        while not coord.should_stop() and not done:

          t = time.time()
          outputs = sess.run(fetches)
          dt = time.time()-t

          locs = outputs[0]
          confs = outputs[1]
          patch_offsets = outputs[2]
          patch_dims = outputs[3]
          patch_is_flipped = outputs[4]
          patch_bbox_restrictions = outputs[5]
          patch_max_to_keep = outputs[6]
          image_height_widths = outputs[7]
          image_ids = outputs[8]

          images = outputs[9]

          for b in range(cfg.BATCH_SIZE):

            print "Patch Dims: ", patch_dims[b]
            print "Patch Offset: ", patch_offsets[b]
            print "Max to keep: ", patch_max_to_keep[b]
            print "Patch restrictions: ", patch_bbox_restrictions[b]
            print "Image HxW: ", image_height_widths[b]
            print

            if current_image_id is None or current_image_id != image_ids[b]:
              original_image = images[b]
              original_image = sess.run(convert_image_to_uint8, {image_to_convert : images[b]})
              current_image_id = image_ids[b]

            img_id = int(np.asscalar(image_ids[b]))

            predicted_bboxes = locs[b] + bbox_priors
            predicted_bboxes = np.clip(predicted_bboxes, 0., 1.)
            predicted_confs = confs[b]

            # Keep only the predictions that are completely contained in the [0.1, 0.1, 0.9, 0.9] square
            # for this patch
            #if patch_restrict[b]:
            #  filtered_bboxes, filtered_confs = filter_proposals(predicted_bboxes, predicted_confs)
            #else:
            #  filtered_bboxes = predicted_bboxes
            #  filtered_confs = predicted_confs
            filtered_bboxes, filtered_confs = filter_proposals(predicted_bboxes, predicted_confs, patch_bbox_restrictions[b])

            # No valid predictions?
            if filtered_bboxes.shape[0] == 0:
              continue

            # Lets get rid of some of the predictions
            num_preds_to_keep = np.asscalar(patch_max_to_keep[b])
            sorted_idxs = np.argsort(filtered_confs.ravel())[::-1]
            sorted_idxs = sorted_idxs[:num_preds_to_keep]
            filtered_bboxes = filtered_bboxes[sorted_idxs]
            filtered_confs = filtered_confs[sorted_idxs]

            plt.figure('Cropped Size')
            uint8_image = sess.run(convert_image_to_uint8, {image_to_convert : images[b]})
            plt.imshow(uint8_image)
            num_detections_to_render = min(filtered_bboxes.shape[0], 10)
            for i in range(num_detections_to_render):

              loc = filtered_bboxes[i].ravel()
              conf = filtered_confs[i]

              #print "Location: ", loc
              #print "Conf: ", conf

              # Plot the predicted location in red
              xmin, ymin, xmax, ymax = loc * cfg.INPUT_SIZE
              plt.plot([xmin, xmax, xmax, xmin, xmin], [ymin, ymin, ymax, ymax, ymin], 'r-')

            # Convert the bounding boxes to the original image dimensions
            converted_bboxes = convert_proposals(
              bboxes = filtered_bboxes,
              offset = patch_offsets[b],
              patch_dims = patch_dims[b],
              image_dims = image_height_widths[b],
              is_flipped= patch_is_flipped[b]
            )

            plt.figure('Resized')
            plt.imshow(original_image)
            num_detections_to_render = min(converted_bboxes.shape[0], 10)
            for i in range(num_detections_to_render):

              loc = converted_bboxes[i].ravel()
              conf = filtered_confs[i]

              #print "Location: ", loc
              #print "Conf: ", conf

              # Plot the predicted location in red
              xmin, ymin, xmax, ymax = loc * cfg.INPUT_SIZE
              plt.plot([xmin, xmax, xmax, xmin, xmin], [ymin, ymin, ymax, ymax, ymin], 'r-')

            r = raw_input("press button: ")
            if r != "":
              done=True
              break

            plt.close('all')

          step += 1
          print print_str % (step, (dt / cfg.BATCH_SIZE) * 1000)

      except tf.errors.OutOfRangeError as e:
        pass

      coord.request_stop()
      coord.join(threads)

def parse_args():

    parser = argparse.ArgumentParser(description='Detect objects using a pretrained Multibox model')

    parser.add_argument('--tfrecords', dest='tfrecords',
                        help='paths to tfrecords files', type=str,
                        nargs='+', required=True)

    parser.add_argument('--priors', dest='priors',
                          help='path to the bounding box priors pickle file', type=str,
                          required=True)

    parser.add_argument('--checkpoint_path', dest='checkpoint_path',
                          help='Either a path to a specific model, or a path to a directory where checkpoint files are stored. If a directory, the latest model will be tested against.', type=str,
                          required=True, default=None)

    parser.add_argument('--config', dest='config_file',
                        help='Path to the configuration file',
                        required=True, type=str)

    args = parser.parse_args()

    return args

def main():
  args = parse_args()
  print "Command line arguments:"
  pprint.pprint(vars(args))
  print

  cfg = parse_config_file(args.config_file)
  print "Configurations:"
  pprint.pprint(cfg)
  print

  with open(args.priors) as f:
    bbox_priors = pickle.load(f)
  bbox_priors = np.array(bbox_priors).astype(np.float32)

  detect_visualize(
    tfrecords=args.tfrecords,
    bbox_priors=bbox_priors,
    checkpoint_path=args.checkpoint_path,
    cfg=cfg
  )

if __name__ == '__main__':
  main()


text:
# import
## batteries
import _pickle as pickle
import os
import sys
import csv
import gzip
import glob
import logging
from functools import partial
from collections import defaultdict
import multiprocessing as mp
## 3rd party
from keras import backend as K
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import IPython
## application

def nested_dict():
    return defaultdict(nested_dict)

def compute_mean_std(x_tr):
    """
    Given training data (list of contigs), compute mean and std
    feature-wise.
    """
    n_feat = x_tr[0].shape[1]
    feat_sum = np.zeros(n_feat)
    feat_sq_sum = np.zeros(n_feat)
    n_el = 0

    for xi in x_tr:
        sum_xi = np.sum(xi, 0)
        sum_sq = np.sum(xi ** 2, 0)
        feat_sum += sum_xi
        feat_sq_sum += sum_sq
        n_el += xi.shape[0]

    mean = feat_sum / n_el
    std = np.sqrt((feat_sq_sum / n_el - mean ** 2).clip(min=0))

    return mean, std

def splitall(path):
    """
    Fully split file path
    """
    path = os.path.abspath(path)
    allparts = []
    while 1:
        parts = os.path.split(path)
        if parts[0] == path:
            allparts.insert(0, parts[0])
            break
        elif parts[1] == path:
            allparts.insert(0, parts[1])
            break
        else:
            path = parts[0]
            allparts.insert(0, parts[1])
    return allparts

def get_row_val(row, row_num, col_idx, col):
    try:
        x = row[col_idx[col]]
    except (IndexError, KeyError):
        msg = 'ERROR: Cannot find "{}" value in Row{} of feature file table'
        sys.stderr.write(msg.format(col, row_num) + '\n'); sys.exit(1)
    return str(x)

def find_pkl_file(feat_file, force_overwrite=False):
    """ Finding pkl file """
    if force_overwrite is True:
        logging.info('  --force-overwrite=True; creating pkl from tsv file')
        return 'tsv'
    else:
        logging.info('  --force-overwrite=False; searching for pkl version of tsv file')

    pkl = os.path.splitext(feat_file)[0]
    if pkl.endswith('.tsv'):
        pkl = os.path.splitext(feat_file)[0]

    if os.path.isfile(pkl):
        logging.info('Found pkl file: {}'.format(pkl))
        msg = '  Using the existing pkl file. Set --force-overwrite=True'
        msg += ' to force-recreate the pkl file from the tsv file'
        logging.info(msg)
        return 'pkl'
    else:
        logging.info('  No pkl found. A pkl file will be created from the tsv file')
        return 'tsv'

def read_feature_file_table(feat_file_table, force_overwrite=False, technology='all-asmbl'):
    """ Loads feature file table, which lists all feature tables & associated
    metadata. The table is loaded based on column names
    Params:
      feat_file_table : str, file path of tsv table
      force_overwrite : bool, force create pkl files?
      technology : str, filter to just specified assembler(s)
    Returns:
      dict{file_type : {simulation_rep : {assembler : [feature_file, simulation_metadata] }}}
    """

    if feat_file_table.endswith('.gz'):
        _open = lambda x: gzip.open(x, 'rt')
    else:
        _open = lambda x: open(x, 'r')

    base_dir = os.path.split(feat_file_table)[0]
    D = nested_dict()
    with _open(feat_file_table) as f:
        # load
        tsv = csv.reader(f, delimiter='\t')
        col_names = next(tsv)
        # indexing
        colnames = ['rep', 'assembler', 'feature_file']
        colnames = {x:col_names.index(x) for x in colnames}

        # formatting rows
        for i,row in enumerate(tsv):
            rep = get_row_val(row, i + 2, colnames, 'rep')
            assembler = get_row_val(row, i + 2, colnames, 'assembler')
            if technology != 'all-asmbl' and assembler != technology:
                msg = 'Feature file table, Row{} => "{}" != --technology; Skipping'
                logging.info(msg.format(i+2, assembler))
            feature_file = get_row_val(row, i + 2, colnames, 'feature_file')
            if not os.path.isfile(feature_file):
                feature_file = os.path.join(base_dir, feature_file)
            if not os.path.isfile(feature_file):
                msg = 'Feature file table, Row{} => Cannot find file'
                msg += '; The file provided: {}'
                raise ValueError(msg.format(i + 2, feature_file))
            else:
                logging.info('Input file exists: {}'.format(feature_file))

            if feature_file.endswith('.tsv') or feature_file.endswith('.tsv.gz'):
                file_type = find_pkl_file(feature_file, force_overwrite)
            elif feature_file.endswith('.pkl'):
                file_type = 'pkl'
            else:
                msg = 'Feature file table, Row{} => file extension'
                msg += ' must be ".tsv", ".tsv.gz", or ".pkl"'
                msg += '; The file provided: {}'
                raise ValueError(msg.format(i + 2, feature_file))

            D[file_type][rep][assembler] = [feature_file]

    # summary
    sys.stderr.write('#-- Feature file table summary --#\n')
    n_asmbl = defaultdict(dict)
    for ft,v in D.items():
        for rep,v in v.items():
            for asmbl,v in v.items():
                try:
                    n_asmbl[ft][asmbl] += 1
                except KeyError:
                    n_asmbl[ft][asmbl] = 1
    msg = 'Assembler = {}; File type = {}; No. of files: {}\n'
    for ft,v in n_asmbl.items():
        for asmbl,v in v.items():
            sys.stderr.write(msg.format(asmbl, ft, v))
    sys.stderr.write('#--------------------------------#\n')

    return D

def pickle_in_parallel(feature_files, n_procs, set_target=True):
    """
    Pickling feature files using multiproessing.Pool.
    Params:
      feature_files : list of file paths
      n_procs : int, number of parallel processes
      set_target : bool, passed to pickle_data_b()
    Returns:
      list of files
    """
    if n_procs > 1:
        logging.info('Pickling in parallel with {} threads...'.format(n_procs))
    else:
        logging.info('Pickling...')
    pool = mp.Pool(processes = n_procs)
    # list of lists for input to pool.map
    x = []
    for rep,v in feature_files['tsv'].items():
        for asmbl,F in v.items():
            F = F[0]
            pklF = os.path.join(os.path.split(F)[0], 'features.pkl')
            x.append([F, pklF, rep, asmbl])
    # Pickle in parallel and saving file paths in dict
    func = partial(pickle_data_b, set_target=set_target)
    if n_procs > 1:
        ret = pool.map(func, x)
    else:
        ret = map(func, x)
    for y in ret:
        rep = y[2]
        asmbl = y[3]
        feature_files['pkl'][rep][asmbl] = y[1]
    return feature_files['pkl']

def load_features_tr(feat_file_table, max_len=10000,
                     technology = None, pickle_only=False,
                     force_overwrite=False, n_procs=1):
    """
    Loads features, pre-process them and returns training.
    Fuses data from both assemblers.

    Inputs:
        feat_file_path: path to the table that lists all feature files
        max_len: fixed length of contigs
        technology: assembler, megahit or metaspades.
        pickle_only: only perform pickling prior to testing. One time call.
        force_overwrite: bool, overwrite existing files
    Outputs:
        x, y: lists, where each element comes from one metagenome, and
          a dictionary with idx -> (metagenome, contig_name)
    """
    # reading in feature file table
    feat_files = read_feature_file_table(feat_file_table,
                                         force_overwrite=force_overwrite,
                                         technology=technology)
    # pickling feature tables (if needed)
    feat_files = pickle_in_parallel(feat_files, n_procs, set_target=True)

    # Pre-process once if not done already
    if pickle_only:
        logging.info('--pickle-only=True; exiting')
        exit(0)

    # for each metagenome simulation rep, combining features from each assembler together
    ## "tech" = assembler
    x, y, ye, yext, n2i = [], [], [], [], []
    for rep,v in feat_files.items():
        xtech, ytech = [], []
        for tech,filename in v.items():
            with open(filename, 'rb') as feat:
                xi, yi, n2ii = pickle.load(feat)
                xtech.append(xi)
                ytech.append(yi)

        x_in_contig, y_in_contig = [], []

        for xi, yi in zip(xtech, ytech):
            for j in range(len(xi)):
                len_contig = xi[j].shape[0]

                idx_chunk = 0
                while idx_chunk * max_len < len_contig:
                    chunked = xi[j][idx_chunk * max_len :
                                    (idx_chunk + 1) * max_len, :]

                    x_in_contig.append(chunked)
                    y_in_contig.append(yi[j])

                    idx_chunk += 1

        # Each element is a metagenome
        x.append(x_in_contig)
        yext.append(np.array(y_in_contig))

    # for binary classification
    y = yext

    return x, y

def load_features(feat_file_table, max_len=10000,
                  technology = 'megahit',
                  pickle_only = False,
                  force_overwrite = False,
                  n_procs = 1):
    """
    Loads features, pre-process them and returns validation data.

    Params:
      data_path: path to directory containing features.pkl
      max_len: fixed length of contigs
      technology: assembler, megahit or metaspades.
      pickle_only: only perform pickling prior to testing. One time call.
    Returns:
        x, y, i2n: lists, where each element comes from one metagenome, and
          a dictionary with idx -> (metagenome, contig_name)
    """

    # Finding feature files
    # reading in feature file table
    feat_files = read_feature_file_table(feat_file_table,
                                         force_overwrite=force_overwrite,
                                         technology=technology)
    # pickling feature tables (if needed)
    feat_files = pickle_in_parallel(feat_files, n_procs)
    if pickle_only:
        logging.info('--pickle-only=True; exiting')
        exit()

    # loading pickled feature matrices
    x, y, ye, yext, n2i = [], [], [], [], []
    shift = 0
    i2n_all = {}
    for rep,v in feat_files.items():
        for assembler,filename in v.items():
            with open(filename, 'rb') as feat:
                features = pickle.load(feat)

            xi, yi, n2ii = features
            i2ni = reverse_dict(n2ii)
            x_in_contig, y_in_contig = [], []

            n2i_keys = set([])
            for j in range(len(xi)):
                len_contig = xi[j].shape[0]

                idx_chunk = 0
                while idx_chunk * max_len < len_contig:
                    chunked = xi[j][idx_chunk * max_len :
                                    (idx_chunk + 1) * max_len, :]

                    x_in_contig.append(chunked)
                    y_in_contig.append(yi[j])

                    i2n_all[len(x_in_contig) - 1 + shift] = (int(rep), i2ni[j][0])
                    idx_chunk += 1
                    n2i_keys.add(i2ni[j][0])

            # Each element is a metagenome
            x.append(x_in_contig)
            yext.append(np.array(y_in_contig))

            #Sanity check
            assert(len(n2i_keys - set(n2ii.keys())) == 0)
            assert(len(set(n2ii.keys()) - n2i_keys) == 0)

            shift = len(i2n_all)

    # for binary classification
    y = yext

    return x, y, i2n_all

def load_features_nogt(feat_file_table, max_len=10000,
                       pickle_only=False,
                       force_overwrite=False,
                       n_procs=1):
    """
    Loads features for real datasets. Filters contigs with low coverage.
    WARNING: `coverage` column assumed to be second-from-last column.

    Params:
      feat_file_table: str, path to feature file table
      max_len: str, fixed length of contigs
    Returns:
      x, y, i2n: lists, where each element comes from one metagenome, and
          a dictionary with idx -> (metagenome, contig_name)
    """
    # reading in feature file table
    feat_files = read_feature_file_table(feat_file_table,
                                         force_overwrite=force_overwrite)
    # pickling feature tables (if needed)
    feat_files = pickle_in_parallel(feat_files, n_procs, set_target=False)
    if pickle_only:
        logging.info('--pickle-only=True; exiting')
        exit()

    # loading pickled feature tables
    x, y, ye, yext, n2i = [], [], [], [], []
    shift = 0
    i2n_all = {}
    idx_coverage = -2
    i = 0
    for rep,v in feat_files.items():
        for assembler,filename in v.items():
            # loading pickled features
            with open(filename, 'rb') as inF:
                logging.info('Loading file: {}'.format(filename))
                features = pickle.load(inF)

            # unpacking
            try:
                xi, n2ii = features
                yi = [-1 for i in range(len(xi))]
            except ValueError:
                xi, yi, n2ii = features

            # reverse dict
            i2ni = reverse_dict(n2ii)

            # contigs
            n_contigs_filtered = 0
            x_in_contig, y_in_contig = [], []
            n2i_keys = set([])
            for j in range(len(xi)):
                len_contig = xi[j].shape[0]

                #Filter low coverage
                if np.amin(xi[j][:, idx_coverage]) == 0:
                    n_contigs_filtered += 1
                    continue

                idx_chunk = 0
                while idx_chunk * max_len < len_contig:
                    chunked = xi[j][idx_chunk * max_len :
                                    (idx_chunk + 1) * max_len, :]

                    x_in_contig.append(chunked)
                    y_in_contig.append(yi[j])

                    i2n_all[len(x_in_contig) - 1 + shift] = (i, i2ni[j][0])
                    idx_chunk += 1
                    n2i_keys.add(i2ni[j][0])
            # status
            msg = 'Contigs filtered due to low coverage: {}'
            logging.info(msg.format(n_contigs_filtered))

            # Each element is a metagenome
            x.append(x_in_contig)
            yext.append(np.array(y_in_contig))
            # for next loop iteration
            shift = len(i2n_all)
            i += 1

    # for binary classification
    y = yext

    return x, y, i2n_all

def kfold(x, y, idx_lo, k=5):
    """Creating folds for k-fold validation
    Params:
      k : number of folds
    Returns:
      4 lists
    """
    # check data
    if len(x) < k:
        msg = 'Number of metagenomes is < n-folds: {} < {}'
        raise IOError(msg.format(len(x), k))

    # Define validation data
    x_tr, y_tr = [], []
    x_val, y_val = [], []

    # setting fold lower & upper
    meta_per_fold = int(len(x) / k)
    lower = idx_lo * meta_per_fold
    upper = (idx_lo + 1) * meta_per_fold

    # creating folds
    for i, xi in enumerate(x):
        if i < lower or i >= upper: # idx_lo:
            x_tr = x_tr + xi
            y_tr.append(y[i])
        else:
            x_val = x_val + xi
            y_val.append(y[i])

    y_tr = np.concatenate(y_tr)
    y_val = np.concatenate(y_val)

    return x_tr, x_val, y_tr, y_val

def pickle_data_b(x, set_target=True):
    """
    One time function parsing the csv file and dumping the
    values of interest into a pickle file.
    The file can be gzip'ed
    Params:
      x : list, first 2 elements are features_in & features_out
      set_target : bool, set the target (for train) or not (for predict)?
    Returns:
      features_out
    """
    features_in, features_out = x[:2]

    msg = 'Pickling feature data: {} => {}'
    logging.info(msg.format(features_in, features_out))

    feat_contig, target_contig = [], []
    name_to_id = {}

    # Dictionary for one-hot encoding
    letter_idx = defaultdict(int)
    # Idx of letter in feature vector
    idx_tmp = [('A',0) , ('C',1), ('T',2), ('G',3)]

    for k, v in idx_tmp:
        letter_idx[k] = v

    idx = 0
    #Read tsv and process features
    if features_in.endswith('.gz'):
        _open = lambda x: gzip.open(x, 'rt')
    else:
        _open = lambda x: open(x, 'r')
    with _open(features_in) as f:
        # load
        tsv = csv.reader(f, delimiter='\t')
        col_names = next(tsv)
        # indexing
        w_contig = col_names.index('contig')
        w_ext = col_names.index('Extensive_misassembly')
        w_ref = col_names.index('ref_base')
        w_nA = col_names.index('num_query_A')
        w_nC = col_names.index('num_query_C')
        w_nG = col_names.index('num_query_G')
        w_nT = col_names.index('num_query_T')
        w_var = col_names.index('num_SNPs')
        w_cov = col_names.index('coverage')  # WARNING: Predict assumes coverage in -2 position
        w_dis = col_names.index('num_discordant')
        w_features = [w_nA, w_nC, w_nG, w_nT, w_var, w_cov, w_dis]
        # formatting rows
        for row in tsv:
            name_contig = row[w_contig]

            # If name not in set, add previous contig and target to dataset
            if name_contig not in name_to_id:
                if idx != 0:
                    feat_contig.append(np.concatenate(feat, 0))
                    if set_target == True:
                        target_contig.append(float(tgt))
                feat = []

                #Set target (0 or 1; 1=misassembly)
                if set_target == True:
                    tgt = int(row[w_ext])
                # index
                name_to_id[name_contig] = idx
                idx += 1

            # Feature vec
            feat.append(np.array(4 * [0] + [int(row[ind]) for ind in w_features])[None, :].astype(np.uint8))
            feat[-1][0][letter_idx[row[w_ref]]] = 1

    # Append last
    feat_contig.append(np.concatenate(feat, 0))
    if set_target == True:
        target_contig.append(float(tgt))
    # Checking feature object
    assert(len(feat_contig) == len(name_to_id))

    # Save processed data into pickle file
    with open(features_out, 'wb') as f:
        logging.info('  Dumping pickle file')
        if set_target == True:
            pickle.dump([feat_contig, target_contig, name_to_id], f)
        else:
            pickle.dump([feat_contig, name_to_id], f)

    return x

def class_recall(label, name='metr'):
    """
    Custom metric for Keras, computes recall per class.

    Inputs:
        label: label wrt which recall is to be computed.
    """
    def __init__(self, name='recall', **kwargs):
        super(class_recall, self).__init__(name=name, **kwargs)

    def __call__(self, y_true, y_pred):
        class_id_preds = K.cast(K.greater(y_pred, 0.5), 'int32')
        y_true = K.cast(y_true, 'int32')
        accuracy_mask = K.cast(K.equal(y_true, label), 'int32')
        class_acc_tensor = K.cast(K.equal(y_true, class_id_preds), 'int32') * accuracy_mask
        class_acc = K.sum(class_acc_tensor) / K.maximum(K.sum(accuracy_mask), 1)
        return class_acc

def explained_var(y_true, y_pred):
    """
    Custom metric for Keras, explained variance.
    """
    return 1  - K.mean((y_true - y_pred) ** 2) / K.var(y_true)

def reverse_dict(d):
    """
    Flip keys and values
    """
    r_d = {}
    for k, v in d.items():
        if v not in r_d:
            r_d[v] = [k]
        else:
            r_d[v].append(k)
    return r_d

def compute_predictions(n2i, generator, model, save_path, save_name):
    """
    Computes predictions for a model and generator, aggregating scores for long contigs.

    Inputs:
        n2i: dictionary with contig_name -> list of idx corresponding to that contig.
        generator: deepmased data generator
    Output:
        saves scores for individual contigs
    """
    score_val = model.predict_generator(generator)

    # Compute predictions by aggregating scores for longer contigs
    score_val = score_val.flatten()
    scores = {}

    outfile = os.path.join(save_path, '_'.join([save_name, 'predictions.tsv']))
    write = open(outfile, 'w')
    csv_writer = csv.writer(write, delimiter='\t')
    csv_writer.writerow(['Collection', 'Contig', 'Deepmased_score'])

    for k in n2i:
        inf = n2i[k][0]
        sup = n2i[k][-1] + 1
        if k[0] not in scores:
            scores[k[0]] = {}

        # Make sure contig doesnt appear more than once
        assert(k[1] not in scores[k[0]])

        # Make sure we have predictions for these indices
        if sup > len(score_val):
            continue

        # Make sure all the labels for the contig coincide
        #scores[k[0]][k[1]] = {'pred' : score_val[inf : sup]}
        csv_writer.writerow([k[0], k[1], str(np.max(score_val[inf : sup]))])

    write.close()
    logging.info('File written: {}'.format(outfile))

def compute_predictions_y_known(y, n2i, model, dataGen):
    """
    Computes predictions for a model and generator, NOT aggregating scores for long contigs.

    Inputs:
        n2i: dictionary with contig_name -> list of idx corresponding to that contig.
    Output:
        scores:
        pred: scores for individual contigs
        y: corresponding true labels
    """

    score_val = model.predict_generator(dataGen)

    # Compute predictions by aggregating scores for longer contigs
    score_val = score_val.flatten()
    scores = {}
    for k in n2i:
        inf = n2i[k][0]
        sup = n2i[k][-1] + 1
        if k[0] not in scores:
            scores[k[0]] = {}

        # Make sure contig doesnt appear more than once
        assert(k[1] not in scores[k[0]])

        # Make sure we have predictions for these indices
        if sup > len(score_val):
            continue

        # Make sure all the labels for the contig coincide
        assert((y[inf : sup] == y[inf]).all())
        scores[k[0]][k[1]] = {'y' : int(y[inf]), 'pred' : score_val[inf : sup]}

    return scores

ext: py

max_stars_repo_name: leylabmpi/DeepMAsED

max_issues_repo_name: leylabmpi/DeepMAsED

max_forks_repo_name: leylabmpi/DeepMAsED

reason: import numpy


text:
from pearce.emulator import OriginalRecipe, ExtraCrispy, SpicyBuffalo
from pearce.mocks import cat_dict
from pearce.inference import run_mcmc_iterator
from scipy.optimize import minimize_scalar
import numpy as np
from os import path

#training_file = '/u/ki/swmclau2/des/xi_cosmo_trainer/PearceRedMagicXiCosmoFixedNd.hdf5'
training_file = '/scratch/users/swmclau2/xi_zheng07_cosmo_lowmsat/PearceRedMagicXiCosmoFixedNd.hdf5'

em_method = 'gp'
split_method = 'random'

load_fixed_params = {'z':0.0}#, 'HOD': 0}

np.random.seed(0)
emu = SpicyBuffalo(training_file, method = em_method, fixed_params=load_fixed_params, custom_mean_function = 'linear', downsample_factor = 0.1)

print 'Metric:', emu._emulators[0].get_parameter_vector()

#Remember if training data is an LHC can't load a fixed set, do that after
fixed_params = {}#'f_c':1.0}#,'logM1': 13.8 }# 'z':0.0}

cosmo_params = {'simname':'testbox', 'boxno': 0, 'realization':0, 'scale_factors':[1.0], 'system': 'sherlock'}
cat = cat_dict[cosmo_params['simname']](**cosmo_params)#construct the specified catalog!

cat.load(1.0, HOD='zheng07')

emulation_point = [('logM0', 14.0), ('sigma_logM', 0.2),
                    ('alpha', 1.083),('logM1', 13.7)]#, ('logMmin', 12.233)]

em_params = dict(emulation_point)
em_params.update(fixed_params)

def add_logMmin(hod_params, cat):
    """
    In the fixed number density case, find the logMmin value that will match the nd given hod_params
    :param: hod_params:
        The other parameters besides logMmin
    :param cat:
        the catalog in question
    :return:
        None. hod_params will have logMmin added to it.
    """
    hod_params['logMmin'] = 13.0 #initial guess
    #cat.populate(hod_params) #may be overkill, but will ensure params are written everywhere
    def func(logMmin, hod_params):
        hod_params.update({'logMmin':logMmin})
        return (cat.calc_analytic_nd(hod_params) - 1e-4)**2

    res = minimize_scalar(func, bounds = (12, 16), args = (hod_params,), options = {'maxiter':100}, method = 'Bounded')

    # assuming this doens't fail
    hod_params['logMmin'] = res.x

add_logMmin(em_params, cat)

r_bins = np.logspace(-1.1, 1.6, 19)
rpoints = emu.scale_bin_centers

xi_vals = []
for i in xrange(50):
    cat.populate(em_params)
    xi_vals.append(cat.calc_xi(r_bins))

# TODO need a way to get a measurement cov for the shams
xi_vals = np.log10(np.array(xi_vals))

cat.populate(em_params)
y10, cov10 = cat.calc_xi(r_bins, do_jackknife=True, jk_args = {'n_rands':10, 'n_sub':5})
#y10 = np.loadtxt('xi_gg_true_jk.npy')
#cov10 = np.loadtxt('xi_gg_cov_true_jk.npy')

y = np.log10(y10)
#y = np.mean(xi_vals, axis = 0)

shot_cov = np.cov(xi_vals, rowvar = False)

cov = np.log10(1+cov10/(np.outer(y10, y10))) # TODO check this is right?

np.savetxt('xi_gg_true_jk.npy', y)
#np.savetxt('xi_gg_cov_true_jk.npy', cov)

scov = np.loadtxt('xigg_scov_log.npy')

# TODOremoved jackknife
cov+=scov # add the emu covariance

cov+=shot_cov #

np.savetxt('xi_gg_cov_true_all.npy', cov)

#cov = np.diag(np.diag(cov))

#xi_vals = np.log10(np.array(xi_vals))
#y = np.loadtxt('xi_gg_true.npy') #xi_vals.mean(axis = 0) #take one example as our xi. could also use the mean, but lets not cheat.
#cov = np.loadtxt('xi_gg_cov_true.npy')#/np.sqrt(50)

# get cosmo params
del em_params['logMmin']
cpv = cat._get_cosmo_param_names_vals()

cosmo_param_dict = {key: val for key, val in zip(cpv[0], cpv[1])}

#em_params.update( cosmo_param_dict)

fixed_params.update(em_params)
#fixed_params.update(cosmo_param_dict)
em_params = cosmo_param_dict

param_names = [k for k in em_params.iterkeys() if k not in fixed_params]

nwalkers = 10
nsteps = 100
nburn = 0

savedir = '/scratch/users/swmclau2/PearceMCMC/'
#chain_fname = path.join(savedir, '%d_walkers_%d_steps_chain_cosmo_zheng_xi_lowmsat.npy'%(nwalkers, nsteps ))
chain_fname = path.join(savedir, '%d_walkers_%d_steps_chain_cosmo_zheng_xi_jk_meanv2.npy'%(nwalkers, nsteps))

with open(chain_fname, 'w') as f:
    f.write('#' + '\t'.join(param_names)+'\n')

print 'starting mcmc'
np.random.seed(0)
for pos in run_mcmc_iterator([emu], param_names, [y], [cov], rpoints, fixed_params = fixed_params,nwalkers = nwalkers,\
        nsteps = nsteps, nburn = nburn, ncores = 1):#, resume_from_previous = chain_fname):

        with open(chain_fname, 'a') as f:
            np.savetxt(f, pos)

reason: import numpy,from scipy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# -*- coding: cp936 -*-
import numpy as np
from freetype import *
import math
import os
import codecs
import copy
import chardet

def setup():
  if not os.path.exists('ttf'):
    os.makedirs('ttf')
  if not os.path.exists('font'):
    os.makedirs('font')
  if not os.path.exists('config.txt'):
    fp = open('config.txt', 'w+')
    s = "FontFileNamePre = default\n"
    s += "FontSizeList = [12,18,24,36,48,72]\n"
    fp.write(s)
    fp.close()
  if not os.path.exists('text.txt'):
    fp = open('text.txt', 'w+')
    s = "你好，世界！"
    fp.write(s)
    fp.close()
  if not os.path.getsize('text.txt'):
    fp = open('text.txt', 'w+')
    s = "你好，世界！"
    fp.write(s)
    fp.close()

def readText():
  if os.path.exists('text.txt'):
    fp = open('text.txt', 'rb')
    r = fp.read()
    fp.close();
    f_charInfo = chardet.detect(r)
    fp = open('text.txt', 'r', encoding=f_charInfo['encoding'])
    #print(f_charInfo)
    charList = fp.readlines()
    fp.close()
    #charList = ["nihao"]
    return charList

def getFontDict():
  if os.path.exists('config.txt'):
    fontDict ={'FontFileNamePre':'', 'FontSizeList':[], 'charList':[]}
    fp = open('config.txt', 'r')
    configList = fp.readlines()
    fp.close()
    for s in configList:
      s1 = s.replace(' ','').replace('\n','')
      index = s1.find('#')
      if index == 0:
        continue
      else:
        index = s1.find('=')
        if index != -1:
          key = s1[:index]
          val = s1[index+1:]
          index = val.find('#')
          if index != -1:
            val = val[:index]
          if key in fontDict:
            fontDict[key] = val
    fontDict['FontSizeList'] = eval(fontDict['FontSizeList'])
    if fontDict['FontFileNamePre'] == "default":
      fontDict['FontFileNamePre'] = ""
    if len(fontDict['FontSizeList']) == 0:
      fontDict['FontSizeList'] = [12,18,24,36,48,72]
    fontDict['charList'] = readText()
    return fontDict

def parseFont(dstfilename, filename, lis, size = 12, angle = 0):
  face = freetype.Face(filename)
  face.set_char_size(size*64)
  Font_height = face.size.height//72
  Font_totallen = 0

  angle = (angle/180.0)*math.pi
  matrix = FT_Matrix((int)( math.cos( angle ) * 0x10000),
                     (int)(-math.sin( angle ) * 0x10000),
                     (int)( math.sin( angle ) * 0x10000),
                     (int)( math.cos( angle ) * 0x10000))
  flag = FT_LOAD_RENDER
  pen = FT_Vector(0,0)
  FT_Set_Transform( face._FT_Face, byref(matrix), byref(pen))
  previous = 0
  xmin, xmax = 0, 0
  ymin, ymax = 0, 0
  L = []
  unicodeList = []
  start = '{'
  unicode = 0
  length = 0
  width = 0
  rows = 0
  xadvance = 0
  xoffset = 0
  yoffset = 0
  end = '},\n'
  maxAadvancex = 0
  TotalList = []
  dict1 = {"unicode":0, "length":0, "width":0, "rows":0, "xadvance":0, "xoffset":0, "top":0}
  glyphDitct = {"lengthH":0, 'lengthL':0, 'maxwidth':0, 'maxrows':0,'minXoffset':255, "minYoffset":255,'maxAadvancex':0}
  baseline = 0
  for s in lis:
    s = s.replace('\n','')
    for c in s:
      dict1["unicode"] = getUnicode(c)[0]#hex会将整数变为字符串
      if dict1["unicode"] in unicodeList:
        continue
      unicodeList.append(dict1["unicode"])
      face.load_char(c, flag)
      kerning = face.get_kerning(previous, c)
      previous = c
      dict1["xadvance"] = face.glyph.metrics.horiAdvance//64
      if glyphDitct['maxAadvancex'] < dict1["xadvance"]:
          glyphDitct['maxAadvancex'] = dict1["xadvance"]
      yadvance = face.glyph.metrics.vertAdvance//64
      if yadvance > Font_height:
          Font_height = yadvance
      bitmap1 = face.glyph.bitmap
      dict1["width"]  = face.glyph.bitmap.width
      dict1["rows"]   = face.glyph.bitmap.rows
      dict1["top"] = face.glyph.bitmap_top
      dict1["xoffset"] = face.glyph.bitmap_left
      if glyphDitct['minXoffset'] > dict1["xoffset"]:
          glyphDitct['minXoffset'] = dict1["xoffset"]
      if glyphDitct['maxwidth'] < dict1["width"]:
          glyphDitct['maxwidth'] = dict1["width"]
      if glyphDitct['maxrows'] < dict1["rows"]:
          glyphDitct['maxrows'] = dict1["rows"]
      if dict1["top"] > baseline:
          baseline = dict1["top"]

      bitmap = getBitmap(bitmap1.buffer)
      dict1["length"] = len(bitmap)#像素的数据个数
      if dict1["length"]:
          writeFileArray(dstfilename, bitmap)#将像素的数据写进去
      TotalList.append(copy.deepcopy(dict1))
      Font_totallen +=1
  glyphDitct['lengthH'] = Font_totallen >> 0xFFFF
  glyphDitct['lengthL'] = Font_totallen & 0xFFFF
  for d in TotalList:
      yoffset = baseline - d['top']
      if glyphDitct['minYoffset'] > yoffset:
          glyphDitct['minYoffset'] = yoffset
      ll = []
      ll.append(d['unicode'])
      ll.append(hex(d['length']))
      ll.append(hex(d['width']))
      ll.append(hex(d['rows']))
      ll.append(hex(d['xoffset']))
      ll.append(hex(yoffset))
      ll.append(hex(d['xadvance']))
      ls = start + listToString(ll) + end
      L.append(ls)
  ls = '{0}\n'
  L.append(ls)
  glyphLL = []
  glyphLL.append(hex(glyphDitct['lengthH']))
  glyphLL.append(hex(glyphDitct['lengthL']))
  glyphLL.append(hex(glyphDitct['maxwidth']))
  glyphLL.append(hex(glyphDitct['maxrows']))
  glyphLL.append(hex(glyphDitct['minXoffset']))
  glyphLL.append(hex(glyphDitct['minYoffset']))
  glyphLL.append(hex(glyphDitct['maxAadvancex']))
  ls = start + listToString(glyphLL) + end
  L.insert(0,ls)
  L.append(Font_height)
  L.append(Font_totallen)
  return L

def listToString(List,c = ''):#将列表转化为字符串,c表示是否加逗号
  s = str(List).replace('[',']').strip(']').replace("'",'') + c
  return s
def getUnicode(text):#return a List
  L = []
  cl = []
  for i in text:
      cl.append(ord(i))#ord
  for i in cl:
      L.append(hex(i))
  return L

def getBitmap(bitmap):
  L = []
  n = 0
  highByte = 0
  flag = True  #if flag = 0 indicate that highByte_start  flag = 1 indicate that highByte_end
  #flag = 2 indicate that lowByte_start  flag = 3 indicate that lowByte_end
  lowByte  = 0
  oneByte = 1
  halfByte = 0
  for i in range(len(bitmap)):
      n = n + 1
      if i + 1 == len(bitmap):#最后一个有效数据
          if bool(bitmap[i]):
              halfByte = 8
          else:
              halfByte = 0
          if flag:
              highByte = halfByte | n
              lowByte = 0
          else:
              lowByte = halfByte | n
          oneByte = (highByte << 4) | lowByte
          L.append(hex(oneByte))
          L.append(hex(0))
          return L
      if (bool(bitmap[i]) != bool(bitmap[i+1]) or n + 1 > 7):
          if bool(bitmap[i]):
              halfByte = 8
          else:
              halfByte = 0
          if flag:
              flag = False
              highByte = halfByte | n
              n = 0
          else:
              flag = True
              lowByte = halfByte | n
              oneByte = (highByte << 4) | lowByte
              L.append(hex(oneByte))
              n = 0
  return L
def loadList(List, num,remains, fix = 0):#将数据加载到list中，并返回列表最后一个字节数据的无效长度
  s = str(bin(num)).lstrip('0').strip('b')
  if fix != 0:
      lis = [0]*(fix-(s))
      s = listToString(lis).replace(',','').replace(' ','') + s
  tail = 0
  index = len(List) - 1
  while len(s):
      if len(s) > remains:
          tail = remains
      else:
          tail = len(s)
      value = int(s[:tail],2)
      if remains < BYTEBITS_MAX:
          List[index] = List[index] | (value << (remains -  tail))
      else:
          List.append((value << (remains - tail)))
      if remains - tail:
          remains -= tail
      else:
          remains = BYTEBITS_MAX
          index += 1
      s = s[tail:]

  return remains
def ModifySuffixName(filename, suffix):#修改文件后缀名
  portion = os.path.splitext(filename)
  if portion[1] != suffix:
      newname = portion[0]+suffix
      if os.path.exists(newname):
          return
      os.rename(filename, newname)

def writeFile(filename, text):
  fp = open(filename, 'a+')
  fp.write(text)
  fp.close()

def writeFileArray(filename, List):#每排10个数据
  #writeFile(filename, listToString(List[0:5], ',\n'))
  num = (len(List))//10;
  m = (len(List))%10
  for i in range(num):
      writeFile(filename, listToString(List[(i*10):(i*10)+10], ',\n'))
  if m:
      writeFile(filename, listToString(List[(num*10):], ',\n'))

if __name__ == '__main__':
  setup()
  pwdPath = os.getcwd()
  ttfSourcePath = pwdPath + '\\ttf'
  fontDestPath = pwdPath + '\\font'
  ttfFileList = os.listdir(ttfSourcePath)
  if len(ttfFileList) == 0:
    print("Failed! Please add truetype file with the extension .ttf in the ttf folder.")
    sys.exit()
  fontDict ={'FontFileNamePre':'', 'FontSizeList':[], 'charList':[]}
  fontDict = getFontDict()
  fontDict['ttfFileList'] = ttfFileList
  lis_1 = ["const uint8_t ", "Bitmaps[] PROGMEM = {\n"]
  lis_2 = ["const gdl_Glyph_t ","Glyphs[] PROGMEM = {\n"]
  lis_3 = ["const gdl_Font_t "," PROGMEM = {\n"]
  fpd = open('font.txt', 'w+')
  for ttf in fontDict['ttfFileList']:
    (filename,suffix) = os.path.splitext(ttf)
    if suffix.upper() != '.TTF':
      continue
    for size in fontDict['FontSizeList']:
      newfilename = fontDict['FontFileNamePre'] + filename + "Font"+ str(size) +'pt'
      filename_path = fontDestPath + '\\' +newfilename
      filename_h = filename_path +'.h'
      if os.path.exists(filename_h):
          os.remove(filename_h)
      filename_txt = filename_path +'.txt'
      fp = open(filename_txt, 'w+')
      text = lis_1[0] + newfilename + lis_1[1]
      fp.write(text)
      fp.close()
      src = ttfSourcePath + '\\' + ttf
      Glyph_list = parseFont(filename_txt, src, fontDict['charList'], size, 0)
      text = '};\n\n' + lis_2[0] + newfilename + lis_2[1]
      writeFile(filename_txt, text)
      for i in Glyph_list[:len(Glyph_list) - 2]:
          writeFile(filename_txt, i)
      text = '};\n\n' + lis_3[0] + newfilename + lis_3[1]
      text += "(uint8_t *)"+ newfilename +"Bitmaps,\n"
      text += "(gdl_Glyph_t *)" + newfilename + "Glyphs,\n" + '1, 0, '+ hex(Glyph_list[len(Glyph_list)-2])+',\n};\n\n'
      writeFile(filename_txt, text)
      ModifySuffixName(filename_txt, '.h')
      text = "#include \"Fonts/"+newfilename+'.h"\n'
      fpd.write(text)
  fpd.close()
  os.system(pwdPath+'\\font.txt')

max_stars_repo_name: zhongdatao/GDL


text:
#   Copyright 2019 California Institute of Technology
# ------------------------------------------------------------------

import numpy as np

def trim( input_image, output_dim ):

    input_dim = input_image.shape[1]

    if input_dim == output_dim:
        return input_image
    elif output_dim < input_dim:
        x1 = input_dim // 2 - output_dim // 2
        x2 = x1 + output_dim
        output_image = input_image[x1:x2,x1:x2].copy()
    else:
        output_image = np.zeros((output_dim,output_dim), dtype=input_image.dtype)
        x1 = output_dim // 2 - input_dim // 2
        x2 = x1 + input_dim
        output_image[x1:x2,x1:x2] = input_image

    return output_image

max_stars_repo_name: PhillipKP/proper-models

max_issues_repo_name: PhillipKP/proper-models

max_forks_repo_name: PhillipKP/proper-models


text:
"""Template for instantiating `Dilution` `UnitOperation`.

See Also
--------
:class:`bio_rtd.uo.fc_uo.Dilution`

"""

__version__ = '0.7.1'
__author__ = 'Jure Sencar'

import numpy as np

from bio_rtd.uo.fc_uo import Dilution

"""Direct instance creation."""

tmp_uo = Dilution(
    t=np.linspace(0, 10, 100),
    dilution_ratio=1.6,  # 60 % addition of dilution buffer
    uo_id="dilution_direct",
    gui_title="Dilution, direct instance"  # Optional.
)
# Optional. Dilution buffer composition.
tmp_uo.c_add_buffer = np.array([0, 0, 200])

"""Using dict of parameters and attributes.

Guide:

1. Define a time step and a simulation time vector.
2. Use `PARAMETERS` and `ATTRIBUTES` as a template.
   Replace variable types with values.
   Remove or comment out the ones that are not needed.
3. Instantiate the unit operation with parameters.
   Updated instance with attributes.

See the example below `PARAMETERS` and `ATTRIBUTES`.

"""

PARAMETERS = {
    # Required.
    "uo_id": str,
    # Required.
    # 1 = no dilution, 1.6 = 60 % addition of dilution buffer.
    "dilution_ratio": float,

    # Optional.
    "gui_title": str,  # default: Dilution
}

ATTRIBUTES = {
    # Optional. Composition of dilution buffer.
    # Default is empty array (equivalent to 0).
    "c_add_buffer": np.ndarray,

    # Additional attributes are inherited from `UnitOperation`.
    # See `examples/templates/add_on_attributes.py`.
    # Add them to the list if needed.
}

"""1. Define a time step and a simulation time vector."""
t = np.linspace(0, 100, 1001)  # it must start with 0
dt = t[1]  # time step

"""2. Use `PARAMETERS` and `ATTRIBUTES` as a template.

Copy/Paste templates.
Replace variable types with values.
Remove or comment out the ones that are not needed.

"""

uo_pars = {
    # Required.
    "uo_id": "dilution_1",
    # Required.
    # 1 = no dilution, 1.6 = 60 % addition of dilution buffer.
    "dilution_ratio": 1.6,

    # Optional.
    # "gui_title": str,  # default: Dilution
}

uo_attr = {
    # Optional. Composition of dilution buffer.
    # Default is empty array (equivalent to 0).
    "c_add_buffer": np.array([0, 0, 200]),

    # Additional attributes are inherited from `UnitOperation`.
    # See `examples/templates/add_on_attributes.py`.
    # Add them to the list if needed.
}

"""3. Instantiate unit operation and populate attributes."""

dilution = Dilution(t, **uo_pars)

for key, value in uo_attr.items():
    # Make sure attribute exist.
    assert hasattr(dilution, key), f"`{key}` is wrong."
    # Override value.
    setattr(dilution, key, value)

# Voila :)

max_stars_repo_name: open-biotech/bio-rtd

max_issues_repo_name: open-biotech/bio-rtd

max_forks_repo_name: open-biotech/bio-rtd


text:
# a bokeh histogram with an additional Kernel-Density-Estimation of the data distribution
from scipy.stats.kde import gaussian_kde

def plot_histogram(data, attribute):
    measured = data[attribute]
    hist, edges = np.histogram(measured, density=True, bins=25)

    x = np.linspace(np.min(measured), np.max(measured), 100)
    pdf = gaussian_kde(measured)

    fig = figure()
    fig.quad(top=hist, bottom=0, left=edges[:-1], right=edges[1:], fill_color='grey', line_color='grey')
    fig.line(x, pdf(x), line_color="black", line_width=2, alpha=0.7, line_dash='dashed')
    fig.xaxis.axis_label = attribute
    fig.yaxis.axis_label = 'P(%s)' %(attribute)
    show(fig)

plot_histogram(data, 'ATTRIBUTE')

max_stars_repo_name: fraunhofer-iais/IAIS-Python-Snippets

max_issues_repo_name: fraunhofer-iais/IAIS-Python-Snippets

max_forks_repo_name: fraunhofer-iais/IAIS-Python-Snippets

reason: from scipy


text:
import tensorflow as tf
import numpy as np
import cv2
from src.data_loader.data_generator import DataGenerator
from src.models.discriminator_model import DiscriminatorModel
from src.trainers.discriminator_trainer import DiscriminatorTrainer
from src.testers.discriminator_tester import DiscriminatorTester
from src.utils.config import processing_config
from src.utils.logger import  Logger
from src.utils.utils import get_args, freeze_graph

def main():
    try:
        args = get_args()
        # print(args.config)
        # config = processing_config("/media/syrix/programms/projects/GP/Intelligent_Frame_Skipping_Network/configs/config_model.json")
        config = processing_config(args.config)
    except:
        print("Missing or invalid arguments")
        exit(0)

    tf_config = tf.ConfigProto()
    tf_config.gpu_options.per_process_gpu_memory_fraction = config.per_process_gpu_memory_fraction

    sess = tf.Session()
    data = DataGenerator(config, training=True, testing=False)
    model = DiscriminatorModel(config)
    model.load(sess)
    logger = Logger(sess, config)

    trainer = DiscriminatorTrainer(sess, model, data, config, logger)
    trainer.train()
    # tester = DiscriminatorTester(sess, model, data, config, logger)
    # tester.test()
    #
    #
    freeze_graph(config.checkpoint_dir, "output")
if __name__ == '__main__':
    main()

  # "train_data_path":"/media/syrix/programms/projects/GP/SuperStreaming/benchmark/skipping/dataset/train/",
  # "val_data_path":"/media/syrix/programms/projects/GP/SuperStreaming/benchmark/skipping/dataset/val/",
  # "test_data_path":"/media/syrix/programms/projects/GP/SuperStreaming/benchmark/skipping/dataset/test/",
  # "summary_dir":"/media/syrix/programms/projects/GP/Intelligent_Frame_Skipping_Network/saved_models/summary/",
  # "checkpoint_dir":"/media/syrix/programms/projects/GP/Intelligent_Frame_Skipping_Network/saved_models/checkpoint/"


text:
import mmcv
import numpy as np
import os
import xml.etree.ElementTree as ET
import argparse
import tqdm
import time

output_dir = '/gdata2/zhuqi/work_dirs/tinaface/widerface/'
os.makedirs(output_dir, exist_ok=True)

pkl_path = '/gdata2/zhuqi/work_dirs/tinaface/widerface/test.pkl'
det_result = mmcv.load(pkl_path)  # 把pkl载入进来做对比

anno_path = '/gdata1/D //val.txt'  # 实际的val的gt值
with open(anno_path, 'r') as f:  # 把gt读进来
    anno_list = f.readlines()

print(anno_list)

def parse_each_annotation(param):
    start_time = time.time()
    anno_name = param[0].strip(); det = param[1]
    xml_folder_path = 'data/WIDERFace/WIDER_val/Annotations/'
    xml_path = xml_folder_path + anno_name + '.xml'
    tree = ET.parse(xml_path)
    root = tree.getroot()
    folder_name = root.find('folder').text
    os.makedirs(output_dir + folder_name, exist_ok=True)
    dets = det[0]
    origin_shape = dets.shape

    save_path = output_dir + folder_name + '/' + anno_name + '.txt'
    bbox_list = []
    for ii in range(dets.shape[0]):
        box = dets[ii]
        x = int(box[0])
        y = int(box[1])
        w = int(box[2]) - int(box[0])
        h = int(box[3]) - int(box[1])
        confidence = str(box[4])
        line = f'{x} {y} {w} {h} {confidence}\n'
        bbox_list.append(line)
    with open(save_path, 'w') as f:
        f.write(anno_name + '.jpg' + '\n')
        f.write(str(len(bbox_list)) + '\n')
        for each_det in bbox_list:
            f.write(each_det)
    end_time = time.time()
    print("Parsing ", origin_shape, " Using %g s" % (end_time - start_time))
    return None

def parse_annotation(anno_list, det_result):
    from multiprocessing import cpu_count
    from multiprocessing.pool import Pool
    pool = Pool(2 * cpu_count() // 3)
    _ = pool.map(parse_each_annotation, zip(anno_list, det_result))
    pool.close()
    # for (anno, det) in zip(anno_list, det_result):
    #     anno = anno.strip()
    #     parse_each_annotation(anno, det)

parse_annotation(anno_list, det_result)

max_stars_repo_name: zqcrafts/chanllenge

max_issues_repo_name: zqcrafts/chanllenge

max_forks_repo_name: zqcrafts/chanllenge


text:
import pandas as pd
import numpy as np
from sklearn.preprocessing import PolynomialFeatures
import matplotlib.pyplot as plt
from scipy.optimize import minimize

def costFunction_regularized_logistic_regression(theta_param, lambda_param, X, y):
    number_of_examples = X.shape[0]
    predicted_y = sigmoid(X.dot(theta_param))

    cost = -1 * (1 / number_of_examples) * (np.log(predicted_y).T.dot(y) + np.log(1 - predicted_y).T.dot(1 - y)) \
           + (lambda_param / (2 * number_of_examples)) * np.sum(np.square(theta_param[1:]))

    if np.isnan(cost):
        return np.inf
    return cost

def gradient_regularized_logistic_regression(theta_param, lambda_param, X, y):
    number_of_examples = X.shape[0]
    # -1 simply means unknown and we want numpy to figure out for us
    predicted_y = sigmoid(X.dot(theta_param.reshape(-1, 1)))

    gradient = (1 / number_of_examples) * X.T.dot(predicted_y - y) \
           + (lambda_param / number_of_examples) * \
               np.r_[[[0]], theta_param[1:].reshape(-1, 1)]

    return gradient.flatten()

def mapFeature(X, degree):
    polynomial_features = PolynomialFeatures(degree)

    return polynomial_features, polynomial_features.fit_transform(X)

def plotData(data, label_x, label_y, label_pos, label_neg, axes=None):
    # Get indexes for class 0 and class 1
    neg = data[:, 2] == 0
    pos = data[:, 2] == 1

    # If no specific axes object has been passed, get the current axes.
    if axes is None:
        axes = plt.gca()

    axes.scatter(data[pos][:, 0], data[pos][:, 1], marker='o', label=label_pos)
    axes.scatter(data[neg][:, 0], data[neg][:, 1], marker='x', label=label_neg)
    axes.set_xlabel(label_x)
    axes.set_ylabel(label_y)
    axes.legend(frameon=True, fancybox=True)

def sigmoid(z):
    return 1.0 / (1 + np.exp(-z))

dataframe = pd.read_csv('data/problem1data.txt', header=None)
y = dataframe[2].values
X = dataframe[[0, 1]].values

polynomial_features, X_New = mapFeature(X, 6)

initial_theta = np.zeros(X_New.shape[1])
print('Cost at the initial theta is ', costFunction_regularized_logistic_regression(initial_theta, 1, X_New, y))
print('Gradient at the initial theta is ', gradient_regularized_logistic_regression(initial_theta, 0, X_New, y))

fig, axes = plt.subplots(1, 3, sharey=True, figsize=(17, 5))

for i, lambda_param in enumerate([0, 1, 100]):
    optimal_param = minimize(costFunction_regularized_logistic_regression, initial_theta,
                             (lambda_param, X_New, y.reshape(-1, 1)), method=None,
                             jac=gradient_regularized_logistic_regression,
                             options={'maxiter': 3000})

    plotData(dataframe.values, 'Microchip Test 1', 'Microchip Test 2', 'y = 1', 'y = 0', axes.flatten()[i])
    plt.xlabel('Microchip Test 1')
    plt.ylabel('Microchip Test 1')
    plt.title('Plot of training data')
    plt.legend()
    plt.show()

    # Plot decisionboundary
    x1_min, x1_max = X[:, 0].min(), X[:, 0].max(),
    x2_min, x2_max = X[:, 1].min(), X[:, 1].max(),
    xx1, xx2 = np.meshgrid(np.linspace(x1_min, x1_max), np.linspace(x2_min, x2_max))

    predicted_y = sigmoid(polynomial_features.fit_transform(np.c_[xx1.ravel(), xx2.ravel()]).dot(optimal_param.x))
    predicted_y = predicted_y.reshape(xx1.shape)
    axes.flatten()[i].contour(xx1, xx2, predicted_y, [0.5], linewidths=1, colors='g');
    axes.flatten()[i].set_title('Lambda = {}'.format(lambda_param))

max_stars_repo_name: WhatTheFar/practical-ai-bootcamp

max_issues_repo_name: WhatTheFar/practical-ai-bootcamp

max_forks_repo_name: WhatTheFar/practical-ai-bootcamp


text:
import pylab
import tables
import math
import numpy

def exactSol(a, b, X):
    c0 = -(1/2.0 + a/12.0 + b/30.0)
    c1 = 0.0
    return X**2/2 + a*X**4/12 + b*X**6/30 + c0*X + c1

fh = tables.openFile("s89-poisson-o3-1d_phi.h5")
q = fh.root.StructGridField
nx, nc = q.shape
Xf = pylab.linspace(0, 1, nx)
qe = q[:,0]

dx = Xf[1]-Xf[0]
Xm = pylab.linspace(0.5*dx, 1-0.5*dx, nx-1)
qm = q[:-1,1]

a = 2.0
b = -12.0

Xhr = pylab.linspace(0, 1, 101)
fhr = exactSol(a, b, Xhr)

# make plot comparing exact to numerical solution
pylab.plot(Xhr, fhr, '-r', Xf, qe, 'ok', Xm, qm, 'ok')

# compute error
fex_e = exactSol(a, b, Xf)
fex_m = exactSol(a, b, Xm)
error = (numpy.abs(fex_e-qe).sum() + numpy.abs(fex_m-qm).sum())/(nx+nx-1);

print "%g %g" % (dx, error)

pylab.show()

max_stars_repo_name: ammarhakim/ammar-simjournal

max_stars_repo_licenses: MIT Unlicense

max_issues_repo_name: ammarhakim/ammar-simjournal

max_issues_repo_licenses: MIT Unlicense

max_forks_repo_name: ammarhakim/ammar-simjournal

max_forks_repo_licenses: MIT Unlicense


text:
from PIL import Image
import numpy as np
import matplotlib.pyplot as plt

import cv2
from skimage import metrics
import os

fake_dirs = ['results/psnr/fake_B_{}'.format(i) for i in range(5, 205, 5)]

images = [img.split('.')[0] for img in os.listdir(fake_dirs[0]) ]

res = []
for image in images:
    if not 'checkpoint' in image:
        res.append(image.split('_fake')[0])

gts = [ np.asarray(Image.open('datasets/sample/trainB/'+image+'.jpg').resize((256, 256))) for image in res]
images = [[ np.asarray(Image.open(fake+'/'+image+'_fake_B.png')) for fake in fake_dirs ] for image in res]

psnr = []
ssim = []
hist_r = []
hist_g = []
hist_b = []

for i, gt in enumerate(gts):
    actual_psnr = []
    actual_ssim = []
    actual_hist_r = []
    actual_hist_g = []
    actual_hist_b = []

    for j in range(len(images[i])):

        psnr_value = metrics.peak_signal_noise_ratio(gt, images[i][j])
        ssim_value = metrics.structural_similarity(gt, images[i][j], multichannel=True)

        color = ('b','g','r')
        gt_hst = {}
        img_hst = {}
        for channel,col in enumerate(color):
            gt_hst[col] = cv2.calcHist([gt],[channel],None,[256],[0,256])
            img_hst[col] = cv2.calcHist([images[i][j]],[channel],None,[256],[0,256])

        hist_value_b = cv2.compareHist(gt_hst['b'], img_hst['b'], cv2.HISTCMP_BHATTACHARYYA )
        hist_value_g = cv2.compareHist(gt_hst['g'], img_hst['g'], cv2.HISTCMP_BHATTACHARYYA )
        hist_value_r = cv2.compareHist(gt_hst['r'], img_hst['r'], cv2.HISTCMP_BHATTACHARYYA )

        actual_psnr.append(psnr_value)
        actual_ssim.append(ssim_value)
        actual_hist_r.append(hist_value_r)
        actual_hist_g.append(hist_value_g)
        actual_hist_b.append(hist_value_b)

    psnr.append(np.array(actual_psnr))
    ssim.append(np.array(actual_ssim))
    hist_r.append(np.array(actual_hist_r))
    hist_g.append(np.array(actual_hist_g))
    hist_b.append(np.array(actual_hist_b))

psnr = np.array(psnr)
ssim = np.array(ssim)
hist_r = np.array(hist_r)
hist_g = np.array(hist_g)
hist_b = np.array(hist_b)

avg_psnr = np.sum(psnr, axis=0)/len(images)
avg_ssim = np.sum(ssim, axis=0)/len(images)
avg_hist_r = np.sum(hist_r, axis=0)/len(images)
avg_hist_g = np.sum(hist_g, axis=0)/len(images)
avg_hist_b = np.sum(hist_b, axis=0)/len(images)

def avged(data, window_width):
    cumsum_vec = np.cumsum(np.insert(data, 0, 0))
    return (cumsum_vec[window_width:] - cumsum_vec[:-window_width]) / window_width

y = avged(avg_psnr, 3)
plt.plot([5*i for i in range(1,len(y) +1) ], y)
plt.title('Average PSNR on test images')
plt.xlabel('Epoch')
plt.ylabel('PSNR')
plt.savefig('psnr.png')

plt.cla()

y = avged(avg_ssim, 3)
plt.plot([5*i for i in range(1,len(y) +1) ], y)
plt.title('Average SSIM on test images')
plt.xlabel('Epoch')
plt.ylabel('SSIM')
plt.savefig('ssim.png')

plt.cla()

y = avged(avg_hist_r, 3)
plt.plot([5*i for i in range(1,len(y) +1) ], y, color='red')

y = avged(avg_hist_g, 3)
plt.plot([5*i for i in range(1,len(y) +1) ], y, color='green')

y = avged(avg_hist_b, 3)
plt.plot([5*i for i in range(1,len(y) +1) ], y, color='blue')

plt.title('Average Histogram difference on test images by channel')
plt.xlabel('Epoch')
plt.ylabel('Histogram Difference')
plt.savefig('hist.png')


text:
import numpy as np
import json
import pickle
import torch
from torch.utils.data import Dataset, DataLoader

def invert_dict(d):
    return {v: k for k, v in d.items()}

def load_vocab(path):
    with open(path, 'r') as f:
        vocab = json.load(f)
        vocab['question_idx_to_token'] = invert_dict(vocab['question_token_to_idx'])
        vocab['program_idx_to_token'] = invert_dict(vocab['program_token_to_idx'])
        vocab['answer_idx_to_token'] = invert_dict(vocab['answer_token_to_idx'])
    return vocab

def collate(batch):
    batch = list(zip(*batch))
    answer, question, program_seq, program_input = list(map(torch.stack, batch[:4]))
    return (answer, question, program_seq, program_input, *batch[4:])

class ClevrDataset(Dataset):

    def __init__(self, questions, image_indices, programs, program_inputs, answers, features, idx_cache, coord_cache):
        # convert data to tensor
        self.all_questions = torch.LongTensor(np.asarray(questions))
        self.all_image_idxs = torch.LongTensor(np.asarray(image_indices))
        self.all_programs = torch.LongTensor(np.asarray(programs))
        self.all_program_inputs = torch.LongTensor(np.asarray(program_inputs))
        self.all_answers = torch.LongTensor(np.asarray(answers))
        self.features = features
        self.idx_cache = idx_cache
        self.coord_cache = coord_cache

    def __getitem__(self, index):
        question = self.all_questions[index]
        image_idx = self.all_image_idxs[index].item()
        program_seq = self.all_programs[index]
        program_input = self.all_program_inputs[index]
        answer = self.all_answers[index]
        assert program_seq.size(0) == program_input.size(0), "program and program_input must have the same length"

        feature, coord = self.features[image_idx]['feature'], self.features[image_idx]['coord'][:,:2]
        feature = torch.FloatTensor(feature)
        num_obj = len(coord)
        edge_vector = np.zeros((num_obj, num_obj, 2))
        for i in range(num_obj):
            for j in range(num_obj):
                edge_vector[i,j] = coord[i] - coord[j]
        edge_vector = torch.FloatTensor(edge_vector)

        self.idx_cache[0] = image_idx
        self.coord_cache[0] = coord
        return (answer, question, program_seq, program_input, feature, edge_vector)

    def __len__(self):
        return len(self.all_questions)

class ClevrDataLoader(DataLoader):

    def __init__(self, **kwargs):
        if 'question_pt' not in kwargs:
            raise ValueError('Must give question_pt')
        if 'feature_pt' not in kwargs:
            raise ValueError('Must give feature_pt')
        if 'vocab_json' not in kwargs:
            raise ValueError('Must give vocab_json')

        feature_pt_path = str(kwargs.pop('feature_pt'))
        with open(feature_pt_path, 'rb') as f:
            features = pickle.load(f)

        vocab_json_path = str(kwargs.pop('vocab_json'))
        print('loading vocab from %s' % (vocab_json_path))
        vocab = load_vocab(vocab_json_path)

        question_pt_path = str(kwargs.pop('question_pt'))
        print('loading questions from %s' % (question_pt_path))
        with open(question_pt_path, 'rb') as f:
            obj = pickle.load(f)
            questions = obj['questions']
            image_indices = obj['image_idxs']
            programs = obj['programs']
            program_inputs = obj['program_inputs']
            answers = obj['answers']

        self.ratio = None
        if 'ratio' in kwargs:
            self.ratio = kwargs.pop('ratio')
            total = int(len(questions) * self.ratio)
            print('training ratio = %.3f, containing %d questions' % (self.ratio, total))
            questions = questions[:total]
            image_indices = image_indices[:total]
            programs = programs[:total]
            program_inputs = program_inputs[:total]
            answers = answers[:total]

        self.idx_cache, self.coord_cache = [0], [0] # just store some information for visualization
        dataset = ClevrDataset(questions, image_indices, programs, program_inputs, answers, features,
                               self.idx_cache, self.coord_cache)
        kwargs['collate_fn'] = collate
        super().__init__(dataset, **kwargs)
        self.vocab = vocab


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import logging

import torch
import torch.nn as nn
from collections import OrderedDict

BN_MOMENTUM = 0.1
logger = logging.getLogger(__name__)

class HeatmapMaxDetBlock(nn.Module):
    """
    Heatmap maximum detector block (for human pose estimation task).
    """
    def __init__(self):
        super(HeatmapMaxDetBlock, self).__init__()

    def forward(self, x):
        heatmap = x
        vector_dim = 2
        batch = heatmap.shape[0]
        channels = heatmap.shape[1]
        in_size = x.shape[2:]
        heatmap_vector = heatmap.view(batch, channels, -1)
        scores, indices = heatmap_vector.max(dim=vector_dim, keepdims=True)
        scores_mask = (scores > 0.0).float()
        pts_x = (indices % in_size[1]) * scores_mask
        pts_y = (indices // in_size[1]) * scores_mask
        pts = torch.cat((pts_x, pts_y, scores), dim=vector_dim)
        for b in range(batch):
            for k in range(channels):
                hm = heatmap[b, k, :, :]
                px = int(pts[b, k, 0])
                py = int(pts[b, k, 1])
                if (0 < px < in_size[1] - 1) and (0 < py < in_size[0] - 1):
                    pts[b, k, 0] += (hm[py, px + 1] - hm[py, px - 1]).sign() * 0.25
                    pts[b, k, 1] += (hm[py + 1, px] - hm[py - 1, px]).sign() * 0.25
        return pts

    @staticmethod
    def calc_flops(x):
        assert (x.shape[0] == 1)
        num_flops = x.numel() + 26 * x.shape[1]
        num_macs = 0
        return num_flops, num_macs

def conv3x3(in_planes, out_planes, stride=1):
    """3x3 convolution with padding"""
    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
                     padding=1, bias=False)

class BasicBlock(nn.Module):
    expansion = 1

    def __init__(self, inplanes, planes, stride=1, downsample=None):
        super(BasicBlock, self).__init__()
        self.conv1 = conv3x3(inplanes, planes, stride)
        self.bn1 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM)
        self.relu = nn.ReLU(inplace=True)
        self.conv2 = conv3x3(planes, planes)
        self.bn2 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        out += residual
        out = self.relu(out)

        return out

class Bottleneck(nn.Module):
    expansion = 4

    def __init__(self, inplanes, planes, stride=1, downsample=None):
        super(Bottleneck, self).__init__()
        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
                               padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM)
        self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1,
                               bias=False)
        self.bn3 = nn.BatchNorm2d(planes * self.expansion,
                                  momentum=BN_MOMENTUM)
        self.relu = nn.ReLU(inplace=True)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        out += residual
        out = self.relu(out)

        return out

class Bottleneck_CAFFE(nn.Module):
    expansion = 4

    def __init__(self, inplanes, planes, stride=1, downsample=None):
        super(Bottleneck_CAFFE, self).__init__()
        # add stride to conv1x1
        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, stride=stride, bias=False)
        self.bn1 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1,
                               padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM)
        self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1,
                               bias=False)
        self.bn3 = nn.BatchNorm2d(planes * self.expansion,
                                  momentum=BN_MOMENTUM)
        self.relu = nn.ReLU(inplace=True)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        out += residual
        out = self.relu(out)

        return out

class PoseResNet(nn.Module):

    def __init__(self, block, layers, **kwargs):
        extra_DECONV_WITH_BIAS = False
        extra_NUM_DECONV_LAYERS = 3
        extra_NUM_DECONV_FILTERS = [256, 256, 256]
        extra_NUM_DECONV_KERNELS = [4, 4, 4]
        extra_FINAL_CONV_KERNEL = 1
        MODEL_NUM_JOINTS = 17

        self.inplanes = 64
        self.deconv_with_bias = extra_DECONV_WITH_BIAS

        super(PoseResNet, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
                               bias=False)
        self.bn1 = nn.BatchNorm2d(64, momentum=BN_MOMENTUM)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.layer1 = self._make_layer(block, 64, layers[0])
        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)

        # used for deconv layers
        self.deconv_layers = self._make_deconv_layer(
            extra_NUM_DECONV_LAYERS,
            extra_NUM_DECONV_FILTERS,
            extra_NUM_DECONV_KERNELS,
        )

        self.final_layer = nn.Conv2d(
            in_channels=extra_NUM_DECONV_FILTERS[-1],
            out_channels=MODEL_NUM_JOINTS,
            kernel_size=extra_FINAL_CONV_KERNEL,
            stride=1,
            padding=1 if extra_FINAL_CONV_KERNEL == 3 else 0
        )

        self.heatmap_max_det = HeatmapMaxDetBlock()

    def _make_layer(self, block, planes, blocks, stride=1):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.inplanes, planes * block.expansion,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(planes * block.expansion, momentum=BN_MOMENTUM),
            )

        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes))

        return nn.Sequential(*layers)

    def _get_deconv_cfg(self, deconv_kernel, index):
        if deconv_kernel == 4:
            padding = 1
            output_padding = 0
        elif deconv_kernel == 3:
            padding = 1
            output_padding = 1
        elif deconv_kernel == 2:
            padding = 0
            output_padding = 0

        return deconv_kernel, padding, output_padding

    def _make_deconv_layer(self, num_layers, num_filters, num_kernels):
        assert num_layers == len(num_filters), \
            'ERROR: num_deconv_layers is different len(num_deconv_filters)'
        assert num_layers == len(num_kernels), \
            'ERROR: num_deconv_layers is different len(num_deconv_filters)'

        layers = []
        for i in range(num_layers):
            kernel, padding, output_padding = \
                self._get_deconv_cfg(num_kernels[i], i)

            planes = num_filters[i]
            layers.append(
                nn.ConvTranspose2d(
                    in_channels=self.inplanes,
                    out_channels=planes,
                    kernel_size=kernel,
                    stride=2,
                    padding=padding,
                    output_padding=output_padding,
                    bias=self.deconv_with_bias))
            layers.append(nn.BatchNorm2d(planes, momentum=BN_MOMENTUM))
            layers.append(nn.ReLU(inplace=True))
            self.inplanes = planes

        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x = self.maxpool(x)

        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)

        x = self.deconv_layers(x)
        x = self.final_layer(x)

        keypoints = self.heatmap_max_det(x)
        return keypoints

        # return x

    def init_weights(self, pretrained=''):
        if os.path.isfile(pretrained):
            logger.info('=> init deconv weights from normal distribution')
            for name, m in self.deconv_layers.named_modules():
                if isinstance(m, nn.ConvTranspose2d):
                    logger.info('=> init {}.weight as normal(0, 0.001)'.format(name))
                    logger.info('=> init {}.bias as 0'.format(name))
                    nn.init.normal_(m.weight, std=0.001)
                    if self.deconv_with_bias:
                        nn.init.constant_(m.bias, 0)
                elif isinstance(m, nn.BatchNorm2d):
                    logger.info('=> init {}.weight as 1'.format(name))
                    logger.info('=> init {}.bias as 0'.format(name))
                    nn.init.constant_(m.weight, 1)
                    nn.init.constant_(m.bias, 0)
            logger.info('=> init final conv weights from normal distribution')
            for m in self.final_layer.modules():
                if isinstance(m, nn.Conv2d):
                    # nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
                    logger.info('=> init {}.weight as normal(0, 0.001)'.format(name))
                    logger.info('=> init {}.bias as 0'.format(name))
                    nn.init.normal_(m.weight, std=0.001)
                    nn.init.constant_(m.bias, 0)

            # pretrained_state_dict = torch.load(pretrained)
            logger.info('=> loading pretrained model {}'.format(pretrained))
            # self.load_state_dict(pretrained_state_dict, strict=False)
            checkpoint = torch.load(pretrained)
            if isinstance(checkpoint, OrderedDict):
                state_dict = checkpoint
            elif isinstance(checkpoint, dict) and 'state_dict' in checkpoint:
                state_dict_old = checkpoint['state_dict']
                state_dict = OrderedDict()
                # delete 'module.' because it is saved from DataParallel module
                for key in state_dict_old.keys():
                    if key.startswith('module.'):
                        # state_dict[key[7:]] = state_dict[key]
                        # state_dict.pop(key)
                        state_dict[key[7:]] = state_dict_old[key]
                    else:
                        state_dict[key] = state_dict_old[key]
            else:
                raise RuntimeError(
                    'No state_dict found in checkpoint file {}'.format(pretrained))
            self.load_state_dict(state_dict, strict=False)
        else:
            logger.error('=> imagenet pretrained model dose not exist')
            logger.error('=> please download it first')
            raise ValueError('imagenet pretrained model does not exist')

resnet_spec = {18: (BasicBlock, [2, 2, 2, 2]),
               34: (BasicBlock, [3, 4, 6, 3]),
               50: (Bottleneck, [3, 4, 6, 3]),
               101: (Bottleneck, [3, 4, 23, 3]),
               152: (Bottleneck, [3, 8, 36, 3])}

def get_pose_net(num_layers, **kwargs):
    block_class, layers = resnet_spec[num_layers]

    model = PoseResNet(block_class, layers, **kwargs)

    return model

def oth_pose_coco_resnet_50_256x192(pretrained=False, **kwargs):
    model = get_pose_net(num_layers=50, **kwargs)
    return model

def oth_pose_coco_resnet_50_384x288(pretrained=False, **kwargs):
    model = get_pose_net(num_layers=50, **kwargs)
    return model

def oth_pose_coco_resnet_101_256x192(pretrained=False, **kwargs):
    model = get_pose_net(num_layers=101, **kwargs)
    return model

def oth_pose_coco_resnet_101_384x288(pretrained=False, **kwargs):
    model = get_pose_net(num_layers=101, **kwargs)
    return model

def oth_pose_coco_resnet_152_256x192(pretrained=False, **kwargs):
    model = get_pose_net(num_layers=152, **kwargs)
    return model

def oth_pose_coco_resnet_152_384x288(pretrained=False, **kwargs):
    model = get_pose_net(num_layers=152, **kwargs)
    return model

def _calc_width(net):
    import numpy as np
    net_params = filter(lambda p: p.requires_grad, net.parameters())
    weight_count = 0
    for param in net_params:
        weight_count += np.prod(param.size())
    return weight_count

def _test():
    import torch

    pretrained = False

    models = [
        oth_pose_coco_resnet_50_256x192,
        oth_pose_coco_resnet_50_384x288,
        oth_pose_coco_resnet_101_256x192,
        oth_pose_coco_resnet_101_384x288,
        oth_pose_coco_resnet_152_256x192,
        oth_pose_coco_resnet_152_384x288,
    ]

    for model in models:

        net = model(pretrained=pretrained)

        # net.train()
        net.eval()
        weight_count = _calc_width(net)
        print("m={}, {}".format(model.__name__, weight_count))
        assert (model != oth_pose_coco_resnet_50_256x192 or weight_count == 33999697)
        assert (model != oth_pose_coco_resnet_50_384x288 or weight_count == 33999697)

        x = torch.randn(14, 3, 1024, 2048)
        y = net(x)
        # y.sum().backward()
        assert (tuple(y.size()) == (14, 19, 1024, 2048))

if __name__ == "__main__":
    _test()

max_stars_repo_name: oliviaweng/imgclsmob


text:
from matplotlib.figure import Figure
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2Tk
import tkinter as tk
from selenium import webdriver
from axe_selenium_python import Axe

from scipy import stats
import time
import numpy as np
# from Naked.toolshed.shell import execute_js

def get_user_input():
    input_list = []
    print()
    print("# -------------------- Start URL Input -------------------- #")
    print('Enter a blank character if there are no more links')
    while True:
        input_url = input('Enter URL: ')
        if (input_url == ''):
            break
        if ('http' not in input_url):
            print('Please enter a valid URL')
            continue
        input_list.append(input_url)
    # print(input_list)
    print("# --------------------- End URL Input --------------------- #")
    print()
    return(input_list)

def make_autopct(values):
    def my_autopct(pct):
        total = sum(values)
        val = int(round(pct*total/100.0))
        if pct > 7:
            return '{p:.2f}%  ({v:d})'.format(p=pct, v=val)
        else:
            return ''
    return my_autopct

def get_all_links(list_of_urls):
    fullSet = set()
    invalid_links = ['twitter', 'instagram', 'facebook',
                     'youtube', 'areyouready', 'void(0)']
    for url in list_of_urls:
        fullSet.add(url)
        driver.get(url)
        url_list = driver.find_elements_by_tag_name("a")
        for link in url_list:
            fullLink = str(link.get_attribute("href"))
            # print(fullLink)
            if any(substring in fullLink for substring in invalid_links):
                break

            fullSet.add(fullLink)

    # fullSet.add('https://www.cpf.gov.sg/Members/Schemes')

    # ------- LocalHost Testing ------- #
    # fullSet.add('http://127.0.0.1:8000/about/')
    # fullSet.add('http://127.0.0.1:8000/contact/')
    # ------- LocalHost Testing ------- #

    return fullSet

def remove_invalid(whole_set):
    # Removing possible special cases
    # fix later
    if ("" in whole_set):
        whole_set.remove("")
    if ("None" in whole_set):
        whole_set.remove("None")
    if ("javascript:;" in whole_set):
        whole_set.remove("javascript:;")
    if ("https://www.gov.sg/" in whole_set):
        whole_set.remove("https://www.gov.sg/")
    if ("https://null/common/Lists/CPFPages/DispForm.aspx?ID=239" in whole_set):
        whole_set.remove(
            "https://null/common/Lists/CPFPages/DispForm.aspx?ID=239")
    if ("https://www.cpf.gov.sg/members" in whole_set):
        whole_set.remove("https://www.cpf.gov.sg/members")
    if ("https://www.cpf.gov.sg/members#" in whole_set):
        whole_set.remove("https://www.cpf.gov.sg/members#")
    if ("https://www.cpf.gov.sg/Members/Schemes#" in whole_set):
        whole_set.remove("https://www.cpf.gov.sg/Members/Schemes#")
    if ("https://icaeservices.ica.gov.sg/ipevp/web/evp/enquire-status-make-payment/status-enquiry" in whole_set):
        whole_set.remove(
            "https://icaeservices.ica.gov.sg/ipevp/web/evp/enquire-status-make-payment/status-enquirygit")
    if ("https://www.onemotoring.com.sg/content/onemotoring/home/digitalservices/buy-e-day-licence.html" in whole_set):
        whole_set.remove(
            "https://www.onemotoring.com.sg/content/onemotoring/home/digitalservices/buy-e-day-licence.html")
    if ("https://www.cpf.gov.sg/eSvc/Web/Miscellaneous/ContributionCalculator/Index?isFirstAndSecondYear=0&isMember=1" in whole_set):
        whole_set.remove(
            "https://www.cpf.gov.sg/eSvc/Web/Miscellaneous/ContributionCalculator/Index?isFirstAndSecondYear=0&isMember=1")
    return whole_set

def save_as_json(final_set, final_json):
    count_passes = 0
    count_incomplete = 0
    count_max = 0
    violations_array = []
    url_array = []

    # -------- Python Selenium -------- #
    for link in final_set:
        print(link)
        driver.get(link)

        axe = Axe(driver)

        # try options
        # full_options = { 'xpath : True }

        # Inject axe-core javascript into page.
        axe.inject()
        # Run axe accessibility checks.
        try:
            results = axe.run()
        except:
            break
            # driver.get(link)
            # axe=Axe(driver)
            # results=axe.run()

        if (results is None):
            break

        url = results['url']
    # -------- Python Selenium -------- #

     # TODO: Can use dict for violations and url array, using array now for simplicity/pyplot
        violations_array = np.append(
            violations_array, len(results['violations']))

        url_array = np.append(url_array, url)

        if (len(results['violations']) > count_max):
            count_max = len(results['violations'])
            max_url_name = url

        count_passes += len(results['passes'])
        count_incomplete += len(results['incomplete'])
        # print(len(results['incomplete']))

        # print(type(results))
        # print(results.get('violations').count("critical"))

        # print('violations: ', count_violations)
        # print('critical violations: ', count_critical)

        final_json[url] = results
        print("done")

        count_array = [count_incomplete, sum(violations_array), count_passes]

    print('Number of violations: ', sum(violations_array))
    return final_json, violations_array, url_array, max_url_name, count_array

def print_stats(count_array, violations_array, url_array, des_array, max_url_name, save_path):
    print(['No. of Web Pages', len(url_array)])
    print(['No. of Violations', str(int(sum(violations_array)))])
    print(['Most Common Violation', str(stats.mode(des_array)[0])])
    print(['No. of Passes', str(count_array[0])])
    print(['Most Violations', 'max_url_name'])
    print(['Time taken:', "%.1f" % time_taken + "s"])
    print(['Full log:', save_path])

def plot_visualisations(count_array, violations_array, url_array, des_array, max_url_name, save_path):
    root = tk.Tk()
    root.wm_title("title")

    fig = Figure(figsize = (10, 10), dpi = 100)
    labels = 'Incomplete', 'Violations', 'Passes'
    sizes = count_array
    explode = (0, 0.2, 0)

    ax1=fig.add_subplot(223)

    ax1.pie(sizes, explode = explode, labels = labels, autopct = make_autopct(sizes),
            textprops = {'fontsize': 10}, shadow = True, startangle = 90, radius = 1.5)

    ax3 = fig.add_subplot(211)
    table_vals = []

    table_vals.append(['No. of Web Pages', len(url_array)])
    table_vals.append(['No. of Violations', str(int(sum(violations_array)))])
    table_vals.append(['Most Common Violation', str(stats.mode(des_array)[0])])
    table_vals.append(['No. of Passes', str(count_array[2])])
    table_vals.append(['Most Violations', max_url_name])
    table_vals.append(['Time taken:', "%.1f" % time_taken + "s"])
    table_vals.append(['Full log:', save_path])

    print(['No. of Web Pages', len(url_array)])
    print(['No. of Violations', str(int(sum(violations_array)))])
    print(['Most Common Violation', str(stats.mode(des_array)[0])])
    print(['No. of Passes', str(count_array[0])])
    print(['Most Violations', max_url_name])
    print(['Time taken:', "%.1f" % time_taken + "s"])
    print(['Full log:', save_path])

    # Draw table
    the_table = ax3.table(cellText=table_vals,
                          colWidths=[0.09, 0.3],
                          rowLabels=None,
                          colLabels=None,
                          loc='center')
    the_table.auto_set_font_size(False)
    the_table.set_fontsize(10)
    the_table.scale(3, 3)

    ax3.tick_params(axis = 'x', which = 'both', bottom = False,
                    top=False, labelbottom=False)
    ax3.tick_params(axis='y', which='both', right=False,
                    left=False, labelleft=False)
    for pos in ['right', 'top', 'bottom', 'left']:
        ax3.spines[pos].set_visible(False)

    j = 1
    labels = []
    for l in url_arr:
        labels.append(j)
        j = j+1
    violations = violations_array

    ax2 = fig.add_subplot(224)

    ax2.bar(labels, violations, align='center', alpha=0.5, tick_label=labels)

    canvas = FigureCanvasTkAgg(fig, master=root)
    canvas.draw()
    canvas.get_tk_widget().pack(side=tk.TOP, fill=tk.BOTH, expand=1)

    toolbar = NavigationToolbar2Tk(canvas, root)
    toolbar.update()
    canvas.get_tk_widget().pack(side=tk.TOP, fill=tk.BOTH, expand=1)

    tk.mainloop()

start_time = time.time()
# Initialise driver

input_url_list = get_user_input()

# -------- For Chrome -------- #
driver = webdriver.Chrome()
driver.maximize_window()
# -------- For Chrome -------- #

# -------- Internet Explorer -------- #
# cap = DesiredCapabilities().INTERNETEXPLORER
# cap['ignoreZoomSetting'] = True
# driver = webdriver.Ie(capabilities=cap)
# -------- Internet Explorer -------- #

# main_url = "https://www.healthhub.sg/a-z"

# --------- SP Log In -------- #
main_url = "https://www.google.com"
# main_url = "https://saml.singpass.gov.sg/"
driver.get(main_url)
# --------- SP Log In -------- #

# -------- Add base URLs -------- #
# urls = {"https://www.cpf.gov.sg/members"}
# "https://www.mycareersfuture.sg/search/"}

axe = Axe(driver)

# Thread sleep
# time.sleep(50)

full_json = dict()

full_set = get_all_links(input_url_list)
# full_set = get_all_links(urls)

full_set = remove_invalid(full_set)

full_json, violations_arr, url_arr, max_url, count_arr = save_as_json(
    full_set, full_json)

json_save_path = './python-axe-selenium/data/demo_test.json'
axe.write_results(full_json, json_save_path)

des_arr = []
for items in full_json.values():
    # print(items['violations'])
    for item in items['violations']:
        des_arr.append(item['description'])

driver.close()
driver.quit()
time_taken = (time.time() - start_time)

plot_visualisations(count_arr, violations_arr, url_arr, des_arr,
                    max_url, json_save_path)

print_stats(count_arr, violations_arr, url_arr, des_arr,
            max_url, json_save_path)

print("Test Completed")

max_stars_repo_name: dalsontws/accessibility-axe-selenium

max_issues_repo_name: dalsontws/accessibility-axe-selenium

max_forks_repo_name: dalsontws/accessibility-axe-selenium