text:
""" Script to check image from produced tfrecord file by `bin/script_tfrecord.py`

Reference
- about batching --> https://stackoverflow.com/questions/44331612/how-to-set-a-number-for-epoch-in-tf-python-io-tf-record-iterator
- about buffer size of shuffle --> https://github.com/tensorflow/tensorflow/issues/14857
- pixel of each image is in range of [0 225]
"""

import os
import argparse
import json
import tensorflow as tf
from PIL import Image
import numpy as np
import deep_da

ROOT_DIR = os.getenv('ROOT_DIR', '.')
CONFIG = {
    "mnist": {
      "train": {
        "image": os.path.join(ROOT_DIR, "dataset/mnist/train-images-idx3-ubyte.gz"),
        "label": os.path.join(ROOT_DIR, "dataset/mnist/train-labels-idx1-ubyte.gz")
      },
      "valid": {
        "image": os.path.join(ROOT_DIR, "dataset/mnist/t10k-images-idx3-ubyte.gz"),
        "label": os.path.join(ROOT_DIR, "dataset/mnist/t10k-labels-idx1-ubyte.gz")
      }
    },
    "svhn": {
      "train": os.path.join(ROOT_DIR, "dataset/svhn/train_32x32.mat"),
      "valid": os.path.join(ROOT_DIR, "dataset/svhn/test_32x32.mat")
    }
}

OUTPUT = os.getenv('OUTPUT', './random/check_tfrecord')

if not os.path.exists(OUTPUT):
    os.makedirs(OUTPUT, exist_ok=True)

def get_options(parser):
    share_param = {'nargs': '?', 'action': 'store', 'const': None, 'choices': None, 'metavar': None}
    parser.add_argument('-n', '--num', help='number to show', default=5, type=int, **share_param)
    parser.add_argument('--data', help='dataset name in %s' % CONFIG.keys(), required=True, type=str,
                        **share_param)
    return parser.parse_args()

class TestTFRecord:

    def __init__(self,
                 path,
                 n_thread=1):

        self.__n_thread = n_thread
        self.__path_tfrecord = path
        self.__read_tf, self.__meta = deep_da.TFRecorder().read_tf(self.__path_tfrecord)
        self.__build_graph()
        self.session = tf.Session(config=tf.ConfigProto(log_device_placement=False))
        self.session.run(tf.global_variables_initializer())
        self.session.run(self.data_iterator, feed_dict={self.tfrecord_name: '%s/train.tfrecord' % path})

    def __build_graph(self):

        ############
        # TFRecord #
        ############

        # load tfrecord instance
        self.tfrecord_name = tf.placeholder(tf.string, name='tfrecord_dataset_name')
        data_set_api = tf.data.TFRecordDataset(self.tfrecord_name, compression_type='GZIP')
        # convert record to tensor
        data_set_api = data_set_api.map(self.__read_tf, self.__n_thread)
        # set batch size
        data_set_api = data_set_api.shuffle(buffer_size=10000, seed=0)
        data_set_api = data_set_api.batch(1)
        # make iterator
        iterator = tf.data.Iterator.from_structure(data_set_api.output_types, data_set_api.output_shapes)
        # get next input
        self.input_image, self.tag = iterator.get_next()

        # initialize iterator
        self.data_iterator = iterator.make_initializer(data_set_api)

    def get_data(self):
        img, tag = recorder.session.run([recorder.input_image, recorder.tag])
        img, tag = img[0], tag[0]
        tag = self.__meta['lookup_label_inv'][int(np.argmax(tag))]
        return img, int(tag)

if __name__ == '__main__':
    args = get_options(
        argparse.ArgumentParser(description='This script is ...', formatter_class=argparse.RawTextHelpFormatter))

    tfrecord_path = os.path.join(ROOT_DIR, 'tfrecord/%s' % args.data)

    recorder = TestTFRecord(tfrecord_path)

    for e in range(args.num):
        x, y = recorder.get_data()

        if args.data == 'svhn':
            x = np.rint(x).astype('uint8')
            x = Image.fromarray(x, 'RGB')
        elif args.data == 'mnist':
            x = np.rint(x[:, :, 0]).astype('uint8')
            x = Image.fromarray(x, 'L')
        else:
            raise ValueError('invalid shape')

        x.save('%s/%s-%i-n%s.png' % (OUTPUT, args.data, e, y))


text:
import pdb
import zmq
import sys
import time
import pickle
import numpy as np

port = "12345"
if len(sys.argv) > 1:
    port = sys.argv[1]
    int(port)

context = zmq.Context()
socket = context.socket(zmq.PUB)
socket.bind("tcp://*:%s" % port)

def publish_loop(initialized_socket, sleep_time_seconds):
    count = 0
    while True:
        topic = b"map"
        messagedata = np.random.uniform(0.0, 10.0, 100)
        # print("%d %d" % (topic, messagedata[0]))
        # print(messagedata)
        pickled_contents = pickle.dumps(messagedata)
        initialized_socket.send_multipart([topic, pickled_contents])
        time.sleep(sleep_time_seconds)
        count += 1
        if count % 1000==0:
            print(count)

try:
    publish_loop(socket, 0.0)
except KeyboardInterrupt:
    socket.unbind()
    socket.close()
finally:
    print("Socket closed?")
    print(str(socket.closed))

ext: py

max_stars_repo_name: bc/zmq-telemetry

max_issues_repo_name: bc/zmq-telemetry

max_forks_repo_name: bc/zmq-telemetry

reason: import numpy


text:
# Copyright (c) 2020 Patrick Hart, Julian Bernhard,
# Klemens Esterle, Tobias Kessler
#
# This software is released under the MIT License.
# https://opensource.org/licenses/MIT

import os
import math
import joblib
import unittest
import numpy as np
from gym.spaces.box import Box
from bark_ml.library_wrappers.lib_tf2rl.load_expert_trajectories import *
from bark_ml.library_wrappers.lib_tf2rl.load_save_utils import *
from bark.runtime.commons.parameters import ParameterServer
from bark_ml.observers.nearest_state_observer import NearestAgentsObserver

class LoadExpertTrajectoriesTest(unittest.TestCase):
  """
  Tests for the tf2rl utils.
  """

  def test_assert_file_exists(self):
    """
    Test: Assert that a non existent file raises an error.
    """
    with self.assertRaises(AssertionError):
      load_expert_trajectories('/tmp/')

  def test_file_contains_expert_trajectories(self):
    """
    Test: Assert that error is thrown if the file does not
            contain expert trajectories.
    """
    file_path = os.path.join(
        os.path.dirname(__file__),
        'data', 'invalid_file.jblb')
    with self.assertRaises(KeyError):
      load_expert_trajectories(os.path.dirname(file_path))

if __name__ == '__main__':
  unittest.main()

max_forks_repo_name: Brucknem/bark-ml


text:
import sys
import numpy as np

from tqdm import tqdm
from global_vars import *
from B_extract_bit.global_vars import *
from B_extract_bit.detect_preamble import detect_preamble

def process(file_name):
  try:
    postfix_list = ["train", "validation", "test"]

    for postfix in postfix_list:
      signal = np.load(signal_path + file_name + "_signal_" + postfix + ".npy")
      answer = np.load(answer_path + file_name + "_answer_" + postfix + ".npy")

      npy_signal1 = []
      npy_signal2 = []
      npy_signal3 = []
      npy_signal4 = []

      for idx in tqdm(range(len(signal)), desc=postfix.upper(), ncols=100, unit=" signal"):
        pre_start = detect_preamble(signal[idx])
        level = -1

        if postfix == "train":
          for x in range(n_bit_data):
            signal_start = pre_start + int(x*n_bit_static) - n_half_bit
            sample = np.array(signal[idx][signal_start:signal_start+int(2*n_bit)])

            if level == -1:
              if answer[idx][x] == 0:  # (H)LH(L)
                npy_signal1.append(sample)
              else:                     # (H)LL(H)
                npy_signal3.append(sample)
                level *= -1
            else:
              if answer[idx][x] == 0:  # (L)HL(H)
                npy_signal2.append(sample)
              else:                     # (L)HH(L)
                npy_signal4.append(sample)
                level *= -1

        else:
          result = []

          for x in range(n_bit_data):
            signal_start = pre_start + int(x*n_bit_static) - n_half_bit
            result.extend(signal[idx][signal_start:signal_start+int(2*n_bit)])

          npy_signal1.append(np.array(result))

      if postfix == "train":
        np.save(output_path + file_name + "_bit_" + postfix + "_1", npy_signal1)
        np.save(output_path + file_name + "_bit_" + postfix + "_2", npy_signal2)
        np.save(output_path + file_name + "_bit_" + postfix + "_3", npy_signal3)
        np.save(output_path + file_name + "_bit_" + postfix + "_4", npy_signal4)
      else:
        np.save(output_path + file_name + "_bit_" + postfix, npy_signal1)

  except Exception as ex:
    _, _, tb = sys.exc_info()
    print("[extract_bit:process:" + str(tb.tb_lineno) + "] " + str(ex) + "\n\n")


text:
import glob
import os
import sys

try:
    sys.path.append(glob.glob('../carla/dist/carla-*%d.%d-%s.egg' % (
        sys.version_info.major,
        sys.version_info.minor,
        'win-amd64' if os.name == 'nt' else 'linux-x86_64'))[0])
except IndexError:
    pass

import carla
from pathlib import Path
import numpy as np

from lane_tracking.cores.lane_detection.camera_geometry import CameraGeometry
from lane_tracking.cores.lane_detection.lane_detector import LaneDetector
from lane_tracking.util.carla_util import carla_vec_to_np_array, carla_img_to_array

def lane_track_init():
    cg = CameraGeometry()
    ld = LaneDetector(model_path=Path("lane_tracking/best_model_multi_dice_loss.pth").absolute())
    return cg, ld

def get_trajectory_from_lane_detector(ld, image):
    """
    polyfit excepts means junction
    param: ld = lane detector
    image: windshield rgb cam
    return: traj
    """
    image_arr = carla_img_to_array(image)
    try:
        poly_left, poly_right, debug = ld(image_arr)
        x = np.arange(-2,60,1.0)
        y = -0.5*(poly_left(x)+poly_right(x))
        x += 0.5
        traj = np.stack((x,y)).T
        warning = False
    except:
        warning = True
        traj = np.array([])
        debug = np.array([])
    return traj, debug, warning

def get_speed(vehicle):
    return np.linalg.norm(carla_vec_to_np_array(vehicle.get_velocity()))

def send_control(vehicle, throttle, steer, brake,
                 hand_brake=False, reverse=False):
    throttle = np.clip(throttle, 0.0, 1.0)
    steer = np.clip(steer, -1.0, 1.0)
    brake = np.clip(brake, 0.0, 1.0)
    control = carla.VehicleControl(throttle, steer, brake, hand_brake, reverse)
    vehicle.apply_control(control)

max_issues_repo_name: DevGlitch/delamain

max_forks_repo_name: DevGlitch/delamain


text:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

class Identify:
    def __init__(self, **kwargs):
        self.data = kwargs.get('data')
        self.modalities = kwargs.get('modalities')
        self.fused_modalities = kwargs.get('fused_modalities')
        self.experiment_id = kwargs.get('exp_id')
        self.k = kwargs.get('k')

        self.hits = pd.DataFrame(columns=self.modalities)

    def pad_labels(self, L):
        n = self.k
        if len(L) < n:
            L.extend([0] * (n - len(L)))
        return L

    def chop_and_sort(self):
        for mod in self.modalities+self.fused_modalities:

            ## Sort the gallerys scores returned for probe
            trimmed_sorted = self.data.groupby('Probe_ID').apply(
                lambda x: x.sort_values(mod, ascending=False)).reset_index(drop=True)

            ## drop extra scores (scores lower than rank k)
            trimmed_sorted = trimmed_sorted.groupby('Probe_ID').head(self.k)

            ## This adds 0 labels to transactions who have fewer than k scores
            df = trimmed_sorted.groupby('Probe_ID')['Label'].apply(list).apply(self.pad_labels)
            labels = pd.DataFrame.from_dict(dict(zip(df.index, df.values))).transpose()

            hit_rates = []
            for i in range(1, self.k + 1):
                hits = np.where(labels.iloc[:, :i].sum(axis=1) >= 1.0)
                rate = hits[0].shape[0] / len(labels.index)
                hit_rates.append(rate)

            self.hits[mod] = hit_rates

    def generate_plots(self):
        #### baseline plots
        for modality in self.modalities:
            hits = self.hits[modality]
            plt.plot([i for i in range(1, self.k + 1)], hits, label=modality, marker='o', linestyle='dotted')

        plt.legend()
        plt.xlabel('Rank')
        plt.ylabel('Identification Rate')
        plt.xticks([i for i in range(1, self.k + 1)])

        plt.title('CMC Curve')
        plt.savefig('./generated/experiments/CMC/' + self.experiment_id + '/CMC-baseline.png')
        plt.clf()

        #### fused plots
        for modality in self.fused_modalities:
            hits = self.hits[modality]
            plt.plot([i for i in range(1, self.k + 1)], hits, label=modality, marker='D', linestyle='dotted')

        plt.legend()
        plt.xlabel('Rank')
        plt.ylabel('Identification Rate')
        plt.xticks([i for i in range(1, self.k + 1)])

        plt.title('CMC Curve')
        plt.savefig('./generated/experiments/CMC/' + self.experiment_id + '/CMC-fused.png')
        plt.clf()

        ## Plot all
        for modality in self.modalities:
            hits = self.hits[modality]
            plt.plot([i for i in range(1, self.k + 1)], hits, label=modality, marker='o', linestyle='dotted')
        for modality in self.fused_modalities:
            hits = self.hits[modality]
            plt.plot([i for i in range(1, self.k + 1)], hits, label=modality, marker='D', linestyle='dotted')

        plt.legend()
        plt.xlabel('Rank')
        plt.ylabel('Identification Rate')
        plt.xticks([i for i in range(1, self.k + 1)])

        plt.title('CMC Curve')
        plt.savefig('./generated/experiments/CMC/' + self.experiment_id + '/CMC-all.png')

    def get_accuracies(self):
        return self.hits

    def get_cmc_summary(self):
        rows = self.modalities + self.fused_modalities
        cols = ['Rank'+ str(i) for i in range(1, 21)] + ['Experiment']
        res = pd.DataFrame(columns=cols, index=rows)
        for mod in self.modalities:
            for i in range(1,21):
                res.loc[mod]['Rank'+str(i)] = self.hits[mod][i-1]
            res.loc[mod]['Experiment'] = 'Baseline'

        for mod in self.fused_modalities:
            for i in range(1, 21):
                res.loc[mod]['Rank' + str(i)] = self.hits[mod][i - 1]
            res.loc[mod]['Experiment'] =self.experiment_id

        return res

max_stars_repo_name: melissadale/ScoreFusionApp

max_issues_repo_name: melissadale/ScoreFusionApp

max_forks_repo_name: melissadale/ScoreFusionApp


text:
"""Functions for importing and analyzing traffic traces"""
from __future__ import division

import math
import collections
import time
import dateutil
import types

import numpy as np
from scipy.stats import chisquare

from icarus.tools import TruncatedZipfDist

__all__ = [
       'frequencies',
       'one_timers',
       'trace_stats',
       'zipf_fit',
       'parse_url_list',
       'parse_wikibench',
       'parse_squid',
       'parse_youtube_umass',
       'parse_common_log_format'
           ]

def frequencies(data):
    """Extract frequencies from traces. Returns array of sorted frequencies

    Parameters
    ----------
    data : array-like
        An array of generic data (i.e. URLs of web pages)

    Returns
    -------
    frequencies : array of int
        The frequencies of the data sorted in descending order

    Notes
    -----
    This function does not return the mapping between data elements and their
    frequencies, it only returns frequencies.
    This function can be used to get frequencies to pass to the *zipf_fit*
    function given a set of data, e.g. content request traces.
    """
    return np.asarray(sorted(collections.Counter(data).values(), reverse=True))

def one_timers(data):
    """Return fraction of contents requested only once (i.e., one-timers)

    Parameters
    ----------
    data : array-like
        An array of generic data (i.e. URLs of web pages)

    Returns
    -------
    one_timers : float
        Fraction of content objects requested only once.
    """
    n_items = 0
    n_onetimers = 0
    counter = collections.Counter(data)
    for i in counter.itervalues():
        n_items += 1
        if i == 1:
            n_onetimers += 1
    return n_onetimers / n_items

def trace_stats(data):
    """Print full stats of a trace

    Parameters
    ----------
    data : array-like
        An array of generic data (i.e. URLs of web pages)

    Return
    ------
    stats : dict
        Metrics of the trace
    """
    if isinstance(data, types.GeneratorType):
        data = collections.deque(data)
    freqs = frequencies(data)
    alpha, p = zipf_fit(freqs)
    n_reqs = len(data)
    n_contents = len(freqs)
    n_onetimers = len(freqs[freqs == 1])
    return dict(n_contents=n_contents,
                n_reqs=n_reqs,
                n_onetimers=n_onetimers,
                alpha=alpha,
                p=p,
                onetimers_contents_ratio=n_onetimers / n_contents,
                onetimers_reqs_ratio=n_onetimers / n_reqs,
                mean_reqs_per_content=n_reqs / n_contents
                )

def zipf_fit(obs_freqs, need_sorting=False):
    """Returns the value of the Zipf's distribution alpha parameter that best
    fits the data provided and the p-value of the fit test.

    Parameters
    ----------
    obs_freqs : array
        The array of observed frequencies sorted in descending order
    need_sorting : bool, optional
        If True, indicates that obs_freqs is not sorted and this function will
        sort it. If False, assume that the array is already sorted

    Returns
    -------
    alpha : float
        The alpha parameter of the best Zipf fit
    p : float
        The p-value of the test

    Notes
    -----
    This function uses the method described in
    http://stats.stackexchange.com/questions/6780/how-to-calculate-zipfs-law-coefficient-from-a-set-of-top-frequencies
    """
    try:
        from scipy.optimize import minimize_scalar
    except ImportError:
        raise ImportError("Cannot import scipy.optimize minimize_scalar. "
                          "You either don't have scipy install or you have a "
                          "version too old (required 0.12 onwards)")
    obs_freqs = np.asarray(obs_freqs)
    if need_sorting:
        # Sort in descending order
        obs_freqs = -np.sort(-obs_freqs)
    n = len(obs_freqs)
    def log_likelihood(alpha):
        return np.sum(obs_freqs * (alpha * np.log(np.arange(1.0, n + 1)) + \
                       math.log(sum(1.0 / np.arange(1.0, n + 1) ** alpha))))
    # Find optimal alpha
    alpha = minimize_scalar(log_likelihood)['x']
    # Calculate goodness of fit
    if alpha <= 0:
        # Silently report a zero probability of a fit
        return alpha, 0
    exp_freqs = np.sum(obs_freqs) * TruncatedZipfDist(alpha, n).pdf
    p = chisquare(obs_freqs, exp_freqs)[1]
    return alpha, p

def parse_url_list(path):
    """Parse traces from a text file where each line contains a URL requested
    without timestamp or counters

    Parameters
    ----------
    path : str
        The path to the trace file to parse

    Returns
    -------
    trace : iterator of strings
        An iterator whereby each element is dictionary expressing all
        attributes of an entry of the trace
    """
    with open(path) as f:
        for line in f:
            yield line
    raise StopIteration()

def parse_wikibench(path):
    """Parses traces from the Wikibench dataset

    Parameters
    ----------
    path : str
        The path to the trace file to parse

    Returns
    -------
    trace : iterator of dicts
        An iterator whereby each element is dictionary expressing all
        attributes of an entry of the trace
    """
    with open(path) as f:
        for line in f:
            entry = line.split(" ")
            yield dict(
                counter=int(entry[0]),
                timestamp=entry[1],
                url=entry[2]
                      )
    raise StopIteration()

def parse_squid(path):
    """Parses traces from a Squid log file.
    Parse a Squid log file.

    Squid is an HTTP reverse proxy. Its logs contains traces of all HTTP
    requests served and can be used for trace-driven simulations based on
    realistic HTTP workloads.
    Traces from the IRCache dataset are in this format.

    Parameters
    ----------
    path : str
        The path to the trace file to parse

    Returns
    -------
    trace : iterator of dicts
        An iterator whereby each element is dictionary expressing all
        attributes of an entry of the trace

    Notes
    -----
    Documentation describing the Squid log format is available here:
    http://wiki.squid-cache.org/Features/LogFormat
    """
    with open(path) as f:
        for line in f:
            entry = line.split(" ")
            timestamp = entry[0]
            duration = int(entry[1])
            client_addr = entry[2]
            log_tag, http_code = entry[3].split("/")
            http_code = int(http_code)
            bytes_len = int(entry[4])
            req_method = entry[5]
            url = entry[6]
            client_ident = entry[7] if entry[7] != '-' else None
            hierarchy_data, hostname = entry[8].split("/")
            content_type = entry[9] if entry[9] != '-' else None
            yield dict(
                time=timestamp,
                duration=duration,
                client_addr=client_addr,
                log_tag=log_tag,
                http_code=http_code,
                bytes_len=bytes_len,
                req_method=req_method, url=url,
                client_ident=client_ident,
                hierarchy_data=hierarchy_data,
                hostname=hostname,
                content_type=content_type
                      )
    raise StopIteration()

def parse_youtube_umass(path):
    """Parse YouTube collected at UMass campus network [1]_.

    These data were collected at UMass campus network over a a measurement
    period between June 2007 and March 2008.

    This function parses the request traces, named youtube.parsed.X.Y.dat.
    Each entry of the trace provides the following information elements:
     * Timestamp
     * YouTube server IP (anonymized)
     * Client IP (anonymized)
     * Request
     * Video ID
     * Content server IP

    Traces are available at http://traces.cs.umass.edu/index.php/Network/Network

    Parameters
    ----------
    path : str
        The path to the trace file to parse

    Returns
    -------
    trace : iterator of dicts
        An iterator whereby each element is dictionary expressing all
        attributes of an entry of the trace

    References
    ----------
    ..[1] Michael Zink, Kyoungwon Suh, Yu Gu and Jim Kurose,
          Watch Global Cache Local: YouTube Network Traces at a Campus Network -
          Measurements and Implications, in Proc. of IEEE MMCN'08
    """
    with open(path) as f:
        for line in f:
            entry = line.split(" ")
            timestamp = entry[0]
            youtube_server_addr = int(entry[1])
            client_addr = entry[2]
            request = entry[3]
            video_id = entry[4]
            content_server_addr = entry[5]
            yield dict(
                time=timestamp,
                youtube_server_addr=youtube_server_addr,
                client_addr=client_addr,
                request=request,
                video_id=video_id,
                content_server_addr=content_server_addr,
                      )
    raise StopIteration()

def parse_common_log_format(path):
    """Parse files saved in the Common Log Format (CLF)

    Parameters
    ----------
    path : str
        The path to the Common Log Format file to parse

    Returns
    -------
    events : iterator
        iterator over the events parsed from the file

    Notes
    -----
    Common Log Format specifications:
    http://www.w3.org/Daemon/User/Config/Logging.html#common-logfile-format

    """
    with open(path) as f:
        for line in f:
            entry = line.split(" ")
            client_addr = entry[0]
            user_ident = entry[1]
            auth_user = entry[2]
            date = entry[3][1:-1]
            request = entry[4]
            status = int(entry[5])
            n_bytes = int(entry[6])
            # Convert timestamp into float
            t = time.mktime(dateutil.parser.parse(date.replace(":", " ", 0)).timetuple())
            event = dict(
                client_addr=client_addr,
                user_ident=user_ident,
                auth_user=auth_user,
                request=request,
                status=status,
                bytes=n_bytes
                        )
            yield t, event
    raise StopIteration()

reason: import numpy,import scipy,from scipy


text:
import numpy as np

from pymoo.performance_indicator.gd import GD
from pymoo.performance_indicator.igd import IGD
from pymoo.performance_indicator.hv import Hypervolume

def pareto_front_if_possible(problem):
    try:
        return problem.pareto_front()
    except:
        return None

class Output:

    def __init__(self, default_width=12) -> None:
        super().__init__()
        self.default_width = default_width
        self.attrs = []

    def append(self, name, number, format_if_float=True, width=None):
        if width is None:
            width = self.default_width
        if format_if_float and isinstance(number, float):
            number = self.format_float(number, width)

        self.attrs.append([name, number, width])

    def format_float(self, number, width):
        if number >= 10 or number * 1e5 < 1:
            return f"%.{width - 7}E" % number
        else:
            return f"%.{width - 3}f" % number

    def extend(self, *args):
        for arg in args:
            self.append(*arg)

    def clear(self):
        self.attrs = []

    def create_regex(self):
        return " | ".join(["{}"] * len(self.attrs))

    def do(self):
        if len(self.attrs) > 0:
            regex = self.create_regex()
            val = regex.format(*[str(val).rjust(width) for _, val, width in self.attrs])
            print(val)

    def header(self):
        regex = self.create_regex()
        s = regex.format(*[str(name).center(width) for name, _, width in self.attrs])
        print("=" * len(s))
        print(s)
        print("=" * len(s))

class Display:

    def __init__(self, output=None, attributes=None):
        super().__init__()
        self.output = output
        if self.output is None:
            self.output = Output()

        self.display_header = True
        self.pareto_front_is_available = None
        self.pf = None
        self.attributes = attributes

    def do(self, problem, evaluator, algorithm, pf=None):

        try:

            # if pf is false nothing happens, otherwise we try to get the pareto front from the problem
            if pf is None or (isinstance(pf, bool) and pf):

                # if we have not tried it before
                if self.pareto_front_is_available is None:
                    # see if you can get it and set the boolean for the future
                    self.pf = pareto_front_if_possible(problem)
                    self.pareto_front_is_available = self.pf is not None
            else:
                self.pf = pf
                self.pareto_front_is_available = True

            self.output.clear()

            # get the actual attributes to display
            self._do(problem, evaluator, algorithm)

            if self.attributes is not None:
                self.output.attrs = [attr for attr in self.output.attrs if attr[0] in self.attributes]

            if self.display_header:
                self.output.header()

            # print the actually line
            self.output.do()

        # catch any exception to make sure the algorithm does not fail because of printing
        except:
            print("WARNING: Error while preparing the output to be printed.")

        self.display_header = False

    def _do(self, problem, evaluator, algorithm):
        self.output.extend(*[('n_gen', algorithm.n_gen, False, 5), ('n_eval', evaluator.n_eval, False, 7)])

class SingleObjectiveDisplay(Display):

    def _do(self, problem, evaluator, algorithm):
        super()._do(problem, evaluator, algorithm)

        F, CV, feasible = algorithm.pop.get("F", "CV", "feasible")
        feasible = np.where(feasible[:, 0])[0]

        if problem.n_constr > 0:
            self.output.append("cv (min)", CV.min())
            self.output.append("cv (avg)", np.mean(CV))

        if len(feasible) > 0:
            _F = F[feasible]
            self.output.append("favg", np.mean(_F))
            self.output.append("fopt", np.min(_F))
        else:
            self.output.extend(*[('favg', "-"), ('fopt', "-")])

class MultiObjectiveDisplay(Display):

    def _do(self, problem, evaluator, algorithm):
        super()._do(problem, evaluator, algorithm)

        F, CV, feasible = algorithm.pop.get("F", "CV", "feasible")
        feasible = np.where(feasible[:, 0])[0]

        if problem.n_constr > 0:
            self.output.append("cv (min)", CV.min())
            self.output.append("cv (avg)", np.mean(CV))

        if len(feasible) > 0:
            if self.pareto_front_is_available:
                _F = F[feasible]
                self.output.append("igd", IGD(self.pf).calc(_F))
                self.output.append("gd", GD(self.pf).calc(_F))
                if problem.n_obj == 2:
                    self.output.append("hv", Hypervolume(pf=self.pf).calc(_F))
        else:
            if self.pareto_front_is_available:
                self.output.extend(*[('igd', "-"), ('gd', "-")])
                if problem.n_obj == 2:
                    self.output.append("hv", "-")


text:
import json
from collections import OrderedDict
import pandas as pd
from typing import Dict

import geopandas as gpd
import numpy as np
import rasterio
from rasterio import features

def main(no_data_val=-9999, fname_out='county_level_pop_out.tif', year=2017, grwth_rate_fct=0.0):
    '''
    '''
    # Get raster reference project from configuration file (ref_proj)
    with self.input()['ref_proj'].open('r') as fid:
        ref_proj = json.load(fid)

    # Read the shapefile with total population data
    shp_pop_obj = self.input()['geoshape_file']
    pop_data = gpd.read_file(shp_pop_obj.path)

    # -------------------------------------
    # Calculate Population from 2008 Census
    # data and growth rate
    # Apply growth rate adjustment
    # -------------------------------------
    nstep = self.year - 2008
    pop_data['GROWTHRATE'] = pop_data['GROWTHRATE'] * (1 + self.grwth_rate_fct)
    pop_data['Population'] = pop_data['SS2008'] * np.exp(pop_data['GROWTHRATE'] * nstep)
    pop_data['Population'] = pop_data['Population'].apply(lambda x: np.round(x))
    pop_data['Population'] = pop_data['Population'].astype('int32')

    # -----------------------------------------------------------------------
    # Rasterize Total population along with age and gender ratios
    # Total population and age & gender ratios should go into one raster file
    # -----------------------------------------------------------------------
    (ncols, nrows) = ref_proj['ncolsrows']
    rast_meta = {
        'driver': 'GTiff',
        'height': nrows,
        'width': ncols,
        'count': 1,
        'dtype': np.int32,
        'crs': ref_proj['srs'],
        'transform': ref_proj['pixel'],
        'nodata': -9999.0,
    }

    # Open raster file for writing
    with self.output().open("wb") as fout:
        with rasterio.open(fout.name, 'w', **rast_meta) as out_raster:

            out_array = out_raster.read(1)

            # Rasterize geopandas geometries with population values
            shapes = (
                (geom, pop_values)
                for geom, pop_values in zip(pop_data['geometry'], pop_data['Population'])
            )
            burned_data = features.rasterize(
                shapes=shapes, fill=0, out=out_array, transform=out_raster.transform
            )

            out_raster.write_band(1, burned_data)

            # Tag raster bands
            band_tags = {'band_1': 'Total_Population'}
            out_raster.update_tags(**band_tags)

class CreateCountyLevelCohorts(luigi.Task):

    ''' '''
    no_data_val = luigi.IntParameter(default=-9999)
    fname_out = luigi.Parameter(default='county_level_cohort_out.tif')
    year = luigi.Parameter(default=2017)

    def requires(self):
        return NotImplementedError(
            'Method must specify \n'
            '{geoshape_file: Shapefile with aggregated population data,\n'
            'age_gender_file: csv file with age and gender cohort population,\n'
            'ref_proj: Reference project for raster}'
        )

    def output(self):
        '''
        Write County level population along with age and gender ratios to raster file:

        band 1 -- 'Population', band 2 -- '0-4_Total', band 3 -- '5-9_Total',
        band 4 -- '10-14_Total', band 5 -- '15-19_Total', band 6 -- '20-24_Total',
        band 7 -- '25-29_Total', band 8 -- '30-34_Total', band 9 -- '35-39_Total',
        band 10 -- '40-44_Total', band 11 -- '45-49_Total', band 12 -- '50-54_Total',
        band 13 -- '55-59_Total', band 14 -- '60-64_Total', band 15 -- '65-69_Total',
        band 16 -- '70-74_Total', band 17 -- '75-79_Total', band 18 -- '80-84_Total',
        band 19 -- '85-89_Total', band 20 -- '90-94_Total', band 21 -- '95-99_Total',
        band 22 -- '100+_Total', band 23 -- 'Total_Female', band 24 -- '0-4_Female',
        band 25 -- '5-9_Female', band 26 -- '10-14_Female', band 27 -- '15-19_Female',
        band 28 -- '20-24_Female', band 29 -- '25-29_Female', band 30 --'30-34_Female',
        band 31 -- '35-39_Female', band 32 -- '40-44_Female', band 33 -- '45-49_Female',
        band 34 -- '50-54_Female', band 35 -- '55-59_Female', band 36 -- '60-64_Female',
        band 37 -- '65-69_Female', band 38 -- '70-74_Female', band 39 -- '75-79_Female',
        band 40 -- '80-84_Female', band 41 -- '85-89_Female', band 42 -- '90-94_Female',
        band 43 -- '95-99_Female', band 44 -- '100+_Female', band 45 -- 'Total_Male',
        band 46 -- '0-4_Male', band 47 -- '5-9_Male', band 48 --'10-14_Male',
        band 49 -- '15-19_Male', band 50 -- '20-24_Male', band 51 --'25-29_Male',
        band 52 -- '30-34_Male', band 53 -- '35-39_Male', band 54 --'40-44_Male',
        band 55 -- '45-49_Male', band 56 -- '50-54_Male', band 57 -- '55-59_Male',
        band 58 -- '60-64_Male', band 59 -- '65-69_Male', band 60 -- '70-74_Male',
        band 61 -- '75-79_Male', band 62 -- '80-84_Male', band 63 -- '85-89_Male',
        band 64 -- '90-94_Male', band 65 -- '95-99_Male', band 66 -- '100+_Male'
        '''
        return LocalTarget(path=self.fname_out, format=luigi.format.Nop)

    def run(self):
        '''
        '''
        # Get raster reference project from configuration file (ref_proj)
        with self.input()['ref_proj'].open('r') as fid:
            ref_proj = json.load(fid)

        # Read the shapefile with total population data
        shp_pop_obj = self.input()['geoshape_file']
        pop_data = gpd.read_file(shp_pop_obj.path)

        # Read the csv file with age and gender cohort populations
        cohort_pop_obj = self.input()['age_gender_file']
        age_gender_data = pd.read_csv(cohort_pop_obj.path)
        age_gender_data.drop('Year', inplace=True, axis=1)

        # Normalize age and gender breakdown (Create fractions) according to total population
        total_pop = age_gender_data['Total'].iloc[0]
        age_gender_data['Total'] = age_gender_data['Total']/total_pop
        age_gender_data['Male'] = age_gender_data['Male']/total_pop
        age_gender_data['Female'] = age_gender_data['Female']/total_pop

        # Create age and gender maps with the same areas defined in total population
        for row in age_gender_data.itertuples():
            pop_data[row.Age+'_Total'] = np.ones(pop_data.shape[0]) * row.Total
            pop_data[row.Age+'_Male'] = np.ones(pop_data.shape[0]) * row.Male
            pop_data[row.Age+'_Female'] = np.ones(pop_data.shape[0]) * row.Female

        # Rasterize Total population along with age and gender ratios
        # Total population and age & gender ratios should go into one raster file
        (ncols, nrows) = ref_proj['ncolsrows']
        rast_meta = {
            'driver': 'GTiff',
            'height': nrows,
            'width': ncols,
            'count': 65,
            'dtype': np.float32,
            'crs': ref_proj['srs'],
            'transform': ref_proj['pixel'],
            'nodata': -9999.0,
        }
        # Tags for raster bands
        clmn_names = [
           '0-4_Total', '5-9_Total', '10-14_Total', '15-19_Total',
           '20-24_Total', '25-29_Total', '30-34_Total', '35-39_Total',
           '40-44_Total', '45-49_Total', '50-54_Total', '55-59_Total',
           '60-64_Total', '65-69_Total', '70-74_Total', '75-79_Total',
           '80-84_Total', '85-89_Total', '90-94_Total', '95-99_Total',
           '100+_Total', 'Total_Female', '0-4_Female', '5-9_Female', '10-14_Female',
           '15-19_Female', '20-24_Female', '25-29_Female', '30-34_Female',
           '35-39_Female', '40-44_Female', '45-49_Female', '50-54_Female',
           '55-59_Female', '60-64_Female', '65-69_Female', '70-74_Female',
           '75-79_Female', '80-84_Female', '85-89_Female', '90-94_Female',
           '95-99_Female', '100+_Female', 'Total_Male', '0-4_Male', '5-9_Male', '10-14_Male',
           '15-19_Male', '20-24_Male', '25-29_Male', '30-34_Male', '35-39_Male', '40-44_Male',
           '45-49_Male', '50-54_Male', '55-59_Male', '60-64_Male', '65-69_Male',
           '70-74_Male', '75-79_Male', '80-84_Male', '85-89_Male', '90-94_Male',
           '95-99_Male', '100+_Male'
        ]

        # Open raster file for writing
        with self.output().open("wb") as fout:
            with rasterio.open(fout.name, 'w', **rast_meta) as out_raster:

                out_array = out_raster.read(1)

                # Loop through population and age and gender ratios
                for i, clmn in enumerate(clmn_names):

                    # Rasterize geopandas geometries with population values
                    shapes = (
                        (geom, pop_values)
                        for geom, pop_values in zip(pop_data['geometry'], pop_data[clmn])
                    )
                    burned_data = features.rasterize(
                        shapes=shapes, fill=0, out=out_array, transform=out_raster.transform
                    )

                    out_raster.write_band(i+1, burned_data)

                # Tag raster bands
                band_tags = OrderedDict({'band_'+str(i+1): val for i, val in enumerate(clmn_names)})
                out_raster.update_tags(**band_tags)

max_stars_repo_name: fengggli/pegasus

max_issues_repo_name: fengggli/pegasus

max_forks_repo_name: fengggli/pegasus


text:
import argparse
import os
import subprocess

import numpy as np
from jinja2 import Template

import datasets as ds
import utils
from vgg16 import vgg16_multilabel_hdf5
from utils import flip_labels as flip_label_matrix

EXP_DIR = os.path.join('data', 'experiments', 'espgame')
DS_DIR = os.path.join('data', 'ESP-Game')
AUX_DIR = os.path.join(EXP_DIR, 'aux')
PROTOTXT_NET = os.path.join(AUX_DIR, 'vgg16_multilabel_00.jinja2')
PROTOTXT_SOLVER = os.path.join(AUX_DIR, 'vgg16_solver_00.jinja2')
SNAPSHOT_FILE = os.path.join(EXP_DIR, '..', 'models',
    'VGG_ILSVRC_16_layers.caffemodel')

TRAIN_LIST = os.path.join(DS_DIR, 'espgame_train_list.txt')
TEST_LIST = os.path.join(DS_DIR, 'espgame_test_list.txt')
TRAIN_ANNOT = os.path.join(DS_DIR, 'espgame_train_annot.hvecs')
TEST_ANNOT = os.path.join(DS_DIR, 'espgame_test_annot.hvecs')
DATASET_INFO = dict(train_list=TRAIN_LIST, test_list=TEST_LIST,
    train_annot=TRAIN_ANNOT, test_annot=TEST_ANNOT)

def check_image_labels(dirname, prm=DATASET_INFO):
    """Create/Check that ESP-Game labels are in HDF5 format

    Note
    ----
    Be careful launching multiple-process dumping source files

    """
    filename = os.path.join(dirname, 'label_train.h5')
    if not os.path.isfile(filename):
        ds.espgame_dump_labels(prm['train_annot'], filename)
    filename = os.path.join(dirname, 'label_test.h5')
    if not os.path.isfile(filename):
        ds.espgame_dump_labels(prm['test_annot'], filename)

def check_txt_sources(dirname, prm=DATASET_INFO):
    """Create/Check that if train and test list exist

    Note
    ----
    Be careful launching multiple-process dumping source files

    """
    filename_train = os.path.join(dirname, 'img_train.txt')
    if not os.path.isfile(filename_train):
        ds.espgame_dump_list(prm['train_list'], filename_train)
    filename_test = os.path.join(dirname, 'img_test.txt')
    if not os.path.isfile(filename_test):
        ds.espgame_dump_list(prm['test_list'], filename_test)
    return filename_train, filename_test

def create_prefix(name, dirname):
    """Create prefix to identify and experiment"""
    exp_id = os.path.join(dirname, name)
    if not os.path.isdir(dirname):
        os.makedirs(dirname)
    return exp_id

def create_prototxt_net(filename, version=0, **kwargs):
    if version == 0:
        prm = dict(batch_size=64, label_src="{{ h5_src_train }}",
            input='image_data_no_label', img_src="{{ img_src_train }}",
            img_root='data/ESP-Game/ESP-ImageSet/', loss='l2-norm',
            img_transf=dict(crop_size=224, mean_value=[104, 117, 123],
            mirror=True), new_width=256, new_height=256, n_output=268)
        vgg16_multilabel_hdf5(filename, **prm)

def dump_annotation_batches(name, Y, prefix=AUX_DIR, clobber=True, txt=True):
    """Save HDF5 files used for caffe-stochastic solver"""
    exp_id = create_prefix(name, prefix)
    src_file, h5_file = exp_id + '.txt', exp_id + '.h5'
    src_exist = os.path.isfile(src_file)
    if clobber or not src_exist:
        utils.h5py_save(h5_file, h5mode='w', label=np.float32(Y.T))
    if txt and not src_exist:
        with open(src_file, 'w') as fid:
            fid.write(h5_file)
    return src_file

def launch_caffe(name, solver, net, snapshot=None, finetune=False, gpu_id=-1,
        prefix=''):
    """Laucn caffe binary (finetunning)"""
    log = os.path.join(prefix, name + '.log')
    cmd = ['sh', 'train.sh', str(gpu_id), solver, net, log]
    if snapshot is not None:
       cmd += [snapshot]
       if finetune:
           cmd += ['1']
    status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)

def load_labels(dirname):
    """Load train/test label matrix"""
    filename = os.path.join(dirname, 'label_train.h5')
    train = utils.h5py_load(filename, 'labels')
    filename = os.path.join(dirname, 'label_test.h5')
    test = utils.h5py_load(filename, 'labels')
    return train, test

def update_net_prototxt(txt_template, name, prefix, h5_train, h5_test, img_train,
        img_test):
    """Update network prototxt template"""
    with open(txt_template, 'r') as fid:
        prototxt = fid.read()
    template = Template(prototxt)
    netfile = os.path.join(prefix, name + '_net.prototxt')
    with open(netfile, 'w') as fid:
        print >>fid, template.render(img_src_train=img_train,
            img_src_test=img_test, h5_src_train=h5_train, h5_src_test=h5_test)
    return netfile

def update_solver_prototxt(txt_template, name, prefix, netfile):
    """Update solver prototxt template"""
    with open(txt_template, 'r') as fid:
        prototxt = fid.read()
    template = Template(prototxt)
    solverfile = os.path.join(prefix, name + '_solver.prototxt')
    snapshot = os.path.join(prefix, name + '_')
    with open(solverfile, 'w') as fid:
        print >>fid, template.render(snapshot=snapshot, net_src=netfile)
    return solverfile

# Program

def input_parser():
    help_id = 'ID used to identify experiment and its results'
    help_gpu = 'Device ID of the GPU used for the experiment'
    help_fp = 'Probability of flipping labels'
    help_ft = 'Flipping strategy (True: just positive, False: All)'
    help_ff = 'Finetune model otherwise Resume training'
    help_pn = 'Fullpath of prototxt network jinja2 template'
    help_ps = 'Fullpath of prototxt solver jinja2 template'
    help_sm = 'Fullpath of snapshot caffe-model'
    help_ad = 'Fullpath of auxiliar folder of ESP-Game experiments'

    p = argparse.ArgumentParser(
       formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    p.add_argument('exp_id', help=help_id, type=str)
    p.add_argument('-gpu', '--gpu_id', help=help_gpu, type=int, default=0)
    p.add_argument('-fp', '--flip_prob', help=help_fp, type=float, default=0.0)
    p.add_argument('-ft', '--flip_type', help=help_ft, action='store_false')
    p.add_argument('-ff', '--finetune_flag', help=help_ff, action='store_false')
    p.add_argument('-pn', '--prototxt_net', help=help_pn,
        default=PROTOTXT_NET)
    p.add_argument('-ps', '--prototxt_solver', help=help_ps,
        default=PROTOTXT_SOLVER)
    p.add_argument('-sm', '--snapshot_file', help=help_sm,
        default=SNAPSHOT_FILE)
    p.add_argument('-ad', '--aux_dir', help=help_ad, default=AUX_DIR)
    return p

def main(exp_id, gpu_id, flip_prob, flip_type, finetune_flag,
        prototxt_net, prototxt_solver, aux_dir, snapshot_file):
    train_id, test_id = exp_id + '_trn', exp_id + '_tst'
    exp_dir = os.path.join(aux_dir, '..', exp_id)
    # Check if source and annotation files exist
    img_src_train, img_src_test = check_txt_sources(aux_dir)
    check_image_labels(aux_dir)
    # Load and Flip label matrix
    Y_train, Y_test = load_labels(aux_dir)
    Yf_train = flip_label_matrix(Y_train, flip_prob, flip_type)
    # Dump annotations in caffe-hdf5 format
    h5_src_train = dump_annotation_batches(train_id, Yf_train,
        prefix=exp_dir)
    h5_src_test = dump_annotation_batches(test_id, Y_test,
        prefix=exp_dir)
    # Update network prototxt
    netfile = update_net_prototxt(prototxt_net, exp_id, exp_dir,
        h5_train=h5_src_train, h5_test=h5_src_test, img_train=img_src_train,
        img_test=img_src_test)
    # Update solver prototxt
    solverfile = update_solver_prototxt(prototxt_solver, exp_id, exp_dir,
        netfile)
    # Launch process
    launch_caffe(exp_id, solverfile, netfile, finetune=finetune_flag,
        gpu_id=gpu_id, prefix=exp_dir, snapshot=snapshot_file)

if __name__ == '__main__':
    p = input_parser()
    args = p.parse_args()
    main(**vars(args))

max_stars_repo_name: escorciav/mlml-cnn

max_issues_repo_name: escorciav/mlml-cnn

max_forks_repo_name: escorciav/mlml-cnn


text:
from base import BaseExpert
import numpy as np

class TrueExpert(BaseExpert):
    """docstring for TrueExpert"""

    def __init__(self, oracle):
        super(TrueExpert, self).__init__(oracle)

    def label(self, data, y=None):
        if y is None:
            raise Exception("True labels are missing")
        else:
            return y

    def fit(self, X, y=None, vct=None):
        return self

class NoisyExpert(BaseExpert):

    def __init__(self, oracle, noise_p, seed=8273645):

        super(NoisyExpert, self).__init__(oracle)
        self.noise_p = noise_p
        self.rnd = np.random.RandomState(seed)

    def label(self, data, y=None):

        if len(y) == 1:
            coin = self.rnd.random_sample()

            if coin < self.noise_p:
                return 1 - y
            else:
                return y
        else:
            coin = self.rnd.random_sample(y.shape)
            new_target = y.copy()
            new_target[coin < self.noise_p] = 1 - new_target[coin < self.noise_p]
            return new_target

    def fit(self, X, y=None, vct=None):
        return self

class PredictingExpert(BaseExpert):
    """docstring for PredictingExpert"""

    def __init__(self, oracle):
        super(PredictingExpert, self).__init__(oracle)

    def label(self, data, y=None):
        return self.oracle.predict(data)

    def fit(self, data, y=None, vct=None):
        if y is not None:
            self.oracle.fit(data.bow, y)
        else:
            self.oracle.fit(data.bow, data.target)
        return self

class SentenceExpert(PredictingExpert):
    """docstring for SentenceExpert"""

    def __init__(self, oracle, tokenizer=None):
        super(SentenceExpert, self).__init__(oracle)
        self.tokenizer = tokenizer

    def convert_to_sentence(self, X, y, vct, limit=None):
        sent_train = []
        labels = []
        tokenizer = vct.build_tokenizer()
        ## Convert the documents into sentences: train
        for t, sentences in zip(y, self.tokenizer.tokenize_sents(X)):
            if limit is None:
                sents = [s for s in sentences if len(tokenizer(s)) > 1]
            elif limit > 0:
                sents = [s for s in sentences if len(s.strip()) > limit]
            elif limit == 0:
                sents = [s for s in sentences]
            sent_train.extend(sents)  # at the sentences separately as individual documents
            labels.extend([t] * len(sents))  # Give the label of the document to all its sentences

        return sent_train, labels  # , dump

    def fit(self, X_text, y=None, vct=None):
        sx, sy = self.convert_to_sentence(X_text, y, vct)
        sx = vct.transform(sx)
        self.oracle.fit(sx, sy)
        return self

class ReluctantSentenceExpert(SentenceExpert):

    def __init__(self, oracle, reluctant_threshold, tokenizer=None, seed=43212):
        super(ReluctantSentenceExpert, self).__init__(oracle, tokenizer=tokenizer)
        self.reluctant_threhold = reluctant_threshold
        self.rnd = np.random.RandomState(seed)

    def label(self, data, y=None):
        prediction = np.array([None] * data.shape[0], dtype=object)
        proba = self.oracle.predict_proba(data)
        pred = self.oracle.predict(data)
        unc = proba.min(axis=1)
        prediction[unc < self.reluctant_threhold] = pred[unc < self.reluctant_threhold]

        return prediction

class ReluctantDocumentExpert(PredictingExpert):

    def __init__(self, oracle, reluctant_threshold, seed=43212):
        super(ReluctantDocumentExpert, self).__init__(oracle)
        self.reluctant_threhold = reluctant_threshold
        self.rnd = np.random.RandomState(seed)

    def fit(self, X_text, y=None, vct=None):
        sx = vct.transform(X_text)
        self.oracle.fit(sx, y)
        return self

    def label(self, data, y=None):
        prediction = np.array([None] * data.shape[0], dtype=object)
        proba = self.oracle.predict_proba(data)
        pred = self.oracle.predict(data)
        unc = proba.min(axis=1)
        prediction[unc < self.reluctant_threhold] = pred[unc < self.reluctant_threhold]

        return prediction

class PerfectReluctantDocumentExpert(PredictingExpert):

    def __init__(self, oracle, reluctant_threshold, tokenizer=None, seed=43212):
        super(PerfectReluctantDocumentExpert, self).__init__(oracle)
        self.reluctant_threhold = reluctant_threshold
        self.rnd = np.random.RandomState(seed)

    def fit(self, X_text, y=None, vct=None):
        sx = vct.transform(X_text)
        self.oracle.fit(sx, y)
        return self

    def label(self, query, y=None):
        data = query
        if isinstance(query, dict):
            data = query.bow
        prediction = np.array([None] * data.shape[0], dtype=object)
        proba = self.oracle.predict_proba(data)
        unc = 1. - proba.max(axis=1)
        prediction[unc < self.reluctant_threhold] = y[unc < self.reluctant_threhold]

        return prediction

class TrueReluctantExpert(TrueExpert):

    def __init__(self, oracle, reluctant_p,  seed=43212):
        super(TrueReluctantExpert, self).__init__(oracle)
        self.rnd = np.random.RandomState(seed)
        self.reluctant_p = reluctant_p

    def label(self, data, y=None):

        prediction = np.array(y, dtype=object)  # copy the true labels

        coin = self.rnd.random_sample(len(y))  # flip a coin for neutral probability

        prediction[coin < self.reluctant_p] = None  ## if coin is < p then is neutral

        return prediction


text:
from __future__ import division
import os
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision.models.inception import inception_v3
import numpy as np
import skimage.metrics
from scipy import linalg

MODEL_ZOOS = dict()

class InceptionV3(nn.Module):
    """Pretrained InceptionV3 network returning feature maps"""

    # Index of default block of inception to return,
    # corresponds to output of final average pooling
    DEFAULT_BLOCK_INDEX = 4

    # Maps feature dimensionality to their output blocks indices
    BLOCK_INDEX_BY_DIM = {
        64: 0,    # First max pooling features
        192: 1,   # Second max pooling featurs
        768: 2,   # Pre-aux classifier features
        2048: 3,  # Final average pooling features
        1000: 4   # Final classifier score
    }

    def __init__(self,
                 output_blocks=(DEFAULT_BLOCK_INDEX,),
                 resize_input=True,
                 normalize_input=True,
                 requires_grad=False):
        """Build pretrained InceptionV3
        Parameters
        ----------
        output_blocks : list of int
            Indices of blocks to return features of. Possible values are:
                - 0: corresponds to output of first max pooling
                - 1: corresponds to output of second max pooling
                - 2: corresponds to output which is fed to aux classifier
                - 3: corresponds to output of final average pooling
        resize_input : bool
            If true, bilinearly resizes input to width and height 299 before
            feeding input to model. As the network without fully connected
            layers is fully convolutional, it should be able to handle inputs
            of arbitrary size, so resizing might not be strictly needed
        normalize_input : bool
            If true, scales the input from range (0, 1) to the range the
            pretrained Inception network expects, namely (-1, 1)
        requires_grad : bool
            If true, parameters of the model require gradient. Possibly useful
            for finetuning the network
        """
        super(InceptionV3, self).__init__()

        self.resize_input = resize_input
        self.normalize_input = normalize_input
        self.output_blocks = sorted(output_blocks)
        self.last_needed_block = max(output_blocks)

        assert self.last_needed_block <= 4, \
            'Last possible output block index is 4'

        self.blocks = nn.ModuleList()

        inception = inception_v3(pretrained=True)

        # Block 0: input to maxpool1
        block0 = [
            inception.Conv2d_1a_3x3,
            inception.Conv2d_2a_3x3,
            inception.Conv2d_2b_3x3,
            nn.MaxPool2d(kernel_size=3, stride=2)
        ]
        self.blocks.append(nn.Sequential(*block0))

        # Block 1: maxpool1 to maxpool2
        if self.last_needed_block >= 1:
            block1 = [
                inception.Conv2d_3b_1x1,
                inception.Conv2d_4a_3x3,
                nn.MaxPool2d(kernel_size=3, stride=2)
            ]
            self.blocks.append(nn.Sequential(*block1))

        # Block 2: maxpool2 to aux classifier
        if self.last_needed_block >= 2:
            block2 = [
                inception.Mixed_5b,
                inception.Mixed_5c,
                inception.Mixed_5d,
                inception.Mixed_6a,
                inception.Mixed_6b,
                inception.Mixed_6c,
                inception.Mixed_6d,
                inception.Mixed_6e,
            ]
            self.blocks.append(nn.Sequential(*block2))

        # Block 3: aux classifier to final avgpool
        if self.last_needed_block >= 3:
            block3 = [
                inception.Mixed_7a,
                inception.Mixed_7b,
                inception.Mixed_7c,
                nn.AdaptiveAvgPool2d(output_size=(1, 1))
            ]
            self.blocks.append(nn.Sequential(*block3))

        # Block 4: 1000 classifier
        if self.last_needed_block >= 4:
            self.blocks.append(inception.fc)

        for param in self.parameters():
            param.requires_grad = requires_grad

    def forward(self, inp):
        """Get Inception feature maps
        Parameters
        ----------
        inp : torch.autograd.Variable
            Input tensor of shape Bx3xHxW. Values are expected to be in
            range (0, 1)
        Returns
        -------
        List of torch.autograd.Variable, corresponding to the selected output
        block, sorted ascending by index
        """
        outp = []
        x = inp

        if self.resize_input:
            x = F.interpolate(x, size=(299, 299), mode='bilinear', align_corners=False)

        if self.normalize_input:
            x = 2 * x - 1  # Scale from range (0, 1) to range (-1, 1)

        for idx, block in enumerate(self.blocks):
            x = block(x)

            if idx == 3:
                x = x[:, :, 0, 0]

            if idx in self.output_blocks:
                if len(x.shape) == 4 and (x.size(2) != 1 or x.size(3) != 1):
                    preds = F.adaptive_avg_pool2d(x, output_size=(1, 1)).squeeze_(-1).squeeze_(-1)
                else:
                    preds = x
                outp.append(preds)

            if idx == self.last_needed_block:
                break

        return outp

class BaseMetric(object):

    INCEPTION_V3 = 'inception_v3'
    PERCEPTUAL = 'perceptual'
    OSreID = 'OS-reid'
    PCBreID = 'PCB-reid'

    MODEL_KEYS = [INCEPTION_V3, PERCEPTUAL, OSreID, PCBreID]

    LOWER = 'lower score is better.'
    HIGHER = 'higher score is better'

    def __init__(self, device=torch.device("cpu"), size=(512, 512)):
        global MODEL_ZOOS

        super(BaseMetric, self).__init__()

        self.device = device
        self.size = size
        self.model_zoos = MODEL_ZOOS

    def forward(self, *input):
        raise NotImplementedError

    def calculate_score(self, *input):
        raise NotImplementedError

    def register_model(self, key):
        assert key in self.MODEL_KEYS, '{} must in {}'.format(key, self.MODEL_KEYS)

        if key not in self.model_zoos.keys():
            if key == self.INCEPTION_V3:
                self.model_zoos[key] = InceptionV3(output_blocks=[3], resize_input=False,
                                                   normalize_input=False, requires_grad=False)
                self.model_zoos[key] = self.model_zoos[key].to(self.device)
                self.model_zoos[key].eval()

            elif key == self.PERCEPTUAL:
                from .lpips import PerceptualLoss

                use_gpu = self.device != "cpu"
                model = PerceptualLoss(model='net-lin', net='alex', use_gpu=use_gpu)

                self.model_zoos[key] = model

            elif key == self.OSreID:
                from .OSreid import OsNetEncoder

                dirpath = os.path.abspath(os.path.dirname(__file__))
                model = OsNetEncoder(
                    # input_width=704,
                    # input_height=480,
                    # weight_filepath="weights/model_weights.pth.tar-40",
                    weight_filepath=os.path.join(dirpath, "OSreid", "osnet_ibn_x1_0_imagenet.pth"),
                    input_height=self.size[0],
                    input_width=self.size[1],
                    batch_size=32,
                    num_classes=2022,
                    patch_height=256,
                    patch_width=128,
                    norm_mean=[0.485, 0.456, 0.406],
                    norm_std=[0.229, 0.224, 0.225],
                    GPU=True)
                self.model_zoos[key] = model

            elif key == self.PCBreID:
                from .PCBreid import PCBReIDMetric

                dirpath = os.path.abspath(os.path.dirname(__file__))
                model_dir = os.path.join(dirpath, 'PCBreid', 'model')

                model = PCBReIDMetric(name='PCB', model_dir=model_dir)
                model = model.to(self.device)

                self.model_zoos[key] = model

            else:
                raise ValueError(key)

    def preprocess(self, x):
        raise NotImplementedError

    @staticmethod
    def to_numpy(x, transpose=False):
        if not isinstance(x, np.ndarray):
            x = x.cpu().numpy()

        if transpose:
            # (3, 256, 256) -> (256, 256, 3)
            x = np.transpose(x, (1, 2, 0))

        return x

    def quality(self):
        raise NotImplementedError

    @staticmethod
    def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):
        """Numpy implementation of the Frechet Distance.
        The Frechet distance between two multivariate Gaussians X_1 ~ N(mu_1, C_1)
        and X_2 ~ N(mu_2, C_2) is
                d^2 = ||mu_1 - mu_2||^2 + Tr(C_1 + C_2 - 2*sqrt(C_1*C_2)).
        Stable version by Dougal J. Sutherland.
        Params:
        -- mu1   : Numpy array containing the activations of a layer of the
                   inception net (like returned by the function 'get_predictions')
                   for generated samples.
        -- mu2   : The sample mean over activations, precalculated on an
                   representative data set.
        -- sigma1: The covariance matrix over activations for generated samples.
        -- sigma2: The covariance matrix over activations, precalculated on an
                   representative data set.
        Returns:
        --   : The Frechet Distance.
        """

        mu1 = np.atleast_1d(mu1)
        mu2 = np.atleast_1d(mu2)

        sigma1 = np.atleast_2d(sigma1)
        sigma2 = np.atleast_2d(sigma2)

        assert mu1.shape == mu2.shape, \
            'Training and test mean vectors have different lengths'
        assert sigma1.shape == sigma2.shape, \
            'Training and test covariances have different dimensions'

        diff = mu1 - mu2

        # Product might be almost singular
        covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
        if not np.isfinite(covmean).all():
            msg = ('fid calculation produces singular product; '
                   'adding %s to diagonal of cov estimates') % eps
            print(msg)
            offset = np.eye(sigma1.shape[0]) * eps
            covmean = linalg.sqrtm((sigma1 + offset).dot(sigma2 + offset))

        # Numerical error might give slight imaginary component
        if np.iscomplexobj(covmean):
            if not np.allclose(np.diagonal(covmean).imag, 0, atol=1e-3):
                m = np.max(np.abs(covmean.imag))
                raise ValueError('Imaginary component {}'.format(m))
            covmean = covmean.real

        tr_covmean = np.trace(covmean)

        return (diff.dot(diff) + np.trace(sigma1) +
                np.trace(sigma2) - 2 * tr_covmean)

    @staticmethod
    def fid_score_func(pred_feats, gt_feats):
        """

        Args:
            pred_feats:
            gt_feats:

        Returns:

        """
        m1, s1 = np.mean(pred_feats, axis=0), np.cov(pred_feats, rowvar=False)
        m2, s2 = np.mean(gt_feats, axis=0), np.cov(gt_feats, rowvar=False)
        return BaseMetric.calculate_frechet_distance(m1, s1, m2, s2)

    @staticmethod
    def is_score_func(feats_softmax):
        """
            inception score function.

        Args:
            feats_softmax (np.ndarray): features after softmax

        Returns:
            score (float)
        """
        kl = feats_softmax * (np.log(feats_softmax) - np.log(np.expand_dims(np.mean(feats_softmax, 0), 0)))
        kl = np.mean(np.sum(kl, 1))
        score = np.exp(kl)
        return score

class SSIMMetric(BaseMetric):

    def __init__(self):
        super(SSIMMetric, self).__init__()
        BaseMetric.__init__(self)

    def preprocess(self, x):
        """
            normalize x from [0, 255] color intensity with np.uint8 to [-1, 1] with np.float32,
        Args:
            x (np.ndarray or torch.tensor): [0, 255] color intensity, np.uint8 (torch.tensor),
            shape = (3, image_size, image_size)

        Returns:
            out (np.ndarray): [-1, 1] color intensity, np.float32, (image_size, image_size, 3)
        """
        if isinstance(x, np.ndarray):
            out = x.astype(np.float32, copy=True)
        else:
            out = x.numpy().astype(np.float32, copy=True)

        out /= 255
        out *= 2
        out -= 1

        out = np.transpose(out, (1, 2, 0))

        return out

    def forward(self, pred, ref):
        """

        Args:
            pred (np.ndarray): color intensity is [-1, 1]
            ref (np.ndarray): color intensity is [-1, 1]

        Returns:

        """
        # print(pred.shape, img.shape)
        # score = measure.compare_ssim(pred, ref, multichannel=True)
        score = skimage.metrics.structural_similarity(pred, ref, multichannel=True)
        return score

    def calculate_score(self, preds, gts):
        scores = []
        length = len(preds)

        assert length == len(gts)

        for i, (pred, ref) in enumerate(zip(preds, gts)):
            pred = self.preprocess(pred)
            ref = self.preprocess(ref)
            scores.append(self.forward(pred, ref))

        return np.mean(scores)

    def quality(self):
        return self.HIGHER

class PSNRMetric(BaseMetric):
    def __init__(self):
        super(PSNRMetric, self).__init__()
        BaseMetric.__init__(self)

    def preprocess(self, x):
        """
            normalize x from [0, 255] color intensity with np.uint8 to [-1, 1] with np.float32,
        Args:
            x (np.ndarray or torch.tensor): [0, 255] color intensity, np.uint8 (torch.tensor),
            shape = (3, image_size, image_size)

        Returns:
            out (np.ndarray): [-1, 1] color intensity, np.float32, (image_size, image_size, 3)
        """
        if isinstance(x, np.ndarray):
            out = x.astype(np.float32, copy=True)
        else:
            out = x.numpy().astype(np.float32, copy=True)

        out /= 255
        out *= 2
        out -= 1

        out = np.transpose(out, (1, 2, 0))

        return out

    def forward(self, pred, ref):
        """

        Args:
            pred (np.ndarray): color intensity is [-1, 1]
            ref (np.ndarray): color intensity is [-1, 1]

        Returns:
            score (np.float32): the ssim score, higher is better.
        """

        # score = measure.compare_psnr(pred, ref)
        score = skimage.metrics.peak_signal_noise_ratio(pred, ref)
        return score

    def calculate_score(self, preds, gts):
        scores = []
        length = len(preds)

        assert length == len(gts)

        for i, (pred, ref) in enumerate(zip(preds, gts)):
            pred = self.preprocess(pred)
            ref = self.preprocess(ref)
            scores.append(self.forward(pred, ref))

        return np.mean(scores)

    def quality(self):
        return self.HIGHER

class PerceptualMetric(BaseMetric):
    def __init__(self, device):

        BaseMetric.__init__(self, device=device)
        self.register_model(self.PERCEPTUAL)

    def preprocess(self, x):
        """

        Args:
            x (np.ndarray (or torch.tensor)): np.ndarray (or torch.tensor),
            each element is [bs, 3, image_size, image_size] wth np.uint8 (torch.uint8) and color intensity [0, 255];

        Returns:
            out (torch.tensor): (bs, 3, image_size, image_size), color intensity [-1, 1]
        """
        if isinstance(x, np.ndarray):
            out = torch.tensor(x).float()
        else:
            out = x.clone().float()
        out /= 255
        out *= 2
        out -= 1
        out = out.to(self.device)
        return out

    def forward(self, pred, ref):
        """

        Args:
            pred (torch.tensor): color intensity is [-1, 1]
            ref (torch.tensor): color intensity is [-1, 1]

        Returns:
            score (torch.tensor):
        """

        with torch.no_grad():
            # return 1.0 - torch.mean(self.model_zoos[self.PERCEPTUAL].forward(pred, target))
            return torch.mean(self.model_zoos[self.PERCEPTUAL].forward(pred, ref))

    def calculate_score(self, preds, gts, batch_size=32):
        assert len(preds) == len(gts)

        length = len(preds)

        scores = []

        for i in range(int(math.ceil((length / batch_size)))):
            pred_batch = preds[i * batch_size: (i + 1) * batch_size]
            gt_batch = gts[i * batch_size: (i + 1) * batch_size]

            pred_batch = self.preprocess(pred_batch)
            gt_batch = self.preprocess(gt_batch)

            s = self.forward(pred_batch, gt_batch)
            scores.append(s)
        scores = torch.stack(scores)
        return torch.mean(scores).cpu().numpy()

    def quality(self):
        return self.LOWER

class InceptionScoreMetric(BaseMetric):

    def __init__(self, device=torch.device("cpu")):
        BaseMetric.__init__(self, device)

        self.register_model(self.INCEPTION_V3)

        self.height, self.width = 299, 299
        # self.mean = torch.tensor([0.485, 0.456, 0.406]).float().to(self.device).view(1, 3, 1, 1)
        # self.std = torch.tensor([0.229, 0.224, 0.225]).float().to(self.device).view(1, 3, 1, 1)

    def preprocess(self, x):
        """

        Args:
            x (np.ndarray or torch.tensor): np.ndarray or torch.tensor, each element is [3, image_size, image_size] wth np.uint8 and color intensity [0, 255];

        Returns:
            out (torch.tensor): (bs, 3, 299, 299), color intensity [0, 1] and normalized using
            mean = [0.485, 0.456, 0.406] and std = [0.229, 0.224, 0.225]
        """

        if isinstance(x, np.ndarray):
            out = torch.tensor(x).float()
        else:
            out = x.clone().float()

        out /= 255
        out *= 2
        out -= 1
        out = out.to(self.device)

        with torch.no_grad():
            out = F.interpolate(out, size=(self.height, self.width), mode='bilinear', align_corners=False)
            # out = (out - self.mean) / self.std

        return out

    def forward(self, imgs):
        """

        Args:
            imgs (torch.tensor): (bs, 3, image_size, image_size), color intensity [0, 1] and normalized using
            mean = [0.485, 0.456, 0.406] and std = [0.229, 0.224, 0.225].

        Returns:
            probs (torch.tensor): (bs, number of feature outputs)
        """

        imgs = self.preprocess(imgs)
        with torch.no_grad():
            feats = self.model_zoos[self.INCEPTION_V3](imgs)[-1]
            probs = F.softmax(feats, dim=1)

        probs = probs.cpu().numpy()
        return probs

    def calculate_score(self, preds, batch_size=32):
        scores = []
        length = len(preds)
        for i in range(int(math.ceil((length / batch_size)))):
            pred_batch = preds[i * batch_size: (i + 1) * batch_size]

            part = self.forward(pred_batch)
            scores.append(self.is_score_func(part))
        return np.mean(scores), np.std(scores)

    def quality(self):
        return self.HIGHER

class FIDMetric(BaseMetric):

    def __init__(self, device=torch.device("cpu")):
        BaseMetric.__init__(self, device)

        self.register_model(self.INCEPTION_V3)

        self.height, self.width = 299, 299
        # self.mean = torch.tensor([0.485, 0.456, 0.406]).float().to(self.device).view(1, 3, 1, 1)
        # self.std = torch.tensor([0.229, 0.224, 0.225]).float().to(self.device).view(1, 3, 1, 1)

    def preprocess(self, x):
        """

        Args:
            x (np.ndarray or torch.tensor): np.ndarray or torch.tensor, each element is [3, image_size, image_size] wth np.uint8 and color intensity [0, 255];

        Returns:
            out (torch.tensor): (bs, 3, 299, 299), color intensity [0, 1] and normalized using
            mean = [0.485, 0.456, 0.406] and std = [0.229, 0.224, 0.225]
        """

        if isinstance(x, np.ndarray):
            out = torch.tensor(x).float()
        else:
            out = x.clone().float()

        out /= 255
        out *= 2
        out -= 1
        out = out.to(self.device)

        with torch.no_grad():
            out = F.interpolate(out, size=(self.height, self.width), mode='bilinear', align_corners=False)
            # out = (out - self.mean) / self.std

        return out

    def forward(self, imgs):
        """

        Args:
            imgs (torch.tensor): (bs, 3, image_size, image_size), color intensity [0, 1] and normalized using
            mean = [0.485, 0.456, 0.406] and std = [0.229, 0.224, 0.225].

        Returns:
            probs (torch.tensor): (bs, number of feature outputs)
        """

        imgs = self.preprocess(imgs)
        with torch.no_grad():
            feats = self.model_zoos[self.INCEPTION_V3](imgs)[-1]
            # print(feats.shape)
            feats = feats.cpu().numpy()
        return feats

    def calculate_score(self, preds, gts, batch_size=32):
        pred_feats = []
        gt_feats = []

        length = len(preds)
        for i in range(int(math.ceil((length / batch_size)))):
            pred_batch = preds[i * batch_size: (i + 1) * batch_size]
            gt_batch = gts[i * batch_size: (i + 1) * batch_size]

            pred_f = self.forward(pred_batch)
            gt_f = self.forward(gt_batch)

            pred_feats.append(pred_f)
            gt_feats.append(gt_f)

        pred_feats = np.concatenate(pred_feats, axis=0)
        gt_feats = np.concatenate(gt_feats, axis=0)

        return self.fid_score_func(pred_feats, gt_feats)

    def quality(self):
        return self.LOWER

class FreIDMetric(BaseMetric):
    def __init__(self, device=torch.device("cpu"), reid_name="PCB-reid"):

        BaseMetric.__init__(self, device)

        if reid_name == "PCB-reid":
            self.REID = self.PCBreID
        else:
            self.REID = self.OSreID

        self.register_model(self.REID)

    def preprocess(self, x):
        x = x.clone()
        x = x.to(self.device)
        return x

    def forward(self, pred):
        """

        Args:
            x (torch.tensor): np.ndarray or torch.tensor, each element is
                [bs, 3, image_size, image_size] wth torch.uint8 and color intensity [0, 255];

        Returns:
            feat (np.ndarray): [bs, C]
        """

        with torch.no_grad():
            pred = self.preprocess(pred)
            feat = self.model_zoos[self.REID](pred)
            feat = feat.cpu().numpy()
        return feat

    def calculate_score(self, preds, gts, batch_size=32):
        pred_feats = []
        gt_feats = []

        length = len(preds)
        for i in range(int(math.ceil((length / batch_size)))):
            pred_batch = preds[i * batch_size: (i + 1) * batch_size]
            gt_batch = gts[i * batch_size: (i + 1) * batch_size]

            pred_f = self.forward(pred_batch)
            gt_f = self.forward(gt_batch)

            pred_feats.append(pred_f)
            gt_feats.append(gt_f)

        pred_feats = np.concatenate(pred_feats, axis=0)
        gt_feats = np.concatenate(gt_feats, axis=0)

        return self.fid_score_func(pred_feats, gt_feats)

    def normalize_feats(self, feats):
        """

        Args:
            feats (np.ndarray): (bs, dim)

        Returns:
            feats (np.ndarray): (bs, dim)
        """

        norm = np.sqrt(np.sum(feats ** 2, axis=1, keepdims=True))
        feats = feats / (norm + 1e-6)
        return feats

    def cosine_similarity(self, pred_feats, ref_feats):
        pred_norm = self.normalize_feats(pred_feats)
        ref_norm = self.normalize_feats(ref_feats)

        return np.mean(np.sum(pred_norm * ref_norm, axis=1))

    def quality(self):
        return self.LOWER

class ReIDScore(FreIDMetric):

    def __init__(self, device=torch.device("cpu"), reid_name="PCB-reid"):
        super().__init__(device, reid_name)

    def calculate_score(self, preds, gts, batch_size=32):
        assert len(preds) == len(gts)

        length = len(preds)

        scores = []

        for i in range(int(math.ceil((length / batch_size)))):
            pred_batch = preds[i * batch_size: (i + 1) * batch_size]
            gt_batch = gts[i * batch_size: (i + 1) * batch_size]

            pred_f = self.forward(pred_batch)
            gt_f = self.forward(gt_batch)

            s = self.cosine_similarity(pred_f, gt_f)
            scores.append(s)

        return np.mean(scores)

    def quality(self):
        return self.HIGHER

max_stars_repo_name: Puneet-G/Impersonator-NNProject

max_issues_repo_name: Puneet-G/Impersonator-NNProject

max_forks_repo_name: Puneet-G/Impersonator-NNProject

reason: import numpy,from scipy


text:
import numpy as np

from numpy.testing import assert_allclose, assert_array_less
import pytest

try:
    from astropy.stats import LombScargle as astropy_LombScargle
except ImportError:
    astropy_LombScargle = None

from ..modeler import SlowTemplatePeriodogram, FastTemplatePeriodogram
from ..template import Template

def generate_template(nharmonics, rseed=0):
    rng = np.random.RandomState(rseed)
    c_n, s_n = rng.randn(2, nharmonics) * 1. / np.arange(1, nharmonics + 1)
    return Template(c_n, s_n)

def generate_data(template, N, tmin, tmax, freq, dy=0.1,
                  phase=0, amp=1, offset=10, rseed=0):
    rng = np.random.RandomState(rseed)
    t = tmin + (tmax - tmin) * np.random.rand(N)
    t.sort()
    y = offset + amp * template(t * freq - phase) + dy * rng.randn(N)
    dy = dy * np.ones_like(y)
    return t, y, dy

@pytest.mark.skipif(not astropy_LombScargle, reason="astropy is not installed")
def test_vs_lombscargle():
    # one-component template should be identical to a
    # Lomb-Scargle periodogram with floating-mean
    template = generate_template(1)
    t, y, dy = generate_data(template, N=50, tmin=0, tmax=100, freq=0.1)

    freq = np.linspace(0.01, 1, 10)
    power1 = SlowTemplatePeriodogram(template=template).fit(t, y, dy).power(freq)
    power2 = astropy_LombScargle(t, y, dy, fit_mean=True).power(freq)

    assert_allclose(power1, power2)

@pytest.mark.parametrize('nharmonics', [1, 2, 3])
@pytest.mark.parametrize('nguesses', [None, 2])
def test_zero_noise(nharmonics, nguesses):
    # in the zero-noise perfect template case, the true frequency should
    # have power = 1
    template = generate_template(nharmonics)
    t, y, _ = generate_data(template, N=50, tmin=0, tmax=100, freq=0.1, dy=0)
    dy = None
    power = SlowTemplatePeriodogram(nguesses=nguesses, template=template).fit(t, y, dy).power(0.1)
    assert_allclose(power, 1)

@pytest.mark.parametrize('nharmonics', [1, 2, 3])
@pytest.mark.parametrize('nguesses', [None, 2])
def test_slow_vs_fast(nharmonics, nguesses):
    # Slow periodogram has convergence issues, and will undershoot the power
    # in some cases. This means the fast periodogram should always be larger
    # than the slow periodogram, up to a small floating point error.
    template = generate_template(nharmonics)
    t, y, dy = generate_data(template, N=50, tmin=0, tmax=100, freq=0.1)
    freq = 0.01 * np.arange(1, 101)

    power_slow = SlowTemplatePeriodogram(template, nguesses=nguesses).fit(t, y, dy).power(freq)
    power_fast = FastTemplatePeriodogram(template).fit(t, y, dy).power(freq)

    assert(all(power_fast + 1E-5 >= power_slow))
    """
    inds = np.arange(len(freq))
    mask = power_slow > power_fast + 1E-4
    for i in inds[mask]:
        p_slow = power_slow[i]
        p_fast = power_fast[i]
        print("frequency %d (%.3e) is problematic because (p_slow = %e) > (p_fast = %e) + 1E-4"%((i,
                     freq[i], p_slow, p_fast)))
        print("          p_slow is %.e times larger than p_fast at this frequency"%(p_slow/p_fast - 1))

    # occaisionally (and randomly) there will be one frequency for which
    # the slow periodogram finds a slightly better solution; I've hacked around
    # that for now since it seems like a relatively minor problem, but it may
    # hint at a larger issue...
    assert(all(np.sort(power_fast - power_slow)[1:] > -1E-4))
    #assert_array_less(power_slow, power_fast + 1E-4)
    """

reason: import numpy,from numpy,from astropy


text:
from Pypeline.Pypes.Stack import Stack
import numpy as np
import unittest

class StackTest(unittest.TestCase):
	"""
	Tests for methods in the Stack class.
	"""

	@classmethod
	def setUpClass(cls):
		pass #TODO

	@classmethod
	def tearDownClass(cls):
		pass #TODO

	def setUp(self):
		self.s = Stack()
		pass #TODO

	def tearDown(self):
		self.s = None
		pass #TODO

	def test_isEmpty(self):
		assert self.s.isEmpty()

	def test_sizeStack(self):
		self.s.push(2)
		assert self.s.sizeStack() == 1

	def test_push(self):
		for i in range(100):
			self.s.push(i)
		assert self.s.sizeStack() == 100

	def test_pop(self):
		[self.s.push(x) for x in range(100, 0, -1)]
		assert(all([(self.s.pop() == (i+1)) for i in range(self.s.sizeStack())]))
		assert self.s.sizeStack() == 0

	def test_peek(self):
		self.s.push(20)
		assert self.s.peek() == 20

	def test_to_array(self):
		ref = []
		for x in range(10):
			self.s.push(x)
			ref.append(x)

		assert self.s.to_list() == ref

	def test_to_list(self):
		ref = np.zeros(10, dtype=int)
		for x in range(10):
			self.s.push(x)
			ref[x] = x

		assert (self.s.to_array() == ref).all()


text:
#!/usr/bin/env python3

import unittest
from unittest.mock import patch, MagicMock

import numpy as np
from sklearn.datasets import make_blobs
import sklearn

from tmc import points
from tmc.utils import load, get_out, patch_helper, spy_decorator

module_name="src.blob_classification"
blob_classification = load(module_name, 'blob_classification')
main = load(module_name, "main")
ph = patch_helper(module_name)

@points('p06-01.1')
class BlobClassification(unittest.TestCase):

    def test_correctness(self):
        a=np.array([[2.  , 2.  , 0.  , 2.5 , 0.76],
                    [2.  , 3.  , 1.  , 1.5 , 0.96],
                    [2.  , 2.  , 6.  , 3.5 , 0.84],
                    [2.  , 2.  , 3.  , 1.2 , 1.  ],
                    [2.  , 4.  , 4.  , 2.7 , 0.8 ]])
        idx=np.arange(5)
        np.random.shuffle(idx)
        for row in a[idx]:
            X,y = make_blobs(100, int(row[0]), centers=int(row[1]),
                                      random_state=int(row[2]), cluster_std=row[3])
            acc=blob_classification(X, y)
            self.assertAlmostEqual(acc, row[-1], msg="Incorrect accuracy score!")

    def test_calls(self):
        row = [2.  , 2.  , 0.  , 2.5 , 0.76]
        X,y = make_blobs(100, int(row[0]), centers=int(row[1]),
                         random_state=int(row[2]), cluster_std=row[3])
        predict_method = spy_decorator(sklearn.naive_bayes.GaussianNB.predict, "predict")
        fit_method = spy_decorator(sklearn.naive_bayes.GaussianNB.fit, "fit")
        with patch(ph("sklearn.model_selection.train_test_split"),
                   wraps=sklearn.model_selection.train_test_split) as tts,\
            patch(ph("sklearn.metrics.accuracy_score"),
                  wraps=sklearn.metrics.accuracy_score) as acs,\
            patch.object(sklearn.naive_bayes.GaussianNB, "fit", new=fit_method),\
            patch.object(sklearn.naive_bayes.GaussianNB, "predict", new=predict_method),\
            patch(ph("sklearn.naive_bayes.GaussianNB"),
                  wraps=sklearn.naive_bayes.GaussianNB) as gnb:

            acc=blob_classification(X, y)

            # Check that train_test_split is called with correct parameters
            tts.assert_called_once()
            args, kwargs = tts.call_args
            self.assertIn("random_state", kwargs,
                          msg="You did not specify random_state argument"
                          "to train_test_split function!")
            self.assertEqual(kwargs["random_state"], 0,
                             msg="Incorrect random_state argument!")
            if "train_size" in kwargs:
              self.assertEqual(kwargs["train_size"], 0.75, msg="Incorrect train_size argument!")
            elif "test_size" in kwargs:
              self.assertEqual(kwargs["test_size"], 0.25, msg="Incorrect test_size argument!")

            # Check that accuracy_score is called
            acs.assert_called_once()

            # Check that GaussianNB is called
            gnb.assert_called_once()

            # Check that fit and predict methods of GaussianNB object are called
            predict_method.mock.assert_called()
            fit_method.mock.assert_called()

if __name__ == '__main__':
    unittest.main()


text:
def sparsity_pattern(elements, N_n):
    import numpy as np
    matrix = np.zeros((N_n, N_n), dtype=str)
    matrix[:,:] = '0'
    for e in elements:
        for i in e:
            for j in e:
                matrix[i,j] = 'X'
    matrix = matrix.tolist()
    matrix = '\n'.join([' '.join([matrix[i][j]
                                  for j in range(len(matrix[i]))])
                        for i in range(len(matrix))])
    return matrix

print('\nP1 elements, left-to-right numbering')
N_n = 4
elements = [[0,1], [1,2], [2,3]]
print(sparsity_pattern(elements, N_n))

print('\nP1 elements, right-to-left numbering')
elements = [[1,0], [2,1], [3,2]]
print(sparsity_pattern(elements, N_n))

print('\nP2 elements, left-to-right numbering')
N_n = 7
elements = [[0,1,2], [2,3,4], [4,5,6]]
print(sparsity_pattern(elements, N_n))

print('\nP1 elements, right-to-left numbering')
elements = [[2,1,0], [4,3,2], [6,5,4]]
print(sparsity_pattern(elements, N_n))

max_stars_repo_name: hplgit/fem-book

max_issues_repo_name: hplgit/fem-book

max_forks_repo_name: hplgit/fem-book


text:
import numpy as np
from mask_box import bbox_area, mask2bbox

def compute_cc(mask, minarea=5 ** 2):
    mask_cc = []
    import skimage.morphology as skimorph

    labels = skimorph.label(mask, connectivity=2)
    for n in np.unique(labels):
        if n == 0: continue
        mask_indiv = labels == n
        if mask_indiv.sum() < minarea: continue
        if bbox_area(mask2bbox(mask_indiv)) == 0: print('        ...zero area!!'); continue
        mask_cc.append(mask_indiv)

    print('    Found %d connected components in mask' % (len(mask_cc)))

    return mask_cc

max_stars_repo_name: coallaoh/ResearchTools

max_issues_repo_name: coallaoh/ResearchTools

max_forks_repo_name: coallaoh/ResearchTools