text:
"""
Re-tooled version of the script found on VideoToTextDNN:
https://github.com/OSUPCVLab/VideoToTextDNN/blob/master/data/py3_process_features.py

Perform batched feature extract using Cadene pretrainedmodels
"""
import torch
import pretrainedmodels
import pretrainedmodels.utils as utils
import torch.nn as nn
import argparse
import time
import os
import numpy as np
import logging

logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

available_features = ['nasnetalarge', 'resnet152', 'pnasnet5large', 'densenet121', 'senet154', 'polynet']

args = None

def init_model(gpu_ids, model_name):
    # model_name = 'pnasnet5large'
    # could be fbresnet152 or inceptionresnetv2
    model = pretrainedmodels.__dict__[model_name](num_classes=1000, pretrained='imagenet')
    model.eval()
    load_img = utils.LoadImage()

    # transformations depending on the model
    # rescale, center crop, normalize, and others (ex: ToBGR, ToRange255)
    tf_img = utils.TransformImage(model)

    """
    TODO(WG): Would be nice to use something like DataParallel, but that only does forward pass on given module.
    Need to stop before logits step.
    Should create wrapper for pretrainedmodels that does the MPI-like ops across GPUs on model.features modules:
    1) replicated
    2) scatter
    3) parallel_apply
    4) gather
    Would have to know what layers are being used on each model.
    """
    if torch.cuda.is_available():
        model = model.cuda(device=gpu_ids[0])

    return load_img, tf_img, model

def extract_features(args):
    root_frames_dir = args.frames_dir
    root_feats_dir = args.feats_dir
    work = args.work
    autofill = int(args.autofill)
    ftype = args.type
    gpu_list = args.gpu_list

    frames_dirs = os.listdir(root_frames_dir)

    if not os.path.isdir(root_feats_dir):
        os.mkdir(root_feats_dir)
    # else:
    #     if autofill:
    #         logger.info('AUTOFILL ON: Attempting to autofill missing features.')
    #         frames_dirs = validate_feats.go(featsd=root_feats_dir, framesd=root_frames_dir)

    # Difficulty of each job is measured by # of frames to process in each chunk.
    # Can't be randomized since autofill list woudld be no longer valid.
    # np.random.shuffle(frames_dirs)
    work = len(frames_dirs) if not work else work

    load_img, tf_img, model = init_model(args.gpu_list, args.type)

    work_done = 0
    while work_done != work:
        frames_dirs_avail = diff_feats(root_frames_dir, root_feats_dir)
        if len(frames_dirs_avail) == 0:
            break

        frames_dir = np.random.choice(frames_dirs_avail)
        ext = '.' + frames_dir.split('.')[-1]
        feat_filename = frames_dir.split('/')[-1].split(ext)[0]
        video_feats_path = os.path.join(args.feats_dir, feat_filename)

        if os.path.exists(video_feats_path):
            logger.info('Features already extracted:\t{}'.format(video_feats_path))
            continue

        try:
            frames_to_do = [os.path.join(args.frames_dir, frames_dir, p) for p in
                            os.listdir(os.path.join(args.frames_dir, frames_dir))]
        except Exception as e:
            logger.exception(e)
            continue

        # Must sort so frames follow numerical order. os.listdir does not guarantee order.
        frames_to_do.sort()

        if len(frames_to_do) == 0:
            logger.warning("Frame folder has no frames! Skipping...")
            continue

        # Save a flag copy
        with open(video_feats_path, 'wb') as pf:
            np.save(pf, [])

        try:
            batches = create_batches(frames_to_do, load_img, tf_img, batch_size=args.batch_size)
        except OSError as e:
            logger.exception(e)
            logger.warning("Corrupt image file. Skipping...")
            os.remove(video_feats_path)
            continue

        logger.debug("Start video {}".format(work_done))

        feats = process_batches(batches, ftype, gpu_list, model)

        with open(video_feats_path, 'wb') as pf:
            np.save(pf, feats)
            logger.info('Saved complete features to {}.'.format(video_feats_path))
        work_done += 1

def process_batches(batches, ftype, gpu_list, model):
    done_batches = []
    for i, batch in enumerate(batches):
        if torch.cuda.is_available():
            batch = batch.cuda(device=gpu_list[0])

        output_features = model.features(batch)
        output_features = output_features.data.cpu()

        conv_size = output_features.shape[-1]

        if ftype == 'nasnetalarge' or ftype == 'pnasnet5large':
            relu = nn.ReLU()
            rf = relu(output_features)
            avg_pool = nn.AvgPool2d(conv_size, stride=1, padding=0)
            out_feats = avg_pool(rf)
        else:
            avg_pool = nn.AvgPool2d(conv_size, stride=1, padding=0)
            out_feats = avg_pool(output_features)

        out_feats = out_feats.view(out_feats.size(0), -1)
        logger.info('Processed {}/{} batches.\r'.format(i + 1, len(batches)))

        done_batches.append(out_feats)
    feats = np.concatenate(done_batches, axis=0)
    return feats

def create_batches(frames_to_do, load_img_fn, tf_img_fn, batch_size=8):
    n = len(frames_to_do)
    if n < batch_size:
        logger.warning("Sample size less than batch size: Cutting batch size.")
        batch_size = n

    logger.info("Generating {} batches...".format(n // batch_size))
    batches = []
    frames_to_do = np.array(frames_to_do)

    for idx in range(0, n, batch_size):
        frames_idx = list(range(idx, min(idx+batch_size, n)))
        batch_frames = frames_to_do[frames_idx]

        batch_tensor = torch.zeros((len(batch_frames),) + tuple(tf_img_fn.input_size))
        for i, frame_ in enumerate(batch_frames):
            input_img = load_img_fn(frame_)
            input_tensor = tf_img_fn(input_img)  # 3x400x225 -> 3x299x299 size may differ
            # input_tensor = input_tensor.unsqueeze(0)  # 3x299x299 -> 1x3x299x299
            batch_tensor[i] = input_tensor

        batch_ag = torch.autograd.Variable(batch_tensor, requires_grad=False)
        batches.append(batch_ag)

    return batches

def diff_feats(frames_dir, feats_dir):
    feats = set(os.listdir(feats_dir))
    frames_to_ext = {'.'.join(i.split('.')[:-1]): i.split('.')[-1] for i in os.listdir(frames_dir)}
    frames = set(frames_to_ext.keys())
    needed_feats = frames - feats
    needed_feats = [i + '.' + frames_to_ext[i] for i in needed_feats]
    return needed_feats

if __name__ == '__main__':
    arg_parser = argparse.ArgumentParser()
    arg_parser.add_argument('frames_dir',help = 'Directory where there are frame directories.')
    arg_parser.add_argument('feats_dir',help = 'Root directory of dataset\'s processed videos.')
    arg_parser.add_argument('-w', '--work', help = 'Number of features to process. Defaults to all.', default=0, type=int)
    arg_parser.add_argument('-gl', '--gpu_list', required=True, nargs='+', type=int, help="Space delimited list of GPU indices to use. Example for 4 GPUs: -gl 0 1 2 3")
    arg_parser.add_argument('-bs', '--batch_size', type=int, help="Batch size to use during feature extraction. Larger batch size = more VRAM usage", default=8)
    arg_parser.add_argument('--type', required=True, help = 'ConvNet to use for processing features.', choices=available_features)
    arg_parser.add_argument('--autofill', action='store_true', default=False, help="Perform diff between frames_dir and feats_dir and fill them in.")

    args = arg_parser.parse_args()

    start_time = time.time()

    logger.info("Found {} GPUs, using {}.".format(torch.cuda.device_count(), len(args.gpu_list)))

    extract_features(args)

    logger.info("Job took %s mins" % ((time.time() - start_time)/60))


text:
from .esi import Esi
from .Appraisal import Appraisal
from .Source import Source
import asyncio
import numpy as np
import itertools
import logging
from datetime import datetime, timedelta, date

logger = logging.getLogger("Esipraisal")

class Esipraisal(object):

    def __init__(self):
        self.__price_table = None
        self.ops = Esi()
        self.client = Esi.get_client()

    async def appraise(self, type_id, region_ids):
        ccp_val = await self.__value_from_ccp(type_id)
        if ccp_val is None:
            estimated_val = None
        else:
            estimated_val = ccp_val.value

        #Method 1: Orders on market
        order_value = await self.__value_from_orders(type_id, region_ids, estimated_val)

        if order_value is not None:
            return order_value

        #Method 2: Historical average
        hist_val = await self.__value_from_history(type_id, region_ids, estimated_val)

        if hist_val is not None:
            return hist_val

        #Method 3:  CCP's value
        if ccp_val is not None:
            ccp_val.region_list = region_ids
            return ccp_val

        app = Appraisal()
        app.source = Source.Invalid
        return app

    async def __value_from_orders(self, type_id, region_ids, ccp_value):
        app = Appraisal()
        app.type = type_id
        app.region_list = region_ids
        app.source = Source.MarketOrders

        async with self.client.session() as esi:
            orders = await self.__fetch_orders(esi, type_id, region_ids)

        price_dicts = self.__to_price_dicts(orders, ccp_value)
        buy_vol = price_dicts.get("buy_volume", 0)
        sell_vol = price_dicts.get("sell_volume", 0)
        min_vol = self.__min_volume(ccp_value)
        logger.info("Volumes: buy = {} sell = {} min = {}".format(buy_vol, sell_vol, min_vol))
        if buy_vol + sell_vol < min_vol:
            #Exit if volume is too low
            return None

        buy_dict = price_dicts.get("buy")
        sell_dict = price_dicts.get("sell")

        buy_vols = []
        buy_prices = []

        sell_vols = []
        sell_prices = []

        volumes = []
        prices = []

        for price, volume in buy_dict.items():
            volumes.append(volume)
            prices.append(price)
            buy_vols.append(volume)
            buy_prices.append(price)

        for price, volume in sell_dict.items():
            volumes.append(volume)
            prices.append(price)
            sell_vols.append(volume)
            sell_prices.append(price)

        vol = np.sum(volumes)
        if vol <= 0:
            return None

        buy_vol = np.sum(buy_vols)
        if buy_vol <= 0:
            return None

        sell_vol = np.sum(sell_vols)
        if sell_vol <= 0:
            return None

        app.value = np.average(prices, axis=0, weights=volumes)
        app.buy_value = np.average(buy_prices, axis=0, weights=buy_vols)
        app.sell_value = np.average(sell_prices, axis=0, weights=sell_vols)

        app.volume = vol
        app.buy_volume = buy_vol
        app.sell_volume = sell_vol

        return app

    def __min_volume(self, estimated_value):
        if estimated_value is None:
            return 50

        if estimated_value < 1e6:
            return 1000
        if estimated_value < 1e9:
            return 100
        return 10

    async def __value_from_history(self, type_id, region_ids, ccp_val):
        app = Appraisal()
        app.type = type_id
        app.region_list = region_ids
        app.source = Source.HistoricalOrders

        async with self.client.session() as esi:
            region_futures = []
            for region in region_ids:
                region_futures.append(self.ops.get_market_history_by_region(esi, region, type_id))

            results = await asyncio.gather(*region_futures)

        prices = []
        volumes = []

        for result in results:
            if result is None:
                continue

            if len(result) < 1:
                continue

            valid_count = 0
            indx = len(result) - 1
            while indx > 0 and valid_count < 7:
                data = result[indx]
                logger.info(data)
                indx -= 1

                price = data.get("average")
                volume = data.get("volume", 0)
                date_str = data.get("date", '1900-01-01')
                data_date = datetime.strptime(date_str, '%Y-%m-%d')

                if data_date + timedelta(days=30) < datetime.utcnow():
                    #Data is too old
                    break

                if self.__is_outlier(price, ccp_val):
                    logger.info("outlier: price={} ccp_val={}".format(price, ccp_val))
                    continue

                if price is None:
                    continue

                if volume <= 0:
                    continue

                prices.append(price)
                volumes.append(volume)
                valid_count += 1

        vol = np.sum(volumes)
        if vol < self.__min_volume(ccp_val):
            logger.info("Min vol not met: volume={} min vol={}".format(vol, self.__min_volume(ccp_val)))
            return None

        app.value = np.average(prices, weights=volumes)
        app.volume = vol

        return app

    async def __value_from_ccp(self, type_id):
        app = Appraisal()
        app.type = type_id
        app.source = Source.CCP

        async with self.client.session() as esi:
            self.__price_table = await self.ops.get_prices(esi)

        for item_price in self.__price_table:
            if int(item_price.get("type_id")) == int(type_id):
                price = item_price.get("average_price")
                if price is None:
                    price = item_price.get("adjusted_price")
                app.value = price
                return app

        logger.warning("Could not get price type={}".format(type_id))
        return None

    #Fetch orders from region(s) using ESI
    async def __fetch_orders(self, esi, type_id, region_ids):

        region_futures = []
        for region in region_ids:
            region_futures.append(self.ops.get_orders_by_region(esi, region, type_id))

        results = await asyncio.gather(*region_futures)

        combined_results = []

        for result in results:
            if result is None:
                continue
            combined_results = combined_results + result

        return combined_results

    #Get an array of prices for use with statistical analysis
    def __to_price_dicts(self, orders_list, ccp_val):
        n_orders = len(orders_list)
        n = 1

        buy_orders = {}
        sell_orders = {}
        buy_volume = 0
        sell_volume = 0

        for order in orders_list:

            buy_order = order.get("is_buy_order")
            price = order.get("price")
            volume = order.get("volume_remain")

            #Outlier filtering
            if self.__is_outlier(price, ccp_val):
                continue

            logger.debug("Processing {} of {} (Volume={})".format(n, n_orders, volume))
            n += 1

            if buy_order:
                buy_volume += volume
                if price in buy_orders:
                    buy_orders[price] = buy_orders[price] + volume
                else:
                    buy_orders[price] = volume
            else:
                sell_volume += volume
                if price in sell_orders:
                    sell_orders[price] = sell_orders[price] + volume
                else:
                    sell_orders[price] = volume

        return {"buy":buy_orders, "buy_volume": buy_volume, "sell":sell_orders, "sell_volume": sell_volume}

    def __is_outlier(self, price, average_value):
        if average_value is None:
            return False
        #These should be pretty borad outliers just want to filter out the very low/high
        max_price = average_value * 1.75
        min_price = average_value * 0.25

        if price > max_price:
            logger.debug("Outlier (over): {}".format(price))
            return True
        if price < min_price:
            logger.debug("Outlier (under): {}".format(price))
            return True

        return False

ext: py

max_stars_repo_name: FlyingKiwis/Esipraisal

max_issues_repo_name: FlyingKiwis/Esipraisal

max_forks_repo_name: FlyingKiwis/Esipraisal

reason: import numpy


text:
import numpy as np
import pandas as pd
import random
import keras
from keras.optimizers import Adam
from keras.models import Input, Model
from keras.layers import Conv3D, Concatenate, MaxPooling3D, AveragePooling3D, UpSampling3D, Activation, Reshape, Permute, Dropout
from keras.callbacks import EarlyStopping, ModelCheckpoint

class BaselineModel():
	# Params:
	# optimizer: keras optimizer (e.g. Adam)
	# loss: loss function for optimization
	# metrics: list of metrics (e.g. ['accuracy'])
	def __init__(self, optimizer, loss, metrics=['accuracy'], epochs=1, batch_size=16, model_name="unknown", use_dropout=0.0):
		self.optimizer = optimizer
		self.model_name = model_name
		self.loss = loss
		self.metrics = metrics
		self.epochs = epochs
		self.batch_size = batch_size
		self.model = None
		self.checkpointer = ModelCheckpoint('model-' + self.model_name+ '-1.h5', verbose=1, save_best_only=True)
		self.dropout = use_dropout

	# This function defines the baseline model in Keras
	def build_model(self, input_shape):

		# Define the input placeholder as a tensor with shape input_shape. Think of this as your input image!
	    X_input = Input(input_shape)

	    # [1] First CONV--POOL--ReLU layer
	    X = Conv3D(filters=16, kernel_size=3, strides=(1,1,1), padding='same', activation='elu')(X_input)
	    X = AveragePooling3D(strides=(1,1,1), padding="same")(X)
	    X = Dropout(self.dropout)(X)

	    # [2] Second CONV--POOL--ReLU layer
	    X = Conv3D(filters=32, kernel_size=3, strides=(1,1,1), padding='same', activation='elu')(X)
	    X = AveragePooling3D(strides=(1,1,1), padding="same")(X)
	    X = Dropout(self.dropout)(X)

	    # [3] Third CONV--POOL--ReLU layer
	    X = Conv3D(filters=64, kernel_size=3, strides=(1,1,1), padding='same', activation='elu')(X)
	    X = AveragePooling3D(strides=(1,1,1), padding="same")(X)
	    X = Dropout(self.dropout)(X)

	    # Final prediction (sigmoid)
	    X = Conv3D(filters=1, kernel_size=1, activation='sigmoid')(X)

	    # Create instance of Baseline Model
	    model = Model(inputs=X_input, outputs=X, name='BaselineModel')

	    self.model = model

	def compile(self):
		self.model.compile(optimizer=self.optimizer, loss=self.loss, metrics=self.metrics)
		self.model.summary()

	# def fit(self, X_train, Y_train, validation_set=None, global_step=None, callbacks=[self.checkpointer]):
	# 	earlystopper = EarlyStopping(patience=10, verbose=1)

	# 	if validation_set is None:
	# 		return self.model.fit(x=X_train, y=Y_train, validation_split=0.2, epochs=self.epochs, batch_size=self.batch_size, callbacks=callbacks)
	# 	else:
	# 		if global_step is not None:
	# 			return self.model.fit(x=X_train, y=Y_train, validation_data=validation_set, epochs=self.epochs, batch_size=self.batch_size, callbacks=callbacks, global_step=global_step)
	# 		else:
	# 			return self.model.fit(x=X_train, y=Y_train, validation_data=validation_set, epochs=self.epochs, batch_size=self.batch_size, callbacks=callbacks)

	def evaluate(self, X_test, Y_test):
		return self.model.evaluate(x=X_test, y=Y_test)


text:
from tensorflow.keras.models import load_model
from tensorflow.keras.preprocessing.image import img_to_array
from tensorflow.keras.preprocessing.image import load_img
from numpy import expand_dims
from matplotlib import pyplot

def load_image(filename, size=(256,256)):
    pixels = load_img(filename, target_size=size)
    pixels = img_to_array(pixels)
    pixels = (pixels - 127.5) / 127.5
    pixels = expand_dims(pixels, 0)
    return pixels

src_image = load_image('k.PNG')
print('Loaded', src_image.shape)
model = load_model('model_043840.h5')
gen_image = model.predict(src_image)
gen_image = (gen_image + 1) / 2.0
pyplot.imshow(gen_image[0])
pyplot.axis('off')
pyplot.show()

reason: from numpy


text:
"""
Core library module
"""
import multiprocessing as mp
import numpy as np
import pandas as pd

#pylint: disable=too-few-public-methods
class FunctionObject(object):
    """
    Expose a simple interface for creating a function object to apply from

    Args:
        value: A value in the series

    Returns:
        The value of this function object applied to the value
    """
    def __call__(self, value):
        pass

def apply(source, function_obj, **kwargs):
    """
    Apply a function in a series/dataframe values in parallel

    Args:
        source: Pandas DataFrame/Series
        function_obj (FunctionObject): the function object to be applied in source
        kwargs (dict): extra options for the function

    Returns:
        Values produced by application of the function
    """
    if not isinstance(function_obj, FunctionObject):
        raise ValueError('function_obj should be an instance of FunctionObject class')
    workers = mp.cpu_count() - 1
    if 'workers' in kwargs:
        workers = kwargs.pop('workers')
    pool = mp.Pool(processes=workers)
    partitions = np.array_split(source, workers)
    results = pool.map(_apply_function,
                       [(partition, function_obj, kwargs)
                        for partition in partitions])
    result = pd.concat(results)
    pool.close()
    return result

def _apply_function(args):
    """
    Helper function for applying the function in a given source partition
    """
    source, function_obj, kwargs = args
    return source.apply(function_obj, kwargs)


text:
from __future__ import division
import numpy as np
from matplotlib import pyplot as plt

from pyhsmm.util.stats import cov
import matplotlib

def heatmap(data, row_labels, col_labels, ax=None,
            cbar_kw={}, cbarlabel="", **kwargs):
    """
    Create a heatmap from a numpy array and two lists of labels.

    Parameters
    ----------
    data
        A 2D numpy array of shape (N, M).
    row_labels
        A list or array of length N with the labels for the rows.
    col_labels
        A list or array of length M with the labels for the columns.
    ax
        A `matplotlib.axes.Axes` instance to which the heatmap is plotted.  If
        not provided, use current axes or create a new one.  Optional.
    cbar_kw
        A dictionary with arguments to `matplotlib.Figure.colorbar`.  Optional.
    cbarlabel
        The label for the colorbar.  Optional.
    **kwargs
        All other arguments are forwarded to `imshow`.
    """

    if not ax:
        ax = plt.gca()

    # Plot the heatmap
    im = ax.imshow(data, **kwargs)

    # Create colorbar
    cbar = ax.figure.colorbar(im, ax=ax, **cbar_kw)
    cbar.ax.set_ylabel(cbarlabel, rotation=-90, va="bottom")

    # We want to show all ticks...
    ax.set_xticks(np.arange(data.shape[1]))
    ax.set_yticks(np.arange(data.shape[0]))
    # ... and label them with the respective list entries.
    ax.set_xticklabels(col_labels)
    ax.set_yticklabels(row_labels)

    # Let the horizontal axes labeling appear on top.
    ax.tick_params(top=True, bottom=False,
                   labeltop=True, labelbottom=False)

    # Rotate the tick labels and set their alignment.
    plt.setp(ax.get_xticklabels(), rotation=-30, ha="right",
             rotation_mode="anchor")

    # Turn spines off and create white grid.
    for edge, spine in ax.spines.items():
        spine.set_visible(False)

    ax.set_xticks(np.arange(data.shape[1] + 1) - .5, minor=True)
    ax.set_yticks(np.arange(data.shape[0] + 1) - .5, minor=True)
    ax.grid(which="minor", color="w", linestyle='-', linewidth=3)
    ax.tick_params(which="minor", bottom=False, left=False)

    return im, cbar

def annotate_heatmap(im, data=None, valfmt="{x:.2f}",
                     textcolors=["black", "white"],
                     threshold=None, **textkw):
    """
    A function to annotate a heatmap.

    Parameters
    ----------
    im
        The AxesImage to be labeled.
    data
        Data used to annotate.  If None, the image's data is used.  Optional.
    valfmt
        The format of the annotations inside the heatmap.  This should either
        use the string format method, e.g. "$ {x:.2f}", or be a
        `matplotlib.ticker.Formatter`.  Optional.
    textcolors
        A list or array of two color specifications.  The first is used for
        values below a threshold, the second for those above.  Optional.
    threshold
        Value in data units according to which the colors from textcolors are
        applied.  If None (the default) uses the middle of the colormap as
        separation.  Optional.
    **kwargs
        All other arguments are forwarded to each call to `text` used to create
        the text labels.
    """

    if not isinstance(data, (list, np.ndarray)):
        data = im.get_array()

    # Normalize the threshold to the images color range.
    if threshold is not None:
        threshold = im.norm(threshold)
    else:
        threshold = im.norm(data.max()) / 2.

    # Set default alignment to center, but allow it to be
    # overwritten by textkw.
    kw = dict(horizontalalignment="center",
              verticalalignment="center")
    kw.update(textkw)

    # Get the formatter in case a string is supplied
    if isinstance(valfmt, str):
        valfmt = matplotlib.ticker.StrMethodFormatter(valfmt)

    # Loop over the data and create a `Text` for each "pixel".
    # Change the text's color depending on the data.
    texts = []
    for i in range(data.shape[0]):
        for j in range(data.shape[1]):
            kw.update(color=textcolors[int(im.norm(data[i, j]) > threshold)])
            text = im.axes.text(j, i, valfmt(data[i, j], None), **kw)
            texts.append(text)

    return texts

def plot_gaussian_2D(mu, lmbda, color='b', centermarker=True,label='',alpha=1.,ax=None,artists=None):
    '''
    Plots mean and cov ellipsoid into current axes. Must be 2D. lmbda is a covariance matrix.
    '''
    assert len(mu) == 2
    ax = ax if ax else plt.gca()

    # TODO use artists!

    t = np.hstack([np.arange(0,2*np.pi,0.01),0])
    circle = np.vstack([np.sin(t),np.cos(t)])
    ellipse = np.dot(np.linalg.cholesky(lmbda),circle)

    if artists is None:
        point = ax.scatter([mu[0]],[mu[1]],marker='D',color=color,s=4,alpha=alpha) \
                if centermarker else None
        line, = ax.plot(ellipse[0,:] + mu[0], ellipse[1,:] + mu[1],linestyle='-',
                linewidth=2,color=color,label=label,alpha=alpha)
    else:
        line, point = artists
        if centermarker:
            point.set_offsets(np.atleast_2d(mu))
        line.set_xdata(ellipse[0,:] + mu[0])
        line.set_ydata(ellipse[1,:] + mu[1])
        line.set_alpha(alpha)
        line.set_color(color)

    return line, point

def plot_gaussian_projection(mu, lmbda, vecs, **kwargs):
    '''
    Plots a ndim gaussian projected onto 2D vecs, where vecs is a matrix whose two columns
    are the subset of some orthonomral basis (e.g. from PCA on samples).
    '''
    return plot_gaussian_2D(project_data(mu,vecs),project_ellipsoid(lmbda,vecs),**kwargs)

def pca_project_data(data,num_components=2):
    # convenience combination of the next two functions
    return project_data(data,pca(data,num_components=num_components))

def pca(data,num_components=2):
    U,s,Vh = np.linalg.svd(cov(data))
    return Vh.T[:,:num_components]

def project_data(data,vecs):
    return np.dot(data,vecs)

def project_ellipsoid(ellipsoid,vecs):
    # vecs is a matrix whose columns are a subset of an orthonormal basis
    # ellipsoid is a pos def matrix
    return np.dot(vecs,np.dot(ellipsoid,vecs.T))

def subplot_gridsize(num):
    return sorted(min([(x,int(np.ceil(num/x))) for x in range(1,int(np.floor(np.sqrt(num)))+1)],key=sum))

max_stars_repo_name: dpaysan/pyhsmm

max_issues_repo_name: dpaysan/pyhsmm

max_forks_repo_name: dpaysan/pyhsmm


text:
import re
import json
import click
import numpy as np
import pandas as pd
from scipy.linalg import norm
from utils import initialise_logger, multicore_apply

from umap import UMAP

def create_vector_data(word_vectors):

    with open(word_vectors, 'r') as f:

        vector_data = f.read().strip().split('\n')
        rows, dims = vector_data[0].split()

        vector_data = [(word, np.array([float(v) for v in vec.split()]))
                       for word, vec in map(lambda s: s.split(' ', 1), vector_data[1:])]
        vector_data = pd.DataFrame(vector_data)

        vector_data.columns = ['word', 'vector']

        return vector_data

def create_word_counts(word_counts):

    result = []
    with open(word_counts, 'r') as f:
        for line in f.read().strip().split('\n'):
            count, word = line.strip().split()
            result.append({'word': word, 'count': int(count)})

    return pd.DataFrame.from_dict(result)

def calculate_umap(params, umap_data, similarity_matrix):
    neighbors, dist = params

    reducer = UMAP(n_neighbors = neighbors, min_dist = dist, random_state = 42)
    reduced = reducer.fit_transform(similarity_matrix)

    umap_data['n_neighbors'] = neighbors
    umap_data['min_dist'] = dist

    umap_data['x'] = reduced[:, 0]
    umap_data['y'] = reduced[:, 1]

    umap_data['x'] = umap_data['x'] / (umap_data.x.max() - umap_data.x.min())
    umap_data['y'] = umap_data['y'] / (umap_data.y.max() - umap_data.y.min())

    umap_data = umap_data[['n_neighbors', 'min_dist', 'word', 'count', 'x', 'y']]

    umap_data['rank'] = np.arange(len(umap_data))

    umap_data = umap_data[umap_data.word != '</s>']

    umap_data['x'] = umap_data.x - umap_data.x.mean()
    umap_data['y'] = umap_data.y - umap_data.y.mean()

    umap_data.columns = ['n_neighbors', 'min_dist', 'word', 'word_count', 'x_coord', 'y_coord', 'rank']

    return umap_data

@click.command()
@click.option('--word_vectors', help='Path to fasttext.vec')
@click.option('--word_counts', help='Path to word_counts.txt')
@click.option('--umap_file', help='Path to save umap.json')
@click.option('--n_neighbours', type=int, help='This parameter controls how UMAP balances local versus global structure in the data.')
@click.option('--min_dist', type=float, help='Controls how tightly UMAP is allowed to pack points together')
@click.option('--log_level', default='INFO', help='Log level (default: INFO)')
def main(word_vectors, word_counts, umap_file, n_neighbours, min_dist, log_level):

    global logger
    logger = initialise_logger(log_level, __file__)

    logger.info('Creating vector_data..')
    vector_data = create_vector_data(word_vectors)
    word_counts = create_word_counts(word_counts)
    vector_data = vector_data.merge(word_counts, on = 'word', how = 'inner')

    logger.info('Computing similarity matrix..')
    normalize = lambda x: x / norm(x)
    word_vectors = np.vstack(vector_data.vector.apply(normalize))
    similarity_matrix = np.dot(word_vectors, word_vectors.transpose())

    logger.info('Running umap model..')
    umap_data = calculate_umap([n_neighbours, min_dist], vector_data, similarity_matrix)

    logger.info('Saving output to {}'.format(umap_file))
    umap_data.to_csv(umap_file, index = False)

    logger.info('Done!')

if __name__ == '__main__':
    main()

max_stars_repo_name: dragonfly-science/papers-past

max_issues_repo_name: dragonfly-science/papers-past

max_forks_repo_name: dragonfly-science/papers-past

reason: import numpy,from scipy


text:
# coding=utf-8
"""Visualize the ActEV world coordinates."""

import argparse
import cv2
import os
import sys
import glob
import tqdm
import numpy as np

parser = argparse.ArgumentParser()
parser.add_argument("traj_path")
parser.add_argument("h_path", help="path to all the homography matrix")
parser.add_argument("vis_path_world")
parser.add_argument("vis_path_img")

def plot_traj(img, traj, color):
  """Plot arrowred trajectory."""
  # traj is [N, 2]
  points = zip(traj[:-1], traj[1:])
  for p1, p2 in points:
    img = cv2.arrowedLine(img, tuple(p1), tuple(p2), color=color, thickness=1,
                          line_type=cv2.LINE_AA, tipLength=0.2)

  return img

def get_scene(videoname_):
  """ActEV scene extractor from videoname."""
  s = videoname_.split("_S_")[-1]
  s = s.split("_")[0]
  return s[:4]

def world_to_img(w_xy, h_):
  """Transform image xy to world ground plane xy."""
  w_x, w_y = w_xy
  img_x, img_y, img_z = np.tensordot(
      h_, np.array([w_x, w_y, 1]), axes=1)
  return img_x / img_z, img_y / img_z

if __name__ == "__main__":
  args = parser.parse_args()

  delim = "\t"
  h, w = 1080, 1920

  if not os.path.exists(args.vis_path_img):
    os.makedirs(args.vis_path_img)
  if not os.path.exists(args.vis_path_world):
    os.makedirs(args.vis_path_world)

  h_files = glob.glob(os.path.join(args.h_path, "*.txt"))
  h_dict = {}  # image to world coordinates
  for h_file in h_files:
    scene = os.path.splitext(os.path.basename(h_file))[0]
    h_matrix = []
    with open(h_file, "r") as f:
      for line in f:
        h_matrix.append(line.strip().split(","))
    h_matrix = np.array(h_matrix, dtype="float")

    _, h_dict[scene] = cv2.invert(h_matrix)  # world to image

  traj_files = glob.glob(os.path.join(args.traj_path, "*.txt"))

  for traj_file in tqdm.tqdm(traj_files):
    videoname = os.path.splitext(os.path.basename(traj_file))[0]
    scene = get_scene(videoname)

    # all trajectory in one image
    vis_world = np.zeros((h, w, 3), dtype="uint8")
    vis_img = np.zeros((h, w, 3), dtype="uint8")

    data = []
    with open(traj_file, "r") as f:
      for line in f:
        data.append(line.strip().split(delim))
    # [frameIdx, personId, x, y]
    data = np.array(data, dtype="float32")

    # normalize the world coordinates to plot on h, w
    min_x = np.amin(np.array(data)[:, 2])
    max_x = np.amax(np.array(data)[:, 2])
    min_y = np.amin(np.array(data)[:, 3])
    max_y = np.amax(np.array(data)[:, 3])
    length_x = max_x - min_x
    length_y = max_y - min_y

    personIds = np.unique(data[:, 1]).tolist()

    for personId in personIds[:10]:

      all_points = data[data[:, 1] == personId, :]

      # sort according to frameidx
      all_points = all_points[np.argsort(all_points[:, 0]), :]

      all_points_world = all_points.copy()
      all_points_world[:, 2] = w * (all_points_world[:, 2] - min_x) / length_x
      all_points_world[:, 3] = h * (all_points_world[:, 3] - min_y) / length_y
      vis_world = plot_traj(vis_world, all_points_world[:, 2:], (0, 255, 0))

      all_points_img = np.array(
          [world_to_img(point, h_dict[scene]) for point in all_points[:, 2:]],
          dtype="float32")

      vis_img = plot_traj(vis_img, all_points_img, (0, 255, 0))

    target_file = os.path.join(args.vis_path_img, "%s.png" % (videoname))
    cv2.imwrite(target_file, vis_img)
    target_file = os.path.join(args.vis_path_world, "%s.png" % (videoname))
    cv2.imwrite(target_file, vis_world)

max_stars_repo_name: JunweiLiang/next-prediction

max_issues_repo_name: JunweiLiang/next-prediction

max_forks_repo_name: JunweiLiang/next-prediction


text:
import matplotlib.pyplot as plt
import matplotlib.ticker as plticker
import numpy as np
from scipy.interpolate import make_interp_spline

def create_graph(
    data,
    usercolor,
    title=None,
    dimensions=(6, 3),
    draw=False,
    background_color="#2f3136",
):
    plt.rcParams["figure.figsize"] = [dimensions[0], dimensions[1]]
    T = np.array(list(range(0, len(data))))
    xnew = np.linspace(T.min(), T.max(), 240)
    spl = make_interp_spline(T, data, k=3)
    power_smooth = spl(xnew)

    # remove under 0
    power = []
    for x in power_smooth:
        power.append(min(x, 0))

    power_smooth = np.array(power)

    fig = plt.figure()
    fig.patch.set_facecolor(background_color)
    if title is not None:
        fig.suptitle(title, color="white")
    plt.autoscale(tight=True)
    plt.plot(xnew, power_smooth, color=usercolor)
    ax = plt.gca()
    loc = plticker.MultipleLocator(base=1.0)
    ax.xaxis.set_major_locator(loc)
    ax.set_facecolor(background_color)
    ax.set_xlabel("Hour (UTC)")
    # ax.set_ylabel('XP gain')

    ax.spines["bottom"].set_color("white")
    ax.spines["left"].set_color("white")
    ax.xaxis.label.set_color("white")
    ax.yaxis.label.set_color("white")
    ax.spines["right"].set_visible(False)
    ax.spines["top"].set_visible(False)
    ax.tick_params(axis="x", colors="white")
    ax.tick_params(axis="y", colors="white")
    plt.fill_between(xnew, power_smooth, color=usercolor, alpha=0.2)
    if draw:
        plt.show()
    plt.savefig("downloads/graph.png", facecolor=background_color, bbox_inches="tight")
    plt.close()

def time_series_graph(frame, data, color, background_color="#2f3136"):
    x = np.array(frame)
    y = np.array(data)
    fig = plt.figure()
    fig.patch.set_facecolor(background_color)
    plt.autoscale(tight=True)
    plt.plot(x, y, color=color)

    ax = plt.gca()
    ax.set_facecolor(background_color)

    ax.spines["bottom"].set_color("white")
    ax.spines["left"].set_color("white")
    ax.xaxis.label.set_color("white")
    ax.yaxis.label.set_color("white")
    ax.spines["right"].set_visible(False)
    ax.spines["top"].set_visible(False)
    ax.tick_params(axis="x", colors="white")
    ax.tick_params(axis="y", colors="white")
    ax.ticklabel_format(useOffset=False, style="plain", axis="y")
    ax.get_yaxis().get_major_formatter().set_useOffset(False)
    ax.get_yaxis().get_major_formatter().set_scientific(False)
    plt.xticks(rotation=45)

    plt.savefig("downloads/graph.png", facecolor=background_color, bbox_inches="tight")
    plt.close()

max_stars_repo_name: Keyruu/miso-bot

max_issues_repo_name: Keyruu/miso-bot

max_forks_repo_name: Keyruu/miso-bot


text:
# -*- coding: utf-8 -*-
"""

This module contains classes for sub-selecting features or samples from given
datasets using the CUR decomposition method. Each class supports a Principal
Covariates Regression (PCov)-inspired variant, using a mixing parameter and
target values to bias the selections.

Authors: Rose K. Cersonsky
         Michele Ceriotti

"""

from abc import abstractmethod

import numpy as np
from scipy.sparse.linalg import eigs as speig

from sklearn.utils import check_X_y, check_array
from skcosmo.pcovr.pcovr_distances import pcovr_covariance, pcovr_kernel_distance
from .orthogonalizers import feature_orthogonalizer, sample_orthogonalizer

class _BaseCUR:
    """
    Base class for CUR selection methods
    Requires a product, typically the gram or covariance matrix, \
    from which to compute the importance score

    If the model is iterative, the orthogonalize method must be overwritten
    to orthogonalize the input matrices after each iteration.

    :param iterative: whether to orthogonalize after each selection,
                      defaults to `true`
    :type iterative: boolean

    :param k: number of eigenvectors to compute the importance score with,
              defaults to 1
    :type k: int

    :param tol: threshold below which values will be considered 0,
                      defaults to 1E-12
    :type tol: float
    """

    def __init__(self, iterative=True, tol=1e-12, k=1):

        self.k = k
        self.iter = iterative
        self.tol = tol
        self.idx = []
        self.pi = []

    def select(self, n):
        """Method for CUR select based upon a product of the input matrices

        Parameters
        ----------
        n : number of selections to make, must be > 0

        Returns
        -------
        idx: list of n selections
        """

        if n <= 0:
            raise ValueError("You must call select(n) with n > 0.")

        if len(self.idx) > n:
            return self.idx[:n]

        for i in range(len(self.idx), n):
            if self.iter:
                v, U = speig(self.product, k=self.k, tol=self.tol)
                U = U[:, np.flip(np.argsort(v))]
                pi = (np.real(U)[:, : self.k] ** 2.0).sum(axis=1)
            pi[self.idx] = 0.0
            self.idx.append(pi.argmax())
            self.pi.append(max(pi))

            self.orthogonalize()
            self.product = self.get_product()

            if np.isnan(self.product).any():
                print(
                    f"The product matrix has rank {i}. "
                    + f"n_select reduced from {n} to {i}."
                )
                return self.idx

        return self.idx

    @abstractmethod
    def get_product(self):
        """
        Abstract method for computing the inner or outer product of the
        input matrices
        """
        return

    @abstractmethod
    def orthogonalize(self):
        """Method to orthogonalize matrices"""
        return

class SampleCUR(_BaseCUR):
    """
    For sample selection, the importance score :math:`\\pi` is the sum over
    the squares of the first :math:`k` components of the left singular vectors

    .. math::

        \\pi_j =
        \\sum_i^k \\left(\\mathbf{U}_\\mathbf{\\tilde{K}}\\right)_{ij}^2.

    where :math:`{\\mathbf{\\tilde{K}} = \\alpha \\mathbf{XX}^T +
    (1 - \\alpha)\\mathbf{\\hat{Y}\\hat{Y}}^T}` for some mixing parameter
    :math:`{\\alpha}`. When :math:`{\\alpha = 1}`, this defaults to the Gram
    matrix :math:`{\\mathbf{K} = \\mathbf{X}\\mathbf{X}^T}`.

    :param iterative: whether to orthogonalize after each selection,
                      defaults to `true`
    :type iterative: boolean

    :param k: number of eigenvectors to compute the importance score with,
              defaults to 1
    :type k: int

    :param X: Data matrix :math:`\\mathbf{X}` from which to select a
                   subset of the `n` rows
    :type X: array of shape (n x m)

    :param mixing: mixing parameter,
                   as described in PCovR as :math:`{\\alpha}`, defaults to 1
    :type mixing: float

    :param tol: threshold below which values will be considered 0,
                      defaults to 1E-12
    :type tol: float

    :param Y: array to include in biased selection when mixing < 1; required
              when mixing < 1, throws AssertionError otherwise
    :type Y: array of shape (n x p), optional when :math:`{\\alpha = 1}`

    """

    def __init__(self, X, mixing=1.0, iterative=True, tol=1e-12, Y=None):
        super().__init__(iterative=iterative, tol=tol)

        self.mixing = mixing

        if mixing < 1:
            try:
                self.A, self.Y = check_X_y(X, Y, copy=True, multi_output=True)
            except AssertionError:
                raise Exception(r"For $\alpha < 1$, $Y$ must be supplied.")
        else:
            self.A, self.Y = check_array(X, copy=True), None

        if not self.iter:
            self.A_current = None
            self.Y_current = None
        else:
            self.A_current = self.A.copy()
            if self.Y is not None:
                self.Y_current = self.Y.copy()
            else:
                self.Y_current = None

        self.product = self.get_product()

    def get_product(self):
        """
        Computes the modified PCovR Gram Matrix
        :math:`{\\mathbf{\\tilde{K}} = \\alpha \\mathbf{XX}^T +
        (1 - \\alpha)\\mathbf{\\hat{Y}\\hat{Y}}^T}`
        """
        return pcovr_kernel_distance(
            self.mixing, self.A_current, self.Y_current, self.tol
        )

    def orthogonalize(self):
        """
        Orthogonalizes the remaining samples by those already selected

        .. math::
            \\mathbf{X} \\leftarrow \\mathbf{X} -
            \\mathbf{X} \\left(\\frac{\\mathbf{x}_{r}^T \\mathbf{x}_{r}}
                                {\\lVert \\mathbf{x}_{r}\\rVert^2}\\right)`.

        When `mixing < 1`, this also includes orthogonalization of
        :math:`\\mathbf{Y}`

        .. math::
            \\mathbf{Y} \\leftarrow \\mathbf{Y} -
            \\mathbf{X} \\left(\\mathbf{X}_\\mathbf{r}^T
            \\mathbf{X}_\\mathbf{r}\\right)^{-1}
            \\mathbf{X}_\\mathbf{r}^T \\mathbf{Y}_\\mathbf{r}`.

        """
        if self.iter:
            self.A_current, self.Y_current = sample_orthogonalizer(
                self.idx, self.A_current, self.Y_current, self.tol
            )

class FeatureCUR(_BaseCUR):
    """
    For feature selection, the importance score :math:`\\pi` is the sum over
    the squares of the first :math:`k` components of the right singular vectors

    .. math::

        \\pi_j =
        \\sum_i^k \\left(\\mathbf{U}_\\mathbf{\\tilde{C}}\\right)_{ij}^2.

    where :math:`{\\mathbf{\\tilde{C}} = \\alpha \\mathbf{X}^T\\mathbf{X} +
    (1 - \\alpha)(\\mathbf{X}^T\\mathbf{X})^{-1/2}\\mathbf{X}^T
    \\mathbf{\\hat{Y}\\hat{Y}}^T\\mathbf{X}(\\mathbf{X}^T\\mathbf{X})^{-1/2}}`
    for some mixing parameter :math:`{\\alpha}`. When :math:`{\\alpha = 1}`,
    this defaults to the covariance matrix
    :math:`{\\mathbf{C} = \\mathbf{X}^T\\mathbf{X}}`.

    :param iterative: whether to orthogonalize after each selection,
                      defaults to `true`
    :type iterative: boolean

    :param k: number of eigenvectors to compute the importance score with,
              defaults to 1
    :type k: int

    :param X: Data matrix :math:`\\mathbf{X}` from which to select a
                   subset of the `n` columns
    :type X: array of shape (n x m)

    :param mixing: mixing parameter, as described in PCovR as
                   :math:`{\\alpha}`, defaults to 1
    :type mixing: float

    :param tol: threshold below which values will be considered 0,
                      defaults to 1E-12
    :type tol: float

    :param Y: array to include in biased selection when mixing < 1;
              required when mixing < 1, throws AssertionError otherwise
    :type Y: array of shape (n x p), optional when :math:`{\\alpha = 1}`
    """

    def __init__(self, X, mixing=1.0, iterative=True, tol=1e-12, Y=None):
        super().__init__(iterative=iterative, tol=tol)

        self.mixing = mixing

        if mixing < 1:
            try:
                self.A, self.Y = check_X_y(X, Y, copy=True, multi_output=True)
            except AssertionError:
                raise Exception(r"For $\alpha < 1$, $Y$ must be supplied.")
        else:
            self.A, self.Y = check_array(X, copy=True), None

        if not self.iter:
            self.A_current = None
            self.Y_current = None
        else:
            self.A_current = self.A.copy()
            if self.Y is not None:
                self.Y_current = self.Y.copy()
            else:
                self.Y_current = None

        self.product = self.get_product()

    def get_product(self):
        """
        Computes the modified PCovR Covariance Matrix
        :math:`{\\mathbf{\\tilde{C}} = \\alpha \\mathbf{X}^T\\mathbf{X} +
        (1 - \\alpha)(\\mathbf{X}^T\\mathbf{X})^{-1/2}\\mathbf{X}^T
        \\mathbf{\\hat{Y}\\hat{Y}}^T\\mathbf{X}(\\mathbf{X}^T
        \\mathbf{X})^{-1/2}}`
        """
        return pcovr_covariance(self.mixing, self.A_current, self.Y_current, self.tol)

    def orthogonalize(self):
        """
        Orthogonalizes the remaining features by those already selected, such
        that

        :math:`{\\mathbf{X} \\leftarrow \\mathbf{X} -
        \\left(\\frac{\\mathbf{X}_{c}\\mathbf{X}_{c}^T\\cdot}
        {\\lVert\\mathbf{X}_{c}\\rVert^2}\\right)\\mathbf{X}}`.

        When `mixing < 1`, this also includes orthogonalization of
        :math:`\\mathbf{Y}`

        :math:`{\\mathbf{Y} \\leftarrow \\mathbf{Y} - \\mathbf{X}_\\mathbf{c}
        \\left(\\mathbf{X}_\\mathbf{c}^T \\mathbf{X}_\\mathbf{c}\\right)^{-1}
        \\mathbf{X}_\\mathbf{c}^T \\mathbf{Y}}`.

        """
        if self.iter:
            self.A_current, self.Y_current = feature_orthogonalizer(
                self.idx, self.A_current, self.Y_current, self.tol
            )

max_stars_repo_name: Luthaf/sklearn-cosmo

max_issues_repo_name: Luthaf/sklearn-cosmo

max_forks_repo_name: Luthaf/sklearn-cosmo


text:
from ..common.Canny import *

from PIL import Image
import numpy as np
from copy import deepcopy
from bitstring import BitStream
from struct import unpack
from PIL import ImageFont
from PIL import Image
from PIL import ImageDraw

def getFirstCPIbkg(dataBuff, startLoc, workArray, dispArray, ptDatCPI, hkDatCPI, nextSlice, actCol, actSlice, stampNum, stampLoc, stampContent, targetFileName):
    readParticles = 0
    Location = startLoc
    findType = -1
    numHSlice = -1
    numVSlice = -1
    ptDatCPI.nSli = 0
    ptDatCPI.nCol = 0
    ptDatCPI.HPtC = 0
    ptDatCPI.VPtC = 0
    stillInBuffer = 1
    while stillInBuffer:
        if dataBuff[Location] == '\xD5':
            if dataBuff[Location+1] == '\xA3':
                findType = 1
                thisFrameIsBackground = 0
                s = BitStream(bytes=dataBuff[Location+20:Location+22])
                hkDatCPI.BkBf, hkDatCPI.FROI, hkDatCPI.FstI, hkDatCPI.NewD, hkDatCPI.bkSb, hkDatCPI.iFull, hkDatCPI.isBk, hkDatCPI.imPr = s.readlist('8*bool')
                hkDatCPI.fStX = unpack("h",dataBuff[Location+22:Location+24])[0]      #12
                hkDatCPI.fStY = unpack("h",dataBuff[Location+24:Location+26])[0]      #13
                hkDatCPI.fEnX = unpack("h",dataBuff[Location+26:Location+28])[0]      #14
                hkDatCPI.fEnY = unpack("h",dataBuff[Location+28:Location+30])[0]      #15
                if hkDatCPI.isBk and hkDatCPI.imPr and hkDatCPI.fStX == 0 and fStY == 0 and hkDatCPI.fEnX > 1022 and hkDatCPI.fEnY > 1021:
                    thisFrameIsBackground = 1

# This should hopefully be thread safe for future use if we carefully divide dataBuff
# equally between threads. In the main application below startLoc is hard wired to
# value zero, but this provides for the DIFF files produced by CERN. This should be
#retained as a useful threading parameter.
def readCPIImageFrame(dataBuff, startLoc, workArray, dispArray, ptDatCPI, hkDatCPI, nextSlice, actCol, actSlice, stampNum, stampLoc, stampContent, targetFileName, currentCPIBackGround):
    bufferSize = len(dataBuff)
    readParticles = 0
    Location = startLoc
    findType = -1
    numHSlice = -1
    numVSlice = -1
    ptDatCPI.nSli = 0
    ptDatCPI.nCol = 0
    ptDatCPI.HPtC = 0
    ptDatCPI.VPtC = 0

    findType = 1
    thisFrameIsBackground = 0
    hkDatCPI.fByt = unpack("I",dataBuff[Location+2:Location+6])[0]        #02 - Total bytes in image block
    hkDatCPI.fTyp = unpack("h",dataBuff[Location+6:Location+8])[0]/10.0   #03 - Image data block version number
    hkDatCPI.nROI = unpack("h",dataBuff[Location+8:Location+10])[0]       #04 - Number of ROIs contained in block
    hkDatCPI.nByt = unpack("I",dataBuff[Location+10:Location+14])[0]      #05 - Total bytes of image data in block
    hkDatCPI.iDay = unpack("B",dataBuff[Location+14:Location+15])[0]      #06 - Day
    hkDatCPI.iHou = unpack("B",dataBuff[Location+15:Location+16])[0]      #07 - Hour
    hkDatCPI.iMin = unpack("B",dataBuff[Location+16:Location+17])[0]      #08 - Minute
    hkDatCPI.iSec = unpack("B",dataBuff[Location+17:Location+18])[0]      #09 - Second
    hkDatCPI.iMsc = unpack("h",dataBuff[Location+18:Location+20])[0]      #10 - Millisecond
    hkDatCPI.iTyt = unpack("h",dataBuff[Location+20:Location+22])[0]      #11 - Image type - see s.readlist below
    s = BitStream(bytes=dataBuff[Location+20:Location+22])
    hkDatCPI.fStX = unpack("h",dataBuff[Location+22:Location+24])[0]      #12 - ROI top left corner X co-ordinate
    hkDatCPI.fStY = unpack("h",dataBuff[Location+24:Location+26])[0]      #13 - ROI top left corner Y co-ordinate
    hkDatCPI.fEnX = unpack("h",dataBuff[Location+26:Location+28])[0]      #14 - ROI lower right corner X co-ordinate
    hkDatCPI.fEnY = unpack("h",dataBuff[Location+28:Location+30])[0]      #15 - ROI lower right corner Y co-ordinate
#    hkDatCPI.imPr, hkDatCPI.isBk, hkDatCPI.iFull, hkDatCPI.bkSb, hkDatCPI.NewD, hkDatCPI.FstI, hkDatCPI.FROI, hkDatCPI.BkBf = s.readlist('8*bool')
    hkDatCPI.BkBf, hkDatCPI.FROI, hkDatCPI.FstI, hkDatCPI.NewD, hkDatCPI.bkSb, hkDatCPI.iFull, hkDatCPI.isBk, hkDatCPI.imPr = s.readlist('8*bool')
    printBkgState = 0
    if printBkgState:
        imPres = "        "
        isBCK = "        "
        isFull = "        "
        isSubd = "        "
        isNDat = "        "
        isFirst = "        "
        isFoundROI = "        "
        isBckBuff = "        "
        if hkDatCPI.imPr:
            imPres = "ImagePr "
        if hkDatCPI.isBk:
            isBCK = "BackGnd "
        if hkDatCPI.iFull:
            isFull = "FullFra "
        if hkDatCPI.bkSb:
            isSubd = "BkgSubd "
        if hkDatCPI.NewD:
            isNDat = "NewData "
        if hkDatCPI.FstI:
            isFirst = "FirstIm "
        if hkDatCPI.FROI:
            isFoundROI = "GotROI  "
        if hkDatCPI.BkBf:
            isBckBuff = "BkgBuff "
        if hkDatCPI.isBk:
            print imPres, isBCK, isFull, isSubd, isNDat, isFirst, isFoundROI, isBckBuff, hkDatCPI.fStX, hkDatCPI.fStY, hkDatCPI.fEnX, hkDatCPI.fEnY
    if hkDatCPI.isBk and hkDatCPI.imPr and hkDatCPI.fStX == 0 and hkDatCPI.fStY == 0 and hkDatCPI.fEnX > 1022 and hkDatCPI.fEnY > 1021:
        thisFrameIsBackground = 1
    hkDatCPI.bkgR = unpack("h",dataBuff[Location+30:Location+32])[0]/10.0 #16 - Background rate in tenths of seconds
    hkDatCPI.nPDS = unpack("h",dataBuff[Location+32:Location+34])[0]      #17 - Maximum number of PDS strobes to tolerate for background to be taken
    hkDatCPI.tNFP = unpack("I",dataBuff[Location+34:Location+38])[0]      #18 - Total number of frames processed so far
    hkDatCPI.thrs = unpack("B",dataBuff[Location+38:Location+39])[0]      #19 - Image threshold (0 to 255)
    hkDatCPI.rErr = unpack("B",dataBuff[Location+39:Location+40])[0]      #20 - ROI error - rejection failure code
    hkDatCPI.mnes = unpack("h",dataBuff[Location+40:Location+42])[0]      #21 - Minimum ROI size
    hkDatCPI.asRt = unpack("f",dataBuff[Location+42:Location+46])[0]      #22 - Aspect ratio rejection criterion
    hkDatCPI.flRt = unpack("f",dataBuff[Location+46:Location+50])[0]      #23 - Minimum ROI fill ratio
    hkDatCPI.pCnt = unpack("I",dataBuff[Location+50:Location+54])[0]      #24 - Number of pixels in ROI above threshold
    hkDatCPI.imMn = unpack("B",dataBuff[Location+54:Location+55])[0]      #25 - Image mean intensity
    hkDatCPI.bkMn = unpack("B",dataBuff[Location+55:Location+56])[0]      #26 - Background mean intensity
    hkDatCPI.spr1 = unpack("h",dataBuff[Location+56:Location+58])[0]      #27 - SPARE 1
    hkDatCPI.xPad = unpack("h",dataBuff[Location+58:Location+60])[0]      #28 - ROI horizontal border pad
    hkDatCPI.yPad = unpack("h",dataBuff[Location+60:Location+62])[0]      #29 - ROI vertical border pad
    hkDatCPI.sCnt = unpack("I",dataBuff[Location+62:Location+66])[0]      #30 - Strobe count (per image basis)
    hkDatCPI.fSav = unpack("I",dataBuff[Location+66:Location+70])[0]      #31 - Number of frames saved to disc so far
    hkDatCPI.minV = unpack("B",dataBuff[Location+70:Location+71])[0]      #32 - Minimum image mean for an acceptable frame
    hkDatCPI.maxV = unpack("B",dataBuff[Location+71:Location+72])[0]      #33 - Maximum image mean for acceptable image
    hkDatCPI.rSav = unpack("I",dataBuff[Location+72:Location+76])[0]      #34 - Number of ROIs saved so far
    hkDatCPI.PCsF = unpack("h",dataBuff[Location+76:Location+78])[0]      #35 - PDS checksum flag
    hkDatCPI.PDH1 = unpack("h",dataBuff[Location+78:Location+80])[0]      #36 - PDS header 1
    hkDatCPI.PDH2 = unpack("h",dataBuff[Location+80:Location+82])[0]      #37 - PDS header 2
    hkDatCPI.PDH3 = unpack("h",dataBuff[Location+82:Location+84])[0]      #38 - PDS header 3
    hkDatCPI.tSOY = unpack("I",dataBuff[Location+84:Location+88])[0]      #39 - Seconds into current year
    hkDatCPI.PAT1 = unpack("h",dataBuff[Location+88:Location+90])[0]      #40 - Arrival time 1 - number of 62.5ms chunks into second
    hkDatCPI.PAT2 = unpack("h",dataBuff[Location+90:Location+92])[0]      #41 - Arrival time 2 - number of 0.8us chunks into 16th
    hkDatCPI.tTrn = unpack("h",dataBuff[Location+92:Location+94])[0]      #42 - Transit time
    hkDatCPI.Strs = unpack("h",dataBuff[Location+94:Location+96])[0]      #43 - Number of strobes since last PDS packet
    hkDatCPI.PI45 = unpack("h",dataBuff[Location+96:Location+98])[0]      #44 - Pulse height 45
    hkDatCPI.PI90 = unpack("h",dataBuff[Location+98:Location+100])[0]     #45 - Pulse height 90
    hkDatCPI.pdCS = unpack("h",dataBuff[Location+100:Location+102])[0]    #46 - PDS checksum
    hkDatCPI.pbMd = unpack("h",dataBuff[Location+102:Location+104])[0]    #47 - Probe mode <---- End of frame header

    frameStamp = "%02d" % hkDatCPI.iHou +":"+"%02d" % hkDatCPI.iMin +":"+"%02d" % hkDatCPI.iSec +":"+"%03d" % hkDatCPI.iMsc
    Location += 104
    isValidParticle = 0
# Read all ROIs contained in the image - need to import the ROI location list to this function and pop(0) entries from it
    if hkDatCPI.nROI > 0: print "nROI = ", hkDatCPI.nROI
    for pNum in range(hkDatCPI.nROI):
        print(pNum)
        isValidParticle, Location, actCol, actSlice, nextSlice, stampNum, stampLoc, stampContent, currentCPIBackGround = readCPIROIHeader(hkDatCPI, ptDatCPI, Location, dataBuff, workArray, dispArray, nextSlice, actCol, actSlice, stampNum, stampLoc, stampContent, frameStamp, targetFileName, thisFrameIsBackground, currentCPIBackGround)

    return findType, isValidParticle, hkDatCPI, workArray, ptDatCPI.nCol, ptDatCPI.nSli, Location, hkDatCPI.tSOY, actCol, actSlice, nextSlice, stampNum, stampLoc, stampContent, currentCPIBackGround

def readCPIROIHeader(hkDatCPI, ptDatCPI, Location, dataBuff, workArray, dispArray, nextSlice, actCol, actSlice, stampNum, stampLoc, stampContent, frameStamp, targetFileName, thisFrameIsBackground, currentCPIBackGround):
    hkDatCPI.rHed = unpack("H",dataBuff[Location:Location+2])[0]        #1 <---- Start of ROI header
    hkDatCPI.rByt = unpack("H",dataBuff[Location+2:Location+4])[0]      #2 -
    hkDatCPI.rUnk = unpack("H",dataBuff[Location+4:Location+6])[0]      #3 -
    hkDatCPI.rTyp = unpack("H",dataBuff[Location+6:Location+8])[0]/10.0 #4 -
    hkDatCPI.rStX = unpack("H",dataBuff[Location+8:Location+10])[0]     #5
    hkDatCPI.rStY = unpack("H",dataBuff[Location+10:Location+12])[0]    #6
    hkDatCPI.rEnX = unpack("H",dataBuff[Location+12:Location+14])[0]    #7
    hkDatCPI.rEnY = unpack("H",dataBuff[Location+14:Location+16])[0]    #8
    hkDatCPI.pByt = unpack("H",dataBuff[Location+16:Location+18])[0]    #9
    hkDatCPI.rFlg = unpack("H",dataBuff[Location+18:Location+20])[0]    #10
    hkDatCPI.rLen = unpack("H",dataBuff[Location+20:Location+22])[0]    #11
    hkDatCPI.sLen = unpack("H",dataBuff[Location+22:Location+24])[0]    #12
    hkDatCPI.eLen = unpack("H",dataBuff[Location+24:Location+26])[0]    #13
    hkDatCPI.rWid = unpack("H",dataBuff[Location+26:Location+28])[0]    #14
    hkDatCPI.sWid = unpack("H",dataBuff[Location+28:Location+30])[0]    #15
    hkDatCPI.eWid = unpack("H",dataBuff[Location+30:Location+32])[0]    #16
    hkDatCPI.Dark = unpack("H",dataBuff[Location+32:Location+34])[0]    #17
    hkDatCPI.Area = unpack("H",dataBuff[Location+34:Location+36])[0]    #18
    hkDatCPI.Prim = unpack("H",dataBuff[Location+36:Location+38])[0]    #19 <---- End of ROI header V25

    isValidParticle = 0
    if 0 < hkDatCPI.iDay <= 31 and 0 <= hkDatCPI.iHou < 24 and 0 <= hkDatCPI.iMin < 60 and 0 <= hkDatCPI.iSec < 60 and hkDatCPI.nByt > 0:
        isValidParticle = 1
    Location += 54

    ptDatCPI.nCol, ptDatCPI.nSli, actCol, actSlice, nextSlice, stampNum, stampLoc, stampContent, currentCPIBackGround = readCPIROI(hkDatCPI.rStX, hkDatCPI.rStY, hkDatCPI.rEnX, hkDatCPI.rEnY, dataBuff, workArray, dispArray, Location, nextSlice, actCol, actSlice, stampNum, stampLoc, stampContent, frameStamp, targetFileName, thisFrameIsBackground, currentCPIBackGround)

    Location += (ptDatCPI.nCol * ptDatCPI.nSli)
    print(ptDatCPI.nCol * ptDatCPI.nSli, Location)
    return isValidParticle, Location, actCol, actSlice, nextSlice, stampNum, stampLoc, stampContent, currentCPIBackGround

def readCPIROI(stX, stY, enX, enY, dataBuff, workArray, dispArray, Location, nextSlice, actCol, actSlice, stampNum, stampLoc, stampContent, frameStamp, targetFileName, thisFrameIsBackground, currentCPIBackGround):
    dX = enX - stX
    dY = enY - stY
    if stX < 0 or stY < 0 or enX > 1024 or enY > 1024 or dX < 1 or dY < 1:
        print "Particle size error: Starts",stX,stY,"ends:",enX,enY,"size:",dX,dY
        return 0, 0, actCol, actSlice, nextSlice, stampNum, stampLoc, stampContent, currentCPIBackGround
    nPix = (1 + dX) * (1 + dY)
    xLoc = 0
    yLoc = 0
    for i in xrange (0,nPix):
        pixVal = unpack("B",dataBuff[Location+i:Location+1+i])[0]
        workArray[yLoc,xLoc,0:3] = (pixVal,pixVal,pixVal)
        if xLoc == dX:
            xLoc = 0
            yLoc += 1
        else:
            xLoc += 1

    doCanny = 0
    doBkgSubtract = 0
    doMarkEdges = 1
    doMarkBox = 1
# If this is a background, store it.
    if thisFrameIsBackground:
        currentCPIBackGround[:,:,:] = workArray[:,:,:]
        print "Read new background"
# OK. Got the image out. Now we can start to process it.
    elif doCanny:
        if doBkgSubtract:
            workArray = bkgRawSubtract(workArray, currentCPIBackGround, stX, stY, dX, dY)
        sigma = 3
        lowThresh = 0.00002
        highThresh = 0.3
        edges, boxVals, floodArea = CannyEdge(workArray, dX, dY, sigma, lowThresh, highThresh, currentCPIBackGround, stX, stY)
#        print "Box: ", boxVals['area'], "Flood: ", floodArea, "Ratio: ", floodArea/boxVals['area']
        if doMarkEdges:
# Mark edges orange - RGB = (255,165,0)
            workArray[np.where(edges)] = (255,165,0)
        if doMarkBox and len(boxVals['corners']) > 3:
# Mark MinAreaRect green - RGB = (127,255,0)
            img = Image.fromarray(workArray)
            draw = ImageDraw.Draw(img)
            draw.polygon([tuple(p) for p in boxVals['corners']], fill=None, outline = (127,255,0))
            boxedData = np.asarray(img)
            workArray[:,:,:] = boxedData[:,:,:]
# Add the new image to the output sheet.
    if not thisFrameIsBackground:
        actSlice, actCol, nextSlice, stampNum, stampLoc, stampContent = addCPIImage(workArray, dispArray, dX, dY, nextSlice, actCol, actSlice, stampNum, stampLoc, stampContent, frameStamp, targetFileName)
    return dX, dY, actCol, actSlice, nextSlice, stampNum, stampLoc, stampContent, currentCPIBackGround

def bkgRawSubtract(image, bkg, xTL, yTL, dX, dY):
# JRD - This is fucking clumsy - need to do better - TRY - if abs(image-bkg)>someValue: image=image else: image=0
    for i in xrange(0,dY):
        for j in xrange(0,dX):
            if image[i, j, 0] - bkg[i, j, 0] > 0:
                image[i, j, :] = image[i, j, 0] - bkg[i, j, 0]
            else:
                image[i, j, :] = 0
    return image

# Exception to deal with (alleged) stereo particle events
class GetOutOfLoop(Exception):
     pass

# Set up frames and title in image display page
def setCPIBkg(workArray, bkgColor):
    workArray[:,:,] = bkgColor

def addCPIImage(workArray, dispArray, numCols, numSlices, nextSlice, actCol, actSlice, stampNum, stampLoc, stampContent, frameStamp, targetFileName):
    print "Adding image", numCols, " x ", numSlices
    if (actCol + numCols) <= 1714 and (nextSlice + numSlices) < 1230:
        stampContent.append(frameStamp)
        stampLoc[stampNum,0] = actCol
        stampLoc[stampNum,1] = actSlice + numSlices + 1
        stampNum += 1
        dispArray[actSlice : actSlice + numSlices, actCol : actCol + numCols, :] = workArray[0 : numSlices, 0 : numCols]
        actCol += 2 + numCols
        if 2 + 9 + actSlice + numSlices > nextSlice:
            nextSlice = 2 + 9 + actSlice + numSlices
        workArray[0:1024, 0:1024, 0:3] = 255
        return actSlice, actCol, nextSlice, stampNum, stampLoc, stampContent
    elif (actCol + numCols) > 1714 and (nextSlice + numSlices) < 1230:
        actCol = 40
        actSlice = nextSlice
        stampContent.append(frameStamp)
        stampLoc[stampNum,0] = actCol
        stampLoc[stampNum,1] = actSlice + numSlices + 1
        stampNum += 1
        dispArray[actSlice : actSlice + numSlices, actCol : actCol + numCols, :] = workArray[0 : numSlices, 0 : numCols]
        actCol += 2 + numCols
        if 2 + 9 + actSlice + numSlices > nextSlice:
            nextSlice = 2 + 9 + actSlice + numSlices
        workArray[0:1024, 0:1024, 0:3] = 255
        return actSlice, actCol, nextSlice, stampNum, stampLoc, stampContent
    else:
        plotCPIImage(dispArray, stampLoc, stampContent, stampNum, targetFileName)
        actCol = 40
        actSlice = 80
        nextSlice = 80
        setCPIBkg(dispArray, 255)
        stampNum = 0
        stampContent = []
        stampLoc[stampNum,0] = actCol
        stampLoc[stampNum,1] = actSlice + numSlices + 1
        stampContent.append(frameStamp)
        stampNum += 1
        dispArray[actSlice : actSlice + numSlices, actCol : actCol + numCols, :] = workArray[0 : numSlices, 0 : numCols]
        actCol += 2 + numCols
        if 2 + 9 + actSlice + numSlices > nextSlice:
            nextSlice = 2 + 9 + actSlice + numSlices
        workArray[0:1024, 0:1024, 0:3] = 255
        return actSlice, actCol, nextSlice, stampNum, stampLoc, stampContent

def plotCPIImage(dispArray, stampLoc, stampContent, stampNum, targetFileName):
    im=Image.Image()
    im=Image.fromarray(dispArray)
    for i in xrange (0,stampNum):
#        print i, stampContent[i], stampLoc[i,0], stampLoc[i,1]
        drawText(im,stampContent[i], stampLoc[i,0], stampLoc[i,1], 8)
    drawText(im,"CPI DATA - time range "+stampContent[0]+" to "+stampContent[stampNum-1], 40, 20, 24)
    drawText(im,"RAW FILE: "+targetFileName, 40, 45, 24)
    drawText(im,"Extracted using pyCPI - "+u"\u00A9"+" James Dorsey - University of Manchester - 2013", 1350, 25, 10)
    draw = ImageDraw.Draw(im)
    draw.line((1350, 50, 1393, 50), fill=0, width = 2)
    drawText(im,"100 "+u"\u03BC"+"m scale line", 1400, 43, 10)
    del draw
    imFileName = "output/" + stampContent[0]+"-"+stampContent[stampNum-1]+".png"
    imFileName = imFileName.replace(":", "")
    im.save(imFileName)
    print "Writing image file ", imFileName

# Direct copy of the function in pyCPI - move this to a common file
def drawText(img, textString, xLoc, yLoc, fontSize):
    font = ImageFont.truetype("/usr/share/fonts/truetype/ttf-dejavu/DejaVuSans.ttf",fontSize)
    draw = ImageDraw.Draw(img)
    draw.text((xLoc, yLoc),textString,(0,0,0),font=font)
    draw = ImageDraw.Draw(img)
    draw = ImageDraw.Draw(img)

class houseRecordCPI(object):
    '''ITERABLE data structure to receive CPI housekeeping data'''

    def __init__(self):
        self.pLen = 63905 #02 - Packet length (83)
        self.tFST = 63905 #03 - Forward sample tube temperature
        self.tUOB = 63905 #04 - Upper optics block temperature
        self.tLOB = 63905 #05 - Lower optics block temperature
        self.tCST = 63905 #06 - Central sample tube temperature
        self.tFBL = 63905 #07 - FibreLink temperature (old aft sample tube)
        self.tNCo = 63905 #08 - Nose cone temperarure
        self.tPy2 = 63905 #09 - Pylon temperature 2
        self.tPy3 = 63905 #10 - Pylon temperature 3
        self.tCCD = 63905 #11 - CCD camera temperature
        self.tILn = 63905 #12 - Imaging lens temperature
        self.fILs = 63905 #13 - Imaging laser temperature
        self.tLsV = 63905 #14 - PDS45 (or vertical) laser temperature
        self.tLsH = 63905 #15 - PDS90 (or horizontal) laser temperature
        self.tPBd = 63905 #16 - Power board temperature
        self.tVPl = 63905 #17 - PDS45 (or vertical) platen temperature
        self.tVOp = 63905 #18 - PDS45 optics temperature
        self.tHPl = 63905 #19 - PDS90 platen temperature
        self.tHOp = 63905 #20 - PDS90 optics temperature
        self.tVIM = 63905 #21 - PDS45 input mirror temperature
        self.tHIM = 63905 #22 - PDS90 input mirror temperature
        self.tAir = 63905 #23 - Internal air temperature
        self.tDSP = 63905 #24 - DSP card temperature
        self.tVAT = 63905 #25 - PDS45 array top temperature
        self.tVAB = 63905 #26 - PDS45 array bottom temperature
        self.tHAT = 63905 #27 - PDS90 array top temperature
        self.tHAB = 63905 #28 - PDS90 array bottom temperature
        self.relH = 63905 #29 - Relative humidity
        self.canP = 63905 #30 - Internal pressure - PSI
        self.VTcC = 63905 #31 - PDS45 TEC current - A
        self.HTcC = 63905 #32 - PDS90 TEC current - A
        self.VLsO = 63905 #33 - PDS45 laser on (unused)
        self.HLsO = 63905 #34 - PDS90 laser on (unused)
        self.vMSe = 63905 #35 - -7V monitor - V
        self.vPSe = 63905 #36 - +7V monitor V
        self.v000 = 63905 #37 - PDS45 diode element 0 - V
        self.v021 = 63905 #38 - PDS45 diode element 21 - V
        self.v042 = 63905 #39 - PDS45 diode element 42 - V
        self.v064 = 63905 #40 - PDS45 diode element 64 - V
        self.v085 = 63905 #41 - PDS45 diode element 85 - V
        self.v106 = 63905 #42 - PDS45 diode element 106 - V
        self.v127 = 63905 #43 - PDS45 diode element 127 - V
        self.cImL = 63905 #44 - Imaging laser measured current
        self.h000 = 63905 #45 - PDS90 diode element 0 - V
        self.h021 = 63905 #46 - PDS90 diode element 21 - V
        self.h042 = 63905 #47 - PDS90 diode element 42 - V
        self.h064 = 63905 #48 - PDS90 diode element 64 - V
        self.h085 = 63905 #49 - PDS90 diode element 85 - V
        self.h106 = 63905 #50 - PDS90 diode element 106 - V
        self.h127 = 63905 #51 - PDS90 diode element 127 - V
        self.ILWm = 63905 #52 - Imaging laser measured pulse width - V
        self.ILCs = 63905 #53 - Imaging laser current set point - V
        self.ILWs = 63905 #54 - Imaging laser pulse width control set point - V
        self.pbMd = 63905 #55 - Probe mode - bit mapped
        self.htMd = 63905 #56 - Heater mode - bit mapped
        self.oPWM = 63905 #57 - Optical block PWM percentage - %
        self.nHpt = 63905 #58 - Number of H particles detected
        self.nVpt = 63905 #59 - Number of V particles detected
        self.tDed = 63905 #60 - Dead time - s
        self.MSFi = 63905 #61 - Maximum slices to fire on - bit mapped
        self.LTDl = 63905 #62 - Laser trigger delay and minimum pixels before firing - bit mapped
        self.vLPS = 63905 #63 - PDS45 laser power set point
        self.hLPS = 63905 #64 - PDS90 laser power set point
        self.vMBC = 63905 #65 - PDS45 masked bits count
        self.hMBC = 63905 #66 - PDS90 masked bits count
        self.nHOP = 63905 #67 - Number of horizontal overload periods
        self.nVOP = 63905 #68 - Number of vertical overload periods
        self.nStP = 63905 #69 - Number of detected stereo particles
        self.cCon = 63905 #70 - 2DS compression configuration - bit mapped
        self.PAW1 = 63905 #71 - PDS alignment info word 1 (unused)
        self.PAW2 = 63905 #72 - PDS alignment info word 2 (unused)
        self.tMSW = 63905 #73 - Timing word bits 47 to 32 (MSW)
        self.tISW = 63905 #74 - Timing word bits 31 to 16 (ISW)
        self.tLSW = 63905 #75 - Timing word bits 15 to 0 (LSW)
        self.tsW1 = 63905 #76 - TAS floating point word 1
        self.tsW2 = 63905 #77 - TAS floating point word 2
        self.CA2D = 63905 #78 - 2DS commands accepted
        self.CACP = 63905 #79 - CPI commands accepted
        self.n2DB = 63905 #80 - Number od 2DS data blocks sent in the last second
        self.ASOF = 63905 #81 - Array skew offset / fuzzy skew - bit mapped
        self.CPIS = 63905 #82 - CPI image stats - bit mapped
        self.Chck = 63905 #83 - Checksum

    def __iter__(self):
        for each in self.__dict__.keys():
            yield self.__getattribute__(each)

# Empty data container for single particle data - ACHTUNG - allocated at runtime
class particleRecord2DS:
    pass

max_stars_repo_name: jrdorsey/pyCloud

max_issues_repo_name: jrdorsey/pyCloud

max_forks_repo_name: jrdorsey/pyCloud


text:
"""PDS releases."""

import os

import numpy as np

from .errors import PDSError
from .html import JPLReleaseParser, ReleasesParser
from .times import cassini_time, utc2cassini
from .vars import RELEASES_URL, ROOT_DATA
from ..cassini import img_id
from ..wget import url_exists, wget_txt

class PDS:
    """PDS releases object.

    Parameters
    ----------
    instr: str
        Instrument name.
    prefix: str, optional
        Releases prefix.
    src: str, optional
        Release source (JPL/SETI/USGS).
    fmt: str, optional
        Data format (LBL, QUB).
    update: bool, optional
        Update releases list.
    verbose: bool, optional
        Display verbose results.

    Examples
    --------
    >>> 'C1487096932_1_ir.cub' in PDS('VIMS')
    True

    >>> PDS('VIMS')['1487096932_1']
    ['covims_unks', 'covims_0006']

    """

    def __init__(self, instr, prefix='co', src='jpl', fmt='lbl',
                 update=False, verbose=True):
        self.instr = instr.lower()
        self.prefix = prefix
        self.src = src
        self.fmt = fmt.lower()
        self.update = update
        self.verbose = verbose
        self.__releases = []

        self.download() if not self.is_file or update else self.load_csv()

    def __repr__(self):
        return (f'<{self.__class__.__name__}> {self.instr.upper()}\n'
                f' - Releases: {len(self)}\n'
                f' - Source: {self.src.upper()}')

    def __len__(self):
        return len(self.__releases)

    def __contains__(self, item):
        return True if self.locate(item) else False

    def __getitem__(self, item):
        urls = self.locate(item)
        if urls:
            return urls[0] if len(urls) == 1 else urls
        raise IndexError(f'Data `{item}` was not found in {self.instr.upper()} releases.')

    @property
    def fname(self):
        """Releases file name."""
        return f'{self.prefix}{self.instr}_{self.src}_releases.csv'

    @property
    def filename(self):
        """List of releases absolute file name."""
        return os.path.join(ROOT_DATA, self.fname)

    @property
    def is_file(self):
        """Check if the file exists."""
        return os.path.exists(self.filename)

    @property
    def url(self):
        """Releases location based on instrument name."""
        return f'{RELEASES_URL}/{self.instr}.html'

    @property
    def _keyword(self):
        """Parser keyword based on instrument name."""
        if self.instr == 'vims':
            return 'Raw cube data'
        raise ValueError(f'Unknown parsing keyword for instrument: {self.instr}')

    def parse_times(self, data):
        """Parse releases times based on instrument name."""
        if self.instr == 'vims':
            return utc2cassini(data)
        raise ValueError(f'Unknown parsing times for instrument: {self.instr}')

    def _fmt(self, fname):
        """Get formatted name based on file and instrument name."""
        if self.instr == 'vims':
            _id = img_id(fname)
            if self.fmt == 'lbl':
                return f'v{_id}.lbl'
            if self.fmt == 'qub':
                return f'v{_id}.qub'
        raise ValueError(f'Invalid instrument: {self.instr} with format: {self.fmt}')

    @property
    def dtype(self):
        """Date types."""
        return {
            'names': ('release', 'start', 'stop', 'url'),
            'formats': ('U25', int, int, 'U999'),
        }

    def download(self):
        """Download and parse PDS releases list.

        The list is cached in a CSV file.

        """
        if self.verbose:
            print(f'Download list of releases from `{self.url}`.')

        try:
            html = wget_txt(self.url)
            results = ReleasesParser(html, src=self.src, keyword=self._keyword)
        except PDSError:
            raise PDSError(f'No releases found in {self.url}')

        releases = []
        for data, link in results:
            start, stop = self.parse_times(data)
            release = link.split('/')[-2]
            releases.append((release, int(start), int(stop), link))

        with open(self.filename, 'w') as f:
            f.write(f'release, start, stop, url, src:{self.url}\n')
            f.write('\n'.join(
                [', '.join([str(r) for r in row]) for row in releases]))

        self.__releases = np.array(releases, dtype=self.dtype)

    def load_csv(self):
        """Load CSV file."""
        with open(self.filename, 'r') as f:
            src_url = f.readlines()[1].split(':')[1]

        if 'jpl.nasa.gov' in src_url:
            self.src = 'jpl'
        elif 'usgs.gov' in src_url:
            self.src = 'usgs'
        elif 'seti.org' in src_url:
            self.src = 'seti'
        else:
            raise ValueError(f'Data source unknown: `{src_url}`.')

        self.__releases = np.loadtxt(self.filename, delimiter=', ', skiprows=1,
                                     dtype=self.dtype)

    @property
    def releases(self):
        """List available releases."""
        return self.__releases['release']

    @property
    def starts(self):
        """List available releases start IDs."""
        return self.__releases['start']

    @property
    def stops(self):
        """List available releases stop IDs."""
        return self.__releases['stop']

    @property
    def links(self):
        """List available releases links."""
        return self.__releases['url']

    def link(self, release):
        """Get URL of the release.

        Parameters
        ----------
        release: str
            Release name.

        Returns
        -------
        str
            Link to the release based on the source data.

        Raises
        ------
        IndexError
            If the name of the release was not found in the available releases.

        """
        links = self.links[self.releases == release]
        if not links:
            raise PDSError(
                f'Release `{release}` not found in {self.src.upper()} releases.')

        return links[0] if len(links) == 1 else links

    def release(self, time):
        """Find the releases which contains the spacecraft time.

        Parameters
        ----------
        time: str or int
            Cassini time.

        Returns
        -------
        list
            List of all the releases names which overlap the input time.

        """
        t = cassini_time(time)
        return list(self.releases[(self.starts <= t) & (t <= self.stops)])

    def locate(self, fname):
        """Find the location a filename.

        Parameters
        ----------
        fname: str
            Input file name.

        Returns
        -------
        list
            List of URL where the label file is located.

        """
        urls = []
        for release in self.release(fname):
            for link in PDSRelease(self, release, update=self.update).link(fname):
                url = link + self._fmt(fname)
                if url_exists(url):
                    urls.append(url)

        return urls

class PDSRelease:
    """PDS release object.

    Parameters
    ----------
    instr: str
        Instrument name.
    name: str
        Name of the release.
    src: str, optional
        Release source (JPL/SETI/USGS).
    url: str, optional
        Location of the release.
    update: bool, optional
        Update releases list.
    verbose: bool, optional
        Display verbose results.

    """

    def __init__(self, pds, name, update=False, verbose=True):
        self._pds = pds
        self.name = name
        self.verbose = verbose
        self.url = None
        self.__folders = []

        self.download() if not self.is_file or update else self.load_csv()

    def __str__(self):
        return self.name

    def __repr__(self):
        return (f'<{self.__class__.__name__}> {self}\n'
                f' - Folders: {len(self)}\n'
                f' - Source: {self.src.upper()}')

    def __len__(self):
        return len(self.__folders)

    def __contains__(self, item):
        return True if self.link(item) else False

    def __getitem__(self, item):
        folders = self.link(item)
        if folders:
            return folders
        raise IndexError(f'Data `{item}` was not found in {self}.')

    @property
    def instr(self):
        """Release PDS instrument."""
        return self._pds.instr

    @property
    def src(self):
        """Release PDS source."""
        return self._pds.src

    @property
    def fname(self):
        """Release file name."""
        return f'{self}_{self.src}.csv'

    @property
    def filename(self):
        """Release absolute file name."""
        return os.path.join(ROOT_DATA, self.fname)

    @property
    def is_file(self):
        """Check if the file exists."""
        return os.path.exists(self.filename)

    @property
    def dtype(self):
        """Date types."""
        return {
            'names': ('folders', 'start', 'stop', 'url'),
            'formats': ('U30', int, int, 'U999'),
        }

    def download(self):
        """Download a specific release."""
        if self.verbose:
            print(f'Download release {self} from {self.src.upper()}.')

        # Get PDS url based on source
        self.url = self._pds.link(self.name)

        try:
            url = self.url + 'data/'
            html = wget_txt(url)

            if self.src == 'jpl':
                results = JPLReleaseParser(html)
            else:
                raise ValueError('Release parse is only available for JPL sources.')

        except PDSError:
            raise PDSError(f'No release folders found in {url}')

        folders = []
        for data in results:
            start, stop = self._pds.parse_times(data)
            link = url + data
            folders.append((data, int(start), int(stop), link))

        with open(self.filename, 'w') as f:
            f.write(f'start, stop, url, src:{url}\n')
            f.write('\n'.join(
                [', '.join([str(r) for r in row]) for row in folders]))

        self.__folders = np.array(folders, dtype=self.dtype)

    def load_csv(self):
        """Load CSV file."""
        with open(self.filename, 'r') as f:
            self.url = f.readlines()[1].split(':')[1]

        self.__folders = np.loadtxt(self.filename, delimiter=', ', skiprows=1,
                                    dtype=self.dtype)

    @property
    def folders(self):
        """List available folders."""
        return self.__folders['folders']

    @property
    def starts(self):
        """List available folders start IDs."""
        return self.__folders['start']

    @property
    def stops(self):
        """List available folders stop IDs."""
        return self.__folders['stop']

    @property
    def links(self):
        """List available folders links."""
        return self.__folders['url']

    def link(self, time):
        """Find the folders links which contains the spacecraft time.

        Parameters
        ----------
        time: str or int
            Cassini time.

        Returns
        -------
        list
            List of all the folders which overlap the input time.

        """
        """Find the folders which contains the spacecraft time.

        Parameters
        ----------
        time: str or int
            Cassini time.

        Returns
        -------
        list
            List of all the folders which overlap the input time.

        """
        t = cassini_time(time)
        return list(self.links[(self.starts <= t) & (t <= self.stops)])

max_stars_repo_name: seignovert/pyvims

max_issues_repo_name: seignovert/pyvims

max_forks_repo_name: seignovert/pyvims


text:
from scaii.env.sky_rts.env.scenarios.random_towers import RandomTowers
import numpy as np

env = RandomTowers()
print("Possible reward types:", env.reward_types())
print("Possible actions:", env.actions())
print("Action description", env.action_desc())

for i in range(0, 2):
    print("episode", i)

    s = env.reset()

    print("acting")
    act = env.new_action()
    act.attack_tower(2)

    s = env.act(act)

    while not s.is_terminal():
        raise Exception("Should not get in loop")
        noop = env.new_action()
        s = env.act(noop)

    print("Reward is:", s.reward, "Terminal?:", s.is_terminal())
    print("With types:", s.typed_reward)

max_stars_repo_name: SCAII/SCAII

max_issues_repo_name: SCAII/SCAII

max_forks_repo_name: SCAII/SCAII


text:
import numpy as np

from test.runtime.frontend_test.onnx_test.util import make_node, make_tensor_value_info, make_model
from test.util import wrap_template, generate_kernel_test_case
from webdnn.frontend.onnx import ONNXConverter

@wrap_template
def template(x0_shape, x1_shape, broadcast=0, axis=None, description: str = ""):
    vx0 = np.random.rand(*x0_shape)
    vx1 = np.random.rand(*x1_shape)
    if axis is not None:
        # onnx-style broadcast
        vx1 = vx1[(None,) * axis + (...,) + (None,) * (vx0.ndim - vx1.ndim - axis)]

    vy = vx0 + vx1

    x0 = make_tensor_value_info("x0", x0_shape)
    x1 = make_tensor_value_info("x1", x1_shape)
    y = make_tensor_value_info("y", vy.shape)

    kwargs = {"broadcast": broadcast}
    if axis is not None:
        kwargs["axis"] = axis
    operator = make_node("Add", ["x0", "x1"], ["y"], **kwargs)

    model = make_model([operator], [x0, x1], [y])

    graph = ONNXConverter().convert(model)

    generate_kernel_test_case(
        description=f"[ONNX] Add {description}",
        graph=graph,
        inputs={
            graph.inputs[0]: vx0,
            graph.inputs[1]: vx1
        },
        expected={graph.outputs[0]: vy},
    )

def test():
    template(x0_shape=[2, 3, 4, 5], x1_shape=[2, 3, 4, 5])

def test_broadcast1():
    template(x0_shape=[2, 3, 4, 5], x1_shape=[1], broadcast=1)

def test_broadcast2():
    template(x0_shape=[2, 3, 4, 5], x1_shape=[5], broadcast=1)

def test_broadcast3():
    template(x0_shape=[2, 3, 4, 5], x1_shape=[4, 5], broadcast=1)

def test_broadcast4():
    template(x0_shape=[2, 3, 4, 5], x1_shape=[3, 4], broadcast=1, axis=1)

def test_broadcast5():
    template(x0_shape=[2, 3, 4, 5], x1_shape=[2], broadcast=1, axis=0)

max_stars_repo_name: steerapi/webdnn

max_issues_repo_name: steerapi/webdnn

max_forks_repo_name: steerapi/webdnn


text:
import numpy as np

src= np.float32(
        [[712,466],
         [1035,662],
         [307,662],
         [569, 469]])
dst= np.float32([
		[842,0],
		[842,664],
		[400,664],
		[400,0]],)

max_stars_repo_name: hiteshhedwig/Lane-PedDetec

max_issues_repo_name: hiteshhedwig/Lane-PedDetec

max_forks_repo_name: hiteshhedwig/Lane-PedDetec


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from __future__ import absolute_import
from __future__ import print_function

import numpy as np
import os
import json
import random

from .feature_extractor import extract_features

def convert_to_dict(data, header, channel_info):
    """ convert data from readers output in to array of arrays format """
    ret = [[] for i in range(data.shape[1] - 1)]
    for i in range(1, data.shape[1]):
        ret[i - 1] = [(t, x) for (t, x) in zip(data[:, 0], data[:, i])
                      if x != ""]
        channel = header[i]
        if len(channel_info[channel]['possible_values']) != 0:
            ret[i - 1] = list(
                map(lambda x: (x[0], channel_info[channel]['values'][x[1]]),
                    ret[i - 1]))
        ret[i - 1] = list(map(lambda x: (float(x[0]), float(x[1])),
                              ret[i - 1]))
    return ret

def extract_features_from_rawdata(chunk, header, period, features):
    with open(
            os.path.join(os.path.dirname(__file__),
                         "resources/channel_info.json")) as channel_info_file:
        channel_info = json.loads(channel_info_file.read())
    data = [convert_to_dict(X, header, channel_info) for X in chunk]
    return extract_features(data, period, features)

def read_chunk(reader, chunk_size):
    data = {}
    for i in range(chunk_size):
        ret = reader.read_next()
        for k, v in ret.items():
            if k not in data:
                data[k] = []
            data[k].append(v)
    data["header"] = data["header"][0]
    return data

def sort_and_shuffle(data, batch_size):
    """ Sort data by the length and then make batches and shuffle them.
        data is tuple (X1, X2, ..., Xn) all of them have the same length.
        Usually data = (X, y).
    """
    assert len(data) >= 2
    data = list(zip(*data))

    random.shuffle(data)

    old_size = len(data)
    rem = old_size % batch_size
    head = data[:old_size - rem]
    tail = data[old_size - rem:]
    data = []

    head.sort(key=(lambda x: x[0].shape[0]))

    mas = [head[i:i + batch_size] for i in range(0, len(head), batch_size)]
    random.shuffle(mas)

    for x in mas:
        data += x
    data += tail

    data = list(zip(*data))
    return data

def add_common_arguments(parser):
    """ Add all the parameters which are common across the tasks
    """
    parser.add_argument('--network', type=str, required=True)
    parser.add_argument('--dim',
                        type=int,
                        default=256,
                        help='number of hidden units')
    parser.add_argument('--depth',
                        type=int,
                        default=1,
                        help='number of bi-LSTMs')
    parser.add_argument('--epochs',
                        type=int,
                        default=100,
                        help='number of chunks to train')
    parser.add_argument('--load_state',
                        type=str,
                        default="",
                        help='state file path')
    parser.add_argument('--mode',
                        type=str,
                        default="train",
                        help='mode: train or test')
    parser.add_argument('--batch_size', type=int, default=64)
    parser.add_argument('--l2',
                        type=float,
                        default=0,
                        help='L2 regularization')
    parser.add_argument('--l1',
                        type=float,
                        default=0,
                        help='L1 regularization')
    parser.add_argument('--save_every',
                        type=int,
                        default=1,
                        help='save state every x epoch')
    parser.add_argument('--prefix',
                        type=str,
                        default="",
                        help='optional prefix of network name')
    parser.add_argument('--dropout', type=float, default=0.0)
    parser.add_argument('--rec_dropout',
                        type=float,
                        default=0.0,
                        help="dropout rate for recurrent connections")
    parser.add_argument('--batch_norm',
                        type=bool,
                        default=False,
                        help='batch normalization')
    parser.add_argument('--timestep',
                        type=float,
                        default=1.0,
                        help="fixed timestep used in the dataset")
    parser.add_argument('--imputation', type=str, default='previous')
    parser.add_argument('--small_part', dest='small_part', action='store_true')
    parser.add_argument('--whole_data',
                        dest='small_part',
                        action='store_false')
    parser.add_argument('--optimizer', type=str, default='adam')
    parser.add_argument('--lr',
                        type=float,
                        default=0.001,
                        help='learning rate')
    parser.add_argument('--beta_1',
                        type=float,
                        default=0.9,
                        help='beta_1 param for Adam optimizer')
    parser.add_argument('--verbose', type=int, default=2)
    parser.add_argument('--size_coef', type=float, default=4.0)
    parser.add_argument(
        '--normalizer_state',
        type=str,
        default=None,
        help='Path to a state file of a normalizer. Leave none if you want to '
        'use one of the provided ones.')
    parser.set_defaults(small_part=False)

class DeepSupervisionDataLoader:
    r"""
    Data loader for decompensation and length of stay task.
    Reads all the data for one patient at once.

    Parameters
    ----------
    dataset_dir : str
        Directory where timeseries files are stored.
    listfile : str
        Path to a listfile. If this parameter is left `None` then
        `dataset_dir/listfile.csv` will be used.
    """
    def __init__(self, dataset_dir, listfile=None, small_part=False):

        self._dataset_dir = dataset_dir
        if listfile is None:
            listfile_path = os.path.join(dataset_dir, "listfile.csv")
        else:
            listfile_path = listfile
        with open(listfile_path, "r") as lfile:
            self._data = lfile.readlines()[1:]  # skip the header

        self._data = [line.split(',') for line in self._data]
        self._data = [(x, float(t), y) for (x, t, y) in self._data]
        self._data = sorted(self._data)

        mas = {"X": [], "ts": [], "ys": [], "name": []}
        i = 0
        while i < len(self._data):
            j = i
            cur_stay = self._data[i][0]
            cur_ts = []
            cur_labels = []
            while j < len(self._data) and self._data[j][0] == cur_stay:
                cur_ts.append(self._data[j][1])
                cur_labels.append(self._data[j][2])
                j += 1

            cur_X, header = self._read_timeseries(cur_stay)
            mas["X"].append(cur_X)
            mas["ts"].append(cur_ts)
            mas["ys"].append(cur_labels)
            mas["name"].append(cur_stay)

            i = j
            if small_part and len(mas["name"]) == 256:
                break

        self._data = mas

    def _read_timeseries(self, ts_filename):
        ret = []
        with open(os.path.join(self._dataset_dir, ts_filename), "r") as tsfile:
            header = tsfile.readline().strip().split(',')
            assert header[0] == "Hours"
            for line in tsfile:
                mas = line.strip().split(',')
                ret.append(np.array(mas))
        return (np.stack(ret), header)

def create_directory(directory):
    if not os.path.exists(directory):
        os.makedirs(directory)

def pad_zeros(arr, min_length=None):
    """
    `arr` is an array of `np.array`s

    The function appends zeros to every `np.array` in `arr`
    to equalize their first axis lenghts.
    """
    dtype = arr[0].dtype
    max_len = max([x.shape[0] for x in arr])
    ret = [
        np.concatenate(
            [x,
             np.zeros((max_len - x.shape[0], ) + x.shape[1:], dtype=dtype)],
            axis=0) for x in arr
    ]
    if (min_length is not None) and ret[0].shape[0] < min_length:
        ret = [
            np.concatenate([
                x,
                np.zeros(
                    (min_length - x.shape[0], ) + x.shape[1:], dtype=dtype)
            ],
                           axis=0) for x in ret
        ]
    return np.array(ret)