text:
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import csv
import numpy as np
import os
import sys

from observations.util import maybe_download_and_extract

def lost_letter(path):
  """LostLetter

  Which "lost" letters will be returned by the public?

  A dataset with 140 observations on the following 8 variables.

  `Location`

  Where letter was "lost": `DesMoines`, `GrinnellCampus`, or
  `GrinnellTown`

  `Address`

  Address on teh letter: `Confederacy` or `Peaceworks`

  `Returned`

  `1`\ =letter was returned or `0`\ =letter was not returned

  `DesMoines`

  Indicator for letters left in Des Moines

  `GrinnellTown`

  Indicator for letters left in the town of Grinnell

  `GrinellCampus`

  Indicator for letters left on the Grinnell campus

  `Peaceworks`

  Indicator for letters addressed to Iowa Peaceworks

  `Confederacy`

  Indicator for letters addressed to Friends of the Confederacy

  Args:

    path: str.
      Path to directory which either stores file or otherwise file will
      be downloaded and extracted there.
      Filename is `lost_letter.csv`.

  Returns:

    Tuple of np.ndarray `x_train` with 140 rows and 8 columns and
    dictionary `metadata` of column headers (feature names).
  """
  import pandas as pd
  path = os.path.expanduser(path)
  filename = 'lost_letter.csv'
  if not os.path.exists(os.path.join(path, filename)):
    url = 'http://dustintran.com/data/r/Stat2Data/LostLetter.csv'
    maybe_download_and_extract(path, url,
                               save_file_name='lost_letter.csv',
                               resume=False)

  data = pd.read_csv(os.path.join(path, filename), index_col=0,
                     parse_dates=True)
  x_train = data.values
  metadata = {'columns': data.columns}
  return x_train, metadata

ext: py

reason: import numpy


text:
#!/usr/bin/python
# -*- coding: utf-8 -*-

# ---------------------------------------------------------------------
# Copyright (c) 2012 Michael Hull.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  - Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  - Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# ----------------------------------------------------------------------

from scipy import arange
from scipy.special import jn

from enable.api import ComponentEditor
from traits.api import HasTraits, Instance
from traitsui.api import Item, View

from chaco.api import create_line_plot, HPlotContainer
from chaco.tools.api import PanTool

class PlotExample(HasTraits):
    container = Instance(HPlotContainer)

    traits_view = View(Item('container', editor=ComponentEditor(),
                            show_label=False, width=800, height=600),
                       title="Chaco Tutorial")

    def _container_default(self):
        x = arange(-5.0, 15.0, 20.0/100)

        y = jn(0, x)
        left_plot = create_line_plot((x, y), bgcolor="white",
                                     add_grid=True, add_axis=True)
        left_plot.tools.append(PanTool(left_plot))
        self.left_plot = left_plot

        y = jn(1, x)
        right_plot = create_line_plot((x, y), bgcolor="white",
                                      add_grid=True, add_axis=True)
        right_plot.tools.append(PanTool(right_plot))
        right_plot.y_axis.orientation = "right"
        self.right_plot = right_plot

        # Tone down the colors on the grids
        right_plot.hgrid.line_color = (0.3, 0.3, 0.3, 0.5)
        right_plot.vgrid.line_color = (0.3, 0.3, 0.3, 0.5)
        left_plot.hgrid.line_color = (0.3, 0.3, 0.3, 0.5)
        left_plot.vgrid.line_color = (0.3, 0.3, 0.3, 0.5)

        container = HPlotContainer(spacing=20, padding=50, bgcolor="lightgray")
        container.add(left_plot)
        container.add(right_plot)
        return container

demo = PlotExample()

class PlotExample2(PlotExample):
    def _container_default(self):
        container = super(PlotExample2, self)._container_default()

        rplot, lplot = self.right_plot, self.left_plot
        rplot.index_mapper.range = lplot.index_mapper.range
        lplot.value_mapper.range.low = min(rplot.value_mapper.range.low,
                                           lplot.value_mapper.range.low,  )
        rplot.value_mapper.range = lplot.value_mapper.range

        return container

demo = PlotExample2()

if __name__ == "__main__":
    demo.configure_traits()

max_stars_repo_name: mikehulluk/morphforge

max_issues_repo_name: mikehulluk/morphforge

max_forks_repo_name: mikehulluk/morphforge

reason: from scipy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Feb 28 20:27:22 2020

@author: Filipe Lucas De Souza
"""
from util import Solution
import matplotlib.pyplot as plt
from random import seed
from GA_C import GA
import numpy as np

_popSize            = 100
_maxIterations      = 2300
_widthPlates        = 6000
_heightPlates       = 3210
_crossoverAlgorithm = 'P' # PMX
_mutationAlgorithm  = 'R' # Reciprocal Exchange
_mutationRate       = 0.3
_selectionAlgorithm = 'T' # Tournament Selection
_localSearchRate    = 1
_eliteRate          = 0.1
instance_name = "data/dataset_A/A6"
instanceSize =  43254870

seed(182510)
ga = GA(instance_name,_popSize,_mutationRate,_maxIterations,_widthPlates,
   _heightPlates,_crossoverAlgorithm,_mutationAlgorithm,_selectionAlgorithm,
   _localSearchRate,_eliteRate)
ga.search()

ga.getCost(Solution(ga.templateSolution[0:], ga.widthPlates, ga.heightPlates)).printSolution(ga.stacks)

ga.best.printSolution(ga.stacks)

AvgCost = instanceSize/(instanceSize+ np.mean(np.array(ga.output)[:,:-2],axis=1) )
BestCost =  instanceSize/(instanceSize+ np.array(ga.output)[:,-1])

plt.style.use("ggplot")
plt.figure()

plt.plot(np.arange(0,len(AvgCost)),AvgCost,label="AvgCost")
plt.plot(np.arange(0,len(BestCost)),BestCost,label="BestCost")
plt.xlabel("iteration")
plt.ylabel("Occupation Rate")
plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.5))


text:
from .base_backend import BaseBackend, UnittestBaseBackend
import numpy as np
class NumPy(BaseBackend):
	def __init__(self):
		self.np = np
		self.float16 = np.float16
		self.float32 = np.float32
		self.float64 = np.float64
		self.default_dtype = self.float32

	def get_dtype(self, dtype):
		if dtype is None:
			return self.default_dtype
		else:
			return dtype

	def array(self, a, dtype = None):
		return np.array(a, dtype = self.get_dtype(dtype))

	def ones(self, shape, dtype = None):
		return np.ones(shape, self.get_dtype(dtype))

	def zeros(self, shape, dtype = None):
		return np.zeros(shape, self.get_dtype(dtype))

	def tensordot(self, a, b, axes):
		return np.tensordot(a, b, axes)

	def expand_dims(self, a, axis):
		return np.expand_dims(a, axis)

	def concatenate(self, a, axis):
		return np.concatenate(a, axis)

	def inverse(self, a):
		return np.linalg.inv(a)

	def transpose(self, a, axes = None):
		return np.transpose(a, axes = axes)

	def stack(self, a, axis = 0):
		return np.stack(a, axis = axis)

	def repeat(self, a, repeats, axis = None):
		return np.repeat(a, repeats, axis = axis)

	def logical_not(self, a):
		return np.logical_not(a)

	def logical_or(self, a, b):
		return np.logical_or(a, b)

	def logical_and(self, a, b):
		return np.logical_and(a, b)

	def any(self, a, axis = None):
		return np.any(a, axis = axis)

	def range(self,*args, dtype = None):
		return np.arange(*args, dtype = self.get_dtype(dtype))

	def reshape(self, a, newshape):
		return np.reshape(a, newshape)

	def gather(self, a, indices, axis = None):
		return np.take(a, indices, axis = axis)

	def eye(self, N, M = None, dtype = None):
		return np.eye(N, M, dtype = self.get_dtype(dtype))

	def max(self, a, axis = None, keepdims = False):
		return np.max(a, axis = axis, keepdims = keepdims)

	def min(self, a, axis = None, keepdims = False):
		return np.min(a, axis = axis, keepdims = keepdims)

	def mean(self, a, axis = None, keepdims = False):
		return np.mean(a, axis = axis, keepdims = keepdims)

	def sin(self, x):
		return np.sin(x)

	def cos(self, x):
		return np.cos(x)

	def seed(self, x):
		np.random.seed(x)

	def random_uniform(self, minval, maxval = None, size = None, dtype = None):
		return np.random.uniform(minval, maxval, size = size)

class UnittestNumPy(UnittestBaseBackend):
	default_rtol = 1e-5
	default_atol = 1e-8

	def assertIsClose(self, a, b, rtol = default_rtol, atol = default_atol):
		assert np.isclose(a, b, rtol = rtol, atol = atol).all()

max_stars_repo_name: HashimHL/posetools

max_issues_repo_name: HashimHL/posetools

max_forks_repo_name: HashimHL/posetools


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import copy
import inspect
import string

import matplotlib.patches as mpatches
import matplotlib.pyplot as plt
import numpy as np

__all__ = ['plot_chains', 'noise_flower']

def plot_chains(core, hist=True, pars=None, exclude=None,
                ncols=3, bins=40, suptitle=None, color='k',
                publication_params=False, titles=None,
                linestyle=None, plot_map=False,
                save=False, show=True, linewidth=1,
                log=False, title_y=1.01, hist_kwargs={},
                plot_kwargs={}, legend_labels=None, real_tm_pars=True,
                legend_loc=None, **kwargs):
    """Function to plot histograms or traces of chains from cores.

    Parameters
    ----------
    core : {`la_forge.core.Core`,
            `la_forge.core.HyperModelCore`,
            `la_forge.core.TimingCore`,
            `la_forge.slices.SlicedCore`}

    hist : bool, optional
        Whether to plot histograms. If False then traces of the chains will be
        plotted.

    pars : list of str, optional
        List of the parameters to be plotted.

    exclude : list of str, optional
        List of the parameters to be excluded from plot.

    ncols : int, optional
        Number of columns of subplots to use.

    bins : int, optional
        Number of bins to use in histograms.

    suptitle : str, optional
        Title to use for the plots.

    color : str or list of str, optional
        Color to use for histograms.

    publication_params=False,

    titles=None,

    linestyle : str,

    plot_map=False,

    save=False,
    show=True,
    linewidth=1,
    log=False,
    title_y=1.01,
    hist_kwargs={},
    plot_kwargs={},
    legend_labels=None,
    legend_loc=None,

    """
    if pars is not None:
        params = pars
    elif exclude is not None and pars is not None:
        raise ValueError('Please remove excluded parameters from `pars`.')
    elif exclude is not None:
        if isinstance(core, list):
            params = set()
            for c in core:
                params.intersection_update(c.params)
        else:
            params = core.params
        params = list(params)
        for p in exclude:
            params.remove(p)
    elif pars is None and exclude is None:
        if isinstance(core, list):
            params = core[0].params
            for c in core[1:]:
                params = [p for p in params if p in c.params]
        else:
            params = core.params

    if isinstance(core, list):
        fancy_par_names=core[0].fancy_par_names
        if linestyle is None:
            linestyle = ['-' for ii in range(len(core))]

        if isinstance(plot_map, list):
            pass
        else:
            plot_map = [plot_map for ii in range(len(core))]
    else:
        fancy_par_names=core.fancy_par_names

    L = len(params)

    if suptitle is None:
        psr_name = copy.deepcopy(params[0])
        if psr_name[0] == 'B':
            psr_name = psr_name[:8]
        elif psr_name[0] == 'J':
            psr_name = psr_name[:10]
    else:
        psr_name = None

    nrows = int(L // ncols)
    if L %ncols > 0:
        nrows +=1

    if publication_params:
        fig = plt.figure()
    else:
        fig = plt.figure(figsize=[15, 4*nrows])

    for ii, p in enumerate(params):
        cell = ii+1
        axis = fig.add_subplot(nrows, ncols, cell)
        if hist:
            if isinstance(core, list):
                for jj, c in enumerate(core):
                    gpar_kwargs= _get_gpar_kwargs(c, real_tm_pars)
                    phist=plt.hist(c.get_param(p, **gpar_kwargs),
                                   bins=bins, density=True, log=log,
                                   linewidth=linewidth,
                                   linestyle=linestyle[jj],
                                   histtype='step', **hist_kwargs)

                    if plot_map[jj]:
                        pcol=phist[-1][-1].get_edgecolor()
                        plt.axvline(c.get_map_param(p), linewidth=1,
                                    color=pcol, linestyle='--')
            else:
                gpar_kwargs= _get_gpar_kwargs(core, real_tm_pars)
                phist=plt.hist(core.get_param(p, **gpar_kwargs),
                               bins=bins, density=True, log=log,
                               linewidth=linewidth,
                               histtype='step', **hist_kwargs)
                if plot_map:
                    pcol=phist[-1][-1].get_edgecolor()
                    plt.axvline(c.get_map_param(p), linewidth=1,
                                color=pcol, linestyle='--')
        else:
            gpar_kwargs= _get_gpar_kwargs(core, real_tm_pars)
            plt.plot(core.get_param(p, to_burn=True, **gpar_kwargs),
                     lw=linewidth, **plot_kwargs)

        if (titles is None) and (fancy_par_names is None):
            if psr_name is not None:
                par_name = p.replace(psr_name+'_', '')
            else:
                par_name = p
            axis.set_title(par_name)
        elif titles is not None:
            axis.set_title(titles[ii])
        elif fancy_par_names is not None:
            axis.set_title(fancy_par_names[ii])

        axis.set_yticks([])
        xticks = kwargs.get('xticks')
        if xticks is not None:
            axis.set_xticks(xticks)

    if suptitle is None:
        guess_times = np.array([psr_name in p for p in params], dtype=int)
        yes = np.sum(guess_times)
        if yes/guess_times.size > 0.5:
            suptitle = 'PSR {0} Noise Parameters'.format(psr_name)
        else:
            suptitle = 'Parameter Posteriors    '

    if legend_labels is not None:
        patches = []
        colors = ['C{0}'.format(ii) for ii in range(len(legend_labels))]
        for ii, lab in enumerate(legend_labels):
            patches.append(mpatches.Patch(color=colors[ii], label=lab))

        fig.legend(handles=patches, loc=legend_loc)

    fig.tight_layout(pad=0.4)
    fig.suptitle(suptitle, y=title_y, fontsize=18)
    # fig.subplots_adjust(top=0.96)
    xlabel = kwargs.get('xlabel')
    if xlabel is not None:
        fig.text(0.5, -0.02, xlabel, ha='center', usetex=False)

    if save:
        plt.savefig(save, dpi=150, bbox_inches='tight')
    if show:
        plt.show()

    plt.close()

def noise_flower(hmc,
                 colLabels=['Add', 'Your', 'Noise'],
                 cellText=[['Model', 'Labels', 'Here']],
                 colWidths=None,
                 psrname=None, norm2max=False,
                 show=True, plot_path=None):
    """
    Parameters
    ----------

    hmc : la_forge.core.HyperModelCore

    colLabels : list, optional
        Table column headers for legend.

    cellText : nested list, 2d array, optional
        Table entries. Column number must match `colLabels`.

    psrname : str, optional
        Name of pulsar. Only used in making the title of the plot.

    key : list of str, optional
        Labels for each of the models in the selection process.

    norm2max : bool, optional
        Whether to normalize the values to the maximum `nmodel` residency.

    show : bool, optional
        Whether to show the plot.

    plot_path : str
        Enter a file path to save the plot to file.

    """
    # Number of models
    nmodels = hmc.nmodels

    if psrname is None:
        pos_names = [p.split('_')[0] for p in hmc.params
                     if p.split('_')[0][0] in ['B', 'J']]
        psrname = pos_names[0]

    # Label dictionary
    mod_letter_dict = dict(zip(range(1, 27), string.ascii_uppercase))
    mod_letters = [mod_letter_dict[ii+1] for ii in range(nmodels)]
    mod_index = np.arange(nmodels)
    # Histogram
    n, _ = np.histogram(hmc.get_param('nmodel', to_burn=True),
                        bins=np.linspace(-0.5, nmodels-0.5, nmodels+1),
                        density=True)
    if norm2max:
        n /= n.max()

    fig = plt.figure(figsize=[8, 4])
    ax = fig.add_subplot(121, polar=True)
    bars = ax.bar(2.0 * np.pi * mod_index / nmodels, n,
                  width=0.9 * 2 * np.pi / nmodels,
                  bottom=np.sort(n)[1]/2.)

    # Use custom colors and opacity
    for r, bar in zip(n, bars):
        bar.set_facecolor(plt.cm.Blues(r / 1.))

    # Pretty formatting
    ax.set_xticks(np.linspace(0., 2 * np.pi, nmodels+1)[:-1])
    labels=[ii + '=' + str(round(jj, 2)) for ii, jj in zip(mod_letters, n)]
    ax.set_xticklabels(labels, fontsize=11, rotation=0, color='grey')
    ax.grid(alpha=0.4)
    ax.tick_params(labelsize=10, labelcolor='k')
    ax.set_yticklabels([])

    plt.box(on=None)

    ax2 = fig.add_subplot(122)
    ax2.xaxis.set_visible(False)
    ax2.yaxis.set_visible(False)
    table = ax2.table(cellText=cellText,
                      colLabels=colLabels,
                      colWidths=colWidths,
                      loc='center')
    table.auto_set_font_size(False)
    table.set_fontsize(10)
    table.scale(1.05, 1.05)

    plt.box(on=None)
    ax2.set_title('PSR ' + psrname + '\n Noise Model Selection',
                  color='k', y=0.8, fontsize=13,
                  bbox=dict(facecolor='C3', edgecolor='k', alpha=0.2))
    if plot_path is not None:
        plt.savefig(plot_path, bbox_inches='tight', dpi=150)
    if show:
        plt.show()

def _get_gpar_kwargs(core, real_tm_pars):
    '''
    Convenience function to return a kwargs dictionary if their is a call
    to convert timing parameters.
    '''
    if 'tm_convert'in inspect.getfullargspec(core.get_param)[0]:
        gpar_kwargs = {'tm_convert': real_tm_pars}
    else:
        gpar_kwargs = {}
    return gpar_kwargs


text:
########################################
########################################
####### Author : Abhinandan Dubey (alivcor)
####### Stony Brook University

import DeepScore_Word2Vec
import time
import datetime
import EventIssuer
import csv
import sys
from nltk import word_tokenize
from sklearn.cluster import KMeans
import matplotlib.pyplot as plt
import numpy as np
# fix random seed for reproducibility
import pickle

def encodeOneHot(score):
    onehot_y = [0] * 13
    onehot_y[int(score)] = 1
    return np.array(onehot_y)

def preprocessEssayText(_LOGFILENAME, essay_filename):
    EventIssuer.issueMessage("Loading local dictionary : w2v_dict_final_1492930578.7.dsd", _LOGFILENAME)
    local_cache = pickle.load(open("dictionaries/w2v_dict_final_1492930578.7.dsd", "r"))
    EventIssuer.issueSuccess("Local dictionary loaded.", _LOGFILENAME)
    total_hits = 0.
    total_tokens_processed = 0.
    X = []
    beforeStart = time.time()
    essay_vector = np.zeros(300, )
    EventIssuer.issueMessage("Reading the Essay file provided to me " + essay_filename, _LOGFILENAME)
    with open(essay_filename, 'r') as f:
        essay = f.read()
    try:
        word_tokens = word_tokenize(essay)
    except UnicodeDecodeError:
        essay = essay.decode('latin-1').encode("utf-8")
        word_tokens = word_tokenize(essay)
    wcount = len(word_tokens)
    EventIssuer.issueSleep("Processing the file now.", _LOGFILENAME)
    for word in word_tokens:
        total_tokens_processed += 1
        try:
            wvec = local_cache[word]
            total_hits += 1
        except KeyError:
            wvec = DeepScore_Word2Vec.getWordVec(word, logfilename=_LOGFILENAME)
            local_cache[word] = wvec
        if type(wvec) != list:
            essay_vector = np.add(essay_vector, wvec)
        else:
            for wvector in wvec:
                essay_vector = np.add(essay_vector, wvector)
    essay_vector = essay_vector / wcount
    afterEnd = time.time()
    EventIssuer.issueSuccess("Preprocessed the essay in " + str(afterEnd-beforeStart) + " | Hit Rate : " + str(total_hits*100/total_tokens_processed), _LOGFILENAME)
    X.append(essay_vector)
    return np.array(X)

def preprocessDataset(_LOGFILENAME, timestamp):
    # local_cache = pickle.load(open("dictionaries/w2v_dict_1492912478.79.dsd", "r" ))
    # print glove.getWordVec("hello", _LOGFILENAME)
    local_cache = {}
    total_hits = 0.
    total_done = 0.
    total_tokens_processed = 0.
    cache_hit_points = []

    beforeStart = time.time()
    with open("../Dataset/Set1Complete.csv", "rb") as csvfile:
        datareader = csv.reader(csvfile)
        next(datareader, None)
        X = []
        Y = []
        total_essays = 1783.
        for row in datareader:
            essay_vector = np.zeros(300,)
            essay = row[2]
            score = float(float(row[3])+float(row[4]))
            try:
                word_tokens = word_tokenize(essay)
            except UnicodeDecodeError:
                essay = essay.decode('latin-1').encode("utf-8")
                word_tokens = word_tokenize(essay)
            wcount = len(word_tokens)
            for word in word_tokens:
                total_tokens_processed += 1
                try:
                    wvec = local_cache[word]
                    total_hits += 1
                    cache_hit_points.append([total_done, round(total_hits*100/total_tokens_processed, 2)])
                except KeyError:
                    wvec = DeepScore_Word2Vec.getWordVec(word, logfilename=_LOGFILENAME)
                    local_cache[word] = wvec
                if type(wvec) != list:
                    essay_vector = np.add(essay_vector, wvec)
                else:
                    for wvector in wvec:
                        essay_vector = np.add(essay_vector, wvector)
            total_done += 1.
            essay_vector = essay_vector/wcount
            X.append(essay_vector)
            Y.append(encodeOneHot(score))
            EventIssuer.issueSharpAlert("Complete: " + str(round(total_done*100/total_essays, 2)) + "%", _LOGFILENAME)
            if (total_done % 10 == 0):
                EventIssuer.issueSuccess(
                    "Cache Hit ! Current Hit Rate : " + str(round(total_hits * 100 / total_tokens_processed, 2)) + "%",
                    _LOGFILENAME)
                EventIssuer.issueMessage("Saving the dictionary at " + str(total_done), _LOGFILENAME)
                with open('dictionaries/w2v_dict_' + str(timestamp) + '.dsd', 'w') as f:
                    pickle.dump(local_cache, f)
            # if(total_done >= 700):
            #     EventIssuer.issueWarning("Stopping at 700 essays.", _LOGFILENAME)
            #     break

    afterEnd = time.time()
    with open('dictionaries/w2v_dict_final_' + str(timestamp) + '.dsd', 'w') as f:
        pickle.dump(local_cache, f)
    EventIssuer.issueSuccess(str(int(total_done)) + " essays processed in " + str(afterEnd-beforeStart), _LOGFILENAME, ifBold=True)

    X = np.array(X)
    Y = np.array(Y)

    with open('ppData/X_' + str(timestamp) + '.ds', 'w') as f:
        pickle.dump(X, f)

    with open('ppData/Y_' + str(timestamp) + '.ds', 'w') as f:
        pickle.dump(Y, f)

    with open('optimizations/cache_hit_rate' + str(timestamp) + '.csv', 'w') as f:
        for i in cache_hit_points:
            f.write(str(i[0]) + "," + str(i[1]) + "\n")

def randPartition(alldata_X, alldata_Y, _FRACTION):
    """
    adopted from https://gist.github.com/alivcor/9516927cc211c4cf274167d84574e068 by Abhinandan Dubey (alivcor)
    alldata_X : All of your X (Features) data
    alldata_Y : All of your Y (Prediction) data
    _FRACTION : The fraction of data rows you want for train (0.75 means you need 75% of your data as train and 25% as test)
    """
    np.random.seed(0)
    indices = np.arange(alldata_X.shape[0])
    np.random.shuffle(indices)

    dataX = alldata_X[indices]
    dataY = alldata_Y[indices]

    partition_index = int(dataX.shape[0] * _FRACTION)

    trainX = dataX[0:partition_index]
    testX = dataX[partition_index:dataX.shape[0]]

    trainY = dataY[0:partition_index]
    testY = dataY[partition_index:dataY.shape[0]]

    return [trainX, trainY, testX, testY]

def optimize_elbow(data, icn, visualize=False):
    num_clusters = icn
    plt.ion()
    sse_list = []
    nc_list = []
    k_means_list = []
    if(visualize):
        while(num_clusters <= 15):
            k_means = KMeans(n_clusters=num_clusters).fit(data)
            k_means_list.append(k_means)
            clusters_counts = {}
            clusters = {}
            point_index = 0
            for point in k_means.labels_:
                try:
                    clusters_counts[point] += 1
                    clusters[point].append(point_index)
                except KeyError:
                    clusters_counts[point] = 1
                    clusters[point] = [point_index]
                point_index += 1
            # GET SUM SQUARED ERROR
            sum_sq_error = 0
            for cluster_label in clusters:
                cluster_mean = k_means.cluster_centers_[cluster_label]
                for data_point_index in clusters[cluster_label]:
                    data_point = data[data_point_index]
                    for findex in xrange(len(data_point)):
                        sum_sq_error += math.pow(data_point[findex]-cluster_mean[findex], 2)
            print "SSE at", str(num_clusters), "=", str(sum_sq_error)

            #PLOT / UPDATE THE PLOT FOR SSE
            sse_list.append(sum_sq_error)
            nc_list.append(num_clusters)
            num_clusters += 1
            plt.plot(nc_list, np.array(sse_list), marker='o', linestyle='--', color='r')
            plt.pause(0.5)
    else:
        k_means = KMeans(n_clusters=5).fit(data)
        return k_means
    plt.close()
    return k_means_list[3]

def generate_stratified_samples(data_x, data_y, fraction):
    k_means = optimize_elbow(data_y, 2, True)
    clusters_counts = {}
    clusters = {}
    point_index = 0
    for point in k_means.labels_:
        try:
            clusters_counts[point] += 1
            clusters[point].append(point_index)
        except KeyError:
            clusters_counts[point] = 1
            clusters[point] = [point_index]
        point_index += 1
    stratified_samples = {}
    for cluster_label in clusters:
        sampled_dict_indices = np.random.randint(len(clusters[cluster_label]), size=int(len(clusters[cluster_label]) * fraction))
        sampled_indices = np.array(clusters[cluster_label])[sampled_dict_indices]
        print "In Cluster", str(cluster_label), ": Total : ", str(len(clusters[cluster_label])), "| Sampled : ", str(len(sampled_indices))
        stratified_samples[cluster_label] = data_x[sampled_indices]
    return stratified_samples

def stratifiedPartition(alldata_X, alldata_Y, _FRACTION):

def partitionDataset(X, Y):
    trainX, trainY, testX, testY = randPartition(X, Y, 0.90)
    trainX, trainY, devX, devY = randPartition(trainX, trainY, 0.90)
    return trainX, trainY, devX, devY, testX, testY

max_stars_repo_name: alivcor/aes

max_issues_repo_name: alivcor/aes

max_forks_repo_name: alivcor/aes


text:
import pdb

import numpy

import geometry.basis_spline

class SplineModel(object):
    """
    """
    def __init__(
        self,
        knots,
        order,
        coefficients=None,
        A=None,
        B=None,
        P=None,
        Q=None,
        R=None,
        boundary_knots=None):
        """
        """
        if coefficients is None:
            coefficients = numpy.zeros((len(knots) + order - 1,))
        self._spline = geometry.basis_spline.bSpline(
            knots,
            order,
            boundary_knots,
            coefficients)

        self._A = A
        self._B = B

        if P is None:
            P = numpy.diag(numpy.ones((len(self._spline),)))
        if not isinstance(P, numpy.ndarray):
            P *= numpy.diag(numpy.ones((len(self._spline),)))
        self._P = P

        if Q is None:
            Q = numpy.eye(len(self._spline))
        self._Q = Q

        self._R = R

    def time_update(self, dt, u=None, A=None, B=None, Q=None):
        """perform a time update on the model

        Arguments:
            dt: time step size
            u: optionally, the input
            A: optionally, the state transition matrix
            B: optionally, the input matrix
            Q: optionally, the process noise matrix to use

        Returns:
            no returns
        """
        if A is not None:
            self._spline.coords += A.dot(self._spline.coords) * dt
            self._P = A.dot(self._P).dot(A.T)
        elif self._A is not None:
            self._spline.coords += self._A.dot(self._spline.coords) * dt
            self._P = self._A.dot(self._P).dot(self._A.T)

        if Q is None:
            self._P += self._Q * dt
        else:
            if isinstance(Q, float):
                Q = numpy.eye(self._spline.coords.shape[0]) * Q
            self._P += Q * dt

        if u is not None:
            if B is not None:
                self._spline.coords += B.dot(u) * dt
            elif self._B is not None:
                self._spline.coords += self._B.dot(u) * dt

    def measurement(self, x, z, R=None):
        """perform a measurement update

        Arguments:
            x: location of the measurement
            z: measurement
            R: optionally, the measurement noise matrix to use

        Returns:
            no returns
        """
        vandermonde = numpy.array(self._spline.basis(x), ndmin=2)
        y = z - self._spline.eval(x)
        if R is None:
            s = vandermonde.dot(self._P).dot(vandermonde.T) + self._R
        else:
            s = vandermonde.dot(self._P).dot(vandermonde.T) + R
        K = self._P.dot(vandermonde.T) / s
        self._spline.coords += numpy.squeeze(K.dot(y))
        self._P = (numpy.eye(len(self._spline)) - K.dot(vandermonde)).dot(
            self._P)

    def evaluate(self, x):
        """Retrieve the model at a point

        Arguments:
            x: point to evaluate the model at

        Returns:
            y: model value at that point
        """
        return self._spline.eval(x)

    def basis(self, x):
        """Retrieve the basis at a point

        Arguments:
            x: the point to evaluate the model at

        Returns:
            N: the model basis at point x
        """
        return self._spline.basis(x)

    def sigma(self, x):
        """Retrieve the model uncertainty at a point

        Arguments:
            x: the point to retrieve uncertainty at

        Returns:
            sigma: model variance at this point
        """
        N = self._spline.basis(x)
        sigma = N.dot(self._P).dot(N.T)
        if isinstance(sigma, numpy.ndarray):
            return numpy.diag(sigma)
        else:
            return sigma

    @property
    def coefficients(self):
        """Getter for the spline coefficients

        Arguments:
            no arguments

        Returns:
            c: spline coefficients
        """
        return numpy.array(self._spline.coords)

    @coefficients.setter
    def coefficients(self, c):
        """Setter for spline coefficients

        Arguments:
            c: new spline coefficients. Must match current size of the
                coordinates

        Returns:
            no returns
        """
        assert self._spline.coords.shape == c.shape,\
            'input and spline size do not match'
        self._spline.coords = c

    @property
    def knots(self):
        """Getter for the spline knots

        Arguments:
            no arguments

        Returns:
            k: spline knot locations
        """
        return numpy.array(self._spline.knots)

    @property
    def order(self):
        """Getter for spline order

        Arguments:
            no arguments

        Returns:
            n: spline order
        """
        return self._spline.order

    @property
    def covariance(self):
        """Getter for the covaraince

        Arguments:
            no arguments

        Returns:
            P: model covarianace
        """
        return self._P

    @covariance.setter
    def covariance(self, P):
        """Set the covariance

        Arguments:
            P: model covaraiance. either a single value or a (n,n) array where
                n is the dimension of the spline coefficients. If it is a single
                value then this value will be set on the diagonal

        Returns:
            no returns
        """
        if not isinstance(P, numpy.ndarray):
            P = numpy.diag(numpy.ones((len(self._spline),))) * P
        self._P = P

max_stars_repo_name: aivian/robots

max_issues_repo_name: aivian/robots

max_forks_repo_name: aivian/robots


text:
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""For portability and easy deployment, this script renders a single
light-camera configuration. Depending on your computing environment, you can
dispatch this script to multiple machines to render all light-camera
configurations in parallel.

Note that there is still some parallelism when rendering a single configuration,
because Blender automatically renders using multiple threads: all CPUs will be
used for rendering.

Example Usage:

    "$WHERE_YOU_INSTALLED_BLENDER"/blender-2.78c-linux-glibc219-x86_64/blender \
        --background \
        --python "$ROOT"/neural-light-transport/data_gen/render.py \
        -- \
        --scene="$ROOT"/data/scenes/dragon.blend \
        --cam_json="$ROOT"/data/trainvali_cams/P28R.json \
        --light_json="$ROOT"/data/trainvali_lights/L330.json \
        --cam_nn_json="$ROOT"/data/neighbors/cams.json \
        --light_nn_json="$ROOT"/data/neighbors/lights.json \
        --imh='512' \
        --uvs='256' \
        --spp='64' \
        --outdir="$ROOT"/output/render/dragon/trainvali_000020852_P28R_L330
"""

from sys import argv
from argparse import ArgumentParser
from shutil import copyfile
from os.path import join
import pickle as pk
import numpy as np
from tqdm import tqdm

# Blender
import bpy
from mathutils import Vector

import xiuminglib as xm

from util import load_json, dump_json, safe_cast_to_int, remap, add_b_ch, \
    save_float16_npy, name_from_json_path

parser = ArgumentParser(description="")
parser.add_argument(
    '--scene', type=str, required=True, help="path to the .blend scene")
parser.add_argument(
    '--cached_uv_unwrap', type=str, default=None, help=(
        "path to the cached .pickle of UV unwrapping, which needs doing only "
        "once per scene"))
parser.add_argument(
    '--cam_json', type=str, required=True, help="path to the camera .json")
parser.add_argument(
    '--light_json', type=str, required=True, help="path to the light .json")
parser.add_argument(
    '--cam_nn_json', type=str, required=True,
    help="path to the .json of mapping from camera to its nearest neighbor")
parser.add_argument(
    '--light_nn_json', type=str, required=True,
    help="path to the .json of mapping from light to its nearest neighbor")
parser.add_argument(
    '--imh', type=int, default=256,
    help="image height (width derived from camera's sensor height and width)")
parser.add_argument(
    '--uvs', type=int, default=256, help="size of (square) texture map")
parser.add_argument(
    '--spp', type=int, default=64, help="samples per pixel for rendering")
parser.add_argument(
    '--outdir', type=str, required=True, help="output directory")
parser.add_argument(
    '--debug', type=bool, default=False,
    help="whether to dump additional outputs for debugging")

def main(args):
    # Open scene
    xm.blender.scene.open_blend(args.scene)

    # Remove existing cameras and lights, if any
    for o in bpy.data.objects:
        o.select = o.type in ('LAMP', 'CAMERA')
    bpy.ops.object.delete()

    # Load camera and light
    cam = load_json(args.cam_json)
    light = load_json(args.light_json)

    # Add camera and light
    cam_obj = xm.blender.camera.add_camera(
        xyz=cam['position'], rot_vec_rad=cam['rotation'],
        name=cam['name'], f=cam['focal_length'],
        sensor_width=cam['sensor_width'], sensor_height=cam['sensor_height'],
        clip_start=cam['clip_start'], clip_end=cam['clip_end'])
    xm.blender.light.add_light_point(
        xyz=light['position'], name=light['name'], size=light['size'])

    # Common rendering settings
    xm.blender.render.easyset(n_samples=args.spp, color_mode='RGB')

    # Image and texture resolution
    imw = args.imh / cam['sensor_height'] * cam['sensor_width']
    imw = safe_cast_to_int(imw)
    xm.blender.render.easyset(h=args.imh, w=imw)

    # Render full RGB
    # TODO: Render in .exr to avoid saturated pixels (and tone mapping)
    rgb_camspc_f = join(args.outdir, 'rgb_camspc.png')
    xm.blender.render.render(rgb_camspc_f)
    rgb_camspc = xm.io.img.load(rgb_camspc_f, as_array=True)[:, :, :3]

    # Render diffuse RGB
    obj = bpy.data.objects['object']
    make_diffuse(obj)
    diffuse_camspc_f = join(args.outdir, 'diffuse_camspc.png')
    xm.blender.render.render(diffuse_camspc_f, obj_names=obj.name)
    diffuse_camspc = xm.io.img.load(diffuse_camspc_f, as_array=True)[:, :, :3]

    # Render alpha
    alpha_f = join(args.outdir, 'alpha.png')
    xm.blender.render.render_alpha(alpha_f, samples=args.spp)
    alpha = xm.io.img.load(alpha_f, as_array=True)
    alpha = xm.img.normalize_uint(alpha)

    # Cast rays through all pixels to the object
    xs, ys = np.meshgrid(range(imw), range(args.imh))
    # (0, 0)
    # +--------> (w, 0)
    # |           x
    # |
    # v y (0, h)
    xys = np.dstack((xs, ys)).reshape(-1, 2)
    ray_tos, x_locs, x_objnames, x_facei, x_normals = \
        xm.blender.camera.backproject_to_3d(
            xys, cam_obj, obj_names=obj.name, world_coords=True)
    intersect = {
        'ray_tos': ray_tos, 'obj_names': x_objnames, 'face_i': x_facei,
        'locs': x_locs, 'normals': x_normals}

    # Compute mapping between UV and camera space
    uv2cam, cam2uv = calc_bidir_mapping(
        obj, xys, intersect, args.uvs, cached_unwrap=args.cached_uv_unwrap)
    uv2cam = add_b_ch(uv2cam)
    cam2uv = add_b_ch(cam2uv)
    uv2cam[alpha < 1] = 0 # mask out interpolated values that fall outside
    xm.io.img.write_arr(uv2cam, join(args.outdir, 'uv2cam.png'), clip=True)
    xm.io.img.write_arr(cam2uv, join(args.outdir, 'cam2uv.png'), clip=True)
    save_float16_npy(uv2cam[:, :, :2], join(args.outdir, 'uv2cam.npy'))
    save_float16_npy(cam2uv[:, :, :2], join(args.outdir, 'cam2uv.npy'))

    # Compute view and light cosines
    lvis_camspc, cvis_camspc = calc_cosines(
        cam_obj.location, light['position'], xys, intersect, obj.name)
    lvis_camspc = xm.img.denormalize_float(np.clip(lvis_camspc, 0, 1))
    cvis_camspc = xm.img.denormalize_float(np.clip(cvis_camspc, 0, 1))
    xm.io.img.write_img(cvis_camspc, join(args.outdir, 'cvis_camspc.png'))
    xm.io.img.write_img(lvis_camspc, join(args.outdir, 'lvis_camspc.png'))

    # Remap buffers to UV space
    cvis = remap(cvis_camspc, cam2uv)
    lvis = remap(lvis_camspc, cam2uv)
    diffuse = remap(diffuse_camspc, cam2uv)
    rgb = remap(rgb_camspc, cam2uv)
    xm.io.img.write_img(cvis, join(args.outdir, 'cvis.png'))
    xm.io.img.write_img(lvis, join(args.outdir, 'lvis.png'))
    xm.io.img.write_img(diffuse, join(args.outdir, 'diffuse.png'))
    xm.io.img.write_img(rgb, join(args.outdir, 'rgb.png'))
    if args.debug:
        # Remap it backwards to check if we get back the camera-space buffer
        # TODO: UV wrapped images may have seams/holes due to interpolation
        # errors (fixable by better engineering), but this should be fine
        # because the network will learn to eliminate such artifacts in
        # trying to match the camera-space ground truth
        cvis_camspc_repro = remap(cvis, uv2cam)
        lvis_camspc_repro = remap(lvis, uv2cam)
        diffuse_camspc_repro = remap(diffuse, uv2cam)
        rgb_camspc_repro = remap(rgb, uv2cam)
        xm.io.img.write_img(
            cvis_camspc_repro, join(args.outdir, 'cvis_camspc_repro.png'))
        xm.io.img.write_img(
            lvis_camspc_repro, join(args.outdir, 'lvis_camspc_repro.png'))
        xm.io.img.write_img(
            diffuse_camspc_repro, join(args.outdir, 'diffuse_camspc_repro.png'))
        xm.io.img.write_img(
            rgb_camspc_repro, join(args.outdir, 'rgb_camspc_repro.png'))

    # Dump camera and light
    copyfile(args.cam_json, join(args.outdir, 'cam.json'))
    copyfile(args.light_json, join(args.outdir, 'light.json'))

    # Dump neighbor information
    cam_nn = load_json(args.cam_nn_json)
    light_nn = load_json(args.light_nn_json)
    cam_name = name_from_json_path(args.cam_json)
    light_name = name_from_json_path(args.light_json)
    nn = {'cam': cam_nn[cam_name], 'light': light_nn[light_name]}
    dump_json(nn, join(args.outdir, 'nn.json'))

def calc_cosines(cam_loc, light_loc, xys, intersect, obj_name):
    imw = xys[:, 0].max() + 1
    imh = xys[:, 1].max() + 1

    view_cosines = np.zeros((imh, imw))
    light_cosines = np.zeros((imh, imw))

    for oname, xy, loc, normal in tqdm(
            zip(
                intersect['obj_names'], xys, intersect['locs'],
                intersect['normals']),
            total=xys.shape[0], desc="Filling pixels for cosine maps"):
        if loc is None or oname != obj_name:
            continue

        p2l = (Vector(light_loc) - loc).normalized()
        p2c = (cam_loc - loc).normalized()
        normal = normal.normalized()

        light_cosines[xy[1], xy[0]] = p2l.dot(normal)
        view_cosines[xy[1], xy[0]] = p2c.dot(normal)

    return light_cosines, view_cosines

def calc_bidir_mapping(
        obj, xys, intersect, uvs, cached_unwrap=None, max_l1_interp=4):
    imw = xys[:, 0].max() + 1
    imh = xys[:, 1].max() + 1

    # Load or compute the UV unwrapping by Blender
    if cached_unwrap is None:
        # Might be expensive if your model has many vertices
        fi_li_vi_u_v = xm.blender.object.smart_uv_unwrap(obj, area_weight=1)
    else:
        with open(cached_unwrap, 'rb') as h:
            fi_li_vi_u_v = pk.load(h)

    # UV convention:
    # (0, 1)
    #   ^ v
    #   |
    #   |
    #   +------> (1, 0)
    # (0, 0)   u

    # Collect locations and their associated values
    uv2cam_locs, uv2cam_vals = [], []
    cam2uv_locs, cam2uv_vals = [], []
    for xy, oname, fi in tqdm(
            zip(xys, intersect['obj_names'], intersect['face_i']),
            total=xys.shape[0], desc="Filling pixels for camera-UV mappings"):
        if fi is None or oname != obj.name:
            continue

        uv = fi_li_vi_u_v[fi][:, 2:]

        # Collect locations and values for UV to camera
        camspc_loc = (xy[0] / float(imw), 1 - xy[1] / float(imh))
        uvspc_loc = np.hstack((uv[:, :1], 1 - uv[:, 1:]))
        uv2cam_locs.append(np.vstack([camspc_loc] * uvspc_loc.shape[0]))
        uv2cam_vals.append(uvspc_loc)

        # Now for camera to UV
        uvspc_loc = uv
        camspc_loc = (xy[0] / float(imw), xy[1] / float(imh))
        cam2uv_locs.append(uvspc_loc)
        cam2uv_vals.append(np.vstack([camspc_loc] * uvspc_loc.shape[0]))

    # Location convention for xm.img.grid_query_unstruct():
    # (0, 1)
    #     ^ v
    #     |
    #     +------> (1, 0)
    # (0, 0)      u

    # Value convention for xm.img.grid_query_unstruct(), for use by remap():
    # (0, 0)
    # +--------> (1, 0)
    # |           x
    # |
    # v y (0, 1)

    interp_method = {
        'func': 'griddata',
        'func_underlying': 'nearest',
        'fill_value': (0,), # black
        'max_l1_interp': max_l1_interp}

    # UV to camera space: interpolate unstructured values into an image
    locs = np.vstack(uv2cam_locs)
    vals = np.vstack(uv2cam_vals)
    uv2cam = xm.img.grid_query_unstruct(
        locs, vals, (imh, imw), method=interp_method)

    # Camera to UV space: interpolate unstructured values into an image
    locs = np.vstack(cam2uv_locs)
    vals = np.vstack(cam2uv_vals)
    cam2uv = xm.img.grid_query_unstruct(
        locs, vals, (uvs, uvs), method=interp_method)

    return uv2cam, cam2uv

def make_diffuse(obj):
    node_tree = obj.active_material.node_tree
    diffuse_out = node_tree.nodes['Diffuse BSDF'].outputs['BSDF']
    mat_in = node_tree.nodes['Material Output'].inputs['Surface']
    node_tree.links.new(diffuse_out, mat_in)

if __name__ == '__main__':
    # Blender-Python binary
    if '--' in argv:
        argv = argv[argv.index('--') + 1:]

    main(parser.parse_args(argv))

max_stars_repo_name: ceciliavision/neural-light-transport


text:
# -*- coding: utf-8 -*-
"""
Created on Sat Jul 25 19:55:07 2020

@author: eruss
"""

import numpy as np
import matplotlib.pyplot as plt
plt.rcParams.update({'font.size': 22, 'figure.autolayout': True, 'figure.figsize': (15,5)})
np.random.seed(999)

ontime = 0

prob = []
nr_points = []
for i in range(1, int(1e6)):
    car_problem = np.random.random()

    # Will he have a car problem?
    if car_problem < 0.2:
        # if he has  a car problem, will he still arrive on time?
        train = np.random.choice(np.arange(1,11))
        if train <= 2:
            ontime += 1
    else: # no car problem
        ontime_noproblem = np.random.random()
        if ontime_noproblem < 0.3:
            ontime += 1

    if i%10 == 0:
        nr_points.append(i)
        prob.append(ontime/(i))

#plt.plot(prob)
plt.semilogx(nr_points, prob)
plt.ylim([0,0.3])
plt.grid(True, which='both')
plt.xlabel('# of Drawings')
plt.ylabel('P(X)')
plt.savefig('experiment.png')

max_stars_repo_name: euanrussano/academic-kickstart

max_issues_repo_name: euanrussano/academic-kickstart

max_forks_repo_name: euanrussano/academic-kickstart


text:
import torch
import pandas as pd
import numpy as np
from torch.utils.data import Dataset

# each dataset contains the data for a single person, so we'll need to concat datasets
# that means we should create a new column containing the user identifier
class PosesDataset(Dataset):
    def __init__(self, csv_file, transform=None):
        self.poses_frame = pd.read_csv(csv_file)
        self.transform = transform

        # parse user from csv_file name
        user = csv_file.split('_')[3]
        self.poses_frame['user'] = [user] * len(self.poses_frame)

    def __len__(self):
        return len(self.poses_frame)

    def __getitem__(self,idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        user = self.poses_frame.iloc[idx]['user']
        poses = self.poses_frame.iloc[idx, 1:-1]
        poses = np.array([poses])

        sample = {'user': user, 'poses': poses}

        return sample

class ThrowsDataset(Dataset):
    def __init__(self, poses_file, events_file, transform=None):
        self.poses_frame = pd.read_csv(poses_file)
        self.events_frame = pd.read_csv(events_file)
        self.transform = transform

        # parse user from csv_file name
        user = poses_file.split('_')[3]
        self.poses_frame['user'] = [user] * len(self.poses_frame)

        grabs = self.events_frame[self.events_frame['event'] == 'GRABBED_OBJECT']
        throws = self.events_frame[self.events_frame['event'] == 'TARGET_THROW']
        # for the time between a grab and a release, get the poses
        starttimes = []
        for _, grab in grabs.iterrows():
            starttimes.append(grab['time'])

        endtimes = []
        for _, grab in throws.iterrows():
            endtimes.append(grab['time'])

        times = zip(starttimes, endtimes)

        self.throws_poses = pd.DataFrame()
        for (start, end) in times:
            throw = self.poses_frame[(self.poses_frame['time'] > start) & (self.poses_frame['time'] < end)]
            self.throws_poses = self.throws_poses.append(throw, ignore_index = True)

    def __len__(self):
        return len(self.throws_poses)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        user = self.poses_frame.iloc[idx]['user']
        poses = self.throws_poses.iloc[idx, 1:-1]
        poses = np.array([poses])

        sample = {'user': user, 'throw pose': poses}

class ToTensor(object):
    """ convert ndarrays to Tensors """

    def __call__(self, sample):
        user, poses = sample['user'], sample['poses']

        return {'user': torch.from_numpy(user), 'poses': torch.from_numpy(poses)}

pose_data = PosesDataset(csv_file='data/1_Teleport_0_10_poses.csv')
print(pose_data[0])

events_file = 'data/1_HandWalking_0_133_1596455970log/1_HandWalking_0_133_events.csv'
poses_file = 'data/1_HandWalking_0_133_1596455970log/1_HandWalking_0_133_poses.csv'

throws_data = ThrowsDataset(poses_file=poses_file, events_file=events_file)

max_stars_repo_name: avadacatavra/private-locomotion

max_issues_repo_name: avadacatavra/private-locomotion

max_forks_repo_name: avadacatavra/private-locomotion


text:
import instaseis
import obspy
import numpy as np
import re
import mtuq.greens_tensor.base
import mtuq.greens_tensor.instaseis

from collections import defaultdict
from copy import deepcopy
from os.path import basename, exists
from obspy.core import Stream, Trace
from mtuq.util.geodetics import km2deg
from mtuq.util.signal import resample
from mtuq.util.util import path_mtuq, unzip, url2uuid, urlopen_with_retry

# syngine is an webservice that provides Green's functions and synthetic
# seismograms for download as compressed SAC files. syngine uses instaseis
# under the hood for preparing user downloads, so Green's function conventions,
# moment tensor conventions, metadata format, data processing, and so on are
# very similar for syngine and instaseis.  In MTUQ, all of the machinery for
# generating synthetics from syngine Green's tensors is inherited from
# mtuq.greens_tensor.instaseis

GREENS_TENSOR_FILENAMES = [
    'greensfunction_XX.GF001..ZSS.sac',
    'greensfunction_XX.GF001..ZDS.sac',
    'greensfunction_XX.GF001..ZDD.sac',
    'greensfunction_XX.GF001..ZEP.sac',
    'greensfunction_XX.GF001..RSS.sac',
    'greensfunction_XX.GF001..RDS.sac',
    'greensfunction_XX.GF001..RDD.sac',
    'greensfunction_XX.GF001..REP.sac',
    'greensfunction_XX.GF001..TSS.sac',
    'greensfunction_XX.GF001..TDS.sac',
    ]

SYNTHETICS_FILENAMES = [
    'XX.S0001.SE.BXZ.sac',
    'XX.S0001.SE.BXR.sac',
    'XX.S0001.SE.BXT.sac',
    ]

class GreensTensor(mtuq.greens_tensor.instaseis.GreensTensor):
    def _precompute_weights(self):
        super(GreensTensor, self)._precompute_weights()

        # the negative sign is needed because of a bug in syngine? or because
        # of inconsistent moment tensor conventions?
        self._weighted_tensor[2] *= -1

        # Order of terms expected by syngine URL parser (from online
        # documentation):
        #    Mrr, Mtt, Mpp, Mrt, Mrp, Mtp
        #
        # Relations given in instaseis/tests/test_instaseis.py:
        #    m_tt=Mxx, m_pp=Myy, m_rr=Mzz, m_rt=Mxz, m_rp=Myz, m_tp=Mxy
        #
        # Relations suggested by mtuq/tests/unittest_greens_tensor_syngine.py
        # (note sign differences):
        #    m_tt=Mxx, m_pp=Myy, m_rr=Mzz, m_rt=M-xz, m_rp=Myz, m_tp=-Mxy

class GreensTensorFactory(mtuq.greens_tensor.base.GreensTensorFactory):
    def __init__(self, model):
        self.model = model

    def get_greens_tensor(self, station, origin):
        # download and unizp data
        dirname = unzip(download_greens_tensor(self.model, station, origin))

        # read data
        stream = Stream()
        stream.id = station.id
        for filename in GREENS_TENSOR_FILENAMES:
            stream += obspy.read(dirname+'/'+filename, format='sac')

        # what are the start and end times of the data?
        t1_new = float(station.starttime)
        t2_new = float(station.endtime)
        dt_new = float(station.delta)

        # what are the start and end times of the Green's function?
        t1_old = float(stream[0].stats.starttime)
        t2_old = float(stream[0].stats.endtime)
        dt_old = float(stream[0].stats.delta)

        for trace in stream:
            # resample Green's functions
            data_old = trace.data
            data_new = resample(data_old, t1_old, t2_old, dt_old,
                                          t1_new, t2_new, dt_new)
            trace.data = data_new
            trace.stats.starttime = t1_new
            trace.stats.delta = dt_new
            trace.stats.npts = len(data_new)

        traces = [trace for trace in stream]
        return GreensTensor(traces, station, origin)

def download_greens_tensor(model, station, origin):
    """ Downloads Green's functions through syngine URL interface
    """
    try:
        distance_in_deg = km2deg(station.distance)
    except:
        distance_in_deg = km2deg(station.catalog_distance)
    depth_in_m = origin.depth

    url = ('http://service.iris.edu/irisws/syngine/1/query'
         +'?model='+model
         +'&dt='+str(station.delta)
         +'&greensfunction=1'
         +'&sourcedistanceindegrees='+str(distance_in_deg)
         +'&sourcedepthinmeters='+str(int(round(depth_in_m)))
         +'&origintime='+str(origin.time)[:-1]
         +'&starttime='+str(origin.time)[:-1])
    filename = (path_mtuq()+'/'+'data/greens_tensor/syngine/cache/'
         +str(url2uuid(url)))
    if not exists(filename):
        print ' Downloading Green''s functions for station %s' % station.station
        urlopen_with_retry(url, filename+'.zip')
    return filename

def download_synthetics(model, station, origin, mt):
    """ Downloads synthetics through syngine URL interface
    """
    url = ('http://service.iris.edu/irisws/syngine/1/query'
         +'?model='+model
         +'&dt='+str(station.delta)
         +'&components=ZRT'
         +'&receiverlatitude='+str(station.latitude)
         +'&receiverlongitude='+str(station.longitude)
         +'&sourcelatitude='+str(origin.latitude)
         +'&sourcelongitude='+str(origin.longitude)
         +'&sourcedepthinmeters='+str(int(round(origin.depth)))
         +'&origintime='+str(origin.time)[:-1]
         +'&starttime='+str(origin.time)[:-1]
         +'&sourcemomenttensor='+re.sub('\+','',",".join(map(str, mt))))
    filename = (path_mtuq()+'/'+'data/greens_tensor/syngine/cache/'
         +str(url2uuid(url)))
    if not exists(filename):
        print ' Downloading waveforms for station %s' % station.station
        urlopen_with_retry(url, filename+'.zip')
    return filename

def get_synthetics_syngine(model, station, origin, mt):
    from mtuq.data.sac import reader

    dirname = unzip(download_synthetics(model, station, origin, mt))
    stream = reader(dirname)[0]

    # what are the start and end times of the data?
    t1_new = float(station.starttime)
    t2_new = float(station.endtime)
    dt_new = float(station.delta)

    # what are the start and end times of the Green's function?
    t1_old = float(stream[0].stats.starttime)
    t2_old = float(stream[0].stats.endtime)
    dt_old = float(stream[0].stats.delta)

    for trace in stream:
        # resample Green's functions
        data_old = trace.data
        data_new = resample(data_old, t1_old, t2_old, dt_old,
                                      t1_new, t2_new, dt_new)
        trace.data = data_new
        trace.stats.starttime = t1_new
        trace.stats.delta = dt_new
        trace.stats.npts = len(data_new)

        setattr(trace, 'network', station.network)
        setattr(trace, 'station', station.station)
        setattr(trace, 'location', station.location)

    synthetics = Stream()
    for channel in station.channels:
        component = channel[-1].upper()
        trace = stream.select(component=component)[0]
        synthetics += trace

    return synthetics


text:
import torch
import numpy as np

from utils.net_util import gpuify, toFloatTensor
from models.model_io import ModelInput
from runners.train_util import generate_det_4_iou

from .agent import ThorAgent

CLASSES = [
    'Pillow', 'Television', 'GarbageCan', 'Box', 'RemoteControl',
    'Toaster', 'Microwave', 'Fridge', 'CoffeeMachine', 'Mug', 'Bowl',
    'DeskLamp', 'CellPhone', 'Book', 'AlarmClock',
    'Sink', 'ToiletPaper', 'SoapBottle', 'LightSwitch'
]

class NavigationAgent(ThorAgent):
    """ A navigation agent who learns with pretrained embeddings. """

    def __init__(self, create_model, args, rank, gpu_id):
        max_episode_length = args.max_episode_length
        hidden_state_sz = args.hidden_state_sz
        self.action_space = args.action_space
        from utils.class_finder import episode_class

        episode_constructor = episode_class(args.episode_type)
        episode = episode_constructor(args, gpu_id, args.strict_done)

        super(NavigationAgent, self).__init__(
            create_model(args), args, rank, episode, max_episode_length, gpu_id
        )
        self.hidden_state_sz = hidden_state_sz
        self.detector_det = None
        self.gt_det = None
        self.optim_step = None
        self.last_det = None

    def eval_at_state(self, model_options):
        model_input = ModelInput()
        if self.episode.current_frame is None:
            model_input.state = self.state()
        else:
            model_input.state = self.episode.current_frame
        model_input.hidden = self.hidden

        current_pos = '{}|{}|{}|{}'.format(
            # self.environment.scene_name,
            self.episode.environment.controller.state.position()['x'],
            self.episode.environment.controller.state.position()['z'],
            self.episode.environment.controller.state.rotation,
            self.episode.environment.controller.state.horizon
        )

        target_embedding_array = np.zeros((len(CLASSES), 1))
        target_embedding_array[CLASSES.index(self.episode.target_object)] = 1
        glove_embedding_tensor = np.concatenate((self.episode.glove_reader[current_pos][()], target_embedding_array), axis=1)
        # model_input.target_class_embedding = self.episode.glove_embedding
        # if ((self.episode.glove_reader[current_pos][CLASSES.index(self.episode.target_object)] != np.array([0, 0, 0, 0])).all()) and (self.episode.det_frame == None):
        #     self.episode.current_det = self.eps_len
        model_input.target_class_embedding = toFloatTensor(glove_embedding_tensor, self.gpu_id)

        # if self.eps_len == 0:
        #     model_input.det_his = toFloatTensor(torch.zeros(4), self.gpu_id)
        # else:
        #     model_input.det_his = self.last_det
        # model_input.det_cur = toFloatTensor(self.episode.glove_reader[current_pos][CLASSES.index(self.episode.target_object)], self.gpu_id)
        # self.last_det = model_input.det_cur

        model_input.action_probs = self.last_action_probs

        # if self.eps_len == 0:
        #     det_iou = toFloatTensor(torch.zeros(1, 4), self.gpu_id)
        # else:
        #     det_iou = generate_det_4_iou(self.last_det)
        #     det_iou = toFloatTensor(det_iou, self.gpu_id)
        #
        # model_input.action_probs = torch.cat((self.last_action_probs, det_iou), dim=1)

        self.episode.detections.append(self.episode.glove_reader[current_pos][CLASSES.index(self.episode.target_object)])
        # self.detector_det = self.episode.glove_reader[current_pos][CLASSES.index(self.episode.target_object)]
        # if self.episode.task_data[0] in self.episode.det_gt[current_pos]:
        #     self.gt_det = self.episode.det_gt[current_pos][self.episode.task_data[0]]
        # else:
        #     self.gt_det = np.zeros(4)
        # optimal_solution = self.episode.environment.controller.shortest_path_to_target(current_pos, self.episode.task_data[0])
        if self.episode.optimal_actions is not None:
            optimal_solution = self.episode.optimal_actions[current_pos]
            self.optim_step = torch.zeros((1, 6))
            self.optim_step[0, optimal_solution] = 1
        # if (optimal_solution[0] is not None) and (len(optimal_solution[0]) < 99):
        #     optim_next_state = optimal_solution[0][0]
        # else:
        #     optim_next_state = current_pos
        # self.optim_step = self.optim_action(current_pos, optim_next_state)

        # optim_steps = torch.zeros((1, 1))
        # optim_steps[0, 0] = self.episode.environment.controller.shortest_path_to_target(current_pos, self.episode.task_data[0])[1]
        # model_input.optim_steps = toFloatTensor(optim_steps, self.gpu_id)

        # det_his = torch.zeros((1, 2))
        # if self.episode.current_det:
        #     det_his[0, 1] = 1
        # if self.episode.last_det:
        #     det_his[0, 0] = 1
        # model_input.det_relation = toFloatTensor(det_his, self.gpu_id)

        return model_input, self.model.forward(model_input, model_options)

    def preprocess_frame(self, frame):
        """ Preprocess the current frame for input into the model. """
        state = torch.Tensor(frame)
        return gpuify(state, self.gpu_id)

    def reset_hidden(self):
        if self.gpu_id >= 0:
            with torch.cuda.device(self.gpu_id):
                self.hidden = (
                    torch.zeros(1, self.hidden_state_sz).cuda(),
                    torch.zeros(1, self.hidden_state_sz).cuda(),
                )
        else:
            self.hidden = (
                torch.zeros(1, self.hidden_state_sz),
                torch.zeros(1, self.hidden_state_sz),
            )
        self.last_action_probs = gpuify(
            torch.zeros((1, self.action_space)), self.gpu_id
        )

    def repackage_hidden(self):
        self.hidden = (self.hidden[0].detach(), self.hidden[1].detach())
        self.last_action_probs = self.last_action_probs.detach()

    def optim_action(self, current_state, next_state):
        action_prob = torch.zeros((1, 6))
        if current_state == next_state:
            action_prob[0, 5] = 1
        else:
            current_x, current_z, current_rot, current_hor = current_state.split('|')
            next_x, next_z, next_rot, next_hor = next_state.split('|')
            if int(next_rot) == (int(current_rot) + 45):
                action_prob[0, 2] = 1
            elif int(next_rot) == (int(current_rot) - 45):
                action_prob[0, 1] = 1
            elif (current_hor == '0') and (next_hor == '30'):
                action_prob[0, 4] = 1
            elif (current_hor == '30') and (next_hor == '0'):
                action_prob[0, 3] = 1
            elif (current_x != next_x) or (current_z != next_z):
                action_prob[0, 0] = 1

        return action_prob

    def state(self):
        return self.preprocess_frame(self.episode.state_for_agent())

    def exit(self):
        pass


text:
# -*- coding: utf-8 -*-
from __future__ import print_function

import time

import numpy as np
import six
from six.moves import range

from . import SuperTask

class MockNIDAQ:
    def __init__(self):
        # self.data = mockdata.getMockData('cell')
        # self.data = hstack([self.data, self.data])
        # self.loopTime = 20.
        # self.dataPtr = 0.0
        self.clocks = {}
        self.devs = {
            'Dev1': {
                'aiChans': {'/Dev1/ai0': 0, '/Dev1/ai1': 0, '/Dev1/ai2': 0, '/Dev1/ai3': 0},
                'aoChans': {'/Dev1/ao0': 0, '/Dev1/ao1': 0, '/Dev1/ao2': 0, '/Dev1/ao3': 0},
                'ports': {'/Dev1/port0': 0},
                'lines': {
                    '/Dev1/port0/line0': 0,
                    '/Dev1/port0/line1': 0,
                    '/Dev1/port0/line2': 0,
                    '/Dev1/port0/line3': 0,
                },
            }
        }
        self.sampleRate = 20000.
        self.Val_Cfg_Default = -1
        self.Val_ChanForAllLines = 1
        self.Val_ChanPerLine = 0
        self.Val_Diff = 10106
        self.Val_FiniteSamps = 10178
        self.Val_NRSE = 10078
        self.Val_RSE = 10083
        self.Val_Rising = 10280
        self.Val_Task_Unreserve = 5
        self.Val_Volts = 10348

    def listAIChannels(self, dev):
        return list(self.devs[dev]['aiChans'].keys())

    def listAOChannels(self, dev):
        return list(self.devs[dev]['aoChans'].keys())

    def listDILines(self, dev):
        return list(self.devs[dev]['lines'].keys())

    def listDIPorts(self, dev):
        return list(self.devs[dev]['ports'].keys())

    def listDOLines(self, dev):
        return list(self.devs[dev]['lines'].keys())

    def listDOPorts(self, dev):
        return list(self.devs[dev]['ports'].keys())

    def listDevices(self):
        return list(self.devs.keys())

    def createSuperTask(self):
        return SuperTask.SuperTask(self)

    def start(self):
        self.dataPtr = time.time()

    def read(self, size):
        # dataLen = size / self.sampleRate
        # dataEnd = self.dataPtr + dataLen
        # now = time.time()
        # if dataEnd > now:
        #     time.sleep(dataEnd-now)
        # start = int((self.dataPtr % self.loopTime) * self.sampleRate)
        # stop = int(start + size)
        # self.dataPtr = dataEnd
        # print "read", start, stop
        # print "DAQ Returning %d:%d at %f" % (start, stop, time.time())
        # return (self.data[:, start:stop], size)
        return np.zeros(size)

    def GetReadAvailSampPerChan(self):
        return self.sampleRate * (time.time() - self.dataPtr)

    def createTask(self):
        return Task(self)

    def interpretMode(self, mode):
        modes = {
            'rse': self.Val_RSE,
            'nrse': self.Val_NRSE,
            'diff': self.Val_Diff,
            'chanperline': self.Val_ChanPerLine,
            'chanforalllines': self.Val_ChanForAllLines
        }
        if isinstance(mode, six.string_types):
            mode = mode.lower()
            mode = modes.get(mode, None)
        return mode

    def interpretChannel(self, chan):
        parts = chan.lstrip('/').split('/')
        dev = parts.pop(0)
        if len(parts) == 1 and parts[0].startswith('line'):
            # normalize "/Dev1/line0' => ('Dev1', 'port0/line0')
            chan = 'port0/' + parts[0]
        else:
            chan = '/'.join(parts)
        return dev, chan

    def writeAnalogSample(self, chan, value, vRange=(-10., 10.), timeout=10.0):
        """Set the value of an AO or DO port"""
        t = self.createTask()
        t.CreateAOVoltageChan(chan, "", vRange[0], vRange[1], self.Val_Volts, None)
        # t.WriteAnalogScalarF64(True, timeout, value, None)

    def readAnalogSample(self, chan, mode=None, vRange=(-10., 10.), timeout=10.0):
        """Get the value of an AI port"""
        if mode is None:
            mode = self.Val_Cfg_Default
        else:
            mode = self.interpretMode(mode)
        t = self.createTask()
        t.CreateAIVoltageChan(chan, "", mode, vRange[0], vRange[1], self.Val_Volts, None)
        # val = ctypes.c_double(0.)
        # t.ReadAnalogScalarF64(timeout, byref(val), None)
        # return val.value
        return 0.0

    def writeDigitalSample(self, chan, value, timeout=10.):
        """Set the value of an AO or DO port"""
        dev, chan = self.interpretChannel(chan)
        chan = '/%s/%s' % (dev, chan)
        self.devs[dev]['lines'][chan] = value
        # t = self.createTask()
        # t.CreateDOChan(chan, "", self.Val_ChanForAllLines)
        # t.WriteDigitalScalarU32(True, timeout, value, None)

    def readDigitalSample(self, chan, timeout=10.0):
        """Get the value of an AI port"""
        dev, chan = self.interpretChannel(chan)
        chan = '/%s/%s' % (dev, chan)
        return self.devs[dev]['lines'][chan]
        # t = self.createTask()
        # t.CreateDIChan(chan, "", self.Val_ChanForAllLines)
        # val = ctypes.c_ulong(0)
        # t.ReadDigitalScalarU32(timeout, byref(val), None)
        # return val.value

    def startClock(self, clock, duration):
        self.clocks[clock] = (time.time(), duration)

    def stopClock(self, clock):
        if clock not in self.clocks:
            return
        now = time.time()
        start, dur = self.clocks[clock]
        diff = (start + dur) - now
        if diff > 0:
            time.sleep(diff)

    def checkClock(self, clock):
        now = time.time()
        start, dur = self.clocks[clock]
        diff = (start + dur) - now
        return diff <= 0

class Task:
    def __init__(self, nd):
        self.nd = nd
        self.chans = []
        self.chOpts = []
        self.clock = None
        self.nativeClock = None
        self.data = None
        self.mode = None

    # def __getattr__(self, attr):
    #     return lambda *args: self

    def CreateAIVoltageChan(self, *args, **kargs):
        self.chans.append(args[0])
        self.chOpts.append(kargs)
        self.mode = 'ai'

    def CreateAOVoltageChan(self, *args, **kargs):
        self.chans.append(args[0])
        self.chOpts.append(kargs)
        self.mode = 'ao'

    def CreateDIChan(self, *args, **kargs):
        self.chans.append(args[0])
        self.chOpts.append(kargs)
        self.mode = 'di'

    def CreateDOChan(self, *args, **kargs):
        self.chans.append(args[0])
        self.chOpts.append(kargs)
        self.mode = 'do'

    def CfgSampClkTiming(self, clock, rate, b, c, nPts):
        if 'ai' in self.chans[0]:
            self.nativeClock = self.device() + '/ai/SampleClock'
        elif 'ao' in self.chans[0]:
            self.nativeClock = self.device() + '/ao/SampleClock'

        if clock == '':
            clock = None
        self.clock = clock
        self.rate = rate
        self.nPts = nPts
        # print self.chans, self.clock

    def GetSampClkMaxRate(self):
        return 2e6

    def device(self):
        return '/' + self.chans[0].split('/')[1]

    def write(self, data):
        self.data = data

        # Send data off to callbacks if they were specified
        # print "write:", self.chOpts
        for i in range(len(self.chOpts)):
            if 'mockFunc' in self.chOpts[i]:
                self.chOpts[i]['mockFunc'](data[i], 1.0 / self.rate)

        return len(data)

    def read(self):
        dur = self.nPts / self.rate
        tVals = np.linspace(0, dur, self.nPts)
        if 'd' in self.mode:
            data = np.empty((len(self.chans), self.nPts), dtype=np.int32)
        else:
            data = np.empty((len(self.chans), self.nPts))

        for i in range(len(self.chOpts)):
            if 'mockFunc' in self.chOpts[i]:
                data[i] = self.chOpts[i]['mockFunc']()
            else:
                data[i] = 0
        return (data, self.nPts)

    def start(self):
        # only start clock if it matches the native clock for this channel
        if self.clock is None or self.clock == self.nativeClock:
            dur = self.nPts / self.rate
            self.nd.startClock(self.nativeClock, dur)

    def stop(self):
        if self.clock is None:
            self.nd.stopClock(self.nativeClock)
        else:
            self.nd.stopClock(self.clock)

    def isDone(self):
        if self.clock is None:
            return self.nd.checkClock(self.nativeClock)
        else:
            return self.nd.checkClock(self.clock)

    def GetTaskNumChans(self):
        return len(self.chans)

    def isOutputTask(self):
        return self.mode in ['ao', 'do']

    def isInputTask(self):
        return self.mode in ['ai', 'di']

    def TaskControl(self, *args):
        pass

    def WriteAnalogScalarF64(self, a, timeout, val, b):
        pass

    def WriteDigitalScalarU32(self, a, timeout, val, b):
        pass

# class SuperTask:
#     def __init__(self, nd):
#         self.nd = nd
#
#     def __getattr__(self, attr):
#         print "SuperTask."+attr
#         return lambda *args, **kargs: self

NIDAQ = MockNIDAQ()

# class ModWrapper(object):
#     def __init__(self, wrapped):
#         self.wrapped = wrapped
#
#     def __getattr__(self, name):
#         try:
#             return getattr(self.wrapped, name)
#         except AttributeError:
#             if name[:3] == 'Val':
#                 return None
#             else:
#                 return lambda *args: NIDAQ
#
#     def __iter__

# sys.modules[__name__] = ModWrapper(sys.modules[__name__])


text:
import streamlit as st
import matplotlib.pyplot as plt
from PIL import Image
from source import preprocess_input_image, batch_predict, conv_float_int, combine_image, load_trained_model, burn_area
import numpy as np
from keras import backend as K
from tensorflow.python.lib.io import file_io
import boto3
from keras.models import load_model

with open("style.css") as f:
    st.markdown('<style>{}</style>'.format(f.read()), unsafe_allow_html=True)

st.title("Wild Fire Detection App")

st.sidebar.markdown("** App Status **")

model, session = load_trained_model("temp_model.h5")
K.set_session(session)

st.sidebar.markdown('Please upload a raw satellite image')
uploaded_file = st.sidebar.file_uploader("Upload png file", type=["png"])

if uploaded_file is not None:
    uploaded_image = Image.open(uploaded_file)
    st.markdown("** Original Raw Image: **")
    st.image(uploaded_image, width = 500)

    ### Preprocess the raw image
    #st.sidebar.text("Pre-processing the image...")
    with st.spinner("Pre-processing the image..."):
        input_image_array = np.array(uploaded_image)
        original_width, original_height, pix_num = input_image_array.shape
        new_image_array, row_num, col_num = preprocess_input_image(input_image_array)
        st.sidebar.success("Pre-processing has been done.")

    with st.spinner("Making the prediction..."):
        #### Make Prediction
        preds = batch_predict(new_image_array, model)
        # combine the images, and converted to 0-255 for display
        output_pred = conv_float_int(combine_image(preds, row_num, col_num, original_width, original_height, remove_ghost=True)[:,:,0])
        st.sidebar.success("Prediction has been done.")
        # add image mask to the probability array

    #### Show the picture
    st.markdown("** The Predicted Probability is **: ")
    plt.imshow(output_pred)
    st.pyplot()

    #threshold = st.sidebar.slider("Threshold", 0, 1, 0.25)
    preds_t = (preds > 0.25).astype(np.uint8)
    output_mask = conv_float_int(combine_image(preds_t, row_num, col_num, original_width, original_height, remove_ghost=False)[:,:,0])
    st.markdown("** The Predicted Mask is **: ")
    plt.imshow(output_mask)
    st.pyplot()
    #plt.imshow(output_mask)

    st.sidebar.markdown("** CO2 Emission Calculator **")
    forest_type = st.sidebar.selectbox("Please select the type of forest: ", ('Tropical Forest', 'Temperate Forest', 'Boreal Forest', 'Shrublands', 'Grasslands'))
    resolution = st.sidebar.text_input("Please enter the image resolution value: ", '10')

    area, biomass_burnt, equal_days = burn_area(output_mask = output_mask, resolution = float(resolution), forest_type = forest_type)
    st.sidebar.markdown('The total burnt area is:')
    st.sidebar.text("{0:.2f}".format(area/1e6) + ' km^2')
    st.sidebar.markdown('The total CO2 emitted is:')
    st.sidebar.text("{0:.2f}".format(biomass_burnt/1e6) + ' tons')
    st.sidebar.markdown("This is equivalent to:")
    st.sidebar.text("{0:.2f}".format(equal_days) + " days of Califorlia's daily electricity power emission")

max_stars_repo_name: Yanxiang-Yu/WildFireDetection

max_issues_repo_name: Yanxiang-Yu/WildFireDetection

max_forks_repo_name: Yanxiang-Yu/WildFireDetection


text:
"""Implement Omnisci Array type support
"""

__all__ = ['ArrayPointer', 'Array', 'OmnisciArrayType']

from rbc import typesystem, errors
from .omnisci_buffer import (BufferPointer, Buffer,
                             OmnisciBufferType,
                             omnisci_buffer_constructor)
from numba.core import extending, types

class OmnisciArrayType(OmnisciBufferType):
    """Omnisci Array type for RBC typesystem.

    :code:`Omnisci Array<T>` is defined as follows (using C++ syntax)::

      template<typename T>
      struct Array {
        T* ptr;
        size_t sz;
        bool is_null;
      }
    """
    @property
    def buffer_extra_members(self):
        return ('bool is_null',)

ArrayPointer = BufferPointer

class Array(Buffer):
    pass

@extending.lower_builtin(Array, types.Integer, types.StringLiteral)
@extending.lower_builtin(Array, types.Integer, types.NumberClass)
def omnisci_array_constructor(context, builder, sig, args):
    return omnisci_buffer_constructor(context, builder, sig, args)

@extending.type_callable(Array)
def type_omnisci_array(context):
    def typer(size, dtype):
        if isinstance(dtype, types.StringLiteral):
            element_type = typesystem.Type.fromstring(dtype.literal_value)
        elif isinstance(dtype, types.NumberClass):
            element_type = typesystem.Type.fromobject(dtype)
        else:
            raise errors.NumbaNotImplementedError(repr(dtype))
        return OmnisciArrayType((element_type,)).tonumba()
    return typer

max_stars_repo_name: guilhermeleobas/rbc

max_issues_repo_name: guilhermeleobas/rbc

max_forks_repo_name: guilhermeleobas/rbc

reason: from numba


text:
import pygame as pg
import numpy as np
from numba import njit

SCREEN_W, SCREEN_H = 800, 600
FOV_V = np.pi/4 # 45 degrees vertical fov
FOV_H = FOV_V*SCREEN_W/SCREEN_H

def main():
    pg.init()
    screen = pg.display.set_mode((SCREEN_W, SCREEN_H))
    running = True
    clock = pg.time.Clock()
    surf = pg.surface.Surface((SCREEN_W, SCREEN_H))

    # points = np.asarray([[1, 1, 1, 1, 1], [4, 2, 0, 1, 1], [1, .5, 3, 1, 1]])
    # triangles = np.asarray([[0,1,2]])
    points, triangles =  read_obj('obj models/teapot.obj')
    color_scale = 230/np.max(np.abs(points))

    camera = np.asarray([13, 0.5, 2, 3.3, 0])

    z_order = np.zeros(len(triangles))
    shade = z_order.copy()

    while running:
        pg.mouse.set_pos(SCREEN_W/2, SCREEN_H/2)
        surf.fill([50,127,200])
        elapsed_time = clock.tick()*0.001
        light_dir = np.asarray([np.sin(pg.time.get_ticks()/1000), 1, 1])
        light_dir = light_dir/np.linalg.norm(light_dir)

        for event in pg.event.get():
            if event.type == pg.QUIT: running = False
            if event.type == pg.KEYDOWN and event.key == pg.K_ESCAPE: running = False

        project_points(points, camera)
        sort_triangles(points, triangles, camera, light_dir, z_order, shade)

        for index in np.argsort(z_order):
            if z_order[index] == 9999: break
            triangle = [points[triangles[index][0]][3:], points[triangles[index][1]][3:], points[triangles[index][2]][3:]]
            color = shade[index]*np.abs(points[triangles[index][0]][:3])*color_scale +25
            pg.draw.polygon(surf, color, triangle)

        screen.blit(surf, (0,0)); pg.display.update()
        pg.display.set_caption(str(round(1/(elapsed_time+1e-16), 1)) + ' ' + str(camera))
        movement(camera, elapsed_time*10)

def movement(camera, elapsed_time):

    if pg.mouse.get_focused():
        p_mouse = pg.mouse.get_pos()
        camera[3] = (camera[3] + 10*elapsed_time*np.clip((p_mouse[0]-SCREEN_W/2)/SCREEN_W, -0.2, .2))%(2*np.pi)
        camera[4] = camera[4] + 10*elapsed_time*np.clip((p_mouse[1]-SCREEN_H/2)/SCREEN_H, -0.2, .2)
        camera[4] = np.clip(camera[4], -.3, .3)

    pressed_keys = pg.key.get_pressed()

    if pressed_keys[ord('e')]: camera[1] += elapsed_time
    elif pressed_keys[ord('q')]: camera[1] -= elapsed_time

    if (pressed_keys[ord('w')] or pressed_keys[ord('s')]) and (pressed_keys[ord('a')] or pressed_keys[ord('d')]):
        elapsed_time *= 0.707 # keep speed for diagonals

    if pressed_keys[pg.K_UP] or pressed_keys[ord('w')]:
        camera[0] += elapsed_time*np.cos(camera[3])
        camera[2] += elapsed_time*np.sin(camera[3])

    elif pressed_keys[pg.K_DOWN] or pressed_keys[ord('s')]:
        camera[0] -= elapsed_time*np.cos(camera[3])
        camera[2] -= elapsed_time*np.sin(camera[3])

    if pressed_keys[pg.K_LEFT] or pressed_keys[ord('a')]:
        camera[0] += elapsed_time*np.sin(camera[3])
        camera[2] -= elapsed_time*np.cos(camera[3])

    elif pressed_keys[pg.K_RIGHT] or pressed_keys[ord('d')]:
        camera[0] -= elapsed_time*np.sin(camera[3])
        camera[2] += elapsed_time*np.cos(camera[3])

@njit() #TODO: better vertical projection
def project_points(points, camera):

    for point in points:
        # Calculate xy angle of vector from camera point to projection point
        h_angle_camera_point = np.arctan((point[2]-camera[2])/(point[0]-camera[0] + 1e-16))

        # Check if it isn't pointing backwards
        if abs(camera[0]+np.cos(h_angle_camera_point)-point[0]) > abs(camera[0]-point[0]):
            h_angle_camera_point = (h_angle_camera_point - np.pi)%(2*np.pi)

        # Calculate difference between camera angle and pointing angle
        h_angle = (h_angle_camera_point-camera[3])%(2*np.pi)

        # Bring to -pi to pi range
        if h_angle > np.pi: h_angle =  h_angle - 2*np.pi

        # Calculate the point horizontal screen coordinate
        point[3] = SCREEN_W*h_angle/FOV_H + SCREEN_W/2

        # Calculate xy distance from camera point to projection point
        distance = np.sqrt((point[0]-camera[0])**2 + (point[1]-camera[1])**2 + (point[2]-camera[2])**2)

        # Calculate angle to xy plane
        v_angle_camera_point = np.arcsin((camera[1]-point[1])/distance)

        # Calculate difference between camera verticam angle and pointing vertical angle
        v_angle = (v_angle_camera_point - camera[4])%(2*np.pi)
        if v_angle > np.pi: v_angle =  v_angle - 2*np.pi

        # Bring to -pi to pi range
        if v_angle > np.pi: v_angle =  v_angle - 2*np.pi

        # Calculate the point vertical screen coordinate
        point[4] = SCREEN_H*v_angle/FOV_V + SCREEN_H/2

@njit()
def dot_3d(arr1, arr2):
    return arr1[0]*arr2[0] + arr1[1]*arr2[1] + arr1[2]*arr2[2]

@njit()
def sort_triangles(points, triangles, camera, light_dir, z_order, shade):
    for i in range(len(triangles)):
        triangle = triangles[i]

        # Use Cross-Product to get surface normal
        vet1 = points[triangle[1]][:3]  - points[triangle[0]][:3]
        vet2 = points[triangle[2]][:3] - points[triangle[0]][:3]

        # backface culling with dot product between normal and camera ray
        normal = np.cross(vet1, vet2)
        normal = normal/np.sqrt(normal[0]*normal[0] + normal[1]*normal[1] + normal[2]*normal[2])

        CameraRay = points[triangle[0]][:3] - camera[:3]
        dist2cam = np.sqrt(CameraRay[0]*CameraRay[0] + CameraRay[1]*CameraRay[1] + CameraRay[2]*CameraRay[2])
        CameraRay = CameraRay/dist2cam

        # get projected 2d points for filtering of offscreen triangles
        xxs = np.asarray([points[triangle[0]][3:5][0],  points[triangle[1]][3:5][0],  points[triangle[2]][3:5][0]])
        yys = np.asarray([points[triangle[0]][3:5][1],  points[triangle[1]][3:5][1],  points[triangle[2]][3:5][1]])

        # check valid values
        if (dot_3d(normal, CameraRay) < 0   and np.min(xxs) > - SCREEN_W and np.max(xxs) < 2*SCREEN_W
                                            and np.min(yys) > - SCREEN_H and np.max(yys) < 2*SCREEN_H):

            z_order[i] = -dist2cam

            # calculate shading, normalize, dot and to 0 - 1 range
            shade[i] = 0.5*dot_3d(light_dir, normal) + 0.5

        # big value for last positions in sort
        else: z_order[i] = 9999

def read_obj(fileName):
    vertices = []
    triangles = []
    f = open(fileName)
    for line in f:
        if line[:2] == "v ":
            index1 = line.find(" ") + 1
            index2 = line.find(" ", index1 + 1)
            index3 = line.find(" ", index2 + 1)

            vertex = [float(line[index1:index2]), float(line[index2:index3]), float(line[index3:-1]), 1, 1]
            vertices.append(vertex)

        elif line[0] == "f":
            index1 = line.find(" ") + 1
            index2 = line.find(" ", index1 + 1)
            index3 = line.find(" ", index2 + 1)

            triangles.append([int(line[index1:index2]) - 1, int(line[index2:index3]) - 1, int(line[index3:-1]) - 1])

    f.close()

    return np.asarray(vertices), np.asarray(triangles)

if __name__ == '__main__':
    main()
    pg.quit()

max_stars_repo_path: 1 - Simple 3D Engine.py

max_issues_repo_path: 1 - Simple 3D Engine.py

max_forks_repo_path: 1 - Simple 3D Engine.py

reason: import numpy,from numba


text:
"""
Preprocess the fluorescence traces to get a spike train matrix.
Based on the ChaLearn Connectomics challenge starter kit
by Bisakha Ray, Javier Orlandi and Olav Stetter. I tried to clean it
up since it didn't make use of numpy functions and it was woefully
lacking in useful comments.
"""
import os
import sys
import pickle
import gzip
import numpy as np

# Use the Agg backend in running on a server without the DISPLAY variable
if "DISPLAY" not in os.environ:
    import matplotlib
    matplotlib.use('Agg')

import matplotlib.pyplot as plt

# Import OOPSI. Available at: https://github.com/liubenyuan/py-oopsi
# or originally in Matlab from https://github.com/jovo/oopsi
pyoopsi_path = os.path.join("/home", "pyhawkes", "py-oopsi")
sys.path.append(pyoopsi_path)
import oopsi

def process_dataset(K=100,
                    suffix="_iNet1_Size100_CC01inh.txt",
                    dir="data/chalearn/small",
                    outfile="network1c.pkl"):

    # Get the full filenames
    fluor_file = os.path.join(dir, "fluorescence" + suffix)
    net_file   = os.path.join(dir, "network" + suffix)
    pos_file   = os.path.join(dir, "networkPositions" + suffix)

    # Parse the files
    F       = parse_fluorescence_file(fluor_file, K)

    # # Load the oopsi processed fluorescence
    # data_path = os.path.join("data", "chalearn", "small", "network1c.pkl.gz")
    # with gzip.open(data_path, 'r') as f:
    #     _, F, C, network, pos = cPickle.load(f)

    network = parse_network_file(net_file, K)
    pos     = parse_position_file(pos_file, K)

    # Discretize the fluorescence signal
    # Hardcode the bins
    # bins = np.array([-10, 0.17, 10]).reshape((1,3)).repeat(K, axis=0)
    # S, _ = discretize_fluorescence(F, edges=bins, binsui=False)
    # # S, bins = discretize_fluorescence(F, nbins=2, binsui=True)
    # S, bins = discretize_fluorescence(C, nbins=2, binsui=True)
    # # S = remove_double_spikes(S)

    # Get the spike times with oopsi
    # fast-oopsi,
    S,C = extract_spike_oopsi(F, dt=0.02)

    # Plot a segment of fluorescence traces and spikes
    start = 0
    end   = 10000
    k     = 0
    spks  = np.where(S[start:end, k])
    plt.figure()
    plt.plot(F[start:end, k], '-k')
    plt.plot(spks, F[spks,k], 'ro')
    plt.show()

    # Scatter plot the positions
    plt.figure()

    pres,posts = network.nonzero()
    for i,j in zip(pres,posts):
        if np.random.rand() < 0.25:
            plt.plot([pos[i,0], pos[j,0]],
                     [pos[i,1], pos[j,1]],
                     '-k', lw=0.5)
    plt.scatter(pos[:,0], pos[:,1], s=10, c='r', marker='o', facecolor='k')
    plt.xlabel("X")
    plt.ylabel("Y")
    plt.show()

    # Plot the network as a function of X position
    perm = np.argsort(pos[:,0])
    plt.figure()
    plt.imshow(network[np.ix_(perm, perm)], cmap="Greys", interpolation="none")
    plt.xlabel("Postsynaptic")
    plt.ylabel("Presynaptic")
    plt.show()

    with gzip.open(os.path.join(dir, outfile + ".gz"), 'w') as f:
        pickle.dump((S, F, C, network, pos), f, protocol=-1)

def parse_fluorescence_file(filename, K, delimiter=','):
    """
    Parse a fluorescence file to into a numpy matrix

    :param filename:        Name fo the file
    :param delimiter:       Delimeter between neurons (always ',')
    :return:                TxK matrix of fluorescence values
    """

    assert os.path.exists(filename), "File doesn't exist! %s" % filename

    # Define an iterator to yield split lines
    def iter_func():
        with open(filename, 'r') as f:
            for line in f:
                line = line.rstrip().split(delimiter)
                assert len(line) == K, "Line is not of length %d!" % K
                for item in line:
                    yield np.float(item)

    F = np.fromiter(iter_func(), dtype=np.float)
    F = F.reshape((-1, K))
    return F

def parse_network_file(filename, K, vmin=0):
    """
    Parse a network file where each row is of the form I,J,W
    denoting a connection from neuron I to neuron J with weight W.
    :param filename:
    :return:
    """
    network = np.zeros((K,K))

    with open(filename, 'r') as f:
        for line in f:
            # Read the line
            i,j,w = line.rstrip().split(',')
            # Cast to ints
            i,j,w = int(i)-1, int(j)-1, int(w)

            network[i,j] = w

    network = np.clip(network, vmin, np.inf)

    return network

def parse_position_file(filename, K):
    """
    Parse a network file where each row is of the form I,J,W
    denoting a connection from neuron I to neuron J with weight W.
    :param filename:
    :return:
    """
    pos = np.zeros((K,2))

    with open(filename, 'r') as f:
        for k in range(K):
            line = f.readline()
            # Read the line
            x,y = line.rstrip().split(',')
            # Cast to floats
            x,y = float(x), float(y)

            pos[k,0] = x
            pos[k,1] = y

    return pos

def extract_spike_oopsi(F, dt):
    """
    Extract the spike times with OOPSI

    :param F:           Fluorescence data (each row a time bin, each column a neuron).
    :param dt:          Time bin size
    :return             The discretized signal
    """
    D = np.zeros_like(F)
    C = np.zeros_like(F)
    for k in range(F.shape[1]):
        print("Running oopsi on neuron ", k)
        D[:,k], C[:,k] = oopsi.fast(F[:,k], dt=dt, iter_max=6)

    # Cast D to an integer matrix
    # D = D.astype(np.int)

    return D, C

def discretize_fluorescence(F,
                            nbins=2,
                            edges=None,
                            binsui=True,
                            hpfilter=True,
                            debug=False):
    """
    Discretizes the fluorescence signal so it
    can be used to compute the joint PDF. If conditioning is applied, the
    entries above the conditioning level are returned in the G vector.

    Example usage:      D = discretizeFluorescenceSignal(F)

    :param F:           Fluorescence data (each row a time bin, each column a neuron).
    :param binedges:        An array of bin edges (min length 3)
    :param nbins:       If bins is None, use nbins evenly spaced bins for each neuron
    :param binsui:      If true, pop up a UI to set the bin threshold

    :param hpfilter:    Apply a high pass filter to the fluorescence signal,
                        i.e., work with the derivative (default true).

    :return             The discretized signal
    """
    T,K = F.shape

    # Apply the high pass filter
    if hpfilter:
        Fhat = np.diff(F, axis=0)
    else:
        Fhat = F

    # Compute the range of the fluorescence for each neuron
    F_min = np.amin(Fhat, axis=0)
    F_max = np.amax(Fhat, axis=0)

    # Discretize the signal
    D = -1 * np.ones((T-1,K))

    # If global bins are not given, use neuron-specific binning evenly spaced
    # between F_min and F_max
    if edges is not None:
        bins_given = True
    else:
        edges = np.zeros((K, nbins+1))
        bins_given = False

    for k in range(K):
        if not bins_given:
            edges[k,:] = np.linspace(F_min[k]-1e-3, F_max[k]+1e-3, num=nbins+1)

        if binsui:
            # Histogram of the fluorescence
            fig = plt.figure()
            plt.hist(Fhat[:,k], bins=100, normed=True)
            for edge in edges[k,:]:
                plt.plot([edge, edge],
                         [0, plt.gca().get_ylim()[1]], '-r')
            plt.plot()
            plt.xlabel('F')
            plt.ylabel('p(F)')
            plt.title('Neuron %d' % k)

            # Add an event handler to get the threshold
            def onclick(event):
                edges[k,1] = event.xdata
                print("Neuron: ", k, "\tThreshold: %.3f" % edges[k,1])
                plt.close()

            fig.canvas.mpl_connect('button_press_event', onclick)
            plt.show()

        # Digitize this column
        D[:,k] = np.digitize(Fhat[:,k], edges[k,:]) - 1

    assert np.all(D >= 0) and np.all(D < nbins), "Error in digitizing!"

    # Cast D to an integer matrix
    D = D.astype(np.int)

    return D, edges

def remove_double_spikes(D):
    Dhat = D.copy()

    # Identify back to back spikes
    doubles = (Dhat[1:,:] > 0) & (D[:-1] > 0)

    # Remove the first of the pair
    Dhat = Dhat[:-1,:]
    Dhat[doubles] = 0

    return Dhat

for i in [1,4]:
    print("Processing network ", i)
    process_dataset(suffix="_iNet1_Size100_CC0%dinh.txt" % i,
                    outfile="network%d_oopsi.pkl" % i)

max_stars_repo_name: nticea/superhawkes

max_issues_repo_name: nticea/superhawkes

max_forks_repo_name: nticea/superhawkes