text:
import scipy.io as sio
import joblib
import numpy as np
import h5py
import tensorflow as tf
from tqdm import tqdm
import cv2
from vgg19 import VGG19
from keras.preprocessing import image as ki
from keras.models import Model, Sequential
from keras.layers import Flatten, Dense, Input, Lambda, Dropout
from keras.layers import Convolution2D, MaxPooling2D, BatchNormalization
from keras import backend as K
from imagenet_utils import preprocess_input
from scipy.ndimage.interpolation import zoom
from selectivity import si
from sklearn.base import BaseEstimator
from sklearn.metrics import explained_variance_score as fev
import csv

def DeepOracle(target_shape=(14,14,512*3)):

    # Determine proper input shape
    if K.image_dim_ordering() == 'th':
        input_shape = (num_feat, None, None)
    else:
        input_shape = target_shape
        # input_shape = np.squeeze(activation_input).shape

    # Convolution Architecture
    # Block 1
    model = Sequential()
    # model.add(Convolution2D(2, (1, 1), activation='relu', padding='same',
    #     name='block1_conv1', input_shape=target_shape))
    # model.add(BatchNormalization(name='block1_bn1'))

    model.add(Convolution2D(1, (1, 1), activation='relu', padding='same',
        name='block1_conv1', input_shape=target_shape))
    model.add(BatchNormalization(name='block1_bn1'))
    model.add(Flatten(name='flatten'))
    model.add(Dense(300, name='fc'))
    model.add(Dropout(0.5))
    model.add(Dense(37, name='predictions'))

    return model

def get_activation(base_model, layer):
        return Model(input=base_model.input, output=base_model.get_layer(layer).output)

def get_activations(base_model, layers):

    activations = []
    for layer in layers:
        activations.extend([ get_activation(base_model, layer) ])

    return activations

def gen_y_fake(y, sem_y):
    loc = np.zeros_like(y)
    z = np.random.normal(loc,sem_y)
    return (y + z)

def pairwise_pcc(y,y_pred):
    ppcc = [ np.corrcoef(y_pred[:,i],y[:,i]) for i in np.arange(37)]
    return np.nan_to_num(np.array(ppcc)[:,1,0])

def train_test(idxs, frac):
    # Randomize indices and partition
    randomized_idxs = np.random.permutation(idxs)
    c = round(len(idxs)*frac)
    train_idxs = randomized_idxs[:c]
    valid_idxs = randomized_idxs[c:]

    return (train_idxs, valid_idxs)

def build_random(using=None, choose=3, target_scale=None):
    # layers = np.random.choice(base_model_layers[-14:],3,replace=False)
    layers = np.random.choice(using,choose,replace=False)

    return (layers, build(layers=layers, target_scale=target_scale))

def build(layers=None, target_scale=None):

    base_model = VGG19(weights='imagenet')
    print('extracting layers:')
    print(layers)

    f = h5py.File('../data/02activations.hdf5', 'r+')
    activations = []
    for layer in layers:
        layer_activation = []
        try:
            print('extracting ',layer, ' from cache...')
            layer_activation = f['activations/'+layer][:]
        except:
            print(layer,' not in cache, rebuilding from source...')
            images = [ ki.img_to_array(ki.load_img('../data/images/%g.jpg'%id, target_size=(224,224))) for id in np.arange(956) ]
            images = np.array(images)

            activation_fetcher = get_activation(base_model, layer)
            layer_activation = activation_fetcher.predict(images,batch_size=32,verbose=1).astype(np.float16)

            # if rescaling uncomment this

            # num_imgs = layer_activation.shape[0]
            # num_features = layer_activation.shape[3]
            # sc_fac = tuple(list(np.array([num_imgs, target_scale[0], target_scale[1], num_features])/np.array(layer_activation.shape)))
            # print('Rescaling by factor: ', sc_fac)
            # print('resizing feature map...')
            # layer_activation = zoom(layer_activation, sc_fac, order=0)
            # for img in tqdm(images):
            #     img = np.expand_dims(img, axis=0)
            #     layer_activation.extend([ feature ])

            # layer_activation = np.concatenate(layer_activation, axis=0)
            print(layer_activation.shape)
            print('caching ',layer,'...')
            f.create_dataset('activations/'+layer, data=layer_activation)
            del images
            pass

        activations.extend([ layer_activation.astype(np.float32) ])

    f.close()
    # del f, layer_activation
    # gc.collect()

    activations = np.concatenate(activations, axis=3)

    return activations

def eval_network(kfold_sets, activations, activity):
    y_pred_list = []
    ppcc_list = []
    fev_list = []
    ppcc_baseline_list = []
    for train_idxs, valid_idxs in tqdm(kfold_sets, desc='kfold', leave=False):
        train_activity = activity[train_idxs]
        valid_activity = activity[valid_idxs]

        train_activations = activations[train_idxs]
        valid_activations = activations[valid_idxs]

        mod = DeepOracle(activations.shape[1:])

        # reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=3, min_lr=0.0001)
        mod.compile(
                optimizer='adam',
                loss='mse',
                metrics=[])

        mod.fit(train_activations, train_activity,
                batch_size=32,
                epochs=20,
                verbose=1
                )

        y_pred = mod.predict(valid_activations, batch_size=32)
        ppcc = pairwise_pcc(valid_activity,y_pred)
        fev_vals = fev(valid_activity, y_pred, multioutput='raw_values')
        fev_list.extend([fev_vals])
        ppcc_list.extend([ppcc])

        #y_baseline = gen_y_fake(valid_activity, sem_activity[valid_idxs])

        #ppcc_baseline = pairwise_pcc(y_baseline, valid_activity)
        #y_pred_list.extend([y_pred])
        #ppcc_baseline_list.extend([ ppcc_baseline ])

        #si = si(valid_activity)

    return dict(ppcc_list=ppcc_list, fev_list=fev_list)

if __name__ == '__main__':
    try:
        tf.InteractiveSession()
    except:
        pass

    # mat_file = '../data/02mean_d1.mat'
    early_file = '../data/02_stats_early.mat'
    late_file = '../data/02_stats_late.mat'
    print('loading ', early_file, '...')
    print('loading ', late_file, '...')

    e_activity_contents = sio.loadmat(early_file)
    l_activity_contents = sio.loadmat(late_file)

    e_activity = e_activity_contents['resp_mean'].swapaxes(0,1)
    l_activity = l_activity_contents['resp_mean'].swapaxes(0,1)

    # images = mat_contents['images']
    train_frac = 0.8

    # All images
    # idxs = np.arange(956)

    # Small Natural Images
    # idxs = np.arange(540)[::2]

    # Small Natural Images and gratings
    idxs = np.arange(540)[::2]
    idxs = np.concatenate([idxs, np.arange(540,732)])

    kfold_sets = [ train_test(idxs, train_frac) for _ in np.arange(5) ]
    target_scale = (956,56,56,128)

    #base_model_layers = [ layer.name for layer in base_model.layers[1:-5] ]

    # DeepGaze II layers

    tails = [
        ['block1_conv2'],
        ['block2_conv2'],
        ['block3_conv4']
        ]
    bottom_layers = [
        ['block1_conv1'],
        ['block1_conv2'],
        ['block2_conv1'],
        ['block2_conv2']
        ]
    block3 = [
        ['block3_conv1'],
        ['block3_conv2'],
        ['block3_conv3'],
        ['block3_conv4']
    ]

    block4 = [
        ['block4_conv1'],
        ['block4_conv2'],
        ['block4_conv3'],
        ['block4_conv4']
    ]

    block5 = [
        ['block5_conv1'],
        ['block5_conv2'],
        ['block5_conv3'],
        ['block5_conv4']
        ]
    #DG2 = np.array(base_model_layers)[[16, 17, 19]]

    use_layers = []
    use_layers.extend(bottom_layers)
    use_layers.extend(block3)
    use_layers.extend(block4)
    use_layers.extend(block5)
    use_layers = use_layers[1:-1]

    early_results = []
    late_results = []
    for block in tqdm(use_layers, unit='layer'):
        print('starting...')
        activations = build(block, target_scale=(112,112))

        print('evaluating model on early activity for ', block)
        early_eval_metrics = eval_network(kfold_sets, activations, e_activity)

        print('evaluating model on late activity for ', block)
        late_eval_metrics = eval_network(kfold_sets, activations, l_activity)

        early_eval_metrics['network']=block
        late_eval_metrics['network']=block

        early_results.extend([ early_eval_metrics ])
        late_results.extend([ late_eval_metrics ])

        print('overwriting early results...')
        joblib.dump(early_results, 'tmp/early_all_layers.pkl')

        print('overwriting late results...')
        joblib.dump(late_results, 'tmp/late_all_layers.pkl')

reason: import numpy,import scipy,from scipy


text:
"""
Just run optimal pricing in simple situations
"""
# import packages
from tqdm import tqdm
import numpy             as np
import pandas            as pd
import matplotlib.pyplot as plt
import seaborn as sns
import json
from importlib import reload
import os
import sys

stdout = sys.stdout
import cplex

from numpy.core.multiarray import ndarray
from online_myopic_m import online_myopic
import offline_optimal_m as oom
import generate_simulation_data_m as gsdm
from generate_simulation_data import generate_synthetic_data_edge_cloud
from other_functions import display_allocation

pd.set_option("display.max_columns", None)

# Disable priting
def block_print():
    sys.stdout = open(os.devnull, 'w')

# Restore
def enable_print():
    sys.stdout = stdout

# set the parameters
n_tasks = 300
n_time = int(n_tasks/10)
seed = 0
n_nodes = 6
resource_coefficient_original = 0.3
resource_coefficient = (resource_coefficient_original * n_tasks / n_time)
high_value_slackness = 0
low_value_slackness = 6
valuation_ratio = 3
# generate synthetic data for the simulation
df_tasks, df_nodes, n_time, n_tasks, n_nodes = \
    generate_synthetic_data_edge_cloud(n_tasks=n_tasks,
                                       n_time=n_time,
                                       n_nodes=n_nodes,
                                       high_value_slackness_lower_limit=high_value_slackness,
                                       high_value_slackness_upper_limit=high_value_slackness + 2,
                                       low_value_slackness_lower_limit=low_value_slackness,
                                       low_value_slackness_upper_limit=low_value_slackness + 2,
                                       k_resource=resource_coefficient,
                                       vc_ratio=valuation_ratio,
                                       seed=seed)
df_tasks = df_tasks.rename(columns={"storage": "DISK"})
df_nodes = df_nodes.rename(
    columns={"storage": "DISK", "storage_cost": "DISK_cost"})

print("df_tasks:")
print(df_tasks.head())
print("df_nodes:")
print(df_nodes.head())

# run online myopic algo.
social_welfare, number_of_allocated_tasks, allocation_scheme = \
    online_myopic(df_tasks, df_nodes, n_time, n_tasks, n_nodes)
print("social welfare:", social_welfare)
print("number of allocated tasks:", number_of_allocated_tasks)
# print the total value of tasks
total_value = 0
for i in range(n_tasks):
    total_value += (df_tasks.loc[i, "valuation_coefficient"] *
                    df_tasks.loc[i, "usage_time"])
print(f"total value of tasks = {total_value}")

# print(f"allocation_scheme = {allocation_scheme}")

# # another situation
# df_tasks['DISK'] = df_tasks['DISK'].apply(lambda x: 0.1)
# print("df_tasks:")
# print(df_tasks.head())
# print("df_nodes:")
# print(df_nodes.head())
#
# # set price range
# price_upper_value = 600
# price_lower_value = 0
# granularity = (price_upper_value-price_lower_value) / 10
#
# # run online myopic algo.
# social_welfare, number_of_allocated_tasks, optimal_phi, allocation_scheme = \
#     opm.optimal_pricing(df_tasks, df_nodes, n_time, n_tasks, n_nodes,
#                         granularity=granularity,
#                         price_upper_value=price_upper_value,
#                         price_lower_value=price_lower_value, global_phi=False)
# print("social welfare:", social_welfare)
# print("number of allocated tasks:", number_of_allocated_tasks)
# # print the total value of tasks
# total_value = 0
# for i in range(n_tasks):
#     total_value += (df_tasks.loc[i, "valuation_coefficient"] *
#                     df_tasks.loc[i, "usage_time"])
# print(f"total value of tasks = {total_value}")
# display_allocation(allocation_scheme, n_nodes, n_tasks, df_tasks, n_time)
# print(f"optimal_phi = {optimal_phi}")

ext: py

reason: import numpy,from numpy


text:
import cant_utils as cu
import numpy as np
import matplotlib.pyplot as plt
import glob
import bead_util as bu
import tkinter
import tkinter.filedialog
import os, sys
from scipy.optimize import curve_fit
import bead_util as bu
from scipy.optimize import minimize_scalar as minimize

dirs = [13,14,15,16]
cal = 5.0e-14

ddict = bu.load_dir_file( "/home/charles/opt_lev_analysis/scripts/dirfiles/dir_file_june2017.txt" )

load_from_file = False

new_obj = cu.Data_dir('shit_path', [0,0,0], "wheeeee")

new_obj.load_H("optphase2_Hout.p")
#new_obj.plot_H(phase=True, label=True)

Hmat = new_obj.build_avgH(fthresh = 60)

plt.imshow(np.abs(Hmat))
plt.show()

Hmat_diag = np.linalg.inv(Hmat)

plt.imshow(np.abs(Hmat_diag))
plt.show()

print(np.abs(Hmat))

print(np.abs(Hmat_diag))

print(np.abs(np.einsum('ij,jk -> ik', Hmat, Hmat_diag)))

reason: import numpy,from scipy


text:
"""
    heightmaps.py

    Functions for computing general heightmaps and updating the special HeightMap
    attribute of modern chunks.
"""
from __future__ import absolute_import

from numpy import zeros, argmax
import numpy
#
#
# def computeChunkHeightMap(chunk, HeightMap=None):
#     """Computes the HeightMap array for a chunk, which stores the lowest
#     y-coordinate of each column where the sunlight is still at full strength.
#     The HeightMap array is indexed z,x contrary to the blocks array which is x,z,y.
#
#     If HeightMap is passed, fills it with the result and returns it. Otherwise, returns a
#     new array.
#     """
#     sectionHeights = []
#     for cy in chunk.sectionPositions():
#         section = chunk.getSection(cy)
#         if section is None:
#             continue
#
#         opaques = chunk.blocktypes.opacity[section.Blocks]
#         h = extractHeights(opaques)
#         h += (cy << 4)
#         sectionHeights.append(h)
#
#     if len(sectionHeights):
#         heights = sectionHeights.pop(0)
#         for h in sectionHeights:
#             numpy.maximum(heights, h, out=heights)
#
#         heights = heights.swapaxes(0, 1)
#     else:
#         heights = numpy.zeros((16, 16), 'uint32')
#
#     if HeightMap is None:
#         return heights.astype('uint8')
#     else:
#         HeightMap[:] = heights
#         return HeightMap

def extractHeights(array):
    """ Given an array of bytes shaped (y, z, x), return the coordinates of the highest
    non-zero value in each y-column into heightMap
    """

    # The fastest way I've found to do this is to make a boolean array with >0,
    # then turn it upside down with ::-1 and use argmax to get the _first_ nonzero
    # from each column.

    l, w = array.shape[1:]
    heightMap = numpy.empty((l, w), 'int16')

    heights = argmax((array > 0)[::-1], 0)
    heights = array.shape[0] - heights

    # if the entire column is air, argmax finds the first air block and the result is a top height column
    # top height columns won't ever have air in the top block so we can find air columns by checking for both
    heights[(array[-1] == 0) & (heights == array.shape[0])] = 0

    heightMap[:] = heights

    return heightMap


text:
import openmdao.api as om
import numpy as np

class TireConstraintODE(om.ExplicitComponent):
    def initialize(self):
        self.options.declare("num_nodes", types=int)

    def setup(self):
        nn = self.options["num_nodes"]

        # constants
        self.add_input("mu0_y", val=1.68, desc="lateral friction coefficient", units=None)
        self.add_input("mu0_x", val=1.68, desc="longitudinal friction coefficient", units=None)
        self.add_input("k_mu", val=-0.0, desc="tire load sensitivity", units=None)
        self.add_input("a", val=1.8, desc="cg to front distance", units="m")
        self.add_input("b", val=1.6, desc="cg to rear distance", units="m")
        self.add_input("M", val=0.0, desc="mass", units="kg")
        self.add_input("g", val=9.8, desc="mass", units="m/s**2")

        # states
        self.add_input("S_fl", val=np.zeros(nn), desc="longitudinal force fl", units="N")
        self.add_input("S_fr", val=np.zeros(nn), desc="longitudinal force fr", units="N")
        self.add_input("S_rl", val=np.zeros(nn), desc="longitudinal force rl", units="N")
        self.add_input("S_rr", val=np.zeros(nn), desc="longitudinal force rr", units="N")

        self.add_input("F_fl", val=np.zeros(nn), desc="lateral force fl", units="N")
        self.add_input("F_fr", val=np.zeros(nn), desc="lateral force fr", units="N")
        self.add_input("F_rl", val=np.zeros(nn), desc="lateral force rl", units="N")
        self.add_input("F_rr", val=np.zeros(nn), desc="lateral force rr", units="N")

        self.add_input("N_rr", val=np.zeros(nn), desc="normal load rr", units="N")
        self.add_input("N_fr", val=np.zeros(nn), desc="normal load fr", units="N")
        self.add_input("N_rl", val=np.zeros(nn), desc="normal load rl", units="N")
        self.add_input("N_fl", val=np.zeros(nn), desc="normal load fl", units="N")

        # outputs
        self.add_output("c_rr", val=np.zeros(nn), desc="tire load constraint rr", units=None)
        self.add_output("c_rl", val=np.zeros(nn), desc="tire load constraint rr", units=None)
        self.add_output("c_fr", val=np.zeros(nn), desc="tire load constraint rr", units=None)
        self.add_output("c_fl", val=np.zeros(nn), desc="tire load constraint rr", units=None)

        # # Setup partials
        arange = np.arange(self.options["num_nodes"], dtype=int)

        # #partials
        self.declare_partials(of="c_rr", wrt="S_rr", rows=arange, cols=arange)
        self.declare_partials(of="c_rr", wrt="F_rr", rows=arange, cols=arange)
        self.declare_partials(of="c_rr", wrt="N_rr", rows=arange, cols=arange)

        self.declare_partials(of="c_rl", wrt="S_rl", rows=arange, cols=arange)
        self.declare_partials(of="c_rl", wrt="F_rl", rows=arange, cols=arange)
        self.declare_partials(of="c_rl", wrt="N_rl", rows=arange, cols=arange)

        self.declare_partials(of="c_fr", wrt="S_fr", rows=arange, cols=arange)
        self.declare_partials(of="c_fr", wrt="F_fr", rows=arange, cols=arange)
        self.declare_partials(of="c_fr", wrt="N_fr", rows=arange, cols=arange)

        self.declare_partials(of="c_fl", wrt="S_fl", rows=arange, cols=arange)
        self.declare_partials(of="c_fl", wrt="F_fl", rows=arange, cols=arange)
        self.declare_partials(of="c_fl", wrt="N_fl", rows=arange, cols=arange)

    def compute(self, inputs, outputs):
        mu0_y = inputs["mu0_y"]
        mu0_x = inputs["mu0_x"]
        k_mu = inputs["k_mu"]
        a = inputs["a"]
        b = inputs["b"]
        M = inputs["M"]
        g = inputs["g"]

        S_fl = inputs["S_fl"]
        S_fr = inputs["S_fr"]
        S_rl = inputs["S_rl"]
        S_rr = inputs["S_rr"]

        N_fl = inputs["N_fl"]
        N_fr = inputs["N_fr"]
        N_rl = inputs["N_rl"]
        N_rr = inputs["N_rr"]

        F_fl = inputs["F_fl"]
        F_fr = inputs["F_fr"]
        F_rl = inputs["F_rl"]
        F_rr = inputs["F_rr"]

        N0_rr = (M * g / 2) * (a / (a + b))
        N0_rl = (M * g / 2) * (a / (a + b))
        N0_fr = (M * g / 2) * (b / (a + b))
        N0_fl = (M * g / 2) * (b / (a + b))

        mu_x_rr = mu0_x + k_mu * N_rr / N0_rr
        mu_x_rl = mu0_x + k_mu * N_rl / N0_rl
        mu_x_fr = mu0_x + k_mu * N_fr / N0_fr
        mu_x_fl = mu0_x + k_mu * N_fl / N0_fl

        mu_y_rr = mu0_y + k_mu * N_rr / N0_rr
        mu_y_rl = mu0_y + k_mu * N_rl / N0_rl
        mu_y_fr = mu0_y + k_mu * N_fr / N0_fr
        mu_y_fl = mu0_y + k_mu * N_fl / N0_fl

        outputs["c_rr"] = (S_rr / (N_rr * mu_x_rr)) ** 2 + (F_rr / (N_rr * mu_y_rr)) ** 2
        outputs["c_rl"] = (S_rl / (N_rl * mu_x_rl)) ** 2 + (F_rl / (N_rl * mu_y_rl)) ** 2
        outputs["c_fr"] = (S_fr / (N_fr * mu_x_fr)) ** 2 + (F_fr / (N_fr * mu_y_fr)) ** 2
        outputs["c_fl"] = (S_fl / (N_fl * mu_x_fl)) ** 2 + (F_fl / (N_fl * mu_y_fl)) ** 2

    def compute_partials(self, inputs, jacobian):
        mu0_y = inputs["mu0_y"]
        mu0_x = inputs["mu0_x"]
        k_mu = inputs["k_mu"]
        a = inputs["a"]
        b = inputs["b"]
        M = inputs["M"]
        g = inputs["g"]

        S_fl = inputs["S_fl"]
        S_fr = inputs["S_fr"]
        S_rl = inputs["S_rl"]
        S_rr = inputs["S_rr"]

        N_fl = inputs["N_fl"]
        N_fr = inputs["N_fr"]
        N_rl = inputs["N_rl"]
        N_rr = inputs["N_rr"]

        F_fl = inputs["F_fl"]
        F_fr = inputs["F_fr"]
        F_rl = inputs["F_rl"]
        F_rr = inputs["F_rr"]

        N0_rr = (M * g / 2) * (a / (a + b))
        N0_rl = (M * g / 2) * (a / (a + b))
        N0_fr = (M * g / 2) * (b / (a + b))
        N0_fl = (M * g / 2) * (b / (a + b))

        jacobian["c_rr", "S_rr"] = (2 * S_rr) / (N_rr ** 2 * (mu0_x + (N_rr * k_mu) / N0_rr) ** 2)
        jacobian["c_fr", "S_fr"] = (2 * S_fr) / (N_fr ** 2 * (mu0_x + (N_fr * k_mu) / N0_fr) ** 2)
        jacobian["c_fl", "S_fl"] = (2 * S_fl) / (N_fl ** 2 * (mu0_x + (N_fl * k_mu) / N0_fl) ** 2)
        jacobian["c_rl", "S_rl"] = (2 * S_rl) / (N_rl ** 2 * (mu0_x + (N_rl * k_mu) / N0_rl) ** 2)

        jacobian["c_rr", "F_rr"] = (2 * F_rr) / (N_rr ** 2 * (mu0_y + (N_rr * k_mu) / N0_rr) ** 2)
        jacobian["c_fr", "F_fr"] = (2 * F_fr) / (N_fr ** 2 * (mu0_y + (N_fr * k_mu) / N0_fr) ** 2)
        jacobian["c_fl", "F_fl"] = (2 * F_fl) / (N_fl ** 2 * (mu0_y + (N_fl * k_mu) / N0_fl) ** 2)
        jacobian["c_rl", "F_rl"] = (2 * F_rl) / (N_rl ** 2 * (mu0_y + (N_rl * k_mu) / N0_rl) ** 2)

        jacobian["c_rr", "N_rr"] = (
            -(2 * F_rr ** 2) / (N_rr ** 3 * (mu0_y + (k_mu * N_rr) / N0_rr) ** 2) -
            (2 * S_rr ** 2) / (N_rr ** 3 * (mu0_x + (k_mu * N_rr) / N0_rr) ** 2) -
            (2 * F_rr ** 2 * k_mu) /
            (N_rr ** 2 * N0_rr * (mu0_y + (k_mu * N_rr) / N0_rr) ** 3) -
            (2 * k_mu * S_rr ** 2) /
            (N_rr ** 2 * N0_rr * (mu0_x + (k_mu * N_rr) / N0_rr) ** 3)
        )
        jacobian["c_rl", "N_rl"] = (
            -(2 * F_rl ** 2) / (N_rl ** 3 * (mu0_y + (k_mu * N_rl) / N0_rl) ** 2) -
            (2 * S_rl ** 2) / (N_rl ** 3 * (mu0_x + (k_mu * N_rl) / N0_rl) ** 2) -
            (2 * F_rl ** 2 * k_mu) /
            (N_rl ** 2 * N0_rl * (mu0_y + (k_mu * N_rl) / N0_rl) ** 3) -
            (2 * k_mu * S_rl ** 2) /
            (N_rl ** 2 * N0_rl * (mu0_x + (k_mu * N_rl) / N0_rl) ** 3)
        )
        jacobian["c_fr", "N_fr"] = (
            -(2 * F_fr ** 2) / (N_fr ** 3 * (mu0_y + (k_mu * N_fr) / N0_fr) ** 2) -
            (2 * S_fr ** 2) / (N_fr ** 3 * (mu0_x + (k_mu * N_fr) / N0_fr) ** 2) -
            (2 * F_fr ** 2 * k_mu) /
            (N_fr ** 2 * N0_fr * (mu0_y + (k_mu * N_fr) / N0_fr) ** 3) -
            (2 * k_mu * S_fr ** 2) /
            (N_fr ** 2 * N0_fr * (mu0_x + (k_mu * N_fr) / N0_fr) ** 3)
        )
        jacobian["c_fl", "N_fl"] = (
            -(2 * F_fl ** 2) / (N_fl ** 3 * (mu0_y + (k_mu * N_fl) / N0_fl) ** 2) -
            (2 * S_fl ** 2) / (N_fl ** 3 * (mu0_x + (k_mu * N_fl) / N0_fl) ** 2) -
            (2 * F_fl ** 2 * k_mu) /
            (N_fl ** 2 * N0_fl * (mu0_y + (k_mu * N_fl) / N0_fl) ** 3) -
            (2 * k_mu * S_fl ** 2) /
            (N_fl ** 2 * N0_fl * (mu0_x + (k_mu * N_fl) / N0_fl) ** 3)
        )

max_stars_repo_name: kaushikponnapalli/dymos

max_issues_repo_name: kaushikponnapalli/dymos

max_forks_repo_name: kaushikponnapalli/dymos

reason: import numpy


text:
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Generate adversarial example for yolov3_darknet53 by DeepFool"""
import os
import argparse
import datetime
import numpy as np

from mindspore import Tensor
from mindspore.nn import Cell
from mindspore.ops import operations as P
from mindspore.context import ParallelMode
from mindspore import context
from mindspore.train.serialization import load_checkpoint, load_param_into_net
import mindspore as ms

from mindarmour.adv_robustness.attacks import DeepFool

from src.yolo import YOLOV3DarkNet53
from src.logger import get_logger
from src.yolo_dataset import create_yolo_dataset
from src.config import ConfigYOLOV3DarkNet53

def parse_args():
    """Parse arguments."""
    parser = argparse.ArgumentParser('mindspore coco testing')

    # device related
    parser.add_argument('--data_dir', type=str, default='', help='train data dir')
    parser.add_argument('--pretrained', default='', type=str, help='model_path, local pretrained model to load')
    parser.add_argument('--samples_num', default=1, type=int, help='Number of sample to be generated.')
    parser.add_argument('--log_path', type=str, default='outputs/', help='checkpoint save location')
    parser.add_argument('--testing_shape', type=str, default='', help='shape for test ')

    args, _ = parser.parse_known_args()

    args.data_root = os.path.join(args.data_dir, 'val2014')
    args.annFile = os.path.join(args.data_dir, 'annotations/instances_val2014.json')

    return args

def conver_testing_shape(args):
    """Convert testing shape to list."""
    testing_shape = [int(args.testing_shape), int(args.testing_shape)]
    return testing_shape

class SolveOutput(Cell):
    """Solve output of the target network to adapt DeepFool."""
    def __init__(self, network):
        super(SolveOutput, self).__init__()
        self._network = network
        self._reshape = P.Reshape()

    def construct(self, image, input_shape):
        prediction = self._network(image, input_shape)
        output_big = prediction[0]
        output_big = self._reshape(output_big, (output_big.shape[0], -1, 85))
        output_big_boxes = output_big[:, :, 0: 5]
        output_big_logits = output_big[:, :, 5:]
        return output_big_boxes, output_big_logits

def test():
    """The function of eval."""
    args = parse_args()

    devid = int(os.getenv('DEVICE_ID')) if os.getenv('DEVICE_ID') else 0
    context.set_context(mode=context.GRAPH_MODE, device_target='Ascend', save_graphs=True, device_id=devid)

    # logger
    args.outputs_dir = os.path.join(args.log_path,
                                    datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S'))
    rank_id = int(os.environ.get('RANK_ID')) if os.environ.get('RANK_ID') else 0
    args.logger = get_logger(args.outputs_dir, rank_id)

    context.reset_auto_parallel_context()
    parallel_mode = ParallelMode.STAND_ALONE
    context.set_auto_parallel_context(parallel_mode=parallel_mode, gradients_mean=True, device_num=1)

    args.logger.info('Creating Network....')
    network = SolveOutput(YOLOV3DarkNet53(is_training=False))

    data_root = args.data_root
    ann_file = args.annFile

    args.logger.info(args.pretrained)
    if os.path.isfile(args.pretrained):
        param_dict = load_checkpoint(args.pretrained)
        param_dict_new = {}
        for key, values in param_dict.items():
            if key.startswith('moments.'):
                continue
            elif key.startswith('yolo_network.'):
                param_dict_new[key[13:]] = values
            else:
                param_dict_new[key] = values
        load_param_into_net(network, param_dict_new)
        args.logger.info('load_model {} success'.format(args.pretrained))
    else:
        args.logger.info('{} not exists or not a pre-trained file'.format(args.pretrained))
        assert FileNotFoundError('{} not exists or not a pre-trained file'.format(args.pretrained))
        exit(1)

    config = ConfigYOLOV3DarkNet53()
    if args.testing_shape:
        config.test_img_shape = conver_testing_shape(args)

    ds, data_size = create_yolo_dataset(data_root, ann_file, is_training=False, batch_size=1,
                                        max_epoch=1, device_num=1, rank=rank_id, shuffle=False,
                                        config=config)

    args.logger.info('testing shape : {}'.format(config.test_img_shape))
    args.logger.info('totol {} images to eval'.format(data_size))

    network.set_train(False)
    # build attacker
    attack = DeepFool(network, num_classes=80, model_type='detection', reserve_ratio=0.9, bounds=(0, 1))
    input_shape = Tensor(tuple(config.test_img_shape), ms.float32)

    args.logger.info('Start inference....')
    batch_num = args.samples_num
    adv_example = []
    for i, data in enumerate(ds.create_dict_iterator(num_epochs=1)):
        if i >= batch_num:
            break
        image = data["image"]
        image_shape = data["image_shape"]

        gt_boxes, gt_logits = network(image, input_shape)
        gt_boxes, gt_logits = gt_boxes.asnumpy(), gt_logits.asnumpy()
        gt_labels = np.argmax(gt_logits, axis=2)

        adv_img = attack.generate((image.asnumpy(), image_shape.asnumpy()), (gt_boxes, gt_labels))
        adv_example.append(adv_img)
    np.save('adv_example.npy', adv_example)

if __name__ == "__main__":
    test()

max_stars_repo_name: hboshnak/mindarmour

max_issues_repo_name: hboshnak/mindarmour

max_forks_repo_name: hboshnak/mindarmour


text:
import torch
torch.manual_seed(10)
torch.cuda.manual_seed_all(10)
import torch.nn as nn
import random
from torch.autograd import Variable
import time
import numpy as np

def sigmoid(x):
    return 1.0/(1.0+torch.exp(-x))

def check_symmetric(a):
    return torch.allclose(a, a.T)

class Hessian(nn.Module):
    def __init__(self,numSteps,stepSize,**kwargs):
        super(Hessian, self).__init__()
        self.numSteps = numSteps
        self.stepSize = stepSize
        self.kwargs = kwargs
        #self.one = torch.Tensor([1.0])
        self.one = Variable(torch.tensor(1.0).cuda())

    def forward(self,previousSample,energy_function,numSteps=1,stepSize=0.01,k=5,**kwargs):
        new_draw = previousSample.clone().detach().requires_grad_(True)
        #prev_U,gradient,hess = energy_function(kwargs["decoder"],new_draw)
        energy,hess = energy_function(kwargs["decoder"],new_draw)

        if isinstance(energy,str):
            if energy=="nan":
                return "eig failed"
        #print(check_symmetric(hess))
        #norm_gradient = gradient/torch.norm(gradient)

        #norm_gradient = gradient/torch.norm(gradient)
        for _ in range(1):
            #U,S,V = torch.linalg.svd(hess)
            #Mntm = torch.randn_like(new_draw)
            #w = Mntm - torch.mm(Mntm,torch.unsqueeze(norm_gradient,-1))*norm_gradient

            decomposition = torch.linalg.eigh(hess)
            S = decomposition.eigenvalues
            V = decomposition.eigenvectors
            S,indices = torch.sort(S,descending=False)
            V = V[:,indices]
            num_components = 4
            Vk = V[:,:num_components]
            Sk = S[:num_components]
            beta = torch.randn(num_components)
            beta = beta/torch.norm(beta)
            direction = torch.zeros(new_draw.size()).float().cuda()

            for compIndex in range(num_components):
                #if Sk[compIndex]>0:
                direction += beta[compIndex]*Vk[:,compIndex]
                #else:
                #    direction += beta[compIndex]*(-1*Vk[:,compIndex])

            norm_direction = torch.norm(direction)
            direction = direction/norm_direction

            delta = 1e-3
            betasq_lambda = 0.0
            for compIndex in range(num_components):
                betasq_lambda += ((beta[compIndex]/norm_direction)**2 * torch.abs(Sk[compIndex]))

            stepSize = torch.sqrt((2.0*delta)/betasq_lambda)
            #stepSize = max(0.3,min(0.5,stepSize))
            #stepSize = max(0.3,min(0.5,stepSize))
            stepSize = min(stepSize,2.0)
            print(stepSize)
            #stepSize = 0.1

            new_draw = (new_draw+stepSize*direction).clone().detach().requires_grad_(True)
            #current_U,gradient,hess = energy_function(kwargs["decoder"],new_draw)
            #print("Energy:",current_U)
            #norm_gradient = gradient/torch.norm(gradient)
        draw_final = new_draw.clone().detach()

        return draw_final

max_stars_repo_name: sanjeevmk/GLASS

max_issues_repo_name: sanjeevmk/glass

max_forks_repo_name: sanjeevmk/glass


text:
import os
import collections
import torch
import torchvision
import numpy as np
import scipy.misc as m
import scipy.io as io
#import matplotlib.pyplot as plt

from torch.utils import data

from ptsemseg.utils import recursive_glob
from ptsemseg.augmentations import *

class NYUv2Loader(data.Dataset):
    """NYUv2 loader

    Download From (only 13 classes):
        test source: http://www.doc.ic.ac.uk/~ahanda/nyu_test_rgb.tgz
        train source: http://www.doc.ic.ac.uk/~ahanda/nyu_train_rgb.tgz
        test_labels source: https://github.com/ankurhanda/nyuv2-meta-data/raw/master/test_labels_13/nyuv2_test_class13.tgz
        train_labels source: https://github.com/ankurhanda/nyuv2-meta-data/raw/master/train_labels_13/nyuv2_train_class13.tgz

    """

    def __init__(
        self,
        root,
        split="training",
        is_transform=False,
        img_size=(480, 640),
        augmentations=None,
        img_norm=True,
    ):
        self.root = root
        self.is_transform = is_transform
        self.n_classes = 14
        self.augmentations = augmentations
        self.img_norm = img_norm
        self.img_size = (
            img_size if isinstance(img_size, tuple) else (img_size, img_size)
        )
        self.mean = np.array([104.00699, 116.66877, 122.67892])
        self.files = collections.defaultdict(list)
        self.cmap = self.color_map(normalized=False)

        split_map = {"training": "train", "val": "test"}
        self.split = split_map[split]

        for split in ["train", "test"]:
            file_list = recursive_glob(rootdir=self.root + split + "/", suffix="png")
            self.files[split] = file_list

    def __len__(self):
        return len(self.files[self.split])

    def __getitem__(self, index):
        img_path = self.files[self.split][index].rstrip()
        img_number = img_path.split("_")[-1][:4]
        lbl_path = os.path.join(
            self.root, self.split + "_annot", "new_nyu_class13_" + img_number + ".png"
        )

        img = m.imread(img_path)
        img = np.array(img, dtype=np.uint8)

        lbl = m.imread(lbl_path)
        lbl = np.array(lbl, dtype=np.uint8)

        if not (len(img.shape) == 3 and len(lbl.shape) == 2):
            return self.__getitem__(np.random.randint(0, self.__len__()))

        if self.augmentations is not None:
            img, lbl = self.augmentations(img, lbl)

        if self.is_transform:
            img, lbl = self.transform(img, lbl)

        return img, lbl

    def transform(self, img, lbl):
        img = m.imresize(
            img, (self.img_size[0], self.img_size[1])
        )  # uint8 with RGB mode
        img = img[:, :, ::-1]  # RGB -> BGR
        img = img.astype(np.float64)
        img -= self.mean
        if self.img_norm:
            # Resize scales images from 0 to 255, thus we need
            # to divide by 255.0
            img = img.astype(float) / 255.0
        # NHWC -> NCHW
        img = img.transpose(2, 0, 1)

        classes = np.unique(lbl)
        lbl = lbl.astype(float)
        lbl = m.imresize(lbl, (self.img_size[0], self.img_size[1]), "nearest", mode="F")
        lbl = lbl.astype(int)
        assert np.all(classes == np.unique(lbl))

        img = torch.from_numpy(img).float()
        lbl = torch.from_numpy(lbl).long()
        return img, lbl

    def color_map(self, N=256, normalized=False):
        """
        Return Color Map in PASCAL VOC format
        """

        def bitget(byteval, idx):
            return (byteval & (1 << idx)) != 0

        dtype = "float32" if normalized else "uint8"
        cmap = np.zeros((N, 3), dtype=dtype)
        for i in range(N):
            r = g = b = 0
            c = i
            for j in range(8):
                r = r | (bitget(c, 0) << 7 - j)
                g = g | (bitget(c, 1) << 7 - j)
                b = b | (bitget(c, 2) << 7 - j)
                c = c >> 3

            cmap[i] = np.array([r, g, b])

        cmap = cmap / 255.0 if normalized else cmap
        return cmap

    def decode_segmap(self, temp):
        r = temp.copy()
        g = temp.copy()
        b = temp.copy()
        for l in range(0, self.n_classes):
            r[temp == l] = self.cmap[l, 0]
            g[temp == l] = self.cmap[l, 1]
            b[temp == l] = self.cmap[l, 2]

        rgb = np.zeros((temp.shape[0], temp.shape[1], 3))
        rgb[:, :, 0] = r / 255.0
        rgb[:, :, 1] = g / 255.0
        rgb[:, :, 2] = b / 255.0
        return rgb

if __name__ == "__main__":
    import torchvision
    import matplotlib.pyplot as plt

    augmentations = Compose([Scale(512), RandomRotate(10), RandomHorizontallyFlip()])

    local_path = "/home/meet/datasets/NYUv2/"
    dst = NYUv2Loader(local_path, is_transform=True, augmentations=augmentations)
    bs = 4
    trainloader = data.DataLoader(dst, batch_size=bs, num_workers=0)
    for i, data in enumerate(trainloader):
        imgs, labels = data
        imgs = imgs.numpy()[:, ::-1, :, :]
        imgs = np.transpose(imgs, [0, 2, 3, 1])
        f, axarr = plt.subplots(bs, 2)
        for j in range(bs):
            axarr[j][0].imshow(imgs[j])
            axarr[j][1].imshow(dst.decode_segmap(labels.numpy()[j]))
        plt.show()
        a = raw_input()
        if a == "ex":
            break
        else:
            plt.close()

reason: import numpy,import scipy


text:
"""
Generate kp for tf-lift to compute their descriptors.
You need to set the scale of your kp
"""
import os
import random
import time
import argparse
import numpy as np
import cv2

import tensorflow as tf

import tools.cst as cst

import tools.init_weights as init_weights
import tools.model_vgg as model
import methods.elf_vgg.tools as tools_elf

parser = argparse.ArgumentParser()
parser.add_argument('--trials', type=str, default='100', help='xp id')
parser.add_argument('--nms_dist', type=int, default=4, help='Non Maximum Suppression (NMS) distance (default: 4).')
parser.add_argument('--border_remove', type=int, default=4, help='Border of the img where you ignore the kp')
parser.add_argument('--max_num_feat', type=int, default=1000, help='Maximum number of features.')
parser.add_argument('--H', type=int, default=480, help='new height')
parser.add_argument('--W', type=int, default=640, help='new width')
parser.add_argument('--grad_name', type=str, default='pool2', help='grad_name')
parser.add_argument('--feat_name', type=str, default='pool4', help='feat_name')
parser.add_argument('--thr_k_size', type=int, default=5, help='kernel size for blur before otsu threshold computation.')
parser.add_argument('--thr_sigma', type=int, default=5, help='gaussian var. blur for threshold computation.')
parser.add_argument('--noise_k_size', type=int, default=5, help='kernel size blur before thresholding.')
parser.add_argument('--noise_sigma', type=int, default=5,  help='gaussian var. blur before thresholding.')
parser.add_argument('--model', type=str, default='vgg', help='net model in {alexnet, vgg, xception}')
parser.add_argument('--resize', type=int, default=0, help='set to 1 to resize img')
parser.add_argument('--save2txt', type=int, default=0, help='set to 1 to save kp to file txt')
args = parser.parse_args()

new_size = (args.W, args.H)
norm = 'L2'

# setup output dir
res_dir = os.path.join('res/elf/', args.trials)
if not os.path.exists(res_dir):
    os.makedirs(res_dir)

with tf.Graph().as_default():

    img_op = tf.placeholder(dtype=tf.float32, shape=[1,new_size[1],new_size[0],3])
    _, grads_op = model.model_grad(img_op, is_training=False)

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        init_weights.restore_vgg(sess, '%s/vgg/data.ckpt'%cst.WEIGHT_DIR)

        # Start the queue runners.
        coord = tf.train.Coordinator()
        try:
            threads = []
            for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
                threads.extend(qr.create_threads(sess, coord=coord, daemon=True, start=True))

            global_start_time = time.time()
            for scene_name in cst.SCENE_LIST:
                duration = time.time() - global_start_time
                print('*** %s *** %d:%02d'%(scene_name, duration/60, duration%60))

                out_dir = os.path.join(res_dir, scene_name)
                if not os.path.exists(out_dir):
                    os.makedirs(out_dir)

                for img_key in range(1,cst.MAX_IMG_NUM+1):
                    # get 1st img, (resize it), convert to BW
                    img_fn = os.path.join(cst.DATA_DIR,scene_name,'%d.ppm'%img_key)
                    img = cv2.imread(img_fn)
                    if args.resize==1:
                        img = cv2.resize(img, new_size, interpolation=cv2.INTER_LINEAR)

                    patch = tools_elf.preproc(img)
                    grad = sess.run(grads_op[args.grad_name],
                                feed_dict={img_op: patch})[0][0,:,:,:]
                    pts_fail, pts = tools_elf.postproc(grad, args.noise_k_size,
                            args.noise_sigma, args.thr_k_size, args.thr_sigma,
                            args.nms_dist, args.border_remove, args.max_num_feat)

                    SCALE = 4.2
                    scale_v = np.ones(pts.shape[1])*SCALE
                    pts = np.vstack((pts, scale_v))
                    pts = pts[[0,1,3,2],:]
                    pts_fn = os.path.join(out_dir, '%d_kp.txt'%img_key)
                    np.savetxt(pts_fn, pts.T)

        except Exception as e:  # pylint: disable=broad-except
            coord.request_stop(e)

        coord.request_stop()
        coord.join(threads, stop_grace_period_secs=10)

max_stars_repo_name: abenbihi/elf

max_issues_repo_name: abenbihi/elf

max_forks_repo_name: abenbihi/elf


text:
from pylab import imread,imshow,figure,show,subplot,title
from numpy import reshape,flipud,unravel_index,zeros
from minisom import MiniSom

# read the image
img = imread('tree.jpg')

# reshaping the pixels matrix
pixels = reshape(img,(img.shape[0]*img.shape[1],3))

# SOM initialization and training
print('training...')
som = MiniSom(3,3,3,sigma=0.1,learning_rate=0.2) # 3x3 = 9 final colors
som.random_weights_init(pixels)
starting_weights = som.weights.copy() # saving the starting weights
som.train_random(pixels,100)

print('quantization...')
qnt = som.quantization(pixels) # quantize each pixels of the image
print('building new image...')
clustered = zeros(img.shape)
for i,q in enumerate(qnt): # place the quantized values into a new image
	clustered[unravel_index(i,dims=(img.shape[0],img.shape[1]))] = q
print('done.')

# show the result
figure(1)
subplot(221)
title('original')
imshow(flipud(img))
subplot(222)
title('result')
imshow(flipud(clustered))

subplot(223)
title('initial colors')
imshow(flipud(starting_weights),interpolation='none')
subplot(224)
title('learned colors')
imshow(flipud(som.weights),interpolation='none')

show()

max_stars_repo_name: Borlaff/EuclidVisibleInstrument

max_issues_repo_name: Borlaff/EuclidVisibleInstrument

max_forks_repo_name: Borlaff/EuclidVisibleInstrument

reason: from numpy


text:
"""GARCH LLH benchmark, Theano version."""
import os
import numpy as np
import argparse
import json
from util import benchmark, garch_data

parser = argparse.ArgumentParser()
parser.add_argument('-n', metavar='n', default=1000, type=int,
                    help='number of iterations')
parser.add_argument('-mode', metavar='mode', default='cpu',
                    choices=['cpu', 'gpu'], help='use cpu/gpu')
args = parser.parse_args()

device = 'cuda' if args.mode == 'gpu' else 'cpu'
flags = 'openmp=True, mode=FAST_RUN, device=' + device + ', floatX=float32'
os.environ["THEANO_FLAGS"] = flags

import theano
import theano.tensor as T
from theano.ifelse import ifelse
from theano.tensor.shared_randomstreams import RandomStreams
srng = RandomStreams(seed=0)

out = {}
ret, x0, val_llh = garch_data()

ht0 = np.mean(np.square(ret))
ht = T.scalar('ht')
y = T.vector('y')
lr = T.scalar('lr')
mu = T.scalar('mu')
params = T.vector('params')

def garch_like(y, ht):
    return params[0] + params[1]*y + params[2]*ht

y2 = T.square(y)
hts, updates = theano.scan(fn=garch_like,
                           sequences=[y2],
                           n_steps=y2.size - 1,
                           outputs_info=ht0)

hts = T.concatenate(([ht0], hts))
tllh = (-0.5 * (y.size - 1) * T.log(2 * np.pi) -
       0.5 * T.sum(T.log(hts) + T.square(y/T.sqrt(hts))))
gllh = T.grad(tllh, wrt=params)
llh = theano.function(inputs=[y, params], outputs=tllh)

t = benchmark(lambda: llh(ret, x0), args.n, val_llh)
out['theano-' + args.mode] = t

def unroll(tx, th0, n):
    th = th0
    th_tmp = []
    for t in range(n):
        th = garch_like(tx[t], th)
        th_tmp.append(th)
    return T.stack(th_tmp)

t_unroll0 = unroll(y2, ht0, len(ret) - 1)
t_unroll = T.concatenate(([ht0], t_unroll0))
t_unroll_out = (-0.5 * (y.size - 1) * T.log(2 * np.pi) -
               0.5 * T.sum(T.log(t_unroll) + T.square(y/T.sqrt(t_unroll))))
tth = theano.function(inputs=[y, params], outputs=t_unroll_out)

t = benchmark(lambda: tth(ret, x0), args.n, val_llh)

out['theano-unroll-' + args.mode] = t
print(json.dumps(out))

max_stars_repo_name: AndreyKolev/MLLibMark

max_issues_repo_name: AndreyKolev/MLLibMark

max_forks_repo_name: AndreyKolev/MLLibMark

reason: import numpy,import theano,from theano


text:
## Architecture

# Take in inputs from the screen and preprocess them
# Pass them into an NN
# Update the weights of the NN using gradient descent
# weights['1'] - Matrix that holds weights of pixels passing into hidden layer. Dimensions: [200 x 80 x 80] -> [200 x 6400]
# weights['2'] - Matrix that holds weights of hidden layer passing into output. Dimensions: [1 x 200]

# Process is:

# processed_observations = image vector - [6400 x 1] array
# Compute hidden_layer_values = weights['1'] dot processed_observations ([200 x 6400] dot [6400 x 1]) -> [200 x 1] - this gives initial activation values.
# Next we need to transform those either via a sigmoid or an ReLU of some sort. Let's use ReLU
# ReLU(hidden_layer_values)
# Next we need to pass this one layer further
# output_layer_value = weights['2'] dot hidden_layer_values ([1 x 200] dot [200 x 1] -> [1 x 1])
# Now our output layer is the probability of going up or down. Let's make sure this output is between 0 and 1 by passing it through a sigmoid
# p = sigmoid(output_layer_value)

# Learning after round has finished:

# Figure out the result
# Compute the error
# Use the error to calculate the gradient
    # The below dimensions all assume we had exactly 10 frames in the round (not necessarily true!)
    # dC_dw2 = hidden_layer_values^T dot gradient_log_p ([1 x 2000] dot [2000 x 1] -> 1x1)
    # delta_1 = gradient_log_p outer_product weights['2'] = [2000 x 1] outer_product [1 x 200] ([2000 x 200])
    # dC_dw1 = delta_1^T dot input_observations ([200 x 2000]x dot [2000 x 64000] -> [200 x 64000])

# After some batch size of rounds has finished,
    # Use rmsprop to move weights['1'] and weights['2'] in the direction of the gradient
# Repeat!

import gym
import numpy as np

def downsample(image):
    # Take only alternate pixels - basically halves the resolution of the image (which is fine for us)
    return image[::2, ::2, :]

def remove_color(image):
    """Convert all color (RGB is the third dimension in the image)"""
    return image[:, :, 0]

def remove_background(image):
    image[image == 144] = 0
    image[image == 109] = 0
    return image

def preprocess_observations(input_observation, prev_processed_observation, input_dimensions):
    """ convert the 210x160x3 uint8 frame into a 6400 float vector """
    processed_observation = input_observation[35:195] # crop
    processed_observation = downsample(processed_observation)
    processed_observation = remove_color(processed_observation)
    processed_observation = remove_background(processed_observation)
    processed_observation[processed_observation != 0] = 1 # everything else (paddles, ball) just set to 1
    # Convert from 80 x 80 matrix to 6400 x 1 matrix
    processed_observation = processed_observation.astype(np.float).ravel()

    # subtract the previous frame from the current one so we are only processing on changes in the game
    if prev_processed_observation is not None:
        input_observation = processed_observation - prev_processed_observation
    else:
        input_observation = np.zeros(input_dimensions)
    # store the previous frame so we can subtract from it next time
    prev_processed_observations = processed_observation
    return input_observation, prev_processed_observations

def sigmoid(x):
    return 1.0/(1.0 + np.exp(-x))

def relu(vector):
    vector[vector < 0] = 0
    return vector

def apply_neural_nets(observation_matrix, weights):
    """ Based on the observation_matrix and weights, compute the new hidden layer values and the new output layer values"""
    hidden_layer_values = np.dot(weights['1'], observation_matrix)
    hidden_layer_values = relu(hidden_layer_values)
    output_layer_values = np.dot(hidden_layer_values, weights['2'])
    output_layer_values = sigmoid(output_layer_values)
    return hidden_layer_values, output_layer_values

def choose_action(probability):
    random_value = np.random.uniform()
    if random_value < probability:
        # signifies up in openai gym
        return 2
    else:
         # signifies down in openai gym
        return 3

def compute_gradient(gradient_log_p, hidden_layer_values, observation_values, weights):
    """ See here: http://neuralnetworksanddeeplearning.com/chap2.html"""
    delta_L = gradient_log_p
    dC_dw2 = np.dot(hidden_layer_values.T, delta_L).ravel()
    delta_l2 = np.outer(delta_L, weights['2'])
    delta_l2 = relu(delta_l2)
    dC_dw1 = np.dot(delta_l2.T, observation_values)
    return {
        '1': dC_dw1,
        '2': dC_dw2
    }

def update_weights(weights, expectation_g_squared, g_dict, decay_rate, learning_rate):
    """ See here: http://sebastianruder.com/optimizing-gradient-descent/index.html#rmsprop"""
    epsilon = 1e-5
    for layer_name in weights.keys():
        g = g_dict[layer_name]
        expectation_g_squared[layer_name] = decay_rate * expectation_g_squared[layer_name] + (1 - decay_rate) * g**2
        weights[layer_name] += (learning_rate * g)/(np.sqrt(expectation_g_squared[layer_name] + epsilon))
        g_dict[layer_name] = np.zeros_like(weights[layer_name]) # reset batch gradient buffer

def discount_rewards(rewards, gamma):
    """ Actions you took 20 steps before the end result are less important to the overall result than an action you took a step ago.
    This implements that logic by discounting the reward on previous actions based on how long ago they were taken"""
    discounted_rewards = np.zeros_like(rewards)
    running_add = 0
    for t in reversed(range(0, rewards.size)):
        if rewards[t] != 0:
            running_add = 0 # reset the sum, since this was a game boundary (pong specific!)
        running_add = running_add * gamma + rewards[t]
        discounted_rewards[t] = running_add
    return discounted_rewards

def discount_with_rewards(gradient_log_p, episode_rewards, gamma):
    """ discount the gradient with the normalized rewards """
    discounted_episode_rewards = discount_rewards(episode_rewards, gamma)
    # standardize the rewards to be unit normal (helps control the gradient estimator variance)
    discounted_episode_rewards -= np.mean(discounted_episode_rewards)
    discounted_episode_rewards /= np.std(discounted_episode_rewards)
    return gradient_log_p * discounted_episode_rewards

def main():
    env = gym.make("Pong-v0")
    observation = env.reset() # This gets us the image

    # hyperparameters
    episode_number = 0
    batch_size = 10
    gamma = 0.99 # discount factor for reward
    decay_rate = 0.99
    num_hidden_layer_neurons = 200
    input_dimensions = 80 * 80
    learning_rate = 1e-4

    episode_number = 0
    reward_sum = 0
    running_reward = None
    prev_processed_observations = None

    weights = {
        '1': np.random.randn(num_hidden_layer_neurons, input_dimensions) / np.sqrt(input_dimensions),
        '2': np.random.randn(num_hidden_layer_neurons) / np.sqrt(num_hidden_layer_neurons)
    }

    # To be used with rmsprop algorithm (http://sebastianruder.com/optimizing-gradient-descent/index.html#rmsprop)
    expectation_g_squared = {}
    g_dict = {}
    for layer_name in weights.keys():
        expectation_g_squared[layer_name] = np.zeros_like(weights[layer_name])
        g_dict[layer_name] = np.zeros_like(weights[layer_name])

    episode_hidden_layer_values, episode_observations, episode_gradient_log_ps, episode_rewards = [], [], [], []

    while True:
        env.render()
        processed_observations, prev_processed_observations = preprocess_observations(observation, prev_processed_observations, input_dimensions)
        hidden_layer_values, up_probability = apply_neural_nets(processed_observations, weights)

        episode_observations.append(processed_observations)
        episode_hidden_layer_values.append(hidden_layer_values)

        action = choose_action(up_probability)

        # carry out the chosen action
        observation, reward, done, info = env.step(action)

        reward_sum += reward
        episode_rewards.append(reward)

        # see here: http://cs231n.github.io/neural-networks-2/#losses
        fake_label = 1 if action == 2 else 0
        loss_function_gradient = fake_label - up_probability
        episode_gradient_log_ps.append(loss_function_gradient)

        if done: # an episode finished
            episode_number += 1

            # Combine the following values for the episode
            episode_hidden_layer_values = np.vstack(episode_hidden_layer_values)
            episode_observations = np.vstack(episode_observations)
            episode_gradient_log_ps = np.vstack(episode_gradient_log_ps)
            episode_rewards = np.vstack(episode_rewards)

            # Tweak the gradient of the log_ps based on the discounted rewards
            episode_gradient_log_ps_discounted = discount_with_rewards(episode_gradient_log_ps, episode_rewards, gamma)

            gradient = compute_gradient(
              episode_gradient_log_ps_discounted,
              episode_hidden_layer_values,
              episode_observations,
              weights
            )

            # Sum the gradient for use when we hit the batch size
            for layer_name in gradient:
                g_dict[layer_name] += gradient[layer_name]

            if episode_number % batch_size == 0:
                update_weights(weights, expectation_g_squared, g_dict, decay_rate, learning_rate)

            episode_hidden_layer_values, episode_observations, episode_gradient_log_ps, episode_rewards = [], [], [], [] # reset values
            observation = env.reset() # reset env
            running_reward = reward_sum if running_reward is None else running_reward * 0.99 + reward_sum * 0.01
            print('resetting env. episode reward total was %f. running mean: %f' % (reward_sum, running_reward))
            reward_sum = 0
            prev_processed_observations = None

main()

max_stars_repo_name: Froskekongen/oslodatascience-rl

max_issues_repo_name: Froskekongen/oslodatascience-rl

max_forks_repo_name: Froskekongen/oslodatascience-rl


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2020 Ryan L. Collins <rlcollins@g.harvard.edu>
# and the Talkowski Laboratory
# Distributed under terms of the MIT license.

"""
Download and curate a single genome annotation track
"""

import pybedtools as pbt
import pandas as pd
import numpy as np
import argparse
from os import path
import subprocess

def download_track(trackpath):
    """
    Download track (or locate its local copy, if already downloaded)
    """

    if trackpath.startswith('gs://'):
        localpath = path.basename(trackpath)
        if not any([path.exists(localpath), path.isfile(localpath)]):
            subprocess.run(['gsutil', '-m', 'cp', trackpath, './'], check=True)

    elif any(trackpath.startswith(prefix) for prefix in 'http:// https:// ftp://'.split()):
        localpath = path.basename(trackpath)
        if not any([path.exists(localpath), path.isfile(localpath)]):
            subprocess.run(['wget', '--no-check-certificate', trackpath], check=True)

    else:
        localpath = trackpath

    # Correct for URL convention of replacing spaces with "%20"
    localpath = localpath.replace('%20', ' ')

    return localpath

def get_track(trackpath, n_download_retries=5, prefix=None):
    """
    Download (if necessary) and load annotation track
    """

    # Download track, if necessary, trying a total of n_download_retries times
    for tries in range(1, n_download_retries+1):
        try:
            localpath = download_track(trackpath)
            print('\nSuccessfully localized {} after {} attempts\n'.format(trackpath, tries))
            break
        except:
            print('\nAttepmt {} to localize {} was unsuccessful. Retrying...\n'.format(tries, trackpath))
            continue

    # Set track name equal to base filename without bed suffix
    if '.bed' in localpath:
        trackname = path.basename(localpath).split('.bed')[0]
    elif '.txt' in localpath:
        trackname = path.basename(localpath).split('.txt')[0]
    elif '.tsv' in localpath:
        trackname = path.basename(localpath).split('.tsv')[0]

    if prefix is not None:
        if prefix != '':
            trackname = '.'.join([prefix, trackname])
    trackname = trackname.replace(' ', '_').replace('(', '').replace(')', '').replace('/', '')

    return pbt.BedTool(localpath), trackname

def build_blacklist(blacklists):
    """
    Build universal blacklist from one or more inputs
    """

    if blacklists is None:
        xbt = pbt.BedTool('', from_string=True)

    else:
        xlist = [x for s in blacklists for x in s]
        n_bl = len(xlist)
        if n_bl == 0:
            xbt = pbt.BedTool()
        elif n_bl == 1:
            xbt = pbt.BedTool(xlist[0])
        else:
            xbt = pbt.BedTool(xlist[0]).\
                      cat(*[pbt.BedTool(bl) for bl in xlist[1:]])

        xbt = xbt.sort().merge().saveas()

    return xbt

def clean_track(track, trackname, xbt, xcov=0.5, genome=None,
                min_size=10, max_size=100000):
    """
    Clean & curate pbt.BedTool of elements
    """

    def _clean_contig(feature):
        newchrom = str(feature.chrom).replace('chr', '')
        feature.chrom = newchrom
        return feature

    def _is_autosomal(feature):
        return any(str(feature.chrom) == str(x) for x in range(1, 23))

    track = track.cut(range(3)).\
                  each(_clean_contig).\
                  filter(_is_autosomal).\
                  coverage(xbt).\
                  filter(lambda x: float(x[-1]) < xcov).\
                  cut(range(3)).\
                  saveas()

    def _size_filter(feature, min_size, max_size):
        return all([len(feature) >= min_size, len(feature) <= max_size])

    if len(track) > 0:

        if genome is None:
            track = track.sort().merge().\
                          filter(_size_filter, min_size, max_size).\
                          saveas()
        else:
            track = track.sort(g=genome).merge().\
                          filter(_size_filter, min_size, max_size).\
                          saveas()

        if len(track) > 0:
            track_df = track.to_dataframe()
            track_df['name'] = trackname
            return pbt.BedTool.from_dataframe(track_df)

        else:
            return pbt.BedTool('', from_string=True).saveas()

    else:
        return pbt.BedTool('', from_string=True).saveas()

def get_track_stats(track, trackname, orig_trackpath, outfile):
    """
    Compute basic statistics for elements in track
    """

    # Add header to outfile
    hcols = '#trackname n_elements min_size median_size mean_size max_size total_bp original_path'
    outfile.write('\t'.join(hcols.split()) + '\n')

    # Compute stats
    if (len(track) > 0):
        sizes = np.array([len(x) for x in track], dtype=float)
        n_ele = len(track)
        min_size = np.nanmin(sizes)
        med_size = np.nanmedian(sizes)
        mean_size = np.nanmean(sizes)
        max_size = np.nanmax(sizes)
        total_bp = np.nansum(sizes)
        svals = [str(round(x, 1)) for x in [n_ele, min_size, med_size, mean_size, max_size, total_bp]]

    else:
        svals = '0 NA NA NA NA NA'.split()

    # Write stats to outfile
    outfile.write('\t'.join([trackname] + svals + [orig_trackpath]) + '\n')
    outfile.close()

def main():
    """
    Main block
    """

    # Parse command line arguments and options
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('path', help='Path to annotation track. Can be local, ' +
                        'gs://, http://, https://, or ftp://')
    parser.add_argument('-g', '--genome', help='BEDTools-style genome file to ' +
                        'be used when sorting.')
    parser.add_argument('-x', '--blacklist', nargs='*', action='append',
                        help='Blacklist BED files to exclude elements. May be ' +
                        'specified multiple times.')
    parser.add_argument('--blacklist-cov', default=0.1, type=float,
                        help='Minimum fraction of element that must be covered ' +
                        'by any blacklist before being excluded.')
    parser.add_argument('--min-size', default=10, type=int,
                        help='Minimum size of element to retain after merging.')
    parser.add_argument('--max-size', default=100000, type=int,
                        help='Minimum size of element to retain after merging.')
    parser.add_argument('-o', '--outbed', help='Path to output BED file. ' +
                        '[default: trackname + .curated.bed.gz]')
    parser.add_argument('-n', '--override-trackname', help='Manually specify trackname.')
    parser.add_argument('-s', '--stats', action='store_true', help='Compute ' +
                        'track statistics.')
    parser.add_argument('--n-download-retries', default=5, type=int,
                        help='Number of times to retry downloading each track.')
    parser.add_argument('-p', '--prefix',  help='Prefix for track and element names.')
    parser.add_argument('-z', '--bgzip', action='store_true', help='Compress ' +
                        'output BED file with bgzip.')
    args = parser.parse_args()

    # Open connections to output files (unless not specified)
    if args.outbed is not None:
        if path.splitext(args.outbed)[-1] in '.gz .bz .bgz .bgzip .gzip'.split():
            outbed_path = path.splitext(args.outbed)[0]
        else:
            outbed_path = args.outbed

    # Load annotation track
    track, trackname = get_track(args.path, args.n_download_retries, args.prefix)
    if args.override_trackname is not None:
        trackname = args.override_trackname

    # Open connections to unspecified output files
    if args.outbed is None:
        outbed_path = trackname + '.curated.bed'
        bgzip = True
    else:
        bgzip = args.bgzip
    if args.stats:
        stats_out = open(trackname + '.curated.stats.tsv', 'w')

    # Load blacklists
    xbt = build_blacklist(args.blacklist)

    # Clean annotation track
    track = clean_track(track, trackname, xbt, args.blacklist_cov, args.genome,
                        args.min_size, args.max_size)

    # Compute track stats, if optioned
    get_track_stats(track, trackname, args.path, stats_out)

    # Write cleaned track to file
    track.saveas(outbed_path)
    if bgzip:
        subprocess.run(['bgzip', '-f', outbed_path])

if __name__ == '__main__':
    main()


text:
import numpy as np

from ._base import id_provider
from . import Mesh
from ..resources import Buffer

class InstancedMesh(Mesh):
    """An instanced mesh with a matrix for each instance."""

    def __init__(self, geometry, material, count, **kwargs):
        super().__init__(geometry, material, **kwargs)
        count = int(count)
        # Create array of `count` instance_info objects
        dtype = np.dtype(
            [
                ("matrix", np.float32, (4, 4)),
                ("id", np.uint32),
                ("_12_bytes_padding", np.uint8, (12,)),
            ]
        )
        instance_infos = np.zeros(count, dtype)
        self.instance_infos = Buffer(instance_infos, nitems=count)
        # Set ids
        self._idmap = {}
        for instance_index in range(count):
            id = id_provider.claim_id(self)
            self._idmap[id] = instance_index
            instance_infos[instance_index]["id"] = id
        # Init eye matrices
        for i in range(4):
            instance_infos["matrix"][:, i, i] = 1

    def __del__(self):
        super().__del__()
        instance_infos = self.instance_infos.data
        for i in range(len(instance_infos)):
            id_provider.release_id(self, instance_infos[i]["id"])

    def set_matrix_at(self, index: int, matrix):
        """set the matrix for the instance at the given index."""
        matrix = np.array(matrix).reshape(4, 4)
        self.instance_infos.data["matrix"][index] = matrix

    def _wgpu_get_pick_info(self, pick_value):
        info = self.material._wgpu_get_pick_info(pick_value)
        # The id maps to one of our instances
        id = pick_value & 1048575  # 2**20-1
        info["instance_index"] = self._idmap.get(id)
        return info


text:
'''
Plot the aggregation error as a function of rho for the KS and p-norm
functionals
'''
from __future__ import print_function
import tikzplots as tkz
import numpy as np

def integrate(integrand):
    '''Integrate over equally spaced data'''
    sigma = [17.0/48.0, 59.0/48.0, 43.0/48, 49/48.9]
    r = len(sigma)

    integral = 0.0
    for i, s in enumerate(sigma):
        integral += s*integrand[i]
        integral += s*integrand[-1-i]

    for i in range(r, len(integrand)-r):
        integral += integrand[i]

    return integral

def get_disk_aggregate(functional, rho, R, n=1000):
    '''Evaluate the KS functional on a disk'''
    scale = 1.0/R**2
    r = np.linspace(0.0, R, n)

    a0 = 3920.0/363.0

    # This is the solution for constant order
    # phi = (0.1875*R**2 - 0.25*r**2 + (0.0625/R**2)*r**4)

    # Compute the solution scaled to r/R
    x = np.linspace(0.0, 1.0, n)
    phi = a0*R**2*(- (1.0/196)*x**14
                   + (1.0/24)*x**12
                   - (3.0/20)*x**10
                   + (5.0/16)*x**8
                   - (5.0/12)*x**6
                   + (3.0/8)*x**4
                   - (1.0/4)*x**2
                   + (363.0/3920))

    if functional == 'ks':
        ksmax = np.max(phi)
        integrand = r*np.exp(rho*scale*(phi - ksmax))
        kssum = 2*np.pi*(R/(n-1))*integrate(integrand)
        return scale*ksmax, scale*ksmax + np.log(kssum)/rho
    else:
        maxphi = scale*np.max(phi)
        integrand = r*np.power(np.fabs(scale*phi/maxphi), rho)
        psum = 2*np.pi*(R/(n-1))*integrate(integrand)
        return maxphi, maxphi*np.power(psum, 1.0/rho)

n = 100000
functional = 'ks'

xmin = -6
xmax = -3

m = 91
eps_vals = np.logspace(xmin, xmax, m)
ks_agg_err = np.zeros(m)
pnorm_agg_err = np.zeros(m)

with open('results/disk_aggregation_errors.dat', 'w') as fp:
    fp.write('Variables = rho, 1/rho, ks, pnorm\n')
    R = 100.0
    for k, eps in enumerate(eps_vals):
        rho = 1.0/eps
        max_value, ks = get_disk_aggregate('ks', rho, R, n)
        max_value, pnorm = get_disk_aggregate('pnorm', rho, R, n)
        ks_agg_err[k] = np.fabs(ks - max_value)
        pnorm_agg_err[k] = np.fabs(pnorm - max_value)
        fp.write('%25.16e %25.16e %25.16e %25.16e\n'%(
            rho, eps, ks_agg_err[k], pnorm_agg_err[k]))

# Set the colors to use for each set of bars
colors = []
for i in range(10):
    colors.append('tableau%d'%(i))

tikzcolors = '''
\definecolor{tableau0}{RGB}{31,119,180}
\definecolor{tableau1}{RGB}{255,158,74}
\definecolor{tableau2}{RGB}{103,191,92}
\definecolor{tableau3}{RGB}{237,102,93}
\definecolor{tableau4}{RGB}{148,103,189}
\definecolor{tableau5}{RGB}{168,120,110}
\definecolor{tableau6}{RGB}{237,151,202}
\definecolor{tableau7}{RGB}{162,162,162}
\definecolor{tableau8}{RGB}{205,204,93}
\definecolor{tableau9}{RGB}{109,204,218}
'''

ymax = max(np.max(ks_agg_err), np.max(pnorm_agg_err))
ymin = min(np.min(ks_agg_err), np.min(pnorm_agg_err))

ymin = int(np.floor(np.log10(ymin)))
ymax = int(np.ceil(np.log10(ymax)))

# Create a range
xticks = np.linspace(xmin, xmax, xmax - xmin + 1)
xtick_labels = []
for exp in range(xmin, xmax + 1, 1):
    xtick_labels.append('$10^{%d}$'%(exp))

yticks = np.linspace(ymin, ymax, ymax - ymin + 1)
ytick_labels = []
for exp in range(ymin, ymax + 1, 1):
    ytick_labels.append('$10^{%d}$'%(exp))

# The overall dimensions
xdim = 2.0
xscale = xdim/(xmax - xmin)

ydim = 1.75
yscale = ydim/(ymax - ymin)

# Get the header info
s = tkz.get_header()
s += tkz.get_begin_tikz(xdim=1.5, ydim=1.5, xunit='in', yunit='in')

s += r'\tikzstyle{dashed}= [dash pattern=on 6pt off 2pt]'
s += tikzcolors

symbols = ['circle', 'square', 'triangle', 'delta', 'diamond']

xvals = np.log10(eps_vals)
yvals = np.log10(ks_agg_err)
s += tkz.get_2d_plot(xvals, yvals,
                     color=colors[0],
                     symbol=None,
                     xscale=xscale, yscale=yscale,
                     xmin=xmin, xmax=xmax,
                     ymin=ymin, ymax=ymax)
s += tkz.get_2d_plot(xvals[::3], yvals[::3], line_dim=None,
                     color=colors[0],
                     symbol=symbols[0],
                     symbol_size=0.03,
                     xscale=xscale, yscale=yscale,
                     xmin=xmin, xmax=xmax,
                     ymin=ymin, ymax=ymax)

xvals = np.log10(eps_vals)
yvals = np.log10(pnorm_agg_err)
s += tkz.get_2d_plot(xvals, yvals,
                     color=colors[1],
                     symbol=None,
                     xscale=xscale, yscale=yscale,
                     xmin=xmin, xmax=xmax,
                     ymin=ymin, ymax=ymax)

s += tkz.get_2d_plot(xvals[::3], yvals[::3], line_dim=None,
                     color=colors[1],
                     symbol=symbols[1],
                     symbol_size=0.03,
                     xscale=xscale, yscale=yscale,
                     xmin=xmin, xmax=xmax,
                     ymin=ymin, ymax=ymax)

# Set the labels (lower-left corner)
for k, label in enumerate(['KS', '$p$-norm']):
    x = xmin + 0.05*(xmax - xmin)
    y = ymin + (ymax - ymin)*(0.95 - 0.05*k)
    length = 0.035*(xmax - xmin)
    s += tkz.get_legend_entry(x, y, length, label=label,
                              font_size='small',
                              color=colors[k % len(colors)],
                              symbol=symbols[k % len(symbols)],
                              symbol_size=0.03,
                              xscale=xscale, yscale=yscale)

# Plot the axes
s += tkz.get_2d_axes(xmin, xmax, ymin, ymax,
                     xscale=xscale, yscale=yscale,
                     xticks=xticks, yticks=yticks,
                     xtick_labels=xtick_labels,
                     ytick_labels=ytick_labels,
                     tick_font='small',
                     tick_frac=0.01,
                     xlabel_offset=0.075,
                     label_font='large',
                     xlabel='1/$\\rho$',
                     ylabel_offset=0.165,
                     ylabel='Aggregation error')

s += tkz.get_end_tikz()

fp = open('results/disk_exact_aggregation_error.tex', 'w')
fp.write(s)
fp.close()


text:
from dataclasses import dataclass, field
from typing import Type

import numpy as np
from paiargparse import pai_dataclass, pai_meta
from tfaip.data.pipeline.processor.dataprocessor import DataProcessorParams

from calamari_ocr.ocr.dataset.imageprocessors.data_preprocessor import ImageProcessor

@pai_dataclass(alt="FinalPreparation")
@dataclass
class FinalPreparationProcessorParams(DataProcessorParams):
    normalize: bool = True
    invert: bool = True
    transpose: bool = True
    pad: int = field(default=16, metadata=pai_meta(help="Padding (left right) of the line"))
    pad_value: int = 0

    @staticmethod
    def cls() -> Type["ImageProcessor"]:
        return FinalPreparation

class FinalPreparation(ImageProcessor[FinalPreparationProcessorParams]):
    def _apply_single(self, data: np.ndarray, meta):
        if len(data.shape) != 3:
            data = np.expand_dims(data, axis=-1)  # add channels dimension

        channels = data.shape[-1]

        if data.size > 0:
            # non empty image
            if self.params.normalize:
                amax = np.amax(data)
                if amax > 0:
                    data = data * 1.0 / amax

            if self.params.invert:
                data = np.amax(data) - data

        if self.params.transpose:
            data = np.swapaxes(data, 1, 0)

        if self.params.pad > 0:
            if self.params.transpose:
                w = data.shape[1]
                data = np.vstack(
                    [
                        np.full((self.params.pad, w, channels), self.params.pad_value),
                        data,
                        np.full((self.params.pad, w, channels), self.params.pad_value),
                    ]
                )
            else:
                w = data.shape[0]
                data = np.hstack(
                    [
                        np.full((w, self.params.pad, channels), self.params.pad_value),
                        data,
                        np.full((w, self.params.pad, channels), self.params.pad_value),
                    ]
                )

        data = (data * 255).astype(np.uint8)

        if channels == 1:
            data = np.squeeze(data, axis=-1)

        return data

    def local_to_global_pos(self, x, params):
        if self.params.pad > 0 and self.params.transpose:
            return x - self.params.pad
        else:
            return x

max_stars_repo_name: alexander-winkler/calamari

max_issues_repo_name: alexander-winkler/calamari

max_forks_repo_name: alexander-winkler/calamari


text:
"""Simulation box utilities
"""

import numpy as np
import math

def pbc_place_inside(box: np.ndarray, r_out: np.ndarray) -> np.ndarray:
    """Places point inside simulation box applying periodic boundary condition.
    :param box Simulation box.
    :param r_out Point possibly outside box.
    :return r_in Point inside simulation.
    """
    r_in = np.array([r_out[0], r_out[1], r_out[2]])
    for k in (0, 1, 2):
        r_k = r_in[k]
        box_k = box[k]
        n = np.rint(math.floor(r_k/box_k))
        r_in[k] -= n * box_k
    return r_in

def pbc_distance(box: np.ndarray, r_i: np.ndarray, r_j: np.ndarray) -> np.ndarray:
    """Returns distance between two points applying periodic boundary condition.
    :param box Simulation box.
    :param r_i Point i.
    :param r_j Point j.
    :return rij = r_i - r_j
    """
    r_ij = np.array([0.0, 0.0, 0.0])
    for k in (0, 1, 2):
        box_k = box[k]
        dr = r_i[k] - r_j[k]
        ratio = dr / box_k
        n = np.rint(ratio)
        r_ij[k] = dr - n * box_k
    return r_ij

max_stars_repo_name: bco-bc/panalysis

max_issues_repo_name: bco-bc/panalysis

max_forks_repo_name: bco-bc/panalysis


text:
from glob import glob
import pandas as pd
from multiprocessing import cpu_count, Pool
from os.path import isdir, join, isfile
from functools import partial
from os import scandir
from collections.abc import Iterable
from sqlalchemy import (create_engine, engine, Column,
                        Text, Table, MetaData, String,
                        Boolean, DateTime, Integer,
                        Float, ForeignKey, CheckConstraint, PrimaryKeyConstraint)
from numpy import dtype
from subprocess import Popen, PIPE

class DataModel:

    def __init__(self, file_paths_or_dataframe, files_format=None, cpu=None, **kwargs):

        self.extension = files_format
        self.load_function = kwargs
        self.cores = cpu
        self.data = file_paths_or_dataframe

    @property
    def cores(self):

        return self._cpu

    @property
    def data(self):

        return self._data

    @property
    def load_function(self):

        return self._load_function

    @property
    def extension(self):

        return self._extension

    @cores.setter
    def cores(self, cpu):

        if not cpu:
            self._cpu = cpu_count()
        else:
            self._cpu = cpu

    @data.setter
    def data(self, files_path_or_data):

        if isinstance(files_path_or_data, (pd.DataFrame, pd.Series)):
            self._data = files_path_or_data
        else:
            if isinstance(files_path_or_data, str):
                if isdir(files_path_or_data):
                    self.input = glob(join(files_path_or_data, '*{}'.format(self.extension)))
                elif isfile(files_path_or_data):
                    self.input = [files_path_or_data, ]
            elif isinstance(files_path_or_data, (list, tuple)):
                self.input = files_path_or_data
            else:
                raise Exception('file_paths_or_dataframe format is not valid')
            if self.input:
                self._data = self.__get_data()

    @load_function.setter
    def load_function(self, kwargs):
        func = self.__get_function()
        if not func:
            self._load_function = None
        else:
            self._load_function = partial(self.func, function=partial(self.__get_function(), **kwargs))

    @extension.setter
    def extension(self, format_str):

        if format_str is None:
            self._extension = format_str
        elif format_str.lower() in ['json', 'csv', 'xlsx', 'feather', 'parquet']:
            self._extension = '.{}'.format(format_str)
        else:
            raise Exception('extension not valid should be: ')

    @staticmethod
    def func(item, function):

        try:
            temp_data = function(item)
            if isinstance(temp_data, pd.Series):
                temp_columns = temp_data.index
            else:
                temp_columns = temp_data.columns
            # print(item)
            return temp_columns.tolist(), temp_data.values
        except (pd.errors.ParserError, pd.errors.EmptyDataError, Exception) as error:
            # Logic to catch and log errors in a multiprocessing pool to be implemented
            # pass statement is currently deemed acceptable
            pass

    def __convert_to_df(func):

        def _wrapped_convert_to_df(self):

            output_list = func(self)
            output_list_filtered = [*self.filter_by_type(output_list, Iterable)]
            max_index = len(output_list_filtered)-1
            check_headers = True
            index = 0
            while check_headers:
                headers = output_list_filtered[index][0]
                if headers:
                    check_headers = False
                else:
                    index += 1
                if index == max_index and not output_list_filtered[index][0]:
                    raise Exception('No valid headers found')

            data_list = [data_values for columns, data_values
                         in output_list_filtered if columns == headers]
            return pd.DataFrame(data_list, columns=headers)

        return _wrapped_convert_to_df

    @staticmethod
    def filter_by_type(sequence, data_type):
        for element in sequence:
            if isinstance(element, data_type):
                yield element

    def __wrapper_parallel_load(func):

        def wrapped_parallel_load(self):
            variables, function, n_cpu = func(self)
            pool = Pool(n_cpu)
            multiprocess_output = pool.map(function, variables, n_cpu)
            pool.close()
            pool.join()
            return multiprocess_output

        return wrapped_parallel_load

    @__convert_to_df
    @__wrapper_parallel_load
    def __get_data(self):

        return self.input, self.load_function, self.cores

    def split_by(self, granularity):
        """
        Splits and remove duplicates from the loaded table based on a columns list
        :param granularity:
            Type: list
            A list of columns for keep
        :return: pandas DataFrame
        """
        return self.data[granularity].drop_duplicates().reset_index(drop=True)

    def __get_function(self):

        functions_dict = {'.json': pd.read_json,
                          '.csv': pd.read_csv,
                          '.xlsx': pd.read_excel,
                          '.feather': pd.read_feather,
                          '.parquet': pd.read_parquet}
        if not self.extension:
            return None
        else:
            return functions_dict.get(self.extension)

class DatabaseManagement:

    dtypes_dict = {'object': String,
                   'str': String,
                   'int64': Integer,
                   'int': Integer,
                   'float64': Float,
                   'float': Float,
                   'datetime64': DateTime,
                   'bool':	Boolean}

    def __init__(self, username, password, dialect,
                 host, port, database_name=None):

        self.url = engine.url.URL(**{'password': password,
                                        'username': username,
                                        'drivername': dialect,
                                        'host': host,
                                        'port': port,
                                        'database': database_name})
        self.engine = self.url

        self.meta = MetaData()

    @property
    def engine(self):

        return self._engine

    @engine.setter
    def engine(self, url):

        self._engine = create_engine(url)

    def to_sql_copy(self, table_name, from_table, columns_kwargs):

        if isinstance(from_table, pd.DataFrame):

            columns_dict_base = self.__get_parameters_dict(from_table.dtypes.to_dict(),
                                                           columns_kwargs)

            for key, item in columns_dict_base.items():
                columns_dict_base[key]['type'] = self.__pandas_to_sql_types(item['type'])

            selectable_columns = [self.__column_parameters(items) for items in columns_dict_base.items()]

            object_table = Table(table_name, self.meta, *selectable_columns)

            if not self.engine.dialect.has_table(self.engine, table_name):

                self.meta.create_all(self.engine)

            self.__copy_to_table(table_name, from_table, '#')

    def __copy_to_table(self, table_name, data, delimiter):

        self._open_process()
        statement = "\COPY {} from STDIN DELIMITER '{}';\n".format(table_name, delimiter)
        print(statement)
        # # COPY DATA IN NEWLY CREATED TABLE
        data = data.apply(lambda col: col.str.replace(delimiter, ' ') if col.dtype == 'object' else col, axis=0)
        data_str = data.to_csv(sep=delimiter, index=False, header=False).split('\n')[:-1]
        self.psql.stdin.write(bytes(statement, 'utf-8'))
        [self.psql.stdin.write(line) for line in map(lambda row: bytes(row+'\n', 'utf-8'), data_str)]
        self.psql.stdin.write(bytes("\.\n", 'utf-8'))

    @staticmethod
    def __get_parameters_dict(main_dict, additional_dict, level_1_key='type'):

        output_dict = dict(zip(main_dict.keys(),
                           zip(len(main_dict)*[level_1_key, ], main_dict.values())))

        output_dict = {key: [value, ] for key, value in output_dict.items()}

        for key in output_dict.keys():
            if key in additional_dict.keys():
                for item in additional_dict.get(key).items():
                    output_dict.get(key).append(item)

        output_dict_w_kwargs = {key: dict(value) for key, value in output_dict.items()}

        return output_dict_w_kwargs

    @staticmethod
    def __column_parameters(items):

        column_name, args = items
        column_type = args.get('type')
        del args['type']
        extra_constraints = []
        if 'CheckConstraint' in args.keys():
            extra_constraints.append(CheckConstraint(args['CheckConstraint']))
            del args['CheckConstraint']

        return Column(column_name, column_type, *extra_constraints, **args)

    def _open_process(self):

        self.psql = Popen(['psql', str(self.url)], stdout=PIPE, stdin=PIPE)

    def __pandas_to_sql_types(self, type_to_convert):

        if isinstance(type_to_convert, dtype):

            type_to_convert = type_to_convert

        elif isinstance(type_to_convert, type):

            type_to_convert = type_to_convert.__name__

        return self.dtypes_dict.get(str(type_to_convert), Text)

    def drop_table(self):
        pass

if __name__ == '__main__':

    # THIS IS A WIP SCRIPT USED TO MOVE DATA FROM INDIVIDUAL FILES
    # TO A POSTRES DATABASE IN THE CLOUD
    # data is on disk; few 100k files; each file represents a song

    paths = [i.path for idx, i in enumerate(scandir(r'/Users/********/PycharmProjects/udacity/data')) if idx < 500000]

    # data_for_upload = DataModel(paths, 'json', **{'typ': 'series'})

    # data has already been harvested and consolidated into a csv
    data_for_upload = DataModel(pd.read_csv('test.csv'))

    artists_data = data_for_upload.split_by(['artist_id', 'song_id', 'title', 'duration', 'year'])

    # Initializing the DatabaseNanagement class
    # db is hosted in by AWS RDS

    db = DatabaseManagement('********', '********',
                           'postgresql', '*******.cvrvhtkqtojs.us-east-2.rds.amazonaws.com',
                           5432, 'postgres')

    # COPY 50K RECORDS IN DB IN 5s
    # CREATES TABLE IF DOESNT EXIST

    db.to_sql_copy("songs_table", artists_data, columns_kwargs={'artist_id':{'primary_key':False},
                                                                'song_id':{'nullable':False},
                                                                'year':{'nullable': True}})

max_stars_repo_name: KSowska/Portfolio

max_issues_repo_name: KSowska/Portfolio

max_forks_repo_name: KSowska/Portfolio


text:
import pickle
import os
import numpy as np

from postprocess_utils import *
from constants import *

def save_binary(obj, filename, append=False):
    if filename[-4:] != ".pkl":
        print("Adding .pkl extension as it was not found.", flush=True)
        filename = filename + ".pkl"
    if os.path.exists(filename) and append:
        print(f"Found file with name {filename}. Appending results to this file.", flush=True)
        contents = load_binary(filename)
        if append=="embeds":
            obj = np.vstack((contents,obj))
        elif append:  # contents of filename are assumed to be contained in the form of a list. obj is assumed to be a list
            obj = contents + obj
    with open(filename, 'wb') as outfile:
        pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)

def load_binary(filename):
    result = None
    with open(filename, 'rb') as infile:
        result = pickle.load(infile)
    return result

def mkdir(dir):
    os.chdir(".")
    if not os.path.isdir(dir):
        os.mkdir(dir)

def load_windows(data_path, pipeline, require_text=False, text_path=None, require_image=False, image_path=None,
                 require_audio=False, hand3d_image=False, use_lazy=False, test_smpl=False, temporal=False):
    feats = pipeline.split('2')
    p0_size, p1_size = FEATURE_MAP[pipeline]
    if os.path.exists(data_path):
        print('using super quick load', data_path, flush=True)
        data = load_binary(data_path)
        data = make_equal_len(data, method="cutting+reflect")
        if pipeline in ["arm_wh2wh", "wh2wh"]:
            p0_windows = data[:,:,:]
            p1_windows = data[:,:,6*6:]
        elif pipeline=="arm2wh" or pipeline[:13]=="arm_wh2finger":
            p0_windows = data[:,:,:p0_size]
            p1_windows = data[:,:,p0_size:p0_size+p1_size]
        if require_text and not require_image:
            text_windows = load_binary(text_path)
            p0_windows = (p0_windows, text_windows)
        elif require_image and not require_text:
            image_windows = load_binary(image_path)
            image_windows = make_equal_len(image_windows, method="cutting+reflect")
            p0_windows = (p0_windows, image_windows)
        return p0_windows, p1_windows


text:
from rastervision.data.label import Labels

import numpy as np
from rasterio.features import rasterize
import shapely

class SemanticSegmentationLabels(Labels):
    """A set of spatially referenced semantic segmentation labels.

    Since labels are represented as rasters, the labels for a scene can take up a lot of
    memory. Therefore, to avoid running out of memory, labels are computed as needed for
    windows.
    """

    def __init__(self, windows, label_fn, aoi_polygons=None):
        """Constructor

        Args:
            windows: a list of Box representing the windows covering a scene
            label_fn: a function that takes a window (Box) and returns a label array
                of the same shape with each value a class id.
            aoi_polygons: a list of shapely.geom that contains the AOIs
                (areas of interest) for a scene.

        """
        self.windows = windows
        self.label_fn = label_fn
        self.aoi_polygons = aoi_polygons

    def __add__(self, other):
        """Add labels to these labels.

        Returns a concatenation of this and the other labels.
        """
        return SemanticSegmentationLabels(
            self.windows + other.windows,
            self.label_fn,
            aoi_polygons=self.aoi_polygons)

    def __eq__(self, other):
        for window in self.get_windows():
            if not np.array_equal(
                    self.get_label_arr(window), other.get_label_arr(window)):
                return False
        return True

    def filter_by_aoi(self, aoi_polygons):
        """Returns a new SemanticSegmentationLabels object with aoi_polygons set."""
        return SemanticSegmentationLabels(
            self.windows, self.label_fn, aoi_polygons=aoi_polygons)

    def add_window(self, window):
        self.windows.append(window)

    def get_windows(self):
        return self.windows

    def get_label_arr(self, window, clip_extent=None):
        """Get the label array for a window.

        Note: the window should be kept relatively small to avoid running out of memory.

        Args:
            window: Box
            clip_extent: a Box representing the extent of the corresponding Scene

        Returns:
            np.ndarray of class_ids with zeros filled in outside the AOIs and clipped
                to the clip_extent
        """
        window_geom = window.to_shapely()

        if not self.aoi_polygons:
            label_arr = self.label_fn(window)
        else:
            # For each aoi_polygon, intersect with window, and put in window frame of
            # reference.
            window_aois = []
            for aoi in self.aoi_polygons:
                window_aoi = aoi.intersection(window_geom)
                if not window_aoi.is_empty:

                    def transform_shape(x, y, z=None):
                        return (x - window.xmin, y - window.ymin)

                    window_aoi = shapely.ops.transform(transform_shape,
                                                       window_aoi)
                    window_aois.append(window_aoi)

            if window_aois:
                # If window intersects with AOI, set pixels outside the AOI polygon to 0,
                # so they are ignored during eval.
                label_arr = self.label_fn(window)
                mask = rasterize(
                    [(p, 0) for p in window_aois],
                    out_shape=label_arr.shape,
                    fill=1,
                    dtype=np.uint8)
                label_arr[mask.astype(np.bool)] = 0
            else:
                # If window does't overlap with any AOI, then return all zeros.
                label_arr = np.zeros((window.get_height(), window.get_width()))

        if clip_extent is not None:
            clip_window = window.intersection(clip_extent)
            label_arr = label_arr[0:clip_window.get_height(), 0:
                                  clip_window.get_width()]

        return label_arr

max_stars_repo_name: carderne/raster-vision

max_issues_repo_name: carderne/raster-vision

max_forks_repo_name: carderne/raster-vision