text:
# coding: utf-8
import numpy as np
import cv2
import os

INPUT_PATH = 'F:/Scan-flood-Fill/flow_chart/pig_input'
OUTPUT_PATH_FILL = 'F:/Scan-flood-Fill/flow_chart/pig_output'
colorThreshold = 20

def del_file(path):
    # delete files
    for i in os.listdir(path):
        path_file = os.path.join(path, i)
        if os.path.isfile(path_file):
            os.remove(path_file)
        else:
            del_file(path_file)

def pad_with(vector, pad_width, iaxis, kwargs):
    pad_value = kwargs.get('padder', 0)
    vector[:pad_width[0]] = pad_value
    vector[-pad_width[1]:] = pad_value
    return vector

def readAndPad(imagePath, backGroundColour):
    img = cv2.imread(imagePath, 0)
    padImg = np.pad(img,1,pad_with, padder=backGroundColour)
    height, width = padImg.shape[:2]
    return padImg, height, width

def cropAndReverse(img, height, width, backGroundColor, labelColor):
    croppedImg = np.delete((np.delete(img, [0, width-1], axis=1)), [0, height-1], axis=0)
    cv2.imwrite(os.path.join(OUTPUT_PATH_FILL, 'pig_after_cropping.png'), croppedImg.astype('uint8'),[int(cv2.IMWRITE_PNG_COMPRESSION), 0])

    for x in range(height-2):
        for y in range(width-2):
            # label is the color that is temporarily used to fill exterior
            if croppedImg[x,y] == labelColor:
                croppedImg[x,y] = backGroundColor
            if croppedImg[x, y] == 128:
                croppedImg[x, y] = 255
    return croppedImg

def cvFloodFill(img, height, width, seedPosition, fillColor):
    mask = np.zeros([height+2, width +2], np.uint8)
    cv2.floodFill(img, mask, seedPosition, newVal = fillColor, loDiff = 20, upDiff = 20, flags = 4)

def holesInHoles(imgPath, savePath, backGroundColor, boundaryColor, labelColor, fillColor):
    padImg, height, width = readAndPad(imgPath, backGroundColor)
    for x in range(height):
        for y in range(width):
            if abs(padImg[x, y] - backGroundColor) < 50:
                padImg[x, y] = backGroundColor
            else:
                padImg[x, y] = boundaryColor
    cv2.imwrite(os.path.join(OUTPUT_PATH_FILL, 'pig_before_first_fill.png'), padImg.astype('uint8'), [int(cv2.IMWRITE_PNG_COMPRESSION), 0])
    seedPosition = (0, 0)
    cvFloodFill(padImg, height, width, seedPosition, labelColor)
    j= 1
    fname = 'pig_after_' + str(j) + 'main_filling.png'
    cv2.imwrite(os.path.join(savePath, fname), padImg.astype('uint8'),[int(cv2.IMWRITE_PNG_COMPRESSION), 0])
    for x in range(height):
        for y in range(width):
            pixelValue = padImg[x, y]
            if abs(pixelValue - backGroundColor) < colorThreshold:
                seedPosition = (y,x)
                i = 1
                while abs(padImg[x, y-i] - boundaryColor) < colorThreshold:
                    i = i + 1
                if abs(padImg[x, y-i] - labelColor) < colorThreshold:
                    #print(padImg[x, y-i])
                    cvFloodFill(padImg, height, width, seedPosition, fillColor)
                else:
                    #print(padImg[x, y-i])
                    cvFloodFill(padImg, height, width, seedPosition, labelColor)
                j = j + 1
                fname = 'pig_after_' + str(j) + 'main_filling.png'
                cv2.imwrite(os.path.join(savePath, fname), padImg.astype('uint8'), [int(cv2.IMWRITE_PNG_COMPRESSION), 0])

    resultImg = cropAndReverse(padImg, height, width, backGroundColor, labelColor)
    cv2.imwrite(os.path.join(savePath, 'pig_final.png'), resultImg.astype('uint8'), [int(cv2.IMWRITE_PNG_COMPRESSION), 0])

def holeFill_main():
    boundaryColor = 255
    fillColor = 128  # for practice
    labelColor = 80
    backGroundColor = 0
    input_path = INPUT_PATH
    output_path_fill = OUTPUT_PATH_FILL
    for root, dirs, files in os.walk(input_path):
        for f in files:
            holesInHoles(os.path.join(input_path, f), output_path_fill, backGroundColor, boundaryColor, labelColor, fillColor)

if __name__ == '__main__':
    holeFill_main()

max_stars_repo_name: SherylHYX/Scan-flood-Fill

max_issues_repo_name: SherylHYX/Scan-flood-Fill

max_forks_repo_name: SherylHYX/Scan-flood-Fill


text:
from __future__ import print_function
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
import re
import torch
import torch.nn.functional as F
from torch.autograd import Variable
from torch.utils import model_zoo
from nn_transfer import transfer
# from tensorflow.python.keras.models import Model
# from tensorflow.python.keras.layers import Input, Add, Activation, Dropout, Flatten, Dense,\
#                                         Convolution2D, MaxPooling2D, AveragePooling2D, BatchNormalization
from keras.models import Model
from keras.layers import Input, Add, Activation, Dropout, Flatten, Dense, AveragePooling2D, MaxPooling2D, Reshape
from keras.layers.convolutional import Convolution2D

# Transfer tutorial <https://github.com/gzuidhof/nn-transfer/blob/master/example.ipynb>
## PyTorch Things
# convert numpy arrays to torch Variables
params = model_zoo.load_url('https://s3.amazonaws.com/modelzoo-networks/wide-resnet-50-2-export-5ae25d50.pth')

for k, v in sorted(params.items()):
    print(k, tuple(v.shape))
    params[k] = Variable(v, requires_grad=True)
print('\nTotal parameters:', sum(v.numel() for v in params.values()))

# PyTorch Model definition, from <https://github.com/szagoruyko/functional-zoo/blob/master/wide-resnet-50-2-export.ipynb>
def define_pytorch_model(params):
    def conv2d(input, params, base, stride=1, pad=0):
        return F.conv2d(input, params[base + '.weight'],
                        params[base + '.bias'], stride, pad)

    def group(input, params, base, stride, n):
        o = input
        for i in range(0, n):
            b_base = ('%s.block%d.conv') % (base, i)
            x = o
            o = conv2d(x, params, b_base + '0')
            o = F.relu(o)
            o = conv2d(o, params, b_base + '1', stride=i == 0 and stride or 1, pad=1)
            o = F.relu(o)
            o = conv2d(o, params, b_base + '2')
            if i == 0:
                o += conv2d(x, params, b_base + '_dim', stride=stride)
            else:
                o += x
            o = F.relu(o)
        return o

    # determine network size by parameters
    blocks = [sum([re.match('group%d.block\d+.conv0.weight' % j, k) is not None
                   for k in params.keys()]) for j in range(4)]

    def f(input, params):
        o = F.conv2d(input, params['conv0.weight'], params['conv0.bias'], 2, 3)
        o = F.relu(o)
        o = F.max_pool2d(o, 3, 2, 1)
        o_g0 = group(o, params, 'group0', 1, blocks[0])
        o_g1 = group(o_g0, params, 'group1', 2, blocks[1])
        o_g2 = group(o_g1, params, 'group2', 2, blocks[2])
        o_g3 = group(o_g2, params, 'group3', 2, blocks[3])
        o = F.avg_pool2d(input=o_g3, kernel_size=7, stride=1, padding=0)
        o = o.view(o.size(0), -1)
        o = F.linear(o, params['fc.weight'], params['fc.bias'])
        return o

    return f

## Keras Things
def define_keras_model(input_dim, nb_classes=1000):
    def group(input, stride, n, group_num):
        o = input
        for i in range(0, n):
            b_base = ('%s.block%d.conv') % ('group' + str(group_num), i)
            kernel0 = 2 ** (7 + group_num)
            kernel_1 = 2 ** (7 + group_num)
            kernel_2 = 2 ** (8 + group_num)
            kernel_dim = 2 ** (8 + group_num)

            x = o
            # conv0
            o = Convolution2D(kernel0, (1, 1), padding='same', strides=(1, 1), kernel_initializer='he_normal',
                              use_bias=True, name=b_base + '0', activation='relu')(x)
            # conv1
            stride_0 = i == 0 and stride or 1  # lazy copy from pytorch loop
            o = Convolution2D(kernel_1, (3, 3), padding='same', strides=(stride_0, stride_0),
                              kernel_initializer='he_normal',
                              use_bias=True, name=b_base + '1', activation='relu')(o)
            # conv2
            o = Convolution2D(kernel_2, (1, 1), padding='same', strides=(1, 1), kernel_initializer='he_normal',
                              use_bias=True, name=b_base + '2')(o)
            # print(o.shape)
            if i == 0:

                o = Add()([o, Convolution2D(kernel_dim, (1, 1), padding='same', strides=(stride, stride),
                                            kernel_initializer='he_normal',
                                            use_bias=True, name=b_base + '_dim')(x)])
            else:
                o = Add()([o, x])

            o = Activation('relu')(o)
        return o

    # input
    ip = Input(shape=input_dim)

    # conv0
    x = Convolution2D(64, (7, 7), padding='same', strides=(2,2), kernel_initializer='he_normal',
                      use_bias=True, name='conv0', activation='relu')(ip)
    # x = BatchNormalization(axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer='uniform')(x)

    # max pool
    x = MaxPooling2D(pool_size=(3,3), strides=(2,2), padding='valid')(x)

    # group0 (stride 1, n=3)
    x = group(x,stride=1,n=3, group_num=0)

    # group1 (stride 2, n=4)
    x = group(x, stride=2, n=4, group_num=1)

    # group2 (stride 2, n=6)
    x = group(x, stride=2, n=6, group_num=2)

    # group3 (stride 2, n=3)
    x = group(x, stride=2, n=3, group_num=3)

    # avgpool2d
    x = AveragePooling2D((7, 7),strides=(1,1),padding='valid')(x)
    x = Flatten()(x)
    # x = Reshape((2048,))(x)
    x = Dense(nb_classes, name='fc', use_bias=True)(x)
    # x = Activation('linear')(x)
    model = Model(ip, x)

    return model

# Tensorflow
def define_tensorflow_model(inputs, params):
    '''Bottleneck WRN-50-2 model definition
    '''

    def tr(v):
        if v.ndim == 4:
            return v.transpose(2, 3, 1, 0)
        elif v.ndim == 2:
            return v.transpose()
        return v

    params = {k: tf.constant(tr(v)) for k, v in params.items()}

    def conv2d(x, params, name, stride=1, padding=0):
        x = tf.pad(x, [[0, 0], [padding, padding], [padding, padding], [0, 0]])
        z = tf.nn.conv2d(x, params['%s.weight' % name], [1, stride, stride, 1],
                         padding='VALID')
        if '%s.bias' % name in params:
            return tf.nn.bias_add(z, params['%s.bias' % name])
        else:
            return z

    def group(input, params, base, stride, n):
        o = input
        for i in range(0, n):
            b_base = ('%s.block%d.conv') % (base, i)
            x = o
            o = conv2d(x, params, b_base + '0')
            o = tf.nn.relu(o)
            o = conv2d(o, params, b_base + '1', stride=i == 0 and stride or 1, padding=1)
            o = tf.nn.relu(o)
            o = conv2d(o, params, b_base + '2')
            if i == 0:
                o += conv2d(x, params, b_base + '_dim', stride=stride)
            else:
                o += x
            o = tf.nn.relu(o)
        return o

    # determine network size by parameters
    blocks = [sum([re.match('group%d.block\d+.conv0.weight' % j, k) is not None
                   for k in params.keys()]) for j in range(4)]

    o = conv2d(inputs, params, 'conv0', 2, 3)
    o = tf.nn.relu(o)
    o = tf.pad(o, [[0, 0], [1, 1], [1, 1], [0, 0]])
    o = tf.nn.max_pool(o, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID')
    o_g0 = group(o, params, 'group0', 1, blocks[0])
    o_g1 = group(o_g0, params, 'group1', 2, blocks[1])
    o_g2 = group(o_g1, params, 'group2', 2, blocks[2])
    o_g3 = group(o_g2, params, 'group3', 2, blocks[3])
    o = tf.nn.avg_pool(o_g3, ksize=[1, 7, 7, 1], strides=[1, 1, 1, 1], padding='VALID')
    o = tf.reshape(o, [-1, 2048])
    o = tf.matmul(o, params['fc.weight']) + params['fc.bias']
    return o

# Define the PyTorch model
pytorch_model = define_pytorch_model(params)

# Define the tensorflow model
params_tf = {k: v.numpy() for k, v in model_zoo.load_url('https://s3.amazonaws.com/modelzoo-networks/wide-resnet-50-2-export-5ae25d50.pth').items()}
inputs_tf = tf.placeholder(tf.float32, shape=[None, 224, 224, 3])
tf_model = define_tensorflow_model(inputs_tf, params_tf)

# Define the Keras model, reloading the weights since we can't use get_state_dict() on functional pytorch models
input_state_dict_for_keras = model_zoo.load_url('https://s3.amazonaws.com/modelzoo-networks/wide-resnet-50-2-export-5ae25d50.pth')
keras_model = define_keras_model((224,224,3), nb_classes=1000)
# Transfer the pytorch weights to keras
transfer.pytorch_to_keras(pytorch_model, keras_model, state_dict=input_state_dict_for_keras)

# Create dummy data
torch.manual_seed(0)
data = torch.rand(6,3,224,224)
data_keras_and_tf = data.permute(0,2,3,1).numpy()
data_pytorch = Variable(data, requires_grad=False)

# Do a forward pass in all frameworks
pytorch_pred = pytorch_model(data_pytorch, params).data.numpy()
keras_pred = keras_model.predict(data_keras_and_tf)
sess = tf.Session()
tf_pred = sess.run(tf_model, feed_dict={inputs_tf: data_keras_and_tf})

assert keras_pred.shape == pytorch_pred.shape
# check that difference between PyTorch and Tensorflow is small
assert np.abs(tf_pred - pytorch_pred).max() < 1e-4
print(np.abs(keras_pred - pytorch_pred).max())
print(np.abs(keras_pred[...,::-1] - pytorch_pred).max())
print(np.abs(tf_pred - pytorch_pred).max())

plot_comparison = False

if plot_comparison:
    plt.subplot(3, 1, 1)
    plt.axis('Off')
    plt.imshow(pytorch_pred[:,0:40])
    plt.title('pytorch')
    plt.subplot(3, 1, 2)
    plt.axis('Off')
    plt.imshow(tf_pred[:,0:40])
    plt.title('tensorflow')
    plt.subplot(3, 1, 3)
    plt.axis('Off')
    plt.imshow(keras_pred[:,0:40])
    plt.title('keras')
    plt.show()

plot_diff_images = True

if plot_diff_images:
    data_keras_and_tf_flip_ch = keras_model.predict(data_keras_and_tf[...,::-1])
    data_keras_and_tf_flip_lr = keras_model.predict(data_keras_and_tf[:,::-1,:,:])
    data_keras_and_tf_flip_lr_ch = keras_model.predict(data_keras_and_tf[:, ::-1, :, ::-1])
    data_keras_and_tf_flip_ud = keras_model.predict(data_keras_and_tf[:,:,::-1,:])
    data_keras_and_tf_flip_ud_ch = keras_model.predict(data_keras_and_tf[:,:,::-1,::-1])
    data_keras_and_tf_flip_lr_ud = keras_model.predict(data_keras_and_tf[:,::-1,::-1,:])
    data_keras_and_tf_flip_lr_ud_ch = keras_model.predict(data_keras_and_tf[:,::-1,::-1,::-1])
    swap_lr = data_keras_and_tf.transpose(0,2,1,3)
    data_keras_and_tf_swap_lr = keras_model.predict(swap_lr)
    data_keras_and_tf_flip_ch_swap_lr = keras_model.predict(swap_lr[..., ::-1])
    data_keras_and_tf_flip_lr_swap_lr = keras_model.predict(swap_lr[:, ::-1, :, :])
    data_keras_and_tf_flip_lr_ch_swap_lr = keras_model.predict(swap_lr[:, ::-1, :, ::-1])
    data_keras_and_tf_flip_ud_swap_lr = keras_model.predict(swap_lr[:, :, ::-1, :])
    data_keras_and_tf_flip_ud_ch_swap_lr = keras_model.predict(swap_lr[:, :, ::-1, ::-1])
    data_keras_and_tf_flip_lr_ud_swap_lr = keras_model.predict(swap_lr[:, ::-1, ::-1, :])
    data_keras_and_tf_flip_lr_ud_ch_swap_lr = keras_model.predict(swap_lr[:, ::-1, ::-1, ::-1])
    print(np.abs(data_keras_and_tf_flip_ch - pytorch_pred).max())
    print(np.abs(data_keras_and_tf_flip_lr - pytorch_pred).max())
    print(np.abs(data_keras_and_tf_flip_lr_ch - pytorch_pred).max())
    print(np.abs(data_keras_and_tf_flip_ud - pytorch_pred).max())
    print(np.abs(data_keras_and_tf_flip_ud_ch - pytorch_pred).max())
    print(np.abs(data_keras_and_tf_flip_lr_ud - pytorch_pred).max())
    print(np.abs(data_keras_and_tf_flip_lr_ud_ch - pytorch_pred).max())

    print(np.abs(data_keras_and_tf_swap_lr - pytorch_pred).max())
    print(np.abs(data_keras_and_tf_flip_ch_swap_lr - pytorch_pred).max())
    print(np.abs(data_keras_and_tf_flip_lr_swap_lr - pytorch_pred).max())
    print(np.abs(data_keras_and_tf_flip_lr_ch_swap_lr - pytorch_pred).max())
    print(np.abs(data_keras_and_tf_flip_ud_swap_lr - pytorch_pred).max())
    print(np.abs(data_keras_and_tf_flip_ud_ch_swap_lr - pytorch_pred).max())
    print(np.abs(data_keras_and_tf_flip_lr_ud_swap_lr - pytorch_pred).max())
    print(np.abs(data_keras_and_tf_flip_lr_ud_ch_swap_lr - pytorch_pred).max())
    # plt.subplot(3, 1, 1)
    # plt.axis('Off')
    # plt.imshow(pytorch_pred[:, 0:40])
    # plt.title('pytorch')
    # plt.subplot(3, 1, 2)
    # plt.axis('Off')
    # plt.imshow(tf_pred[:, 0:40])
    # plt.title('tensorflow')
    # plt.subplot(3, 1, 3)
    # plt.axis('Off')
    # plt.imshow(keras_pred[:, 0:40])
    # plt.title('keras')
    # plt.show()

assert np.abs(keras_pred - pytorch_pred).max() < 1e-4 # This assertion will fail

print('done!')

ext: py

max_stars_repo_name: luketaverne/handcam

max_issues_repo_name: luketaverne/handcam

max_forks_repo_name: luketaverne/handcam

reason: import numpy


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Oct 10

@author: jaehyuk
"""

import numpy as np
import scipy.stats as ss
import scipy.optimize as sopt
import scipy.integrate as spint
import pyfeng as pf
from . import normal
from . import bsm

'''
MC model class for Beta=1
'''
class ModelBsmMC:
    beta = 1.0   # fixed (not used)
    vov, rho = 0.0, 0.0
    sigma, intr, divr = None, None, None
    bsm_model = None
    '''
    You may define more members for MC: time step, etc
    '''

    def __init__(self, sigma, vov=0, rho=0.0, beta=1.0, intr=0, divr=0, time_steps=1_000, n_samples=10_000):
        self.sigma = sigma
        self.vov = vov
        self.rho = rho
        self.intr = intr
        self.divr = divr
        self.time_steps = time_steps
        self.n_samples = n_samples
        self.bsm_model = pf.Bsm(sigma, intr=intr, divr=divr)

    def bsm_vol(self, strike, spot, texp=None, sigma=None):
        ''''
        From the price from self.price() compute the implied vol
        Use self.bsm_model.impvol() method
        '''
        price = self.price(strike, spot, texp, sigma)
        vol = self.bsm_model.impvol(price, strike, spot, texp)
        return vol

    def price(self, strike, spot, texp=None, sigma=None, cp=1, time_steps=1_000, n_samples=10_000):
        '''
        Your MC routine goes here
        Generate paths for vol and price first. Then get prices (vector) for all strikes
        You may fix the random number seed
        '''
        np.random.seed(12345)
        div_fac = np.exp(-texp * self.divr)
        disc_fac = np.exp(-texp * self.intr)
        forward = spot / disc_fac * div_fac

        if sigma is None:
            sigma = self.sigma

        self.time_steps = time_steps         # number of time steps of MC
        self.n_samples = n_samples          # number of samples of MC

        # Generate correlated normal random variables W1, Z1
        z = np.random.normal(size=(self.n_samples, self.time_steps))
        x = np.random.normal(size=(self.n_samples, self.time_steps))
        w = self.rho * z + np.sqrt(1-self.rho**2) * x

        path_size = np.zeros([self.n_samples, self.time_steps + 1])
        delta_tk = texp / self.time_steps
        log_sk = np.log(spot) * np.ones_like(path_size)      # log price
        sk = spot * np.ones_like(path_size)              # price
        sigma_tk = self.sigma * np.ones_like(path_size)       # sigma
        for i in range(self.time_steps):
            log_sk[:, i+1] = log_sk[:, i] + sigma_tk[:, i] * np.sqrt(delta_tk) * w[:, i] - 0.5 * (sigma_tk[:, i]**2) * delta_tk
            sigma_tk[:, i+1] = sigma_tk[:, i] * np.exp(self.vov * np.sqrt(delta_tk) * z[:, i] - 0.5 * (self.vov**2) * delta_tk)
            sk[:, i+1] = np.exp(log_sk[:, i+1])

        price = np.zeros_like(strike)

        for j in range(len(strike)):
            price[j] = np.mean(np.maximum(sk[:, -1] - strike[j], 0))
        return  disc_fac * price

'''
MC model class for Beta=0
'''
class ModelNormalMC:
    beta = 0.0   # fixed (not used)
    vov, rho = 0.0, 0.0
    sigma, intr, divr = None, None, None
    normal_model = None

    def __init__(self, sigma, vov=0, rho=0.0, beta=1.0, intr=0, divr=0, time_steps=1_000, n_samples=10_000):
        self.sigma = sigma
        self.vov = vov
        self.rho = rho
        self.intr = intr
        self.divr = divr
        self.time_steps = time_steps
        self.n_samples = n_samples
        self.normal_model = pf.Norm(sigma, intr=intr, divr=divr)

    def norm_vol(self, strike, spot, texp=None, sigma=None):
        ''''
        From the price from self.price() compute the implied vol.
        Use self.normal_model.impvol() method
        '''
        price = self.price(strike, spot, texp, sigma)
        vol = self.normal_model.impvol(price, strike, spot, texp)
        return vol

    def price(self, strike, spot, texp=None, sigma=None, cp=1, time_steps=1_000, n_samples=10_000):
        '''
        Your MC routine goes here
        Generate paths for vol and price first. Then get prices (vector) for all strikes
        You may fix the random number seed
        '''
        np.random.seed(12345)
        div_fac = np.exp(-texp * self.divr)
        disc_fac = np.exp(-texp * self.intr)
        forward = spot / disc_fac * div_fac

        if sigma is None:
            sigma = self.sigma

        self.time_steps = time_steps         # number of time steps of MC
        self.n_samples = n_samples          # number of samples of MC

        # Generate correlated normal random variables W1, Z1
        z = np.random.normal(size=(self.n_samples, self.time_steps))
        x = np.random.normal(size=(self.n_samples, self.time_steps))
        w = self.rho * z + np.sqrt(1-self.rho**2) * x

        path_size = np.zeros([self.n_samples, self.time_steps + 1])
        delta_tk = texp / self.time_steps

        sk = spot * np.ones_like(path_size)              # price
        sigma_tk = self.sigma * np.ones_like(path_size)       # sigma
        for i in range(self.time_steps):
            sk[:, i+1] = sk[:, i] + sigma_tk[:, i] * np.sqrt(delta_tk) * w[:, i]
            sigma_tk[:, i+1] = sigma_tk[:, i] * np.exp(self.vov * np.sqrt(delta_tk) * z[:, i] - 0.5 * (self.vov**2) * delta_tk)

        price = np.zeros_like(strike)

        for j in range(len(strike)):
            price[j] = np.mean(np.maximum(sk[:, -1] - strike[j], 0))
        return  disc_fac * price

'''
Conditional MC model class for Beta=1
'''
class ModelBsmCondMC:
    beta = 1.0   # fixed (not used)
    vov, rho = 0.0, 0.0
    sigma, intr, divr = None, None, None
    bsm_model = None
    '''
    You may define more members for MC: time step, etc
    '''

    def __init__(self, sigma, vov=0, rho=0.0, beta=1.0, intr=0, divr=0, time_steps=1_000, n_samples=10_000):
        self.sigma = sigma
        self.vov = vov
        self.rho = rho
        self.intr = intr
        self.divr = divr
        self.time_steps = time_steps
        self.n_samples = n_samples
        self.bsm_model = pf.Bsm(sigma, intr=intr, divr=divr)

    def bsm_vol(self, strike, spot, texp=None):
        ''''
        should be same as bsm_vol method in ModelBsmMC (just copy & paste)
        '''
        price = self.price(strike, spot, texp, sigma)
        vol = self.bsm_model.impvol(price, strike, spot, texp)
        return vol

    def price(self, strike, spot, texp=None, cp=1, time_steps=1_000, n_samples=10_000):
        '''
        Your MC routine goes here
        Generate paths for vol only. Then compute integrated variance and BSM price.
        Then get prices (vector) for all strikes
        You may fix the random number seed
        '''
        np.random.seed(12345)
        div_fac = np.exp(-texp * self.divr)
        disc_fac = np.exp(-texp * self.intr)
        forward = spot / disc_fac * div_fac

        self.time_steps = time_steps         # number of time steps of MC
        self.n_samples = n_samples          # number of samples of MC

        # Generate correlated normal random variables Z
        z = np.random.normal(size=(self.n_samples, self.time_steps))

        delta_tk = texp / self.time_steps
        sigma_tk = self.sigma * np.ones([self.n_samples, self.time_steps+1])

        for i in range(self.time_steps):
            sigma_tk[:, i+1] = sigma_tk[:, i] * np.exp(self.vov * np.sqrt(delta_tk) * z[:, i] - 0.5 * (self.vov ** 2) * delta_tk)

        I = spint.simps(sigma_tk * sigma_tk, dx=texp/self.time_steps) / (self.sigma**2)  # integrate by using Simpson's rule

        spot_cond = spot * np.exp(self.rho * (sigma_tk[:, -1] - self.sigma) / self.vov - (self.rho*self.sigma)**2 * texp * I / 2)
        vol = self.sigma * np.sqrt((1 - self.rho**2) * I)

        price = np.zeros_like(strike)

        for j in range(len(strike)):
            price[j] = np.mean(bsm.price(strike[j], spot_cond, texp ,vol))

        return  disc_fac * price

'''
Conditional MC model class for Beta=0
'''
class ModelNormalCondMC:
    beta = 0.0   # fixed (not used)
    vov, rho = 0.0, 0.0
    sigma, intr, divr = None, None, None
    normal_model = None

    def __init__(self, sigma, vov=0, rho=0.0, beta=0.0, intr=0, divr=0, time_steps=1_000, n_samples=10_000):
        self.sigma = sigma
        self.vov = vov
        self.rho = rho
        self.intr = intr
        self.divr = divr
        self.time_steps = time_steps
        self.n_samples = n_samples
        self.normal_model = pf.Norm(sigma, intr=intr, divr=divr)

    def norm_vol(self, strike, spot, texp=None):
        ''''
        should be same as norm_vol method in ModelNormalMC (just copy & paste)
        '''
        price = self.price(strike, spot, texp, sigma)
        vol = self.normal_model.impvol(price, strike, spot, texp)
        return vol

    def price(self, strike, spot, texp=None, cp=1, time_steps=1_000, n_samples=10_000):
        '''
        Your MC routine goes here
        Generate paths for vol only. Then compute integrated variance and normal price.
        You may fix the random number seed
        '''
        np.random.seed(12345)
        div_fac = np.exp(-texp * self.divr)
        disc_fac = np.exp(-texp * self.intr)
        forward = spot / disc_fac * div_fac

        self.time_steps = time_steps         # number of time steps of MC
        self.n_samples = n_samples          # number of samples of MC

        # Generate correlated normal random variables Z
        z = np.random.normal(size=(self.n_samples, self.time_steps))

        delta_tk = texp / self.time_steps
        sigma_tk = self.sigma * np.ones([self.n_samples, self.time_steps+1])

        for i in range(self.time_steps):
            sigma_tk[:, i+1] = sigma_tk[:, i] * np.exp(self.vov * np.sqrt(delta_tk) * z[:, i] - 0.5 * (self.vov ** 2) * delta_tk)

        I = spint.simps(sigma_tk * sigma_tk, dx=texp/self.time_steps) / (self.sigma**2)  # integrate by using Simpson's rule

        spot_cond = spot + self.rho * (sigma_tk[:, -1] - self.sigma) / self.vov
        vol = self.sigma * np.sqrt((1 - self.rho**2) * I)

        price = np.zeros_like(strike)

        for j in range(len(strike)):
            price[j] = np.mean(normal.price(strike[j], spot_cond, texp ,vol))

        return  disc_fac * price

max_stars_repo_name: polarbluebear/ASP

max_issues_repo_name: polarbluebear/ASP

max_forks_repo_name: polarbluebear/ASP

reason: import numpy,import scipy


text:
from typing import List
from typing_extensions import runtime
from manim import *
import numpy as np

np.set_printoptions(threshold=np.inf)

class Transformasiabrrot(ThreeDScene, MovingCamera):
    def __init__(self):
        super().__init__()

    def construct(self):
        self.set_camera_orientation(phi=60*DEGREES, theta=-45*DEGREES, focal_distance=15)

        self.construct_axis()
        self.setup_polyhedra()
        self.write_mat()
        self.do_translate(2, 2, 3)
        self.wait()
        self.do_translate(-2, -3, 3)
        self.do_scale(2)
        self.wait()
        self.do_scale(1/2)
        self.wait()
        self.do_rotation(90)
        self.wait(2)
        self.do_rotation(-90)
        self.wait(1)
        # self.do_abrrot([2, 1, 0], [3, 3, 1], 5)

        self.move_camera(phi=60*DEGREES, theta=-45*DEGREES, focal_distance=15, frame_center=self.main_obj.get_center(), run_time=2)

    def construct_axis(self):
        axis = ThreeDAxes()
        axis.set_z_index(3)
        labz = axis.get_z_axis_label(Tex("$z$"))
        laby = axis.get_y_axis_label(Tex("$y$"))
        labx = axis.get_x_axis_label(Tex("$x$"))

        self.play(FadeIn(axis), Write(labz), Write(laby), Write(labx))

    def setup_polyhedra(self):
        # som som major
        poly_points = [
            [ 3, 0, 0], # V0 kanan
            [ 0, 2, 0], # V1 atas
            [-3, 0, 0], # V2 kiri
            [ 0,-2, 0], # V3 bawah
            [ 0, 0, 1], # V4 keluar
            [ 0, 0,-1]  # V5 crot
        ]
        faces_list = [
            [0, 1, 4],
            [0, 1, 5],
            [1, 2, 4],
            [1, 2, 5],
            [2, 3, 4],
            [2, 3, 5],
            [3, 0, 4],
            [3, 0, 5],
        ]
        self.main_obj = main_obj = Polyhedron(vertex_coords=poly_points, faces_list=faces_list)
        self.play(DrawBorderThenFill(main_obj), run_time=2)
        self.move_camera(phi=60*DEGREES, theta=-45*DEGREES, focal_distance=15, frame_center=main_obj.get_center(), run_time=2)
        self.wait(0.5)

    def write_mat(self):

        def get_poly_coords():
            coords_faces = self.main_obj.extract_face_coords()
            # [print(n, n.shape) for n in coords_faces[0] + coords_faces[5]]
            coords = np.stack(coords_faces[0] + coords_faces[5])
            coords = coords.round(2)
            # print(coords)
            coords = coords.transpose()
            coords = np.append(coords, [[1, 1, 1, 1, 1, 1]], axis=0)
            # print(coords)
            return coords

        def matrix_updater(mob:Matrix):
            # https://www.reddit.com/r/manim/comments/oid6hv/comment/h4vxk5y/?utm_source=share&utm_medium=web2x&context=3
            newMat = Matrix(get_poly_coords(), h_buff=2)
            newMat.scale(0.5)
            newMat.to_corner(UP + LEFT)
            newMat.shift(OUT*20)
            mob.become(newMat, copy_submobjects=False)

        pmat = Matrix(get_poly_coords(), h_buff=2)
        self.add_fixed_in_frame_mobjects(pmat)
        pmat.scale(0.5)
        pmat.to_corner(UP + LEFT)

        self.play(Write(pmat))
        # self.add(pmat)
        self.wait()
        pmat.add_updater(matrix_updater)

    def do_abrrot(self, startp, endp, dur):

        def main_obj_rot_updater(main_obj_r, dt):
            main_obj_r.rotate(angle=dt, axis=AB, about_point=point_rot)

        # https://gamedev.stackexchange.com/questions/72528/how-can-i-project-a-3d-point-onto-a-3d-line
        self.rot_axis = rot_axis = Line3D(startp, endp)
        AB = rot_axis.end - rot_axis.start
        AP = self.main_obj.get_center() - rot_axis.start
        point_rot = rot_axis.start + np.dot(AP, AB) / np.dot(AB, AB) * AB

        self.play(Write(rot_axis))
        # self.add(rot_axis)

        self.move_camera(phi=60*DEGREES, theta=-45*DEGREES, focal_distance=20, frame_center=rot_axis.get_center())
        self.begin_ambient_camera_rotation(45*DEGREES/3, about='theta')
        self.main_obj.add_updater(main_obj_rot_updater)
        self.wait(dur)
        self.main_obj.remove_updater(main_obj_rot_updater)
        self.stop_ambient_camera_rotation(about="theta")

    def do_translate(self, x, y, z):
        center = self.main_obj.get_center()
        tlPoint = [center[0] + x, center[1]+y, center[2]+z]
        tlLine = Arrow3D(center, tlPoint)
        self.play(Write(tlLine, runtime=5))
        self.wait(5)
        self.remove(tlLine)
        self.play(self.main_obj.animate.shift(x*RIGHT + y*UP + z*OUT), run_time=2)
        self.move_camera(phi=60*DEGREES, theta=-45*DEGREES, focal_distance=15, frame_center=self.main_obj.get_center(), run_time=2)

    def do_scale(self, scale_fact):
        center = self.main_obj.get_center()
        scPointX = [center[0] * scale_fact, center[1], center[2]]
        scLineX = Arrow3D(center, scPointX)
        scPointY = [center[0], center[1]*scale_fact, center[2]]
        scLineY = Arrow3D(center, scPointY)
        scPointZ = [center[0], center[1] * scale_fact, center[2]]
        scLineZ = Arrow3D(center, scPointZ)
        self.play(Write(scLineX, scLineY, scLineZ, runtime=5))
        phantom_axes = ThreeDAxes()
        phantom_axes.set_opacity(0.0)
        scl_group = VGroup(phantom_axes, self.main_obj)
        self.play(scl_group.animate.scale(scale_fact), run_time=2)
        self.move_camera(phi=60*DEGREES, theta=-45*DEGREES, focal_distance=15, frame_center=self.main_obj.get_center(), run_time=2)

    def do_rotation(self, deg):
        self.play(self.main_obj.animate.rotate(angle=deg*DEGREES, axis=UP, about_point=ORIGIN), run_time=2)

max_stars_repo_name: ZZIDZZ/gkv-transformasi-matrix

max_issues_repo_name: ZZIDZZ/gkv-transformasi-matrix

max_forks_repo_name: ZZIDZZ/gkv-transformasi-matrix


text:
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Text, Tuple, Union

import numpy as np
from pandas import DataFrame

from pymove.utils.constants import (
    DATETIME,
    DIST_PREV_TO_NEXT,
    DIST_TO_NEXT,
    DIST_TO_PREV,
    LATITUDE,
    LONGITUDE,
    SPEED_TO_PREV,
    TID,
    TIME_TO_PREV,
    TRAJ_ID,
)

if TYPE_CHECKING:
    from pymove.core.dask import DaskMoveDataFrame
    from pymove.core.pandas import PandasMoveDataFrame

def get_bbox_by_radius(
    coordinates: Tuple[float, float], radius: Optional[float] = 1000
) -> List:
    """
    Defines minimum and maximum coordinates,
    given a distance radius from a point.

    Parameters
    ----------
    coords : tuple (lat, lon)
        The coordinates of point

    radius: float, optional (1000 by default)

    Returns
    -------
    array
        coordinates min and max of the bbox

    References
    ----------
        https://mathmesquita.me/2017/01/16/filtrando-localizacao-em-um-raio.html
    """
    earth_radius = 6371000
    r = radius / earth_radius

    lat, lon = np.radians(coordinates)

    latmin = lat - r
    latmax = lat + r

    delta_lon = np.arcsin(np.sin(r) / np.cos(lat))

    lonmin = lon - delta_lon
    lonmax = lon + delta_lon

    return np.rad2deg([latmin, lonmin, latmax, lonmax])

def by_bbox(
    move_data: DataFrame,
    bbox: Tuple[int, int, int, int],
    filter_out: Optional[bool] = False,
    inplace: Optional[bool] = False
) -> Optional[DataFrame]:
    """
    Filters points of the trajectories according to specified bounding box.

    Parameters
    ----------
    move_data : dataframe
       The input trajectories data
    bbox : tuple
        Tuple of 4 elements, containing the minimum and maximum values
        of latitude and longitude of the bounding box.
    filter_out : boolean, optional
        If set to false the function will return the trajectories points
        within the bounding box, and the points outside otherwise, by default False
    inplace : boolean, optional
        if set to true the original dataframe will be altered to contain
        the result of the filtering, otherwise a copy will be returned, by default False

    Returns
    -------
    DataFrame
        Returns dataframe with trajectories points filtered by bounding box or None

    """

    filter_ = (
        (move_data[LATITUDE] >= bbox[0])
        & (move_data[LONGITUDE] >= bbox[1])
        & (move_data[LATITUDE] <= bbox[2])
        & (move_data[LONGITUDE] <= bbox[3])
    )
    if filter_out:
        filter_ = ~filter_

    return move_data.drop(index=move_data[~filter_].index, inplace=inplace)

def by_datetime(
    move_data: DataFrame,
    start_datetime: Optional[Text] = None,
    end_datetime: Optional[Text] = None,
    filter_out: Optional[bool] = False,
    inplace: Optional[bool] = False,
) -> Optional[DataFrame]:
    """
    Filters trajectories points according to specified time range.

    Parameters
    ----------
    move_data : dataframe
        The input trajectory data
    start_datetime : str
        The start date and time (Datetime format) of the time range, by default None
    end_datetime : str
        The end date and time (Datetime format) of the time range, by default None
    filter_out : bool, optional
        If set to true, the function will return the points of
        the trajectories with timestamp outside the time range.
        The points whithin the time range will be return if filter_out is False.
        by default False
    inplace : bool, optional
        if set to true the original dataframe will be altered to contain
        the result of the filtering, otherwise a copy will be returned, by default False

    Returns
    -------
    DataFrame
        Returns dataframe with trajectories points filtered by time range or None

    """

    if start_datetime is not None and end_datetime is not None:
        filter_ = (
            (move_data[DATETIME] >= start_datetime)
            & (move_data[DATETIME] <= end_datetime)
        )
    elif end_datetime is not None:
        filter_ = move_data[DATETIME] <= end_datetime
    else:
        filter_ = move_data[DATETIME] >= start_datetime

    if filter_out:
        filter_ = ~filter_

    return move_data.drop(index=move_data[~filter_].index, inplace=inplace)

def by_label(
    move_data: DataFrame,
    value: Any,
    label_name: Text,
    filter_out: Optional[bool] = False,
    inplace: Optional[bool] = False
) -> Optional[DataFrame]:
    """
    Filters trajectories points according to specified value and column label.

    Parameters
    ----------
    move_data : dataframe
        The input trajectory data
    value : The value to be use to filter the trajectories
        Specifies the value used to filter the trajectories points
    label_name : str
        Specifies the label of the column used in the filtering
    filter_out : bool, optional
        If set to true, the function will return the points of
        the trajectories with timestamp outside the time range.
        The points whithin the time range will be return if filter_out is False.
        by default False
    inplace : bool, optional
        if set to true the original dataframe will be altered to contain
        the result of the filtering, otherwise a copy will be returned, by default False

    Returns
    -------
    DataFrame
        Returns dataframe with trajectories points filtered by label or None

    """

    filter_ = move_data[label_name] == value
    if filter_out:
        filter_ = ~filter_

    return move_data.drop(index=move_data[~filter_].index, inplace=inplace)

def by_id(
    move_data: DataFrame,
    id_: Optional[int] = None,
    label_id: Optional[Text] = TRAJ_ID,
    filter_out: Optional[bool] = False,
    inplace: Optional[bool] = False
) -> Optional[DataFrame]:
    """
    Filters trajectories points according to specified trajectory id.

    Parameters
    ----------
    move_data : dataframe
        The input trajectory data
    id_ : int
        Specifies the number of the id used to filter the trajectories points
    label_id : str, optional
        The label of the column which contains the id of the trajectories,
        by default TRAJ_ID
    filter_out : bool, optional
        If set to true, the function will return the points of
        the trajectories with timestamp outside the time range.
        The points whithin the time range will be return if filter_out is False.
        by default False
    inplace : bool, optional
        if set to true the original dataframe will be altered to contain
        the result of the filtering, otherwise a copy will be returned, by default False

    Returns
    -------
    DataFrame
        Returns dataframe with trajectories points filtered by id or None

    """

    return by_label(move_data, id_, label_id, filter_out, inplace)

def by_tid(
    move_data: DataFrame,
    tid_: Optional[Text] = None,
    filter_out: Optional[bool] = False,
    inplace: Optional[bool] = False
) -> Optional[DataFrame]:
    """
    Filters trajectories points according to a specified  trajectory tid.

    Parameters
    ----------
    move_data : dataframe
        The input trajectory data
    tid_ : str
        Specifies the number of the tid used to filter the trajectories points
    label_tid : str, optional
        The label of the column in the user dataframe which contains
        the tid of the trajectories, by default None
    filter_out : bool, optional
        If set to true, the function will return the points of
        the trajectories with timestamp outside the time range.
        The points whithin the time range will be return if filter_out is False.
        by default False
    inplace : bool, optional
        if set to true the original dataframe will be altered to contain
        the result of the filtering, otherwise a copy will be returned, by default False

    Returns
    -------
    DataFrame
        Returns a dataframe with trajectories points filtered or None

    """

    return by_label(move_data, tid_, TID, filter_out, inplace)

def outliers(
    move_data: Union['PandasMoveDataFrame', 'DaskMoveDataFrame'],
    jump_coefficient: Optional[float] = 3.0,
    threshold: Optional[float] = 1,
    filter_out: Optional[bool] = False,
    inplace: Optional[bool] = False,
) -> Optional[Union['PandasMoveDataFrame', 'DaskMoveDataFrame']]:
    """
    Filters trajectories points that are outliers.

    Parameters
    ----------
    move_data : dataframe
        The input trajectory data
    jump_coefficient : float, optional
        by default 3
    threshold : float, optional
        Minimum value that the distance features must have
        in order to be considered outliers, by default 1
    filter_out : bool, optional
        If set to true, the function will return the points of
        the trajectories with timestamp outside the time range.
        The points whithin the time range will be return if filter_out is False.
        by default False
    inplace : bool, optional
        if set to true the original dataframe will be altered to contain
        the result of the filtering, otherwise a copy will be returned, by default False

    Returns
    -------
    DataFrame
        Returns a dataframe with the trajectories outliers or None

    """

    if DIST_TO_PREV not in move_data:
        move_data.generate_dist_features()

    if move_data.index.name is not None:
        print('...Reset index for filtering\n')
        move_data.reset_index(inplace=True)

    if (
        DIST_TO_PREV in move_data
        and DIST_TO_NEXT
        and DIST_PREV_TO_NEXT in move_data
    ):
        jump = jump_coefficient * move_data[DIST_PREV_TO_NEXT]
        filter_ = (
            (move_data[DIST_TO_NEXT] > threshold)
            & (move_data[DIST_TO_PREV] > threshold)
            & (move_data[DIST_PREV_TO_NEXT] > threshold)
            & (jump < move_data[DIST_TO_NEXT])
            & (jump < move_data[DIST_TO_PREV])
        )

        if filter_out:
            filter_ = ~filter_

        print('...Filtering jumps \n')
        return move_data.drop(index=move_data[~filter_].index, inplace=inplace)

    else:
        print('...Distances features were not created')
        return move_data

def clean_consecutive_duplicates(
    move_data: DataFrame,
    subset: Optional[Union[int, Text]] = None,
    keep: Optional[Union[Text, bool]] = 'first',
    inplace: Optional[bool] = False
) -> Optional[DataFrame]:
    """
    Removes consecutive duplicate rows of the Dataframe, optionally only
    certain columns can be consider.

    Parameters
    ----------
    move_data : dataframe
        The input trajectory data
    subset : Array of strs, optional
        Specifies  Column label or sequence of labels, considered for
        identifying duplicates, by default None
    keep : 'first', 'last', optional
        If keep is set as first, all the duplicates except for
        the first occurrence will be dropped.
        On the other hand if set to last, all duplicates except for
        the last occurrence will be dropped.
        If set to False, all duplicates are dropped.
        by default 'first'
    inplace : boolean, optional
        if set to true the original dataframe will be altered,
        the duplicates will be dropped in place,
        otherwise a copy will be returned, by default False

    Returns
    -------
    DataFrame
        The filtered trajectories points without consecutive duplicates or None

    """

    if keep == 'first':
        n = 1
    else:
        n = -1
    if subset is None:
        filter_ = (move_data.shift(n) != move_data).any(axis=1)
    else:
        filter_ = (move_data[subset].shift(n) != move_data[subset]).any(axis=1)

    return move_data.drop(index=move_data[~filter_].index, inplace=inplace)

def _filter_single_by_max(move_data: DataFrame, **kwargs):
    """
    Filters from a dataframe rows with features below value.

    Parameters
    ----------
    move_data : dataframe
        Dataframe to be filtered.
    **kwargs : arguments
        - arg1 : feature
        - arg2 : value

    Returns
    -------
    DataFrame
        Filtered dataframe.

    """

    return move_data[move_data[kwargs['arg1']] <= kwargs['arg2']]

def _filter_speed_max_radius(move_data: DataFrame, **kwargs):
    """
    Filters from a dataframe rows with current or previous row features
    exceeding value.

    Parameters
    ----------
    move_data : dataframe
        Dataframe to be filtered.
    **kwargs : arguments
        - arg1 : feature
        - arg2 : value

    Returns
    -------
    DataFrame
        Filtered dataframe.

    """
    filter_ = (
        (np.nan_to_num(move_data[kwargs['arg1']].shift(1)) > kwargs['arg2'])
        | (np.nan_to_num(move_data[kwargs['arg1']]) > kwargs['arg2'])
    )
    return move_data[filter_]

def _filter_data(move_data: DataFrame, f: callable, kwargs: Dict):
    """
    Filter the dataframe using condition from given function

    Parameters
    ----------
    move_data : dataframe
        Dataframe to be filtered.
    f : function
        Filtering function
    **kwargs : arguments
        - arg1 : feature
        - arg2 : value
        - outliers : special behavior if cleaning by outliers

    Returns
    -------
    dataframe
        Filtered dataframe.
    int
        Number of rows to be dropped

    """

    if kwargs['outliers']:
        filter_data_points = f(
            move_data,
            jump_coefficient=kwargs['arg1'],
            threshold=kwargs['arg2'],
            inplace=False
        )
    else:
        filter_data_points = f(
            move_data,
            arg1=kwargs['arg1'],
            arg2=kwargs['arg2'],
            inplace=False
        )
    rows_to_drop = filter_data_points.shape[0]
    return filter_data_points, rows_to_drop

def _clean_gps(move_data: DataFrame, f: callable, **kwargs):
    """
    Cleans gps points from a dataframe using condition from given function

    Parameters
    ----------
    move_data : dataframe
        Dataframe to be filtered.
    f : function
        Filtering function
    **kwargs : arguments
        - arg1 : feature
        - arg2 : value
        - outliers : special behavior if cleaning by outliers

    Returns
    -------
    dataframe
        Filtered dataframe.

    """

    if move_data.index.name is not None:
        print('...Reset index for filtering\n')
        move_data.reset_index(inplace=True)

    filter_data_points, rows_to_drop = _filter_data(move_data, f, kwargs)

    sum_drop = 0
    while rows_to_drop > 0:
        print('...Dropping %s rows of gps points\n' % rows_to_drop)
        shape_before = move_data.shape[0]
        move_data.drop(index=filter_data_points.index, inplace=True)
        sum_drop = sum_drop + rows_to_drop
        print(
            '...Rows before: %s, Rows after:%s, Sum drop:%s\n'
            % (shape_before, move_data.shape[0], sum_drop)
        )

        filter_data_points, rows_to_drop = _filter_data(move_data, f, kwargs)

    print('%s GPS points were dropped' % sum_drop)

    return move_data

def clean_gps_jumps_by_distance(
    move_data: Union['PandasMoveDataFrame', 'DaskMoveDataFrame'],
    label_id: Optional[Text] = TRAJ_ID,
    jump_coefficient: Optional[float] = 3.0,
    threshold: Optional[float] = 1,
    label_dtype: Optional[callable] = np.float64,
    inplace: Optional[bool] = False,
) -> Optional[Union['PandasMoveDataFrame', 'DaskMoveDataFrame']]:
    """
    Removes the trajectories points that are outliers from the dataframe.

    Parameters
    ----------
    move_data : dataframe
        The input trajectory data
    label_id : str, optional
         Indicates the label of the id column in the user dataframe, by default TRAJ_ID
    jump_coefficient : float, optional
        by default 3
    threshold : float, optional
        Minimum value that the distance features must have
        in order to be considered outliers, by default 1
    label_dtype : type, optional
        Represents column id type, by default np.float64.
    inplace : boolean, optional
        if set to true the operation is done in place, the original
        dataframe will be altered and None is returned, by default False

    Returns
    -------
    DataFrame
        The filtered trajectories without the gps jumps or None

    """

    if not inplace:
        move_df = move_data[:]
    else:
        move_df = move_data
    # sum_drop = 0

    if DIST_TO_PREV not in move_df:
        move_df.generate_dist_features(
            label_id=label_id, label_dtype=label_dtype
        )

    print(
        '\nCleaning gps jumps by distance to jump_coefficient %s...\n'
        % jump_coefficient
    )

    move_df = _clean_gps(
        move_df,
        outliers,
        arg1=jump_coefficient,
        arg2=threshold,
        outliers=True
    )
    if not inplace:
        return move_df

def clean_gps_nearby_points_by_distances(
    move_data: Union['PandasMoveDataFrame', 'DaskMoveDataFrame'],
    label_id: Optional[Text] = TRAJ_ID,
    radius_area: Optional[float] = 10.0,
    label_dtype: Optional[callable] = np.float64,
    inplace: Optional[bool] = False,
) -> Optional[Union['PandasMoveDataFrame', 'DaskMoveDataFrame']]:
    """
    Removes points from the trajectories when the distance between them and the
    point before is smaller than the value set by the user.

    Parameters
    ----------
    move_data : dataframe
        The input trajectory data
    label_id : str, optional
         Indicates the label of the id column in the user dataframe, by default TRAJ_ID
    radius_area : float, optional
        Species the minimum distance a point must have to it"srs previous point
        in order not to be dropped, by default 10
    label_dtype : type, optional
        Represents column id type, ,y default np.float64.
    inplace : boolean, optional
        if set to true the operation is done in place, the original
        dataframe will be altered and None is returned, be default False

    Returns
    -------
    DataFrame
        The filtered trajectories without the gps nearby points by distance or None

    """
    if not inplace:
        move_df = move_data[:]
    else:
        move_df = move_data
    # sum_drop = 0

    if DIST_TO_PREV not in move_df:
        move_df.generate_dist_features(
            label_id=label_id, label_dtype=label_dtype
        )

    print(
        '\nCleaning gps points from radius of %s meters\n'
        % radius_area
    )

    move_df = _clean_gps(
        move_df,
        _filter_single_by_max,
        arg1=DIST_TO_PREV,
        arg2=radius_area,
        outliers=False
    )
    if not inplace:
        return move_df

def clean_gps_nearby_points_by_speed(
    move_data: Union['PandasMoveDataFrame', 'DaskMoveDataFrame'],
    label_id: Optional[Text] = TRAJ_ID,
    speed_radius: Optional[float] = 0.0,
    label_dtype: Optional[Callable] = np.float64,
    inplace: Optional[bool] = False,
) -> Optional[Union['PandasMoveDataFrame', 'DaskMoveDataFrame']]:
    """
    Removes points from the trajectories when the speed of travel between them
    and the point before is smaller than the value set by the user.

    Parameters
    ----------
    move_data : dataframe
        The input trajectory data
    label_id : str, optional
         Indicates the label of the id column in the user dataframe, be defalt TRAJ_ID
    speed_radius : float, optional
        Species the minimum speed a point must have from it"srs previous point,
        in order not to be dropped, by default 0
    label_dtype : type, optional
        Represents column id type, by default np.float64.
    inplace : boolean, optional
        if set to true the operation is done in place, the original
        dataframe will be altered and None is returned, by default False

    Returns
    -------
    DataFrame
        The filtered trajectories without the gps nearby points by speed or None

    """

    if not inplace:
        move_df = move_data[:]
    else:
        move_df = move_data

    if SPEED_TO_PREV not in move_df:
        move_df.generate_dist_time_speed_features(
            label_id=label_id, label_dtype=label_dtype
        )

    print(
        '\nCleaning gps points using %s speed radius\n'
        % speed_radius
    )

    move_df = _clean_gps(
        move_df,
        _filter_single_by_max,
        arg1=SPEED_TO_PREV,
        arg2=speed_radius,
        outliers=False
    )
    if not inplace:
        return move_df

def clean_gps_speed_max_radius(
    move_data: Union['PandasMoveDataFrame', 'DaskMoveDataFrame'],
    label_id: Optional[Text] = TRAJ_ID,
    speed_max: Optional[float] = 50.0,
    label_dtype: Optional[Callable] = np.float64,
    inplace: Optional[bool] = False,
) -> Optional[Union['PandasMoveDataFrame', 'DaskMoveDataFrame']]:
    """
    Recursively removes trajectories points with speed higher than the value
    specified by the user. Given any point p of the trajectory, the point will
    be removed if one of the following happens: if the travel speed from the
    point before p to p is greater than the  max value of speed between adjacent
    points set by the user. Or the travel speed between point p and the next
    point is greater than the value set by the user. When the cleaning is done,
    the function will update the time and distance features in the dataframe and
    will call itself again. The function will finish processing when it can no
    longer find points disrespecting the limit of speed.

    Parameters
    ----------
    move_data : dataframe
        The input trajectory data
    label_id : str, optional
        Indicates the label of the id column in the user dataframe, by default TRAJ_ID
    speed_max : float, optional
        Indicates the maximum value a point speed_to_prev and speed_to_next
        should have, in order not to be dropped, by default 50
    label_dtype : type, optional
        Represents column id type, by default np.float64.
    inplace : boolean, optional
        if set to true the operation is done in place, the original
        dataframe will be altered and None is returned, by default False

    Returns
    -------
    DataFrame
        The filtered trajectories without the gps nearby points or None

    """

    if not inplace:
        move_df = move_data[:]
    else:
        move_df = move_data

    if SPEED_TO_PREV not in move_df:
        move_df.generate_dist_time_speed_features(
            label_id=label_id, label_dtype=label_dtype
        )

    print(
        '\nClean gps points with speed max > %s meters by seconds'
        % speed_max
    )

    move_df = _clean_gps(
        move_df,
        _filter_speed_max_radius,
        arg1=SPEED_TO_PREV,
        arg2=speed_max,
        outliers=False
    )
    if not inplace:
        return move_df

def clean_trajectories_with_few_points(
    move_data: Union['PandasMoveDataFrame', 'DaskMoveDataFrame'],
    label_tid: Optional[Text] = TID,
    min_points_per_trajectory: Optional[int] = 2,
    inplace: Optional[bool] = False
) -> Optional[Union['PandasMoveDataFrame', 'DaskMoveDataFrame']]:
    """
    Removes from the given dataframe, trajectories with fewer points than was
    specified by the user.

    Parameters
    ----------
    move_data : dataframe
        The input trajectory data
    label_tid : str, optional
        The label of the column which contains the tid of the trajectories, by default TID
    min_points_per_trajectory: int, optional
        Specifies the minimum number of points a trajectory must have
        in order not to be dropped, by default 2
    inplace : boolean, optional
        if set to true the operation is done in place, the original
        dataframe will be altered and None is returned, by default False

    Returns
    -------
    DataFrame
        The filtered trajectories without the minimum number of gps points or None

    Raises
    ------
    KeyError
        If the label feature is not in the dataframe

    """

    if not inplace:
        move_df = move_data[:]
    else:
        move_df = move_data

    if label_tid not in move_df:
        raise KeyError('%s not in dataframe' % label_tid)

    print(
        '\nCleaning gps points from trajectories of fewer than %s points\n'
        % min_points_per_trajectory
    )

    if move_df.index.name is not None:
        print('\n...Reset index for filtering\n')
        move_df.reset_index(inplace=True)

    move_datacount_tid = move_df.groupby(by=label_tid).size()
    filter_ = move_datacount_tid < min_points_per_trajectory
    tids_with_few_points = move_datacount_tid[filter_].index
    shape_before_drop = move_df.shape
    idx = move_df[move_df[label_tid].isin(tids_with_few_points)].index

    if idx.shape[0] > 0:
        print(
            '\n...There are %s ids with few points'
            % tids_with_few_points.shape[0]
        )
        print(
            '\n...Tids before drop: %s'
            % move_df[label_tid].unique().shape[0]
        )
        move_df.drop(index=idx, inplace=True)
        print(
            '\n...Tids after drop: %s'
            % move_df[label_tid].unique().shape[0]
        )
        print(
            '\n...Shape - before drop: %s - after drop: %s'
            % (shape_before_drop, move_df.shape)
        )

    if not inplace:
        return move_df

def clean_trajectories_short_and_few_points(
    move_data: Union['PandasMoveDataFrame', 'DaskMoveDataFrame'],
    label_id: Optional[Text] = TID,
    min_trajectory_distance: Optional[float] = 100,
    min_points_per_trajectory: Optional[int] = 2,
    label_dtype: Optional[Callable] = np.float64,
    inplace: Optional[bool] = False,
) -> Optional[Union['PandasMoveDataFrame', 'DaskMoveDataFrame']]:
    """
    Eliminates from the given dataframe trajectories with fewer points and
    shorter length than specified values by the user.

    Parameters
    ----------
    move_data : dataframe
        The input trajectory data
    label_id : str, optional
        The label of the column which contains the tid of the trajectories, by default TID
    min_trajectory_distance: float, optional
        Specifies the minimun length a trajectory must have
        in order not to be dropped, by default 100
    min_points_per_trajectory: int, optional
        Specifies the minimun number of points a trajectory must have
        in order not to be dropped, by default 2
    label_dtype : type, optional
        Represents column id type, by default np.float64.
    inplace: boolean, optional
        if set to true the operation is done in place, the original
        dataframe will be altered and None is returned, by default False

    Returns
    -------
    DataFrame
        The filtered trajectories with the minimum gps points and distance or None

    Notes
    -----
        remove_tids_with_few_points must be performed before updating features.

    """

    if not inplace:
        move_df = move_data[:]
    else:
        move_df = move_data

    print('\nRemove short trajectories...')
    clean_trajectories_with_few_points(
        move_df, label_id, min_points_per_trajectory, inplace=True
    )

    if DIST_TO_PREV not in move_df:
        move_df.generate_dist_features(
            label_id=label_id, label_dtype=label_dtype
        )

    print('\n...Dropping unnecessary trajectories...')

    if move_df.index.name is not None:
        print('reseting index')
        move_df.reset_index(inplace=True)

    move_dataagg_tid = move_df.groupby(by=label_id).agg(
        {DIST_TO_PREV: 'sum'}
    )
    filter_ = move_dataagg_tid[DIST_TO_PREV] < min_trajectory_distance
    tid_selection = move_dataagg_tid[filter_].index

    print(
        '\n...short trajectories and trajectories with a minimum distance (%s): %s'
        % (move_dataagg_tid.shape[0], min_trajectory_distance)
    )
    print('\n...There are %s tid do drop' % tid_selection.shape[0])
    shape_before_drop = move_df.shape

    idx = move_df[move_df[label_id].isin(tid_selection)].index
    if idx.shape[0] > 0:
        tids_before_drop = move_df[label_id].unique().shape[0]
        print(
            '\n...Tids - before drop: %s - after drop: %s'
            % (tids_before_drop, move_df[label_id].unique().shape[0])
        )
        move_df.drop(index=idx, inplace=True)
        print(
            '\n...Shape - before drop: %s - after drop: %s'
            % (shape_before_drop, move_df.shape)
        )

    if not inplace:
        return move_df

def clean_id_by_time_max(
    move_data: Union['PandasMoveDataFrame', 'DaskMoveDataFrame'],
    label_id: Optional[Text] = TRAJ_ID,
    time_max: Optional[float] = 3600,
    label_dtype: Optional[Callable] = np.float64,
    inplace: Optional[bool] = False,
) -> Optional[Union['PandasMoveDataFrame', 'DaskMoveDataFrame']]:
    """
    Clears GPS points with time by ID greater than a user-defined limit.

    Parameters
    ----------
    move_data: dataframe.
        The input data.
    label_id: str, optional
        The label of the column which contains the id of the trajectories,
        by default TRAJ_ID
    time_max: float, optional
        Indicates the maximum value time a set of points with the
        same id should have in order not to be dropped, by default 3600
    label_dtype : type, optional
        Represents column id type, by default np.float64.
    inplace : boolean, optional
        if set to true the operation is done in place, the original
        dataframe will be altered and None is returned, by default False

    Returns
    -------
    dataframe or None
        The filtered trajectories with the maximum time.

    """

    if not inplace:
        move_df = move_data[:]
    else:
        move_df = move_data

    if TIME_TO_PREV not in move_df:
        move_df.generate_dist_time_speed_features(
            label_id=label_id, label_dtype=label_dtype
        )

    print(
        '\nClean gps points with time max by id < %s seconds'
        % time_max
    )
    move_dataid_drop = (
        move_df.groupby([label_id], as_index=False)
        .agg({TIME_TO_PREV: 'sum'})
        .query('%s < %s' % (TIME_TO_PREV, time_max))
    )
    print(
        '...Ids total: %s\nIds to drop:%s'
        % (
            move_df[label_id].nunique(),
            move_dataid_drop[label_id].nunique()
        )
    )
    if move_dataid_drop.shape[0] > 0:
        before_drop = move_df.shape[0]
        filter_ = move_df[label_id].isin(move_dataid_drop[label_id])
        idx = move_df[filter_].index
        move_df.drop(idx, inplace=True)
        print(
            '...Rows before drop: %s\n Rows after drop: %s'
            % (before_drop, move_df.shape[0])
        )

    if not inplace:
        return move_df

max_stars_repo_name: JoaoCarabetta/PyMove

max_issues_repo_name: JoaoCarabetta/PyMove

max_forks_repo_name: JoaoCarabetta/PyMove


text:
from os import listdir
import json
import simplejson
import pandas as pd
import matplotlib
from datetime import datetime
import numpy as np

#convert timestamp to datetime
def ts_to_date(ts):
    str_date = datetime.utcfromtimestamp(ts/1000).strftime('%Y-%m-%d %H:%M:%S')
    return datetime.strptime(str_date,'%Y-%m-%d %H:%M:%S')

#granularity can also be "daily"
#files: name of txt files
#return a dictionary mapping link name: link dataframe
def txts_to_dfs(files, granularity="hourly"):
    dfs = {}
    for i in range(len(files)):
        name = files[i]
        f = open(name)
        print f
        if(name=="link_data/.DS_Store"):
            print "ERROR"
        else:
            json_data = json.loads(f.read())
            #print json_data
            df = pd.DataFrame(json_data["points"], dtype="float", columns=json_data["columns"])
            #set index to time
            df.set_index('time', inplace=True)
            #convert milisecond unix time to datetime
            df.index =  df.index.map(ts_to_date)
            #daily granularity: use the median hour of the day (useful for initial plot)
            #!!!DOUBLE CHECK THIS
            if(granularity=="daily"):
                df = df.groupby(pd.Grouper(freq='D')).transform(np.cumsum).resample('D').median()
            #set link_name, e.g. from 'link_data/elpa_hous.txt' extract 'elpa_hous'
            link_name = name[10:len(name)-4]
            dfs[link_name] = df
    return dfs

#hardcoded with known link interfaces to combine
def combine_interfaces(dfs):
    #links that have multiple interfaces
    aofa_lond_summed = dfs.pop('aofa_lond-2-1-1').add(dfs.pop('aofa_lond-7-2-1'), fill_value=0)
    dfs['aofa_lond'] = aofa_lond_summed
    pnwg_denv_summed = dfs.pop('pnwg_denv-4-1-1').add(dfs.pop('pnwg_denv-10-1-4'), fill_value=0)
    pnwg_denv_summed = pnwg_denv_summed.add(dfs.pop('pnwg_denv-10-1-6'))
    dfs['pnwg_denv'] = pnwg_denv_summed
    return dfs


text:
import argparse
import os
import torch
import torchvision
import torchvision.transforms as transforms
from torchvision.transforms.functional import rotate, to_tensor, normalize
import numpy as np
from PIL import Image
from utils import use_gpu, load_ckpt_for_feature_extraction, extract
import pylab

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('ckpt_dir', type=str)
    parser.add_argument('split', type=str,
                        help='path to an .npz of admin and agent data ids')
    parser.add_argument('-r', '--rotation', type=float, nargs='+',
                        help='one or multiple degrees to rotate the probe '
                        'images. Default to no rotation')
    parser.add_argument('save', type=str,
                        help='path to save the features.')
    return parser.parse_args()

def rotate_npy_img(np_img, angle):
    """
        np_img: numpy array: n_images x height x width x channel
    """
    pil_img = Image.fromarray(np_img)

    return rotate(pil_img, angle)

def probing_dataLoader(probe_ids, mean_std, batch_size=128, rot=None):
    """
        Loader for probing samples
        Parameters:

        probe_ids: index of probe data to read
        mean_std: mean and std to normalize the images
        batch_size: batch size
        rot: a list of degrees to rotate the image. Default to no rotation

        Return:
        An instance of DataLoader
    """
    full_set = torchvision.datasets.CIFAR100('images', True)
    probes = full_set.data[probe_ids]

    # rotate each image for specified degrees
    if rot is None:
        rot = [0]
    elif 0 not in rot:
        rot.insert(0, 0)

    batch = []
    for im in probes:
        for degree in rot:
            rot_probe = rotate_npy_img(im, degree)
            rot_probe = to_tensor(rot_probe)
            normalize_probe = normalize(rot_probe, mean_std[0], mean_std[1])
            batch.append(normalize_probe)
            if len(batch) == batch_size:
                yield torch.stack(batch)
                batch = []
    if len(batch) > 0:
        yield torch.stack(batch)

if __name__ == '__main__':
    args = parse_args()
    split_file = np.load(args.split)
    probe_ids = split_file['admin_probe_ids']
    net = load_ckpt_for_feature_extraction(args.ckpt_dir)
    mean_std = np.load(os.path.join(args.ckpt_dir, 'mean_std.npz'))
    mean_std = (mean_std['mean'], mean_std['std'])
    X_train, _ = extract(net,
                         probing_dataLoader(probe_ids, mean_std,
                                            rot=args.rotation)
                        )
    np.save(args.save, X_train)


text:
# --------------------------------------------------------------
# PURPOSE : Compute total dynamic response of a damped system
#
# INPUT   : m =  Mass [Kg]
#           c =  Damping [N/(m/s)]
#           k =  Stiffness [N/m]
#           t =  Time vector
#          p0 =  External load [N]
#
# OUPUT   : u(t) = Total response of the system
#
# CALL    :  total_response(m,c,k,p0)
#--------------------------------------------------------------
# Last modified : Benjamin Bondsman     2021-02-06
#--------------------------------------------------------------
import math
import numpy as np
import matplotlib.pyplot as plt
# ---------------Total response of a damped system ------------
def total_response(m,c,k,p0):
    # NATURAL CIRCULAR FREQUENCY [rad/s]
    w_n = math.sqrt(k/m)
    # CRITICAL DAMPING COEFFICIENT
    Ccr = 2*m*w_n
    # RELATIVE CRITICAL VISCOUS DAMPING
    zeta = c/Ccr
    # DAMPED CIRCULAR FREQUENCY  [rad/s]
    w_d = w_n * math.sqrt(1-zeta**2)
    # NATURAL TIME PERIOD [s]
    T_n = (2*math.pi)
    # Def time vector
    t = np.arange (0,0.5*T_n,0.01)
    #
    # Solution of a system u = u_h + u_p
    # Solve Particular solution u_p
    u_p = p0/k
    # Solve homogeneous solution u_h
    A = u_p
    B = (w_n*zeta*A)/w_d
    # Initialize vectors
    response = []
    time = []
    # Homogeneous solution for all time steps
    for t in np.arange (0,0.5*T_n,0.01):
        u_tot = math.exp(-zeta*w_n*t)*((A*math.cos(w_n*t))+(B*math.sin(w_d*t)))+u_p
        response.append(u_tot)
        time.append(t)
    # Plot the results
    plt.figure()
    plt.plot(time,response,color='teal',marker=11)
    plt.xlabel('Time [s]')
    plt.ylabel('Displacement [mm]')
    plt.grid(True)
    plt.title('Total Response')
    plt.show()
#--------------------------------------------------------------
# ------------------ Call the function ------------------------
total_response(m,c,k,p0)

max_stars_repo_name: benbonnn/dynamicResponse

max_issues_repo_name: benbonnn/dynamicResponse

max_forks_repo_name: benbonnn/dynamicResponse


text:
#!/usr/bin/env python

import datetime
import sys
import numpy
import math
import os
import matplotlib
import matplotlib.pyplot as plt
from matplotlib import cm
from matplotlib.colors import LogNorm

import data

KEYFILE = os.path.expanduser('~/.ssh/livedatakey')
WEBSPACE = 'pkienzle@webster.ncnr.nist.gov:/var/www/html/ipeek'
WORKDIR="/tmp/"

def getLiveData(datafile):
    path = "/net/charlotte/var/ftp/pub/sansdata/"+datafile
    sansfile="pkienzle@sparkle.ncnr.nist.gov:%s"%path
    workfile=WORKDIR+"livedata."+datafile[:3].lower()
    cmd = ("scp -p -i %s %s %s"
               % (KEYFILE, sansfile, workfile))
    #print cmd
    os.system(cmd)
    return workfile

def putLiveData(imgname,htmlname,rawdata=None):
    files = [WORKDIR+f for f in (imgname, htmlname)]
    if rawdata is not None: files.append(rawdata)
    # assuming no spaces in filenames
    cmd = "scp -p -i %s %s %s"% (KEYFILE, " ".join(files), WEBSPACE)
    #print cmd
    os.system(cmd)

def createLiveHTML(metadata,imgname,htmlname):

    collen=16.258
    if (metadata['sample.table'] == '1'):
        offset=0.55
    else:
        offset=0

    dataimage = imgname
    outfile = open(htmlname,"w")

    print >> outfile, """
<?
include("/var/www/include/utility.inc")
?>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">

<title>%s status</title>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
<meta http-equiv="refresh" content="30" />
<link rel="stylesheet" href="http://www.ncnr.nist.gov/programs/sans/scripts/style.css" type="text/css">

NCNR Home

Instruments

Science

Experiments

Sitemap

%s : %s

Date/Time	%s

Count Time	%s s

Elapsed Count Time	%s s

Monitor Counts	%d

Detector Counts	%d

Guides	%d

SDD	%.1f m

Lambda	%.1f A

Source Aperture	%.1f cm

Sample Aperture	%.1f cm

Sample Position	%.1f

Sample Temperature	%.1f C

The data is updated every 5 minutes and this page will refresh every 30s

Last data update: %s

Last page refresh:

<?
include("/var/www/html/programs/sans/SANS_bottom1.inc");
?> """ % (imgname[:3],
          metadata['run.defdir'],
          metadata['sample.labl'],
          dataimage,
          metadata['run.datetime'],
          metadata['run.ctime']*metadata['run.npre'],
          metadata['run.rtime'],
          metadata['run.moncnt'],
          metadata['run.detcnt'],
          round((collen-metadata['resolution.ap12dis']-offset)/1.55),
          metadata['det.dis'],
          metadata['resolution.lmda'],
          metadata['resolution.ap1'],
          metadata['resolution.ap2'],
          metadata['sample.position'],
          metadata['sample.temp'],
          datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))

    outfile.close()

def generateDataImage(data,metadata,imgname):
    fig = plt.figure(figsize=(8,4))
    ax = fig.add_subplot(121)
    ax.imshow(data,origin='lower',cmap=cm.jet)
    ax2 = fig.add_subplot(122)
    vmax = data.max()
    if vmax < 1: vmax=1
    ax2.imshow(data,origin='lower',cmap=cm.jet,norm=LogNorm(vmin=1, vmax=vmax))
    #plt.colorbar(data,ax=ax2,norm=LogNorm(vmin=1))
    fig.savefig(imgname)

def run_update(inst=None):

    imgname = inst+"_livedata.png"
    htmlname = inst+"_livedata.html"
    datafile = inst[:3]+"Current/live001.sa3_ice_a001"

    localfile = getLiveData(datafile)
    detdata,metadata = data.readNCNRData(localfile)
    generateDataImage(detdata,metadata,WORKDIR+imgname)
    createLiveHTML(metadata,imgname,WORKDIR+htmlname)
    putLiveData(imgname,htmlname,rawdata=localfile)

if __name__ == "__main__":
    inst = sys.argv[1] if len(sys.argv) == 2 else None
    run_update(inst)


text:
import gym
import gym_pendrogone
import tensorflow as tf
import numpy as np
from agent import Agent
from ppo import rollouts_generator, add_vtarg_adv, render, Sensei

import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

num_iterations = 1
sample_horizon = 100
# Learning hyperparameters
epochs = 1
batch_size = 10
learning_rate = 3e-4
# GAE params
gamma = 0.99
lam = 0.95
# PPO specific hyperparameter, not gonna change this :v
epsilon = 0.2

def main():
    env = gym.make('PendrogoneZero-v0')
    # env = gym.make('DroneZero-v0')

    continuous = isinstance(env.action_space, gym.spaces.Box)
    # print(continuous)

    ob_dim = env.observation_space.shape[0]
    ac_dim = env.action_space.shape[0] if continuous else env.action_space.n

    # print('ob_dim', ob_dim)
    # print('ac_dim', ac_dim)

    veronika = Agent(continuous, ob_dim, ac_dim, n_layers=2)
    regina = Sensei(veronika, continuous, ob_dim, ac_dim,
                    epochs, batch_size,
                    learning_rate, epsilon)

    init = tf.global_variables_initializer()
    with tf.Session() as sess:
        sess.run(init)
        generator = rollouts_generator(sess, veronika, env, sample_horizon)

        for i in range(num_iterations):
            seg = generator.__next__()
            # print(seg["rew"])
            add_vtarg_adv(seg, lam, gamma)

            adv = seg["adv"]
            adv = (adv - adv.mean()) / (adv.std() + 1e-8)
            adv = adv[:, None]

            regina.train_samples(sess, seg["ob"], seg["ac"], adv, seg["vtarg"], seg["log_probs"])

            rewards = np.array(seg["ep_rets"])

            if i % 10 == 0 or i == num_iterations-1:
                if rewards.shape[0] > 0:
                    mean, std = rewards.mean(), rewards.std()
                    print(
                        'Iteration {0:3d}: reward:  m{1:6.3f}, std{2:4.2f}; ep_len: {3:5.2f}; action: m:{4}, std:{5}'
                        .format( i, mean, std,
                                 np.mean(seg["ep_lens"]),
                                 np.mean(seg["ac"], axis=0), np.std(seg["ac"], axis=0) )
                    )

        render(veronika, env, sess)

if __name__ == '__main__':
    main()


text:
# """
# nosetests -sv --nologcapture tests/quick_test.py
# nosetests --verbosity=2 --detailed-errors --nologcapture --processes=4 --process-restartworker --process-timeout=1000 tests/quick_test.py
# """

import os
import sys

import numpy as np
import pandas as pd
from sklearn.datasets import load_boston
from sklearn.model_selection import train_test_split

from brainless.algorithm.regressor import  Regressor

sys.path = [os.path.abspath(os.path.dirname(__file__))] + sys.path
sys.path = [os.path.abspath(os.path.dirname(os.path.dirname(__file__)))] + sys.path

os.environ['is_test_suite'] = 'True'

# os.environ['KERAS_BACKEND'] = 'theano'

def get_boston_regression_dataset():
    boston = load_boston()
    df_boston = pd.DataFrame(boston.data)
    df_boston.columns = boston.feature_names
    df_boston['MEDV'] = boston['target']
    df_boston_train, df_boston_test = train_test_split(df_boston, test_size=0.33, random_state=42)
    return df_boston_train, df_boston_test

def regression_test():
    # a random seed of 42 has ExtraTreesRegressor getting the best CV score,
    # and that model doesn't generalize as well as GradientBoostingRegressor.
    np.random.seed(0)
    model_name = 'LGBMRegressor'

    df_boston_train, df_boston_test = get_boston_regression_dataset()
    many_dfs = []
    for i in range(100):
        many_dfs.append(df_boston_train)
    df_boston_train = pd.concat(many_dfs)

    column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'}

    ml_predictor = Regressor(type_of_estimator='regressor', column_descriptions=column_descriptions)

    ml_predictor.train(df_boston_train, model_names=[model_name], perform_feature_scaling=False)

    test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV)

    print('test_score')
    print(test_score)

    lower_bound = -3.2
    if model_name == 'DeepLearningRegressor':
        lower_bound = -7.8
    if model_name == 'LGBMRegressor':
        lower_bound = -4.95
    if model_name == 'XGBRegressor':
        lower_bound = -3.4

    assert lower_bound < test_score < -2.7

if __name__ == '__main__':
    regression_test()

max_stars_repo_name: loaiabdalslam/Brainless

max_issues_repo_name: loaiabdalslam/Brainless

max_forks_repo_name: loaiabdalslam/Brainless


text:
import seaborn as sns
import sklearn
import sklearn.manifold as skm
import numpy as np
import matplotlib.pyplot as plt

class Visualize():
    """
    Visualize is a class to visualization methods.
    """

    def __init__(self, problem):
        self.problem = problem
        sns.set_style('white')

    def mds_embed(self, trees):
        """
        Return embedded points of trees using the pq-gram distance.
        """
        N = len(trees)
        distance_mat = np.zeros((N, N))
        for i in range(N):
            for j in range(i+1):
                distance_mat[i][j] = self.problem.pq_gram_distance(trees[i], trees[j])

        for j in range(N):
            for i in range(j+1):
                distance_mat[i][j] = distance_mat[j][i]

        mds = skm.MDS(n_components=2, dissimilarity='precomputed')
        XY = mds.fit_transform(distance_mat)
        return XY, distance_mat

    def mds_plot(self, trees, labels, epoch=0):
        """
        Plot the embedding space using  Multi-Dimensional Scaling.
        """
        Xs, distance_mat = self.mds_embed(trees)
        label_set = sorted(list(set(labels)))
        X_dic = {}
        for l in label_set:
            X_dic[l] = []
        for i in range(len(trees)):
            X_dic[labels[i]].append(Xs[i])
        plt.figure(figsize=(6, 6))
        k = 1
        plt.tick_params(labelbottom=False,
                        labelleft=False,
                        labelright=False,
                        labeltop=False)
        color_list = ['orange', 'dogerblue', 'gold', 'deeppink']
        for X in X_dic.values():
            Xp = np.array(X)
            plt.scatter(Xp[:, 0], Xp[:, 1], linewidth=1, edgecolors='black', label='class_'+str(k))
            k += 1
        leg = plt.legend(frameon=True, bbox_to_anchor=(1, 1), loc='upper right')
        leg.get_frame().set_edgecolor('black')
        plt.title('MDS embedding')
        plt.show()
        plt.figure(figsize=(6, 6))
        plt.imshow(distance_mat)
        plt.colorbar()
        plt.show()

    def mds_gif_imgs(self, trees, labels, epoch=0):
        """
        Generate mds embedding images to create gif.
        """
        Xs, distance_mat = self.mds_embed(trees)
        label_set = sorted(list(set(labels)))
        X_dic = {}
        for l in label_set:
            X_dic[l] = []
        for i in range(len(trees)):
            X_dic[labels[i]].append(Xs[i])
        plt.figure(figsize=(6, 6))
        k = 1
        for X in X_dic.values():
            Xp = np.array(X)
            plt.scatter(Xp[:, 0], Xp[:, 1], linewidth=1, edgecolors='black', label='class_'+str(k))
            k += 1
        plt.savefig('img/embedding/mds_epoch_'+str(epoch)+'.png')
        plt.show()

max_stars_repo_name: hkrsnd/ml-pqgram

max_issues_repo_name: hkrsnd/ml-pqgram

max_forks_repo_name: hkrsnd/ml-pqgram


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Jan 12 01:17:17 2019

@author: Matteo Papini
"""

import torch
import numpy as np
from joblib import Parallel, delayed
from potion.common.misc_utils import seed_all_agent
import random
import math

def sequential_episode_generator(env, policy, horizon=float('inf'), max_episodes=float('inf'),
                                 action_filter=None, render=False, deterministic=False, key=None):

    ds = sum(env.observation_space.shape)
    ds = max(ds, 1)
    da = sum(env.action_space.shape)
    da = max(da, 1)

    n = 0
    while n < max_episodes:
        # Episode
        states = torch.zeros((horizon, ds),
                             dtype=torch.float)
        actions = torch.zeros((horizon, da),
                              dtype=torch.float)
        rewards = torch.zeros(horizon, dtype=torch.float)
        mask = torch.zeros(horizon, dtype=torch.float)
        infos = torch.zeros(horizon, dtype=torch.float)
        s = env.reset()
        done = False
        t = 0
        if render:
            try:
                env.render()
            except:
                pass
        while not done and t < horizon:
            s = np.array(s, dtype=np.float)
            s = torch.tensor(s, dtype=torch.float).view(-1)
            a = policy.act(s, deterministic)
            if not torch.is_tensor(a):
                a = torch.tensor(a)
            a = a.view(-1)
            if action_filter is not None:
                a = action_filter(a)

            next_s, r, done, info = env.step(a.numpy())
            if render:
                try:
                    env.render()
                except:
                    print(s, a, r, done, info)
                    print()
            states[t] = s
            actions[t] = a
            rewards[t] = r
            if key is not None and key in info:
                infos[t] = info[key]
            mask[t] = 1

            s = next_s
            t += 1

        yield states, actions, rewards, mask, infos
        n += 1

def parallel_episode_generator(env, policy, horizon=float('inf'), action_filter=None, seed=None, deterministic=False, key=None):
        ds = sum(env.observation_space.shape)
        ds = max(ds, 1)
        da = sum(env.action_space.shape)
        da = max(da, 1)

        env.seed(seed)
        seed_all_agent(seed)
        states = torch.zeros((horizon, ds),
                             dtype=torch.float)
        actions = torch.zeros((horizon, da),
                              dtype=torch.float)
        rewards = torch.zeros(horizon, dtype=torch.float)
        mask = torch.zeros(horizon, dtype=torch.float)
        infos = torch.zeros(horizon, dtype=torch.float)
        s = env.reset()
        done = False
        t = 0
        while not done and t < horizon:
            s = np.array(s, dtype=np.float)
            s = torch.tensor(s, dtype=torch.float).view(-1)
            a = policy.act(s, deterministic)
            a = torch.tensor(a, dtype=torch.float).view(-1)
            if action_filter is not None:
                a = action_filter(a)
            next_s, r, done, info = env.step(a.numpy())

            states[t] = s
            actions[t] = a
            rewards[t] = r
            mask[t] = 1
            if key is not None and key in info:
                infos[t] = info[key]

            s = next_s
            t += 1
        return states, actions, rewards, mask, infos

def light_episode_generator(env, policy, horizon=float('inf'), disc=1., action_filter=None, key=None):
        s = env.reset()
        done = False
        t = 0
        ret = 0.
        uret = 0.
        info_sum = 0.
        while not done and t < horizon:
            s = np.array(s, dtype=np.float)
            s = torch.tensor(s, dtype=torch.float).view(-1)
            a = policy.act(s)
            a = torch.tensor(a, dtype=torch.float).view(-1)
            if action_filter is not None:
                a = action_filter(a)
            next_s, r, done, info = env.step(a.numpy())
            ret += disc**t * r
            uret += r
            if key is not None and key in info:
                info_sum += info[key]

            s = next_s
            t += 1
        return ret, uret, t, info_sum

def generate_batch(env, policy, horizon, episodes, action_filter=None, render=False, n_jobs=False, seed=None, deterministic=False, key=None):
    """Batch: list of (features, actions, rewards, mask) tuples"""
    if not n_jobs:
        gen = sequential_episode_generator(env, policy, horizon, episodes, action_filter, render, deterministic, key)
        batch = [ep for ep in gen]
    else:
        if seed is None:
            seed = random.randint(0,999999)
        batch = Parallel(n_jobs=n_jobs)(delayed(parallel_episode_generator)(env, policy, horizon, action_filter, seed=seed*10000+i, deterministic=deterministic, key=key) for i in range(episodes))
    return batch

def evaluate_hyperpolicy(env, hyperpolicy, horizon, disc, episodes, resample=True, action_filter=None, render=False, n_jobs=False, seed=None, key=None):
    if not n_jobs:
        gen = sequential_hyperpolicy_evaluator(env, hyperpolicy, horizon, disc, episodes, resample,
                                 action_filter, render, key)
        batch = [ep for ep in gen]
    else:
        if seed is None:
            seed = random.randint(0,999999)
        batch = Parallel(n_jobs=n_jobs)(delayed(parallel_hyperpolicy_evaluator)(env, hyperpolicy, horizon, disc, episodes, resample,
                                 action_filter, render, seed*10000+i, key) for i in range(episodes))
    return batch

"""Testing"""
if __name__ == '__main__':
    from potion.actors.continuous_policies import SimpleGaussianPolicy as Gaussian
    import gym.spaces
    env = gym.make('MountainCarContinuous-v0')
    policy = Gaussian(2, 1)

    batch = generate_batch(env, policy, 3, 4)
    print(batch)


text:
"""
Code: slicing of large raster images in image patches of window size F (= 256). In this code, the ~10% of the area of each image
is kept as validation data. To achieve this we keep the lowest (bottom right) 10% of each tile as validation data. This is done by
using all the indices corresponding to the lowest 10% of area (i.e. after the ~70% of the length of each area).

Area_test = (0.3 * Height) * (0.3 * Width) ~ 0.1 * Height*Width
"""

import rasterio
import numpy as np
import glob
import cv2
import uuid
from pathos.pools import   ThreadPool as pp

# ********************************** CONSTANTS *************************************
# Class definitions
# New fast access, from stackoverflow: https://stackoverflow.com/questions/53059201/how-to-convert-3d-rgb-label-imagein-semantic-segmentation-to-2d-gray-image-an
# ******************************************************************
NClasses = 6 # Looking at the data I am treating "background" as a separate class.
Background = np.array([255,0,0]) #:{'name':'Background','cType':0},
ImSurf = np.array ([255,255,255])# :{'name':'ImSurf','cType':1},
Car = np.array([255,255,0]) # :{'name':'Car','cType':2},
Building = np.array([0,0,255]) #:{'name':'Building','cType':3},
LowVeg = np.array([0,255,255]) # :{'name':'LowVeg','cType':4},
Tree = np.array([0,255,0]) # :{'name':'Tree','cType':5}
# ******************************************************************

# READING DATA
# @@@@@@@@@@@@@@@@@ REPLACE THIS WITH YOUR DATA DIRECTORY
read_prefix= r'/flush1/dia021/isprs_potsdam/raw_data/'
prefix_imgs = r'4_Ortho_RGBIR/'
prefix_dems = r'1_DSM_normalisation/'
prefix_labels = r'5_Labels_for_participants/'

flnames_imgs = sorted(glob.glob(read_prefix+prefix_imgs+'*.tif'))
flnames_dems = sorted(glob.glob(read_prefix+prefix_dems+'*.tif'))
flnames_labels = sorted(glob.glob(read_prefix+prefix_labels+'*.tif'))

IDs = []
for name in flnames_labels:
    IDs +=[name.replace(read_prefix + '5_Labels_for_participants/top_potsdam_','').replace('_label.tif','')]

# Helper functions to create boundary and distance transform
# Expect ground trouth label in 1hot format
# Necessary for fixing an error in the data:
def img_transform(_img):
    new_size = 6000
    _nchannels=_img.shape[0]
    img = np.transpose(_img,[1,2,0])
    img = cv2.resize(img,(new_size,new_size),interpolation= cv2.INTER_NEAREST)
    #img = transform.resize(img,(new_size,new_size,_nchannels),preserve_range=True)
    img = np.transpose(img,[2,0,1])
    #img = img.astype('uint8')

    return img

def get_boundary(labels, _kernel_size = (3,3)):

    label = labels.copy()
    for channel in range(label.shape[0]):
        temp = cv2.Canny(label[channel],0,1)
        label[channel] = cv2.dilate(temp, cv2.getStructuringElement(cv2.MORPH_CROSS,_kernel_size) ,iterations = 1)

    label = label.astype(np.float32)
    label /= 255.
    #label = label.astype(np.uint8)
    return label

def get_distance(labels):
    label = labels.copy()
    #print (label.shape)
    dists = np.empty_like(label,dtype=np.float32)
    for channel in range(label.shape[0]):
        dist = cv2.distanceTransform(label[channel], cv2.DIST_L2, 0)
        dist = cv2.normalize(dist, dist, 0, 1.0, cv2.NORM_MINMAX)
        dists[channel] = dist

    return dists

def ID_2_filenames(_ID):

    if (len(_ID[2:]) == 1 ):
        ID_dsm = '0'+_ID[0]+'_'+'0'+_ID[2]
    else:
        ID_dsm = '0'+_ID

    label_name = r'top_potsdam_' + _ID + '_label.tif'
    dsm_name = r'dsm_potsdam_' + ID_dsm + '_normalized_lastools.jpg'
    img_name = r'top_potsdam_'+_ID + '_RGBIR.tif'

    return label_name, img_name, dsm_name

def read_n_stack(_ID):
    """
    Given and ID string, returns stacked img (RGBIR+DEMS) and label (RGB)
    It fixes a bug in the data, one having dimension one pixel less
    """
    tflname_label, tflname_img, tflname_dems = ID_2_filenames(_ID)

    tflname_label = read_prefix + prefix_labels + tflname_label
    tflname_img = read_prefix + prefix_imgs + tflname_img
    tflname_dems = read_prefix + prefix_dems + tflname_dems

    # read label:
    with rasterio.open(tflname_label) as src:
        label = src.read()
    if label.shape[1:] != (6000,6000):
        label = img_transform(label)

    # read image
    with rasterio.open(tflname_img) as src:
        img = src.read()
    if img.shape[1:] != (6000,6000):
        img = img_transform(img)

    # read DEMs
    with rasterio.open(tflname_dems) as src:
        dems = src.read()
    if dems.shape[1:] != (6000,6000):
        dems = img_transform(dems)

    img = np.concatenate([img,dems],axis=0)

    return img, label

# Fast version to translate class RGB tuples to integer indices
def rgb_to_2D_label(_label):
    label_seg = np.zeros(_label.shape[1:],dtype=np.uint8)
    label_seg [np.all(_label.transpose([1,2,0])==Background,axis=-1)] = 0
    label_seg [np.all(_label.transpose([1,2,0])==ImSurf,axis=-1)] = 1
    label_seg [np.all(_label.transpose([1,2,0])==Car,axis=-1)] = 2
    label_seg [np.all(_label.transpose([1,2,0])==Building,axis=-1)] = 3
    label_seg [np.all(_label.transpose([1,2,0])==LowVeg,axis=-1)] = 4
    label_seg [np.all(_label.transpose([1,2,0])==Tree,axis=-1)] = 5

    return label_seg

# translates image to 1H encoding
def rgb_to_1Hlabel(_label):
    teye = np.eye(NClasses,dtype=np.uint8)

    label_seg = np.zeros([*_label.shape[1:],NClasses],dtype=np.uint8)
    label_seg [np.all(_label.transpose([1,2,0])==Background,axis=-1)] = teye[0]
    label_seg [np.all(_label.transpose([1,2,0])==ImSurf,axis=-1)] = teye[1]
    label_seg [np.all(_label.transpose([1,2,0])==Car,axis=-1)] = teye[2]
    label_seg [np.all(_label.transpose([1,2,0])==Building,axis=-1)] = teye[3]
    label_seg [np.all(_label.transpose([1,2,0])==LowVeg,axis=-1)] = teye[4]
    label_seg [np.all(_label.transpose([1,2,0])==Tree,axis=-1)] = teye[5]

    return label_seg.transpose([2,0,1])

def ID_preprocessing(_ID):

    img, label = read_n_stack(_ID)
    label = rgb_to_1Hlabel(label) # This makes label in 1H

    return img,label

# Read img names and corresponding masks.
Filter = 256 # Window size of patches
stride =  Filter // 2 # This is the stride so as to capture edge effects.
length_scale = 0.317 # when squared gives ~ 0.1, i.e. 10% of all area

###prefix_global_write = r'/flush1/dia021/isprs_potsdam/Data_6k/'
prefix_global_write = r'/scratch1/dia021/isprs_potsdam/Data_6k/'

# ********************** END OF CONSTANTS ******************************************

# This function is used for parallelization
def img_n_tens_slice(_img_ID):

    # Training images location
    write_dir_img_train = prefix_global_write + 'training/imgs/'
    write_dir_label_train = prefix_global_write  + 'training/masks/'

    # Validation images location
    write_dir_img_val = prefix_global_write + 'validation/imgs/'
    write_dir_label_val = prefix_global_write  + 'validation/masks/'

    print ("reading img,masks ID::{}".format(_img_ID))

    _img, _masks = ID_preprocessing(_img_ID)

    nTimesRows = int((_img.shape[1] - Filter)//stride + 1)
    nTimesCols = int((_img.shape[2] - Filter)//stride + 1)

    # @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
    # @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
    # This is for keeping a validation set
    nTimesRows_val = int((1.0-length_scale)*nTimesRows)
    nTimesCols_val = int((1.0-length_scale)*nTimesCols)
    # @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
    # @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@

    for row in range(nTimesRows-1):
        for col in range(nTimesCols-1):

            # Extract temporary
            timg = _img[:, row*stride:row*stride+Filter, col*stride:col*stride+Filter]
            tmask_1hot  = _masks[:,row*stride:row*stride+Filter, col*stride:col*stride+Filter]

            # TODO: create boundary/distance on the fly?
            tbound = get_boundary(tmask_1hot)
            tdist = get_distance(tmask_1hot)
            # Aggregate all masks together in a single entity
            tmask_all = np.concatenate([tmask_1hot,tbound,tdist],axis=0)

            run_ID = str(uuid.uuid4())
            if row >= nTimesRows_val and col >= nTimesCols_val :
                timg_name  = write_dir_img_val + 'img-' + run_ID +'.npy'
                tmask_name = write_dir_label_val + 'img-'+ run_ID +'-mask.npy'
            else:
                timg_name  = write_dir_img_train + 'img-' + run_ID +'.npy'
                tmask_name = write_dir_label_train + 'img-'+ run_ID +'-mask.npy'

            np.save(timg_name, timg)
            np.save(tmask_name, tmask_all)

    # Keep the overlapping non integer final row/column images as validation images as well
    rev_row = _img.shape[1] - Filter
    rev_col = _img.shape[2] - Filter
    for row in range(nTimesRows-1):
        timg        = _img  [:, row*stride:row*stride+Filter, rev_col:]
        tmask_1hot  = _masks[:, row*stride:row*stride+Filter, rev_col:]

        tbound = get_boundary(tmask_1hot)
        tdist = get_distance(tmask_1hot)
        # Aggregate all masks together in a single entity
        tmask_all = np.concatenate([tmask_1hot,tbound,tdist],axis=0)
        run_ID = str(uuid.uuid4())
        timg_name  = write_dir_img_val + 'img-' + run_ID +'.npy'
        tmask_name = write_dir_label_val + 'img-'+ run_ID +'-mask.npy'

        np.save(timg_name, timg)
        np.save(tmask_name, tmask_all)

    for col in range(nTimesCols-1):
        timg        = _img  [:, rev_row:, col*stride:col*stride + Filter]
        tmask_1hot  = _masks[:, rev_row:, col*stride:col*stride + Filter]

        tbound = get_boundary(tmask_1hot)
        tdist = get_distance(tmask_1hot)
        # Aggregate all masks together in a single entity
        tmask_all = np.concatenate([tmask_1hot,tbound,tdist],axis=0)
        run_ID = str(uuid.uuid4())
        timg_name  = write_dir_img_val + 'img-' + run_ID +'.npy'
        tmask_name = write_dir_label_val + 'img-'+ run_ID +'-mask.npy'

        np.save(timg_name, timg)
        np.save(tmask_name, tmask_all)

if __name__ == '__main__':

    # Process each node in parallel
    nnodes  = int(16) # Change with the number of your CPUs
    pool = pp(nodes=nnodes)

    # These are the training images  -- processing in parallel
    pool.map(img_n_tens_slice,IDs)

max_stars_repo_name: meule/resuneta