text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.4'
#       jupytext_version: 1.1.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # S_EnsembleFlexProbs [<img src="https://www.arpm.co/lab/icons/icon_permalink.png" width=30 height=30 style="display: inline;">](https://www.arpm.co/lab/redirect.php?code=S_EnsembleFlexProbs&codeLang=Python)
# For details, see [here](https://www.arpm.co/lab/redirect.php?permalink=ExerEnsembleFP).

# ## Prepare the environment

# +
import os
import os.path as path
import sys

sys.path.append(path.abspath('../../functions-legacy'))
from collections import namedtuple

import numpy as np
from numpy import arange, array, ones, diff, abs, log, exp, sqrt, r_
from numpy import sum as npsum, min as npmin, max as npmax

from scipy.io import loadmat

import matplotlib.pyplot as plt
from matplotlib.pyplot import figure, bar, legend, xlim, ylim, scatter, ylabel, \
    xlabel, title, xticks, yticks
import matplotlib.dates as mdates

plt.style.use('seaborn')
np.seterr(all='ignore')

from CONFIG import GLOBAL_DB, TEMPORARY_DB
from ARPM_utils import save_plot, struct_to_dict, date_mtop
from intersect_matlab import intersect
from HistogramFP import HistogramFP
from RollPrices2YieldToMat import RollPrices2YieldToMat
from EffectiveScenarios import EffectiveScenarios
from ConditionalFP import ConditionalFP
from Stats import Stats
from ColorCodedFP import ColorCodedFP
# -

# ## Upload data

# +
try:
    db = loadmat(os.path.join(GLOBAL_DB, 'db_Stocks'), squeeze_me=True)
except FileNotFoundError:
    db = loadmat(os.path.join(TEMPORARY_DB, 'db_Stocks'), squeeze_me=True)

SPX = struct_to_dict(db['SPX'])
try:
    db = loadmat(os.path.join(GLOBAL_DB, 'db_VIX'), squeeze_me=True)
except FileNotFoundError:
    db = loadmat(os.path.join(TEMPORARY_DB, 'db_VIX'), squeeze_me=True)

VIX = struct_to_dict(db['VIX'])

try:
    db = loadmat(os.path.join(GLOBAL_DB, 'db_SwapCurve'), squeeze_me=True)
except FileNotFoundError:
    db = loadmat(os.path.join(TEMPORARY_DB, 'db_SwapCurve'), squeeze_me=True)

DF_Rolling = struct_to_dict(db['DF_Rolling'])
# -

# ## Recover the invariants and the time series of the conditioning variables

# +
# invariants (S&P500 returns)
epsi = diff(log(SPX.Price_close))

# CONDITIONING VARIABLES
# 1) VIX (VIX.value)

# 2) 5years Swap Zero Rate
ZeroRates,_ = RollPrices2YieldToMat(DF_Rolling.TimeToMat, DF_Rolling.Prices)
zr5 = ZeroRates[DF_Rolling.TimeToMat == 5,:]

# merging datasets
date,_,_ = intersect(intersect(SPX.Date[1:], VIX.Date), DF_Rolling.Dates)
_, i_spx,_ = intersect(SPX.Date[1:], date)
_, i_vix,_ = intersect(VIX.Date, date)
_, i_zr,_ = intersect(DF_Rolling.Dates, date)

epsi = epsi[i_spx].reshape(1,-1)
z1 = VIX.value[i_vix].reshape(1,-1)
z2 = zr5[0,i_zr].reshape(1,-1)
t_ = len(date)
# -

# ## Compute the Flexible Probabilities conditioning on each of the two factors

# +
alpha = 0.3

# prior
lam = log(2) / 1080
prior = exp(-lam*abs(arange(t_, 1 + -1, -1))).reshape(1,-1)
prior = prior / npsum(prior)

# flex. probs conditioned on VIX (z1)

VIXcond = namedtuple('conditioner', ['Series', 'TargetValue', 'Leeway'])
VIXcond.Series = z1
VIXcond.TargetValue = np.atleast_2d(z1[0,-1])
VIXcond.Leeway = alpha
p1 = ConditionalFP(VIXcond, prior)

# flex. probs conditioned on the swap rate (z2)
ZRcond = namedtuple('conditioner', ['Series', 'TargetValue', 'Leeway'])
ZRcond.Series = z2
ZRcond.TargetValue = np.atleast_2d(z2[[0],[-1]])
ZRcond.Leeway = alpha
p2 = ConditionalFP(ZRcond, prior)
# -

# ## Compute the respective Effective Number of Scenarios and the diversity indicator

# +
# effective number of scenarios

typ = namedtuple('type','Entropy')
typ.Entropy = 'Exp'
ens1 = EffectiveScenarios(p1, typ)
ens2 = EffectiveScenarios(p2, typ)

# diversity indicator
rho2_12 = npsum(sqrt(p1*p2))  # overlap: Bhattacharyya coefficient

dd12 = sqrt(1 - rho2_12)  # Hellinger distance

d1 = dd12  # Diversity
d2 = d1
# -

# ## Weights of the Flexible Probabilities Ensemble Posterior

weights = r_[ens1*d1,  ens2*d2]
weights = weights / npsum(weights)

# ## Optimal set of Flex. Probs as log-mixture

opt_p = exp(weights[0]*log(p1) + weights[1]*log(p2))
opt_p = opt_p / npsum(opt_p)
ens_optp = EffectiveScenarios(opt_p, typ)

# ## S&P returns histogram and statistics

option = namedtuple('option', 'n_bins')
option.n_bins = 10*log(epsi.shape[1])
p_eps, x_eps = HistogramFP(epsi, opt_p, option)
m, Sdev, VaR, CVaR, Sk, K = Stats(epsi, opt_p)

# ## Generate the figure

# +
date_tick = arange(99, len(date),380)
date_dt = array([date_mtop(i) for i in date])
myFmt = mdates.DateFormatter('%d-%b-%y')

figure(figsize=(16,10))
# VIX
ax = plt.subplot2grid((2,5),(0,0),colspan=2)
ph0 = ax.plot(date_dt, p1[0],lw=0.5,color='gray')
xticks([])
yticks([])
ax2 = ax.twinx()
ax2.plot(date_dt, z1[0],color= [0, 0, 0.6],lw=0.5)
ph1 =ax2.plot(date_dt, z1[0,-1]*ones(t_),color= 'r', linestyle='--')
xlim([min(date_dt), max(date_dt)])
ax.set_ylim([0, 1.5*npmax(p1)])
ax2.set_ylim([npmin(z1), 1.3*npmax(z1)])
ax2.set_yticks(arange(20,100,20))
ax2.set_ylabel('VIX',color=[0, 0, 0.6])
ax2.grid(False)
LEG = 'target %2.2f'% z1[0,-1]
LEG1 = 'Entr. Pool. Flex. Probs'
legend(handles=[ph1[0],ph0[0]],labels=[LEG, LEG1],loc='upper right')
title('Conditioning variable: VIX')
ENS_text = 'Effective Num.Scenarios =  % 3.0f'% ens1
plt.text(min(date_dt) , npmax(z1)*1.2, ENS_text,horizontalalignment='left')
# 5 YEARS ZERO SWAP RATE
ax = plt.subplot2grid((2,5),(1,0),colspan=2)
ph0=ax.plot(date_dt, p2[0],lw=0.5,color='gray')
yticks([])
xticks([])
ax2 = ax.twinx()
ax2.plot(date_dt, z2[0],color= [0, 0, 0.6],lw=0.5)
ph1=ax2.plot(date_dt, z2[0,-1]*ones(t_),color='r',linestyle='--')
xlim([min(date_dt), max(date_dt)])
ax.set_ylim([0, 1.5*npmax(p2)])
ax2.set_ylim([0.9*npmin(z2), 1.3*npmax(z2)])
ax2.set_ylabel('Swap rate',color=[0, 0, 0.6])
ax2.set_yticks([0.05])
ax2.grid(False)
LEG = 'target %2.3f'% z2[0,-1]
LEG1 = 'Entr. Pool. Flex. Probs'
legend(handles=[ph1[0],ph0[0]],labels=[LEG, LEG1],loc='upper right')
title('Conditioning variable: swap rate.')
ENS_text = 'Effective Num.Scenarios =  % 3.0f'%ens2
plt.text(min(date_dt) , npmax(z2)*1.2, ENS_text, horizontalalignment='left')
# ENSEMBLE FLEXIBLE PROBABILITIES
ax = plt.subplot2grid((2,5),(0,2),colspan=2)
bar(date_dt, opt_p[0], width=(date_dt[1].toordinal()-date_dt[0].toordinal()), facecolor=[0.6, 0.6, 0.6], edgecolor=[0.6, 0.6, 0.6])
xlim([min(date_dt), max(date_dt)]), ylim([0, 1.05*npmax(opt_p)])
yticks([])
xticks(date_dt[date_tick])
ax.xaxis.set_major_formatter(myFmt)
ylabel('probability')
title('ENSEMBLE FLEXIBLE PROBABILITIES')
ENS_text = 'Effective Num.Scenarios = % 3.0f'%ens_optp
plt.text(min(date_dt) , 1.03*npmax(opt_p), ENS_text, horizontalalignment='left')
# S&P returns
ax = plt.subplot2grid((2,5),(1,2),colspan=2)
# scatter colormap and colors
CM, C = ColorCodedFP(opt_p, npmin(opt_p), npmax(opt_p), arange(0,0.85,0.055), 0, 1, [1, 0.1])
ax.set_facecolor('white')
scatter(date_dt, epsi, 20, c=C, marker='.',cmap=CM)
xlim([min(date_dt), max(date_dt)])
ylim([npmin(epsi), npmax(epsi)])
xticks(date_dt[date_tick])
ax.xaxis.set_major_formatter(myFmt)
ylabel('returns')
title('S&P')
# HFP-histogram
ax = plt.subplot2grid((2,5),(1,4))
plt.barh(x_eps[:-1], p_eps[0], height=x_eps[1]-x_eps[0],facecolor=[0.7, 0.7, 0.7], edgecolor=[0.6, 0.6, 0.6])
xlim([0, 1.05*npmax(p_eps)])
ylim([npmin(epsi), npmax(epsi)])
xlabel('probability')
ax.set_xticks([])
# text relative to S&P RETS HIST
TEXT1 = 'Mean  % 3.3f \nSdev    %3.3f \nVaR      %3.3f \nCVaR   %3.3f \nSkew   %3.3f \nKurt     %3.3f' %(m,Sdev,VaR,CVaR,Sk,K)
plt.text(0.45*npmax(p_eps), 0.05, TEXT1, horizontalalignment='left',verticalalignment='bottom')
plt.tight_layout()
# save_plot(ax=plt.gca(), extension='png', scriptname=os.path.basename('.')[:-3], count=plt.get_fignums()[-1])

max_stars_repo_name: dpopadic/arpmRes

max_issues_repo_name: dpopadic/arpmRes

max_forks_repo_name: dpopadic/arpmRes

reason: import numpy,from numpy,from scipy


text:
"""Module containing tests for famfaceangles.bids"""

import numpy as np
from numpy.testing import assert_array_equal, assert_array_almost_equal
from os.path import dirname, abspath, join
from ..bids import make_events

def _fake_face_stim(onset, stim_file, stim_size,
                    stim_poss, reaction_time='', response=''):
    """
    Code to emulate stimulus presentation

    Parameters
    ----------
    onset
    stim_file
    reaction_time
    response
    stim_size
    stim_poss

    Returns
    -------
    """

    lines = []
    stim_template = \
        '{onset:.4f}\tEXP\t{stim_file}: {prop} = array([{x}, {y}])\n'

    stim_code = 5

    lines.append('{0}\tEXP\tSTIM: {1}\n'.format(onset, stim_code))
    lines.append(
        stim_template.format(
            onset=onset,
            stim_file=stim_file,
            prop='size',
            x=stim_size[0],
            y=stim_size[1]
        )
    )

    onset_ = onset
    for i, pos in enumerate(stim_poss):
        lines.append(
            stim_template.format(
                onset=onset_,
                stim_file=stim_file,
                prop='pos',
                x=pos[0], y=pos[1]
            )
        )
        onset_ += 0.62

    if response and reaction_time:
        lines.append(
            '{0}\tDATA\tRESPONSE: {1}\n'.format(onset + reaction_time,
                                                response)
        )

    return lines

def test_make_events():
    testlog = join(dirname(abspath(__file__)), 'testlog.txt')
    with open(testlog, 'r') as f:
        lines = f.readlines()

    df = make_events(lines)
    assert(len(df) > 0)

    response2lbl = {'1': 'same', '2': 'different'}
    onsets = [3.5, 8.5, 13.5, 19.5]
    identities = ['id1', 'id3', 'id2', 'id2']
    orientations = ['a4', 'a3', 'a5', 'a1']
    stim_files = [
        'stim/familiar/{0}_{1}.png'.format(i, o)
        for i, o in zip(identities, orientations)
    ]

    sizes = [(500, 500) for _ in onsets]
    poss = [2 + np.random.randn(2)*10 for _ in onsets]
    reaction_times = [2.3, 1.2, '', 3.4]
    responses = ['1', '1', '', '2']

    lines = []
    for on, sf, sz, rt, res in zip(onsets, stim_files, sizes,
                                   reaction_times, responses):
        lines.extend(
            _fake_face_stim(on, sf, sz, poss,
                            reaction_time=rt, response=res))

    df = make_events(lines)

    assert(len(df)) == len(onsets)
    assert_array_equal(df.onset, onsets)
    assert_array_equal(df.identity, identities)
    assert_array_equal(df.orientation, orientations)
    assert_array_equal(df.stim_file, stim_files)
    assert_array_equal(df.duration,
                       np.round(np.diff(onsets), 0).tolist() + [5])
    assert_array_equal([np.round(r, 2) if isinstance(r, float)
                        else r for r in df.reaction_time],
                       np.array(reaction_times, dtype=str))
    assert_array_equal(df.response, [response2lbl.get(r, '')
                                     for r in responses])
    assert(df.stim_type.unique() == 'face')

    # add fixation at the end
    lines.extend(
        ['{0}\tEXP\tSTIM: 0'.format(onsets[-1] + 5.)]
    )
    df_ = make_events(lines)
    assert(df.to_dict() == df_.iloc[:-1].to_dict())
    assert(df_.iloc[-1].stim_type == 'fixation')

def test_make_events_startfixation():
    """Check what happens when the run starts with fixation"""

    lines = """15.8360         EXP     START OF EXPERIMENT
    30.8631         EXP     STIM: 0
    30.8881         EXP     0
    35.8652         EXP     STIM: 0
    35.8883         EXP     0
    40.8662         EXP     STIM: 10
    40.9018         EXP     stim/stranger/id2_a5.png: pos = array([ 3.56839343,  9.24224815])
    40.9018         EXP     stim/stranger/id2_a5.png: size = array([ 532.10222004,  532.10222004])
    41.5018         EXP     stim/stranger/id2_a5.png: pos = array([ 7.9594968 ,  0.60281356])
    42.1019         EXP     stim/stranger/id2_a5.png: pos = array([ 7.75116244,  0.74977234])"""

    df = make_events(lines.split('\n'))
    assert(len(df) == 3)

def test_make_events_tworesponses():
    """Check that we manage correctly when we have two responses"""
    exp_start = 15.8360
    stim_on = 40.9018
    resps = [43.0111, 43.4111, 43.6111]
    lines = """{0:.4f}         EXP     START OF EXPERIMENT
    30.8631         EXP     STIM: 0
    30.8881         EXP     0
    35.8652         EXP     STIM: 0
    35.8883         EXP     0
    40.8662         EXP     STIM: 10
    {1:.4f}         EXP     stim/stranger/id2_a5.png: pos = array([ 3.56839343,  9.24224815])
    40.9018         EXP     stim/stranger/id2_a5.png: size = array([ 532.10222004,  532.10222004])
    41.5018         EXP     stim/stranger/id2_a5.png: pos = array([ 7.9594968 ,  0.60281356])
    42.1019         EXP     stim/stranger/id2_a5.png: pos = array([ 7.75116244,  0.74977234])
    {2:.4f}        DATA    RESPONSE: 2
    {3:.4f}        DATA    RESPONSE: 1
    {4:.4f}        DATA    RESPONSE: 2""".format(exp_start, stim_on, *resps)

    df = make_events(lines.split('\n'))
    assert(len(df) == 3)
    face = df[df.stim_type == 'face']
    assert np.round(stim_on - exp_start, 2) == float(face.onset)
    assert np.round(resps[-1] - exp_start, 2) == float(face.onset) + \
                                              float(face.reaction_time)

ext: py

max_stars_repo_name: mvdoc/identity-decoding

max_issues_repo_name: mvdoc/identity-decoding

max_forks_repo_name: mvdoc/identity-decoding

reason: import numpy,from numpy


text:
import numpy as np
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.preprocessing import MinMaxScaler

'''
Computing the distance of certain order between two inputs; dimensions of the inputs should match.
'''
def distance(x, y, order):
    axis = tuple(np.arange(1, len(x.shape), dtype=int))
    input_shape = x.shape

    if order == 0:
        return np.sum((x - y) != 0, axis)
    elif order == 1:
        return np.sum(np.abs(x - y), axis)
    elif order == 2:
        return np.sqrt(np.sum(np.square(x - y), axis))
    elif order == 'inf':
        return np.max(np.abs(x - y), axis)
    elif order == 'mse':
        return np.mean(np.sum(np.square(x - y), axis))
    elif order == 'ptb':
        l1 = distance(x, y, 1)

        return np.round((1 / x.shape[0]) * (1 / np.prod(input_shape[1:])) * np.sum(l1), 4)
    elif order == 'ptbr':
        l0 = distance(x, y, 0)

        return np.round(100 * (1 / x.shape[0]) * (1 / np.prod(input_shape[1:])) * np.sum(l0), 2)
    else:
        print('Give correct order')

'''
Function for extracting the indices of perturbed samples.
'''
def ind_perturbed(adversarial_distance):
    ind = np.where(np.logical_and(np.greater(adversarial_distance, 0), np.less_equal(adversarial_distance, 1)))[0]

    return ind

'''
Function for extracting the indices of samples that no perturbation was found.
'''
def ind_non_perturbable(adversarial_distance):
    ind = np.where(np.isinf(adversarial_distance))[0]

    return ind

'''
Custom batch generator.
'''
def batch_generator(length, batch_size, shuffle=False):

    if isinstance(batch_size, float):
        batch_size = int(np.ceil(length*batch_size))
    else:
        if batch_size > length:
            batch_size = length

    ind = np.arange(length, dtype=int)
    n = int(np.ceil(length / batch_size))
    if shuffle:
        np.random.shuffle(ind)

    if n == 1:
        return [ind.tolist()]
    batch_ind = np.reshape(ind[:(n-1)*batch_size], [n-1, batch_size]).tolist()
    batch_ind.append(ind[(n-1)*batch_size:].tolist())

    return batch_ind

'''
Function for reversing a shuffling operation.
'''
def unshuffle_index(batch_ind):

    batch_ind_flat = np.array([x for sublist in batch_ind for x in sublist])
    return np.argsort(batch_ind_flat)

'''
Function for scaling the input, so all features are within [0, 1].
'''
def min_max_scaling(train, test=None):
    mms = MinMaxScaler()

    shape_train = train.shape
    train = np.reshape(train, [shape_train[0], np.prod(shape_train[1:])])
    mms.fit(train)
    train = mms.transform(train)
    train = np.reshape(train, shape_train)

    if test is not None:
        shape_test = test.shape
        test = np.reshape(test, [shape_test[0], np.prod(shape_test[1:])])
        test = mms.transform(test)
        test = np.reshape(test, shape_test)

        return train, test

    return train

'''
Simple scaling, dividing by the max.
'''
def max_scaling(train, test=None):
    if test is not None:
        return train / np.max(train), test / np.max(test)

    return train / np.max(train)

'''
Function for scaling the input, so all features are of 0 mean and unit variance.
'''
def zero_mean_unit_variance(train, test=None, use_std=True):
    train_mean = np.mean(train, axis=0)
    if test is not None:
        test_mean = np.mean(test, axis=0)
    if use_std:
        train_std = np.std(train, axis=0)
        if test is not None:
            test_std = np.std(test, axis=0)
    else:
        train_std = np.ones_like(train_mean)
        if test is not None:
            test_std = np.ones_like(test_mean)
    train = (train - train_mean) / (train_std + 1e-32)

    if test is not None:
        test = (test - test_mean) / (test_std + 1e-32)

        return train, test

    return train

'''
Function for scaling the input, so input is positive and within [0, 1].
'''
def translate_pos(train):
    train_pos = np.array(train, copy=True)
    orig_shape = train_pos.shape
    train_pos = np.moveaxis(train_pos, -1, 1)
    perm_shape = train_pos.shape
    train_pos = np.reshape(train_pos, [perm_shape[0], perm_shape[1], np.prod(perm_shape[2:])])
    min_per_image = np.min(train_pos, axis=2)
    train_pos[min_per_image < 0] = train_pos[min_per_image < 0] - np.tile(
        min_per_image[min_per_image < 0][..., np.newaxis], (1, np.prod(perm_shape[2:])))
    train_pos = train_pos / (np.tile(np.max(train_pos, axis=2)[..., np.newaxis], (1, np.prod(perm_shape[2:]))) + 1e-32)
    train_pos = np.moveaxis(train_pos, 1, -1)
    train_pos = np.reshape(train_pos, orig_shape)

    return train_pos

'''
Subtracting the pixel mean.
'''
def pixel_mean(train, test):

    train_mean = np.mean(train, 0)

    return train - train_mean, test - train_mean

'''
Wrapper function for all scaling functions
'''
def normalization(norm_type, train, test=None):
    if norm_type != 'no':

        if norm_type == 'max':
            if test is not None:
                train, test = max_scaling(train, test)
            else:
                train = max_scaling(train)
        elif norm_type == 'min_max':
            if test is not None:
                train, test = min_max_scaling(train, test)
            else:
                train = min_max_scaling(train)
        elif norm_type == 't_pos':
            train = translate_pos(train)
            if test is not None:
                test = translate_pos(test)
        elif norm_type == 'zero_unit':
            if test is not None:
                train, test = zero_mean_unit_variance(train, test, use_std=True)
            else:
                train = zero_mean_unit_variance(train, use_std=True)
        else:
            raise KeyError("Give correct type of norm_type, among {max, min_max and zero_unit}")

    if test is not None:
        return train, test
    else:
        return train

'''
Function for extracting training and validation splits.
'''
def validation_split(x_train, y_train, val_split, stratified=True):

    if stratified:
        sss = StratifiedShuffleSplit(n_splits=1, test_size=val_split)
        for train_ind, val_ind in sss.split(x_train, y_train):
            return train_ind, val_ind
    else:
        n = len(x_train)
        ind = np.arange(n)
        val_ind = np.random.choice(n, int(np.ceil(n*val_split)), replace=False)
        train_ind = np.isin(ind, val_ind, invert=True)

        return train_ind, val_ind

max_stars_repo_name: giannislelekas/topdown

max_issues_repo_name: giannislelekas/topdown

max_forks_repo_name: giannislelekas/topdown

reason: import numpy


text:
#######################################################################
# Copyright (C) 2019-present, Blosc Development team <blosc@blosc.org>
# All rights reserved.
#
# This source code is licensed under a BSD-style license (found in the
# LICENSE file in the root directory of this source tree)
#######################################################################

import caterva as cat
import pytest
import numpy as np

@pytest.mark.parametrize("shape, chunks, blocks, dtype",
                         [
                             ([931], [223], [45], np.int32),
                             ([134, 121, 78], [12, 13, 18], [4, 4, 9], np.float64),
                         ])
def test_numpy(shape, chunks, blocks, dtype):
    size = int(np.prod(shape))
    nparray = np.arange(size, dtype=dtype).reshape(shape)
    a = cat.asarray(nparray, chunks=chunks, blocks=blocks)
    nparray2 = np.asarray(a[:]).view(dtype)
    np.testing.assert_almost_equal(nparray, nparray2)

max_stars_repo_name: DimitriPapadopoulos/python-caterva

max_issues_repo_name: DimitriPapadopoulos/python-caterva

max_forks_repo_name: DimitriPapadopoulos/python-caterva


text:
# -*- coding: utf-8 -*-
"""
Created on Thu Aug 13 13:52:23 2015

@author: wirkert
"""

import unittest
import copy
import numpy as np

from msi.test import helpers
import msi.msimanipulations as mani

class TestMsiManipulations(unittest.TestCase):

    def setUp(self):
        self.msi = helpers.getFakeMsi()
        self.specialmsi = helpers.getFakeMsi()

        # set one pixel to special values
        self.specialValue = np.arange(self.specialmsi.get_image().shape[-1]) * 2
        self.specialmsi.get_image()[2, 2, :] = self.specialValue

        # create a segmentation which sets all elements to invalid but the
        # one pixel with the special value
        self.segmentation = np.zeros(self.specialmsi.get_image().shape[0:-1])
        self.segmentation[2, 2] = 1

    def tearDown(self):
        pass

    def test_apply_segmentation(self):

        mani.apply_segmentation(self.specialmsi, self.segmentation)

        validImageEntries = self.specialmsi.get_image() \
            [~self.specialmsi.get_image().mask]

        np.testing.assert_equal(validImageEntries, self.specialValue,
                        "image has been correctly segmented")

    def test_calculate_mean_spectrum(self):

        mani.calculate_mean_spectrum(self.specialmsi)

        np.testing.assert_equal(np.array([0.96, 2., 3.04, 4.08, 5.12]),
                                self.specialmsi.get_image(),
                        "mean spectrum is correctly calculated on image with " +
                        "no mask applied")

    def test_calculate_mean_spectrum_masked_image(self):

        mani.apply_segmentation(self.specialmsi, self.segmentation)
        mani.calculate_mean_spectrum(self.specialmsi)

        np.testing.assert_equal(self.specialValue, self.specialmsi.get_image(),
                        "mean spectrum is correctly calculated on image with " +
                        "mask applied")

    def test_interpolate(self):
        # create not sorted new wavelengths
        newWavelengths = np.array([4.0, 2.5, 3.5, 1.5])
        mani.interpolate_wavelengths(self.msi, newWavelengths)

        np.testing.assert_equal(newWavelengths, self.msi.get_wavelengths(),
                        "wavelengths correctly updated")
        # check if first image pixel was correctly calculated
        # (hopefully true for all then)
        np.testing.assert_equal(np.array([2.0, 3.5, 2.5, 4.5]),
                                self.msi.get_image()[0, 0, :],
                        "image elements correctly interpolated")

    def test_normalize_integration_times(self):
        old_shape = self.msi.get_image().shape
        integration_times = np.array([1., 2., 3., 4., 5.])
        self.msi.add_property({'integration times': integration_times})
        mani.normalize_integration_times(self.msi)

        np.testing.assert_equal(self.msi.get_image()[1, 3, :],
                            np.ones_like(integration_times),
                            "normalized integration times")
        np.testing.assert_equal(self.msi.get_properties()['integration times'],
                            np.ones_like(integration_times),
                            "integration time property set to ones")
        self.assertEqual(self.msi.get_image().shape, old_shape,
                        "shape did not change from normalizing")

    def test_normalize_integration_times_none_given(self):
        msi_copy = copy.deepcopy(self.msi)
        mani.normalize_integration_times(msi_copy)
        np.testing.assert_equal(msi_copy.get_image(), self.msi.get_image(),
                                "nothing change by normalizing without" + \
                                "integration times given")

    def test_dark_correction(self):
        desired_image_data = copy.copy(self.msi.get_image())
        desired_image_data -= 1

        dark = copy.copy(self.msi)
        dark.set_image(np.ones_like(dark.get_image()))

        mani.dark_correction(self.msi, dark)

        np.testing.assert_equal(self.msi.get_image(),
                                       desired_image_data,
                                       "dark image correctly accounted for")
        np.testing.assert_equal(dark.get_image(),
                                       np.ones_like(dark.get_image()),
                        "dark image unchanged by dark correction")

    def test_dark_correction_with_single_value(self):
        desired_image_data = copy.copy(self.specialmsi.get_image())
        desired_image_data -= 1

        dark = copy.copy(self.specialmsi)
        dark.set_image(np.ones_like(dark.get_image()))
        mani.calculate_mean_spectrum(dark)

        mani.dark_correction(self.specialmsi, dark)

        np.testing.assert_equal(self.specialmsi.get_image(),
                                       desired_image_data,
                "dark image correctly accounted for from singular dark value")
        np.testing.assert_equal(dark.get_image(),
                                       np.ones_like(dark.get_image()),
                "dark image unchanged by dark correction")

    def test_flatfield_correction(self):
        desired_image_data = np.ones_like(self.specialmsi.get_image())
        desired_image_data[2, 2, 0] = np.nan

        mani.flatfield_correction(self.specialmsi, self.specialmsi)

        np.testing.assert_equal(self.specialmsi.get_image(),
                                       desired_image_data,
                       "correct image by itself should lead to only 1s ")

    def test_flatfield_correction_differing_integration_times(self):
        MSI_INTEGRATION_TIME = 3.0
        FLATFIELD_INTEGRATION_TIME = 2.0
        desired_image_data = np.ones_like(self.specialmsi.get_image()) * \
             FLATFIELD_INTEGRATION_TIME / MSI_INTEGRATION_TIME
        desired_image_data[2, 2, 0] = np.nan
        self.specialmsi.add_property({"integration times":
                                      np.ones_like(
                                        self.specialmsi.get_image()[0, 0, :])
                                        * MSI_INTEGRATION_TIME})
        flatfield = copy.deepcopy(self.specialmsi)
        flatfield.add_property({"integration times":
                                np.ones_like(
                                    flatfield.get_image()[0, 0, :])
                                    * FLATFIELD_INTEGRATION_TIME})
        # for testing if flatfield does not changed by correction we copy it
        flatfield_copy = copy.deepcopy(flatfield)

        mani.flatfield_correction(self.specialmsi, flatfield_copy)

        np.testing.assert_almost_equal(self.specialmsi.get_image(),
                                       desired_image_data, 15,
                       "corrected image is a division of integration times")
        np.testing.assert_equal(flatfield.get_image(),
                                flatfield_copy.get_image(),
                                "flatfield doesn't change by correction")

    def test_flatfield_correction_with_single_value(self):
        desired_image_data = np.ones_like(self.msi.get_image())
        flatfield = copy.copy(self.msi)
        mani.calculate_mean_spectrum(flatfield)
        unchanged_flatfield = copy.deepcopy(flatfield)

        mani.flatfield_correction(self.msi, flatfield)

        np.testing.assert_equal(self.msi.get_image(),
                                       desired_image_data,
                "flatfield correctly accounted for from singular reference value")
        np.testing.assert_equal(flatfield, unchanged_flatfield,
                "flatfield not changed by algorithm")

    def test_image_correction(self):
        dark = copy.copy(self.msi)
        dark.set_image(np.ones_like(dark.get_image()) * 0.5)
        flatfield = copy.copy(self.msi)
        flatfield_copy = copy.deepcopy(flatfield)
        dark_copy = copy.deepcopy(dark)

        mani.image_correction(self.msi, flatfield, dark)

        np.testing.assert_equal(flatfield.get_image(),
                                flatfield_copy.get_image(),
                                "image correction didn't change flatfield")
        np.testing.assert_equal(dark.get_image(), dark_copy.get_image(),
                                "image correction didn't change dark image")
        np.testing.assert_almost_equal(self.msi.get_image(),
                                       np.ones_like(self.msi.get_image()),
                                       15, "image correctly corrected :-)")


text:
import argparse
from concurrent.futures import as_completed, ProcessPoolExecutor
import json
import os
import sys
sys.path.insert(0, '../RISCluster/')

import h5py
import numpy as np
from tqdm import tqdm

import utils

debug = False
if debug:
    source = '../../../Data/Full/Full.h5'
    include = '[24]'
    exclude = None
    after = None
    before = None
    dest = '../../../Data/RS09.h5'

def _copy_attributes(in_object, out_object):
    '''Copy attributes between 2 HDF5 objects.'''
    for key, value in in_object.attrs.items():
        out_object.attrs[key] = value

def _find_indeces(index, source, stations):
    with h5py.File(source, 'r') as f:
        metadata = json.loads(f['/4.0/Catalogue'][index])
    if metadata["Station"] in stations:
        return index
    else:
        return np.nan

if __name__ == "__main__":
    if not debug:
        parser = argparse.ArgumentParser(
            description="Creates new dataset from existing dataset."
        )
        parser.add_argument("source", help="Enter path to source dataset.")
        parser.add_argument("dest", help="Enter path to destination dataset.")
        parser.add_argument("--include", help="Enter stations to include.")
        parser.add_argument("--exclude", help="Enter stations to exclude.")
        parser.add_argument("--after", help="Include after YYYYMMDDTHHMMSS.")
        parser.add_argument("--before", help="Include before YYYYMMDDTHHMMSS.")
        args = parser.parse_args()

        if args.include is None and args.exclude is None:
            raise Exception("Must specify stations to include or exclude.")
        source = args.source
        dest = args.dest
        include = args.include
        exclude = args.exclude
        after = args.after
        before = args.before

    if not os.path.exists(source):
        raise ValueError(f"Source file not found: {source}")

    with h5py.File(source, 'r') as rf:
        M = len(rf['/4.0/Trace'])
    index = np.arange(1, M)

    if include is not None:
        include = json.loads(include)
    else:
        include = None
    if exclude is not None:
        exclude = json.loads(exclude)
    else:
        exclude = None
    if after is not None:
        after = after
    else:
        after = None
    if before is not None:
        before = before
    else:
        before = None

    if include is not None and exclude is not None:
        removals = [utils.get_station(i) for i in exclude]
        stations = [utils.get_station(i) for i in include]
        stations = [i for i in stations if i not in removals]
        print(f"Searching {stations}")
    elif include is not None:
        stations = [utils.get_station(i) for i in include]
        print(f"Searching {stations}")
    elif exclude is not None:
        removals = [utils.get_station(i) for i in exclude]
        stations = [utils.get_station(i) for i in range(34)]
        stations = [i for i in stations if i not in removals]
        print(f"Searching {stations}")
    else:
        stations = [utils.get_station(i) for i in range(34)]
        print(f"Searching {stations}")

    A = [{
            "index": index[i],
            "source": source,
            "stations": stations
        } for i in range(len(index))]

    index_keep = np.zeros(len(index))
    with ProcessPoolExecutor(max_workers=14) as exec:
        print("Finding detections that meet filter criteria...")
        futures = [exec.submit(_find_indeces, **a) for a in A]
        kwargs1 = {
            "total": int(len(index)),
            "bar_format": '{l_bar}{bar:20}{r_bar}{bar:-20b}',
            "leave": True
        }
        for i, future in enumerate(tqdm(as_completed(futures), **kwargs1)):
            index_keep[i] = future.result()
    index_keep = np.sort(index_keep[~np.isnan(index_keep)]).astype(int)

    with h5py.File(source, 'r') as fs, h5py.File(dest, 'w') as fd:
        M = len(index_keep)
        dset_names = ['Catalogue', 'Trace', 'Spectrogram', 'Scalogram']
        # dset_names = ['Catalogue']
        for dset_name in dset_names:
            group_path = '/4.0'
            dset = fs[f"{group_path}/{dset_name}"]
            dset_shape = dset.shape[1:]
            dset_shape = (M,) + dset_shape
            group_id = fd.require_group(group_path)
            dset_id = group_id.create_dataset(dset_name, dset_shape, dtype=dset.dtype, chunks=None)
            _copy_attributes(dset, dset_id)
            kwargs2 = {
                "desc": dset_name,
                "bar_format": '{l_bar}{bar:20}{r_bar}{bar:-20b}'
            }
            for i in tqdm(range(len(index_keep)), **kwargs2):
                dset_id[i] = dset[index_keep[i]]


text:
""" philoseismos: engineering seismologist's toolbox.

author: Ivan Dubrovin
e-mail: io.dubrovin@icloud.com """

import numpy as np

# format string to unpack BFHs in one go
BFHFS = 'iiihhhhhhhhhhhhhhhhhhhhhhhhiiiQQiii' + 'i' * 50 + 'BBhhihQQi' + 'i' * 17

# keys for BFH dictionary
BFHCOLS = ['job_id',
           'line_no',
           'reel_no',
           'traces_per_ensemble',
           'aux_traces_per_ensemble',
           'sample_interval',
           'sample_interval_orig',
           'samples_per_trace',
           'samples_per_trace_orig',
           'sample_format',
           'ensemble_fold',
           'trace_sorting',
           'vertical_sum',
           'sweep_freq_start',
           'sweep_freq_end',
           'sweep_length',
           'sweep_type',
           'sweep_channel',
           'sweep_taper_start',
           'sweep_taper_end',
           'taper_type',
           'correlated_flag',
           'binary_gain_recovery_flag',
           'amp_recovery_method',
           'measurement_system',  # 1 for meters, 2 for feet
           'impulse_polarity',
           'vibratory_polarity',
           'ext_traces_per_ensemble',
           'ext_aux_traces_per_ensemble',
           'ext_samples_per_trace',
           'ext_sample_interval',
           'ext_sample_interval_orig',
           'ext_samples_per_trace_orig',
           'ext_ensemble_fold',
           'integer_constant']
BFHCOLS += [f'unassigned_{i}' for i in range(1, 51)]
BFHCOLS += ['segy_revision_major',
            'segy_revision_minor',
            'fixed_trace_length_flag',
            'no_ext_tfhs',
            'no_additional_trace_headers',
            'time_basis',
            'no_traces',
            'byte_offset_of_data',
            'no_trailer_stanzas']
BFHCOLS += [f'unassigned_{i}' for i in range(51, 68)]

# sample format code dictionary. maps codes to (size, format letter, description)
SFC = {
    1: (4, None, '4-byte IBM floating-point'),
    2: (4, 'i', '4-byte signed integer'),
    3: (2, 'h', '2-byte signed integer'),
    5: (4, 'f', '4-byte IEEE floating-point'),
    6: (4, 'd', '8-byte IEEE floating-point'),
    8: (1, 'b', '1-byte signed integer'),
    9: (8, 'q', '8-byte signed integer'),
    10: (4, 'L', '4-byte, unsigned integer'),
    11: (2, 'H', '2-byte, unsigned integer'),
    12: (8, 'Q', '8-byte, unsigned integer'),
    16: (1, 'B', '1-byte, unsigned integer')
}

# format string to unpack trace headers in one go. after these 232 bytes go 8 bytes of text
THFS = 'iiiiiiihhhhiiiiiiiihhiiiihhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhiiiiihhihhhhhhhhihh'

# columns for Geometry DataFrame
THCOLS = ['TRACENO',  # ordinal number of a trace
          'fTRACENO',
          'FFID',  # unique number of a shot
          'CHAN',  # channel number
          'SOURCE',  # number of a source point
          'CDP',  # number of a common depth point
          'SEQNO',  # ordinal number of a trace in the ensemble
          'TRC_TYPE',  # ID code for a trace
          'STACKCNT',  # number of vertically stacked traces
          'TRFOLD',  # number of horizontally stacked traces
          'Data Use',
          'OFFSET',  # offset - distance between the source and the receiver
          'REC_ELEV',  # receiver elevation
          'SOU_ELEV',  # source elevation
          'DEPTH',  # source depth from the surface
          'REC_DATUM',  # elevation of the datum at the receiver
          'SOU_DATUM',  # elevation of the datum at the source
          'SOU_H2OD',  # water depth at the source
          'REC_H2OD',  # water depth at the receiver
          'ELEVSC',  # scalar to apply to the elevations
          'COORDSC',  # scalar to apply to the coordinates
          'SOU_X',
          'SOU_Y',
          'REC_X',
          'REC_Y',
          'Coordinate Units',
          'Weathering Velocity',
          'Subweathering Velocity',
          'UPHOLE',  # vertical time at the source, ms
          'REC_UPHOLE',  # vertical time at the receiver, ms
          'SOU_STAT',  # a static correction at the source, ms
          'REC_STAT',  # a static correction at the receiver, ms
          'TOT_STAT',  # total static correction, ms
          'Lag Time A',
          'Lag Time B',
          'Delay Recording Time',
          'TLIVE_S',  # starting time of muting, ms
          'TFULL_S',  # ending time of muting, ms
          'NUMSMP',  # number of samples in a trace (RadExPro needs this!)
          'DT',  # sample interval in microseconds (RadExPro needs this!)
          'IGAIN',  # code of the gain type of the instrument
          'PREAMP',  # amplification coefficient for instrument, dB
          'EARLYG',  # initial amplification of the instrument, dB
          'COR_FLAG',  # correlation flag (1 - no, 2 - yes)
          'SWEEPFREQSTART',  # starting frequency of the sweep, Hz
          'SWEEPFREQEND',  # ending frequency of the sweep, Hz
          'SWEEPLEN',  # length of the sweep, ms
          'SWEEPTYPE',  # code for the sweep type
          'Sweep Taper (start)',
          'Sweep Taper (end)',
          'Taper Type',
          'AAXFILT',  # frequency of the anti-aliasing filter, Hz
          'AAXSLOP',  # slope of the anti-aliasing filter, dB / oct
          'FREQXN',  # frequency of the reject filter, Hz
          'FXNSLOP',  # slope of the reject filter, dB / oct
          'FREQXL',  # low-cut frequency, Hz
          'FREQXH',  # high-cut frequency, Hz
          'FXLSLOP',  # low-cut slope, dB / oct
          'FXHSLOP',  # high-cut slope, dB / oct
          'YEAR',
          'DAY',
          'HOUR',
          'MINUTE',
          'SECOND',
          'Time Basis',
          'Weighting Factor',
          'Group No. of Roll Switch',
          'Group No. of First Trace',
          'Group No. of Last Trace',
          'Gap Size',
          'Over Travel',
          'CDP_X',
          'CDP_Y',
          'In-line No.',
          'Cross-line No.',
          'Shotpoint Number',
          'Shotpoint Scalar',
          'Trace Measurement Unit',
          'Transduction Constant (mantissa)',
          'Transduction Constant (10 power exponent)',
          'Transduction Units',
          'Device/Trace ID',
          'Times Scalar',
          'Source Type/Orientation',
          'Source Energy Direction (vertical)',
          'Source Energy Direction (cross-line)',
          'Source Energy Direction (in-line)',
          'Source Measurement (mantissa)',
          'Source Measurement (10 power exponent)',
          'Source Measurement Unit']
# then go 8 bytes of text - so called "Header name"

# dictionaries to map sample format code to data matrix dtype
DTYPEMAP = {
    1: np.float32,
    2: np.int32,
    3: np.int16,
    5: np.float32,
    6: np.float64,
    8: np.int8,
    9: np.int64,
    10: np.uint32,
    11: np.uint16
}

IDTYPEMAP = {
    'float32': 5,
    'int32': 2,
    'int16': 3,
    'float64': 6,
    'int8': 8,
    'int64': 9,
    'uint32': 10,
    'uint16': 11
}

max_stars_repo_name: iod-ine/philoseismos

max_issues_repo_name: iod-ine/philoseismos

max_forks_repo_name: iod-ine/philoseismos


text:
# -*- coding: utf-8 -*-
import cv2
import numpy as np

# leemos la imagen mediante la funcion imread
img = cv2.imread ('C:/Users/Victor Pacheco Garci/Desktop/UPC/2017-18/GDSA/Team5/Farmacia Albinyana/farmacia_albinyana_101.jpg',1)

# convertimos la imagen a escala de grises
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

# detectamos esquinas. Corner Harris(img,blockSize,ksize,k) utiliza los siguientes argumentos:
    # img - imagen en escala de grises tipo float 32.
    # blockSize - Tamaño del vecindario considerado para la detección de esquinas.
    # ksize - parámetro de apertura de la derivada de Sobel utilizada.
    # k - parámetro libre en la equación del detector Harris.
gray = np.float32(gray)
dst = cv2.cornerHarris(gray,2,3,0.001)

# result is dilated for marking the corners, not important
dst = cv2.dilate(dst,None)

# Threshold for an optimal value, it may vary depending on the image. En este caso, dst>0.005
img[dst>0.005*dst.max()]=[0,0,255]

cv2.imshow('dst',img)
if cv2.waitKey(0) & 0xff == 27:
    cv2.destroyAllWindows()

max_stars_repo_name: gdsa-upc/PicZam

max_issues_repo_name: gdsa-upc/PicZam

max_forks_repo_name: gdsa-upc/PicZam


text:
import numpy as np
import lenstronomy.Util.kernel_util as kernel_util
import lenstronomy.Util.util as util
import warnings

class PSF(object):
    """
    Point Spread Function class.
    This class describes and manages products used to perform the PSF modeling (convolution for extended surface
    brightness and painting of PSF's for point sources).
    """

    def __init__(self, psf_type='NONE', fwhm=None, truncation=5, pixel_size=None, kernel_point_source=None,
                 psf_error_map=None, point_source_supersampling_factor=1, kernel_point_source_init=None):
        """

        :param psf_type: string, type of PSF: options are 'NONE', 'PIXEL', 'GAUSSIAN'
        :param fwhm: float, full width at half maximum, only required for 'GAUSSIAN' model
        :param truncation: float, Gaussian truncation (in units of sigma), only required for 'GAUSSIAN' model
        :param pixel_size: width of pixel (required for Gaussian model, not required when using in combination with ImageModel modules)
        :param kernel_point_source: 2d numpy array, odd length, centered PSF of a point source
        :param psf_error_map: uncertainty in the PSF model. Same shape as point source kernel.
        This error will be added to the pixel error around the position of point sources as follows:
        sigma^2_i += 'psf_error_map'_j * (point_source_flux_i)**2
        :param point_source_supersampling_factor: int, supersampling factor of kernel_point_source
        :param kernel_point_source_init: memory of an initial point source kernel that gets passed through the psf iteration
        """
        self.psf_type = psf_type
        self._pixel_size = pixel_size
        self.kernel_point_source_init = kernel_point_source_init
        if self.psf_type == 'GAUSSIAN':
            if fwhm is None:
                raise ValueError('fwhm must be set for GAUSSIAN psf type!')
            self._fwhm = fwhm
            self._sigma_gaussian = util.fwhm2sigma(self._fwhm)
            self._truncation = truncation
            self._point_source_supersampling_factor = 0
        elif self.psf_type == 'PIXEL':
            if kernel_point_source is None:
                raise ValueError('kernel_point_source needs to be specified for PIXEL PSF type!')
            if len(kernel_point_source) % 2 == 0:
                raise ValueError('kernel needs to have odd axis number, not ', np.shape(kernel_point_source))
            if point_source_supersampling_factor > 1:
                self._kernel_point_source_supersampled = kernel_point_source
                self._point_source_supersampling_factor = point_source_supersampling_factor
                kernel_point_source = kernel_util.degrade_kernel(self._kernel_point_source_supersampled, self._point_source_supersampling_factor)
            self._kernel_point_source = kernel_point_source / np.sum(kernel_point_source)

        elif self.psf_type == 'NONE':
            self._kernel_point_source = np.zeros((3, 3))
            self._kernel_point_source[1, 1] = 1
        else:
            raise ValueError("psf_type %s not supported!" % self.psf_type)
        if psf_error_map is not None:
            self._psf_error_map = psf_error_map
            if self.psf_type == 'PIXEL':
                if len(self._psf_error_map) != len(self._kernel_point_source):
                    raise ValueError('psf_error_map must have same size as kernel_point_source!')
            self.psf_error_map_bool = True
        else:
            self.psf_error_map_bool = False

    @property
    def kernel_point_source(self):
        if not hasattr(self, '_kernel_point_source'):
            if self.psf_type == 'GAUSSIAN':
                kernel_numPix = self._truncation * self._fwhm / self._pixel_size
                if kernel_numPix % 2 == 0:
                    kernel_numPix += 1
                self._kernel_point_source = kernel_util.kernel_gaussian(kernel_numPix, self._pixel_size, self._fwhm)
        return self._kernel_point_source

    @property
    def kernel_pixel(self):
        """
        returns the convolution kernel for a uniform surface brightness on a pixel size

        :return: 2d numpy array
        """
        if not hasattr(self, '_kernel_pixel'):
            self._kernel_pixel = kernel_util.pixel_kernel(self.kernel_point_source, subgrid_res=1)
        return self._kernel_pixel

    def kernel_point_source_supersampled(self, supersampling_factor, updata_cache=True):
        """
        generates (if not already available) a supersampled PSF with ood numbers of pixels centered

        :param supersampling_factor: int >=1, supersampling factor relative to pixel resolution
        :param updata_cache: boolean, if True, updates the cached supersampling PSF if generated.
        Attention, this will overwrite a previously used supersampled PSF if the resulution is changing.
        :return: super-sampled PSF as 2d numpy array
        """
        if hasattr(self, '_kernel_point_source_supersampled') and self._point_source_supersampling_factor == supersampling_factor:
            kernel_point_source_supersampled = self._kernel_point_source_supersampled
        else:
            if self.psf_type == 'GAUSSIAN':
                kernel_numPix = self._truncation / self._pixel_size * supersampling_factor
                kernel_numPix = int(round(kernel_numPix))
                if kernel_numPix > 10000:
                    raise ValueError('The pixelized Gaussian kernel has a grid of %s pixels with a truncation at '
                                     '%s times the sigma of the Gaussian, exceeding the limit allowed.' % (kernel_numPix, self._truncation))
                if kernel_numPix % 2 == 0:
                    kernel_numPix += 1
                kernel_point_source_supersampled = kernel_util.kernel_gaussian(kernel_numPix, self._pixel_size / supersampling_factor, self._fwhm)

            elif self.psf_type == 'PIXEL':

                kernel = kernel_util.subgrid_kernel(self.kernel_point_source, supersampling_factor, odd=True, num_iter=5)
                n = len(self.kernel_point_source)
                n_new = n * supersampling_factor
                if n_new % 2 == 0:
                    n_new -= 1
                if hasattr(self, '_kernel_point_source_supersampled'):
                    warnings.warn("Super-sampled point source kernel over-written due to different subsampling"
                                  " size requested.", Warning)
                kernel_point_source_supersampled = kernel_util.cut_psf(kernel, psf_size=n_new)
            elif self.psf_type == 'NONE':
                kernel_point_source_supersampled = self._kernel_point_source
            else:
                raise ValueError('psf_type %s not valid!' % self.psf_type)
            if updata_cache is True:
                self._kernel_point_source_supersampled = kernel_point_source_supersampled
                self._point_source_supersampling_factor = supersampling_factor
        return kernel_point_source_supersampled

    def set_pixel_size(self, deltaPix):
        """
        update pixel size

        :param deltaPix: pixel size in angular units (arc seconds)
        :return: None
        """
        self._pixel_size = deltaPix
        if self.psf_type == 'GAUSSIAN':
            try:
                del self._kernel_point_source
            except:
                pass

    @property
    def psf_error_map(self):
        if not hasattr(self, '_psf_error_map'):
            self._psf_error_map = np.zeros_like(self.kernel_point_source)
        return self._psf_error_map

    @property
    def fwhm(self):
        """

        :return: full width at half maximum of kernel (in units of pixel)
        """
        if self.psf_type == 'GAUSSIAN':
            return self._fwhm
        else:
            return kernel_util.fwhm_kernel(self.kernel_point_source) * self._pixel_size

max_stars_repo_name: jiwoncpark/lenstronomy

max_issues_repo_name: jiwoncpark/lenstronomy

max_forks_repo_name: jiwoncpark/lenstronomy


text:
import tweepy
import os
import re
import logging
import nltk
import string
import datetime
import pandas as pd
import numpy as np
import preprocessor as p
from tweepy import OAuthHandler
from tweepy import API
from tweepy import Cursor
from nltk.corpus import stopwords
from textblob import TextBlob

form = logging.Formatter("%(asctime)s : %(levelname)-5.5s : %(message)s")
logger = logging.getLogger()

consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(form)
logger.addHandler(consoleHandler)

logger.setLevel(logging.DEBUG)

class dataAcq():
    def __init__(self, cols=None, auth=None):
        if cols is not None:
            self.cols = cols
        else:
            self.cols = ['id', 'created_at', 'source', 'original_text', 'clean_text',
                         'polarity', 'subjectivity', 'lang', 'favorite_count', 'retweet_count', 'original_author',
                         'possibly_sensitive', 'hashtags', 'user_mentions', 'place', 'place_coord_boundaries']

        if auth is None:
            # Variables that contains the user credentials to access Twitter API
            consumer_key = os.environ.get('TWITTER_API_KEY')
            consumer_secret = os.environ.get('TWITTER_API_SECRET')
            access_token = os.environ.get('TWITTER_ACCESS_TOKEN')
            access_token_secret = os.environ.get('TWITTER_ACCESS_TOKEN_SECRET')

            # This handles Twitter authetification and the connection to Twitter Streaming API
            auth = OAuthHandler(consumer_key, consumer_secret)
            auth.set_access_token(access_token, access_token_secret)

        self.auth = auth
        self.api = API(auth, wait_on_rate_limit=True)

    def cleanTweets(self, twitterText):
        logger.info("Cleaning a tweet \n\n")
        tweet = p.clean(twitterText)

        # Happy Emoticons
        emoticonsHappy = set([
            ':-)', ':)', ';)', ':o)', ':]', ':3', ':c)', ':>', '=]', '8)', '=)', ':}',
            ':^)', ':-D', ':D', '8-D', '8D', 'x-D', 'xD', 'X-D', 'XD', '=-D', '=D',
            '=-3', '=3', ':-))', ":'-)", ":')", ':*', ':^*', '>:P', ':-P', ':P', 'X-P',
            'x-p', 'xp', 'XP', ':-p', ':p', '=p', ':-b', ':b', '>:)', '>;)', '>:-)',
            '<3'])

        # Sad Emoticons
        emoticonsSad = set([
            ':L', ':-/', '>:/', ':S', '>:[', ':@', ':-(', ':[', ':-||', '=L', ':<',
            ':-[', ':-<', '=\\', '=/', '>:(', ':(', '>.<', ":'-(", ":'(", ':\\', ':-c',
            ':c', ':{', '>:\\', ';('])

        # Emoji patterns
        emojiPattern = re.compile("[" u"\U0001F600-\U0001F64F"  # emoticons
                                      u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                                      u"\U0001F680-\U0001F6FF"  # transport & map symbols
                                      u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                                      u"\U00002702-\U000027B0"
                                      u"\U000024C2-\U0001F251" "]+", flags=re.UNICODE)

        # Combine the sad and happy emoticons
        emoticons = emoticonsHappy.union(emoticonsSad)

        # tokenize the words in the tweet and get the stop words
        stopWords = set(stopwords.words('english'))
        wordTokens = nltk.word_tokenize(tweet)

        # after tweepy preprocessing the colon symbol left remain after
        # removing mentions
        tweet = re.sub(r':', '', tweet)
        tweet = re.sub(r'‚Ä¶', '', tweet)

        # replace consecutive non-ASCII characters with a space
        tweet = re.sub(r'[^\x00-\x7F]+', ' ', tweet)

        # remove emojis from tweet
        tweet = emojiPattern.sub(r'', tweet)

        # looping through conditions
        filteredTweet = []
        for w in wordTokens:
            # check tokens against stop words , emoticons and punctuations
            if w not in stopWords and w not in emoticons and w not in string.punctuation:
                filteredTweet.append(w)

        return ' '.join(filteredTweet)

    def getTweets(self, keyword, geocode, startDate, endDate, csvfile):

        df = pd.DataFrame(columns=self.cols)
        if csvfile is not None:
            # If the file exists, then read the existing data from the CSV file.
            if os.path.exists(csvfile):
                df = pd.read_csv(csvfile, header=0)

        for status in Cursor(self.api.search, q=keyword, geocode=geocode, lang="en", since_id=startDate,
                             tweet_mode="extended", include_rts=False).items(200):
            logger.info("Working on new Tweet :)")

            newEntry = []
            status = status._json

            # if this tweet is a retweet update retweet count
            if status['created_at'] in df['created_at'].values:
                i = df.loc[df['created_at'] == status['created_at']].index[0]

                cond1 = status['favorite_count'] != df.at[i, 'favorite_count']
                cond2 = status['retweet_count'] != df.at[i, 'retweet_count']
                if cond1 or cond2:
                    df.at[i, 'favorite_count'] = status['favorite_count']
                    df.at[i, 'retweet_count'] = status['retweet_count']

                continue

            # clean the tweet
            cleanText = self.cleanTweets(status['text'])

            # Calculate the Sentiment
            blob = TextBlob(cleanText)
            polarity = blob.sentiment.polarity
            subjectivity = blob.sentiment.subjectivity

            newEntry += [status['id'], status['created_at'], status['source'], status['text'], cleanText,
                         polarity, subjectivity, status['lang'], status['favorite_count'], status['retweet_count']]

            newEntry.append(status['user']['screen_name'])

            try:
                is_sensitive = status['possibly_sensitive']
            except KeyError:
                is_sensitive = None

            newEntry.append(is_sensitive)

            hashtags = ", ".join([hashtag_item['text'] for hashtag_item in status['entities']['hashtags']])
            newEntry.append(hashtags)  # append the hashtags

            #
            mentions = ", ".join([mention['screen_name'] for mention in status['entities']['user_mentions']])
            newEntry.append(mentions)  # append the user mentions

            try:
                xyz = status['place']['bounding_box']['coordinates']
                coordinates = [coord for loc in xyz for coord in loc]
            except TypeError:
                coordinates = None
            newEntry.append(coordinates)

            try:
                location = status['user']['location']
            except TypeError:
                location = ''
            newEntry.append(location)

            # now append a row to the dataframe
            single_tweet_df = pd.DataFrame([newEntry], columns=self.cols)
            df = df.append(single_tweet_df, ignore_index=True)

        df.to_csv("micTest12.csv", columns=self.cols, index=False, encoding="utf-8")
        return df

if __name__ == "__main__":

    keywords = ["#airquality", "#cleanair", "#airpollution", "#pollution", "#hvac", "#airpurifier", "#indoorairquality",
                "#climatechange", "#indoorair", "#environment", "#airconditioning", "#coronavirus", "#heating",
                "#ac", "#airfilter", "#allergies", "#hvacservice", "#ventilation", "#wellness", "#delhipollution",
                "#airconditioner", "#airqualityindex", "#bhfyp “particulate matter” “fine particulate matter”", "#air",
                "#pm2_5", "#emissions", "#natureishealing", "#nature", "#pollutionfree", "#wearethevirus", "#freshair",
                "#safety", "#covid", "#health"]
    startDate = datetime.datetime(2019, 4, 1, 0, 0, 0)
    endDate = datetime.datetime(2020, 8, 1, 0, 0, 0)
    geocodes = ["-33.918861,18.423300,40km", "-26.205681,28.046822,40km", "5.550000,-0.020000,40km",
                "-1.286389,36.817223,40km", "-4.043740,39.658871,40km", "6.465422,3.406448,40km",
                " -1.935114,30.082111,40km", "0.347596,32.582520,40km"]
    csvfile = "micTest12.csv"

    acquireTweets = dataAcq()
    for keyword in keywords:
        for geocode in geocodes:
            df = acquireTweets.getTweets(keyword=keyword, geocode=geocodes, startDate=startDate, endDate=endDate,
                                         csvfile=csvfile)

    print(df.head())


text:
#dataset iterator
#map pre-batch operations, stft etc
# chunker
#shuffle
# batch(batch_size)
"""
Example call:

export STORAGE_ROOT=<your desired storage root>
python -m padertorch.contrib.examples.voice_activity_detection.train
"""
import os
from pathlib import Path

import numpy as np
from paderbox.utils.timer import timeStamped
from paderbox.transform.module_stft import STFT
from padercontrib.database.fearless import Fearless
from padertorch import Trainer
from padertorch.contrib.examples.voice_activity_detection.model import SAD_Classifier
from padertorch.contrib.je.data.transforms import AudioReader, Normalizer, Collate
from padertorch.contrib.je.modules.conv import CNN1d, CNN2d
from padertorch.modules.fully_connected import fully_connected_stack
from padertorch.train.optimizer import Adam
import torch
from torch.nn import MaxPool2d
from torch.autograd import Variable
from paderbox.array import segment_axis
from einops import rearrange

storage_dir = str(
    Path(os.environ['STORAGE_ROOT']) / 'voice_activity' / timeStamped('')[1:]
)
os.makedirs(storage_dir, exist_ok=True)

DEBUG = False
DATA_TEST = False

def get_datasets():
    db = Fearless()
    train = db.get_dataset_train(subset='stream')
    validate = db.get_dataset_validation(subset='stream')

    def prepare_example(example):
        example['audio_path'] = example['audio_path']['observation']
        example['activity'] = db.get_activity(example)
        return example

    train = train.map(prepare_example)
    validate = validate.map(prepare_example)

    training_data = prepare_dataset(train, training=True)
    validation_data = prepare_dataset(validate, training=False)
    return training_data, validation_data

def prepare_dataset(dataset, training=False):
    batch_size = 8#24

    def chunker(example):
        """Split stream into 4s segments"""
        # 4s at 8kHz -> 32k samples
        chunk_length = 32000
        # chunk_count = int(example['num_samples']/chunk_length)
        # if DEBUG:
        #     chunk_count = min(chunk_count, 20)
        # chunks = []
        # for chunk_id in range(chunk_count):
        #     chunk = {'example_id': example['example_id']+'_'+str(chunk_id)}
        #     num_samples = chunk_length

        #     if chunk_id == chunk_count - 1:
        #         num_samples = example['num_samples'] - (chunk_count-1) * chunk_length

        #     start = chunk_id*chunk_length
        #     end = start + num_samples
        #     chunk.update(num_samples=num_samples)
        #     chunk.update(audio_start_samples=start)
        #     chunk.update(audio_stop_samples=end)
        #     chunk.update(audio_path=example['audio_path'])
        #     chunk.update(activity=example['activity'][start:end])
        #     chunks.append(chunk)
        #     np.random.shuffle(chunks)
        # return chunks
        start = max(0, np.random.randint(example['num_samples'])-chunk_length)
        stop = start + chunk_length
        example.update(num_samples=chunk_length)
        example.update(audio_start_samples=start)
        example.update(audio_stop_samples=stop)
        example.update(audio_path=example['audio_path'])
        example.update(activity=example['activity'][start:stop])
        return example

    def select_speech(example):
        """Cut out a section with speech for evaluation.

        We evaluate the model on 30s audio segments which contain speech."""
        first_speech = example['activity'].intervals[0][0]
        max_time_buffer = 8000 * 15 # 15s
        time_buffer = np.random.randint(max_time_buffer)
        length = 8000 * 30 # 30s
        start = max(0, first_speech-time_buffer)
        stop = start + length
        example['audio_start_samples'] = start
        example['audio_stop_samples'] = stop
        example['activity'] = example['activity'][start:stop]
        return example

    if training:
        dataset = dataset.shuffle(reshuffle=True)

    dataset = dataset.prefetch(
        num_workers=8, buffer_size=10*batch_size
    )

    if training:
        dataset = dataset.map(chunker)
    else:
        dataset = dataset.map(select_speech)

    audio_reader = AudioReader(
        source_sample_rate=8000, target_sample_rate=8000
    )
    dataset = dataset.map(audio_reader)

    STFT_SHIFT = 80
    STFT_WINDOW_LENGTH = 400
    STFT_SIZE = 512
    STFT_PAD = True

    stft = STFT(
        shift=STFT_SHIFT,
        size=STFT_SIZE,
        window_length=STFT_WINDOW_LENGTH,
        pad=STFT_PAD,
        fading='half'# was None
    )

    def segment(array):
        frames = int(array.shape[0]/STFT_SHIFT)
        output = np.zeros(frames)
        for i in range(frames):
            middle = i*STFT_SHIFT
            start = max(0, middle-STFT_WINDOW_LENGTH)
            stop = min(middle+STFT_WINDOW_LENGTH, array.shape[0]-1)
            output[i] = array[start: stop].any()
        return output

    def calculate_stft(example):
        complex_spectrum = stft(example['audio_data'].flatten())
        spectrum_magnitude = np.abs(complex_spectrum)**2
        real_magnitude = spectrum_magnitude.astype(np.float32)
        real_magnitude = real_magnitude[None, None, ...]
        example['features'] = rearrange(real_magnitude,
                                        'b c f t -> b c t f', b=1, c=1)[:, :, :-1, :]
        example['activity'] = segment(example['activity'])
        # example['activity'] = segment_axis(example['activity'],
        #                                    length=STFT_WINDOW_LENGTH,
        #                                    shift=STFT_SHIFT,
        #                                    end='pad' if STFT_PAD else 'cut'
        #                                    ).any(axis=-1)
        return example

    dataset = dataset.map(calculate_stft)

    def finalize(example):
        return {
            'example_id': example['example_id'],
            'features': Variable(torch.from_numpy(example['features'])),
            'seq_len': example['features'].shape[-1],
            'activity': example['activity'][:].astype(np.float32)
        }

    dataset = dataset.map(finalize)

    dataset = dataset.batch(batch_size).map(Collate(to_tensor=True))

    def unpack_tensor(batch):
        batch['features'] = Variable(torch.from_numpy(np.vstack(batch['features'])))
        return batch

    dataset = dataset.map(unpack_tensor)
    return dataset

def get_model():
    cnn2d = CNN2d(in_channels=1,
                  out_channels=2*[16] + 2*[32] + 2*[64],
                  kernel_size=3,
                  norm='batch',
                  pool_size=[1, (4, 1)] + 2*[1, (8, 1)])
    temporal_layer = CNN1d(in_channels=64,
                           out_channels=[128, 10],
                           kernel_size=3,
                           pool_size=1)
    # we cannot pool across the channels using CNN1d
    pooling = MaxPool2d(kernel_size=(10, 1))
    sigmoid = torch.nn.Sigmoid()
    model = SAD_Classifier(cnn2d, temporal_layer, pooling, sigmoid)
    return model

def train(model):
    train_set, validate_set = get_datasets()
    stop_trigger = 50000
    if DEBUG:
        stop_trigger = 5000
    trainer = Trainer(
        model=model,
        optimizer=Adam(lr=1e-3),
        storage_dir=str(storage_dir),
        summary_trigger=(100, 'iteration'),
        checkpoint_trigger=(1000, 'iteration'),
        stop_trigger=(stop_trigger, 'iteration')
    )
    trainer.register_validation_hook(validate_set)
    trainer.test_run(train_set, validate_set)
    trainer.train(train_set)

def main():
    if DATA_TEST:
        train_set, validate_set = get_datasets()
        element = train_set.__iter__().__next__()
        #element['activity'] = element['activity']
        model = get_model()
        print(element['features'].shape, element['activity'].shape)
        output = model.forward(element)
        print(output.shape)
        print(model.review(element, output))
        element = validate_set.__iter__().__next__()
        output = model.forward(element)
        print(output.shape, element['activity'].shape)
    else:
        model = get_model()
        train(model)

if __name__ == '__main__':
    main()

max_stars_repo_name: alexanderwerning/padertorch

max_issues_repo_name: alexanderwerning/padertorch

max_forks_repo_name: alexanderwerning/padertorch


text:
# -*- coding: utf-8 -*-

from io import StringIO

import numpy as np
import pandas as pd
import scipy.stats as stats
from scipy.interpolate import interp1d

def gumbel_r_(mean: float, sd: float, **_):

    # parameters Gumbel W&S
    alpha = 1.282 / sd
    u = mean - 0.5772 / alpha

    # parameters Gumbel scipy
    scale = 1 / alpha
    loc = u

    return dict(loc=loc, scale=scale)

def lognorm_(mean: float, sd: float, **_):

    cov = sd / mean

    sigma_ln = np.sqrt(np.log(1 + cov ** 2))
    miu_ln = np.log(mean) - 1 / 2 * sigma_ln ** 2

    s = sigma_ln
    loc = 0
    scale = np.exp(miu_ln)

    return dict(s=s, loc=loc, scale=scale)

def norm_(mean: float, sd: float, **_):

    loc = mean
    scale = sd

    return dict(loc=loc, scale=scale)

def uniform_(ubound: float, lbound: float, **_):

    if lbound > ubound:
        lbound += ubound
        ubound = lbound - ubound
        lbound -= ubound

    loc = lbound
    scale = ubound - lbound

    return dict(loc=loc, scale=scale)

def random_variable_generator(dict_in: dict, num_samples: int):
    """Generates samples of defined distribution. This is build upon scipy.stats library.

    :param dict_in:     distribution inputs, required keys are distribution dependent, should be align with inputs
                        required in the scipy.stats. Additional compulsory keys are:
                            `dist`: str, distribution type;
                            `ubound`: float, upper bound of the sampled values; and
                            `lbound`: float, lower bound of the sampled values.
    :param num_samples: number of samples to be generated.
    :return samples:    sampled values based upon `dist` in the range [`lbound`, `ubound`] with `num_samples` number of
                        values.
    """

    # assign distribution type
    dist_0 = dict_in["dist"]
    dist = dict_in["dist"]

    # assign distribution boundary (for samples)
    ubound = dict_in["ubound"]
    lbound = dict_in["lbound"]

    # sample CDF points (y-axis value)
    def generate_cfd_q(dist_, dist_kw_, lbound_, ubound_):
        cfd_q_ = np.linspace(
            getattr(stats, dist_).cdf(x=lbound_, **dist_kw_),
            getattr(stats, dist_).cdf(x=ubound_, **dist_kw_),
            num_samples,
        )
        samples_ = getattr(stats, dist_).ppf(q=cfd_q_, **dist_kw_)
        return samples_

    # convert human distribution parameters to scipy distribution parameters
    if dist_0 == "gumbel_r_":
        dist_kw = gumbel_r_(**dict_in)
        dist = "gumbel_r"
        samples = generate_cfd_q(
            dist_=dist, dist_kw_=dist_kw, lbound_=lbound, ubound_=ubound
        )
    elif dist_0 == "uniform_":
        dist_kw = uniform_(**dict_in)
        dist = "uniform"
        samples = generate_cfd_q(
            dist_=dist, dist_kw_=dist_kw, lbound_=lbound, ubound_=ubound
        )

    elif dist_0 == "norm_":
        dist_kw = norm_(**dict_in)
        dist = "norm"
        samples = generate_cfd_q(
            dist_=dist, dist_kw_=dist_kw, lbound_=lbound, ubound_=ubound
        )

    elif dist_0 == "lognorm_":
        dist_kw = lognorm_(**dict_in)
        dist = "lognorm"
        samples = generate_cfd_q(
            dist_=dist, dist_kw_=dist_kw, lbound_=lbound, ubound_=ubound
        )

    elif dist_0 == "lognorm_mod_":
        dist_kw = lognorm_(**dict_in)
        dist = "lognorm"
        samples = generate_cfd_q(
            dist_=dist, dist_kw_=dist_kw, lbound_=lbound, ubound_=ubound
        )
        samples = 1 - samples

    elif dist_0 == "constant_":
        # print(num_samples, lbound, ubound, np.average(lbound))
        samples = np.full((num_samples,), np.average([lbound, ubound]))

    else:
        try:
            dict_in.pop("dist")
            dict_in.pop("ubound")
            dict_in.pop("lbound")
            samples = generate_cfd_q(
                dist_=dist, dist_kw_=dict_in, lbound_=lbound, ubound_=ubound
            )
        except AttributeError:
            raise ValueError("Unknown distribution type {}.".format(dist))

    samples[samples == np.inf] = ubound
    samples[samples == -np.inf] = lbound

    if "permanent" in dict_in:
        samples += dict_in["permanent"]

    np.random.shuffle(samples)

    return samples

def dict_unflatten(dict_in: dict) -> dict:

    dict_out = dict()

    for k in list(dict_in.keys()):
        if ":" in k:
            k1, k2 = k.split(":")

            if k1 in dict_out:
                dict_out[k1][k2] = dict_in[k]
            else:
                dict_out[k1] = dict(k2=dict_in[k])

    return dict_out

def dict_flatten(dict_in: dict) -> dict:
    """Converts two levels dict to single level dict. Example input and output see _test_dict_flatten.
    :param dict_in: Any two levels (or less) dict.
    :return dict_out: Single level dict.
    """

    dict_out = dict()

    for k in list(dict_in.keys()):
        if isinstance(dict_in[k], dict):
            for kk, vv in dict_in[k].items():
                dict_out[f"{k}:{kk}"] = vv
        else:
            dict_out[k] = dict_in[k]

    return dict_out

def _test_dict_flatten():
    x = dict(A=dict(a=0, b=1), B=dict(c=2, d=3))
    y_expected = {"A:a": 0, "A:b": 1, "B:c": 2, "B:d": 3}
    y = dict_flatten(x)
    assert y == y_expected

def main(x: dict, num_samples: int) -> pd.DataFrame:
    """Generates samples based upon prescribed distribution types.

    :param x: description of distribution function.
    :param num_samples: number of samples to be produced.
    :return df_out:
    """

    dict_out = dict()

    for k, v in x.items():

        if isinstance(v, float) or isinstance(v, int) or isinstance(v, np.float):
            dict_out[k] = np.full((num_samples,), v, dtype=float)

        elif isinstance(v, str):
            dict_out[k] = np.full(
                (num_samples,), v, dtype=np.dtype("U{:d}".format(len(v)))
            )

        elif isinstance(v, np.ndarray) or isinstance(v, list):
            dict_out[k] = list(np.full((num_samples, len(v)), v, dtype=float))

        elif isinstance(v, dict):
            if "dist" in v:
                try:
                    dict_out[k] = random_variable_generator(v, num_samples)
                except KeyError:
                    raise ("Missing parameters in input variable {}.".format(k))
            elif "ramp" in v:
                s_ = StringIO(v["ramp"])
                d_ = pd.read_csv(
                    s_,
                    names=["x", "y"],
                    dtype=float,
                    skip_blank_lines=True,
                    skipinitialspace=True,
                )
                t_ = d_.iloc[:, 0]
                v_ = d_.iloc[:, 1]
                if all(v_ == v_[0]):
                    f_interp = v_[0]
                else:
                    f_interp = interp1d(t_, v_, bounds_error=False, fill_value=0)
                dict_out[k] = np.full((num_samples,), f_interp)
            else:
                raise ValueError("Unknown input data type for {}.".format(k))
        else:
            raise TypeError("Unknown input data type for {}.".format(k))

    dict_out["index"] = np.arange(0, num_samples, 1)

    df_out = pd.DataFrame.from_dict(dict_out, orient="columns")

    return df_out

def _test_random_variable_generator():
    x = dict(v=np.pi)
    y = main(x, 1000)
    assert len(y["v"].values) == 1000
    assert all([v == np.pi for v in y["v"].values])

    x = dict(v="hello world.")
    y = main(x, 1000)
    assert len(y["v"].values) == 1000
    assert all([v == "hello world." for v in y["v"].values])

    x = dict(v=[0.0, 1.0, 2.0])
    y = main(x, 1000)
    assert len(y["v"].values) == 1000
    assert all([all(v == np.array([0.0, 1.0, 2.0])) for v in y["v"].values])

    x = dict(v=dict(dist="uniform_", ubound=10, lbound=-1))
    y = main(x, 1000)
    assert len(y["v"].values) == 1000
    assert np.max(y["v"].values) == 10
    assert np.min(y["v"].values) == -1
    assert abs(np.mean(y["v"].values) - (10 - 1) / 2) <= 0.00001

    x = dict(v=dict(dist="norm_", ubound=5 + 1, lbound=5 - 1, mean=5, sd=1))
    y = main(x, 1000)
    assert len(y["v"].values) == 1000
    assert np.max(y["v"].values) == 6
    assert np.min(y["v"].values) == 4
    assert abs(np.mean(y["v"].values) - 5) <= 0.00001

    x = dict(v=dict(dist="gumbel_r_", ubound=2000, lbound=50, mean=420, sd=126))
    y = main(x, 1000)
    assert len(y["v"].values) == 1000
    assert abs(np.max(y["v"].values) - 2000) <= 1
    assert abs(np.min(y["v"].values) - 50) <= 1
    assert abs(np.mean(y["v"].values) - 420) <= 1

if __name__ == "__main__":
    _test_random_variable_generator()
    _test_dict_flatten()

max_stars_repo_name: fsepy/sfeprapy

max_issues_repo_name: fsepy/sfeprapy

max_forks_repo_name: fsepy/sfeprapy

reason: import numpy,import scipy,from scipy


text:
#!/usr/bin/env python3

# Copyright 2020 Songxiang Liu
# Apache 2.0

from typing import List

import torch
import torch.nn.functional as F

import numpy as np

from .utils.abs_model import AbsMelDecoder
from .rnn_decoder_mol import Decoder
from .utils.cnn_postnet import Postnet
from .utils.vc_utils import get_mask_from_lengths

from utils.load_yaml import HpsYaml

class MelDecoderMOLv2(AbsMelDecoder):
    """Use an encoder to preprocess ppg."""
    def __init__(
        self,
        num_speakers: int,
        spk_embed_dim: int,
        bottle_neck_feature_dim: int,
        encoder_dim: int = 256,
        encoder_downsample_rates: List = [2, 2],
        attention_rnn_dim: int = 512,
        decoder_rnn_dim: int = 512,
        num_decoder_rnn_layer: int = 1,
        concat_context_to_last: bool = True,
        prenet_dims: List = [256, 128],
        num_mixtures: int = 5,
        frames_per_step: int = 2,
        mask_padding: bool = True,
    ):
        super().__init__()

        self.mask_padding = mask_padding
        self.bottle_neck_feature_dim = bottle_neck_feature_dim
        self.num_mels = 80
        self.encoder_down_factor=np.cumprod(encoder_downsample_rates)[-1]
        self.frames_per_step = frames_per_step
        self.use_spk_dvec = True

        input_dim = bottle_neck_feature_dim

        # Downsampling convolution
        self.bnf_prenet = torch.nn.Sequential(
            torch.nn.Conv1d(input_dim, encoder_dim, kernel_size=1, bias=False),
            torch.nn.LeakyReLU(0.1),

            torch.nn.InstanceNorm1d(encoder_dim, affine=False),
            torch.nn.Conv1d(
                encoder_dim, encoder_dim,
                kernel_size=2*encoder_downsample_rates[0],
                stride=encoder_downsample_rates[0],
                padding=encoder_downsample_rates[0]//2,
            ),
            torch.nn.LeakyReLU(0.1),

            torch.nn.InstanceNorm1d(encoder_dim, affine=False),
            torch.nn.Conv1d(
                encoder_dim, encoder_dim,
                kernel_size=2*encoder_downsample_rates[1],
                stride=encoder_downsample_rates[1],
                padding=encoder_downsample_rates[1]//2,
            ),
            torch.nn.LeakyReLU(0.1),

            torch.nn.InstanceNorm1d(encoder_dim, affine=False),
        )
        decoder_enc_dim = encoder_dim
        self.pitch_convs = torch.nn.Sequential(
            torch.nn.Conv1d(2, encoder_dim, kernel_size=1, bias=False),
            torch.nn.LeakyReLU(0.1),

            torch.nn.InstanceNorm1d(encoder_dim, affine=False),
            torch.nn.Conv1d(
                encoder_dim, encoder_dim,
                kernel_size=2*encoder_downsample_rates[0],
                stride=encoder_downsample_rates[0],
                padding=encoder_downsample_rates[0]//2,
            ),
            torch.nn.LeakyReLU(0.1),

            torch.nn.InstanceNorm1d(encoder_dim, affine=False),
            torch.nn.Conv1d(
                encoder_dim, encoder_dim,
                kernel_size=2*encoder_downsample_rates[1],
                stride=encoder_downsample_rates[1],
                padding=encoder_downsample_rates[1]//2,
            ),
            torch.nn.LeakyReLU(0.1),

            torch.nn.InstanceNorm1d(encoder_dim, affine=False),
        )

        self.reduce_proj = torch.nn.Linear(encoder_dim + spk_embed_dim, encoder_dim)

        # Decoder
        self.decoder = Decoder(
            enc_dim=decoder_enc_dim,
            num_mels=self.num_mels,
            frames_per_step=frames_per_step,
            attention_rnn_dim=attention_rnn_dim,
            decoder_rnn_dim=decoder_rnn_dim,
            num_decoder_rnn_layer=num_decoder_rnn_layer,
            prenet_dims=prenet_dims,
            num_mixtures=num_mixtures,
            use_stop_tokens=True,
            concat_context_to_last=concat_context_to_last,
            encoder_down_factor=self.encoder_down_factor,
        )

        # Mel-Spec Postnet: some residual CNN layers
        self.postnet = Postnet()

    def parse_output(self, outputs, output_lengths=None):
        if self.mask_padding and output_lengths is not None:
            mask = ~get_mask_from_lengths(output_lengths, outputs[0].size(1))
            mask = mask.unsqueeze(2).expand(mask.size(0), mask.size(1), self.num_mels)
            outputs[0].data.masked_fill_(mask, 0.0)
            outputs[1].data.masked_fill_(mask, 0.0)
        return outputs

    def forward(
        self,
        bottle_neck_features: torch.Tensor,
        feature_lengths: torch.Tensor,
        speech: torch.Tensor,
        speech_lengths: torch.Tensor,
        logf0_uv: torch.Tensor = None,
        spembs: torch.Tensor = None,
        output_att_ws: bool = False,
    ):
        decoder_inputs = self.bnf_prenet(
            bottle_neck_features.transpose(1, 2)
        ).transpose(1, 2)
        logf0_uv = self.pitch_convs(logf0_uv.transpose(1, 2)).transpose(1, 2)
        decoder_inputs = decoder_inputs + logf0_uv

        assert spembs is not None
        spk_embeds = F.normalize(
            spembs).unsqueeze(1).expand(-1, decoder_inputs.size(1), -1)
        decoder_inputs = torch.cat([decoder_inputs, spk_embeds], dim=-1)
        decoder_inputs = self.reduce_proj(decoder_inputs)

        # (B, num_mels, T_dec)
        T_dec = torch.div(feature_lengths, int(self.encoder_down_factor), rounding_mode='floor')
        mel_outputs, predicted_stop, alignments = self.decoder(
            decoder_inputs, speech, T_dec)
        ## Post-processing
        mel_outputs_postnet = self.postnet(mel_outputs.transpose(1, 2)).transpose(1, 2)
        mel_outputs_postnet = mel_outputs + mel_outputs_postnet
        if output_att_ws:
            return self.parse_output(
                [mel_outputs, mel_outputs_postnet, predicted_stop, alignments], speech_lengths)
        else:
            return self.parse_output(
                [mel_outputs, mel_outputs_postnet, predicted_stop], speech_lengths)

        # return mel_outputs, mel_outputs_postnet

    def inference(
        self,
        bottle_neck_features: torch.Tensor,
        logf0_uv: torch.Tensor = None,
        spembs: torch.Tensor = None,
    ):
        decoder_inputs = self.bnf_prenet(bottle_neck_features.transpose(1, 2)).transpose(1, 2)
        logf0_uv = self.pitch_convs(logf0_uv.transpose(1, 2)).transpose(1, 2)
        decoder_inputs = decoder_inputs + logf0_uv

        assert spembs is not None
        spk_embeds = F.normalize(
            spembs).unsqueeze(1).expand(-1, decoder_inputs.size(1), -1)
        bottle_neck_features = torch.cat([decoder_inputs, spk_embeds], dim=-1)
        bottle_neck_features = self.reduce_proj(bottle_neck_features)

        ## Decoder
        if bottle_neck_features.size(0) > 1:
            mel_outputs, alignments = self.decoder.inference_batched(bottle_neck_features)
        else:
            mel_outputs, alignments = self.decoder.inference(bottle_neck_features,)
        ## Post-processing
        mel_outputs_postnet = self.postnet(mel_outputs.transpose(1, 2)).transpose(1, 2)
        mel_outputs_postnet = mel_outputs + mel_outputs_postnet
        # outputs = mel_outputs_postnet[0]

        return mel_outputs[0], mel_outputs_postnet[0], alignments[0]

def load_model(train_config, model_file, device=None):

    if device is None:
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    model_config = HpsYaml(train_config)
    ppg2mel_model = MelDecoderMOLv2(
        **model_config["model"]
    ).to(device)
    ckpt = torch.load(model_file, map_location=device)
    ppg2mel_model.load_state_dict(ckpt["model"])
    ppg2mel_model.eval()
    return ppg2mel_model

max_stars_repo_name: KnightNic/MockingBird

max_issues_repo_name: KnightNic/MockingBird

max_forks_repo_name: KnightNic/MockingBird


text:
# -*- coding:utf-8 -*-

import time
import collections
import numpy as np
import pandas as pd
import copy
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import LabelEncoder
from tensorflow.keras.utils import to_categorical

from .metainfo import CategoricalColumn, ContinuousColumn
from ..preprocessing import MultiLabelEncoder, MultiKBinsDiscretizer, DataFrameWrapper, LgbmLeavesEncoder, \
    CategorizeEncoder
from ..utils import dt_logging, consts
from . import deeptable

logger = dt_logging.get_logger()
from .config import ModelConfig

class AbstractPreprocessor:
    def __init__(self, config: ModelConfig):
        self.config = config
        self.labels_ = None
        self.task_ = None

    @property
    def pos_label(self):
        if self.labels_ is not None and len(self.labels_) == 2:
            return self.labels_[1]
        else:
            return None

    @property
    def labels(self):
        return self.labels_

    @property
    def task(self):
        return self.task_

    def fit_transform(self, X, y, copy_data=True):
        raise NotImplementedError

    def transform_X(self, X, copy_data=True):
        raise NotImplementedError

    def transform_y(self, y, copy_data=True):
        raise NotImplementedError

    def transform(self, X, y, copy_data=True):
        raise NotImplementedError

    def inverse_transform_y(self, y_indicator):
        raise NotImplementedError

    def get_categorical_columns(self):
        raise NotImplementedError

    def get_continuous_columns(self):
        raise NotImplementedError

    def save(self, filepath):
        raise NotImplementedError

    @staticmethod
    def load(filepath):
        raise NotImplementedError

class DefaultPreprocessor(AbstractPreprocessor):
    def __init__(self, config: ModelConfig):
        super().__init__(config)
        self.reset()

    def reset(self):
        self.metainfo = None
        self.categorical_columns = None
        self.continuous_columns = None
        self.y_lable_encoder = None
        self.X_transformers = collections.OrderedDict()

    def prepare_X(self, X):
        if not isinstance(X, pd.DataFrame):
            X = pd.DataFrame(X)
        if len(set(X.columns)) != len(list(X.columns)):
            cols = [item for item, count in collections.Counter(X.columns).items() if count > 1]
            raise ValueError(f'Columns with duplicate names in X: {cols}')
        if X.columns.dtype != 'object':
            X.columns = ['x_' + str(c) for c in X.columns]
            logger.warn(f"Column index of X has been converted: {X.columns}")
        return X

    def fit_transform(self, X, y, copy_data=True):
        start = time.time()
        self.reset()
        if X is None:
            raise ValueError(f'X cannot be none.')
        if y is None:
            raise ValueError(f'y cannot be none.')
        if len(X.shape) != 2:
            raise ValueError(f'X must be a 2D datasets.')
        if len(y.shape) != 1:
            raise ValueError(f'y must be a 1D datasets.')
        if X.shape[0] != y.shape[0]:
            raise ValueError(f"The number of samples of X and y must be the same. X.shape:{X.shape}, y.shape{y.shape}")
        if pd.Series(y).isnull().sum() > 0:
            raise ValueError("Missing values in y.")

        if copy:
            X = copy.deepcopy(X)
            y = copy.deepcopy(y)

        y = self.fit_transform_y(y)

        X = self.prepare_X(X)
        X = self.__prepare_features(X)
        if self.config.auto_imputation:
            X = self._imputation(X)
        if self.config.auto_encode_label:
            X = self._categorical_encoding(X)
        if self.config.auto_discrete:
            X = self._discretization(X)
        if self.config.apply_gbm_features and y is not None:
            X = self._apply_gbm_features(X, y)
        self.X_transformers['last'] = 'passthrough'

        print(f'fit_transform cost:{time.time() - start}')
        return X, y

    def fit_transform_y(self, y):
        self.task_, self.labels_ = deeptable.infer_task_type(y)
        if self.task_ in [consts.TASK_BINARY, consts.TASK_MULTICLASS]:
            self.y_lable_encoder = LabelEncoder()
            y = self.y_lable_encoder.fit_transform(y)
            self.labels_ = self.y_lable_encoder.classes_
        return y

    def transform(self, X, y, copy_data=True):
        X_t = self.transform_X(X, copy_data)
        y_t = self.transform_y(y, copy_data)
        return X_t, y_t

    def transform_y(self, y, copy_data=True):
        logger.info("Transform [y]...")
        start = time.time()
        if copy_data:
            y = copy.deepcopy(y)
        if self.y_lable_encoder is not None:
            y = self.y_lable_encoder.transform(y)
        print(f'transform_y cost:{time.time() - start}')
        y = np.array(y)
        return y

    def transform_X(self, X, copy_data=True):
        start = time.time()
        logger.info("Transform [X]...")
        if copy_data:
            X = copy.deepcopy(X)
        X = self.prepare_X(X)
        steps = (step for step in self.X_transformers.values())
        pipeline = make_pipeline(*steps)
        X_t = pipeline.transform(X)
        print(f'transform_X cost:{time.time() - start}')
        return X_t

    def inverse_transform_y(self, y_indicator):
        if self.y_lable_encoder is not None:
            return self.y_lable_encoder.inverse_transform(y_indicator)
        else:
            return y_indicator

    def __prepare_features(self, X):
        start = time.time()

        logger.info(f'Preparing features...')
        num_vars = []
        convert2cat_vars = []
        cat_vars = []
        excluded_vars = []

        if self.config.cat_exponent >= 1:
            raise ValueError(f'"cat_expoent" must be less than 1, not {self.config.cat_exponent} .')

        unique_upper_limit = round(X.shape[0] ** self.config.cat_exponent)
        for c in X.columns:
            nunique = X[c].nunique()
            dtype = str(X[c].dtype)

            if c in self.config.exclude_columns:
                excluded_vars.append((c, dtype, nunique))
                continue
            if self.config.categorical_columns is not None and isinstance(self.config.categorical_columns, list):
                if c in self.config.categorical_columns:
                    cat_vars.append((c, dtype, nunique))
                else:
                    if np.issubdtype(dtype, np.number):
                        num_vars.append((c, dtype, nunique))
                    else:
                        print(
                            f'Column [{c}] has been discarded. It is not numeric and not in [config.categorical_columns].')
            else:
                if dtype == 'object' or dtype == 'category' or dtype == 'bool':
                    cat_vars.append((c, dtype, nunique))
                elif self.config.auto_categorize and X[c].nunique() < unique_upper_limit:
                    convert2cat_vars.append((c, dtype, nunique))
                else:
                    num_vars.append((c, dtype, nunique))

        if len(convert2cat_vars) > 0:
            ce = CategorizeEncoder([c for c, d, n in convert2cat_vars], self.config.cat_remain_numeric)
            X = ce.fit_transform(X)
            self.X_transformers['categorize'] = ce
            if self.config.cat_remain_numeric:
                cat_vars = cat_vars + ce.new_columns
                num_vars = num_vars + convert2cat_vars
            else:
                cat_vars = cat_vars + convert2cat_vars

        logger.debug(f'{len(cat_vars)} categorical variables and {len(num_vars)} continuous variables found. '
                     f'{len(convert2cat_vars)} of them are from continuous to categorical.')

        self.__append_categorical_cols([(c[0], c[2] + 2) for c in cat_vars])
        self.__append_continuous_cols([c[0] for c in num_vars], consts.INPUT_PREFIX_NUM + 'all')
        print(f'Preparing features cost:{time.time() - start}')
        return X

    def _imputation(self, X):
        start = time.time()
        logger.info('Data imputation...')
        continuous_vars = self.get_continuous_columns()
        categorical_vars = self.get_categorical_columns()
        ct = ColumnTransformer([
            ('categorical', SimpleImputer(missing_values=np.nan, strategy='constant', fill_value='nan-fillvalue'),
             categorical_vars),
            ('continuous', SimpleImputer(missing_values=np.nan, strategy='mean'), continuous_vars),
        ])
        dfwrapper = DataFrameWrapper(ct, categorical_vars + continuous_vars)
        X = dfwrapper.fit_transform(X)
        self.X_transformers['imputation'] = dfwrapper
        print(f'Imputation cost:{time.time() - start}')
        return X

    def _categorical_encoding(self, X):
        start = time.time()
        logger.info('Categorical encoding...')
        vars = self.get_categorical_columns()
        mle = MultiLabelEncoder(vars)
        X = mle.fit_transform(X)
        self.X_transformers['label_encoder'] = mle
        print(f'Categorical encoding cost:{time.time() - start}')
        return X

    def _discretization(self, X):
        start = time.time()
        logger.info('Data discretization...')
        vars = self.get_continuous_columns()
        mkbd = MultiKBinsDiscretizer(vars)
        X = mkbd.fit_transform(X)
        self.__append_categorical_cols([(new_name, bins + 1) for name, new_name, bins in mkbd.new_columns])
        self.X_transformers['discreter'] = mkbd
        print(f'Discretization cost:{time.time() - start}')
        return X

    def _apply_gbm_features(self, X, y):
        start = time.time()
        logger.info('Extracting GBM features...')
        cont_vars = self.get_continuous_columns()
        cat_vars = self.get_categorical_columns()
        gbmencoder = LgbmLeavesEncoder(cat_vars, cont_vars, self.task_, **self.config.gbm_params)
        X = gbmencoder.fit_transform(X, y)
        self.X_transformers['gbm_features'] = gbmencoder
        if self.config.gbm_feature_type == consts.GBM_FEATURE_TYPE_EMB:
            self.__append_categorical_cols([(name, X[name].max() + 1) for name in gbmencoder.new_columns])
        else:
            self.__append_continuous_cols([name for name in gbmencoder.new_columns],
                                          consts.INPUT_PREFIX_NUM + 'gbm_leaves')
        print(f'Extracting gbm features cost:{time.time() - start}')
        return X

    def __append_categorical_cols(self, cols):
        logger.debug(f'{len(cols)} categorical variables appended.')

        if self.config.fixed_embedding_dim:
            embedding_output_dim = self.config.embeddings_output_dim if self.config.embeddings_output_dim > 0 else consts.EMBEDDING_OUT_DIM_DEFAULT
        else:
            embedding_output_dim = 0
            #

        if self.categorical_columns is None:
            self.categorical_columns = []
        self.categorical_columns = self.categorical_columns + \
                                   [CategoricalColumn(name,
                                                      voc_size,
                                                      embedding_output_dim
                                                      if embedding_output_dim > 0
                                                      else min(4 * int(pow(voc_size, 0.25)), 20))
                                    for name, voc_size in cols]

    def __append_continuous_cols(self, cols, input_name):
        if self.continuous_columns is None:
            self.continuous_columns = []
        self.continuous_columns = self.continuous_columns + [ContinuousColumn(name=input_name,
                                                                              column_names=[c for c in cols])]

    def get_categorical_columns(self):
        return [c.name for c in self.categorical_columns]

    def get_continuous_columns(self):
        cont_vars = []
        for c in self.continuous_columns:
            cont_vars = cont_vars + c.column_names
        return cont_vars


text:
import threading
import json
import numpy as np

class ReversiExecServer(threading.Thread):
    """
    执行从客户端发送来的请求
    """

    def __init__(self, client, address, reversi_ai):
        threading.Thread.__init__(self)
        self.client = client
        self.address = address
        self.reversi_ai = reversi_ai

    def __send(self, message):
        self.client.sendall(
            ('HTTP/1.1 200 OK\nAccess-Control-Allow-Origin: *\nContent-Type: application/json\n\n' + message).encode(
                encoding='utf-8'))

    def __pre_exec(self, request):
        """预处理请求头"""
        return request.strip('\r\n ').split('\n')[-1]

    def __exec(self, message):
        """处理请求（待填充）"""
        resp = {}
        try:
            requests = json.loads(message)
            resp['status'] = 0
            resp['message'] = "Hello World!"

            mode = requests['mode']  # 请求模式（当前棋盘状态/比赛过程序列）
            data = np.array(json.loads(requests['data'])) * requests['color']  # 棋盘数据

            if mode == 'board':
                action = self.reversi_ai.play(data)[0]
                resp['response'] = int(action)
            else:
                pass
        except Exception as e:
            resp['status'] = 1
            resp['message'] = "Exec error" + str(e)

        print('message: ', message)
        print('response: ', resp)
        print('--------------------')
        return json.dumps(resp)

    def run(self):
        request = self.client.recv(1024).decode(encoding='utf-8')
        message = self.__pre_exec(request)
        res = self.__exec(message)
        self.__send(res)
        self.client.close()