text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Transforms a mesh (points and vectorial/tensorial data) to new coordinates.
A frequent use case are 2D-slices in 3D to be transformed into x-y-coordinates
(pre-processing), running OGS and transforming back into x-y-z coordinates
(post-processing).

AUTOMATIC DETECTION
    runfile('vtu_trafo.py', args='-i example4preprocessing.vtu -o forward.vtu -s')
    runfile('vtu_trafo.py', args='-i forward.vtu -o backward.vtu -r')
MANUAL SLICE DEFINITION
    runfile('vtu_trafo.py', args='-i example4preprocessing.vtu -o forward.vtu -x 9200 -X 18000 -y 9000 -Y 20000')
    runfile('vtu_trafo.py', args='-i forward.vtu -o backward.vtu -x 9200 -X 18000 -y 9000 -Y 20000 -r')
EXAMPLE FOR TRAFO OF RESULTS
    runfile('vtu_trafo.py', args='-i example4postprocessing.vtu -o backward.vtu -x 9200 -X 18000 -y 9000 -Y 20000 -r')

TODO
    implement for further THM result types, possibly for input data too
        done Darcy velocity (point data)
"""
import sys
import vtk
from vtk.util.numpy_support import vtk_to_numpy
from vtk.util.numpy_support import numpy_to_vtk
import numpy as np
from scipy.spatial.transform import Rotation as R
import argparse

tested_vtk_version = "9.0.1"
vtutrafo_version = "0.0"
vector_names = {'DarcyVelocity'}   # list names of data that transforms like vectors
coordinate_init_value = 0.0   # to detect if not set or invalid
ORG_X="original_x"
ORG_Y="original_y"
ORG_Z="original_z"
stored_data_found = False

# parsing command line arguments

parser = argparse.ArgumentParser(
    description="Transforms mesh points and results between two orthonormal coordinate systems. Needed to run OpenGeoSys on 2D-slices from a 3D-model.",
    epilog="Tested with VTK "
    + tested_vtk_version
)

required = parser.add_argument_group('required arguments')
required.add_argument(
    "-i",
    "--input",
    default="",
    required=True,
    help='input file'
)
required.add_argument(
    "-o",
    "--output",
    default="",
    required=True,
    help="output file"
)

parser.add_argument(
    "-x",
    "--start-x",
    type=float,
    default=coordinate_init_value,
    help="start x coordinate of slice (default is zero, if none of slice coordinates is given, then automatic detection)"
)
parser.add_argument(
    "-X",
    "--end-x",
    type=float,
    default=coordinate_init_value,
    help="end x coordinate of slice (default is zero, if none of slice coordinates is given, then automatic detection)"
)
parser.add_argument(
    "-y",
    "--start-y",
    type=float,
    default=coordinate_init_value,
    help="start y coordinate of slice (default is zero, if none of slice coordinates is given, then automatic detection)"
)
parser.add_argument(
    "-Y",
    "--end-y",
    type=float,
    default=coordinate_init_value,
    help="end y coordinate of slice (default is zero, if none of slice coordinates is given, then automatic detection)"
)
parser.add_argument(
        "-z",
        "--set-2D-z",
        metavar="2D_Z",
        type=float,
        help="enforces given z-coordinate in 2D, except for reverse trafo with given original points (autodetect). Note that forward trafo ends in 2D and reverse trafo starts in 2D"
    )

mutex = parser.add_mutually_exclusive_group()
mutex.add_argument(
        "-r",
        "--reverse",
        action="store_true",
        help="reverse trafo"
    )

mutex.add_argument(
        "-s",
        "--store",
        action="store_true",
        help="store orginal data (only for direct trafo)"
    )

parser.add_argument('-v', '--version', action='version', version=vtutrafo_version)
args = parser.parse_args()

# check command line arguments
store_flag = args.store
reverse_flag = args.reverse
start_x = args.start_x
end_x = args.end_x
start_y = args.start_y
end_y = args.end_y
inputfile = args.input
outputfile = args.output

auto_detect = (start_x == end_x) and (start_y == end_y)

# Read file
reader = vtk.vtkXMLUnstructuredGridReader()
reader.SetFileName(inputfile)
reader.Update()
vtk_mesh = reader.GetOutput()
vtk_pdata = vtk_mesh.GetPointData()
vtk_cdata = vtk_mesh.GetCellData()
vtk_points = vtk_mesh.GetPoints()

print("{} Points".format(vtk_mesh.GetNumberOfPoints()))
pdata_array_names = []
for i in range(vtk_pdata.GetNumberOfArrays()):
    pdata_array_names.append(vtk_pdata.GetArrayName(i))
print("Point Data:", pdata_array_names)

print("{} Cells".format(vtk_mesh.GetNumberOfCells()))
cdata_array_names = []
for i in range(vtk_cdata.GetNumberOfArrays()):
    cdata_array_names.append(vtk_cdata.GetArrayName(i))
print("Cell Data: ", cdata_array_names)

points = vtk_to_numpy(vtk_points.GetData())
N_points = len(points)

# store original points
if args.store:
    print("store original point coordinates")
    r = vtk_pdata.AddArray(numpy_to_vtk(points[:, 0]))
    vtk_pdata.GetArray(r).SetName(ORG_X)
    r = vtk_pdata.AddArray(numpy_to_vtk(points[:, 1]))
    vtk_pdata.GetArray(r).SetName(ORG_Y)
    r = vtk_pdata.AddArray(numpy_to_vtk(points[:, 2]))
    vtk_pdata.GetArray(r).SetName(ORG_Z)

if (ORG_X in pdata_array_names) and (ORG_Y in pdata_array_names) and (ORG_Z in pdata_array_names):
    stored_data_found = True
    org_points = np.stack( [np.array(vtk_to_numpy(vtk_pdata.GetArray(ORG_X))),
                            np.array(vtk_to_numpy(vtk_pdata.GetArray(ORG_Y))),
                            np.array(vtk_to_numpy(vtk_pdata.GetArray(ORG_Z)))], axis=-1 )

###   DEFINE BASES   ###
global_ex = np.array([1, 0, 0])   # 3D coordinate system
global_ey = np.array([0, 1, 0])
global_ez = np.array([0, 0, 1])
global_base = np.matrix([global_ex, global_ey, global_ez])# note, base as row_vectors

# construct base from given slice direction or automatically if not given
if auto_detect:
    if reverse_flag:
        if stored_data_found:
            print("reading original points from " + inputfile)
            # vectors from point with minimum of a coordinate to point with maximum
            dP = np.array([ org_points[np.argmax(org_points[:,0])] - org_points[np.argmin(org_points[:,0])],
                            org_points[np.argmax(org_points[:,1])] - org_points[np.argmin(org_points[:,1])],
                            org_points[np.argmax(org_points[:,2])] - org_points[np.argmin(org_points[:,2])] ])
        else:
            print("WARNING! Automatic detection for reverse trafo, but no stored data found. No output written.")
            sys.exit()
    else:
        print("detecting slice orientation automatically")
    # vectors from point with minimum of a coordinate to point with maximum
        dP = np.array([ points[np.argmax(points[:,0])] - points[np.argmin(points[:,0])],
                        points[np.argmax(points[:,1])] - points[np.argmin(points[:,1])],
                        points[np.argmax(points[:,2])] - points[np.argmin(points[:,2])] ])
    dPnorm = np.linalg.norm(dP, axis=-1)
    dP_sorted = dP[np.argsort(dPnorm)]
    local_Ex = dP_sorted[-1]   # 2D coordinate system
    local_EY = dP_sorted[-2]
else:
    local_Ex = np.array([end_x-start_x, end_y-start_y, 0.0])   # 2D coordinate system
    local_EY = np.array([0.0, 0.0, 1.0])

local_ex = local_Ex / np.linalg.norm(local_Ex)    # unit vector
local_Ey = local_EY - np.dot(local_EY, local_ex)*local_ex   # make orthogonal
local_ey = local_Ey / np.linalg.norm(local_Ey)   # unit vector
local_ez = np.cross(local_ex, local_ey)
local_base = np.matrix([local_ex, local_ey, local_ez])

###   GENERATE TRANSFORMATION   ###
# my way (to verify scipy way)
#myRmatrix = global_base*local_base.I
#myR = R.from_matrix(myRmatrix)

# scipy way
spR, spMeanError = R.align_vectors(global_base, local_base)
#spRmatrix = spR.as_matrix()

###   APPLY TRANSFORMATION   ###

# transform mesh points (nodes)
if reverse_flag and auto_detect:
    points = org_points
else:
    if (args.set_2D_z is not None) and reverse_flag:
        points[:,2] = args.set_2D_z   # set z coordinate in 2D, i.e. before trafo
    points = spR.apply(points, inverse=reverse_flag)
    if (args.set_2D_z is not None) and not reverse_flag:
        points[:,2] = args.set_2D_z   # set z coordinate in 2D, i.e. after trafo
vtk_points.SetData(numpy_to_vtk(points))

if stored_data_found:
    vtk_pdata.RemoveArray(ORG_X)
    vtk_pdata.RemoveArray(ORG_Y)
    vtk_pdata.RemoveArray(ORG_Z)

# transform point data
for pdata_array_name in pdata_array_names:
    if pdata_array_name in vector_names:
        vtk_pdata_array = vtk_pdata.GetArray(pdata_array_name)
        pdata_array = vtk_to_numpy(vtk_pdata_array)
        rows, cols = pdata_array.shape
        if cols == 2:   # append zero z-component
            pdata_array = spR.apply( np.concatenate((pdata_array, np.zeros((rows,1))), axis=1),
                    inverse=reverse_flag)
            print(pdata_array_name + " augmented (2D to 3D) and transformed")
            vtk_pdata.RemoveArray(pdata_array_name)
            r = vtk_pdata.AddArray(numpy_to_vtk(pdata_array))
            vtk_pdata.GetArray(r).SetName(pdata_array_name)
        elif cols == 3:
            pdata_array = spR.apply(pdata_array, inverse=reverse_flag)
            print(pdata_array_name + " transformed")
            vtk_pdata.RemoveArray(pdata_array_name)
            r = vtk_pdata.AddArray(numpy_to_vtk(pdata_array))
            vtk_pdata.GetArray(r).SetName(pdata_array_name)
        else:
            print("Not a vector: ", pdata_array_name)

# write file
writer = vtk.vtkXMLUnstructuredGridWriter()
writer.SetFileName(outputfile)
writer.SetInputData(vtk_mesh)
writer.Write()

max_stars_repo_name: dominik-kern/VTUinterface

max_issues_repo_name: dominik-kern/VTUinterface

max_forks_repo_name: dominik-kern/VTUinterface

reason: import numpy,from scipy


text:
import flappy_headless
import numpy as np
import sys
from collections import defaultdict
import pickle

rewardAlive = 1
rewardKill = -10000
alpha = 0.1
gamma = 1

Q = defaultdict(lambda:[0,0])  # Q["stateKey"] = (rewardJump, rewardNotJump)
#Q = defaultdict(lambda:(np.random.rand(),np.random.rand()))  # Q["stateKey"] = (rewardJump, rewardNotJump)

#sys.exit()

def paramsToState(params):
    playerVelY = params["playerVelY"]
    playery = params["playery"]

    if int(params["upperPipes"][0]["x"]) < 40:
        index = 1
    else:
        index = 0

    upperPipex = round(int(params["upperPipes"][index]["x"]) / 5) * 5
    upperPipey = int(params["upperPipes"][index]["y"])

    yDiff = round((playery - upperPipey) / 5) * 5

    state = str(playerVelY) + "_" + \
            str(yDiff) + "_" + \
            str(upperPipex)
    return state

oldState = None
oldAction = None
gameCounter = 0
gameScores = []

def shouldEmulateKeyPress(params):
    global oldState
    global oldAction

    state = paramsToState(params)
    estReward = Q[state]
    prevReward = Q[oldState]

    index = None
    if oldAction:
        index = 1
    else:
        index = 0

    prevReward[index] = (1 - alpha) * prevReward[index] + alpha * (rewardAlive + gamma * max(estReward))
    Q[oldState] = prevReward
    oldState = state

    #print("Estimated Reward: ", estReward)

    if estReward[0] >= estReward[1]:
        #print("Dont jump")
        oldAction = False
        return False
    else:
        #print("Jump")
        oldAction = True
        return True

def onGameOver(gameInfo):
    global oldState
    global oldAction
    global gameCounter
    global gameScores

    prevReward = Q[oldState]

    index = None
    if oldAction:
        index = 1
    else:
        index = 0

    prevReward[index] = (1 - alpha) * prevReward[index] + alpha * rewardKill
    Q[oldState] = prevReward
    #print("_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-\n\n",Q,"\n\n_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-")

    gameScores.append(gameInfo["score"])

    #print("Game Counter: ", gameCounter,"Score:", gameInfo["score"])

    if gameCounter % 100 == 0:
        print(str(gameCounter) + ": " + str(np.mean(gameScores[-100:])))

    oldState = None
    oldAction = None
    gameCounter += 1

    if gameCounter % 10000 == 0:
        with open("Q\\" + str(gameCounter) + ".pickle", "wb") as file:
            pickle.dump(dict(Q), file)

flappy_headless.main(shouldEmulateKeyPress, onGameOver)

ext: py

reason: import numpy


text:
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.decomposition import PCA
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score

# This script executes a series of preprocessing steps and finally performs a Principal Component Analysis to reduce the
# size of the data set. Moreover, it splits the data in training and test set and stores them in 2 separate files,
# 'Train.csv' and 'Test.csv'. Finally, it calculates the accuracy score of a KNN algorithm to estimate the loss of
# accuracy that was introduced by the dimensionality reduction

seed = np.random.seed = 7    # Setting the seed to ensure reproducibility
scaler = MinMaxScaler()      # The scaling approach that will be used depends on this scaler parameter
digits = pd.read_csv('digit_recognizer_dataset.csv')
stats = digits.describe().loc[['mean', 'std'], :]

# Discard non - informative digits
white_digits = []   # List to put digits that are always white
useful_digits = []  # List for the rest of the digits

for index, column in enumerate(digits.columns.tolist()):
    if stats.loc['mean', column] == 0 and stats.loc['std', column] == 0:
        white_digits.append(column)
    else:
        useful_digits.append(column)

final_digits = digits[useful_digits]
final_digits = final_digits.drop(labels='label', axis=1)
labels = digits['label']

# Train Test split
X_train, X_test, y_train, y_test = train_test_split(final_digits, labels, test_size=0.2,
                                                    random_state=seed, stratify=labels)
# Scaling of Numeric Features Before PCA
scaler.fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)

# PCA
pca = PCA(.95)   # Keeping the components that explain 95% of the variance
pca.fit(X_train_scaled)
print(pca.n_components_)
X_train_scaled = pca.transform(X_train_scaled)
X_test_scaled = pca.transform(X_test_scaled)

# Scaling of Numeric Features After PCA
scaler.fit(X_train_scaled)
X_train_rescaled = scaler.transform(X_train_scaled)
X_test_rescaled = scaler.transform(X_test_scaled)

# Write the Training Data to a CSV file
df_X_train = pd.DataFrame(data=X_train_rescaled)
y_train = y_train.reset_index(drop=True)
df_y_train = pd.DataFrame(data=y_train)
finalDf_train = pd.concat([df_X_train, df_y_train], axis=1, ignore_index=True)
finalDf_train.to_csv('Train.csv', index=False)

# Write the Test data to a CSV file
df_X_test = pd.DataFrame(data=X_test_rescaled)
y_test = y_test.reset_index(drop=True)
df_y_test = pd.DataFrame(data=y_test)
finalDf_test = pd.concat([df_X_test, df_y_test], axis=1, ignore_index=True)
finalDf_test.to_csv('Test.csv', index=False)

# Making predictions with a KNN to estimate accuracy drop due to PCA
X = finalDf_train
y = X[X.columns[-1]]
X = X.drop(columns=X.columns[-1], axis=1)

# Train/Validation split
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2,
                                                  random_state=seed, stratify=y)

# "Training" KNN
knn_clf = KNeighborsClassifier(n_neighbors=3)
knn_clf.fit(X_train, y_train)

# Predictions
y_knn_pred = knn_clf.predict(X_train)
print('Training Accuracy')
print(accuracy_score(y_train, y_knn_pred))
print('--------------------------------------')

y_knn_pred = knn_clf.predict(X_val)
print('Validation Accuracy')
print(accuracy_score(y_val, y_knn_pred))
print('--------------------------------------')

max_stars_repo_name: TheoEfthymiadis/Digit-Recognition-MNIST

max_issues_repo_name: TheoEfthymiadis/Digit-Recognition-MNIST

max_forks_repo_name: TheoEfthymiadis/Digit-Recognition-MNIST


text:
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the PyMVPA package for the
#   copyright and license terms.
#
### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
'''
AFNI NIML dataset I/O support.
Usually this type of datasets are used for functional data (timeseries,
preprocessed data), statistical maps or searchlight results.

Created on Feb 19, 2012

@author: Nikolaas. N. Oosterhof (nikolaas.oosterhof@unitn.it)

Files that are read with the afni_niml_dset.read function contain a dict
with the following fields:

   .data           PxN data for P nodes and N columns (values per node).
   .node_indices   P indices of P nodes that data refers to (base 0)
   .history        String with history information
   .stats          list with statistical information for each column.
   .labels         list with labels of the data columns
   .dset_type      String with the data set type

Similarly, such a dict can be saved to a .niml.dset file using the
afni_niml_dset.write function
'''

import random, numpy as np, os, time, sys, socket
import afni_niml_types as types
import afni_niml as niml

def warning(x): print x, debug

def _string2list(s, SEP=";", warn_if_no_string=True):
    '''splits a string by SEP; if the last element is empty then it is not returned

    The rationale is that AFNI/NIML like to close the string with a ';' which
    would return one (empty) value too many

    If the input is already a list that has lists or tuples,
    by default a warning is thrown because SUMA may puke over it.
    '''
    if isinstance(s, (list, tuple)):
        if warn_if_no_string and \
                    any(isinstance(s_elem, (list, tuple)) for s_elem in s):
            # a short representation of the offending structure
            s_str = '%r' % s if len(s) <= 1 else '[%s ... %s]' % (s[0], s[-1])
            warning('Two-dimensional string structure %s found - '
                        'it may not be readable by SUMA.' % s_str)
        return s

    r = s.split(SEP)
    if not r[-1]:
        r = r[:-1]
    return r

def rawniml2dset(p):
    if type(p) is list:
        return map(rawniml2dset, p)

    assert type(p) is dict and all([f in p for f in ['dset_type', 'nodes']]), p
    #assert type(p) is dict and all([f in p for f in ['nodes']]), p

    r = dict()
    r['dset_type'] = p['dset_type']

    for node in p['nodes']:
        assert 'name' in node

        name = node['name']
        data = node.get('data', None)

        if name == 'INDEX_LIST':
            r['node_indices'] = data
        elif name == 'SPARSE_DATA':
            r['data'] = data
        elif name == 'AFNI_atr':
            atr = node['atr_name']

            if atr == 'HISTORY_NOTE':
                r['history'] = data
            elif atr == 'COLMS_STATSYM':
                r['stats'] = _string2list(data)
            elif atr == 'COLMS_LABS':
                r['labels'] = _string2list(data)
        else:
            r[name] = data
            #raise ValueError("Unexpected node %s" % name)

    return r

def _dset2rawniml_header(s):
    r = dict()

    # set dataset type, default is Node_Bucket
    r['dset_type'] = s.get('dset_type', 'Node_Bucket')

    # make a new id code of 24 characters, all uppercase letters
    r['self_idcode'] = niml.getnewidcode()
    r['filename'] = s.get('filename', 'null')
    r['label'] = r['filename']
    r['name'] = 'AFNI_dataset'
    r['ni_form'] = 'ni_group'

    return r

def _dset2rawniml_data(s):
    return dict(data_type='Node_Bucket_data',
                name='SPARSE_DATA',
                data=s['data'])

def _dset_nrows_ncols(s):
    if type(s) is dict and 'data' in s:
        data = s['data']
    else:
        data = s

    if isinstance(data, np.ndarray):
        sh = s['data'].shape
        nrows = sh[0]
        ncols = 1 if len(sh) == 1 else sh[1]
    elif isinstance(data, list):
        lengths = set([len(d) for d in data])
        if len(lengths) != 1:
            raise ValueError('nonmatching lengths', lengths)
        nrows = lengths.pop()
        ncols = len(data)
    else:
        raise ValueError('not understood: %s' % data)

    return nrows, ncols

def _dset2rawniml_nodeidxs(s):
    nrows, _ = _dset_nrows_ncols(s)

    node_idxs = s.get('node_indices') if 'node_indices' in s else np.arange(nrows, dtype=np.int32)

    if not node_idxs is None:
        if not type(node_idxs) is np.ndarray:
            node_idxs = np.asarray(node_idxs, dtype=np.int32)

        if node_idxs.size != nrows:
            raise ValueError("Size mismatch for node indices (%r) and data (%r)" %
                             (node_idxs.size, nrows))

        if node_idxs.shape != (nrows, 1):
            node_idxs = np.reshape(node_idxs, ((nrows, 1))) # reshape to column vector if necessary

    def is_sorted(v): # O(1) in best case and O(n) in worst case (unlike sorted())
        if v is None:
            return None
        n = len(v)
        return n == 0 or all(v[i] <= v[i + 1] for i in xrange(n - 1))

    return dict(data_type='Node_Bucket_node_indices',
                name='INDEX_LIST',
                data=node_idxs,
                sorted_node_def='Yes' if is_sorted(node_idxs) else 'No')

def _dset2rawniml_datarange(s):
    data = s['data']

    try:
        minpos = np.argmin(data, axis=0)
        maxpos = np.argmax(data, axis=0)

        f = types.numpy_data2printer(data) # formatter function
        r = []
        for i in xrange(len(minpos)):
            mnpos = minpos[i]
            mxpos = maxpos[i]
            r.append('%s %s %d %d' % (f(data[mnpos, i]), f(data[mxpos, i]), mnpos, mxpos))

        # range of data in each column
        return dict(atr_name='COLMS_RANGE',
                    data=r)
    except:
        return dict(atr_name='COLMS_RANGE',
                    data=None)

def _dset2rawniml_labels(s):
    _, ncols = _dset_nrows_ncols(s)

    labels = s.get('labels', None)
    if labels is None:
        labels = ['col_%d' % i for i in xrange(ncols)]
    elif type(labels) != list:
        labels = list(labels)
    if len(labels) != ncols:
        raise ValueError("Wrong number of labels (%s): found %d but expected %d" %
                         (labels, len(labels), ncols))
    return dict(atr_name='COLMS_LABS',
                data=labels)

def _dset2rawniml_history(s):
    try:
        logprefix = ('[%s@%s: %s]' % (os.environ.get('USER', 'UNKNOWN'),
                                      socket.gethostname(),
                                      time.asctime()))
    except:
        logprefix = ''
    # history
    history = s.get('history', '')
    if history and not history.endswith('\n'):
        history += ('\n')
    history += '%s Saved by %s:%s' % (logprefix,
                                    __file__,
                                    sys._getframe().f_code.co_name)

    return dict(atr_name='HISTORY_NOTE',
                data=history)

def _dset2rawniml_datatypes(s):
    data = s['data']
    _, ncols = _dset_nrows_ncols(s)
    # XXX does not support mixed types
    datatype = ['Generic_Int' if types.numpy_data_isint(data) else 'Generic_Float'] * ncols
    return dict(atr_name='COLMS_TYPE',
                data=datatype)

def _dset2rawniml_stats(s):
    data = s['data']
    _, ncols = _dset_nrows_ncols(s)
    stats = s.get('stats', None)

    if stats is None:
        stats = ['none'] * ncols
    return dict(atr_name='COLMS_STATSYM',
                data=stats)

def _dset2rawniml_anything_else(s):
    ignore_keys = ['data', 'stats', 'labels', 'history', 'dset_type', 'node_indices']

    ks = s.keys()
    niml = []
    for k in ks:
        if k in ignore_keys:
            continue
        niml_elem = dict(data=s[k], name=k)

        try:
            niml.append(_dset2rawniml_complete(niml_elem))
        except TypeError:
            debug('NIML', 'Warning: unable to convert value for key %s' % k)

    return niml

def _dset2rawniml_complete(r):
    '''adds any missing information and ensures data is formatted properly'''

    # if data is a list of strings, join it and store it as a string
    # otherwise leave data untouched
    if types.numpy_data_isstring(r['data']):
        r['data'] = list(r['data'])

    while True:
        data = r['data']
        tp = type(data)

        if types.numpy_data_isstring(r['data']):
            r['data'] = list(r['data'])

        elif tp is list:
            if all(isinstance(d, basestring)  for d in data):
                r['data'] = ";".join(data)
            else:
                tp = 'mixed'
                break

        else:
            break # we're done

    if tp == 'mixed':
        #data = [types.nimldataassupporteddtype(d) for d in data]
        #r['data'] = data

        nrows, ncols = _dset_nrows_ncols(data)
        r['ni_dimen'] = str(nrows)
        tpstrs = []
        for d in data:
            if isinstance(d, basestring) or \
                    (type(d) is list and
                            all(isinstance(di, basestring) for di in d)):
                tpstr = 'String'
            elif isinstance(d, np.ndarray):
                tpstr = types.numpy_type2name(d.dtype)
            else:
                raise ValueError('unrecognized type %s' % type(d))
            tpstrs.append(tpstr)
        r['ni_type'] = ','.join(tpstrs)

    elif issubclass(tp, basestring):
        r['ni_dimen'] = '1'
        r['ni_type'] = 'String'
    elif tp is np.ndarray:
        data = types.nimldataassupporteddtype(data)
        if len(data.shape) == 1:
            data = np.reshape(data, (data.shape[0], 1))

        r['data'] = data # ensure we store a supported type

        nrows, ncols = data.shape
        r['ni_dimen'] = str(nrows)
        tpstr = types.numpy_type2name(data.dtype)
        r['ni_type'] = '%d*%s' % (ncols, tpstr) if nrows > 1 else tpstr
    elif not data is None:
        raise TypeError('Illegal type %r in %r' % (tp, data))

    if not 'name' in r:
        r['name'] = 'AFNI_atr'

    return r

def _remove_empty_nodes(nodes):
    tp = type(nodes)
    if tp is list:
        i = 0
        while i < len(nodes):
            node = nodes[i]
            if type(node) is dict and 'data' in node and node['data'] is None:
                nodes.pop(i)
            else:
                i += 1
    elif tp is dict:
        for v in nodes.itervalues():
            _remove_empty_nodes(v)

def dset2rawniml(s):
    if type(s) is list:
        return map(dset2rawniml, s)
    elif type(s) is np.ndarray:
        s = dict(data=s)

    if not 'data' in s:
        raise ValueError('No data?')

    r = _dset2rawniml_header(s)
    builders = [_dset2rawniml_data,
              _dset2rawniml_nodeidxs,
              _dset2rawniml_labels,
              _dset2rawniml_datarange,
              _dset2rawniml_history,
              _dset2rawniml_datatypes,
              _dset2rawniml_stats]

    nodes = [_dset2rawniml_complete(build(s)) for build in builders]
    _remove_empty_nodes(nodes)

    more_nodes = filter(lambda x:not x is None, _dset2rawniml_anything_else(s))

    r['nodes'] = nodes + more_nodes
    return r

def read(fn, itemifsingletonlist=True):
    return niml.read(fn, itemifsingletonlist, rawniml2dset)

def write(fnout, dset, form='binary'):
    fn = os.path.split(fnout)[1]

    if not type(fn) is str:
        if not isinstance(fnout, basestring):
            raise ValueError("Filename %s should be string" % str)
        fn = str(fn) # ensure that unicode is converted to string

    dset['filename'] = fn
    niml.write(fnout, dset, form, dset2rawniml)

def sparse2full(dset, pad_to_ico_ld=None, pad_to_node=None,
                ico_ld_surface_count=1, set_missing_values=0):
    '''
    Creates a 'full' dataset which has values associated with all nodes

    Parameters
    ----------
    dset: dict
        afni_niml_dset-like dictionary with at least a field 'data'
    pad_to_node_ico_ld: int
        number of linear divisions (only applicable if used through
        AFNI's MapIcosehedron) of the surface this dataset refers to.
    pad_to_node: int
        number of nodes of the surface this data
    ico_ld_surface_count: int (default: 1)
        if pad_to_ico_ld is set, this sets the number of surfaces that
        were origingally used. The typical use case is using a 'merged'
        surface originally based on a left and right hemisphere
    set_missing_values: int or float (default: 0)
        value to which nodes not present in dset are set.

    Returns
    -------
    dset: dict
        afni_niml_dset-like dictionary with at least fields 'data' and
        'node_indices'.
    '''

    if not pad_to_ico_ld is None:
        if pad_to_node:
            raise ValueError("Cannot have both ico_ld and pad_to_node")
        pad_to_node = ico_ld_surface_count * (pad_to_ico_ld ** 2 * 10 + 2)
    else:
        if pad_to_node is None:
            raise ValueError("Need either pad_to_ico_ld or pad_to_node")

    data = dset['data']
    nrows, ncols = data.shape

    node_indices = dset.get('node_indices', np.reshape(np.arange(nrows),
                                                            (-1, 1)))

    # a few sanity checks
    n = len(node_indices)

    if nrows != n:
        raise ValueError('element count mismatch between data (%d) and '
                         'node indices (%d)' % (nrows, n))

    if n > pad_to_node:
        raise ValueError('data has more rows (%d) than pad_to_node (%d)',
                                                (n, pad_to_node))

    full_node_indices_vec = np.arange(pad_to_node)
    full_node_indices = np.reshape(full_node_indices_vec, (pad_to_node, 1))

    full_data = np.zeros((pad_to_node, ncols), dtype=data.dtype) + \
                                                        set_missing_values
    full_data[np.reshape(node_indices, (n,)), :] = data[:, :]

    fulldset = dict(dset) # make a (superficial) copy
    fulldset['data'] = full_data
    fulldset['node_indices'] = full_node_indices

    return fulldset

def from_any(s, itemifsingletonlist=True):
    if isinstance(s, dict) and 'data' in s:
        return s
    elif isinstance(s, basestring):
        return read(s, itemifsingletonlist)
    elif isinstance(s, np.ndarray):
        return dict(data=s)
    else:
        raise ValueError('not recognized input: %r' % s)

def label2index(dset, label):
    if type(label) is list:
        return [label2index(dset, x) for x in label]

    if type(label) is int:
        sh = dset['data'].shape
        if label < 0 or label >= sh[1]:
            raise ValueError('label index %d out of bounds (0.. %d)' %
                                    (label, sh[1]))
        return label

    labels = dset.get('labels', None)
    if labels is None:
        raise ValueError('No labels found')

    for i, k in enumerate(labels):
        if k == label:
            return i

    return None

def ttest(dsets, sa_labels=None, return_values='mt',
          set_NaN_to=0., compare_to=0.):
    '''Runs a one-sample t-test across datasets

    Parameters
    ----------
    dsets: str or list of dicts
        (filenames of) NIML dsets, each referring to PxQ data for
        P nodes (features) and Q values per node (samples)
    sa_labels: list of (int or str)
        indices or labels of columns to compare
    return_values: str (default: 'mt')
        'm' or 't' or 'mt' to return sample mean, t-value, or both
    set_NaN_to: float or None (default: 0.)
        the value that NaNs in dsets replaced by. If None then NaNs are kept.
    compare_to: float (default: 0.)
        t-tests are compared against the null hypothesis of a mean of
        compare_to.

    Returns
    -------
    dset: dict
        NIML dset-compatible dict with fields 'data', 'labels',
        'stats' and 'node_indices' set.
    '''

    do_m = 'm' in return_values
    do_t = 't' in return_values

    if not (do_m or do_t):
        raise ValueError("Have to return at least m or t")

    ns = len(dsets)

    for i, dset in enumerate(dsets):
        dset = from_any(dset)
        dset_data = dset['data']
        if i == 0:
            sh = dset_data.shape
            if sa_labels is None:
                if 'labels' in dset:
                    sa_labels = dset['labels']
                    dset_labels = sa_labels
                else:
                    sa_labels = range(sh[1])
                    dset_labels = ['%d' % j for j in sa_labels]
            else:
                dset_labels = sa_labels

            nc = len(dset_labels) if dset_labels else sh[1]
            nn = sh[0]

            data = np.zeros((nn, nc, ns), dset_data.dtype) # number of nodes, columns, subjects

        if 'node_indices' in dset:
            node_idxs = np.reshape(dset['node_indices'], (-1,))
        else:
            node_idxs = np.arange(nn)

        if i == 0:
            node_idxs0 = node_idxs
        else:
            if set(node_idxs0) != set(node_idxs):
                raise ValueError("non-matching node indices for %d and %d" %
                                    (0, i))

        col_idxs = np.asarray(label2index(dset, sa_labels))

        data[node_idxs, :, i] = dset_data[:, col_idxs]

    # subtract the value it is compared to
    # so that it now tests against a mean of zero
    if do_m:
        m = np.mean(data, axis=2)

    if do_t:
        from scipy import stats
        t = stats.ttest_1samp(data - compare_to, 0., axis=2)[0]

    if do_m and do_t:
        r = np.zeros((nn, 2 * nc), dtype=m.dtype)
        r[:, np.arange(0, 2 * nc, 2)] = m
        r[:, np.arange(1, 2 * nc, 2)] = t
    elif do_t:
        r = t
    elif do_m:
        r = m

    pf = []
    stats = []
    if do_m:
        pf.append('m')
        stats.append('None')
    if do_t:
        pf.append('t')
        stats.append('Ttest(%d)' % (ns - 1))

    labs = sum([['%s_%s' % (p, lab) for p in pf] for lab in dset_labels], [])
    stats = stats * nc

    if not set_NaN_to is None:
        r[np.logical_not(np.isfinite(r))] = set_NaN_to

    return dict(data=r, labels=labs, stats=stats, node_indices=node_idxs0)

max_stars_repo_name: HaiyangJin/surfing

max_issues_repo_name: HaiyangJin/surfing

max_forks_repo_name: HaiyangJin/surfing

reason: from scipy


text:
from datetime import date
from unittest import TestCase, skip

import numpy as np
import dask.array as da

from axisutilities import Axis, AxisRemapper, DailyTimeAxisBuilder, WeeklyTimeAxisBuilder, \
    RollingWindowTimeAxisBuilder, MonthlyTimeAxisBuilder

class TestTimeAxisConverter(TestCase):
    @classmethod
    def setUpClass(cls) -> None:
        import os
        os.environ['TZ'] = 'MST'

    def test_get_coverage_01(self):
        from_axis = Axis(
            lower_bound=np.arange(0, 7)*24,
            upper_bound=np.arange(1, 8)*24,
            fraction=0.5
        )

        to_axis = Axis(
            lower_bound=[0],
            upper_bound=[168],
            data_ticks=[84]
        )

        row_idx, col_idx, weights = AxisRemapper._get_coverage(
            from_axis.lower_bound, from_axis.upper_bound,
            to_axis.lower_bound, to_axis.upper_bound
        )

        self.assertListEqual([0]*7, row_idx)
        self.assertListEqual(list(range(7)), col_idx)
        self.assertListEqual([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], weights)

    def test_get_coverage_02(self):
        from_axis = Axis(
            lower_bound=[0],
            upper_bound=[168],
            data_ticks=[84]
        )

        to_axis = Axis(
            lower_bound=np.arange(0, 7) * 24,
            upper_bound=np.arange(1, 8) * 24,
            fraction=0.5
        )

        row_idx, col_idx, coverageWeight = AxisRemapper._get_coverage(
            from_axis.lower_bound, from_axis.upper_bound,
            to_axis.lower_bound, to_axis.upper_bound
        )
        self.assertListEqual(list(range(7)), row_idx)
        self.assertListEqual([0]*7, col_idx)
        self.assertListEqual([1.0/7.0, 1.0/7.0, 1.0/7.0, 1.0/7.0, 1.0/7.0, 1.0/7.0, 1.0/7.0], coverageWeight)

    def test_get_coverage_03(self):
        from_axis = DailyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_end_date(date(2019, 1, 8)) \
            .build()

        to_axis = WeeklyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_end_date(date(2019, 1, 8)) \
            .build()

        row_idx, col_idx, weights = AxisRemapper._get_coverage(
            from_axis.lower_bound, from_axis.upper_bound,
            to_axis.lower_bound, to_axis.upper_bound
        )

        self.assertListEqual([0] * 7, row_idx)
        self.assertListEqual(list(range(7)), col_idx)
        self.assertListEqual([1.0] * 7, weights)

    def test_creation_01(self):
        from_axis = DailyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_end_date(date(2019, 1, 8)) \
            .build()

        to_axis = WeeklyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_end_date(date(2019, 1, 8)) \
            .build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

    def test_average_01(self):
        from_axis = DailyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_end_date(date(2019, 1, 8)) \
            .build()

        to_axis = WeeklyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_end_date(date(2019, 1, 8)) \
            .build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = [1, 2, 3, 4, 5, 6, 7]

        to_data = tc.average(from_data)

        self.assertAlmostEqual(4.0, to_data[0, 0], 0)

    def test_average_02(self):
        from_axis = WeeklyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_end_date(date(2019, 1, 8)) \
            .build()

        to_axis = DailyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_end_date(date(2019, 1, 8)) \
            .build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = [4.0]

        to_data = tc.average(from_data)

        self.assertEqual((tc.to_nelem, 1), to_data.shape)

        for e in to_data:
            self.assertAlmostEqual(4.0, e[0], 0)

    def test_average_03(self):
        from_axis = DailyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_end_date(date(2019, 1, 8)) \
            .build()

        to_axis = WeeklyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_end_date(date(2019, 1, 8)) \
            .build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = np.asarray([1, 2, 3, 4, 5, 6, 7])

        to_data = tc.average(from_data)

        self.assertAlmostEqual(4.0, to_data[0, 0], 0)

    def test_average_04(self):
        from_axis = DailyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_n_interval(14) \
            .build()

        to_axis = WeeklyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_n_interval(2) \
            .build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = list(range(1, 15))

        to_data = tc.average(from_data)

        self.assertAlmostEqual(4.0, to_data[0, 0], 0)
        self.assertAlmostEqual(11.0, to_data[1, 0], 0)

    def test_average_05(self):
        from_axis = WeeklyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_n_interval(2) \
            .build()

        to_axis = DailyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_n_interval(14) \
            .build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = [4.0, 11]

        to_data = tc.average(from_data)

        expected = [4] * 7 + [11] * 7
        self.assertListEqual(expected, to_data.flatten().tolist())

    def test_average_06(self):
        from_axis = DailyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_n_interval(14) \
            .build()

        to_axis = WeeklyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_n_interval(2) \
            .build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = np.moveaxis(np.asarray(list(range(1, 15)) * 12).reshape((3, 4, 14)), 2, 0)

        to_data = tc.average(from_data)

        self.assertEqual((2, 3, 4), to_data.shape)

        self.assertListEqual(
            [4] * 12,
            np.round(to_data[0, :, :].reshape((-1, 1))).astype(np.int).flatten().tolist()
        )

        self.assertListEqual(
            [11] * 12,
            np.round(to_data[1, :, :].reshape((-1, 1))).astype(np.int).flatten().tolist()
        )

    def test_average_07(self):
        from_axis = DailyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_n_interval(14) \
            .build()

        to_axis = WeeklyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_n_interval(2) \
            .build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = np.asarray(list(range(1, 15)) * 12).reshape((3, 4, 14))

        to_data = tc.average(from_data, dimension=2)

        self.assertEqual((3, 4, 2), to_data.shape)

        self.assertListEqual(
            [4] * 12,
            np.round(to_data[:, :, 0].reshape((-1, 1))).astype(np.int).flatten().tolist()
        )

        self.assertListEqual(
            [11] * 12,
            np.round(to_data[:, :, 1].reshape((-1, 1))).astype(np.int).flatten().tolist()
        )

    def test_average_08(self):
        from_axis = DailyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_n_interval(14) \
            .build()

        to_axis = WeeklyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_n_interval(2) \
            .build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = np.moveaxis(np.asarray(list(range(1, 15)) * 12).reshape((3, 4, 14)), 2, 1)

        to_data = tc.average(from_data, dimension=1)

        self.assertEqual((3, 2, 4), to_data.shape)

        self.assertListEqual(
            [4] * 12,
            np.round(to_data[:, 0, :].reshape((-1, 1))).astype(np.int).flatten().tolist()
        )

        self.assertListEqual(
            [11] * 12,
            np.round(to_data[:, 1, :].reshape((-1, 1))).astype(np.int).flatten().tolist()
        )

    def test_average_09(self):
        from_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        to_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=2
        ).build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = np.moveaxis(np.asarray(list(range(1, 15)) * 12).reshape((3, 4, 14)), 2, 1).tolist()

        to_data = tc.average(from_data, dimension=1)

        self.assertEqual((3, 2, 4), to_data.shape)

        self.assertListEqual(
            [4] * 12,
            np.round(to_data[:, 0, :].reshape((-1, 1))).astype(np.int).flatten().tolist()
        )

        self.assertListEqual(
            [11] * 12,
            np.round(to_data[:, 1, :].reshape((-1, 1))).astype(np.int).flatten().tolist()
        )

    def test_average_10(self):
        from_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        to_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=3
        ).build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis, assure_no_bound_mismatch=False)

        from_data = list(range(1, 15))

        to_data = tc.average(from_data)

        self.assertAlmostEqual(4.0, to_data[0, 0], 0)
        self.assertAlmostEqual(11.0, to_data[1, 0], 0)
        self.assertTrue(np.isnan(to_data[2, 0]))

    def test_average_11(self):
        from_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        to_axis = RollingWindowTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            end_date=date(2019, 1, 15),
            window_size=7
        ).build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = list(range(1, 15))

        to_data = tc.average(from_data)
        self.assertEqual(8, to_data.size)

        for idx in range(8):
            self.assertAlmostEqual(idx+4, to_data[idx, 0], 0)

    def test_average_12(self):
        from_axis = DailyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_n_interval(14) \
            .build()

        to_axis = WeeklyTimeAxisBuilder()\
            .set_start_date(date(2019, 1, 1)) \
            .set_n_interval(2) \
            .build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = list(range(1, 15))
        from_data[2] = np.nan

        to_data = tc.average(from_data)

        self.assertAlmostEqual(4.167, to_data[0, 0], 3)
        self.assertAlmostEqual(11.0, to_data[1, 0], 3)

    def test_min_01(self):
        from_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        to_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=2
        ).build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = list(range(1, 15))

        to_data = tc.min(from_data)

        self.assertAlmostEqual(1.0, to_data[0, 0], 0)
        self.assertAlmostEqual(8.0, to_data[1, 0], 0)

    def test_min_02(self):
        from_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        to_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=2
        ).build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = np.moveaxis(np.asarray(list(range(1, 15)) * 12).reshape((3, 4, 14)), 2, 0).tolist()

        to_data = tc.min(from_data)

        self.assertTrue(
            np.all(
                np.ones((3, 4), dtype="int") - to_data[0].round() == 0.0
            )
        )
        self.assertTrue(
            np.all(
                np.ones((3, 4), dtype="int") * 8 - to_data[1].round() == 0.0
            )
        )

    def test_min_03(self):
        from_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        to_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=2
        ).build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = np.moveaxis(np.asarray(list(range(1, 15)) * 12).reshape((3, 4, 14)), 2, 1).tolist()

        to_data = tc.min(from_data, dimension=1)

        self.assertTrue(
            np.all(
                np.ones((3, 4), dtype="int") - to_data[:, 0, :].round() == 0.0
            )
        )
        self.assertTrue(
            np.all(
                np.ones((3, 4), dtype="int") * 8 - to_data[:, 1, :].round() == 0.0
            )
        )

    def test_min_04(self):
        from_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        to_axis = RollingWindowTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            end_date=date(2019, 1, 15),
            window_size=7
        ).build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = np.moveaxis(np.asarray(list(range(1, 15)) * 12).reshape((3, 4, 14)), 2, 1).tolist()

        to_data = tc.min(from_data, dimension=1)

        for i in range(tc.to_nelem):
            self.assertTrue(
                np.all(
                    np.ones((3, 4), dtype="int") * (1 + i) - to_data[:, i, :].round() == 0.0
                )
            )

    def test_min_05(self):
        from_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        to_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=3
        ).build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis, assure_no_bound_mismatch=False)

        from_data = list(range(1, 15))

        to_data = tc.min(from_data)

        self.assertAlmostEqual(1.0, to_data[0, 0], 0)
        self.assertAlmostEqual(8.0, to_data[1, 0], 0)
        self.assertTrue(np.isnan(to_data[2, 0]))

    def test_max_01(self):
        from_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        to_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=2
        ).build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = list(range(1, 15))

        to_data = tc.max(from_data)

        self.assertAlmostEqual(7.0, to_data[0, 0], 0)
        self.assertAlmostEqual(14.0, to_data[1, 0], 0)

    def test_max_02(self):
        from_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        to_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=2
        ).build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = np.moveaxis(np.asarray(list(range(1, 15)) * 12).reshape((3, 4, 14)), 2, 0).tolist()

        to_data = tc.max(from_data)

        self.assertTrue(
            np.all(
                np.ones((3, 4), dtype="int") * 7 - to_data[0].round() == 0.0
            )
        )
        self.assertTrue(
            np.all(
                np.ones((3, 4), dtype="int") * 14 - to_data[1].round() == 0.0
            )
        )

    def test_max_03(self):
        from_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        to_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=2
        ).build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = np.moveaxis(np.asarray(list(range(1, 15)) * 12).reshape((3, 4, 14)), 2, 1).tolist()

        to_data = tc.max(from_data, dimension=1)

        self.assertTrue(
            np.all(
                np.ones((3, 4), dtype="int") * 7 - to_data[:, 0, :].round() == 0.0
            )
        )
        self.assertTrue(
            np.all(
                np.ones((3, 4), dtype="int") * 14 - to_data[:, 1, :].round() == 0.0
            )
        )

    def test_max_04(self):
        from_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        to_axis = RollingWindowTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            end_date=date(2019, 1, 15),
            window_size=7
        ).build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = np.moveaxis(np.asarray(list(range(1, 15)) * 12).reshape((3, 4, 14)), 2, 1).tolist()

        to_data = tc.max(from_data, dimension=1)

        for i in range(tc.to_nelem):
            self.assertTrue(
                np.all(
                    np.ones((3, 4), dtype="int") * (7+i) - to_data[:, i, :].round() == 0.0
                )
            )

    def test_max_05(self):
        from_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        to_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=3
        ).build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis, assure_no_bound_mismatch=False)

        from_data = list(range(1, 15))

        to_data = tc.max(from_data)

        self.assertAlmostEqual(7.0, to_data[0, 0], 0)
        self.assertAlmostEqual(14.0, to_data[1, 0], 0)
        self.assertTrue(np.isnan(to_data[2, 0]))

    def test_apply_function_01(self):
        daily_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        weekly_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=2
        ).build()

        ac = AxisRemapper(from_axis=daily_axis, to_axis=weekly_axis)

        daily_data = np.random.random((3, 4, 5, 14))

        def user_defined_function(data):
            return np.nansum(data, axis=0) * 42

        weekly_user_defined = ac.apply_function(daily_data, user_defined_function, dimension=3)

    @skip
    def test_speed_01(self):
        from_axis = DailyTimeAxisBuilder(
            start_date=date(2000, 1, 1),
            end_date=date(2020, 1, 1)
        ).build()

        to_axis = MonthlyTimeAxisBuilder(
            start_year=2000,
            end_year=2019
        ).build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis)

        from_data = np.random.random((from_axis.nelem, 100, 100))

        import time
        start = time.time()
        for i in range(1):
            tc.min(from_data)
        end = time.time()

        print(f"Took: %f [s]", end -start)

        import time
        start = time.time()
        for i in range(1):
            tc.min(from_data)
        end = time.time()

        print(f"Took: %f [s]", end - start)

    def test_dask_array_01(self):
        daily_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        weekly_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=2
        ).build()

        tc = AxisRemapper(from_axis=daily_axis, to_axis=weekly_axis)

        from_data = da.arange(14)

        to_data = tc.average(from_data)
        self.assertTrue(isinstance(to_data, da.Array))
        to_data = to_data.compute()

        self.assertAlmostEqual(3.0, to_data[0, 0], 0)
        self.assertAlmostEqual(10.0, to_data[1, 0], 0)

    def test_dask_array_02(self):
        daily_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        weekly_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=2
        ).build()

        tc = AxisRemapper(from_axis=daily_axis, to_axis=weekly_axis)

        from_data = da.from_array(
            np.moveaxis(np.asarray(np.arange(14).tolist()*12).reshape(3, 4, 14), 2, 0)
        )

        to_data = tc.average(from_data)
        self.assertTrue(isinstance(to_data, da.Array))
        to_data = to_data.compute()

        np.testing.assert_almost_equal(
            to_data,
            np.asarray([3] * 12 + [10]*12).reshape(2, 3, 4)
        )

    def test_dask_array_03(self):
        daily_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        weekly_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=2
        ).build()

        tc = AxisRemapper(from_axis=daily_axis, to_axis=weekly_axis)

        from_data = da.from_array(
            np.asarray(np.arange(14).tolist()*12).reshape(3, 4, 14)
        )

        to_data = tc.average(from_data, 2)
        self.assertTrue(isinstance(to_data, da.Array))
        to_data = to_data.compute()

        np.testing.assert_almost_equal(
            to_data,
            np.moveaxis(np.asarray([3] * 12 + [10]*12).reshape(2, 3, 4), 0, 2)
        )

    def test_dask_array_04(self):
        daily_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        weekly_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=2
        ).build()

        tc = AxisRemapper(from_axis=daily_axis, to_axis=weekly_axis)

        from_data = da.from_array(
            np.moveaxis(np.asarray(np.arange(14).tolist()*12).reshape(3, 4, 14), 2, 0),
            chunks=(14, 1, 1)
        )

        to_data = tc.average(from_data)
        self.assertTrue(isinstance(to_data, da.Array))
        to_data = to_data.compute()

        np.testing.assert_almost_equal(
            to_data,
            np.asarray([3] * 12 + [10]*12).reshape(2, 3, 4)
        )

    def test_dask_array_05(self):
        daily_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        weekly_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=2
        ).build()

        tc = AxisRemapper(from_axis=daily_axis, to_axis=weekly_axis)

        from_data = da.from_array(
            np.moveaxis(np.asarray(np.arange(14).tolist()*12).reshape(3, 4, 14), 2, 0),
            chunks=(2, 1, 1)
        )

        to_data = tc.average(from_data)
        self.assertTrue(isinstance(to_data, da.Array))
        to_data = to_data.compute()

        np.testing.assert_almost_equal(
            to_data,
            np.asarray([3] * 12 + [10]*12).reshape(2, 3, 4)
        )

    def test_dask_min(self):
        from_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        to_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=3
        ).build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis, assure_no_bound_mismatch=False)
        from_data = da.arange(14, dtype='float64')
        to_data = tc.min(from_data).compute()
        np.testing.assert_almost_equal(to_data, np.array([0.0, 7.0, np.nan]).reshape(3, 1))

    def test_dask_max(self):
        from_axis = DailyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=14
        ).build()

        to_axis = WeeklyTimeAxisBuilder(
            start_date=date(2019, 1, 1),
            n_interval=3
        ).build()

        tc = AxisRemapper(from_axis=from_axis, to_axis=to_axis, assure_no_bound_mismatch=False)
        from_data = da.arange(14, dtype='float64')
        to_data = tc.max(from_data).compute()
        np.testing.assert_almost_equal(to_data, np.array([6.0, 13.0, np.nan]).reshape(3, 1))

max_stars_repo_name: coderepocenter/AxisUtilities

max_issues_repo_name: coderepocenter/AxisUtilities

max_forks_repo_name: coderepocenter/AxisUtilities


text:
# coding=utf-8

"""
tools for run bi-lstm short text classification
"""

import numpy as np
import math

class TextLoader(object):
    def __init__(self, is_training, data_path, map_file_path, batch_size, seq_length, vocab, labels, std_label_map,
                 encoding='utf8', is_reverse=False):
        self.data_path = data_path
        self.map_file_path = map_file_path
        self.batch_size = batch_size
        self.seq_length = seq_length
        self.is_train = is_training
        self.encoding = encoding
        # load label std mapping index
        self.std_label_map = {}
        self.label_num_map = {}
        label_set = set()
        word_set = set()
        if is_training:
            with open(map_file_path, 'r', encoding=encoding) as map_index_file:
                for line in map_index_file:
                    tokens = line.strip().split('\t')
                    assert len(tokens) == 3
                    label = tokens[0]
                    label_set.add(label)
                    std_id = tokens[1]
                    self.std_label_map[std_id] = label
                    words = tokens[2].split(" ")
                    for token in words:
                        word_set.add(token)

            train_file = data_path
            with open(train_file, 'r', encoding=encoding) as fin:
                for line in fin:
                    tokens = line.strip().split('\t')
                    assert len(tokens) == 3
                    std_ids = tokens[0].split(",")
                    words = tokens[2].split(" ")
                    if len(std_ids) > 1:
                        label = '__label__list'  # answer list
                        label_set.add(label)
                    elif std_ids[0] == '0':
                        label = '__label__none'  # refuse answer
                        label_set.add(label)
                    else:
                        assert std_ids[0] in self.std_label_map
                        label = self.std_label_map.get(std_ids[0])  # __label__xx:some label
                    for token in words:
                        word_set.add(token)
                    if label not in self.label_num_map:
                        self.label_num_map[label] = 1
                    else:
                        self.label_num_map[label] = self.label_num_map[label] + 1

            self.labels = dict(
                zip(list(label_set), range(0, len(label_set))))  # {__label__1:0, __label_2:1, __label_3:2, ...}
            # print("self.labels: " + str(self.labels))
            # print("self.std_label_map: " + str(self.std_label_map))
            self.id_2_label = {value: key for key, value in self.labels.items()}
            self.label_size = len(self.labels)
            self.vocab = dict(zip(list(word_set), range(1, len(word_set) + 1)))
            self.id_2_vocab = {value: key for key, value in self.vocab.items()}
            self.vocab_size = len(
                self.vocab) + 1  # self.vocab.size + 1, 0 for pad, not encoding unknown, if care for it, you can modify here
            self.load_preprocessed(data_path, is_reverse)
        elif vocab is not None and labels is not None and std_label_map is not None:
            self.vocab = vocab
            self.id_2_vocab = {value: key for key, value in self.vocab.items()}
            self.vocab_size = len(vocab) + 1
            self.labels = labels
            self.id_2_label = {value: key for key, value in self.labels.items()}
            self.label_size = len(self.labels)
            self.std_label_map = std_label_map
            self.load_preprocessed(data_path, is_reverse)
        self.num_batches = 1
        self.x_batches = None
        self.y_batches = None
        self.len_batches = None
        self.reset_batch_pointer()

    def load_preprocessed(self, data_path, is_reverse):
        train_file = data_path
        self.raw_lines = []
        with open(train_file, 'r', encoding=self.encoding) as fin:
            train_x = []
            train_y = []
            train_len = []
            for line in fin:
                temp_x = []
                temp_y = []
                x_len = []
                tokens = line.strip().split('\t')
                assert len(tokens) == 3
                std_ids = tokens[0].split(",")
                words = tokens[2].split(" ")
                if len(std_ids) > 1:
                    label = '__label__list'  # answer list
                elif std_ids[0] == '0':
                    label = '__label__none'  # refuse answer
                else:
                    if std_ids[0] not in self.std_label_map:
                        label = '__label__none'
                    else:
                        label = self.std_label_map.get(std_ids[0])  # __label__xx:some label
                # if label not in self.labels:
                #     print("label: <" + label + ">")
                #     print("self.labels: ")
                #     print(str(self.labels))
                temp_y.append(self.labels[label])
                for item in words:
                    if item in self.vocab:  # not encoding unknown, if care for it, you can modify here
                        temp_x.append(self.vocab[item])
                if len(temp_x) == 0:
                    print("all word in line is not in vocab, line: " + line)
                    continue
                if len(temp_x) >= self.seq_length:
                    x_len.append(self.seq_length)
                    temp_x = temp_x[:self.seq_length]
                    if is_reverse:
                        temp_x.reverse()
                else:
                    x_len.append(len(temp_x))
                    if is_reverse:
                        temp_x.reverse()
                    temp_x = temp_x + [0] * (self.seq_length - len(temp_x))
                train_x.append(temp_x)
                train_y.append(temp_y)
                train_len.append(x_len)
                self.raw_lines.append(tokens[2])
        tensor_x = np.array(train_x)
        tensor_y = np.array(train_y)
        tensor_len = np.array(train_len)
        # print("tensor_x.shape: " + str(tensor_x.shape))
        # print("len(self.raw_lines): " + str(len(self.raw_lines)))

        self.tensor = np.c_[tensor_x, tensor_y, tensor_len].astype(int)  # tensor_x.size * (40+1+1)

    def list_split_n(self, raw_items, split_len_batches):
        split_items = []
        j = 0
        for i in range(len(split_len_batches)):
            split_items.append(raw_items[j: j + split_len_batches[i]])
            j += split_len_batches[i]
        return split_items

    def create_batches(self):
        self.num_batches = int(self.tensor.shape[0] / self.batch_size)
        if int(self.tensor.shape[0] % self.batch_size):
            self.num_batches = self.num_batches + 1
        if self.num_batches == 0:
            assert False, 'Not enough data, make batch_size small.'
        if self.is_train:
            np.random.shuffle(self.tensor)
        # print("self.num_batches: " + str(self.num_batches))
        tensor = self.tensor[:self.num_batches * self.batch_size]
        # print("len(tensor): " + str(len(tensor)))
        raw_lines = self.raw_lines[
                    :self.num_batches * self.batch_size]  # if train raw_lines order is different from tensor
        # print("len(raw_lines): " + str(len(raw_lines)))

        self.x_batches = np.array_split(tensor[:, :-2], self.num_batches, 0)
        self.y_batches = np.array_split(tensor[:, -2], self.num_batches, 0)
        self.len_batches = np.array_split(tensor[:, -1], self.num_batches, 0)
        split_len_batches = []
        for i in range(len(self.x_batches)):
            split_len_batches.append(len(self.x_batches[i]))
        self.raw_lines_batches = self.list_split_n(raw_lines, split_len_batches)  # should split by np.array_split
        sum = 0
        # for i in range(len(self.x_batches)):
        #     print("i: " + str(i) + "len(self.x_batches[i]): " + str(len(self.x_batches[i])))
        #     sum += len(self.x_batches[i])
        # print("sum: " + str(sum))
        #
        # print("len(self.x_batches): " + str(len(self.x_batches)))
        # print("len(self.y_batches): " + str(len(self.y_batches)))
        # print("len(self.len_batches): " + str(len(self.len_batches)))
        # print("len(self.raw_lines_batches): " + str(len(self.raw_lines_batches)))

    def next_batch(self):
        batch_x = self.x_batches[self.pointer]
        batch_y = self.y_batches[self.pointer]
        xlen = self.len_batches[self.pointer]
        batch_line = self.raw_lines_batches[self.pointer]

        self.pointer += 1
        return batch_x, batch_y, xlen, batch_line

    def reset_batch_pointer(self):
        self.create_batches()
        self.pointer = 0

if __name__ == "__main__":
    pass


text:
import numpy as np
import gpsd
import csv
import datetime
import time
import board
import busio
import smbus
import led
import adafruit_ads1x15.ads1115 as ADS
from adafruit_ads1x15.analog_in import AnalogIn
from adafruit_ads1x15.ads1x15 import Mode
import matplotlib.pyplot as plt

class SeisNode:
    """
    Sesmic node class (for Raspberry Pi)

    ADC Parameters (kwargs):
        gain = 1
        mode = Mode.CONTINUOUS
        data_rate = 860
    """

    def __init__(self,**kwargs):
        self.gain = 1
        self.mode = Mode.CONTINUOUS
        self.data_rate = 860

        for key in kwargs:
            setattr(self, key, kwargs[key])

        self.vz = []
        self.vx = []
        self.vy = []

        ## Create the I2C bus
        i2c = busio.I2C(board.SCL, board.SDA)
        ## Create the ADC object
        self.ads = ADS.ADS1115(i2c)
        self.ads.gain = self.gain
        self.ads.mode = Mode.CONTINUOUS
        self.ads.data_rate = self.data_rate
        #self.ads.comparator_config = 0

    def set_gain(self,gain):
        self.ads.gain = gain
        self.gain = gain

    def collect_adc(self):
        self.vz = []
        self.vx = []
        self.vy = []
        # Create single-ended input on channel 0
        chan0 = AnalogIn(self.ads, ADS.P0)
        chan1 = AnalogIn(self.ads, ADS.P1)
        chan2 = AnalogIn(self.ads, ADS.P2)
        try:
            t = time.time()
            while time.time()-t < 60:
                # Recieve voltage values
                self.vz.append(chan0.value)
                self.vx.append(chan1.value)
                self.vy.append(chan2.value)
        except KeyboardInterrupt:
            pass

        return  vz,vx,vy

    def collect_gps(self):
        gpsd.connect()
        packet = gpsd.get_current()
        #print(packet.mode)
        if packet.mode >= 3:
            time = packet.get_time(local_time=True)
            lat = packet.lat
            ey = packet.error['y']
            lon = packet.lon
            ex = packet.error['x']
            alt = packet.alt
            ez = packet.error['v']

            const = 1/111319.9 #  degrees per meter
            data = [time,lat,ey*const,lon,ex*const,alt,ez]
            return data

    def trigger(self):
        bus = smbus.SMBus(1)
        shot = 0
        status = True
        while True:
            #Parameters for write_byte_data
            #1. Address of the device
            #2. Communication data - active mode control register
            #3. Our data - 0 (standby mode) or 1 (active)
            bus.write_byte_data(0x1D, 0x2A, 1)

            #Read from the status register, real-time status register 0x00
            #Data returned will be an array
            #Contents of 7 bytes read and stored in data array represent:
            #status (ignore), MSBx, LSBx, MSBy, LSBy, MSBz, LSBz
            data = bus.read_i2c_block_data(0x1D, 0x00, 7)

            number_of_bits = 16
            MSB_z = data[5]
            LSB_z = data[6]

            zAccl = (MSB_z * 256 + LSB_z) / number_of_bits
            if zAccl > 2047:
                zAccl -= 4096

            # if z acceleration changes by some great amount
            prev_z = 1000
            if abs((prev_z-zAccl)/zAccl) >= 1.0: ### Change this
                bus.write_byte_data(0x1D, 0x2A, 0)
                # record GPS for 1 second.
                with open("/home/pi/SeisNode/data/shots.csv" %shot, "a+") as writefile:
                    print("Trigger")
                    led.turn_on()
                    writer = csv.writer(writefile)
                    if shot == 0:
                        writer.writerow(["Shot","Time", "Lat", "Lat Err", "Lon", "Lon Err", "Alt", "Alt Err"])
                    start = time.time()
                    while time.time()-start < 1:
                        data = self.collect_gps()
                        data = np.concatenate([[shot],data])
                        writer.writerow(data)
                        time.sleep(0.2)
                    writefile.flush()
                    writefile.close()

                    shot += 1
                    time.sleep(30)
                    led.turn_off()
                    prev_z = 1000
                    status = True
                    continue
            status = False
            prev_z = zAccl

        def collect_data(self):
            pass

max_stars_repo_name: dhchurchwell/SeisNode

max_issues_repo_name: btbernstein/Rpi

max_forks_repo_name: btbernstein/Rpi


text:
# Copyright 2018-2021 Streamlit Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest

import numpy as np
import pandas as pd
from streamlit.util import index_

class Index_Test(unittest.TestCase):
    def test_index_list(self):
        self.assertEqual(index_([1, 2, 3, 4], 1), 0)
        self.assertEqual(index_([1, 2, 3, 4], 4), 3)

    def test_index_list_fails(self):
        with self.assertRaises(ValueError):
            index_([1, 2, 3, 4], 5)

    def test_index_tuple(self):
        self.assertEqual(index_((1, 2, 3, 4), 1), 0)
        self.assertEqual(index_((1, 2, 3, 4), 4), 3)

    def test_index_tuple_fails(self):
        with self.assertRaises(ValueError):
            index_((1, 2, 3, 4), 5)

    def test_index_numpy_array(self):
        self.assertEqual(index_(np.array([1, 2, 3, 4]), 1), 0)
        self.assertEqual(index_(np.array([1, 2, 3, 4]), 4), 3)

    def test_index_numpy_array_fails(self):
        with self.assertRaises(ValueError):
            index_(np.array([1, 2, 3, 4]), 5)

    def test_index_pandas_series(self):
        self.assertEqual(index_(pd.Series([1, 2, 3, 4]), 1), 0)
        self.assertEqual(index_(pd.Series([1, 2, 3, 4]), 4), 3)

    def test_index_pandas_series_fails(self):
        with self.assertRaises(ValueError):
            index_(pd.Series([1, 2, 3, 4]), 5)

max_stars_repo_name: AnOctopus/streamlit

max_issues_repo_name: linzhou-zhong/streamlit

max_forks_repo_name: linzhou-zhong/streamlit


text:
import numpy as np

from tree_utils import LeafNode, InternalDecisionNode
from select_best_binary_split import select_best_binary_split

def train_tree_greedy(
        x_NF, y_N, depth,
        MAX_DEPTH=10,
        MIN_SAMPLES_INTERNAL=1,
        MIN_SAMPLES_LEAF=1):
    ''' Train a binary decision tree on provided dataset in greedy fashion.

    Args
    ----
    x_NF : 2D array, shape (N,F) = (n_examples, n_features)
        Training data features to use to build the tree.
    y_N : 1D array, shape (N,) = (n_examples,)
        Training labels to use to build the tree.
    depth : int
        Current depth of the tree. Needed to use as a recursive function.
        Should be zero when called by end user.
    MAX_DEPTH : int
        Hyperparameter controlling maximum allowed depth.
        Smaller values might protect against overfitting.
    MIN_SAMPLES_INTERNAL : int
        Controls minimum samples allowed for an InternalDecisionNode.
        Higher values might protect against overfitting.
    MIN_SAMPLES_LEAF : int
        Hyperparameter controlling minimum samples allowed for a leaf node.
        Higher values might protect against overfitting.

    Returns
    -------
    root_node : an instance of LeafNode or InternalDecisionNode
        This node determines the root of the requested tree.
        You can query its attributes `left_child` and `right_child` to get
        other nodes in the tree (if they exist).

    '''
    N, F = x_NF.shape
    MIN_SAMPLES_INTERNAL = np.maximum(
        MIN_SAMPLES_INTERNAL, 2 * MIN_SAMPLES_LEAF)
    if depth >= MAX_DEPTH:
        return LeafNode(x_NF, y_N)
    elif N < MIN_SAMPLES_INTERNAL:
        return LeafNode(x_NF, y_N)
    else:
        feat_id, thresh_val, x_LF, y_L, x_RF, y_R = select_best_binary_split(
                x_NF, y_N, MIN_SAMPLES_LEAF)
        if feat_id is None:
            # Case where further split is not possible
            return LeafNode(x_NF, y_N)
        else:
            # TODO recursively call train_tree_greedy to build the left child
            left_child = None # TODO fixme
            # TODO recursively call train_tree_greedy to build the right child
            right_child = None # TODO fixme
            return InternalDecisionNode(
                x_NF, y_N, feat_id, thresh_val, left_child, right_child)


text:
#===================================
#        Chengwen Liu              #
#      liuchw2010@gmail.com        #
#   University of Texas at Austin  #
#===================================

''' Usage: python matchTXYZ.py template.txyz dealwith.(t)xyz
    # Assign the atom types of the template.txyz file to the dealwith txyz or xyz file.
    # Especially suitable for the case that their atoms are different in order.
'''

import os
import sys
import numpy as np

def readTXYZ(TXYZ):
  atoms=[]
  coord=[]
  order=[]
  types=[]
  connections=[]
  for line in open(TXYZ).readlines()[1:]:
    data=line.split()
    order.append(data[0])
    atoms.append(data[1])
    coord.append([float(data[2]), float(data[3]), float(data[4])])
    types.append(data[5])
    connections.append(data[6:])
  return atoms,coord,order,types,connections

def fingerprint(TXYZ):
  fprints = []
  atoms, elements = np.loadtxt(TXYZ, usecols=(0,1), dtype='str', skiprows=1, unpack=True)
  connections = []
  for line in open(TXYZ).readlines()[1:]:
    d = line.split()
    connections.append(d[6:])

  atom_ele_dict = dict(zip(atoms, elements))
  atom_con_dict = {}
  for atom, con in zip(atoms,connections):
    con_ele = [atom_ele_dict[c] for c in con]
    constr = ''.join(sorted(con_ele))
    atom_con_dict[atom] = constr

  level = 5
  if level > 1:
    atom_con_dict2 = {}
    for atom, con in zip(atoms,connections):
      eles = []
      cons = []
      for c in con:
        eles.append(atom_ele_dict[c])
        cons.append(c)
      cons = [x for _,x in sorted(zip(eles,cons))]
      newstr = ''.join([atom_con_dict[c] for c in cons])
      atom_con_dict2[atom] = ''.join(sorted(newstr))

  # level 3 is good for chain molecules
  if level > 2:
    atom_con_dict3 = {}
    for atom, con in zip(atoms,connections):
      eles = []
      cons = []
      for c in con:
        eles.append(atom_ele_dict[c])
        cons.append(c)
      cons = [x for _,x in sorted(zip(eles,cons))]
      newstr = ''.join([atom_con_dict2[c] for c in cons])
      atom_con_dict3[atom] = ''.join(sorted(newstr))

  # level 4 is needed for ring molecules
  if level > 3:
    atom_con_dict4 = {}
    for atom, con in zip(atoms,connections):
      eles = []
      cons = []
      for c in con:
        eles.append(atom_ele_dict[c])
        cons.append(c)
      cons = [x for _,x in sorted(zip(eles,cons))]
      newstr = ''.join([atom_con_dict3[c] for c in cons])
      atom_con_dict4[atom] = ''.join(sorted(newstr))

  if level > 4:
    atom_con_dict5 = {}
    for atom, con in zip(atoms,connections):
      eles = []
      cons = []
      for c in con:
        eles.append(atom_ele_dict[c])
        cons.append(c)
      cons = [x for _,x in sorted(zip(eles,cons))]
      newstr = ''.join([atom_con_dict4[c] for c in cons])
      atom_con_dict5[atom] = ''.join(sorted(newstr))

  for atom in atoms:
    fprints.append(atom_ele_dict[atom] + '-' + str(''.join(sorted(atom_con_dict[atom] + atom_con_dict2[atom] + atom_con_dict3[atom] + atom_con_dict4[atom] + atom_con_dict5[atom]))))

  return fprints

def main():
  template = sys.argv[1]
  dealwith = sys.argv[2]
  if os.path.splitext(dealwith)[1] == ".xyz":
    xyz = dealwith
    dealwith = dealwith.replace("xyz", "txyz")
    obstr = "obabel -ixyz %s -otxyz -O %s"%(xyz, dealwith)
    os.system(obstr)
  fname = dealwith + "_2"
  fp1 = fingerprint(template)
  fp2 = fingerprint(dealwith)
  newidx = []
  for i in fp2:
    if i in fp1:
      idx = fp1.index(i)
      newidx.append(idx)
      fp1[idx] = ' '
    else:
      print(f"Error:{template} and {dealwith} could not match!")

  atoms, coord, _, _, connections =  readTXYZ(dealwith)
  _, _, _,types, _ =  readTXYZ(template)
  with open(fname, 'w') as f:
    f.write("%3s\n"%len(atoms))
    for i in range(len(newidx)):
      idx = int(newidx[i])
      f.write("%3s%3s%12.6f%12.6f%12.6f  %s   %s\n"%(i+1,atoms[i], coord[i][0], coord[i][1], coord[i][2], types[idx], '  '.join(connections[i])))
  return

if __name__ == "__main__":
  main()

max_stars_repo_name: leucinw/leucinwChemTools

max_issues_repo_name: leucinw/leucinwChemTools

max_forks_repo_name: leucinw/leucinwChemTools


text:
#!/usr/bin/env python
import rospy
import tf

from visualization_msgs.msg import Marker
from geometry_msgs.msg import Point
from std_srvs.srv import *
import numpy as np

# Give ourselves the ability to run a dynamic reconfigure server.
# from dynamic_reconfigure.server import Server as DynamicReconfigureServer
# from tf_trace_publisher.cfg import tf_Trace_PublisherConfig as ConfigType

class CableTracker():
	def __init__(self):

		# self.server = DynamicReconfigureServer(ConfigType, self.reconfigure)
		self.srv = rospy.Service('~fix_pose', Empty, self.fix_pose)
		self.cable_length = 1.0
		self.fixed_poses = []

		self.base_frame = "world"
		self.fix_tcp = "fix_tcp"
		self.end_frame = "supposed_tcp"

		# initalize
		self.trace = Marker()
		self.trace.id = rospy.get_param('~id', 1)
		self.trace.type= Marker.LINE_STRIP
		self.trace.action = Marker.MODIFY
		self.trace.scale.x = 0.1
		self.trace.scale.y = 0.1
		self.trace.scale.z = 0.1

		self.trace.color.r = 1
		self.trace.color.g = 0
		self.trace.color.b = 0
		self.trace.color.a = 1

		self.trace.header.frame_id = self.base_frame
		# self.trace.ns = self.fix_tcp

		self.tf_listener = tf.TransformListener()
		self.pub_markerLine = rospy.Publisher('visualization_marker', Marker, queue_size=10)

		try:
			# self.tf_listener.waitForTransform(
			# 	self.base_frame,
			# 	self.fix_tcp,
			# 	rospy.Time(0),
			# 	rospy.Duration(4.0))
            #
			# (trans,rot) = self.tf_listener.lookupTransform(
			# 	self.base_frame,
			# 	self.fix_tcp,
			# 	rospy.Time(0))

			#self.fixed_poses.append([0.5487073659896851, -0.04490622505545616, 0.7260761260986328])
			#self.fixed_poses.append([0.6585336327552795, 0.06950411200523376, 0.7238875031471252])
			#self.fixed_poses.append([0.7527252435684204, 0.06950411200523376, 0.7238875031471252])
			pass
		except:
			return

	def run(self):
		while not rospy.is_shutdown():
			if self.fixed_poses:
				try:
					(trans,rot) = self.tf_listener.lookupTransform(
						self.base_frame,
						self.fix_tcp,
						rospy.Time(0))

					remaining_length = self.cable_length

					self.trace.points = []
					for i, fixture_point in enumerate(self.fixed_poses):
						# print fixture_point
						if i < len(self.fixed_poses)-1:
							remaining_length -= np.linalg.norm(np.array(self.fixed_poses[i])-np.array(self.fixed_poses[i+1]))

						p = Point()
					 	p.x = fixture_point[0]
					 	p.y = fixture_point[1]
					 	p.z = fixture_point[2]
						#print p
					 	self.trace.points.append(p)

					# print "trans",trans
					# print remaining_length

					# letzer punkt
					end_point = remaining_length * ((np.array(trans) - np.array(self.fixed_poses[-1])) / np.linalg.norm(np.array(trans) - np.array(self.fixed_poses[-1]))) + np.array(self.fixed_poses[-1])
					# print "end", end_point

					p = Point()
				 	p.x = end_point[0]
				 	p.y = end_point[1]
				 	p.z = end_point[2]
					self.trace.points.append(p)

					if len(self.trace.points) > 0:
						self.pub_markerLine.publish(self.trace)

				except (tf.LookupException, tf.ConnectivityException, tf.ExtrapolationException):
					rospy.logwarn("Transformation Lookup failed")
					rospy.sleep(rospy.Duration(1))
					continue
			rospy.sleep(0.5)

	# # Create a callback function for the dynamic reconfigure server.
	# def reconfigure(self, config, level):
	# 	# Fill in local variables with values received from dynamic reconfigure clients (typically the GUI).
    #
	# 	self.trace.header.frame_id = config["tf_parent"]
	# 	self.trace.ns = config["tf_child"]
    #
	# 	self.trace.scale.x = config["scale"]
	# 	self.trace.scale.y = config["scale"]
	# 	self.trace.scale.z = config["scale"]
    #
	# 	self.trace.color.r = config["color_R"]
	# 	self.trace.color.g = config["color_G"]
	# 	self.trace.color.b = config["color_B"]
	# 	self.trace.color.a = config["color_A"]
    #
	# 	self.maxLen = config["length"]
	# 	self.rateTime = config["rate"]
	# 	self.rate = rospy.Rate(self.rateTime)
    #
	# 	config["tf_parent"] = self.trace.header.frame_id
	# 	config["tf_child"] = self.trace.ns
	# 	config["scale"] = self.trace.scale.x
	# 	config["color_R"] = self.trace.color.r
	# 	config["color_G"] = self.trace.color.g
	# 	config["color_B"] = self.trace.color.b
	# 	config["color_A"] = self.trace.color.a
    #
	# 	config["length"] = self.maxLen
	# 	config["rate"] = self.rateTime
    #
	# 	rospy.loginfo("Configuration changed!")
	# 	return config

	def fix_pose(self, req):
		try:
			(trans,rot) = self.tf_listener.lookupTransform(
				self.base_frame,
				self.fix_tcp,
				rospy.Time(0))
			self.fixed_poses.append(trans)

			return EmptyResponse()
		except:
			return

if __name__ == '__main__':

	rospy.init_node('CableTracker', anonymous=False)
	node = CableTracker()
	try:
		node.run()
	except rospy.ROSInterruptException: pass


text:
import numpy as np

class Geom:
    def __init__(self, method, **kwargs):
        self.generate_initial_state(method,**kwargs)

    def generate_initial_state(self,method,**kwargs):
        """Generate initial coordinates of particles in a box either randomly or based on a file.

        Parameters
        ----------
        method : string, either 'random' or 'file'
            Method of generating initial state.
        file_name : string
            Name of file used to generate initial state.
        num_particles : integer
            Number of particles to generate.
        box_length : integer or float
            Length of box to generate.

        Returns
        -------
        coordinates : array
            Array of particle coordinates generated for an initial state
        """

        if method is 'random':
            if (kwargs['num_particles'] == None or kwargs['reduced_den'] == None):
                raise ValueError(' "num_particles" and "reduced_den" arguments must be set for method=random!')
            self.num_particles = kwargs['num_particles']
            self.box_length = np.cbrt(self.num_particles / kwargs['reduced_den'])
            self.volume = self.box_length**3
            self.coordinates = (0.5 - np.random.rand(self.num_particles, 3)) * self.box_length

        elif method is 'file':
            if (kwargs['file_name'] == None):
                raise ValueError('"filename" argument must be set for method = file!')
            file_name = kwargs['file_name']
            with open(file_name) as f:
                lines = f.readlines()
                self.box_length = np.fromstring(lines[0], dtype=float, sep=',')[0]
                self.volume = self.box_length**3
                self.num_particles = np.fromstring(lines[1], dtype=float, sep=',')[0]
            self.coordinates = np.loadtxt(file_name, skiprows=2, usecols=(1,2,3))
            if (self.num_particles != self.coordinates.shape[0]):
                raise ValueError('Inconsistent value of number of particles in file!')

        else:
            raise TypeError('Method type not recognized.')

    def minimum_image_distance(self,r_i, coords):
        """Calculate minimum image distance between two particles, i and j.

        Parameters
        ---------
        box_length : integer or float
            Length of box to generate.
        r_i : array
            Coordinates of particle i.
        coords : array
            Coordintes of particle j or array of positions

        Returns
        -------
        rij2 : square of the distance between particles i and j, or particle i and all the coords.

        """
        rij = r_i - coords
        rij = rij - self.box_length * np.round(rij / self.box_length)
        rij2 = np.sum(rij**2,axis=-1)
        return rij2

    def wrap(self,v):
        """ Wrap a vector back to periodic box

        Parameters
        ----------
        v : the vector to be wrapped

        Returns
        -------
        wrapped_v : the vector wrapped back to simulation box
        """
        wrapped_v = v - self.box_length*np.round(v/self.box_length)
        return wrapped_v

    def get_particle_coordinates(self):
        """Get the coordinates of all particles in the system

        Parameters
        ----------
        None

        Returns
        -------
        None
        """
        return self.coordinates

    def save_state(self, file_name):
        """Save current simulation state into a txt file. First line is box dimension, second line is number of particles, and the rest are particle coordinates

        Parameters
        ----------
        None

        Returns
        -------
        None
        """
        import os.path
        if (os.path.exists(file_name)):
            raise ValueError("File already exists!")
        f = open(file_name,"w+")
        f.write("%.18e   %.18e   %.18e\n" %(self.box_length,self.box_length,self.box_length))
        f.write("%d\n"%(self.num_particles))
        f.close()
        f = open(file_name,'ab')
        np.savetxt(f,self.coordinates)
        f.close()


text:
import argparse
import json
import numpy as np
import os
import torch

from datetime import datetime
from pathlib import Path
from sklearn import metrics

from evaluate import run_model
from loader import load_data
from model import TripleMRNet

def train(rundir, task, backbone, epochs, learning_rate, use_gpu,
        abnormal_model_path=None):
    train_loader, valid_loader = load_data(task, use_gpu)

    model = TripleMRNet(backbone=backbone)
    for dirpath, dirnames, files in os.walk(args.rundir):
        if not files:
            break
        max_epoch = 0
        model_path = None
        for fname in files:
            if fname.endswith(".json"):
                continue
            ep = int(fname[27:])
            if ep >= max_epoch:
                max_epoch = ep
                model_path = os.path.join(dirpath, fname)

        if model_path:
            state_dict = torch.load(model_path, map_location=(None if use_gpu else 'cpu'))
            model.load_state_dict(state_dict)

    if use_gpu:
        model = model.cuda()

    optimizer = torch.optim.Adam(model.parameters(), learning_rate, weight_decay=.01)
    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=5, factor=.3, threshold=1e-4)

    best_val_loss = float('inf')

    start_time = datetime.now()

    epoch = 0
    if max_epoch: epoch += max_epoch
    while epoch < epochs:
        change = datetime.now() - start_time
        print('starting epoch {}. time passed: {}'.format(epoch+1, str(change)))

        train_loss, train_auc, _, _ = run_model(
                model, train_loader, train=True, optimizer=optimizer,
                abnormal_model_path=abnormal_model_path)

        print(f'train loss: {train_loss:0.4f}')
        print(f'train AUC: {train_auc:0.4f}')

        val_loss, val_auc, _, _ = run_model(model, valid_loader,
                abnormal_model_path=abnormal_model_path)

        print(f'valid loss: {val_loss:0.4f}')
        print(f'valid AUC: {val_auc:0.4f}')

        scheduler.step(val_loss)

        # save every epoch
        file_name = f'model_val{val_auc:0.4f}_train{train_auc:0.4f}_epoch{epoch+1}'
        save_path = Path(rundir) / file_name
        torch.save(model.state_dict(), save_path)

        if val_loss < best_val_loss:
            best_val_loss = val_loss

            file_name = f'val{val_auc:0.4f}_train{train_auc:0.4f}_epoch{epoch+1}'
            save_path = Path(rundir) / file_name
            torch.save(model.state_dict(), save_path)

        epoch += 1

def get_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument('--rundir', type=str, required=True)
    parser.add_argument('--task', type=str, required=True)
    parser.add_argument('--seed', default=42, type=int)
    parser.add_argument('--gpu', action='store_true')
    parser.add_argument('--learning_rate', default=1e-05, type=float)
    parser.add_argument('--weight_decay', default=0.01, type=float)
    parser.add_argument('--epochs', default=100, type=int)
    parser.add_argument('--max_patience', default=5, type=int)
    parser.add_argument('--factor', default=0.3, type=float)
    parser.add_argument('--backbone', default="alexnet", type=str)
    parser.add_argument('--abnormal_model', default=None, type=str)
    return parser

if __name__ == '__main__':
    args = get_parser().parse_args()

    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    if args.gpu:
        torch.cuda.manual_seed_all(args.seed)

    # if args.task != "abnormal":
    #     if args.abnormal_model is None:
    #         raise ValueError("Enter abnormal model path for `acl` or `meniscus` task")

    os.makedirs(args.rundir, exist_ok=True)

    with open(Path(args.rundir) / 'args.json', 'w') as out:
        json.dump(vars(args), out, indent=4)

    train(args.rundir, args.task, args.backbone, args.epochs, args.learning_rate, args.gpu, abnormal_model_path=args.abnormal_model)