text:
import os
import sys
import shutil
import tempfile
from textwrap import dedent
from tests.utils.runtest import automakesuite, run
from tests.utils.testcase import TestCase
from tests.utils.functionaltestcase import FunctionalTestCase

from Ironclad import PythonMapper

bz2_doc = """The python bz2 module provides a comprehensive interface for
the bz2 compression library. It implements a complete file
interface, one shot (de)compression functions, and types for
sequential (de)compression.
"""

bz2_compress_doc = """compress(data [, compresslevel=9]) -> string

Compress data in one shot. If you want to compress data sequentially,
use an instance of BZ2Compressor instead. The compresslevel parameter, if
given, must be a number between 1 and 9.
"""

bz2_decompress_doc = """decompress(data) -> decompressed data

Decompress data in one shot. If you want to decompress data sequentially,
use an instance of BZ2Decompressor instead.
"""

bz2___author__ = """The bz2 python module was written by:

    Gustavo Niemeyer <niemeyer@conectiva.com>
"""

bz2_test_str = "I wonder why. I wonder why. I wonder why I wonder why. "
bz2_test_text = bz2_test_str * 1000
bz2_test_data = 'BZh91AY&SYM\xf6FM\x00!\xd9\x95\x80@\x01\x00 \x06A\x90\xa0 \x00\x90 \x1ai\xa0)T4\x1bS\x81R\xa6\tU\xb0J\xad\x02Ur*T\xd0%W\xc0\x95^\x02U`%V\x02Up\tU\xb0J\xae\xc0\x95X\tU\xf8\xbb\x92)\xc2\x84\x82o\xb22h'

bz2_test_line = "I wonder why. I wonder why. I wonder why I wonder why.\n"
bz2_test_text_lines = bz2_test_line * 1000

DLL_PATH = os.path.join("build", "ironclad", "python27.dll")

def ModuleTestCase(module):
    import_code = 'import ' + module
    locals_ = locals()
    class _ModuleTestCase(TestCase):
        __name__ = module + 'Test'
        def assertRuns(self, test_code=''):
            mapper = PythonMapper(DLL_PATH)
            try:
                exec '\n'.join([import_code, test_code]) in globals(), locals_
            finally:
                mapper.Dispose()
                if module in sys.modules:
                    del sys.modules[module]
    return _ModuleTestCase

def TrivialModuleTestCase(module):
    test = ModuleTestCase(module)
    test.testImports = test.assertRuns
    return test

class ExternalFunctionalityTest(FunctionalTestCase):

    def testImportHook(self):
        self.assertRuns(dedent("""\
            import bz2
            assert bz2.compress(%(uncompressed)r) == %(compressed)r
            assert bz2.decompress(%(compressed)r) == %(uncompressed)r
            # check that clr imports still work
            import clr
            import System
            """) % {
            "compressed": bz2_test_data,
            "uncompressed": bz2_test_text})

    def testNumPySciPyH5Py(self):
        # combine several tests into one, so we don't have to wait for multiple reloads
        file_path = os.path.abspath(os.path.join('tests', 'data', 'text.txt'))
        file_contents = list(open(file_path).read())

        self.assertRuns(dedent("""\
            #=====================================================
            # test we can actually import them

            import scipy as sp
            import numpy as np

            #=====================================================
            # perform a few basic numpy sanity checks

            assert np.core.multiarray.__name__ == 'numpy.core.multiarray'

            r1 = np.arange(20)
            r2 = np.arange(20)
            assert r1 is not r2
            assert np.all(r1 == r2)

            r1[...] = 20
            assert np.all(r1 > r2)

            def assert_equals(a, b):
                assert a == b, '%%r != %%r' %% (a, b)
            assert_equals((1+3j), np.complex128(1+3j))

            #=====================================================
            # test matrices probably work

            m = np.matrix([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
            assert abs(np.linalg.det(m)) < 0.0000001

            #=====================================================
            # test memmap when patched

            ironclad.patch_native_filenos()
            f = np.core.memmap(%r)
            assert map(chr, f) == %r

            #=====================================================
            # adapted from numpy.core.test_print.TestPrint
            # This test is a placeholder for one which fails
            # due to float formatting differences.

            def mangle(num):
                return str(num).lower().replace('+0', '+')
            def assert_complex_type(type):
                for x in (0, 1,-1, 1e10, 1e20):
                    assert_equals(mangle(type(x)),
                                  mangle(complex(x)).lower())
                    assert_equals(mangle(type(x*1j)).lower(),
                                  mangle(complex(x*1j)).lower())
                    assert_equals(mangle(type(x + x*1j)).lower(),
                                  mangle(complex(x + x*1j)).lower())

            assert_complex_type(np.cfloat)
            assert_complex_type(np.cdouble)

            # this one is a known failure in certain environments,
            # butI'm not sure how to identify those environments.
            #
            # assert_complex_type(np.clongdouble)

            #=====================================================
            # test for no dupe h5 imports
            # we still have dupe .py imports from .pyds for some reason

            import h5py
            import sys
            assert 'h5py.h5g.h5' not in sys.modules # .pyd dupes
            assert 'h5py.h5i.h5' not in sys.modules # .pyd dupes
            assert 'h5py.h5t.h5' not in sys.modules # .pyd dupes
            assert 'h5py.h5g._sync' not in sys.modules # .py dupes
            assert 'h5py.h5i._sync' not in sys.modules # .py dupes
            assert 'h5py.h5t._sync' not in sys.modules # .py dupes

            """ % (file_path, file_contents)), insert_args='-X:Frames')

    def testPatchNativeFilenos(self):
        self.assertRuns(dedent("""\
            import os
            import sys
            builtinBefore = dict(sys.modules['__builtin__'].__dict__)
            osBefore = dict(sys.modules['os'].__dict__)

            ironclad.patch_native_filenos()

            # check kwargs propagated
            f = open(%r, mode='r')
            f.close()

            ironclad.unpatch_native_filenos()

            for k, v in builtinBefore.items():
                assert sys.modules['__builtin__'].__dict__[k] is v
            for k, v in osBefore.items():
                assert sys.modules['os'].__dict__[k] is v

            """ % os.path.abspath(__file__)))

class NumpyTest(FunctionalTestCase):

    def testNumpyCrash(self):
        self.assertRuns(dedent("""
            import numpy
            from System.Threading import Thread, ThreadStart
            a = numpy.matrix([[1, 2], [3, 4]])
            b = numpy.matrix([[1, 2], [3, 4]])

            def multiInvert(a):
                for ii in xrange(1000):
                    a = a.I

            t1 = Thread(ThreadStart(lambda: multiInvert(a)))
            t2 = Thread(ThreadStart(lambda: multiInvert(b)))
            t1.Start()
            t2.Start()
            t1.Join()
            t2.Join()
            """), insert_args='-X:Frames -X:Debug')

class BZ2Test(ModuleTestCase('bz2')):

    def testFunctionsAndDocstringsExist(self):
        self.assertRuns(dedent("""
            assert bz2.__doc__ == %r
            assert bz2.__author__ == %r
            assert callable(bz2.compress)
            assert bz2.compress.__doc__ == %r
            assert callable(bz2.decompress)
            assert bz2.decompress.__doc__ == %r
            """) % (bz2_doc,
                    bz2___author__,
                    bz2_compress_doc,
                    bz2_decompress_doc))

    def testFunctionsWork(self):
        self.assertRuns(dedent("""
            assert bz2.compress(%(uncompressed)r) == %(compressed)r
            assert bz2.decompress(%(compressed)r) == %(uncompressed)r
            """) % {"compressed": bz2_test_data,
                    "uncompressed": bz2_test_text})

    def testTypesExist(self):
        self.assertRuns(dedent("""
            assert bz2.BZ2File.__name__ == 'BZ2File'
            assert bz2.BZ2File.__module__ == 'bz2'
            assert bz2.BZ2Compressor.__name__ == 'BZ2Compressor'
            assert bz2.BZ2Compressor.__module__ == 'bz2'
            assert bz2.BZ2Decompressor.__name__ == 'BZ2Decompressor'
            assert bz2.BZ2Decompressor.__module__ == 'bz2'
            """))

    def testCompressor(self):
        self.assertRuns(dedent("""
            # adapted from test_bz2.py
            compressor = bz2.BZ2Compressor()
            text = %r
            chunkSize = 50
            n = 0
            data = ''
            while 1:
                chunk = text[n*chunkSize:(n+1)*chunkSize]
                if not chunk:
                    break
                data += compressor.compress(chunk)
                n += 1
            data += compressor.flush()
            assert data == %r
            """) % (bz2_test_text, bz2_test_data))

    def testDecompressor(self):
        self.assertRuns(dedent("""
            # adapted from test_bz2.py
            decompressor = bz2.BZ2Decompressor()
            data = %r
            chunkSize = 5
            n = 0
            text = ''
            while 1:
                chunk = data[n*chunkSize:(n+1)*chunkSize]
                if not chunk:
                    break
                text += decompressor.decompress(chunk)
                n += 1
            assert text == %r
            """) % (bz2_test_data, bz2_test_text))

    def testFileRead(self):
        testPath = os.path.join("tests", "data", "bz2", "compressed.bz2")
        self.assertRuns(dedent("""
            f = bz2.BZ2File(%r)
            try:
                assert f.read() == %r
            finally:
                f.close()
            """) % (testPath, bz2_test_text))

    def testFileReadLine(self):
        testPath = os.path.join("tests", "data", "bz2", "compressed.bz2")
        self.assertRuns(dedent("""
            f = bz2.BZ2File(%r)
            try:
                assert f.readline() == %r
                assert f.readline() == ''
            finally:
                f.close()
            """) % (testPath, bz2_test_text))

    def testFileReadLines_Short(self):
        testPath = os.path.join("tests", "data", "bz2", "compressedlines.bz2")
        self.assertRuns(dedent("""
            f = bz2.BZ2File(%r)
            try:
                lines = f.readlines()
                assert len(lines) == 1000
                assert ''.join(lines) == %r
            finally:
                f.close()
            """) % (testPath, bz2_test_text_lines))

    def testFileReadLines_Long(self):
        testPath = os.path.join("tests", "data", "bz2", "compressed.bz2")
        self.assertRuns(dedent("""
            f = bz2.BZ2File(%r)
            try:
                lines = f.readlines()
                assert lines == [%r]
            finally:
                f.close()
            """) % (testPath, bz2_test_text))

    def testFileIterate(self):
        testPath = os.path.join("tests", "data", "bz2", "compressedlines.bz2")
        self.assertRuns(dedent("""
            f = bz2.BZ2File(%r)
            try:
                assert f.xreadlines() is f
                assert f.__iter__() is f
                for _ in range(1000):
                    line = f.next()
                    assert line == %r
                try:
                    f.next()
                    raise AssertionError("iteration did not stop")
                except StopIteration:
                    pass
            finally:
                f.close()
            """) % (testPath, bz2_test_line))

    def testFileSeekTell(self):
        testPath = os.path.join("tests", "data", "bz2", "compressed.bz2")
        self.assertRuns(dedent("""
            def assertSeeksTo(f, seekargs, expectedPosition):
                f.seek(*seekargs)
                assert f.tell() == expectedPosition
                assert f.read() == %r[expectedPosition:]
                f.seek(expectedPosition)

            f = bz2.BZ2File(%r)
            try:
                assertSeeksTo(f, (55000,), 55000)
                assertSeeksTo(f, (0,), 0)
                assertSeeksTo(f, (12345, 0), 12345)
                assertSeeksTo(f, (10000, 1), 22345)
                assertSeeksTo(f, (-10000, 1), 12345)
                assertSeeksTo(f, (-55000, 2), 0)
            finally:
                f.close()
            """) % (bz2_test_text, testPath))

    def testFileWrite(self):
        # read is tested separately elsewhere, so we assume it works
        testDir = tempfile.mkdtemp()
        path = os.path.join(testDir, "test.bz2")
        self.assertRuns(dedent("""
            w = bz2.BZ2File(%r, 'w')
            try:
                w.write(%r)
            finally:
                w.close()
            r = bz2.BZ2File(%r, 'r')
            try:
                assert r.read() == %r
            finally:
                r.close()
            """) % (path, bz2_test_text, path, bz2_test_text))
        shutil.rmtree(testDir)

    def testHeavyUse(self):
        self.assertRuns(dedent("""
            COUNT = 100
            compressors = [bz2.BZ2Compressor() for _ in range(COUNT)]
            decompressors = [bz2.BZ2Decompressor() for _ in range(COUNT)]

            current = %r
            for i in range(COUNT):
                current = decompressors[-i].decompress(
                    compressors[i].compress(current[:i * 40]) +
                    compressors[i].compress(current[i * 40:]) +
                    compressors[i].flush())

            assert current == %r
            """) % (bz2_test_text, bz2_test_text))

    def testFileProperties(self):
        testPath = os.path.join("tests", "data", "bz2", "compressedlines.bz2")
        self.assertRuns(dedent("""
            f = bz2.BZ2File(%r, 'U')
            try:
                #assert f.mode == 'r' # IP2 bug -- no file.mode
                assert f.name == %r
                assert f.closed == False
                assert f.newlines == None

                f.readlines()
                assert f.newlines == '\\n'

                for attr in ['mode', 'name', 'closed', 'newlines']:
                    try:
                        setattr(f, attr, 123)
                    except AttributeError:
                        pass
                    else:
                        raise AssertionError('unexpected settable attribute: ' + attr)
            finally:
                f.close()
            assert f.closed == True
            """) % (testPath, testPath))

    def testFileMember(self):
        testPath = os.path.join("tests", "data", "bz2", "compressedlines.bz2")
        self.assertRuns(dedent("""
            f = bz2.BZ2File(%r)
            try:
                assert f.softspace == 0
                f.softspace = 1
                assert f.softspace == 1
            finally:
                f.close()
            """) % testPath)

    def assertFileWriteLines(self, sequence):
        # read is tested separately elsewhere, so we assume it works
        testDir = tempfile.mkdtemp()
        path = os.path.join(testDir, "test.bz2")
        self.assertRuns(dedent("""
            w = bz2.BZ2File(%r, 'w')
            try:
                w.writelines(%r)
            finally:
                w.close()
            r = bz2.BZ2File(%r, 'r')
            try:
                assert r.read() == %r
            finally:
                r.close()
            """) % (path, sequence, path, bz2_test_text))
        shutil.rmtree(testDir)

    def testFileWriteLines_List(self):
        self.assertFileWriteLines([bz2_test_str] * 1000)

    def testFileWriteLines_Tuple(self):
        self.assertFileWriteLines(tuple([bz2_test_str] * 1000))

class HashlibTest(ModuleTestCase('_hashlib')):

    def testTrivial(self):
        self.assertRuns(dedent("""
            assert _hashlib.openssl_md5('foobarbaz').digest() == u'm\xf2=\xc0?\x9bT\xcc8\xa0\xfc\x14\x83\xdfn!'
            """))

class PyFileTest(FunctionalTestCase):

    def testPyFile(self):
        # this is the CPython file type, interpreted just like an extension type
        mapper = PythonMapper(DLL_PATH)
        try:
            f1 = mapper.CPyFileClass(os.path.join(self.testDir, 'newFile'), 'w')
            f1.write("hello!")
            f1.close()

            f2 = mapper.CPyFileClass(os.path.join(self.testDir, 'newFile'), 'r')
            assert f2.read() == 'hello!'
            f2.close()
        finally:
            mapper.Dispose()

# TODO: crashes test run
# Sqlite3Test = TrivialModuleTestCase('sqlite3') # test PATH manipulation in LoadModule
PySVNTest = TrivialModuleTestCase('pysvn') # test misleading names passed to Py_InitModule4
MMapTest = TrivialModuleTestCase('mmap')
CsvTest = TrivialModuleTestCase('csv')

suite = automakesuite(locals())
if __name__ == '__main__':
    run(suite)

max_stars_repo_name: paweljasinski/ironclad

max_issues_repo_name: paweljasinski/ironclad

max_forks_repo_name: paweljasinski/ironclad

reason: import numpy,from numpy,import scipy


text:
# yellowbrick.features.radviz
# Implements radviz for feature analysis.
#
# Author:   Benjamin Bengfort <bbengfort@districtdatalabs.com>
# Created:  Fri Oct 07 13:18:00 2016 -0400
#
# Copyright (C) 2016 District Data Labs
# For license information, see LICENSE.txt
#
# ID: radviz.py [0f4b236] benjamin@bengfort.com $

"""
Implements radviz for feature analysis.
"""

##########################################################################
## Imports
##########################################################################

import numpy as np
import matplotlib.patches as patches

from yellowbrick.draw import manual_legend
from yellowbrick.utils import is_dataframe
from yellowbrick.utils import nan_warnings
from yellowbrick.features.base import DataVisualizer
from yellowbrick.style.colors import resolve_colors

##########################################################################
## Quick Methods
##########################################################################

def radviz(X, y=None, ax=None, features=None, classes=None,
           color=None, colormap=None, alpha=1.0, **kwargs):
    """
    Displays each feature as an axis around a circle surrounding a scatter
    plot whose points are each individual instance.

    This helper function is a quick wrapper to utilize the RadialVisualizer
    (Transformer) for one-off analysis.

    Parameters
    ----------

    X : ndarray or DataFrame of shape n x m
        A matrix of n instances with m features

    y : ndarray or Series of length n
        An array or series of target or class values

    ax : matplotlib Axes, default: None
        The axes to plot the figure on.

    features : list of strings, default: None
        The names of the features or columns

    classes : list of strings, default: None
        The names of the classes in the target

    color : list or tuple of colors, default: None
        Specify the colors for each individual class

    colormap : string or matplotlib cmap, default: None
        Sequential colormap for continuous target

    alpha : float, default: 1.0
        Specify a transparency where 1 is completely opaque and 0 is completely
        transparent. This property makes densely clustered points more visible.

    Returns
    -------
    ax : matplotlib axes
        Returns the axes that the parallel coordinates were drawn on.
    """
    # Instantiate the visualizer
    visualizer = RadialVisualizer(
        ax, features, classes, color, colormap, alpha, **kwargs
    )

    # Fit and transform the visualizer (calls draw)
    visualizer.fit(X, y, **kwargs)
    visualizer.transform(X)

    # Return the axes object on the visualizer
    return visualizer.ax

##########################################################################
## Static RadViz Visualizer
##########################################################################

class RadialVisualizer(DataVisualizer):
    """
    RadViz is a multivariate data visualization algorithm that plots each
    axis uniformely around the circumference of a circle then plots points on
    the interior of the circle such that the point normalizes its values on
    the axes from the center to each arc.

    Parameters
    ----------

    ax : matplotlib Axes, default: None
        The axis to plot the figure on. If None is passed in the current axes
        will be used (or generated if required).

    features : list, default: None
        a list of feature names to use
        If a DataFrame is passed to fit and features is None, feature
        names are selected as the columns of the DataFrame.

    classes : list, default: None
        a list of class names for the legend
        If classes is None and a y value is passed to fit then the classes
        are selected from the target vector.

    color : list or tuple, default: None
        optional list or tuple of colors to colorize lines
        Use either color to colorize the lines on a per class basis or
        colormap to color them on a continuous scale.

    colormap : string or cmap, default: None
        optional string or matplotlib cmap to colorize lines
        Use either color to colorize the lines on a per class basis or
        colormap to color them on a continuous scale.

    alpha : float, default: 1.0
        Specify a transparency where 1 is completely opaque and 0 is completely
        transparent. This property makes densely clustered points more visible.

    kwargs : dict
        Keyword arguments that are passed to the base class and may influence
        the visualization as defined in other Visualizers.

    Examples
    --------

    >>> visualizer = RadViz()
    >>> visualizer.fit(X, y)
    >>> visualizer.transform(X)
    >>> visualizer.poof()

    Notes
    -----
    These parameters can be influenced later on in the visualization
    process, but can and should be set as early as possible.
    """

    def __init__(self, ax=None, features=None, classes=None, color=None,
                 colormap=None, alpha=1.0, **kwargs):
        super(RadialVisualizer, self).__init__(
            ax, features, classes, color, colormap, **kwargs
        )
        self.alpha = alpha

    @staticmethod
    def normalize(X):
        """
        MinMax normalization to fit a matrix in the space [0,1] by column.
        """
        a = X.min(axis=0)
        b = X.max(axis=0)
        return (X - a[np.newaxis, :]) / ((b - a)[np.newaxis, :])

    def draw(self, X, y, **kwargs):
        """
        Called from the fit method, this method creates the radviz canvas and
        draws each instance as a class or target colored point, whose location
        is determined by the feature data set.
        """
        # Convert from dataframe
        if is_dataframe(X):
            X = X.values

        # Clean out nans and warn that the user they aren't plotted
        nan_warnings.warn_if_nans_exist(X)
        X, y = nan_warnings.filter_missing(X, y)

        # Get the shape of the data
        nrows, ncols = X.shape

        # Set the axes limits
        self.ax.set_xlim([-1,1])
        self.ax.set_ylim([-1,1])

        # Create the colors
        # TODO: Allow both colormap, listed colors, and palette definition
        # TODO: Make this an independent function or property for override!
        color_values = resolve_colors(
            n_colors=len(self.classes_), colormap=self.colormap, colors=self.color
        )
        self._colors = dict(zip(self.classes_, color_values))

        # Create a data structure to hold scatter plot representations
        to_plot = {}
        for kls in self.classes_:
            to_plot[kls] = [[], []]

        # Compute the arcs around the circumference for each feature axis
        # TODO: make this an independent function for override
        s = np.array([
                (np.cos(t), np.sin(t))
                for t in [
                    2.0 * np.pi * (i / float(ncols))
                    for i in range(ncols)
                ]
            ])

        # Compute the locations of the scatter plot for each class
        # Normalize the data first to plot along the 0, 1 axis
        for i, row in enumerate(self.normalize(X)):
            row_ = np.repeat(np.expand_dims(row, axis=1), 2, axis=1)
            xy   = (s * row_).sum(axis=0) / row.sum()
            kls = self.classes_[y[i]]

            to_plot[kls][0].append(xy[0])
            to_plot[kls][1].append(xy[1])

        # Add the scatter plots from the to_plot function
        # TODO: store these plots to add more instances to later
        # TODO: make this a separate function
        for i, kls in enumerate(self.classes_):
            self.ax.scatter(
                to_plot[kls][0], to_plot[kls][1], color=self._colors[kls],
                label=str(kls), alpha=self.alpha, **kwargs
            )

        # Add the circular axis path
        # TODO: Make this a seperate function (along with labeling)
        self.ax.add_patch(patches.Circle(
            (0.0, 0.0), radius=1.0, facecolor='none', edgecolor='grey', linewidth=.5
        ))

        # Add the feature names
        for xy, name in zip(s, self.features_):
            # Add the patch indicating the location of the axis
            self.ax.add_patch(patches.Circle(xy, radius=0.025, facecolor='#777777'))

            # Add the feature names offset around the axis marker
            if xy[0] < 0.0 and xy[1] < 0.0:
                self.ax.text(xy[0] - 0.025, xy[1] - 0.025, name, ha='right', va='top', size='small')
            elif xy[0] < 0.0 and xy[1] >= 0.0:
                self.ax.text(xy[0] - 0.025, xy[1] + 0.025, name, ha='right', va='bottom', size='small')
            elif xy[0] >= 0.0 and xy[1] < 0.0:
                self.ax.text(xy[0] + 0.025, xy[1] - 0.025, name, ha='left', va='top', size='small')
            elif xy[0] >= 0.0 and xy[1] >= 0.0:
                self.ax.text(xy[0] + 0.025, xy[1] + 0.025, name, ha='left', va='bottom', size='small')

        self.ax.axis('equal')

    def finalize(self, **kwargs):
        """
        Finalize executes any subclass-specific axes finalization steps.
        The user calls poof and poof calls finalize.

        Parameters
        ----------
        kwargs: generic keyword arguments.

        """
        # Set the title
        self.set_title(
            'RadViz for {} Features'.format(len(self.features_))
        )

        # Remove the ticks from the graph
        self.ax.set_yticks([])
        self.ax.set_xticks([])

        # Add the legend
        colors = [self._colors[c] for c in self.classes_]
        manual_legend(self, self.classes_, colors, loc='best')

# Alias for RadViz
RadViz = RadialVisualizer

ext: py

max_stars_repo_name: souravsingh/yellowbrick

max_issues_repo_name: souravsingh/yellowbrick

max_forks_repo_name: souravsingh/yellowbrick

reason: import numpy


text:
# Copyright (c) 2020, Fabio Muratore, Honda Research Institute Europe GmbH, and
# Technical University of Darmstadt.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. Neither the name of Fabio Muratore, Honda Research Institute Europe GmbH,
#    or Technical University of Darmstadt, nor the names of its contributors may
#    be used to endorse or promote products derived from this software without
#    specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL FABIO MURATORE, HONDA RESEARCH INSTITUTE EUROPE GMBH,
# OR TECHNICAL UNIVERSITY OF DARMSTADT BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import numpy as np
import torch as to
from matplotlib import pyplot as plt
from matplotlib import ticker

import pyrado
from pyrado.policies.features import FeatureStack, RBFFeat
from pyrado.policies.feed_back.linear import LinearPolicy
from pyrado.spaces import BoxSpace
from pyrado.utils.data_types import EnvSpec

if __name__ == "__main__":
    # Define some arbitrary EnvSpec
    obs_space = BoxSpace(bound_lo=np.array([-5.0, -12.0]), bound_up=np.array([10.0, 6.0]))
    act_space = BoxSpace(bound_lo=np.array([-1.0]), bound_up=np.array([1.0]))
    spec = EnvSpec(obs_space, act_space)

    num_fpd = 5
    num_eval_points = 500

    policy_hparam = dict(feats=FeatureStack(RBFFeat(num_feat_per_dim=num_fpd, bounds=obs_space.bounds, scale=None)))
    policy = LinearPolicy(spec, **policy_hparam)

    eval_grid_0 = to.linspace(-5.0, 10, num_eval_points)
    eval_grid_1 = to.linspace(-12.0, 6, num_eval_points)
    eval_grid = to.stack([eval_grid_0, eval_grid_1], dim=1)

    feat_vals = to.empty(num_eval_points, num_fpd * obs_space.flat_dim)
    # Feed evaluation samples one by one
    for i, obs in enumerate(eval_grid):
        feat_vals[i, :] = policy.eval_feats(obs)

    feat_vals_batch = policy.eval_feats(eval_grid)

    if (feat_vals == feat_vals_batch).all():
        feat_vals = feat_vals_batch
    else:
        raise pyrado.ValErr(msg="Batch mode failed")

    if num_eval_points <= 50:
        # Plot the feature values over the feature index
        _, axs = plt.subplots(obs_space.flat_dim, 1, figsize=(10, 8), tight_layout=False)
        for i, fv in enumerate(feat_vals):
            axs[0].plot(np.arange(num_fpd), fv[:num_fpd].numpy(), label=i)
            axs[1].plot(np.arange(num_fpd), fv[num_fpd:].numpy(), label=i)
        axs[0].legend(
            title="eval point",
            ncol=num_eval_points // 2,
            loc="upper center",
            bbox_to_anchor=(0.0, 1.1, 1.0, 0.3),
            mode="expand",
        )
        axs[0].set_title("reconstructed input dim 1")
        axs[1].set_title("reconstructed input dim 2")
        axs[1].set_xlabel("feature")
        axs[0].set_ylabel("activation")
        axs[1].set_ylabel("activation")
        axs[0].xaxis.set_major_locator(ticker.MaxNLocator(num_fpd, integer=True))
        axs[1].xaxis.set_major_locator(ticker.MaxNLocator(num_fpd, integer=True))

    # Plot the feature values over the input samples
    _, axs = plt.subplots(obs_space.flat_dim, 1, figsize=(10, 8), tight_layout=False)
    for i in range(obs_space.flat_dim):
        for j, fv in enumerate(feat_vals[:, i * num_fpd : (i + 1) * num_fpd].T):
            axs[i].plot(fv.numpy(), label=j, c=f"C{j%10}")
        axs[i].legend()

    axs[0].set_title("reconstructed input dim 1")
    axs[1].set_title("reconstructed input dim 2")
    axs[1].set_xlabel("input sample")
    axs[0].set_ylabel("activation")
    axs[1].set_ylabel("activation")

    plt.show()

max_stars_repo_name: theogruner/SimuRLacra

max_stars_repo_licenses: DOC Zlib BSD-3-Clause

max_issues_repo_name: theogruner/SimuRLacra

max_issues_repo_licenses: DOC Zlib BSD-3-Clause

max_forks_repo_name: theogruner/SimuRLacra

max_forks_repo_licenses: DOC Zlib BSD-3-Clause


text:
#!/usr/bin/env python3
import argparse
import random
import numpy as np

parser = argparse.ArgumentParser()
parser.add_argument('input')
parser.add_argument('--output')
parser.add_argument('-n', type=int, default=0)

parser.add_argument('--input-txt', nargs='*')
parser.add_argument('--output-txt', nargs='*')

args = parser.parse_args()

if not args.output:
    args.output = args.input
if args.input_txt and not args.output_txt:
    args.output_txt = args.input_txt

with open(args.input, 'rb') as input_file:
    n, dim = np.load(input_file)

    indices = list(range(n))
    random.shuffle(indices)

    if args.n > 0:
        indices = indices[:args.n]

    frames = []

    for _ in range(n):
        feats = np.load(input_file)
        frames.append(feats)

with open(args.output, 'wb') as output_file:
    np.save(output_file, (len(indices), dim))
    for index in indices:
        feats = frames[index]
        np.save(output_file, feats)

if args.input_txt and args.output_txt:
    lines = []
    for input_filename in args.input_txt:
        with open(input_filename) as input_file:
            lines.append(input_file.readlines())

    for lines_, output_filename in zip(lines, args.output_txt):
        with open(output_filename, 'w') as output_file:
            for index in indices:
                line = lines_[index]
                output_file.write(line)


text:
"""
Perform a gridsearch to identify the best number of nodes and batch size
"""

import sys
import time
import numpy as np
import pandas as pd

import lasagne
from lasagne import layers
from lasagne import nonlinearities
from lasagne.nonlinearities import ScaledTanH
from nolearn.lasagne import NeuralNet, TrainSplit, RememberBestWeights, \
	BatchIterator
from sklearn import preprocessing
from sklearn.grid_search import GridSearchCV

script_start_time = time.time()

# first argument passed is the column of the output measure to train
output_col = int(sys.argv[1])

# specify the seed for random number generation so we can get consistent
# shuffling and initialized weights
np.random.seed(6509)

# Read the training data for the neural network
# Input data is 250000x4 and output data is 250000x32
X = pd.read_csv('../../parameters_250000.txt', sep=' ')
Y = pd.read_csv('../../results.txt', sep=' ', index_col=False)
# These functional groups do not exist in my model
Y = Y.drop(['light_aromatic_C-C', 'light_aromatic_methoxyl'], axis=1)
y_columns = Y.columns.values
X = X.values.astype(np.float32)
Y = Y.values.astype(np.float32)

# Shuffle the dataset (because x parameters are varied in a structured way)
combined = np.concatenate((X, Y), axis=1)
np.random.shuffle(combined)

# Separate the data into training (with included validation) and test sets.
# (Validation set is separated from the training set by nolearn)
test_fraction = 0.2
training = combined[:-int(test_fraction * combined.shape[0]), :]
test = combined[-int(test_fraction * combined.shape[0]):, :]

x_train = training[:, :4]
y_train = training[:, 4:]
x_test = test[:, :4]
y_test = test[:, 4:]

# Standardize the input & output parameters to have zero mean and unit variance
# x_scaler.transform() can be used later to transform any new data
# x_scaler.inverse_transform() can be used to get the original data back
x_scaler = preprocessing.StandardScaler().fit(x_train)
x_train = x_scaler.transform(x_train)
x_test = x_scaler.transform(x_test)
y_scaler = preprocessing.StandardScaler().fit(y_train)
y_train = y_scaler.transform(y_train)
y_test = y_scaler.transform(y_test)

# set up the Scaled tanh parameters.  See nonlinearities.py for usage notes.
# I am following the guidance of LeCun et al. for these values
scaled_tanh = ScaledTanH(scale_in=2./3, scale_out=1.7159)

# Use RememberBestWeights to use the parameters that yielded the best loss
# on the validation set.
rbw = RememberBestWeights()

# implement early stopping so we don't have to reach max_epochs
class EarlyStopping(object):
    def __init__(self, patience=100):
        self.patience = patience
        self.best_valid = np.inf
        self.best_valid_epoch = 0
        self.best_weights = None

    def __call__(self, nn, train_history):
        current_valid = train_history[-1]['valid_loss']
        current_epoch = train_history[-1]['epoch']
        if current_valid < self.best_valid:
            self.best_valid = current_valid
            self.best_valid_epoch = current_epoch
            self.best_weights = nn.get_all_params_values()
        elif self.best_valid_epoch + self.patience < current_epoch:
            print("Early stopping.")
            print("Best valid loss was {:.6f} at epoch {}.".format(
                self.best_valid, self.best_valid_epoch))
            nn.load_params_from(self.best_weights)
            raise StopIteration()

# NeuralNet automatically splits the data provided into a training
# and a validation set, using 20% of the samples for validation.
# You can adjust this ratio by overriding the eval_size=0.2 parameter.
# .
# adagrad scales the learning rates by dividing by the square root of
# accumulated squared gradients.  This is an adaptive learning rate
# technique.  See the references in updates.py for details.
# .
# setting the first dimension of input_shape to None translates to using a
# variable batch size
# .
# the initial weights are initialized from a uniform distribution with a
# cleverly chosen interval - Lasagne figures out this interval using
# Glorot-style initialization
# (http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf)
# .
# Mean squared error is the default objective function to minimize
net = NeuralNet(
            layers=[
                ('input', layers.InputLayer),
                ('hidden0', layers.DenseLayer),
                ('hidden1', layers.DenseLayer),
                ('output', layers.DenseLayer)
                ],
            input_shape=(None, x_train.shape[1]),
            hidden0_num_units=15,
            hidden0_nonlinearity=scaled_tanh,
            hidden1_num_units=15,
            hidden1_nonlinearity=scaled_tanh,
            output_num_units=1,
            output_nonlinearity=nonlinearities.linear,
            regression=True,
            verbose=1,
            max_epochs=250,
            update=lasagne.updates.adagrad,
#            on_epoch_finished=[EarlyStopping(patience=100), rbw],
#            on_training_finished=[rbw.restore],
            train_split=TrainSplit(eval_size=0.3),
            batch_iterator_train=BatchIterator(batch_size=128),
            )

# Set up the gridsearch
param_grid = {'hidden0_num_units': range(5, 32),
              'hidden1_num_units': range(5, 32),
              }

grid_search = GridSearchCV(net, param_grid, verbose=0, n_jobs=20,
                           pre_dispatch='2*n_jobs',
                           scoring='mean_squared_error')

# select only the output measure you want to train
y_train2 = y_train[:, output_col]
# Train the network parameters using a subset of the entire training set
grid_search.fit(x_train[:6000, :], y_train2[:6000])

script_running_time = time.time() - script_start_time

with open('updated_gridsearch_single_report.txt', 'a') as report:
    print >>report, ('%s - %s took % sec\n\n' % (output_col, y_columns[output_col],
                                                 script_running_time))
    for entry in grid_search.grid_scores_:
       print >>report, entry
    print >>report, '\n\nbest: %s - %s' % (output_col, grid_search.best_params_)


text:
from numpy import *
import numpy
import random
import time

def load_rating_data(file_path='MovieLens/u.data'):
    ratings=[]
    for line in open(file_path,'r'):
        data=line.split('\t')
        uid=int(data[0])
        mid=int(data[1])
        rat=int(data[2])
        ratings.append([uid,mid,rat])
    data=array(ratings)
    return data

def split_rating_data(data,ratio=0.2):
    train_data=[]
    test_data=[]
    for line in data:
        num=random.random()
        if num < 0.2:
            test_data.append(line)
        else:
            train_data.append(line)
    train_data=array(train_data)
    test_data=array(test_data)
    return train_data,test_data

def getNearestNeighbor(userId, cgsimmat):
    neighbors_dist = []
    for i in range(1,u_num):
        if cgsimmat[userId][i]!=0:
            neighbors_dist.append([cgsimmat[userId][i],i])
    neighbors_dist.sort(reverse=True)
    return neighbors_dist

def getTopN(groupID, ratings):
    topn_ratings = []
    for i in range(1,i_num):
        if ratings[groupID][i]!=0:
            topn_ratings.append([ratings[groupID][i],i])
    topn_ratings.sort(reverse=True)
    return topn_ratings

def integrate_strategy():
    if strategy == "AVG":
        for i in range(len(groups)):
            for k in range(len(group_cand[i])):
                total_rating = 0
                for j in range(group_len):
                    total_rating += predict_mat[groups[i][j]][group_cand[i][k]]
                cand_ratings[i][group_cand[i][k]] = total_rating / group_len

    elif strategy == "MP":
        for i in range(len(groups)):
            for j in range(len(group_cand[i])):
                max = -1
                for k in groups[i]:
                    if predict_mat[k][group_cand[i][j]]>max:
                        max = predict_mat[k][group_cand[i][j]]
                cand_ratings[i][group_cand[i][j]] = max

    elif strategy == "LM":
        for i in range(len(groups)):
            for j in range(len(group_cand[i])):
                min = 100
                for k in groups[i]:
                    if predict_mat[k][group_cand[i][j]]<min and predict_mat[k][group_cand[i][j]]!=0:
                        min = predict_mat[k][group_cand[i][j]]
                cand_ratings[i][group_cand[i][j]] = min

if __name__ == '__main__':#只有用到训练集和测试集的编号才需要-1
    #beta=0.6
    para = {}
    i=0
    with open('UBCF.txt') as f:  # 需要重新打开文本进行读取
        for line in f:
            content = line.rstrip()  # 删除字符串末尾的空白
            if(len(content)>1):
                kv = content.split('=')
                para[kv[0]] = kv[1]
    f.close()
    group_len = para['gSize']
    neighbor_num = para['neighborNum']
    top_n = para['topN']
    strategy = para['gStrategy']
    data=load_rating_data()
    file_path = 'MovieLens/u1.base'
    train_data = []
    with open(file_path) as f:
        for line in f:
            tdata = line.split('\t')
            uid = int(tdata[0])
            mid = int(tdata[1])
            rat = int(tdata[2])
            train_data.append([uid, mid, rat])
    f.close()
    train_data = array(train_data)

    file_path = 'MovieLens/u1.test'
    test_data = []
    with open(file_path) as f:
        for line in open(file_path, 'r'):
            tdata = line.split('\t')
            uid = int(tdata[0])
            mid = int(tdata[1])
            rat = int(tdata[2])
            test_data.append([uid, mid, rat])
    f.close()
    test_data = array(test_data)

    u_num = max(data[:,0])+1
    i_num = max(data[:,1])+1
    rating_mat = zeros((u_num,i_num),dtype=int16)#评分矩阵

    participant_list = []
    for i in range(i_num):
        participant_list.append([])

    for ratingtuple in data:
        (i, j, rat) = ratingtuple
        participant_list[j].append(i)

    # sum = 0
    # for i in range(1, i_num):
    #     sum += len(participant_list[i])
    # sum = sum / (i_num - 1)

    group_len = int(group_len)
    modified = 0
    start = time.time()

    file_path = 'MovieLens/'+str(group_len) + '_group.data'
    groups = []
    for line in open(file_path, 'r'):
        mbs = line.split('\t')
        group = []
        for i in range(0, group_len):
            group.append(int(mbs[i]))
        groups.append(group)

    for i in range(len(train_data)):
        rating_mat[train_data[i,0],train_data[i,1]]=train_data[i,2]

    count = 0
    user_avg = zeros(u_num,dtype=int)
    for i in range(1,u_num):
        total=0
        for j in range(1,i_num):
            total+=rating_mat[i][j]
            if rating_mat[i][j]!=0:
                count+=1
        user_avg[i]=total/count

    cgsim_mat=zeros((u_num,u_num),dtype=float)
    avgsim=0
    rating_array = numpy.array(rating_mat)
    for i in range(1,u_num,1):
        for j in range(i+1,u_num,1):
            fenzi = numpy.dot(rating_array[i],rating_array[j])
            fenmu1 = numpy.sqrt(rating_array[i].dot(rating_array[i]))
            fenmu2 = numpy.sqrt(rating_array[j].dot(rating_array[j]))
            if fenmu1 * fenmu2!=0:
                cgsim_mat[i][j] = cgsim_mat[j][i] = fenzi / (fenmu1 * fenmu2)
            #avgsim+=cgsim_mat[i][j]
    #avgsim=avgsim*2/(u_num*(u_num-1))

    user_cand = []
    for i in range(u_num):
        user_cand.append([])

    for i in range(1,u_num):
        for j in range(1,i_num):
            if rating_mat[i][j]==0:
                user_cand[i].append(j)

    neighbor_num=int(neighbor_num)
    predict_mat=zeros((u_num,i_num),dtype=float)
    for i in range(1,u_num):
        fenzi=0
        fenmu=0
        neighbor_list=getNearestNeighbor(i,cgsim_mat)[:neighbor_num]
        for k in user_cand[i]:
            for j in neighbor_list:
                fenzi+=cgsim_mat[i][j[1]]*(rating_mat[j[1]][k]-user_avg[j[1]])
                fenmu+=abs(cgsim_mat[i][j[1]])
            predict_mat[i][k]=user_avg[i] + fenzi/fenmu

    user_training = []
    for i in range(u_num):
        user_training.append([])

    for ratingtuple in train_data:
        (i, j, rat) = ratingtuple
        user_training[i].append(j)

    group_all = numpy.zeros((len(groups), i_num), dtype=int)

    for i in range(len(groups)):
        for j in range(len(groups[i])):
            for k in range(len(user_training[groups[i][j]])):
                group_all[i][user_training[groups[i][j]][k]] = 1

    group_cand = []
    for i in range(len(groups)):
        group_cand.append([])

    for j in range(len(groups)):
        for i in range(1, i_num, 1):
            if group_all[j][i] == 0:
                group_cand[j].append(i)

    cand_ratings = numpy.zeros((len(groups), i_num), dtype=float)
    integrate_strategy()

    if (modified == 1):
        for i in range(len(groups)):
            for j in range(1, i_num):
                if len(participant_list[j]) > 300:
                    predict_mat[i][j] = predict_mat[i][j] / math.log(len(participant_list[j]), 350)
    top_n = int(top_n)
    recommend_list = []
    for i in range(len(groups)):
        tmp = getTopN(i,cand_ratings)[:top_n]
        sub_list = []
        for j in tmp:
            sub_list.append(j[1])
        recommend_list.append(sub_list)

    end = time.time()
    runtime = end - start
    test_mat = numpy.zeros((u_num, i_num), dtype=int)
    for i in range(len(test_data)):
        test_mat[test_data[i][0]][test_data[i][1]] = 1

    user_testset = []
    for i in range(u_num):
        test_set = []
        for j in range(1, i_num, 1):
            if test_mat[i][j] == 1:
                test_set.append(j)
        user_testset.append(test_set)

    group_testset = []
    for i in range(len(groups)):
        tmp_intersaction = user_testset[groups[i][0]]
        for j in range(len(groups[i]) - 1):
            tmp_intersaction = [k for k in tmp_intersaction if k in user_testset[groups[i][j + 1]]]
        group_testset.append(tmp_intersaction)

    Precision = 0
    Recall = 0
    Novelty = 0
    hits = 0
    for i in range(len(groups)):
        for j in range(len(recommend_list[i])):
            if recommend_list[i][j] in group_testset[i]:
                hits += 1
            Novelty += len(participant_list[recommend_list[i][j]])
        Precision += top_n
        if len(group_testset[i]) != 0:
            Recall += len(group_testset[i])
    Novelty = Novelty / (top_n * u_num * len(groups))
    Precision = hits/Precision
    Recall = hits/Recall

    f = open('results.txt','w')
    f.write('precisions='+str(Precision)+'\n')
    f.write('recall=' + str(Recall) + '\n')
    f.write('novelty=' + str(Novelty))
    f.close()
    # print('Precision=', Precision)
    # print('Recall=', Recall)
    # print('Novelty=', Novelty)
    print('time =',runtime)

reason: import numpy,from numpy


text:
from pathlib import Path
import sys

def loadmat(filename):
    """
    Load toolbox mat format data file

    :param filename: relative pathname of datafile
    :type filename: str
    :raises ValueError: File does not exist
    :return: contents of mat data file
    :rtype: dict

    Reads a MATLAB format *mat* file which can contain multiple variables, in
    a binary or ASCII format.  Returns a dict where the keys are the variable
    names and the values are NumPy arrays.

    .. note::
        - Uses SciPy ``io.loadmat`` to do the work.
        - If the filename has no path component, eg. ``map1.mat`, it will be
          first be looked for in the folder ``roboticstoolbox/data``.

    :seealso: :func:`path_to_datafile`
    """
    from scipy.io import loadmat

    return loaddata(filename, loadmat, squeeze_me=True, struct_as_record=False)

def loaddata(filename, handler, **kwargs):
    """
    Load toolbox data file

    :param filename: relative pathname of datafile
    :type filename: str
    :param handler: function to read data
    :type handler: callable
    :raises ValueError: File does not exist
    :return: data object

    Resolves the relative pathname to an absolute name and then invokes the
    data reading function::

        handler(abs_file_name, **kwargs)

    .. note:: If the filename has no path component, eg. ``foo.dat``, it will
        first be looked for in the folder ``roboticstoolbox/data``.

    :seealso: :func:`path_to_datafile`
    """
    path = path_to_datafile(filename)
    return handler(path, **kwargs)

def path_to_datafile(filename):
    """
    Get absolute path to datafile

    :param filename: pathname of datafile
    :type filename: str
    :raises FileNotFoundError: File does not exist
    :return: Absolute path
    :rtype: str

    If ``filename`` contains no path specification eg. ``map1.mat`` it will
    first attempt to locate the file within the ``roboticstoolbox/data``
    folder and if found, return that absolute path.

    Otherwise, ``~`` is expanded, the path made absolute, resolving symlinks
    and the file's existence is tested.

    Example::

        loadmat('map1.mat')        # read ...roboticstoolbox/data/map1.mat
        loadmat('foo.dat')         # read ./foo.dat
        loadmat('~/data/foo.dat')  # read ~/data/foo.dat
    """

    filename = Path(filename)

    if filename.parent == Path():
        # just a filename, no path, assume it is in roboticstoolbox/data

        p = Path(__file__).resolve().parent.parent / 'data' / filename
        if p.exists():
            return str(p.resolve())

    p = filename.expanduser()
    p = p.resolve()
    if not p.exists():
        raise FileNotFoundError(f"File '{p}' does not exist")
    return str(p)

if __name__ == "__main__":

    a = loadmat("map1.mat")
    print(a)
    a = loadmat("roboticstoolbox/data/map1.mat")
    print(a)
    a = loadmat("roboticstoolbox/data/../data/map1.mat")
    print(a)
    a = loadmat("~/code/robotics-toolbox-python/roboticstoolbox/data/map1.mat")
    print(a)

max_stars_repo_name: LabRobPL/robotics-toolbox-python

max_issues_repo_name: LabRobPL/robotics-toolbox-python

max_forks_repo_name: LabRobPL/robotics-toolbox-python

reason: from scipy


text:
import numpy as np
from keras.models import Sequential, load_model
from keras.layers import Input, Dropout, Flatten, Conv2D, MaxPooling2D, Dense, Activation
from keras.optimizers import SGD
from keras.utils import np_utils
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import TensorBoard
import itertools
# In[1]:import lib

# all images will be converted to this size
ROWS = 256
COLS = 256
CHANNELS = 3
# In[1]:import lib
train_image_generator = ImageDataGenerator(horizontal_flip=True, rescale=1./255, rotation_range=45)
test_image_generator = ImageDataGenerator(horizontal_flip=False, rescale=1./255, rotation_range=0)
# In[1]:import lib
train_generator = train_image_generator.flow_from_directory('train', target_size=(ROWS, COLS), class_mode='categorical')
test_generator = test_image_generator.flow_from_directory('test', target_size=(ROWS, COLS), class_mode='categorical')
# In[1]:import lib
train_generator.reset()
test_generator.reset()
# In[1]:import lib
model = Sequential()
model.add(Conv2D(64, (3,3), input_shape=(ROWS, COLS, CHANNELS)))
model.add(Activation('relu'))
model.add(Conv2D(64, (3,3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(4,4)))
model.add(Conv2D(64, (3,3)))
model.add(Activation('relu'))
model.add(Conv2D(64, (3,3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(4,4)))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(400))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(200))
model.add(Activation('softmax'))
# In[1]:import lib
model.compile(loss='categorical_crossentropy', optimizer='adamax', metrics=['accuracy'])
# In[1]:import lib
model.summary()
# In[1]:import lib
tensorboard = TensorBoard(log_dir='./logs/custom')
# In[1]:import lib
model.fit_generator(train_generator, steps_per_epoch=512, epochs=10, callbacks=[tensorboard], verbose=2)

# In[1]:save model
model.save("birds-customCNN.model")

# In[1]:import lib
print(model.evaluate_generator(test_generator, steps=1000))


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

range = getattr(__builtins__, 'xrange', range)
# end of py2 compatability boilerplate

import os

import pytest

import numpy as np

from matrixprofile.algorithms.mpdist import mpdist
import matrixprofile

MODULE_PATH = matrixprofile.__path__[0]

def test_small_series_single_threaded():
    ts = np.array([
        1, 2, 3, 1, 2, 3, 4, 5, 6, 0, 0, 1,
        1, 2, 2, 4, 5, 1, 1, 9
    ]).astype('d')
    query = np.array([
        0.23595094, 0.9865171, 0.1934413, 0.60880883,
        0.55174926, 0.77139988, 0.33529215, 0.63215848
    ]).astype('d')
    w = 4

    desired = 0.437690617625298
    actual = mpdist(ts, query, w, n_jobs=1)

    np.testing.assert_almost_equal(actual, desired)

def test_small_series_multi_threaded():
    ts = np.array([
        1, 2, 3, 1, 2, 3, 4, 5, 6, 0, 0, 1,
        1, 2, 2, 4, 5, 1, 1, 9
    ]).astype('d')
    query = np.array([
        0.23595094, 0.9865171, 0.1934413, 0.60880883,
        0.55174926, 0.77139988, 0.33529215, 0.63215848
    ]).astype('d')
    w = 4

    desired = 0.437690617625298
    actual = mpdist(ts, query, w, n_jobs=-1)

    np.testing.assert_almost_equal(actual, desired)

def test_against_matlab():
    ts = np.loadtxt(os.path.join(MODULE_PATH, '..', 'tests', 'sampledata.txt'))
    tsb = ts[199:300]
    w = 32

    desired = 1.460009659995543e-07
    actual = mpdist(ts, tsb, w, n_jobs=1)

    np.testing.assert_almost_equal(actual, desired)

def test_against_matlab_parallel():
    ts = np.loadtxt(os.path.join(MODULE_PATH, '..', 'tests', 'sampledata.txt'))
    tsb = ts[199:300]
    w = 32

    desired = 1.460009659995543e-07
    actual = mpdist(ts, tsb, w, n_jobs=-1)

    np.testing.assert_almost_equal(actual, desired)

max_stars_repo_name: StateFarmIns/matrixprofile

max_issues_repo_name: StateFarmIns/matrixprofile

max_forks_repo_name: StateFarmIns/matrixprofile


text:
"""
Tests for markov/core.py

Functions
---------
    mc_compute_stationary   [Status: Tested in test_markovchain_pmatrices]
    mc_sample_path          [Status: Tested in test_mc_sample_path]

"""
import numpy as np
from scipy import sparse
import itertools
from numpy.testing import assert_allclose, assert_array_equal, assert_raises
from nose.tools import eq_, ok_, raises

from quantecon.markov import (
    MarkovChain, mc_compute_stationary, mc_sample_path
)

def list_of_array_equal(s, t):
    """
    Compare two lists of ndarrays

    s, t: lists of numpy.ndarrays

    """
    eq_(len(s), len(t))
    all(assert_array_equal(x, y) for x, y in zip(s, t))

# KMR Function
# Useful because it seems to have 1 unit eigvalue, but a second one that
# approaches unity.  Good test of accuracy.
def KMR_Markov_matrix_sequential(N, p, epsilon):
    """
    Generate the Markov matrix for the KMR model with *sequential* move

    N: number of players
    p: level of p-dominance for action 1
       = the value of p such that action 1 is the BR for (1-q, q) for any q > p,
         where q (1-q, resp.) is the prob that the opponent plays action 1 (0, resp.)
    epsilon: mutation probability

    References:
        KMRMarkovMatrixSequential is contributed from https://github.com/oyamad
    """
    P = np.zeros((N+1, N+1), dtype=float)
    P[0, 0], P[0, 1] = 1 - epsilon * (1/2), epsilon * (1/2)
    for n in range(1, N):
        P[n, n-1] = \
            (n/N) * (epsilon * (1/2) +
                     (1 - epsilon) * (((n-1)/(N-1) < p) + ((n-1)/(N-1) == p) * (1/2))
                     )
        P[n, n+1] = \
            ((N-n)/N) * (epsilon * (1/2) +
                         (1 - epsilon) * ((n/(N-1) > p) + (n/(N-1) == p) * (1/2))
                         )
        P[n, n] = 1 - P[n, n-1] - P[n, n+1]
    P[N, N-1], P[N, N] = epsilon * (1/2), 1 - epsilon * (1/2)
    return P

# Tests: methods of MarkovChain, mc_compute_stationary #

def test_markovchain_pmatrices():
    """
    Test the methods of MarkovChain, as well as mc_compute_stationary,
    with P matrix and known solutions
    """
    # Matrix with two recurrent classes [0, 1] and [3, 4, 5],
    # which have periods 2 and 3, respectively
    Q = np.zeros((6, 6))
    Q[0, 1], Q[1, 0] = 1, 1
    Q[2, [0, 3]] = 1/2
    Q[3, 4], Q[4, 5], Q[5, 3] = 1, 1, 1
    Q_stationary_dists = \
        np.array([[1/2, 1/2, 0, 0, 0, 0], [0, 0, 0, 1/3, 1/3, 1/3]])

    testset = [
        {'P': np.array([[0.4, 0.6], [0.2, 0.8]]),  # P matrix
         'stationary_dists': np.array([[0.25, 0.75]]),  # Known solution
         'comm_classes': [np.arange(2)],
         'rec_classes': [np.arange(2)],
         'is_irreducible': True,
         'period': 1,
         'is_aperiodic': True,
         'cyclic_classes': [np.arange(2)],
         },
        {'P': sparse.csr_matrix([[0.4, 0.6], [0.2, 0.8]]),
         'stationary_dists': np.array([[0.25, 0.75]]),
         'comm_classes': [np.arange(2)],
         'rec_classes': [np.arange(2)],
         'is_irreducible': True,
         'period': 1,
         'is_aperiodic': True,
         'cyclic_classes': [np.arange(2)],
         },
        {'P': np.array([[0, 1], [1, 0]]),
         'stationary_dists': np.array([[0.5, 0.5]]),
         'comm_classes': [np.arange(2)],
         'rec_classes': [np.arange(2)],
         'is_irreducible': True,
         'period': 2,
         'is_aperiodic': False,
         'cyclic_classes': [np.array([0]), np.array([1])],
         },
        {'P': np.eye(2),
         'stationary_dists': np.array([[1, 0], [0, 1]]),
         'comm_classes': [np.array([0]), np.array([1])],
         'rec_classes': [np.array([0]), np.array([1])],
         'is_irreducible': False,
         'period': 1,
         'is_aperiodic': True,
         },
        # Reducible mc with a unique recurrent class,
        # where n-1 is a transient state
        {'P': np.array([[1, 0], [1, 0]]),
         'stationary_dists': np.array([[1, 0]]),
         'comm_classes': [np.array([0]), np.array([1])],
         'rec_classes': [np.array([0])],
         'is_irreducible': False,
         'period': 1,
         'is_aperiodic': True,
         },
        {'P': Q,
         'stationary_dists': Q_stationary_dists,
         'comm_classes': [np.array([0, 1]), np.array([2]), np.array([3, 4, 5])],
         'rec_classes': [np.array([0, 1]), np.array([3, 4, 5])],
         'is_irreducible': False,
         'period': 6,
         'is_aperiodic': False,
         },
        {'P': sparse.csr_matrix(Q),
         'stationary_dists': Q_stationary_dists,
         'comm_classes': [np.array([0, 1]), np.array([2]), np.array([3, 4, 5])],
         'rec_classes': [np.array([0, 1]), np.array([3, 4, 5])],
         'is_irreducible': False,
         'period': 6,
         'is_aperiodic': False,
         }
    ]

    # Loop Through TestSet #
    for test_dict in testset:
        mc = MarkovChain(test_dict['P'])
        computed = mc.stationary_distributions
        assert_allclose(computed, test_dict['stationary_dists'])

        assert(mc.num_communication_classes == len(test_dict['comm_classes']))
        assert(mc.is_irreducible == test_dict['is_irreducible'])
        assert(mc.num_recurrent_classes == len(test_dict['rec_classes']))
        list_of_array_equal(
            sorted(mc.communication_classes, key=lambda x: x[0]),
            sorted(test_dict['comm_classes'], key=lambda x: x[0])
        )
        list_of_array_equal(
            sorted(mc.recurrent_classes, key=lambda x: x[0]),
            sorted(test_dict['rec_classes'], key=lambda x: x[0])
        )
        assert(mc.period == test_dict['period'])
        assert(mc.is_aperiodic == test_dict['is_aperiodic'])
        try:
            list_of_array_equal(
                sorted(mc.cyclic_classes, key=lambda x: x[0]),
                sorted(test_dict['cyclic_classes'], key=lambda x: x[0])
            )
        except NotImplementedError:
            assert(mc.is_irreducible is False)

        # Test of mc_compute_stationary
        computed = mc_compute_stationary(test_dict['P'])
        assert_allclose(computed, test_dict['stationary_dists'])

# Basic Class Structure with Setup #
####################################

class Test_markovchain_stationary_distributions_KMRMarkovMatrix2():
    """
    Test Suite for MarkovChain.stationary_distributions using KMR Markov
    Matrix [suitable for nose]
    """

    # Starting Values #

    N = 27
    epsilon = 1e-2
    p = 1/3
    TOL = 1e-2

    def setUp(self):
        """ Setup a KMRMarkovMatrix and Compute Stationary Values """
        self.P = KMR_Markov_matrix_sequential(self.N, self.p, self.epsilon)
        self.mc = MarkovChain(self.P)
        self.stationary = self.mc.stationary_distributions
        stat_shape = self.stationary.shape

        if len(stat_shape) == 1:
            self.n_stat_dists = 1
        else:
            self.n_stat_dists = stat_shape[0]

    def test_markov_matrix(self):
        "Check that each row of matrix sums to 1"
        mc = self.mc
        assert_allclose(np.sum(mc.P, axis=1), np.ones(mc.n))

    def test_sum_one(self):
        "Check that each stationary distribution sums to 1"
        stationary_distributions = self.stationary
        assert_allclose(np.sum(stationary_distributions, axis=1),
                        np.ones(self.n_stat_dists))

    def test_nonnegative(self):
        "Check that the stationary distributions are non-negative"
        stationary_distributions = self.stationary
        assert(np.all(stationary_distributions > -1e-16))

    def test_left_eigen_vec(self):
        "Check that vP = v for all stationary distributions"
        mc = self.mc
        stationary_distributions = self.stationary

        if self.n_stat_dists == 1:
            assert_allclose(np.dot(stationary_distributions, mc.P),
                            stationary_distributions, atol=self.TOL)
        else:
            for i in range(self.n_stat_dists):
                curr_v = stationary_distributions[i, :]
                assert_allclose(np.dot(curr_v, mc.P), curr_v, atol=self.TOL)

def test_simulate_shape():
    P = [[0.4, 0.6], [0.2, 0.8]]
    mcs = [MarkovChain(P), MarkovChain(sparse.csr_matrix(P))]

    for mc in mcs:
        (ts_length, init, num_reps) = (10, None, None)
        assert_array_equal(mc.simulate(ts_length, init, num_reps).shape,
                           (ts_length,))

        (ts_length, init, num_reps) = (10, [0, 1], None)
        assert_array_equal(mc.simulate(ts_length, init, num_reps).shape,
                           (len(init), ts_length))

        (ts_length, init, num_reps) = (10, [0, 1], 3)
        assert_array_equal(mc.simulate(ts_length, init, num_reps).shape,
                           (len(init)*num_reps, ts_length))

        for (ts_length, init, num_reps) in [(10, None, 3), (10, None, 1)]:
            assert_array_equal(mc.simulate(ts_length, init, num_reps).shape,
                               (num_reps, ts_length))

def test_simulate_init_array_num_reps():
    P = [[0.4, 0.6], [0.2, 0.8]]
    mc = MarkovChain(P)

    ts_length = 10
    init = [0, 1]
    num_reps = 3

    X = mc.simulate(ts_length, init, num_reps)
    assert_array_equal(X[:, 0], init*num_reps)

def test_simulate_init_type():
    P = [[0.4, 0.6], [0.2, 0.8]]
    mc = MarkovChain(P)

    seed = 0
    ts_length = 3
    init = 0  # int
    X = mc.simulate(ts_length, init=init, random_state=seed)

    inits_np_int = [t(init) for t in [np.int32, np.int64]]
    for init in inits_np_int:
        X_np_int = mc.simulate(ts_length, init=init, random_state=seed)
        assert_array_equal(X_np_int, X)

def test_simulate_dense_vs_sparse():
    n = 5
    a = 1/3
    b = (1 - a)/2
    P = np.zeros((n, n))
    for i in range(n):
        P[i, (i-1) % n], P[i, i], P[i, (i+1) % n] = b, a, b
    mcs = [MarkovChain(P), MarkovChain(sparse.csr_matrix(P))]

    ts_length = 10
    inits = (None, 0, [0, 1])
    num_repss = (None, 2)

    random_state = 0
    for init, num_reps in itertools.product(inits, num_repss):
        assert_array_equal(*(mc.simulate(ts_length, init, num_reps,
                                         random_state=random_state)
                             for mc in mcs))

def test_simulate_ergodicity():
    P = [[0.4, 0.6], [0.2, 0.8]]
    stationary_dist = [0.25, 0.75]
    init = 0
    mc = MarkovChain(P)

    seed = 4433
    ts_length = 100
    num_reps = 300
    tol = 0.1

    x = mc.simulate(ts_length, init=init, num_reps=num_reps, random_state=seed)
    frequency_1 = x[:, -1].mean()
    ok_(np.abs(frequency_1 - stationary_dist[1]) < tol)

def test_simulate_for_matrices_with_C_F_orders():
    """
    Test MarkovChasin.simulate for matrices with C- and F-orders
    See the issue and fix on Numba:
    github.com/numba/numba/issues/1103
    github.com/numba/numba/issues/1104
    """
    P_C = np.array([[0.5, 0.5], [0, 1]], order='C')
    P_F = np.array([[0.5, 0.5], [0, 1]], order='F')
    init = 1
    ts_length = 10
    sample_path = np.ones(ts_length, dtype=int)

    computed_C_and_F = \
        MarkovChain(np.array([[1.]])).simulate(ts_length, init=0)
    assert_array_equal(computed_C_and_F, np.zeros(ts_length, dtype=int))

    computed_C = MarkovChain(P_C).simulate(ts_length, init)
    computed_F = MarkovChain(P_F).simulate(ts_length, init=init)
    assert_array_equal(computed_C, sample_path)
    assert_array_equal(computed_F, sample_path)

def test_mc_sample_path():
    P = [[0.4, 0.6], [0.2, 0.8]]
    Ps = [P, sparse.csr_matrix(P)]

    init = 0
    sample_size = 10

    seed = 42

    for P in Ps:
        # init: integer
        expected = [0, 0, 1, 1, 1, 0, 0, 0, 1, 1]
        computed = mc_sample_path(
            P, init=init, sample_size=sample_size, random_state=seed
        )
        assert_array_equal(computed, expected)

        # init: distribution
        distribution = (0.5, 0.5)
        expected = [0, 1, 1, 1, 0, 0, 0, 1, 1, 1]
        computed = mc_sample_path(
            P, init=distribution, sample_size=sample_size, random_state=seed
        )
        assert_array_equal(computed, expected)

def test_mc_sample_path_lln():
    P = [[0.4, 0.6], [0.2, 0.8]]
    stationary_dist = [0.25, 0.75]
    init = 0

    seed = 4433
    sample_size = 10**4
    tol = 0.02

    frequency_1 = mc_sample_path(P, init=init, sample_size=sample_size,
                                 random_state=seed).mean()
    ok_(np.abs(frequency_1 - stationary_dist[1]) < tol)

class TestMCStateValues:
    def setUp(self):
        state_values = [[0, 1], [2, 3], [4, 5]]  # Pass python list
        self.state_values = np.array(state_values)

        self.mc_reducible_dict = {
            'mc': MarkovChain([[1, 0, 0], [1, 0, 0], [0, 0, 1]],
                              state_values=state_values),
            'coms': [[0], [1], [2]],
            'recs': [[0], [2]]
        }

        self.mc_periodic_dict = {
            'mc': MarkovChain([[0, 1, 0], [0, 0, 1], [1, 0, 0]],
                              state_values=state_values),
            'coms': [[0, 1, 2]],
            'recs': [[0, 1, 2]],
            'cycs': [[0], [1], [2]]
        }

    def test_com_rec_classes(self):
        for mc_dict in [self.mc_reducible_dict, self.mc_periodic_dict]:
            mc = mc_dict['mc']
            coms = mc_dict['coms']
            recs = mc_dict['recs']
            properties = ['communication_classes',
                          'recurrent_classes']
            suffix = '_indices'
            for prop0, classes_ind in zip(properties, [coms, recs]):
                for return_indices in [True, False]:
                    if return_indices:
                        classes = classes_ind
                        prop = prop0 + suffix
                        key = lambda x: x[0]
                    else:
                        classes = [self.state_values[i] for i in classes_ind]
                        prop = prop0
                        key = lambda x: x[0, 0]
                    list_of_array_equal(
                        sorted(getattr(mc, prop), key=key),
                        sorted(classes, key=key)
                    )

    def test_cyc_classes(self):
        mc = self.mc_periodic_dict['mc']
        cycs = self.mc_periodic_dict['cycs']
        properties = ['cyclic_classes']
        suffix = '_indices'
        for prop0, classes_ind in zip(properties, [cycs]):
            for return_indices in [True, False]:
                if return_indices:
                    classes = classes_ind
                    prop = prop0 + suffix
                    key = lambda x: x[0]
                else:
                    classes = [self.state_values[i] for i in classes_ind]
                    prop = prop0
                    key = lambda x: x[0, 0]
                list_of_array_equal(
                    sorted(getattr(mc, prop), key=key),
                    sorted(classes, key=key)
                )

    def test_simulate(self):
        # Deterministic mc
        mc = self.mc_periodic_dict['mc']
        ts_length = 6

        methods = ['simulate_indices', 'simulate']

        init_idx = 0
        inits = [init_idx, self.state_values[init_idx]]
        path = np.arange(init_idx, init_idx+ts_length)%mc.n
        paths = [path, self.state_values[path]]
        for method, init, X_expected in zip(methods, inits, paths):
            X = getattr(mc, method)(ts_length, init)
            assert_array_equal(X, X_expected)

        init_idx = [1, 2]
        inits = [init_idx, self.state_values[init_idx]]
        path = np.array(
            [np.arange(i, i+ts_length)%mc.n for i in init_idx]
        )
        paths = [path, self.state_values[path]]
        for method, init, X_expected in zip(methods, inits, paths):
            X = getattr(mc, method)(ts_length, init)
            assert_array_equal(X, X_expected)

        inits = [None, None]
        seed = 1234  # init will be 2
        init_idx = 2
        path = np.arange(init_idx, init_idx+ts_length)%mc.n
        paths = [path, self.state_values[path]]
        for method, init, X_expected in zip(methods, inits, paths):
            X = getattr(mc, method)(ts_length, init, random_state=seed)
            assert_array_equal(X, X_expected)

def test_mc_stationary_distributions_state_values():
    P = [[0.4, 0.6, 0], [0.2, 0.8, 0], [0, 0, 1]]
    state_values = ['a', 'b', 'c']
    mc = MarkovChain(P, state_values=state_values)
    stationary_dists_expected = [[0.25, 0.75, 0], [0, 0, 1]]
    stationary_dists = mc.stationary_distributions
    assert_allclose(stationary_dists, stationary_dists_expected)

def test_get_index():
    P = [[0.4, 0.6], [0.2, 0.8]]
    mc = MarkovChain(P)

    eq_(mc.get_index(0), 0)
    eq_(mc.get_index(1), 1)
    assert_raises(ValueError, mc.get_index, 2)
    assert_array_equal(mc.get_index([1, 0]), [1, 0])
    assert_raises(ValueError, mc.get_index, [[1]])

    mc.state_values = [1, 2]
    eq_(mc.get_index(1), 0)
    eq_(mc.get_index(2), 1)
    assert_raises(ValueError, mc.get_index, 0)
    assert_array_equal(mc.get_index([2, 1]), [1, 0])
    assert_raises(ValueError, mc.get_index, [[1]])

    mc.state_values = [[1, 2], [3, 4]]
    eq_(mc.get_index([1, 2]), 0)
    assert_raises(ValueError, mc.get_index, 1)
    assert_array_equal(mc.get_index([[3, 4], [1, 2]]), [1, 0])

@raises(ValueError)
def test_raises_value_error_non_2dim():
    """Test with non 2dim input"""
    MarkovChain(np.array([0.4, 0.6]))

def test_raises_value_error_non_sym():
    """Test with non symmetric input"""
    P = np.array([[0.4, 0.6]])
    assert_raises(ValueError, MarkovChain, P)
    assert_raises(ValueError, MarkovChain, sparse.csr_matrix(P))

def test_raises_value_error_non_nonnegative():
    """Test with non nonnegative input"""
    P = np.array([[0.4, 0.6], [-0.2, 1.2]])
    assert_raises(ValueError, MarkovChain, P)
    assert_raises(ValueError, MarkovChain, sparse.csr_matrix(P))

def test_raises_value_error_non_sum_one():
    """Test with input such that some of the rows does not sum to one"""
    P = np.array([[0.4, 0.6], [0.2, 0.9]])
    assert_raises(ValueError, MarkovChain, P)
    assert_raises(ValueError, MarkovChain, sparse.csr_matrix(P))

def test_raises_value_error_simulate_init_out_of_range():
    P = [[0.4, 0.6], [0.2, 0.8]]
    mc = MarkovChain(P)

    n = mc.n
    ts_length = 3
    assert_raises(ValueError, mc.simulate, ts_length, init=n)
    assert_raises(ValueError, mc.simulate, ts_length, init=-(n+1))
    assert_raises(ValueError, mc.simulate, ts_length, init=[0, n])
    assert_raises(ValueError, mc.simulate, ts_length, init=[0, -(n+1)])

def test_raises_non_homogeneous_state_values():
    P = [[0.4, 0.6], [0.2, 0.8]]
    state_values = [(0, 1), 2]
    assert_raises(ValueError, MarkovChain, P, state_values=state_values)

if __name__ == '__main__':
    import sys
    import nose

    argv = sys.argv[:]
    argv.append('--verbose')
    argv.append('--nocapture')
    nose.main(argv=argv, defaultTest=__file__)

reason: import numpy,from numpy,from scipy


text:
import numpy as np

from spikeinterface import NumpyRecording, NumpySorting

from probeinterface import generate_linear_probe

def generate_recording(
        num_channels=2,
        sampling_frequency=30000.,  # in Hz
        durations=[10.325, 3.5],  #  in s for 2 segments
        set_probe=True
):
    num_segments = len(durations)
    num_timepoints = [int(sampling_frequency * d) for d in durations]

    traces_list = []
    for i in range(num_segments):
        traces = np.random.randn(num_timepoints[i], num_channels).astype('float32')
        times = np.arange(num_timepoints[i]) / sampling_frequency
        traces += np.sin(2 * np.pi * 50 * times)[:, None]
        traces_list.append(traces)
    recording = NumpyRecording(traces_list, sampling_frequency)

    if set_probe:
        probe = generate_linear_probe(num_elec=num_channels)
        probe.set_device_channel_indices(np.arange(num_channels))
        recording.set_probe(probe, in_place=True)

    return recording

def generate_sorting(
        num_units=5,
        sampling_frequency=30000.,  # in Hz
        durations=[10.325, 3.5],  #  in s for 2 segments
):
    num_segments = len(durations)
    num_timepoints = [int(sampling_frequency * d) for d in durations]

    unit_ids = np.arange(num_units)

    units_dict_list = []
    for seg_index in range(num_segments):
        units_dict = {}
        for unit_id in unit_ids:
            #  15 Hz for all units
            n_spike = int(15. * durations[seg_index])
            spike_times = np.sort(np.unique(np.random.randint(0, num_timepoints[seg_index], n_spike)))
            units_dict[unit_id] = spike_times
        units_dict_list.append(units_dict)
    sorting = NumpySorting.from_dict(units_dict_list, sampling_frequency)

    return sorting

def create_sorting_npz(num_seg, file_path):
    # create a NPZ sorting file
    d = {}
    d['unit_ids'] = np.array([0, 1, 2], dtype='int64')
    d['num_segment'] = np.array([2], dtype='int64')
    d['sampling_frequency'] = np.array([30000.], dtype='float64')
    for seg_index in range(num_seg):
        spike_indexes = np.arange(0, 1000, 10)
        spike_labels = np.zeros(spike_indexes.size, dtype='int64')
        spike_labels[0::3] = 0
        spike_labels[1::3] = 1
        spike_labels[2::3] = 2
        d[f'spike_indexes_seg{seg_index}'] = spike_indexes
        d[f'spike_labels_seg{seg_index}'] = spike_labels
    np.savez(file_path, **d)

if __name__ == '__main__':
    print(generate_recording())
    print(generate_sorting())


text:
# -*- coding:utf-8 -*-
'''
@Author: LiamTTT
@Project: TCTPreprocess
@File: cvt2voc_PMScope.py
@Date: 2021/9/15
@Time: 上午10:16
@Desc: convert PMScope csv annotation file to voc format with specified hyper params.
'''
import os
import json
import argparse
import random
from time import time

import numpy as np
from tqdm import tqdm
from loguru import logger

from core.common import check_dir, save_xml_str
from core.annotations import (
    find_neighbors_for_given_center, get_bnd_in_tile,
    create_voc_annotation, create_voc_object,
    BndBox
)

random.seed(42)

def process_csv_file(csv_file):
    annotations = {}
    with open(csv_file, 'r') as f:
        lines = f.readlines()[1:]
        logger.info(f'Num of annotations: {len(lines)}')
        nb_success = 0
        for line in lines:
            bat, img_roi, cx, cy, w, h, anno_n, sld_n, scx, scy, roi_p = line.strip().split(',')
            # rename batch name
            try:
                if bat.upper() not in BMP_INFO_LUT[img_roi]['batch']:
                    logger.error(
                        'slide batch({}) does not match to json file({}).'.format(bat, BMP_INFO_LUT[img_roi]['batch']))
                    continue
                bat = BMP_INFO_LUT[img_roi]['batch']
            except KeyError:
                logger.exception(f'No {roi_p} info in BMP_INFO_LUT')
                continue
            # init data struct, if necessary
            if bat not in annotations.keys():
                annotations[bat] = {}
            if sld_n not in annotations[bat].keys():
                annotations[bat][sld_n] = {}
            if img_roi not in annotations[bat][sld_n].keys():
                annotations[bat][sld_n][img_roi] = {
                    'ROI_path': roi_p,
                    'objects': []
                }
            # record
            obj = {
                'name': anno_n,
                'center': [int(float(cx)), int(float(cy))],
                'bnd_size': [int(float(w)), int(float(h))],
                'wsi_center': [int(float(scx)), int(float(scy))]
            }
            annotations[bat][sld_n][img_roi]['objects'].append(obj)
            nb_success += 1
    logger.info(f'{nb_success} success')
    return annotations

def get_obj_in_tile(name, center_delta, bnd_size, tile_size):
    (xmin, ymin, xmax, ymax), seg = get_bnd_in_tile(center_delta, bnd_size, tile_size)
    if xmin == xmax or ymin == ymax:
        # FIXME 211217 siboliu: The main reason for zero obj is not clear, this condition is temporal solution.
        return None, None
    return create_voc_object(name, xmin, ymin, xmax, ymax), seg

def create_annotation_of_roi(image_roi, center_anno=True, nb_sample=1):
    annotations = []
    # get roi meta info
    roi_info = BMP_INFO_LUT[image_roi]
    # source info
    image_device = 'PortableMicroScope'
    wsi_batch = roi_info['batch']
    wsi_name = roi_info['slide']  # e.g. sfy1104135
    source_kwarg = {'source': {'image_roi': image_roi}}
    folder = wsi_batch
    # filename

    # process object
    annotation_objects = ANNOTATIONS[wsi_batch][wsi_name][image_roi]['objects']
    tile_size = np.array(TILE_SIZE)
    # wsi_bnds_center = np.array([obj['wsi_center'] for obj in annotation_objects])  # bnd center on wsi where the roi comes from
    bnds_center = np.array([obj['center'] for obj in annotation_objects])
    bnds_wh = np.array([obj['bnd_size'] for obj in annotation_objects])
    bnds_cls = [obj['name'] for obj in annotation_objects]
    # create voc annotations
    for idx in range(len(annotation_objects)):
        # get main object
        main_shifts = []
        for shift_id in range(nb_sample):
            shift_x = (random.random() - 0.5) * 0.90 * tile_size[0]
            shift_y = (random.random() - 0.5) * 0.90 * tile_size[1]
            main_shifts.append(np.array([shift_x, shift_y], dtype=int))
        if center_anno:
            main_shifts[0] = np.zeros((2,))
        for shift_id, main_shift in enumerate(main_shifts):
            objects = []
            obj_in_tile, segmented = get_obj_in_tile(bnds_cls[idx], main_shift, bnds_wh[idx], tile_size)
            if obj_in_tile is None:
                logger.warning(f"annotations is invalid: {image_roi}, {idx} annotation, shift id: {shift_id}, bnd: {bnds_wh[idx]}")
                break
            objects.append(obj_in_tile)
            neighbor_idx, neighbor_delta = find_neighbors_for_given_center(idx, main_shift, tile_size, bnds_center)
            for ni, nd in zip(neighbor_idx, neighbor_delta):
                obj_in_tile, seg = get_obj_in_tile(bnds_cls[ni], main_shift + nd, bnds_wh[ni], tile_size)
                if obj_in_tile is None:
                    continue
                segmented |= seg
                objects.append(obj_in_tile)
            # define other attributes in annotation file.
            filename = '{:s}_{:s}_{:0>3d}_{:d}_{:d}_{:d}_{:s}.jpg'.format(wsi_name, image_roi.split('.bmp')[0], idx, shift_id, *bnds_center[idx], bnds_cls[idx])
            padding = [0, 0]  # pad to right and bottom half of tile
            if tile_size[0] % 2:
                padding[0] += 1
            if tile_size[1] % 2:
                padding[1] += 1
            wsi_bndbox = BndBox(
                *((bnds_center[idx] - main_shift - tile_size // 2).tolist() + (bnds_center[idx] - main_shift + (tile_size // 2) + padding).tolist()))
            voc_anno = create_voc_annotation(
                folder, filename,
                image_device, wsi_batch, wsi_name, wsi_bndbox,
                *tile_size,
                objects,
                segmented,
                **source_kwarg
            )
            annotations.append(voc_anno)
    return annotations

def create_annotation_of_batch(batch_name, center_anno=True, nb_sample=1):
    wsis_anno = ANNOTATIONS[batch_name]
    logger.info(f'Process {batch_name}')
    logger.info(f'WSI nums: {len(wsis_anno)}')
    logger.info(f'ROIS nums: {sum([len(r) for _, r in wsis_anno.items()])}')
    nb_success = 0
    batch_annotations = []
    since = time()
    for sld_n, rois_anno in tqdm(wsis_anno.items(), desc=f'Process {batch_name}'):
        for roi_n in rois_anno.keys():
            try:
                roi_annotations = create_annotation_of_roi(roi_n, center_anno, nb_sample)
            except:
                logger.exception('process {}-{} failed! ROI path: {}'.format(sld_n, roi_n, BMP_INFO_LUT[roi_n]['path']))
                continue
            batch_annotations += roi_annotations
            nb_success += 1
    logger.info(f'{nb_success} ROIs success!')
    logger.info(f'Total annotations: {len(batch_annotations)}')
    total_time = time() - since
    logger.info('Time: {:.2f} s, {:.3f} s/ROI'.format(total_time, (total_time / nb_success) if nb_success else float('inf')))
    return batch_annotations

def create_dataset_for_batch(dataset_dir, dataset_name, center_anno=True, nb_sample=1):
    logger.info(f'Create dataset Annotations file: {dataset_dir}')
    save_annos_dir = os.path.join(dataset_dir, 'Annotations')
    check_dir(save_annos_dir, True, logger)
    batch_annotations = create_annotation_of_batch(dataset_name, center_anno, nb_sample)
    nb_success = 0
    since = time()
    for anno in tqdm(batch_annotations, desc=f'Save annotations in {save_annos_dir}'):
        save_path = os.path.join(save_annos_dir, anno.filename.replace('jpg', 'xml'))
        if save_xml_str(anno.to_xml(), save_path, 'w+', logger):
            nb_success += 1
    total_time = time() - since
    logger.info(f'create {nb_success} annotations file in {save_annos_dir}')
    logger.info('Time: {:.2f} s {:.3f} s/file'.format(total_time, (total_time / nb_success) if nb_success else float('inf')))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Convert FCDD xml to VOC annotation files.')
    parser.add_argument('--csv_anno_file', help='path to merged PMScope annotations csv file', type=str)
    parser.add_argument('--json_lut_file', help='path to PMScope ROI info LUT file', type=str)
    parser.add_argument('--dataset_root', help='dir to all PMScope dataset root', type=str)
    parser.add_argument('--dataset_list', help='dataset name list, also the folder name. Empty for all',
                        type=str, nargs='*', default=[])
    parser.add_argument('--tile_size', help='tile size in 0.67um/pixel',
                        type=int, nargs='+', default=[800])
    parser.add_argument('--center_anno', default=False, action='store_true', help='always create center annotation')
    parser.add_argument('--nb_sample', default=1, type=int, help='number of sample to create.')
    args = parser.parse_args()

    # Parse args
    dataset_root = args.dataset_root
    # process global var
    lut_file = args.json_lut_file
    global BMP_INFO_LUT
    BMP_INFO_LUT = json.load(open(lut_file, 'r'))
    # get formatted annotations dict
    csv_file = args.csv_anno_file
    global ANNOTATIONS
    ANNOTATIONS = process_csv_file(csv_file)
    # make tile size global
    global TILE_SIZE
    tile_size = args.tile_size
    if len(tile_size) == 1:
        tile_size = tile_size * 2
    elif len(tile_size) == 2:
        pass
    else:
        raise ValueError(f'Tile size {tile_size} should be [width, height] or [length] for square')
    TILE_SIZE = tile_size
    center_anno = args.center_anno
    nb_sample = args.nb_sample
    if nb_sample < 1 or nb_sample > 9:
        nb_sample = np.clip(nb_sample, 1, 9)
        logger.warning(f"clip nb_sample to {nb_sample}")
    logger.info(f'create {nb_sample} samples for each annotation. make annotaion at center: {center_anno}')
    # get dataset list
    dataset_list = args.dataset_list
    if len(dataset_list) == 0:
        dataset_list = list(ANNOTATIONS.keys())
        logger.info(f'Create all dataset in {csv_file}')
    logger.info(f'Datasets: {dataset_list}')
    logger.info(f'Tile Size: {tile_size}')
    # end parse args

    # process
    for dataset_name in dataset_list:
        dataset_dir = os.path.join(dataset_root, dataset_name)
        check_dir(dataset_dir, True, logger)
        create_dataset_for_batch(dataset_dir, dataset_name, center_anno, nb_sample)
    # end process

max_stars_repo_name: LiamTTT/TCTPreprocess

max_issues_repo_name: LiamTTT/TCTPreprocess

max_forks_repo_name: LiamTTT/TCTPreprocess


text:
import numpy as np
import gym
import pybullet_envs

from pybullet_envs.gym_locomotion_envs import WalkerBaseBulletEnv
from pybullet_envs.env_bases import MJCFBaseBulletEnv
from pybullet_envs.robot_locomotors import Ant, WalkerBase
from pybullet_envs.robot_bases import MJCFBasedRobot

class MyWalkerBase(WalkerBase):

  def __init__(self, fn, robot_name, action_dim, obs_dim, power):
    MJCFBasedRobot.__init__(self, fn, robot_name, action_dim, obs_dim)
    self.power = power
    self.camera_x = 0
    self.start_pos_x, self.start_pos_y, self.start_pos_z = 0, 0, 0
    self.walk_target_x = 20  # kilometer away
    self.walk_target_y = 20
    self.body_xyz = [0, 0, 0]

  electricity_cost = -2.0  # cost for using motors -- this parameter should be carefully tuned against reward for making progress, other values less improtant
  stall_torque_cost = -0.1  # cost for running electric current through a motor even at zero rotational speed, small
  foot_collision_cost = -1.0  # touches another leg, or other objects, that cost makes robot avoid smashing feet into itself
  foot_ground_object_names = set(["floor"])  # to distinguish ground and other objects
  joints_at_limit_cost = -0.1  # discourage stuck joints

  def step(self, a):
    if not self.scene.multiplayer:  # if multiplayer, action first applied to all robots, then global step() called, then _step() for all robots with the same actions
      self.robot.apply_action(a)
      self.scene.global_step()

    state = self.robot.calc_state()  # also calculates self.joints_at_limit

    self._alive = float(
        self.robot.alive_bonus(
            state[0] + self.robot.initial_z,
            self.robot.body_rpy[1]))  # state[0] is body height above ground, body_rpy[1] is pitch
    done = self._isDone()
    if not np.isfinite(state).all():
      print("~INF~", state)
      done = True

    potential_old = self.potential
    self.potential = self.robot.calc_potential()
    progress = float(self.potential - potential_old)

    feet_collision_cost = 0.0
    for i, f in enumerate(
        self.robot.feet
    ):  # TODO: Maybe calculating feet contacts could be done within the robot code
      contact_ids = set((x[2], x[4]) for x in f.contact_list())
      #print("CONTACT OF '%d' WITH %d" % (contact_ids, ",".join(contact_names)) )
      if (self.ground_ids & contact_ids):
        #see Issue 63: https://github.com/openai/roboschool/issues/63
        #feet_collision_cost += self.foot_collision_cost
        self.robot.feet_contact[i] = 1.0
      else:
        self.robot.feet_contact[i] = 0.0

    electricity_cost = self.electricity_cost * float(np.abs(a * self.robot.joint_speeds).mean(
    ))  # let's assume we have DC motor with controller, and reverse current braking
    electricity_cost += self.stall_torque_cost * float(np.square(a).mean())

    joints_at_limit_cost = float(self.joints_at_limit_cost * self.robot.joints_at_limit)
    debugmode = 0
    if (debugmode):
      print("alive=")
      print(self._alive)
      print("progress")
      print(progress)
      print("electricity_cost")
      print(electricity_cost)
      print("joints_at_limit_cost")
      print(joints_at_limit_cost)
      print("feet_collision_cost")
      print(feet_collision_cost)

    self.rewards = [
        self._alive, progress, electricity_cost, joints_at_limit_cost, feet_collision_cost
    ]
    if (debugmode):
      print("rewards=")
      print(self.rewards)
      print("sum rewards")
      print(sum(self.rewards))
    self.HUD(state, a, done)
    self.reward += sum(self.rewards)

    return state, sum(self.rewards), bool(done), {}

class MyAnt(MyWalkerBase):
  foot_list = ['front_left_foot', 'front_right_foot', 'left_back_foot', 'right_back_foot']

  def __init__(self):
    MyWalkerBase.__init__(self, "ant.xml", "torso", action_dim=8, obs_dim=28, power=2.5)

  def alive_bonus(self, z, pitch):
    return +1 if z > 0.26 else -1  # 0.25 is central sphere rad, die if it scrapes the ground

class MyAntBulletEnv(WalkerBaseBulletEnv):

  def __init__(self, render=False):
    self.robot = MyAnt()

    self.camera_x = 0
    self.walk_target_x = 20  # originally 1 kilometer away, now 5m
    self.walk_target_y = 20
    self.stateId = -1
    MJCFBaseBulletEnv.__init__(self, self.robot, render)


text:
#
# Copyright 2017-2018 European Centre for Medium-Range Weather Forecasts (ECMWF).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Authors:
#   Alessandro Amici - B-Open - https://bopen.eu
#

from __future__ import absolute_import, division, print_function

import numpy as np

from xarray import Variable
from xarray.core import indexing
from xarray.core.utils import Frozen, FrozenOrderedDict
from xarray.backends.common import AbstractDataStore, BackendArray
try:
    from xarray.backends.locks import ensure_lock, SerializableLock
except ImportError:
    # no locking for xarray <= 0.11
    def ensure_lock(lock):
        return lock

    class SerializableLock(object):
        def __enter__(self):
            pass

        def __exit__(self, exc_type, exc_val, exc_tb):
            pass

# FIXME: Add a dedicated lock, even if ecCodes is supposed to be thread-safe
#   in most circumstances. See:
#       https://confluence.ecmwf.int/display/ECC/Frequently+Asked+Questions
ECCODES_LOCK = SerializableLock()

class CfGribArrayWrapper(BackendArray):
    def __init__(self, datastore, array):
        self.datastore = datastore
        self.shape = array.shape
        self.dtype = array.dtype
        self.array = array

    def __getitem__(self, key):
        return indexing.explicit_indexing_adapter(
            key, self.shape, indexing.IndexingSupport.OUTER, self._getitem)

    def _getitem(self, key):
        with self.datastore.lock:
            return self.array[key]

class CfGribDataStore(AbstractDataStore):
    """
    Implements the ``xr.AbstractDataStore`` read-only API for a GRIB file.
    """
    def __init__(self, filename, lock=None, **backend_kwargs):
        import cfgrib
        if lock is None:
            lock = ECCODES_LOCK
        self.lock = ensure_lock(lock)
        self.ds = cfgrib.open_file(filename, **backend_kwargs)

    def open_store_variable(self, name, var):
        if isinstance(var.data, np.ndarray):
            data = var.data
        else:
            wrapped_array = CfGribArrayWrapper(self, var.data)
            data = indexing.LazilyOuterIndexedArray(wrapped_array)

        encoding = self.ds.encoding.copy()
        encoding['original_shape'] = var.data.shape

        return Variable(var.dimensions, data, var.attributes, encoding)

    def get_variables(self):
        return FrozenOrderedDict((k, self.open_store_variable(k, v))
                                 for k, v in self.ds.variables.items())

    def get_attrs(self):
        return Frozen(self.ds.attributes)

    def get_dimensions(self):
        return Frozen(self.ds.dimensions)

    def get_encoding(self):
        dims = self.get_dimensions()
        encoding = {
            'unlimited_dims': {k for k, v in dims.items() if v is None},
        }
        return encoding

max_stars_repo_name: shoyer/cfgrib

max_issues_repo_name: shoyer/cfgrib

max_forks_repo_name: shoyer/cfgrib