text:
import logging
import numpy as np
from typing import List, Tuple, Union

from autoarray.structures.arrays.two_d.array_2d import Array2D
from autoarray.structures.grids.two_d.grid_2d import Grid2D
from autoarray.structures.vectors.abstract import AbstractVectorYX2D

from autoarray.mask.mask_2d import Mask2D
from autoarray.structures.grids import abstract_grid
from autoarray.structures.grids.two_d import abstract_grid_2d
from autoarray.geometry import geometry_util

from autoarray import type as ty

logging.basicConfig()
logger = logging.getLogger(__name__)

class VectorYX2D(AbstractVectorYX2D):
    def __new__(
        cls,
        vectors: Union[np.ndarray, List[Tuple[float, float]]],
        grid: Union[Grid2D, List],
        mask,
    ):
        """
        A collection of (y,x) vectors which are located on a regular 2D grid of (y,x) coordinates.

        The vectors are paired to a uniform 2D mask of pixels and sub-pixels. Each vector corresponds to a value at
        the centre of a sub-pixel in an unmasked pixel.

        The `VectorYX2D` is ordered such that pixels begin from the top-row of the corresponding mask and go right
        and down. The positive y-axis is upwards and positive x-axis to the right.

        The (y,x) vectors are stored as a NumPy array which has the `slim` and `native shapes described below.
        Irrespective of this shape, the last dimension of the data structure storing the vectors is always shape 2,
        corresponding to the y and x vectors. [total_vectors, 2].

        Calculations should use the NumPy array structure wherever possible for efficient calculations.

        The vectors input to this function can have any of the following forms (they will be converted to the 1D NumPy
        array structure and can be converted back using the object's properties):

        [[vector_0_y, vector_0_x], [vector_1_y, vector_1_x]]
        [(vector_0_y, vector_0_x), (vector_1_y, vector_1_x)]

        If your vector field lies on a 2D irregular grid of data the `VectorFieldIrregular2D` data structure should be
        used.

        Case 1: [sub-size=1, slim]:
        ---------------------------

        The Vector2D is an ndarray of shape [total_unmasked_pixels, 2].

        The first element of the ndarray corresponds to the pixel index, for example:

        - vector[3, 0:2] = the 4th unmasked pixel's y and x values.
        - vector[6, 0:2] = the 7th unmasked pixel's y and x values.

        Below is a visual illustration of a vector, where a total of 10 pixels are unmasked and are included in
        the vector.

        IxIxIxIxIxIxIxIxIxIxI
        IxIxIxIxIxIxIxIxIxIxI     This is an example `Mask2D`, where:
        IxIxIxIxIxIxIxIxIxIxI
        IxIxIxIxIoIoIxIxIxIxI     x = `True` (Pixel is masked and excluded from the vector)
        IxIxIxIoIoIoIoIxIxIxI     o = `False` (Pixel is not masked and included in the vector)
        IxIxIxIoIoIoIoIxIxIxI
        IxIxIxIxIxIxIxIxIxIxI
        IxIxIxIxIxIxIxIxIxIxI
        IxIxIxIxIxIxIxIxIxIxI
        IxIxIxIxIxIxIxIxIxIxI

        The mask pixel index's will come out like this (and the direction of scaled values is highlighted
        around the mask).

        pixel_scales = 1.0"

        <--- -ve  x  +ve -->
                                                        y      x
        IxIxIxIxIxIxIxIxIxIxI  ^   vector[0, :] = 0
        IxIxIxIxIxIxIxIxIxIxI  I   vector[1, :] = 1
        IxIxIxIxIxIxIxIxIxIxI  I   vector[2, :] = 2
        IxIxIxIxI0I1IxIxIxIxI +ve  vector[3, :] = 3
        IxIxIxI2I3I4I5IxIxIxI  y   vector[4, :] = 4
        IxIxIxI6I7I8I9IxIxIxI -ve  vector[5, :] = 5
        IxIxIxIxIxIxIxIxIxIxI  I   vector[6, :] = 6
        IxIxIxIxIxIxIxIxIxIxI  I   vector[7, :] = 7
        IxIxIxIxIxIxIxIxIxIxI \/   vector[8, :] = 8
        IxIxIxIxIxIxIxIxIxIxI      vector[9, :] = 9

        Case 2: [sub-size>1, slim]:
        ------------------

        If the masks's sub size is > 1, the vector is defined as a sub-vector where each entry corresponds to the
        values at the centre of each sub-pixel of an unmasked pixel.

        The sub-vector indexes are ordered such that pixels begin from the first (top-left) sub-pixel in the first
        unmasked pixel. Indexes then go over the sub-pixels in each unmasked pixel, for every unmasked pixel.
        Therefore, the sub-vector is an ndarray of shape [total_unmasked_pixels*(sub_array_shape)**2, 2]. For example:

        - vector[9, 0:2] - using a 2x2 sub-vector, gives the 3rd unmasked pixel's 2nd sub-pixel y and x values.
        - vector[9, 0:2] - using a 3x3 sub-vector, gives the 2nd unmasked pixel's 1st sub-pixel y and x values.
        - vector[27, 0:2] - using a 3x3 sub-vector, gives the 4th unmasked pixel's 1st sub-pixel y and x values.

        Below is a visual illustration of a sub vector. Indexing of each sub-pixel goes from the top-left corner. In
        contrast to the vector above, our illustration below restricts the mask to just 2 pixels, to keep the
        illustration brief.

        IxIxIxIxIxIxIxIxIxIxI
        IxIxIxIxIxIxIxIxIxIxI     This is an example `Mask2D`, where:
        IxIxIxIxIxIxIxIxIxIxI
        IxIxIxIxIxIxIxIxIxIxI     x = `True` (Pixel is masked and excluded from lens)
        IxIxIxIxIoIoIxIxIxIxI     o = `False` (Pixel is not masked and included in lens)
        IxIxIxIxIxIxIxIxIxIxI
        IxIxIxIxIxIxIxIxIxIxI
        IxIxIxIxIxIxIxIxIxIxI
        IxIxIxIxIxIxIxIxIxIxI
        IxIxIxIxIxIxIxIxIxIxI

        Our vector with a sub-size looks like it did before:

        pixel_scales = 1.0"

        <--- -ve  x  +ve -->

        IxIxIxIxIxIxIxIxIxIxI  ^
        IxIxIxIxIxIxIxIxIxIxI  I
        IxIxIxIxIxIxIxIxIxIxI  I
        IxIxIxIxIxIxIxIxIxIxI +ve
        IxIxIxI0I1IxIxIxIxIxI  y
        IxIxIxIxIxIxIxIxIxIxI -ve
        IxIxIxIxIxIxIxIxIxIxI  I
        IxIxIxIxIxIxIxIxIxIxI  I
        IxIxIxIxIxIxIxIxIxIxI \/
        IxIxIxIxIxIxIxIxIxIxI

        However, if the sub-size is 2,each unmasked pixel has a set of sub-pixels with values. For example, for pixel 0,
        if `sub_size=2`, it has 4 values on a 2x2 sub-vector:

        Pixel 0 - (2x2):

               vector[0, 0:2] = y and x values of first sub-pixel in pixel 0.
        I0I1I  vector[1, 0:2] = y and x values of first sub-pixel in pixel 1.
        I2I3I  vector[2, 0:2] = y and x values of first sub-pixel in pixel 2.
               vector[3, 0:2] = y and x values of first sub-pixel in pixel 3.

        If we used a sub_size of 3, for the first pixel we we would create a 3x3 sub-vector:

                 vector[0] = y and x values of first sub-pixel in pixel 0.
                 vector[1] = y and x values of first sub-pixel in pixel 1.
                 vector[2] = y and x values of first sub-pixel in pixel 2.
        I0I1I2I  vector[3] = y and x values of first sub-pixel in pixel 3.
        I3I4I5I  vector[4] = y and x values of first sub-pixel in pixel 4.
        I6I7I8I  vector[5] = y and x values of first sub-pixel in pixel 5.
                 vector[6] = y and x values of first sub-pixel in pixel 6.
                 vector[7] = y and x values of first sub-pixel in pixel 7.
                 vector[8] = y and x values of first sub-pixel in pixel 8.

        Case 3: [sub_size=1, native]
        ----------------------------

        The Vector2D has the same properties as Case 1, but is stored as an an ndarray of shape
        [total_y_values, total_x_values, 2].

        All masked entries on the vector have values of 0.0.

        For the following example mask:

        IxIxIxIxIxIxIxIxIxIxI
        IxIxIxIxIxIxIxIxIxIxI     This is an example `Mask2D`, where:
        IxIxIxIxIxIxIxIxIxIxI
        IxIxIxIxIoIoIxIxIxIxI     x = `True` (Pixel is masked and excluded from the vector)
        IxIxIxIoIoIoIoIxIxIxI     o = `False` (Pixel is not masked and included in the vector)
        IxIxIxIoIoIoIoIxIxIxI
        IxIxIxIxIxIxIxIxIxIxI
        IxIxIxIxIxIxIxIxIxIxI
        IxIxIxIxIxIxIxIxIxIxI
        IxIxIxIxIxIxIxIxIxIxI

        - vector[0,0, 0:2] = [0.0, 0.0] (it is masked, thus zero)
        - vector[0,0, 0:2] = [0.0, 0.0] (it is masked, thus zero)
        - vector[3,3, 0:2] = [0.0, 0.0] (it is masked, thus zero)
        - vector[3,3, 0:2] = [0.0, 0.0] (it is masked, thus zero)
        - vector[3,4, 0:2] = [0, 0]
        - vector[3,4, 0:2] = [-1, -1]

        Case 4: [sub_size>, native]
        ---------------------------

        The properties of this vector can be derived by combining Case's 2 and 3 above, whereby the vector is stored as
        an ndarray of shape [total_y_values*sub_size, total_x_values*sub_size, 2].

        All sub-pixels in masked pixels have values 0.0.

        Parameters
        ----------
        vectors
            The 2D (y,x) vectors on a regular grid that represent the vector-field.
        grid
            The regular grid of (y,x) coordinates where each vector is located.
        mask
            The 2D mask associated with the array, defining the pixels each array value is paired with and
            originates from.
        """

        if len(vectors) == 0:
            return []

        if type(vectors) is list:
            vectors = np.asarray(vectors)

        obj = vectors.view(cls)
        obj.grid = Grid2D(grid=grid, mask=mask)
        obj.mask = mask

        return obj

    def __array_finalize__(self, obj):

        if hasattr(obj, "mask"):
            self.mask = obj.mask

        if hasattr(obj, "grid"):
            self.grid = obj.grid

    @classmethod
    def manual_slim(
        cls,
        vectors: Union[np.ndarray, List[List], List[Tuple]],
        shape_native: Tuple[int, int],
        pixel_scales: ty.PixelScales,
        sub_size: int = 1,
        origin: Tuple[float, float] = (0.0, 0.0),
    ) -> "VectorYX2D":
        """
        Create a VectorYX2D (see *VectorYX2D.__new__*) by inputting the vector in 1D, for example:

        vectors=np.array([[1.0, 1.0], [2.0, 2.0], [3.0, 3.0], [4.0, 4.0]])
        vectors=[[1.0, 1.0], [2.0, 2.0], [3.0, 3.0], [4.0, 4.0]]

        The `VectorYX2D` object assumes a uniform `Grid2D` which is computed from the input `shape_native`,
        `pixel_scales` and `origin`.

        From 1D input the method cannot determine the 2D shape of the grid and its mask, thus the `shape_native` must be
        input into this method. The mask is setup as a unmasked `Mask2D` of shape_native.

        Parameters
        ----------
        vectors
            The (y,x) vectors input as an ndarray of shape [total_unmasked_pixells*(sub_size**2), 2] or a list of lists.
        shape_native
            The 2D shape of the mask the grid is paired with.
        pixel_scales
            The (y,x) scaled units to pixel units conversion factors of every pixel. If this is input as a ``float``,
            it is converted to a (float, float) structure.
        sub_size
            The size (sub_size x sub_size) of each unmasked pixels sub-grid.
        origin
            The origin of the grid's mask.
        """

        pixel_scales = geometry_util.convert_pixel_scales_2d(pixel_scales=pixel_scales)

        grid = Grid2D.uniform(
            shape_native=shape_native,
            pixel_scales=pixel_scales,
            sub_size=sub_size,
            origin=origin,
        )

        mask = Mask2D.unmasked(
            shape_native=shape_native,
            pixel_scales=pixel_scales,
            sub_size=sub_size,
            origin=origin,
        )

        vectors = abstract_grid_2d.convert_grid_2d(grid_2d=vectors, mask_2d=mask)

        return cls(vectors=vectors, grid=grid, mask=mask)

    @classmethod
    def manual_native(
        cls,
        vectors: Union[np.ndarray, List],
        pixel_scales: ty.PixelScales,
        sub_size: int = 1,
        origin: Tuple[float, float] = (0.0, 0.0),
    ) -> "VectorYX2D":
        """
        Create a VectorYX2D (see *VectorYX2D.__new__*) by inputting the grid coordinates in 2D, for example:

        vectors=np.ndarray([[[1.0, 1.0], [2.0, 2.0]],
                         [[3.0, 3.0], [4.0, 4.0]]])

        vectors=[[[1.0, 1.0], [2.0, 2.0]],
                [[3.0, 3.0], [4.0, 4.0]]]

        The `VectorYX2D` object assumes a uniform `Grid2D` which is computed from the mask's `shape_native`,
        `pixel_scales` and `origin`.

        The 2D shape of the grid and its mask are determined from the input grid and the mask is setup as an
        unmasked `Mask2D` of shape_native.

        Parameters
        ----------
        grid
            The (y,x) coordinates of the grid input as an ndarray of shape
            [total_y_coordinates*sub_size, total_x_pixel*sub_size, 2] or a list of lists.
        pixel_scales
            The (y,x) scaled units to pixel units conversion factors of every pixel. If this is input as a ``float``,
            it is converted to a (float, float) structure.
        sub_size
            The size (sub_size x sub_size) of each unmasked pixels sub-grid.
        origin
            The origin of the grid's mask.
        """

        vectors = abstract_grid.convert_grid(grid=vectors)

        pixel_scales = geometry_util.convert_pixel_scales_2d(pixel_scales=pixel_scales)

        shape_native = (
            int(vectors.shape[0] / sub_size),
            int(vectors.shape[1] / sub_size),
        )

        mask = Mask2D.unmasked(
            shape_native=shape_native,
            pixel_scales=pixel_scales,
            sub_size=sub_size,
            origin=origin,
        )

        grid = Grid2D.uniform(
            shape_native=shape_native,
            pixel_scales=pixel_scales,
            sub_size=sub_size,
            origin=origin,
        )

        vectors = abstract_grid_2d.convert_grid_2d(grid_2d=vectors, mask_2d=mask)

        return VectorYX2D(vectors=vectors, grid=grid, mask=mask)

    @classmethod
    def manual_mask(cls, vectors, mask) -> "VectorYX2D":
        """
        Create a VectorYX2D (see *VectorYX2D.__new__*) by inputting the vectors in 1D or 2D with its mask,
        for example:

        mask = Mask2D([[True, False, False, False])
        array=np.array([1.0, 2.0, 3.0])

        Parameters
        ----------
        array
            The values of the array input as an ndarray of shape [total_unmasked_pixels*(sub_size**2)] or a list of
            lists.
        mask
            The mask whose masked pixels are used to setup the sub-pixel grid.
        """

        grid = Grid2D.from_mask(mask=mask)

        vectors = abstract_grid_2d.convert_grid_2d(grid_2d=vectors, mask_2d=mask)
        return VectorYX2D(vectors=vectors, grid=grid, mask=mask)

    @classmethod
    def full(
        cls,
        fill_value,
        shape_native,
        pixel_scales: ty.PixelScales,
        sub_size: int = 1,
        origin: Tuple[float, float] = (0.0, 0.0),
    ):
        """
        Create a `VectorYX2D` (see `AbstractVectorYX2D.__new__`) where all values are filled with an input fill value,
        analogous to the method np.full().

        From 1D input the method cannot determine the 2D shape of the array and its mask, thus the shape_native must be
        input into this method. The mask is setup as a unmasked `Mask2D` of shape_native.

        Parameters
        ----------
        fill_value
            The value all array elements are filled with.
        pixel_scales
            The (y,x) scaled units to pixel units conversion factors of every pixel. If this is input as a ``float``,
            it is converted to a (float, float) structure.
        sub_size
            The size (sub_size x sub_size) of each unmasked pixels sub-grid.
        origin
            The origin of the grid's mask.
        origin
            The (y,x) scaled units origin of the mask's coordinate system.
        """
        if sub_size is not None:
            shape_native = (shape_native[0] * sub_size, shape_native[1] * sub_size)

        return cls.manual_native(
            vectors=np.full(
                fill_value=fill_value, shape=(shape_native[0], shape_native[1], 2)
            ),
            pixel_scales=pixel_scales,
            sub_size=sub_size,
            origin=origin,
        )

    @classmethod
    def ones(
        cls,
        shape_native,
        pixel_scales: ty.PixelScales,
        sub_size: int = 1,
        origin: Tuple[float, float] = (0.0, 0.0),
    ):
        """
        Create a `VectorYX2D` (see `AbstractVectorYX2D.__new__`) where all values are filled with 1.0, analogous to
        the method np.ones().

        From 1D input the method cannot determine the 2D shape of the array and its mask, thus the shape_native must be
        input into this method. The mask is setup as a unmasked `Mask2D` of shape_native.

        Parameters
        ----------
        fill_value
            The value all array elements are filled with.
        pixel_scales
            The (y,x) scaled units to pixel units conversion factors of every pixel. If this is input as a ``float``,
            it is converted to a (float, float) structure.
        sub_size
            The size (sub_size x sub_size) of each unmasked pixels sub-grid.
        origin
            The (y,x) scaled units origin of the mask's coordinate system.
        """
        return cls.full(
            fill_value=1.0,
            shape_native=shape_native,
            pixel_scales=pixel_scales,
            sub_size=sub_size,
            origin=origin,
        )

    @classmethod
    def zeros(
        cls,
        shape_native,
        pixel_scales: ty.PixelScales,
        sub_size: int = 1,
        origin: Tuple[float, float] = (0.0, 0.0),
    ):
        """
        Create a `VectorYX2D` (see `AbstractVectorYX2D.__new__`) where all values are filled with 1.0, analogous to
        the method np.zeros().

        From 1D input the method cannot determine the 2D shape of the array and its mask, thus the shape_native must be
        input into this method. The mask is setup as a unmasked `Mask2D` of shape_native.

        Parameters
        ----------
        fill_value
            The value all array elements are filled with.
        pixel_scales
            The (y,x) scaled units to pixel units conversion factors of every pixel. If this is input as a ``float``,
            it is converted to a (float, float) structure.
        sub_size
            The size (sub_size x sub_size) of each unmasked pixels sub-grid.
        origin
            The (y,x) scaled units origin of the mask's coordinate system.
        """
        return cls.full(
            fill_value=0.0,
            shape_native=shape_native,
            pixel_scales=pixel_scales,
            sub_size=sub_size,
            origin=origin,
        )

    def apply_mask(self, mask: Mask2D):
        return VectorYX2D.manual_mask(vectors=self.native, mask=mask)

    @property
    def magnitudes(self) -> Array2D:
        """
        Returns the magnitude of every vector which are computed as sqrt(y**2 + x**2).
        """
        return Array2D(
            array=np.sqrt(self[:, 0] ** 2.0 + self[:, 1] ** 2.0), mask=self.mask
        )

    @property
    def y(self) -> Array2D:
        """
        Returns the y vector values as an `Array2D` object.
        """
        return Array2D(array=self.slim[:, 0], mask=self.mask)

    @property
    def x(self) -> Array2D:
        """
        Returns the y vector values as an `Array2D` object.
        """
        return Array2D(array=self.slim[:, 1], mask=self.mask)

max_stars_repo_name: caoxiaoyue/PyAutoArray

max_issues_repo_name: caoxiaoyue/PyAutoArray

max_forks_repo_name: caoxiaoyue/PyAutoArray


text:
"""
WIP

This file should contain classes (that behave like torch models), but they
implement the learning of classical learning algorithms like SVM and
RandomForest.

Deep networks are amazing at learning features. However, I don't think it's
very useful to use linear logicstic regression as a classifier. In many cases I
think an SVM or a RandomForest might produce a superior classification model,
but this has yet to be shown.

TODO:
    - [ ] Classical Abstract API
    - [ ] Integration with the FitHarn
        - [ ] How do we swap netharn's backprop+SGD with sklearn's SVM and RandomForest fit methods?
        - [ ] Netharn needs a "classical" implementation of "run".
            - [ ] Simply use the data loader to load the data
            - [ ] Defaults should encourage use with deep features.
    - [ ] RandomForest
    - [ ] SVM
"""
import ubelt as ub

class ClassicalModule(ub.NiceRepr):
    """
    Abstraction for more classical approaches to learning classifiers
    """

    def __init__(self):
        raise NotImplementedError('abstract api')

    def fit(self, dataset):
        pass

class EstimatorFactory(ub.NiceRepr):
    """
    Returns sklearn classifier

    Example:
        >>> # xdoctest: +SKIP
        >>> from netharn.models.classical import *  # NOQA
        >>> self = factory = EstimatorFactory('RF', verbose=1)
        >>> print('factory = {!r}'.format(factory))
        >>> estimator = factory()
        >>> print('estimator = {!r}'.format(estimator))
    """
    def __init__(self, clf_key='RF-OVR', **kw):
        self.clf_key = clf_key
        self.kw = kw
        self.est_kw1 = None
        self.est_kw2 = None
        self.est_kw = None
        self.est_type = None
        self.wrap_type = None
        self._normalize_attrs()

    def __nice__(self):

        kw_repr = ub.repr2(self.est_kw, nl=0)
        if len(kw_repr) > 50:
            kw_repr = ub.hash_data(kw_repr)[0:8]

        text = 'clf_key={}, est_kw={}'.format(
            self.clf_key, self.est_type, self.wrap_type, kw_repr
        )
        return text

    def __call__(self):
        clf = self._make_estimator()
        return clf

    def _normalize_attrs(self):
        tup = self.clf_key.split('-')
        wrap_type = None if len(tup) == 1 else tup[1]
        est_type = tup[0]

        self.est_type = est_type
        self.wrap_type = wrap_type

        est_kw1, est_kw2 = self._lookup_params(self.est_type)
        self.est_kw1 = est_kw1
        self.est_kw2 = est_kw2
        self.est_kw = ub.dict_union(est_kw1, est_kw2, self.kw)

    def _lookup_params(self, est_type):
        if est_type in {'RF', 'RandomForest'}:
            self.est_type = 'RF'
            est_kw1 = {
                # 'max_depth': 4,
                'bootstrap': True,
                'class_weight': None,
                'criterion': 'entropy',
                'max_features': 'sqrt',
                # 'max_features': None,
                'min_samples_leaf': 5,
                'min_samples_split': 2,
                # 'n_estimators': 64,
                'n_estimators': 256,
            }
            est_kw2 = {
                'random_state': 3915904814,
                'verbose': 0,
                'n_jobs': -1,
            }
        elif est_type in {'SVC', 'SVM'}:
            self.est_type = 'SVC'
            est_kw1 = dict(kernel='linear')
            est_kw2 = {}
        elif est_type in {'Logit', 'LogisticRegression'}:
            self.est_type = 'Logit'
            est_kw1 = {}
            est_kw2 = {}
        elif est_type in {'MLP'}:
            est_kw1 = dict(
                activation='relu', alpha=1e-05, batch_size='auto',
                beta_1=0.9, beta_2=0.999, early_stopping=False,
                epsilon=1e-08, hidden_layer_sizes=(10, 10),
                learning_rate='constant', learning_rate_init=0.001,
                max_iter=200, momentum=0.9, nesterovs_momentum=True,
                power_t=0.5, random_state=3915904814, shuffle=True,
                solver='lbfgs', tol=0.0001, validation_fraction=0.1,
                warm_start=False
            )
            est_kw2 = dict(verbose=False)
        else:
            raise KeyError('Unknown Estimator')
        return est_kw1, est_kw2

    def _make_estimator(self):
        make_estimator = self._make_est_func()
        clf = make_estimator()
        return clf

    def _make_est_func(self):
        import sklearn
        from sklearn import multiclass  # NOQA
        from sklearn import ensemble  # NOQA
        from sklearn import neural_network  # NOQA
        from sklearn import svm  # NOQA
        from sklearn import preprocessing  # NOQA
        from sklearn import pipeline  # NOQA
        from functools import partial

        wrap_type = self.wrap_type
        est_type = self.est_type

        multiclass_wrapper = {
            None: ub.identity,
            'OVR': sklearn.multiclass.OneVsRestClassifier,
            'OVO': sklearn.multiclass.OneVsOneClassifier,
        }[wrap_type]
        est_class = {
            'RF': sklearn.ensemble.RandomForestClassifier,
            'SVC': sklearn.svm.SVC,
            'Logit': partial(sklearn.linear_model.LogisticRegression, solver='lbfgs'),
            'MLP': sklearn.neural_network.MLPClassifier,
        }[est_type]

        est_kw = self.est_kw
        try:
            from sklearn.impute import SimpleImputer
            Imputer = SimpleImputer
            import numpy as np
            NAN = np.nan
        except Exception:
            from sklearn.preprocessing import Imputer
            NAN = 'NaN'
        if est_type == 'MLP':
            def make_estimator():
                pipe = sklearn.pipeline.Pipeline([
                    ('inputer', Imputer(
                        missing_values=NAN, strategy='mean')),
                    # ('scale', sklearn.preprocessing.StandardScaler),
                    ('est', est_class(**est_kw)),
                ])
                return multiclass_wrapper(pipe)
        elif est_type == 'Logit':
            def make_estimator():
                pipe = sklearn.pipeline.Pipeline([
                    ('inputer', Imputer(
                        missing_values=NAN, strategy='mean')),
                    ('est', est_class(**est_kw)),
                ])
                return multiclass_wrapper(pipe)
        else:
            def make_estimator():
                return multiclass_wrapper(est_class(**est_kw))

        return make_estimator

class ClfProblem(ClassicalModule):
    """
    Takes the place of model in Hyperparams

    CommandLine:
        xdoctest -m netharn.models.classical ClfProblem:1

    Example:
        >>> # xdoctest: +SKIP
        >>> from netharn.models.classical import *  # NOQA
        >>> self = ClfProblem.demo('SVM-OVR', verbose=10)
        >>> print('self = {!r}'.format(self))
        >>> clf = self.fit()

    Example:
        >>> # xdoctest: +SKIP
        >>> from netharn.models.classical import *  # NOQA
        >>> print(chr(10) * 3 + '------------')
        >>> clf = ClfProblem.demo('SVM-OVR', verbose=1).fit()
        >>> clf = ClfProblem.demo('SVM-OVO', verbose=1).fit()
        >>> print(chr(10) * 3 + '------------')
        >>> clf = ClfProblem.demo('RF-OVR', verbose=1).fit()
        >>> clf = ClfProblem.demo('RF-OVO', verbose=1).fit()
        >>> print(chr(10) * 3 + '------------')
        >>> clf = ClfProblem.demo('Logit-OVR', verbose=1).fit()
        >>> clf = ClfProblem.demo('Logit-OVO', verbose=1).fit()
        >>> print(chr(10) * 3 + '------------')
        >>> clf = ClfProblem.demo('MLP-OVR', verbose=1).fit()
        >>> clf = ClfProblem.demo('MLP-OVO', verbose=1).fit()
    """
    def __init__(self, clf_key='RF', data=None, **kw):
        self.factory = EstimatorFactory(clf_key, **kw)
        self.data = None

        if data is not None:
            self.connect_data(data)

    def __nice__(self):
        return '{}, data=({})'.format(
            self.factory.__nice__(),
            self.data.__nice__(),)

    def connect_data(self, data):
        self.data = data

    def fit(self):
        X = self.data.X
        y = self.data.y
        clf = self.factory()
        clf = clf.fit(X, y)
        return clf

    @classmethod
    def demo(cls, clf_key='RF', **kw):
        import sklearn.datasets
        import pandas as pd
        iris = sklearn.datasets.load_iris()

        X_df = pd.DataFrame(iris.data, columns=iris.feature_names)
        Y_df = pd.DataFrame({name: iris.target == idx
                             for idx, name in enumerate(iris.target_names)})

        classes = list(map(str, Y_df.columns))
        class_idxs = (Y_df.values * [[0, 1, 2]]).sum(axis=1)

        data = ClfData(X_df, class_idxs, classes)
        self = cls(clf_key, data=data, **kw)
        return self

class ClfData(ub.NiceRepr):
    def __init__(self, X, y, classes):
        self.X = X
        self.y = y
        self.classes = classes

    def __nice__(self):
        return '{}'.format(len(self.X))

if __name__ == '__main__':
    """
    CommandLine:
        xdoctest -m netharn.models.classical all
    """
    import xdoctest
    xdoctest.doctest_module(__file__)

ext: py

max_stars_repo_name: VIAME/netharn

max_issues_repo_name: Kitware/netharn

max_forks_repo_name: Kitware/netharn

reason: import numpy


text:
#!/usr/bin/python3

import sys
import numpy as np
import argparse
import re
import subprocess
sys.path.append('/export/b07/jyang/kaldi-jyang/kaldi/src/featbin/')

def readStream(oneUtt):
    '''
    Readin: output of Kaldi command copy-feats, type str
    Return: Utt name (str) and feature matrix(numpy array, frame * 13 dimension)
    '''
    uttStr = re.sub('(\[|\])', '', oneUtt)
    uttList= uttStr.split('\n')
    uttList = list(filter(None, uttList))
    uttName = re.sub(r'\s+', '', uttList.pop(0))
    arr= []
    for nk, k in enumerate(uttList):
        perFrame = list(map(float, k.split()))
        arr.append(perFrame)
    arrNp = np.asarray(arr)
    return uttName, arrNp

#def main():
#    parser = argparse.ArgumentParser\
#    (description = 'Conver Kaldi MFCC features into python np arrays')
#    parser.add_argument('iList', help = 'Input MFCC scp file')
#    args = parser.parse_args()
#    with open(args.iList, 'r') as s:
#        filestr = s.read()
#        print (type(filestr))
#        uttName, arrNp = readScp(filestr)
#        print (arrNp.shape)

#if __name__ == '__main__':
#    main()

max_stars_repo_name: RobinAlgayres/beer

max_issues_repo_name: RobinAlgayres/beer

max_forks_repo_name: RobinAlgayres/beer


text:
import os
import numpy as np
import torch

from data.kitti_data import KittiDataset
from data.nuscenes_data import NuscenesDataset
from data.postech import PostechDataset

from models.models import HRegNet
from models.utils import calc_error_np, set_seed

import argparse
import datetime

def parse_args():
    parser = argparse.ArgumentParser(description='HRegNet')

    parser.add_argument('--seed', type=int, default=1)
    parser.add_argument('--gpu', type=str, default='1')
    parser.add_argument('--root', type=str, default='')
    parser.add_argument('--npoints', type=int, default=16384)
    parser.add_argument('--dataset', type=str, default='kitti')
    parser.add_argument('--use_fps', action='store_true')
    parser.add_argument('--data_list', type=str, default='')
    parser.add_argument('--use_weights', action='store_true')
    parser.add_argument('--pretrain_weights', type=str, default='')
    parser.add_argument('--voxel_size', type=float, default=0.3)
    parser.add_argument('--save_dir',type=str, default='')
    parser.add_argument('--augment', type=float, default=0.0)
    parser.add_argument('--freeze_detector', action='store_true')
    parser.add_argument('--freeze_feats', action='store_true')

    return parser.parse_args()

def test(args):
    if args.dataset == 'kitti':
        test_seqs = ['08','09','10']
        test_dataset = KittiDataset(args.root, test_seqs, args.npoints, args.voxel_size, args.data_list, args.augment)
    elif args.dataset == 'nuscenes':
        test_seqs = ['test']
        test_dataset = NuscenesDataset(args.root, test_seqs, args.npoints, args.voxel_size, args.data_list, args.augment)
    elif args.dataset == 'postech':
        test_seqs = ['test']
        test_dataset = PostechDataset(args.root, test_seqs, args.npoints, args.voxel_size, args.data_list, args.augment)
    else:
        raise('Not implemented')

    net = HRegNet(args).cuda()
    net.load_state_dict(torch.load(args.pretrain_weights))

    net.eval()

    trans_error_list = []
    rot_error_list = []
    pred_T_list = []
    delta_t_list = []
    trans_thresh = 2.0
    rot_thresh = 5.0
    success_idx = []

    with torch.no_grad():
        for idx in range(test_dataset.__len__()):
            start_t = datetime.datetime.now()
            src_points, dst_points, gt_R, gt_t = test_dataset[idx]
            src_points = src_points.unsqueeze(0).cuda()
            dst_points = dst_points.unsqueeze(0).cuda()
            gt_R = gt_R.numpy()
            gt_t = gt_t.numpy()
            ret_dict = net(src_points, dst_points)
            end_t = datetime.datetime.now()
            pred_R = ret_dict['rotation'][-1]
            pred_t = ret_dict['translation'][-1]
            pred_R = pred_R.squeeze().cpu().numpy()
            pred_t = pred_t.squeeze().cpu().numpy()
            rot_error, trans_error = calc_error_np(pred_R, pred_t, gt_R, gt_t)
            pred_T = np.zeros((3,4))
            gt_T = np.zeros((3,4))
            pred_T[:3,:3] = pred_R
            pred_T[:3,3] = pred_t
            gt_T[:3,:3] = gt_R
            gt_T[:3,3] = gt_t
            pred_T = pred_T.flatten()
            gt_T = gt_T.flatten()
            pred_T_list.append(pred_T)
            print('{:d}: trans: {:.4f} rot: {:.4f}'.format(idx, trans_error, rot_error))
            trans_error_list.append(trans_error)
            rot_error_list.append(rot_error)

            if trans_error < trans_thresh and rot_error < rot_thresh:
                success_idx.append(idx)

            delta_t = (end_t - start_t).microseconds
            delta_t_list.append(delta_t)

    success_rate = len(success_idx)/test_dataset.__len__()
    trans_error_array = np.array(trans_error_list)
    rot_error_array = np.array(rot_error_list)
    trans_mean = np.mean(trans_error_array[success_idx])
    trans_std = np.std(trans_error_array[success_idx])
    rot_mean = np.mean(rot_error_array[success_idx])
    rot_std = np.std(rot_error_array[success_idx])
    delta_t_array = np.array(delta_t_list)
    delta_t_mean = np.mean(delta_t_array)

    print('Translation mean: {:.4f}'.format(trans_mean))
    print('Translation std: {:.4f}'.format(trans_std))
    print('Rotation mean: {:.4f}'.format(rot_mean))
    print('Rotation std: {:.4f}'.format(rot_std))
    print('Runtime: {:.4f}'.format(delta_t_mean))
    print('Success rate: {:.4f}'.format(success_rate))

    if not os.path.exists(args.save_dir):
        os.makedirs(args.save_dir)
    pred_T_array = np.array(pred_T_list)
    np.savetxt(os.path.join(args.save_dir, args.dataset+'_pred.txt'), pred_T_array)
    np.savetxt(os.path.join(args.save_dir, args.dataset+'_trans_error.txt'), trans_error_list)
    np.savetxt(os.path.join(args.save_dir, args.dataset+'_rot_error.txt'), rot_error_list)

    f_summary = open(os.path.join(args.save_dir, args.dataset+'_summary.txt'), 'w')
    f_summary.write('Dataset: '+args.dataset+'\n')
    f_summary.write('Translation threshold: {:.2f}\n'.format(trans_thresh))
    f_summary.write('Rotation threshold: {:.2f}\n'.format(rot_thresh))
    f_summary.write('Translation mean: {:.4f}\n'.format(trans_mean))
    f_summary.write('Translation std: {:.4f}\n'.format(trans_std))
    f_summary.write('Rotation mean: {:.4f}\n'.format(rot_mean))
    f_summary.write('Rotation std: {:.4f}\n'.format(rot_std))
    f_summary.write('Runtime: {:.4f}\n'.format(delta_t_mean))
    f_summary.write('Success rate: {:.4f}\n'.format(success_rate))
    f_summary.close()

    print('Saved results to ' + args.save_dir)

if __name__ == '__main__':
    args = parse_args()

    os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
    set_seed(args.seed)

    test(args)

max_stars_repo_name: Kwonyoung-Ryu/HRegNet

max_issues_repo_name: Kwonyoung-Ryu/HRegNet

max_forks_repo_name: Kwonyoung-Ryu/HRegNet


text:
import pytest
import numpy as np
import jax
import pyscf
from pyscfad import gto
from pyscfad.lib import numpy as jnp

TOL_VAL = 1e-12
TOL_NUC = 1e-10
TOL_CS = 1e-10
TOL_EXP = 1e-9
TOL_NUC2 = 5e-9

TEST_SET = ["int1e_ovlp", "int1e_kin", "int1e_nuc",
            "int1e_rinv",]
TEST_SET_ECP = ["ECPscalar"]
TEST_SET_NUC = ["int1e_nuc"]
TEST_SET_2C2E = ["int2c2e",]

@pytest.fixture
def get_mol0():
    mol = pyscf.M(
        atom = 'O 0. 0. 0.; H 0. , -0.757 , 0.587; H 0. , 0.757 , 0.587',
        basis = 'ccpvdz',
        verbose=0,
    )
    return mol

@pytest.fixture
def get_mol():
    mol = gto.Mole()
    mol.atom = 'O 0. 0. 0.; H 0. , -0.757 , 0.587; H 0. , 0.757 , 0.587'
    mol.basis = 'ccpvdz'
    mol.verbose=0
    mol.build(trace_coords=True, trace_exp=True, trace_ctr_coeff=True)
    return mol

@pytest.fixture
def get_mol_ecp0():
    mol = pyscf.M(
        atom = '''
            Na 0. 0. 0.
            H  0.  0.  1.
        ''',
        basis = {'Na':'lanl2dz', 'H':'sto3g'},
        ecp = {'Na':'lanl2dz'},
        verbose=0,
    )
    return mol

@pytest.fixture
def get_mol_ecp():
    mol = gto.Mole()
    mol.atom = '''
        Na 0. 0. 0.
        H  0.  0.  1.
    '''
    mol.basis = {'Na':'lanl2dz', 'H':'sto3g'}
    mol.ecp = {'Na':'lanl2dz'}
    mol.verbose=0
    mol.build(trace_coords=True, trace_exp=True, trace_ctr_coeff=True)
    return mol

def grad_analyt(mol, intor):
    atmlst = range(mol.natm)
    aoslices = mol.aoslice_by_atom()
    nao = mol.nao
    g = np.zeros((nao,nao,mol.natm,3))
    s1 = -mol.intor(intor, comp=3)
    for k, ia in enumerate(atmlst):
        p0, p1 = aoslices[ia,2:]
        g[p0:p1,:,k] += s1[:,p0:p1].transpose(1,2,0)
        g[:,p0:p1,k] += s1[:,p0:p1].transpose(2,1,0)
    return g

def nuc_grad_analyt(mol):
    atmlst = range(mol.natm)
    aoslices = mol.aoslice_by_atom()
    nao = mol.nao
    g = np.zeros((nao,nao,mol.natm,3))
    h1 = -mol.intor("int1e_ipnuc", comp=3)
    for k, ia in enumerate(atmlst):
        p0, p1 = aoslices [ia,2:]
        with mol.with_rinv_at_nucleus(ia):
            vrinv = mol.intor('int1e_iprinv', comp=3)
            vrinv *= -mol.atom_charge(ia)
        vrinv[:,p0:p1] += h1[:,p0:p1]
        g[:,:,k] = vrinv.transpose(1,2,0) + vrinv.transpose(2,1,0)
    return g

def ECPscalar_grad_analyt(mol):
    atmlst = range(mol.natm)
    aoslices = mol.aoslice_by_atom()
    nao = mol.nao
    g = np.zeros((nao,nao,mol.natm,3))
    h1 = -mol.intor("ECPscalar_ipnuc", comp=3)
    for k, ia in enumerate(atmlst):
        p0, p1 = aoslices [ia,2:]
        with mol.with_rinv_at_nucleus(ia):
            vrinv = mol.intor('ECPscalar_iprinv', comp=3)
        vrinv[:,p0:p1] += h1[:,p0:p1]
        g[:,:,k] = vrinv.transpose(1,2,0) + vrinv.transpose(2,1,0)
    return g

def four_point_fd(mol, intor, _env_of, disp=1e-4):
    grad_fd = []
    for _, ptr_exp in enumerate(_env_of):
        #ptr_exp = _env_of[i]
        mol._env[ptr_exp] += disp
        sp = mol.intor(intor)
        mol._env[ptr_exp] += disp
        sp2 = mol.intor(intor)
        mol._env[ptr_exp] -= disp * 4.
        sm2 = mol.intor(intor)
        mol._env[ptr_exp] += disp
        sm = mol.intor(intor)
        g = (8.*(sp-sm) - sp2 + sm2) / (12.*disp)
        grad_fd.append(g)
        mol._env[ptr_exp] += disp
    return np.asarray(grad_fd).transpose(1,2,0)

def cs_grad_fd(mol, intor):
    disp = 1e-3
    _, _, _env_of = gto.mole.setup_ctr_coeff(mol)
    g = four_point_fd(mol, intor, _env_of, disp)
    return g

def exp_grad_fd(mol, intor):
    disp = 1e-4
    _, _, _env_of = gto.mole.setup_exp(mol)
    g = four_point_fd(mol, intor, _env_of, disp)
    return g

def func(mol, intor):
    return mol.intor(intor)

def func1(mol, intor):
    return jnp.linalg.norm(mol.intor(intor))

def _test_int1e_value(intor, mol0, mol1, tol=TOL_VAL):
    v0 = mol0.intor(intor)
    v1 = mol1.intor(intor)
    assert abs(v1-v0).max() < tol

def _test_int1e_deriv_fd(intor, mol0, mol1, funfd, jacattr, tol):
    v0 = mol0.intor(intor)
    jac_fd = funfd(mol0, intor)
    jac_fwd = jax.jacfwd(func)(mol1, intor)
    jac_rev = jax.jacrev(func)(mol1, intor)
    assert abs(getattr(jac_fwd, jacattr) - jac_fd).max() < tol
    assert abs(getattr(jac_rev, jacattr) - jac_fd).max() < tol

    g0 = np.einsum("ij,ijx->x", v0, jac_fd) / np.linalg.norm(v0)
    jac_fwd = jax.jacfwd(func1)(mol1, intor)
    jac_rev = jax.jacrev(func1)(mol1, intor)
    assert abs(getattr(jac_fwd, jacattr) - g0).max() < tol
    assert abs(getattr(jac_rev, jacattr) - g0).max() < tol

def _test_int1e_deriv_cs(intor, mol0, mol1, tol=TOL_CS):
    _test_int1e_deriv_fd(intor, mol0, mol1,
                         cs_grad_fd, "ctr_coeff", tol)

def _test_int1e_deriv_exp(intor, mol0, mol1, tol=TOL_EXP):
    _test_int1e_deriv_fd(intor, mol0, mol1,
                         exp_grad_fd, "exp", tol)

def _test_int1e_deriv_nuc(intor, mol0, mol1, funanal, args, tol=TOL_NUC):
    v0 = mol0.intor(intor)
    jac0 = funanal(*args)
    jac_fwd = jax.jacfwd(func)(mol1, intor)
    jac_rev = jax.jacrev(func)(mol1, intor)
    assert abs(jac_fwd.coords - jac0).max() < tol
    assert abs(jac_rev.coords - jac0).max() < tol

    g0 = np.einsum("ij,ijnx->nx", v0, jac0) / np.linalg.norm(v0)
    jac_fwd = jax.jacfwd(func1)(mol1, intor)
    jac_rev = jax.jacrev(func1)(mol1, intor)
    assert abs(jac_fwd.coords - g0).max() < tol
    assert abs(jac_rev.coords - g0).max() < tol

# pylint: disable=redefined-outer-name
def test_int1e_deriv1(get_mol0, get_mol, get_mol_ecp0, get_mol_ecp):
    mol0 = get_mol0
    mol1 = get_mol
    for intor in TEST_SET:
        _test_int1e_value(intor, mol0, mol1)
        _test_int1e_deriv_cs(intor, mol0, mol1)
        _test_int1e_deriv_exp(intor, mol0, mol1)

    for intor in set(TEST_SET) - set(TEST_SET_NUC):
        _test_int1e_deriv_nuc(intor, mol0, mol1, grad_analyt,
                              (mol0, intor.replace("int1e_", "int1e_ip")))

    for intor in TEST_SET_NUC:
        _test_int1e_deriv_nuc(intor, mol0, mol1, nuc_grad_analyt, (mol0,))

    molecp0 = get_mol_ecp0
    molecp1 = get_mol_ecp
    for intor in TEST_SET_ECP:
        _test_int1e_value(intor, molecp0, molecp1)
        _test_int1e_deriv_cs(intor, molecp0, molecp1)
        _test_int1e_deriv_exp(intor, molecp0, molecp1)
        _test_int1e_deriv_nuc(intor, molecp0, molecp1, ECPscalar_grad_analyt, (molecp0,))

    for intor in TEST_SET_2C2E:
        _test_int1e_value(intor, mol0, mol1)
        _test_int1e_deriv_cs(intor, mol0, mol1)
        _test_int1e_deriv_exp(intor, mol0, mol1, tol=5e-8)
        _test_int1e_deriv_nuc(intor, mol0, mol1, grad_analyt,
                              (mol0, intor.replace("int2c2e", "int2c2e_ip1")))

max_stars_repo_name: yangdatou/pyscfad

max_issues_repo_name: yangdatou/pyscfad

max_forks_repo_name: yangdatou/pyscfad

reason: import numpy,import jax


text:
import pandas as pd
import numpy as np
import subprocess

# list of languages
#languages = ['Albanian', 'Bosnian', 'Bulgarian', 'Croatian', 'English', 'German', 'Hungarian', 'Polish', 'Portuguese', 'Russian', 'Serbian', 'Slovak', 'Slovenian', 'Swedish']
languages = ['Slovenian', 'Swedish']

# list of language abbreviations
#abbreviations = ['sq', 'bs', 'bg', 'hr', 'en', 'de', 'hu', 'pl', 'pt', 'ru', 'sr', 'sk', 'sl', 'sv']
abbreviations = ['sl', 'sv']

for lang, abrev in zip(languages, abbreviations):
    # encode text and write to file
    infile = '../data/clean/text/{}.csv'.format(lang)
    outfile = '../data/embed/{}.raw'.format(lang)
    subprocess.call('./embed.sh %s %s %s' % (infile, abrev, outfile), shell=True)


text:
import numpy as np
import scipy.io as sio
from PIL import Image
import os
from config import *

attributes = "person	imagenum	Male	Asian	White	Black	Baby	Child	Youth	Middle Aged	Senior	Black Hair	Blond Hair	Brown Hair	Bald" \
       "	No Eyewear	Eyeglasses	Sunglasses	Mustache	Smiling	Frowning	Chubby	Blurry	Harsh Lighting	Flash	Soft Lighting	Outdoor	Curly Hair	Wavy Hair" \
       "	Straight Hair	Receding Hairline	Bangs	Sideburns	Fully Visible Forehead	Partially Visible Forehead	Obstructed Forehead	Bushy Eyebrows	Arched Eyebrows	Narrow Eyes	Eyes Open" \
       "	Big Nose	Pointy Nose	Big Lips	Mouth Closed	Mouth Slightly Open	Mouth Wide Open	Teeth Not Visible	No Beard	Goatee	Round Jaw	Double Chin	Wearing Hat	Oval Face" \
       "	Square Face	Round Face	Color Photo	Posed Photo	Attractive Man	Attractive Woman	Indian	Gray Hair	Bags Under Eyes	Heavy Makeup	Rosy Cheeks	Shiny Skin	Pale Skin" \
       "	5 o' Clock Shadow	Strong Nose-Mouth Lines	Wearing Lipstick	Flushed Face	High Cheekbones	Brown Eyes	Wearing Earrings	Wearing Necktie	Wearing Necklace"

class person:
    def __init__(self, name, dist, imgnum):
        self.name = name
        self.num = imgnum
        self.dist = dist

def load_label(path="./lfw_label.mat"):
    return sio.loadmat(path)["lfwlabel1"]

def l1_distance(attr0, attr1):
    return np.mean(np.abs(attr0 - attr1))

def KNN(center, data, k=100):
    #col0: person, col1: image_num, col2-col75: attribute value
    dist_all = []
    #initialize
    dist0 = l1_distance(center[2:], data[0, 2:])
    p = person(data[0, 0], dist0, data[0, 1])
    dist_all.append(p)
    #KNN match
    for i in range(1, np.size(data, 0)):
        dist = l1_distance(center[2:], data[i, 2:])
        for j in range(dist_all.__len__()):
            if dist < dist_all[j].dist:
                p = person(data[i, 0], dist, data[i, 1])
                dist_all.insert(j, p)
                break
            if j == dist_all.__len__()-1:
                p = person(data[i, 0], dist, data[i, 1])
                dist_all.append(p)

    return dist_all[:k]

def get_corresponding_attr_dataset(data, attribute):
    attr_num = attributes.split("\t").__len__()
    attrs = dict(zip(attributes.split("\t"), range(attr_num)))#attrs: "person":0, "imagenum":1, "Male":2, ...
    neg = 0
    pos = 0
    for i in range(np.size(data, 0)):
        if data[i, attrs[attribute]] < 0:
            neg += 1
        else:
            pos += 1
    negative = np.zeros([neg, attr_num], dtype=np.object)
    positive = np.zeros([pos, attr_num], dtype=np.object)
    neg = 0
    pos = 0
    for i in range(np.size(data, 0)):
        if data[i, attrs[attribute]] < -1:
            negative[neg, :] = data[i, :]
            neg += 1
        elif data[i, attrs[attribute]] > 1:
            positive[pos, :] = data[i, :]
            pos += 1
    return negative, positive

def save_KNN_img(attribute, object_attr_vector):
    data = load_label()
    negative, positive = get_corresponding_attr_dataset(data, attribute)
    img_names = os.listdir("./lfw//" + object_attr_vector[0][0].replace(" ", "_"))
    Image.open("./lfw//" + object_attr_vector[0][0].replace(" ", "_") + "//" + img_names[0]).save("./input_img//" + object_attr_vector[0][0] + ".jpg")
    neg = KNN(object_attr_vector, negative, K)
    pos = KNN(object_attr_vector, positive, K)
    for i in range(K):
        filename = neg[i].name[0].replace(" ", "_")
        img_num = neg[i].num[0]
        Image.open("./lfw//" + filename + "//" + filename + "_" + "%04d" % (img_num) + ".jpg").save("./source//" + str(i) + ".jpg")
        filename = pos[i].name[0].replace(" ", "_")
        img_num = pos[i].num[0]
        Image.open("./lfw//" + filename + "//" + filename + "_" + "%04d" %(img_num) + ".jpg").save("./target//" + str(i) + ".jpg")
    pass

if __name__=="__main__":
    #Mouth Slightly Open
    #Mustache
    #Smiling
    #Senior
    #Middle Aged
    #Eyes Open
    #Eyeglasses
    target_attr = "Mustache"
    data = load_label()
    save_KNN_img(target_attr, data[0, :])

reason: import numpy,import scipy


text:
import os
import numpy as np
import dolfin as df
from finmag.field import Field
from finmag.util.meshes import from_geofile
from finmag.energies import UniaxialAnisotropy, CubicAnisotropy
from finmag.util.helpers import sphinx_sci as s
import pytest

MODULE_DIR = os.path.dirname(os.path.abspath(__file__))

@pytest.fixture
def finmag(request):
    finmag = request.cached_setup(setup=setup,
                                  teardown=teardown, scope="session")
    return finmag

Ms = 0.86e6
K1 = 520e3
K2 = 230e3
K3 = 123e3
u1 = (1, 0, 0)
u2 = (0, 1, 0)
x1 = y1 = z1 = 20
# same as in bar.geo file

def m_gen(r):
    x = np.maximum(np.minimum(r[0] / x1, 1.0), 0.0)  # x, y and z as a fraction
    y = np.maximum(np.minimum(r[1] / y1, 1.0), 0.0)  # between 0 and 1 in the
    z = np.maximum(np.minimum(r[2] / z1, 1.0), 0.0)
    mx = (2 - y) * (2 * x - 1) / 4
    mz = (2 - y) * (2 * z - 1) / 4
    my = np.sqrt(1 - mx ** 2 - mz ** 2)
    return np.array([mx, my, mz])

def setup(K2=K2):
    print "Running finmag..."
    mesh = from_geofile(os.path.join(MODULE_DIR, "bar.geo"))
    coords = np.array(zip(* mesh.coordinates()))

    S3 = df.VectorFunctionSpace(mesh, "Lagrange", 1, dim=3)
    m = Field(S3)
    m.set_with_numpy_array_debug(m_gen(coords).flatten())

    S1 = df.FunctionSpace(mesh, "Lagrange", 1)
    Ms_cg = Field(df.FunctionSpace(mesh, 'CG', 1), Ms)

    anisotropy = UniaxialAnisotropy(K1, u1, K2=K2)
    anisotropy.setup(m, Ms_cg, unit_length=1e-9)

    H_anis = df.Function(S3)
    H_anis.vector()[:] = anisotropy.compute_field()
    return dict(m=m, H=H_anis, S3=S3, table=start_table())

def setup_cubic():
    print "Running finmag..."
    mesh = from_geofile(os.path.join(MODULE_DIR, "bar.geo"))
    coords = np.array(zip(* mesh.coordinates()))

    S3 = df.VectorFunctionSpace(mesh, "Lagrange", 1, dim=3)
    m = Field(S3)
    m.set_with_numpy_array_debug(m_gen(coords).flatten())

    S1 = df.FunctionSpace(mesh, "Lagrange", 1)
    Ms_cg = Field(df.FunctionSpace(mesh, 'CG', 1), Ms)

    anisotropy = CubicAnisotropy(u1=u1, u2=u2, K1=K1, K2=K2, K3=K3)
    anisotropy.setup(m, Ms_cg, unit_length=1e-9)

    H_anis = Field(S3)
    H_anis.set_with_numpy_array_debug(anisotropy.compute_field())
    return dict(m=m, H=H_anis.f, S3=S3, table=start_table())

def teardown(finmag):
    write_table(finmag)

table_delim = "    " + "=" * 10 + (" " + "=" * 30) * 4 + "\n"
table_entries = "    {:<10} {:<30} {:<30} {:<30} {:<30}\n"

def start_table():
    table = ".. _anis_table:\n\n"
    table += ".. table:: Comparison of the anisotropy field computed with finmag against nmag, oommf and magpar\n\n"
    table += table_delim
    table += table_entries.format(
        # hack because sphinx light table syntax does not allow an empty header
        ":math:`\,`",
        ":math:`\\subn{\\Delta}{test}`",
        ":math:`\\subn{\\Delta}{max}`",
        ":math:`\\bar{\\Delta}`",
        ":math:`\\sigma`")
    table += table_delim
    return table

def write_table(finmag):
    finmag["table"] += table_delim
    with open(os.path.join(MODULE_DIR, "table.rst"), "w") as f:
        f.write(finmag["table"])

def table_entry(name, tol, rel_diff):
    return table_entries.format(
        name, s(tol, 0),
        s(np.max(rel_diff)), s(np.mean(rel_diff)), s(np.std(rel_diff)))

max_stars_repo_name: davidcortesortuno/finmag

max_issues_repo_name: davidcortesortuno/finmag

max_forks_repo_name: davidcortesortuno/finmag


text:
import numpy as np
import math

__author__ = "Anton Höß"
__copyright__ = "Copyright 2020"

class Vehicle:
    """Defines a vehicle which can be detected by sensors.

    Parameters
    ----------
    name : str
        The name of the vehicle.
    active : bool
        Defines if the vehicle is active on starting the simulation (can be activated later).
    v : float, optional
        The vehicle's velocity.
    q : float, optional
        The vehicle's acceleration.
    """

    def __init__(self, name, active, v=100., q=10.):
        self.name = name
        self.active = active
        self.v_max = v
        self.q_max = q

        self.A = v * v / q
        self.omega = q / (2 * v)

        self.r = np.zeros(2)
        self.rd = np.zeros(2)       # r'(t)         -> velocity
        self.rdd = np.zeros(2)      # r''(t)        -> acceleration
        self.rdt = np.zeros(2)      # t(t)          -> tangent (normalized)
        self.rdn = np.zeros(2)      # n(t)          -> normal (normalized)
        self.rddxrdt = np.zeros(2)  # r''(t) x t(t) -> acceleration x tangent (normalized)
        self.rddxrdn = np.zeros(2)  # r''(t) x n(t) -> acceleration x normal (normalized)
    # end def

    def __str__(self):
        return "{}: vel={}, accel={}, pos=({:10.4f} {:10.4f})".format(self.name, self.v_max, self.q_max, self.r[0], self.r[1])
    # end def

    def update(self, t):
        """Updates the Vehicle's state (position, velocity, acceleration).

        Parameters
        ----------
        t : float
            The time to calculate the Vehicle's state for.
        """
        vec = np.array([math.sin(self.omega * t), math.sin(2 * self.omega * t)])
        self.r = self.A * vec

        vec = np.array([math.cos(self.omega * t) / 2.0, math.cos(2 * self.omega * t)])
        self.rd = self.v_max * vec

        vec = np.array([math.sin(self.omega * t) / 4.0, math.sin(2 * self.omega * t)])
        self.rdd = -self.q_max * vec

        self.rdt = self.rd / np.linalg.norm(self.rd)

        self.rdn = np.array([-self.rd[1], self.rd[0]]) / np.linalg.norm(self.rd)

        self.rddxrdt = self.rdd * self.rdt

        self.rddxrdn = self.rdd * self.rdn
    # end def
# end class


text:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import cantera as ct
import numpy as np
import math

# reference about fitting thermo data: https://github.com/Upstream-Applied-Science/canteraJanaf
# reference about writing thermo file: https://github.com/jiweiqi/Cantera2Chemkin/blob/master/soln2ck.py

def build_nasa(nasa_coeffs, row):
            """
            Creates string of nasa polynomial coefficients
            :param nasa_coeffs
                cantera species thermo coefficients object
            :param row
                which row to write coefficients in
            """
            line_coeffs = ''
            lines = [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14]]
            line_index = lines[row-2]
            for ix, c in enumerate(nasa_coeffs):
                if ix in line_index:
                    if c >= 0:
                        line_coeffs += ' '
                    line_coeffs += str('{:.8e}'.format(c))
            return line_coeffs

def write(gas, thermo_fileName_new, plot):
    """Function to write cantera gas object to inp file.
    :param gas:
        gas: Cantera gas object,
        thermo_fileName_new: Name of converted thermo file
        plot: whether plot or not
    """
    with open(thermo_fileName_new, 'w+') as f:
        #f.write('THERMO ALL' + '\n' +
        #        '   300.000  1000.000  5000.000' +'\n')

        Tlow = 300
        Tcommon = 1000
        Thigh = 5000
        deltaT = 100
        T_low = np.arange(Tlow,Tcommon+deltaT,deltaT)
        T_high = np.arange(Tcommon,Thigh+deltaT,deltaT)
        temp_rangeAll = '{0:10.3f}'.format(Tlow) + '{0:10.3f}'.format(Tcommon) + '{0:10.3f}'.format(Thigh)
        f.write('THERMO ALL' + '\n' + temp_rangeAll + '\n')

        #write data for each species in the Solution object
        for sp_index in range(len(gas.species_names)):
            species = gas.species(sp_index)
            species_name = gas.species_name(sp_index)
            molecular_weight = gas.molecular_weights[sp_index]
            nasa_coeffs = species.thermo.coeffs # orininal coeffs, size = 15, [0]=Tcommon, [1-7] is high coeffs, [8-14] is low coeffs
            temp_range = '{:<10}'.format(Tlow) + '{:<10}'.format(Thigh) + '{:<8}'.format(Tcommon)

            if (nasa_coeffs[0] != Tcommon):
                print("fitting species: %s"%(species_name), "from ",nasa_coeffs[0], "to ",Tcommon)
                cp_low = [species.thermo.cp(T)/molecular_weight for T in T_low] # [J/(kmol·K)] / [kg/kmol] = [J/(kg·K)]
                cp_high = [species.thermo.cp(T)/molecular_weight for T in T_high] # [J/(kmol·K)] / [kg/kmol] = [J/(kg·K)]

                cp_low_polynomials = np.polyfit(T_low, cp_low, 4)
                cp_high_polynomials = np.polyfit(T_high, cp_high, 4)

                R = ct.gas_constant/molecular_weight  # [J/(kmol·K)] / [kg/kmol] = [J/(kg·K)]

                #reverse order and divide by R (np.ployfit has calculated coeffs*R so we must divide by R to get just the coeffs)
                a1_a5_low = np.flip(cp_low_polynomials,0)/R
                a1_a5_high = np.flip(cp_high_polynomials,0)/R

                T_ref = 298.15
                h = species.thermo.h(T_ref)/molecular_weight # [J/kmol] / [kg/kmol] = [J/kg]
                s = species.thermo.s(T_ref)/molecular_weight # [J/kmol/K] / [kg/kmol] = [J/(kg·K)]

                a6_low = h/R - np.sum(np.array([a1_a5_low[i]*T_ref**(i+1)/(i+1) for i in range(5)]))
                a6_high = h/R - np.sum(np.array([a1_a5_high[i]*T_ref**(i+1)/(i+1) for i in range(5)]))

                a7_low = s/R - np.sum(np.array([a1_a5_low[i]*T_ref**(i)/(i) for i in range(1,5)]))-a1_a5_low[0]*math.log(T_ref)
                a7_high = s/R - np.sum(np.array([a1_a5_high[i]*T_ref**(i)/(i) for i in range(1,5)]))-a1_a5_high[0]*math.log(T_ref)

                nasa_coeffs = np.concatenate([[Tcommon], a1_a5_high, [a6_high], [a7_high], a1_a5_low, [a6_low], [a7_low]])

                if plot:
                    Tlist = np.concatenate([T_low, T_high])
                    cp = [species.thermo.cp(T)/molecular_weight for T in Tlist] # [J/(kg·K)]
                    cp_fit_low = [ np.sum(np.array([a1_a5_low[i]*T**(i) for i in range(5)])) for T in T_low] # cp/R
                    cp_fit_high = [ np.sum(np.array([a1_a5_high[i]*T**(i) for i in range(5)])) for T in T_high] # cp/R
                    cp_fit = np.concatenate([cp_fit_low, cp_fit_high])*R

                    a1_a5_org_low = species.thermo.coeffs[8:13]
                    a1_a5_org_high = species.thermo.coeffs[1:6]
                    cp_bad_low = [ np.sum(np.array([a1_a5_org_low[i]*T**(i) for i in range(5)])) for T in T_low] # cp/R
                    cp_bad_high = [ np.sum(np.array([a1_a5_org_high[i]*T**(i) for i in range(5)])) for T in T_high] # cp/R
                    cp_bad = np.concatenate([cp_bad_low, cp_bad_high])*R

                    enthalpy = [species.thermo.h(T)/molecular_weight for T in Tlist] # J/kg
                    enthalpy_fit_low = [ a6_low + np.sum(np.array([a1_a5_low[i]*T**(i+1)/(i+1) for i in range(5)])) for T in T_low] # h/R
                    enthalpy_fit_high = [ a6_high + np.sum(np.array([a1_a5_high[i]*T**(i+1)/(i+1) for i in range(5)])) for T in T_high] # h/R
                    enthalpy_fit = np.concatenate([enthalpy_fit_low, enthalpy_fit_high])*R

                    fig = plt.figure()

                    ax1 = fig.add_subplot()
                    ax1.scatter(Tlist, cp, c='blue', label="cp_org, Tcommon=%d"%(species.thermo.coeffs[0]))
                    ax1.plot(Tlist, cp_fit, c='red', label="cp_fit, Tcommon=1000")
                    ax1.plot(Tlist, cp_bad, c='purple', label="cp_bad")
                    ax1.set_xlabel("T [K]")
                    ax1.set_ylabel("cp [J/kg]")

                    ax2 = plt.twinx()
                    ax2.scatter(Tlist, enthalpy, c='black', label="h_org, Tcommon=%d"%(species.thermo.coeffs[0]))
                    ax2.plot(Tlist, enthalpy_fit, c='green', label="h_fit, Tcommon=1000")
                    ax2.set_ylabel("h [J/kg]")

                    fig.legend(loc="lower right", bbox_to_anchor=(1,0), bbox_transform=ax1.transAxes)
                    plt.savefig("%s.png"%(species_name), dpi=500, bbox_inches='tight', pad_inches=0.1)
                    plt.close()

            species_comp = ''
            for atom in species.composition:
                species_comp += '{:<3}'.format(atom.upper())
                species_comp += '{:<2}'.format(str(int(species.composition[atom])))

            species_phase = 'G'

            # Ref: Table 3. Summary of the Rules for Thermo Data
            # https://shepherd.caltech.edu/EDL/PublicResources/sdt/formats/chemkin.html#:~:text=The%20Chemkin%20thermo%20file%20format,%2C%20Sandia%20Report%20SAND89%2D8009.

            line_1 = (
                    '{:<18}'.format(species_name) +
                    '{:<6}'.format(' ') +
                    '{:<20}'.format(species_comp) +
                    '{:<1}'.format(species_phase) +
                    '{:<28}'.format(temp_range) +
                    '{:<5}'.format(' ') +
                    '{:<1}'.format(' ') +
                    '{:<1}'.format('1') +
                    '\n')
            f.write(line_1)

            line_2_coeffs = build_nasa(nasa_coeffs, 2)
            line_2 = line_2_coeffs  + '    2\n'
            f.write(line_2)

            line_3_coeffs = build_nasa(nasa_coeffs, 3)
            line_3 = line_3_coeffs + '    3\n'
            f.write(line_3)

            line_4_coeffs = build_nasa(nasa_coeffs, 4)
            line_4 = line_4_coeffs + '                   4\n'
            f.write(line_4)

        f.write('END\n')

if __name__ == '__main__':
    mech='mech.cti'
    gas = ct.Solution(mech)
    thermo_fileName_new = 'thermoCorrected.dat'
    print(thermo_fileName_new)
    plot=True # True or False
    write(gas, thermo_fileName_new, plot)


text:
import netomaton as ntm
import numpy as np
import matplotlib.pyplot as plt
import random

"""
A minimal model of a growing network.
"At each time step, a new vertex is added; then, with probability δ, two vertices are chosen uniformly at random and
joined by an undirected edge. This process is repeated for t time steps."

If the probability of attaching two nodes is p, then the probability of a randomly
chosen node have degree k is: (2p)^k / (1 + 2p)^(k+1).

See: "Are randomly grown graphs really random?", https://arxiv.org/abs/cond-mat/0104546

Callaway, D. S., Hopcroft, J. E., Kleinberg, J. M., Newman, M. E., & Strogatz, S. H. (2001).
Are randomly grown graphs really random?. Physical Review E, 64(4), 041902.

"""

if __name__ == "__main__":

    # probability of attaching two nodes
    delta = 1.0

    network = ntm.topology.disconnected(1)  # begin with a single-node network

    def topology_rule(ctx):
        num_nodes = len(ctx.network.nodes)
        new_label = num_nodes
        ctx.network.add_node(new_label)
        if random.random() < delta:
            # choose 2 nodes at random, without replacement
            choices = [int(i) for i in np.random.choice(list(ctx.network.nodes), size=2, replace=False)]
            ctx.network.add_edge(choices[1], choices[0])
            ctx.network.add_edge(choices[0], choices[1])

        return ctx.network

    trajectory = ntm.evolve(network=network, topology_rule=topology_rule, timesteps=200)

    # plot degree distribution
    ntm.plot_degree_distribution(trajectory[-1].network, out_degree=True,
                                 equation=lambda k: ((2*delta)**k)/((1 + 2*delta)**(k+1)),
                                 equation_text="$p_k = \\frac{(2\\delta)^k}{(1 + 2\\delta)^{k+1}}$")

    # animate time evolution of the network (NOTE: node self-links are not rendered)
    ntm.animate_network(trajectory, interval=350, layout="spring", with_labels=False)

max_stars_repo_name: lantunes/netomaton

max_issues_repo_name: lantunes/netomaton

max_forks_repo_name: lantunes/netomaton


text:
import json
import operator
from pathlib import Path

import matplotlib.pyplot as plt
import networkx as nx
import numpy as np

def graph_from_json(fp: Path, against_weight: int = 6, with_weight: int = 2, name: str = "HumanBody"):
    """Loads a graph from given filepath.

    Each row in `network` represents an edge and contains an array with 3 elements.
    The first two elements combined are the from and to vertices for the edge.
    The third element indicates if the edge is going with or against the stream.
    With is indicated by `0`. Against is indicated by `1`.
    """
    with open(fp) as f:
        network = np.array(json.load(f).get("network"), dtype=int)

    # Change all weights to corresponding energy cost
    network[:, 2] = np.where(network[:, 2] == 0, with_weight, against_weight)

    # Building and filling the graph
    graph = nx.MultiDiGraph(name=name)
    graph.add_weighted_edges_from(network.tolist())  # Instantiate all edges with corresponding weights

    return graph

def give_node_positions():
    """Returns the node positions."""
    pos = {14: (1, 0), 12: (1, 1), 13: (0, 1), 11: (2, 1), 10: (1, 2), 9: (2, 2), 8: (0, 3),
           7: (1, 3), 6: (2, 3), 5: (0, 4), 4: (1, 4), 3: (2, 4), 2: (1, 5), 1: (1, 6)}
    return pos

def node_positions_on_canvas(all_pos: dict, w_canvas: int = 600, h_canvas: int = 600):
    """Calculate the positions of the nodes based on the height and width of the canvas.
    """
    # get the max coordinates on the x and y axis
    all_coords = list(all_pos.values())
    x_max = max(all_coords, key=operator.itemgetter(0))[0]
    y_max = max(all_coords, key=operator.itemgetter(1))[1]

    # Calculate the multiplication value on both axis with an offset
    x_mult = w_canvas // (x_max + 1)
    y_mult = h_canvas // (y_max + 1)

    # Apply the new position
    new_pos = {}
    for pos in all_pos.items():
        new_pos[pos[0]] = {"x": (pos[1][0] + 1) * x_mult,
                           "y": h_canvas - (pos[1][1] + 1) * y_mult}
    return new_pos

def plot_graph(graph):
    """Plots the given graph.
    For each tuple in pos-dictionairy, tuple[0] is the x-value, tuple[1] is the y-value.
    """
    pos = give_node_positions()

    # Draw all vertices and edges.
    nx.draw_networkx_edges(graph, pos=pos)
    nx.draw_networkx_labels(graph, pos=pos)
    nx.draw_networkx_nodes(graph, pos, nodelist=[5, 8, 13], node_color="tab:blue", node_size=800)  # Deoxygenated blood vertices
    nx.draw_networkx_nodes(graph, pos, nodelist=[3, 6, 9, 11], node_color="tab:red", node_size=800)  # Oxygenated blood vertices
    nx.draw_networkx_nodes(graph, pos, nodelist=[1, 2, 4, 7, 10, 12, 14], node_color="tab:purple", node_size=800)  # Organ-vertices

    plt.title(graph.name)
    plt.axis("off")
    plt.show()

if __name__ == "__main__":
    graph = graph_from_json(Path("./loan/network.json"))
    plot_graph(graph)

max_stars_repo_name: Casper-Smet/LOAN

max_issues_repo_name: Casper-Smet/LOAN

max_forks_repo_name: Casper-Smet/LOAN

reason: import numpy,import networkx


text:
"""
Load Open Graph Benchmark data as GraphTensors and TensorFlow Datasets.

GraphTensors are created by sampling the neighbourhoods of each classification
node using NetworkX. Each node to be classified is stored as a separate
GraphTensor comprising of it and its n neighbourhoods. The number of neighbourhoods
to sample is a chosen parameter.
"""
import random
from dataclasses import dataclass
from typing import Any, Generator, Iterator, List, Sequence, Set, Tuple

import networkx as nx
import numpy as np
import tensorflow as tf
import tensorflow_gnn as tfgnn
from ogb.nodeproppred import NodePropPredDataset
from tqdm import tqdm

def ogb_as_networkx_graph(name: str) -> nx.Graph:
    """
    Load an Open Graph Benchmark dataset as a NetworkX graph.
    """
    dataset = NodePropPredDataset(name)
    splits = dataset.get_idx_split()
    ogb_graph, labels = dataset[0]

    num_nodes = ogb_graph["num_nodes"]
    ogb_features = ogb_graph["node_feat"]
    ogb_edgelist = ogb_graph["edge_index"]
    ogb_node_indices = np.arange(num_nodes)
    num_classes = labels.max() + 1

    graph = nx.from_edgelist(ogb_edgelist.T)
    data = zip(ogb_node_indices, ogb_features, labels)
    features = {
        node: {"features": features, "label": label}
        for node, features, label in data
    }
    nx.set_node_attributes(graph, values=features)
    return graph, splits, num_classes

@dataclass
class NodeSampler:
    graph: nx.Graph
    neighbour_samples: Tuple[int, ...] = (10, 2)

    @property
    def num_neighbourhoods(self) -> int:
        return len(self.neighbour_samples)

    def sample(self, seed_node: int) -> nx.Graph:
        sampled_nodes = {seed_node}
        to_sample = sampled_nodes.copy()
        for num_neighbours in self.neighbour_samples:

            for node in to_sample:
                neighbourhood_nodes = self.gather_neighbourhood(
                    node, exclude_nodes=sampled_nodes
                )
                if not neighbourhood_nodes:
                    continue
                sampled_neighbourhood_nodes = self.sample_neighbourhood(
                    neighbourhood_nodes, num_neighbours
                )
                sampled_nodes.update(sampled_neighbourhood_nodes)

            to_sample = sampled_neighbourhood_nodes.copy()

        return self.graph.subgraph(sampled_nodes)

    def gather_neighbourhood(
        self, node: int, exclude_nodes: Set[int]
    ) -> Set[int]:
        neighbourhood_nodes = set(self.graph.neighbors(node))
        neighbourhood_nodes = self.exclude_already_sampled_nodes(
            neighbourhood_nodes, exclude_nodes
        )
        return neighbourhood_nodes

    @staticmethod
    def exclude_already_sampled_nodes(
        neighbourhood_nodes: Set[int], sampled_nodes: Set[int]
    ) -> set[int]:
        return neighbourhood_nodes - sampled_nodes

    @staticmethod
    def sample_neighbourhood(nodes: Set[int], num_neighbours: int) -> Set[int]:
        return set(random.choices(list(nodes), k=num_neighbours))

def _prepare_data_for_node_classification(
    graph: nx.Graph, seed_node: int
) -> List[Tuple[Any, Any]]:
    """
    Position seed node as the first node in the data.

    TensorFlow GNN has a convention whereby the node to be classified, the "seed node",
    is positioned first in the component. This is for use with layers such as
    `tfgnn.keras.layers.ReadoutFirstNode` which extracts the first node from a component.
    """
    seed_data = graph.nodes(data=True)[seed_node]
    data = [(seed_data["features"], seed_data["label"])]
    data += [
        (data["features"], data["label"])
        for node, data in graph.nodes(data=True)
        if node != seed_node
    ]
    return data

def generate_graph_samples(
    graph: nx.Graph, seed_nodes: Sequence[int], sampler: NodeSampler
) -> Iterator[tfgnn.GraphTensor]:
    """
    Lazily samples subgraphs from a NetworkX graph and converts them to
    GraphTensors.

    In practice, this would be a preprocessing step that builds the subgraphs
    using a Apache Beam, constructs the GraphTensors and serialises them as
    tf.Examples.
    """
    for seed_node in seed_nodes:
        subgraph = sampler.sample(seed_node)
        subgraph = nx.convert_node_labels_to_integers(
            subgraph, label_attribute="graph_index"
        )
        subgraph_seed_node = next(
            node
            for node, data in subgraph.nodes(data=True)
            if data["graph_index"] == seed_node
        )

        num_edges = subgraph.number_of_edges()
        edge_list = np.asarray(subgraph.edges)
        edges = tfgnn.EdgeSet.from_fields(
            sizes=[num_edges],
            adjacency=tfgnn.Adjacency.from_indices(
                source=("paper", edge_list[:, 0]),
                target=("paper", edge_list[:, 1]),
            ),
        )

        data = _prepare_data_for_node_classification(
            subgraph, subgraph_seed_node
        )
        features, labels = zip(*data)
        num_nodes = subgraph.number_of_nodes()
        nodes = tfgnn.NodeSet.from_fields(
            features={
                "hidden_state": np.asarray(features),
                "label": np.asarray(labels),
            },
            sizes=[num_nodes],
        )

        context = tfgnn.Context.from_fields(features=None)

        graph_tensor = tfgnn.GraphTensor.from_pieces(
            edge_sets={"cites": edges},
            node_sets={"paper": nodes},
            context=context,
        )

        yield graph_tensor

def merge_graph_batches(graph: tfgnn.GraphTensor) -> tfgnn.GraphTensor:
    """TensorFlow GNN expects batches of graphs to be a single component."""
    return graph.merge_batch_to_components()

def load_dataset_from_graph(
    graph: nx.Graph,
    split: Sequence[int],
    sampler: NodeSampler,
    batch_size: int,
    graph_type_spec: tfgnn.GraphTensorSpec,
) -> tf.data.Dataset:
    """
    Load a TensorFlow Dataset sampled as ego subgraphs from a NetworkX graph.

    Only suitable for small graphs or very low neighbourhood sizes. Since the
    Dataset is small, we can cache the data in memory. For larger neighbourhood
    sizes, it's preferable to write examples to disk using `main` and loading
    with `load_dataset_from_examples`.
    """

    def generator() -> Generator[tfgnn.GraphTensor, None, None]:
        """tf.data.Dataset expects a Callable that returns a Generator."""
        samples = generate_graph_samples(graph, split, sampler=sampler)
        yield from samples

    dataset = tf.data.Dataset.from_generator(
        generator, output_signature=graph_type_spec
    )
    return (
        dataset.cache()
        .repeat()
        .batch(batch_size, drop_remainder=True)
        .map(merge_graph_batches)
        .prefetch(tf.data.AUTOTUNE)
    )

def load_dataset_from_examples(
    path: str, batch_size: int, type_spec: tfgnn.GraphTensorSpec
) -> tf.data.TFRecordDataset:
    return (
        tf.data.TFRecordDataset(path)
        .batch(batch_size, drop_remainder=True)
        .map(lambda example: tfgnn.parse_example(type_spec, example))
        .repeat()
        .map(merge_graph_batches)
        .prefetch(tf.data.AUTOTUNE)
    )

def write_graph_tensors_to_examples(
    graph_generator: Iterator[tfgnn.GraphTensor], path: str
) -> None:
    # TODO: Shard over multiple files.
    with tf.io.TFRecordWriter(path) as writer:
        for graph in graph_generator:
            example = tfgnn.write_example(graph)
            writer.write(example.SerializeToString())

def main() -> None:
    dataset = "ogbn-arxiv"
    neighbour_samples = (10, 2)

    graph, splits, _ = ogb_as_networkx_graph(dataset)
    train_split, val_split = splits["train"], splits["valid"]
    sampler = NodeSampler(graph, neighbour_samples)

    for split_name, split in (("train", train_split), ("val", val_split)):
        generator = generate_graph_samples(graph, split, sampler)
        filename = f"{split_name}_hop_{'-'.join(map(str, neighbour_samples))}.tfrecords"
        write_graph_tensors_to_examples(
            graph_generator=tqdm(generator, total=len(split)),
            path=f"data/{dataset}/{split_name}/{filename}",
        )

if __name__ == "__main__":
    main()

max_stars_repo_name: joshcarty/tfgnn-ogb

max_issues_repo_name: joshcarty/tfgnn-ogb

max_forks_repo_name: joshcarty/tfgnn-ogb


text:
import numpy as np

class PIDClamping:
    __slots__ = (
        "P",
        "I",
        "D",
        "kP",
        "kI",
        "kD",
        "output",
        "last_error",
        "last_time",
        "last_I",
        "delta_I",
        "max_output",
        "min_output",
        "dt",
    )

    def __init__(self, kP=0.0, kI=0.0, kD=0.0, max_output=np.inf, min_output=None):
        self.P = 0.0
        self.I = 0.0
        self.D = 0.0

        self.kP = kP
        self.kI = kI
        self.kD = kD

        self.last_I = 0.0
        self.delta_I = 0.0

        self.max_output = max_output
        if min_output is None:
            self.min_output = -max_output
        else:
            self.min_output = min_output

        self.output = 0.0
        self.last_error = 0.0
        self.last_time = None
        self.dt = 0

    def __update(self, current_error, t):
        """
        current_error: the difference between a desired setpoint SP and
            a measured process variable PV.
        t: the sampling timestamp in seconds.
        """
        self.P = current_error
        self.last_error = current_error
        if self.last_time is None:
            self.last_time = t
            return

        self.dt = t - self.last_time
        if self.dt <= 0.0:
            return

        self.last_I = self.I
        self.delta_I = current_error * self.dt
        self.I += self.delta_I
        self.D = (current_error - self.last_error) / self.dt
        self.last_time = t

    def __compute_output(self):
        u = self.kP * self.P + self.kI * self.I + self.kD * self.D
        if u > self.max_output or u < self.min_output or np.sign(u) == np.sign(self.P):
            self.I = self.last_I
            u -= self.kI * self.delta_I
        self.output = u

    def get_output(self, current_error, t):
        self.__update(current_error, t)
        self.__compute_output()
        return self.output

max_stars_repo_name: wlbksy/robotics

max_issues_repo_name: wlbksy/robotics

max_forks_repo_name: wlbksy/robotics


text:
import os
import imageio
import numpy as np

def quickdraw_npy_to_imagefile(inpath, outpath, filetype='png', subset=None):
    """
    Creates a folder with subfolders for each image class
    from the Quickdraw dataset (https://quickdraw.withgoogle.com)
    downloaded in .npy format.

    To download the .npy formatted dataset:
      gsutil -m cp gs://quickdraw_dataset/full/numpy_bitmap/*.npy quickdraw-png

    Usage example:
      quickdraw_npy_to_imagefile('quickdraw-npy', 'quickdraw-png')

    Parameters
    ----------

    inpath : str
        string specifying the path to the input directory containing
        the .npy files

    outpath : str
        string specifying the path for the output images

    subset : tuple or list (default=None)
        A subset of categories to consider. E.g.
        `("lollipop", "binoculars", "mouse", "basket")`

    """
    if not os.path.exists(outpath):
        os.mkdir(outpath)
    npy_list = [i for i in os.listdir(inpath) if i.endswith('.npy')]

    if subset:
        npy_list = [i for i in npy_list if i.split('.npy')[0] in subset]

    if not len(npy_list):
        raise ValueError('No .npy files found in %s' % inpath)

    npy_paths = [os.path.join(inpath, i) for i in npy_list]

    for i, j in zip(npy_list, npy_paths):

        label = (i.split('-')[-1]).split('.npy')[0]
        folder = os.path.join(outpath, label)
        if not os.path.exists(folder):
            os.mkdir(folder)
        X = np.load(j)

        cnt = 0
        for row in X:
            img_array = row.reshape(28, 28)
            assert cnt < 1000000
            outfile = os.path.join(folder, '%s_%06d.%s' % (
                label, cnt, filetype))
            imageio.imwrite(outfile,
                            img_array[:, :])
            cnt += 1

max_stars_repo_name: XiaoshengLin/deeplearning-models

max_issues_repo_name: tilaprimera/deeplearning-models

max_forks_repo_name: tilaprimera/deeplearning-models


text:
# Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Tests for Dequantize Operations."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow as tf

# TODO(petewarden) - Remove this ugly hack to get around Python linking problems
# with Bazel.
# pylint: disable=g-bad-import-order
from tensorflow.contrib.quantization import load_quantized_ops_so
from tensorflow.contrib.quantization.kernels import load_quantized_kernels_so

class DequantizeOpTest(tf.test.TestCase):

  def __init__(self, method_name="runTest"):
    super(DequantizeOpTest, self).__init__(method_name)
    load_quantized_ops_so.Load()
    load_quantized_kernels_so.Load()

  def _testDequantizeOp(self, inputs, min_range, max_range, dtype):
    with self.test_session():
      input_op = tf.constant(inputs, shape=[len(inputs)], dtype=dtype)
      dequantized = tf.contrib.quantization.dequantize(
          input_op, min_range, max_range)
      tf_ans = dequantized.eval()

    # TODO(vrv): Add support for DT_QINT32 quantization if needed.
    type_dict = {
        tf.quint8: np.uint8,
        tf.qint8: np.int8,
        tf.quint16: np.uint16,
        tf.qint16: np.int16
        }
    self.assertTrue(dtype in type_dict.keys())
    v_max = np.iinfo(type_dict[dtype]).max
    v_min = np.iinfo(type_dict[dtype]).min
    self.assertTrue(min_range >= v_min)
    self.assertTrue(max_range <= v_max)
    type_range = v_max - v_min
    if v_min < 0:
      half_range = (type_range + 1) / 2
    else:
      half_range = 0.0

    np_ans = ((inputs.astype(np.float32) + half_range) *
              (max_range - min_range) / type_range) + min_range
    self.assertAllClose(tf_ans, np_ans)

  def testBasicQuint8(self):
    self._testDequantizeOp(np.array([0, 128, 255]),
                           0.0, 6.0, tf.quint8)
    self._testDequantizeOp(np.array([0, 128, 255]),
                           0.0, 123.456, tf.quint8)
    self._testDequantizeOp(np.array([0, 4, 42, 108, 243]),
                           5.0, 200.2, tf.quint8)

  def testBasicQint8(self):
    self._testDequantizeOp(np.array([-128, 0, 127]),
                           -1.0, 2.0, tf.qint8)
    self._testDequantizeOp(np.array([-2, 4, -17]),
                           -5.0, -3.0, tf.qint8)
    self._testDequantizeOp(np.array([0, -4, 42, -108]),
                           5.0, 40.0, tf.qint8)

if __name__ == "__main__":
  tf.test.main()


text:
import re
import numpy
from scipy import stats
import json

class RLStatsCompute:
    def __init__(self, files_list, confidence = 0.95):
        self.data, self.extended = self.load_files(files_list)
        self.mean, self.std, self.lower, self.upper, self.hist = self.compute_stats(self.data, confidence)

    def load_files(self, files_list):
        data      = []
        extended  = []

        for f in files_list:
            print("loading ", f)
            data_ = numpy.loadtxt(f, unpack = True, comments='{')
            data.append(data_)

            extended_f = []

            with open(f) as file:
                lines = file.readlines()

                for line in lines:
                    tmp = "{" + line.split('{')[1]
                    if len(tmp) > 0:
                        try:
                            tmp = tmp.replace("'", "\"")
                            tmp = json.loads(tmp)
                            extended_f.append(tmp)
                        except Exception:
                            pass

            extended.append(extended_f)

        data      = numpy.array(data)

        return data, extended

    def compute_stats(self, data, confidence = 0.95):
        n       = data.shape[2]

        mean    = numpy.mean(data, axis = 0)
        std     = numpy.std(data, axis = 0)
        se      = stats.sem(data, axis=0)
        h       = se * stats.t.ppf((1 + confidence) / 2., n-1)

        lower = mean - h
        upper = mean + h

        hist = []

        for col in range(data.shape[1]):
            h, e = numpy.histogram(data[0][col], bins=64)

            e    = e[0:-1]
            h    = h/numpy.sum(h)

            hist.append([e, h])

        hist = numpy.array(hist)

        return mean, std, lower, upper, hist

reason: import numpy,from scipy


text:
"""
Firn depth models
Author: Cyril Grima <cyril.grima@gmail.com>
"""

import pandas as pd
from numpy.polynomial.polynomial import polyval
import scipy.constants as ct
import numpy as np

def density_kovacs82(d0, z):
    """Convert depth into firn density from Kovacs [1982]
    measurements over the brine of McMurdo Ice Shelf

    Arguments
    ---------
    d0 : float
        first polynom equals the surface density [kg^m{-3}]
    z : array of floats
        Depth [m]
    """

    p = {'B': [d0, 2.79e1, 0, 0, 0, 0, 0],
         'C': [d0, 2.21e1, -3.09e-1, 0, 0, 0, 0],
         'D': [d0, 2.38e1, -5.11e-1, 4.62e-3, 0, 0, 0],
         'E': [d0, 3.36e1, -1.89e00, 7.83e-2, -1.85e-3, 2.24e-5, -1.07e-7],
         'F': [d0, 3.57e1, -2.15e00, 8.77e-2, -1.94e-3, 2.15e-5, -9.54e-8]}
    p = pd.DataFrame(p)

    # Density caclcuation
    dns = {'B': z*0, 'C': z*0, 'D': z*0, 'E': z*0, 'F': z*0}
    dns = pd.DataFrame(dns)

    for i, val in enumerate(z):
        dns.ix[i] = polyval(val,p.ix[:])
    dns[dns > 917] = 917.
    return dns

def density_sorgelaw(d0, z, zt):
    """Convert depth into firn density from Sorge's law
    as reported by [Cuffey and Patterson, 2010, eq.2.2]

    Arguments
    ---------
    d0 : float
        surface density [kg^m{-3}]
    z : array of floats
        Depth [m]
    zp : float
        characteristic depth of firn
    """
    di = 917. # Density of ice
    return di-(di-d0)*np.exp(-1.9*z/zt)

reason: import numpy,from numpy,import scipy


text:
#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates.

import os
from os.path import join as pjoin
from typing import Dict, Tuple
import numpy as np
from . import load_colmap, image_io as tr
from .geometry_np import reproject, project, sample

def store_visible_points_per_image(
    points3D: Dict[int, load_colmap.Point3D]
) -> Dict[int, np.ndarray]:
    """
    returns dictionary where key is the
        image id: int
    and the value is
        3D points (3, N) that are visible in each image
    (Note: currently images do not contain this info, but a list of -1's)
    """
    map_img_to_pt3D = {}
    for _cur_key, cur_point in points3D.items():
        # assert(cur_key == cur_point) # They are the same by design
        for img_id in cur_point.image_ids:
            if img_id in map_img_to_pt3D:
                map_img_to_pt3D[img_id].append(cur_point.xyz)
            else:
                map_img_to_pt3D[img_id] = [cur_point.xyz]

    for img_id, pt_list in map_img_to_pt3D.items():
        map_img_to_pt3D[img_id] = load_colmap.convert_points3D(np.array(pt_list).T)

    return map_img_to_pt3D

def vote_scale(
    scales: np.ndarray, min_percentile_thresh: int = 10, max_percentile_thresh: int = 90
) -> float:
    """
    Note if len(scales) is really small, e.g., len(scales) == 2, it will return nan
    """
    m = np.percentile(scales, min_percentile_thresh)
    M = np.percentile(scales, max_percentile_thresh)
    ix = np.logical_and(m <= scales, scales <= M)
    scale = np.mean(scales[ix])
    return scale

def calibrate_frame_w_sparse_points(
    pts3d: np.ndarray, intr: np.ndarray, extr: np.ndarray, inv_depth: np.ndarray
) -> float:
    """
    Args:
        pts3d   (3, N)
        intr    (4)
        extr    (3, 4)
        depth   (H, W)

    Returns:
        scale: depth * scale = -pts_in_local_camera_coordinate.z
    """
    # points 3d in local camera coordinate
    # FIXME: deal with the case when a point is behind the camera
    pts3d_cam = reproject(pts3d, extr)
    pts2d = project(pts3d_cam, intr)
    inv_depths, ix = sample(inv_depth, pts2d)
    ds = -pts3d[-1, :][ix]  # Note negative sign
    scales = ds * inv_depths
    return vote_scale(scales)

def calibrate_w_sparse_colmap(
    colmap_dir: str, dense_depth_dir: str
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
    """
    Args:
        colmap_dir: sparse colmap directory containing
            cameras.bin/txt, points3D.bin/txt, images.bin/txt
        dense_depth_dir: folder name for dense depth directory
        scales_fn (optional): dump per frame scale

    Returns:
        Calibrated intrinsics and extrinsics
        intrinsics  (N, 4)
        extrinsics  (N, 3, 4)
        scales      (N)
    """
    cameras, images, points3D = load_colmap.read_model(path=colmap_dir, ext=".bin")
    # compute intrinsics, extrinsics
    depth_names = [
        x for x in os.listdir(dense_depth_dir) if os.path.splitext(x)[-1] == ".raw"
    ]
    size = tr.load_raw_float32_image(pjoin(dense_depth_dir, depth_names[0])).shape[:2][
        ::-1
    ]
    intrinsics, extrinsics = load_colmap.convert_calibration(cameras, images, size)

    # TODO: make the following up to compute the scale a single function
    map_img_to_pt3D = store_visible_points_per_image(points3D)
    ordered_im_ids = load_colmap.ordered_image_ids(images)
    scales = np.empty(intrinsics.shape[0])
    for i, im_id in enumerate(ordered_im_ids):
        if im_id not in map_img_to_pt3D:
            scales[i] = np.nan
            print('[WARNING] %s does not have visible feature point' % images[im_id].name)
        im_name = images[im_id].name
        depth_fn = pjoin(dense_depth_dir, os.path.splitext(im_name)[0] + ".raw")
        inv_depth = tr.load_raw_float32_image(depth_fn)
        pts3D = map_img_to_pt3D[im_id]
        scale = calibrate_frame_w_sparse_points(
            pts3D, intrinsics[i], extrinsics[i], inv_depth
        )
        scales[i] = scale

    mean_scale = scales[~np.isnan(scales)].mean()
    extrinsics[..., -1] /= mean_scale
    return intrinsics, extrinsics, scales