# Author: Raphael Vallat <raphaelvallat9@gmail.com>
# Date: July 2018
# Python code inspired from the CircStats MATLAB toolbox (Berens 2009)
# and the brainpipe Python package.
# Reference:
# Berens, Philipp. 2009. CircStat: A MATLAB Toolbox for Circular Statistics.
# Journal of Statistical Software, Articles 31 (10): 1–21.
import numpy as np
from scipy.stats import norm

from .utils import remove_na

__all__ = [
    "convert_angles",
    "circ_axial",
    "circ_mean",
    "circ_r",
    "circ_corrcc",
    "circ_corrcl",
    "circ_rayleigh",
    "circ_vtest",
]

###############################################################################
# HELPER FUNCTIONS
###############################################################################

def _checkangles(angles, axis=None):
    """Internal function to check that angles are in radians."""
    msg = (
        "Angles are not in unit of radians. Please use the "
        "`pingouin.convert_angles` function to map your angles to "
        "the [-pi, pi] range."
    )
    ptp_rad = np.nanmax(angles, axis=axis) - np.nanmin(angles, axis=axis)
    ptp_mask = ptp_rad <= 2 * np.pi
    if not ptp_mask.all():
        raise ValueError(msg)

def convert_angles(angles, low=0, high=360, positive=False):
    """Element-wise conversion of arbitrary-unit circular quantities
    to radians.

    .. versionadded:: 0.3.4

    Parameters
    ----------
    angles : array_like
        Circular data.
    low : float or int, optional
        Low boundary for ``angles`` range.  Default is 0.
    high : float or int, optional
        High boundary for ``angles`` range.  Default is 360
        (for degrees to radians conversion).
    positive : boolean
        If True, radians are mapped on the :math:`[0, 2\\pi]`. Otherwise,
        the resulting angles are mapped from :math:`[-\\pi, \\pi)` (default).

    Returns
    -------
    radians : array_like
        Circular data in radians.

    Notes
    -----
    The formula to convert a set of angles :math:`\\alpha` from an arbitrary
    range :math:`[\\text{high},\\text{low}]` to radians
    :math:`[0, 2\\pi]` is:

    .. math::

        \\alpha_r = \\frac{2\\pi\\alpha}{\\text{high} - \\text{low}}

    If ``positive=False`` (default), the resulting angles in
    radians :math:`\\alpha_r` are then wrapped to the :math:`[-\\pi, \\pi)`
    range:

    .. math::

        (\\text{angle} + \\pi) \\mod 2 \\pi - \\pi

    Examples
    --------
    1. Convert degrees to radians

    >>> from pingouin import convert_angles
    >>> a = [0, 360, 180, 90, 45, 270]
    >>> convert_angles(a, low=0, high=360)
    array([ 0.        ,  0.        , -3.14159265,  1.57079633,  0.78539816,
           -1.57079633])

    with ``positive=True``:

    >>> convert_angles(a, low=0, high=360, positive=True)
    array([0.        , 6.28318531, 3.14159265, 1.57079633, 0.78539816,
           4.71238898])

    2. Convert hours (24h-format) to radians

    >>> sleep_onset = [22.5, 23.25, 24, 0.5, 1]
    >>> convert_angles(sleep_onset, low=0, high=24)
    array([-0.39269908, -0.19634954,  0.        ,  0.13089969,  0.26179939])

    3. Convert radians from :math:`[0, 2\\pi]` to :math:`[-\\pi, \\pi)`:

    >>> import numpy as np
    >>> rad = [0.1, 3.14, 5, 2, 6]
    >>> convert_angles(rad, low=0, high=2*np.pi)
    array([ 0.1       ,  3.14      , -1.28318531,  2.        , -0.28318531])

    4. Convert degrees from a 2-D array

    >>> np.random.seed(123)
    >>> deg = np.random.randint(low=0, high=360, size=(3, 4))
    >>> convert_angles(deg)
    array([[-0.66322512,  1.71042267, -2.26892803,  0.29670597],
           [ 1.44862328,  1.85004901,  2.14675498,  0.99483767],
           [-2.54818071, -2.35619449,  1.67551608,  1.97222205]])
    """
    assert isinstance(positive, bool)
    assert isinstance(high, (int, float)), "high must be numeric"
    assert isinstance(low, (int, float)), "low must be numeric"
    ptp = high - low
    assert ptp > 0, "high - low must be strictly positive."
    angles = np.asarray(angles)
    assert np.nanmin(angles) >= low, "angles cannot be >= low."
    assert np.nanmax(angles) <= high, "angles cannot be <= high."
    # Map to [0, 2pi] range
    rad = angles * (2 * np.pi) / ptp
    if not positive:
        # https://stackoverflow.com/a/29237626/10581531
        # Map to [-pi, pi) range:
        rad = (rad + np.pi) % (2 * np.pi) - np.pi  # [-pi, pi)
        # Map to (-pi, pi] range:
        # rad = np.angle(np.exp(1j * rad))
        # rad = -1 * ((-rad + np.pi) % (2 * np.pi) - np.pi)
    return rad

def circ_axial(angles, n):
    """Transforms n-axial data to a common scale.

    Parameters
    ----------
    angles : array
        Sample of angles in radians
    n : int
        Number of modes

    Returns
    -------
    angles : float
        Transformed angles

    Notes
    -----
    Tranform data with multiple modes (known as axial data) to a unimodal
    sample, for the purpose of certain analysis such as computation of a
    mean resultant vector (see Berens 2009).

    Examples
    --------
    Transform degrees to unimodal radians in the Berens 2009 neuro dataset.

    >>> import numpy as np
    >>> from pingouin import read_dataset
    >>> from pingouin.circular import circ_axial
    >>> df = read_dataset('circular')
    >>> angles = df['Orientation'].to_numpy()
    >>> angles = circ_axial(np.deg2rad(angles), 2)
    """
    angles = np.asarray(angles)
    return np.remainder(angles * n, 2 * np.pi)

###############################################################################
# DESCRIPTIVE STATISTICS
###############################################################################

def circ_mean(angles, w=None, axis=0):
    """Mean direction for (binned) circular data.

    Parameters
    ----------
    angles : array_like
        Samples of angles in radians. The range of ``angles`` must be either
        :math:`[0, 2\\pi]` or :math:`[-\\pi, \\pi]`. If ``angles`` is not
        expressed in radians (e.g. degrees or 24-hours), please use the
        :py:func:`pingouin.convert_angles` function prior to using the present
        function.
    w : array_like
        Number of incidences per bins (i.e. "weights"), in case of binned angle
        data.
    axis : int or None
        Compute along this dimension. Default is the first axis (0).

    Returns
    -------
    mu : float
        Circular mean, in radians.

    See also
    --------
    scipy.stats.circmean, scipy.stats.circstd, pingouin.circ_r

    Notes
    -----
    From Wikipedia:

    *In mathematics, a mean of circular quantities is a mean which is sometimes
    better-suited for quantities like angles, daytimes, and fractional parts
    of real numbers. This is necessary since most of the usual means may not be
    appropriate on circular quantities. For example, the arithmetic mean of 0°
    and 360° is 180°, which is misleading because for most purposes 360° is
    the same thing as 0°.
    As another example, the "average time" between 11 PM and 1 AM is either
    midnight or noon, depending on whether the two times are part of a single
    night or part of a single calendar day.*

    The circular mean of a set of angles :math:`\\alpha` is defined by:

    .. math::

        \\bar{\\alpha} =  \\text{angle} \\left ( \\sum_{j=1}^n \\exp(i \\cdot
        \\alpha_j) \\right )

    For binned angles with weights :math:`w`, this becomes:

    .. math::

        \\bar{\\alpha} =  \\text{angle} \\left ( \\sum_{j=1}^n w \\cdot
        \\exp(i \\cdot \\alpha_j) \\right )

    Missing values in ``angles`` are omitted from the calculations.

    References
    ----------
    * https://en.wikipedia.org/wiki/Mean_of_circular_quantities

    * Berens, P. (2009). CircStat: A MATLAB Toolbox for Circular
      Statistics. Journal of Statistical Software, Articles, 31(10),
      1–21. https://doi.org/10.18637/jss.v031.i10

    Examples
    --------
    1. Circular mean of a 1-D array of angles, in radians

    >>> import pingouin as pg
    >>> angles = [0.785, 1.570, 3.141, 0.839, 5.934]
    >>> round(pg.circ_mean(angles), 4)
    1.013

    Compare with SciPy:

    >>> from scipy.stats import circmean
    >>> import numpy as np
    >>> round(circmean(angles, low=0, high=2*np.pi), 4)
    1.013

    2. Using a 2-D array of angles in degrees

    >>> np.random.seed(123)
    >>> deg = np.random.randint(low=0, high=360, size=(3, 5))
    >>> deg
    array([[322,  98, 230,  17,  83],
           [106, 123,  57, 214, 225],
           [ 96, 113, 126,  47,  73]])

    We first need to convert from degrees to radians:

    >>> rad = np.round(pg.convert_angles(deg, low=0, high=360), 4)
    >>> rad
    array([[-0.6632,  1.7104, -2.2689,  0.2967,  1.4486],
           [ 1.85  ,  2.1468,  0.9948, -2.5482, -2.3562],
           [ 1.6755,  1.9722,  2.1991,  0.8203,  1.2741]])

    >>> pg.circ_mean(rad)  # On the first axis (default)
    array([1.27532162, 1.94336576, 2.23195927, 0.52110503, 1.80240563])
    >>> pg.circ_mean(rad, axis=-1)  # On the last axis (default)
    array([0.68920819, 2.49334852, 1.5954149 ])
    >>> round(pg.circ_mean(rad, axis=None), 4)  # Across the entire array
    1.6954

    Missing values are omitted from the calculations:

    >>> rad[0, 0] = np.nan
    >>> pg.circ_mean(rad)
    array([1.76275   , 1.94336576, 2.23195927, 0.52110503, 1.80240563])

    3. Using binned angles

    >>> np.random.seed(123)
    >>> nbins = 18  # Number of bins to divide the unit circle
    >>> angles_bins = np.linspace(0, 2 * np.pi, nbins)
    >>> # w represents the number of incidences per bins, or "weights".
    >>> w = np.random.randint(low=0, high=5, size=angles_bins.size)
    >>> round(pg.circ_mean(angles_bins, w), 4)
    0.606
    """
    angles = np.asarray(angles)
    _checkangles(angles)  # Check that angles is in radians
    w = np.asarray(w) if w is not None else np.ones(angles.shape)
    assert angles.shape == w.shape, "Input dimensions do not match"
    return np.angle(np.nansum(np.multiply(w, np.exp(1j * angles)), axis=axis))

def circ_r(angles, w=None, d=None, axis=0):
    """Mean resultant vector length for circular data.

    Parameters
    ----------
    angles : array_like
        Samples of angles in radians. The range of ``angles`` must be either
        :math:`[0, 2\\pi]` or :math:`[-\\pi, \\pi]`. If ``angles`` is not
        expressed in radians (e.g. degrees or 24-hours), please use the
        :py:func:`pingouin.convert_angles` function prior to using the present
        function.
    w : array_like
        Number of incidences per bins (i.e. "weights"), in case of binned angle
        data.
    d : float
        Spacing (in radians) of bin centers for binned data. If supplied,
        a correction factor is used to correct for bias in the estimation
        of r.
    axis : int or None
        Compute along this dimension. Default is the first axis (0).

    Returns
    -------
    r : float
        Circular mean vector length.

    See also
    --------
    pingouin.circ_mean

    Notes
    -----
    The length of the mean resultant vector is a crucial quantity for the
    measurement of circular spread or hypothesis testing in directional
    statistics. The closer it is to one, the more concentrated the data
    sample is around the mean direction (Berens 2009).

    The circular vector length of a set of angles :math:`\\alpha` is defined
    by:

    .. math::

        \\bar{\\alpha} =  \\frac{1}{N}\\left \\| \\sum_{j=1}^n
        \\exp(i \\cdot \\alpha_j) \\right \\|

    Missing values in ``angles`` are omitted from the calculations.

    References
    ----------
    * https://en.wikipedia.org/wiki/Mean_of_circular_quantities

    * Berens, P. (2009). CircStat: A MATLAB Toolbox for Circular
      Statistics. Journal of Statistical Software, Articles, 31(10),
      1–21. https://doi.org/10.18637/jss.v031.i10

    Examples
    --------
    1. Mean resultant vector length of a 1-D array of angles, in radians

    >>> import pingouin as pg
    >>> angles = [0.785, 1.570, 3.141, 0.839, 5.934]
    >>> r = pg.circ_r(angles)
    >>> round(r, 4)
    0.4972

    Note that there is a close relationship between the vector length and the
    circular standard deviation, i.e. :math:`\\sigma = \\sqrt{-2 \\ln R}`:

    >>> import numpy as np
    >>> round(np.sqrt(-2 * np.log(r)), 4)
    1.1821

    which gives similar result as SciPy built-in function:

    >>> from scipy.stats import circstd
    >>> round(circstd(angles), 4)
    1.1821

    Sanity check: if all angles are the same, the vector length should be one:

    >>> angles = [3.14, 3.14, 3.14, 3.14]
    >>> round(pg.circ_r(angles), 4)
    1.0

    2. Using a 2-D array of angles in degrees

    >>> np.random.seed(123)
    >>> deg = np.random.randint(low=0, high=360, size=(3, 5))
    >>> deg
    array([[322,  98, 230,  17,  83],
           [106, 123,  57, 214, 225],
           [ 96, 113, 126,  47,  73]])

    We first need to convert from degrees to radians:

    >>> rad = np.round(pg.convert_angles(deg, low=0, high=360), 4)
    >>> rad
    array([[-0.6632,  1.7104, -2.2689,  0.2967,  1.4486],
           [ 1.85  ,  2.1468,  0.9948, -2.5482, -2.3562],
           [ 1.6755,  1.9722,  2.1991,  0.8203,  1.2741]])

    >>> pg.circ_r(rad)  # On the first axis (default)
    array([0.46695499, 0.98398294, 0.3723287 , 0.31103746, 0.42527149])
    >>> pg.circ_r(rad, axis=-1)  # On the last axis (default)
    array([0.28099998, 0.45456096, 0.88261161])
    >>> round(pg.circ_r(rad, axis=None), 4)  # Across the entire array
    0.4486

    Missing values are omitted from the calculations:

    >>> rad[0, 0] = np.nan
    >>> pg.circ_r(rad)
    array([0.99619613, 0.98398294, 0.3723287 , 0.31103746, 0.42527149])

    3. Using binned angles

    >>> np.random.seed(123)
    >>> nbins = 18  # Number of bins to divide the unit circle
    >>> angles_bins = np.linspace(0, 2 * np.pi, nbins)
    >>> # w represents the number of incidences per bins, or "weights".
    >>> w = np.random.randint(low=0, high=5, size=angles_bins.size)
    >>> round(pg.circ_r(angles_bins, w), 4)
    0.3642
    """
    angles = np.asarray(angles)
    _checkangles(angles)  # Check that angles is in radians
    w = np.asarray(w) if w is not None else np.ones(angles.shape)
    assert angles.shape == w.shape, "Input dimensions do not match."

    # Add np.nan in weight vector (otherwise nansum(w) is wrong)
    w = w.astype(float)
    w[np.isnan(angles)] = np.nan

    # Compute weighted sum of cos and sin of angles:
    r = np.nansum(np.multiply(w, np.exp(1j * angles)), axis=axis)

    # Calculate vector length:
    r = np.abs(r) / np.nansum(w, axis=axis)

    # For data with known spacing, apply correction factor
    if d is not None:
        c = d / 2 / np.sin(d / 2)
        r = c * r
    return r

###############################################################################
# INFERENTIAL STATISTICS
###############################################################################

def circ_corrcc(x, y, correction_uniform=False):
    """Correlation coefficient between two circular variables.

    Parameters
    ----------
    x : 1-D array_like
        First circular variable (expressed in radians).
    y : 1-D array_like
        Second circular variable (expressed in radians).
    correction_uniform : bool
        Use correction for uniform marginals.

    Returns
    -------
    r : float
        Correlation coefficient.
    pval : float
        Uncorrected p-value.

    Notes
    -----
    Adapted from the CircStats MATLAB toolbox [1]_.

    The range of ``x`` and ``y`` must be either
    :math:`[0, 2\\pi]` or :math:`[-\\pi, \\pi]`. If ``angles`` is not
    expressed in radians (e.g. degrees or 24-hours), please use the
    :py:func:`pingouin.convert_angles` function prior to using the present
    function.

    Please note that NaN are automatically removed.

    If the ``correction_uniform`` is True, an alternative equation from
    [2]_ (p. 177) is used. If the marginal distribution of ``x`` or ``y`` is
    uniform, the mean is not well defined, which leads to wrong estimates
    of the circular correlation. The alternative equation corrects for this
    by choosing the means in a way that maximizes the positive or negative
    correlation.

    References
    ----------
    .. [1] Berens, P. (2009). CircStat: A MATLAB Toolbox for Circular
           Statistics. Journal of Statistical Software, Articles, 31(10), 1–21.
           https://doi.org/10.18637/jss.v031.i10

    .. [2] Jammalamadaka, S. R., & Sengupta, A. (2001). Topics in circular
           statistics (Vol. 5). world scientific.

    Examples
    --------
    Compute the r and p-value of two circular variables

    >>> from pingouin import circ_corrcc
    >>> x = [0.785, 1.570, 3.141, 3.839, 5.934]
    >>> y = [0.593, 1.291, 2.879, 3.892, 6.108]
    >>> r, pval = circ_corrcc(x, y)
    >>> print(round(r, 3), round(pval, 4))
    0.942 0.0658

    With the correction for uniform marginals

    >>> r, pval = circ_corrcc(x, y, correction_uniform=True)
    >>> print(round(r, 3), round(pval, 4))
    0.547 0.2859
    """
    x = np.asarray(x)
    y = np.asarray(y)
    assert x.size == y.size, "x and y must have the same length."

    # Remove NA
    x, y = remove_na(x, y, paired=True)
    n = x.size

    # Compute correlation coefficient
    x_sin = np.sin(x - circ_mean(x))
    y_sin = np.sin(y - circ_mean(y))

    if not correction_uniform:
        # Similar to np.corrcoef(x_sin, y_sin)[0][1]
        r = np.sum(x_sin * y_sin) / np.sqrt(np.sum(x_sin**2) * np.sum(y_sin**2))
    else:
        r_minus = np.abs(np.sum(np.exp((x - y) * 1j)))
        r_plus = np.abs(np.sum(np.exp((x + y) * 1j)))
        denom = 2 * np.sqrt(np.sum(x_sin**2) * np.sum(y_sin**2))
        r = (r_minus - r_plus) / denom

    # Compute T- and p-values
    tval = (
        np.sqrt((n * (x_sin**2).mean() * (y_sin**2).mean()) / np.mean(x_sin**2 * y_sin**2))
        * r
    )

    # Approximately distributed as a standard normal
    pval = 2 * norm.sf(abs(tval))
    return r, pval

def circ_corrcl(x, y):
    """Correlation coefficient between one circular and one linear variable
    random variables.

    Parameters
    ----------
    x : 1-D array_like
        First circular variable (expressed in radians).
        The range of ``x`` must be either :math:`[0, 2\\pi]` or
        :math:`[-\\pi, \\pi]`. If ``angles`` is not
        expressed in radians (e.g. degrees or 24-hours), please use the
        :py:func:`pingouin.convert_angles` function prior to using the present
        function.
    y : 1-D array_like
        Second circular variable (linear)

    Returns
    -------
    r : float
        Correlation coefficient
    pval : float
        Uncorrected p-value

    Notes
    -----
    Please note that NaN are automatically removed from datasets.

    Examples
    --------
    Compute the r and p-value between one circular and one linear variables.

    >>> from pingouin import circ_corrcl
    >>> x = [0.785, 1.570, 3.141, 0.839, 5.934]
    >>> y = [1.593, 1.291, -0.248, -2.892, 0.102]
    >>> r, pval = circ_corrcl(x, y)
    >>> print(round(r, 3), round(pval, 3))
    0.109 0.971
    """
    from scipy.stats import pearsonr, chi2

    x = np.asarray(x)
    y = np.asarray(y)
    assert x.size == y.size, "x and y must have the same length."

    # Remove NA
    x, y = remove_na(x, y, paired=True)
    n = x.size

    # Compute correlation coefficent for sin and cos independently
    rxs = pearsonr(y, np.sin(x))[0]
    rxc = pearsonr(y, np.cos(x))[0]
    rcs = pearsonr(np.sin(x), np.cos(x))[0]

    # Compute angular-linear correlation (equ. 27.47)
    r = np.sqrt((rxc**2 + rxs**2 - 2 * rxc * rxs * rcs) / (1 - rcs**2))

    # Compute p-value
    pval = chi2.sf(n * r**2, 2)
    return r, pval

def circ_rayleigh(angles, w=None, d=None):
    """Rayleigh test for non-uniformity of circular data.

    Parameters
    ----------
    angles : 1-D array_like
        Samples of angles in radians. The range of ``angles`` must be either
        :math:`[0, 2\\pi]` or :math:`[-\\pi, \\pi]`. If ``angles`` is not
        expressed in radians (e.g. degrees or 24-hours), please use the
        :py:func:`pingouin.convert_angles` function prior to using the present
        function.
    w : array_like
        Number of incidences per bins (i.e. "weights"), in case of binned angle
        data.
    d : float
        Spacing (in radians) of bin centers for binned data. If supplied,
        a correction factor is used to correct for bias in the estimation
        of r.

    Returns
    -------
    z : float
        Z-statistic
    pval : float
        P-value

    Notes
    -----
    The Rayleigh test asks how large the resultant vector length R must be
    to indicate a non-uniform  distribution (Fisher 1995).

    H0: the population is uniformly distributed around the circle
    HA: the populatoin is not distributed uniformly around the circle

    The assumptions for the Rayleigh test are that (1) the distribution has
    only one mode and (2) the data is sampled from a von Mises distribution.

    Examples
    --------
    1. Simple Rayleigh test for non-uniformity of circular data.

    >>> from pingouin import circ_rayleigh
    >>> x = [0.785, 1.570, 3.141, 0.839, 5.934]
    >>> z, pval = circ_rayleigh(x)
    >>> print(round(z, 3), round(pval, 6))
    1.236 0.304844

    2. Specifying w and d

    >>> z, pval = circ_rayleigh(x, w=[.1, .2, .3, .4, .5], d=0.2)
    >>> print(round(z, 3), round(pval, 6))
    0.278 0.806997
    """
    angles = np.asarray(angles)
    _checkangles(angles)  # Check that angles is in radians
    if w is None:
        r = circ_r(angles)
        n = len(angles)
    else:
        assert len(angles) == len(w), "Input dimensions do not match"
        r = circ_r(angles, w, d)
        n = np.sum(w)

    # Compute Rayleigh's statistic
    R = n * r
    z = (R**2) / n

    # Compute p value using approxation in Zar (1999), p. 617
    pval = np.exp(np.sqrt(1 + 4 * n + 4 * (n**2 - R**2)) - (1 + 2 * n))
    return z, pval

def circ_vtest(angles, dir=0.0, w=None, d=None):
    """V test for non-uniformity of circular data with a specified
    mean direction.

    Parameters
    ----------
    angles : 1-D array_like
        Samples of angles in radians. The range of ``angles`` must be either
        :math:`[0, 2\\pi]` or :math:`[-\\pi, \\pi]`. If ``angles`` is not
        expressed in radians (e.g. degrees or 24-hours), please use the
        :py:func:`pingouin.convert_angles` function prior to using the present
        function.
    dir : float
        Suspected mean direction (angle in radians).
    w : array_like
        Number of incidences per bins (i.e. "weights"), in case of binned angle
        data.
    d : float
        Spacing (in radians) of bin centers for binned data. If supplied,
        a correction factor is used to correct for bias in the estimation
        of r.

    Returns
    -------
    V : float
        V-statistic
    pval : float
        P-value

    Notes
    -----
    H0: the population is uniformly distributed around the circle.
    HA: the population is not distributed uniformly around the circle but
    has a mean of dir.

    Note: Not rejecting H0 may mean that the population is uniformly
    distributed around the circle OR that it has a mode but that this mode
    is not centered at dir.

    The V test has more power than the Rayleigh test and is preferred if
    there is reason to believe in a specific mean direction.

    Adapted from the Matlab Circular Statistics Toolbox.

    Examples
    --------
    1. V-test for non-uniformity of circular data.

    >>> from pingouin import circ_vtest
    >>> x = [0.785, 1.570, 3.141, 0.839, 5.934]
    >>> v, pval = circ_vtest(x, dir=1)
    >>> print(round(v, 3), pval)
    2.486 0.05794648732225438

    2. Specifying w and d

    >>> v, pval = circ_vtest(x, dir=0.5, w=[.1, .2, .3, .4, .5], d=0.2)
    >>> print(round(v, 3), round(pval, 5))
    0.637 0.23086
    """
    angles = np.asarray(angles)
    if w is None:
        r = circ_r(angles)
        mu = circ_mean(angles)
        n = len(angles)
    else:
        assert len(angles) == len(w), "Input dimensions do not match"
        r = circ_r(angles, w, d)
        mu = circ_mean(angles, w)
        n = np.sum(w)

    # Compute Rayleigh and V statistics
    R = n * r
    v = R * np.cos(mu - dir)

    # Compute p value
    u = v * np.sqrt(2 / n)
    pval = 1 - norm.cdf(u)
    return v, pval


"""project URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/4.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from django.conf import settings
from django.conf.urls.static import static
from AppMaquinas.views import (index, MaquinaList,
Login, Logout, SignUp, ProfileCreate, ProfileUpdate,
MaquinaMineList, MaquinaUpdate, MaquinaDetail, MaquinaDelete,
MaquinaCreate, MaquinaSearch, MensajeCreate, MensajeDelete, MensajeList,
about
)

urlpatterns = [
    path('admin/', admin.site.urls),
    path('about', about, name="about"),
    #urls maquinas
    path('', index, name="index"),
    path('maquina/list', MaquinaList.as_view(), name="maquina-list"),
    path('maquina/list/mine', MaquinaMineList.as_view(),name="maquina-mine"),
    path('maquina/detail/<pk>/', MaquinaDetail.as_view(), name="maquina-detail"),
    path('maquina/update/<pk>/', MaquinaUpdate.as_view(), name="maquina-update"),
    path('maquina/delete/<pk>/', MaquinaDelete.as_view(), name="maquina-delete"),
    path('maquina/create/', MaquinaCreate.as_view(), name="maquina-create"),
    path('maquina/buscar/', MaquinaSearch.as_view(), name="maquina-buscar"),
    #url ingreso sistema
    path('login/', Login.as_view(), name="login"),
    path('logout/', Logout.as_view(), name="logout"),
    path('signup/', SignUp.as_view(), name="signup"),
    #urls manejo perfil
    path('profile/create', ProfileCreate.as_view(), name="profile-create"),
    path('profile/update/<pk>/', ProfileUpdate.as_view(), name="profile-update"),
    # urls Mensaje
    path('mensaje/create/', MensajeCreate.as_view(), name= "mensaje-create"),
    path('mensaje/list', MensajeList.as_view(), name="mensaje-list"),
    path('mensaje/delete/<pk>/', MensajeDelete.as_view(), name="mensaje-delete"),

]
urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


from pyspark.sql import SparkSession
from pyspark.sql.functions import *

spark = SparkSession.builder \
    .appName("Employee") \
    .config("hive.metastore.uris", "thrift://hive-metastore:9083") \
    .config("spark.jars", "/Drivers/SQL_Sever/jdbc/sqljdbc42.jar")\
    .enableHiveSupport() \
    .getOrCreate()

spark.sql("use bronze")
print("_______Bronze Schema is used________")

bronze_Employee = spark.sql("select * from bronze.dimemployee")

bronze_Employee= bronze_Employee\
            .withColumn("Address", array(col("AddressLine1"), col("AddressLine2")))\
            .drop(col("AdditionalContactInfo"), col("Suffix"), col("Title"), col("AddressLine1"), col("AddressLine2"))

bronze_Employee.createOrReplaceTempView("sEmp")
bronze_Employee = spark.sql("""
    select *
    from semp
    where OrganizationNode is not null or OrganizationLevel is not null
""")

print("________Has Been finished from Cleaning_______")

bronze_Employee.write.format("hive").mode("overwrite").saveAsTable("silver.DimEmployee")
print("____________written_____________")


import os
import logging
import torch
from torch.distributions import Categorical
import numpy as np
from arviz.stats import psislw
from sklearn.metrics import accuracy_score, precision_score, recall_score
from tqdm.auto import tqdm

from dmvaes.dataset import MnistDataset

N_EVAL_SAMPLES = 25
NUM = 300
N_EXPERIMENTS = 5
LABELLED_PROPORTIONS = np.array([0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.0])
LABELLED_PROPORTIONS = LABELLED_PROPORTIONS / LABELLED_PROPORTIONS.sum()
LABELLED_FRACTION = 0.05
np.random.seed(42)
N_INPUT = 28 * 28
N_LABELS = 9

CLASSIFICATION_RATIO = 50.0
N_EVAL_SAMPLES = 25
N_EPOCHS = 100
LR = 3e-4
BATCH_SIZE = 512
DATASET = MnistDataset(
    labelled_fraction=LABELLED_FRACTION,
    labelled_proportions=LABELLED_PROPORTIONS,
    root="dmvaes_data/mnist",
    download=True,
    do_1d=True,
    test_size=0.5,
)
X_TRAIN, Y_TRAIN = DATASET.train_dataset.tensors
RDM_INDICES = np.random.choice(len(X_TRAIN), 200)
X_SAMPLE = X_TRAIN[RDM_INDICES].to("cuda")
Y_SAMPLE = Y_TRAIN[RDM_INDICES].to("cuda")
DO_OVERALL = True

# Utils functions
def compute_reject_score(y_true: np.ndarray, y_pred: np.ndarray, num=20):
    """
        Computes precision recall properties for the discovery label using
        Bayesian decision theory
    """
    _, n_pos_classes = y_pred.shape

    assert np.unique(y_true).max() == (n_pos_classes - 1) + 1
    thetas = np.linspace(0.1, 1.0, num=num)
    properties = dict(
        precision_discovery=np.zeros(num),
        recall_discovery=np.zeros(num),
        accuracy=np.zeros(num),
        thresholds=thetas,
    )

    for idx, theta in enumerate(thetas):
        y_pred_theta = y_pred.argmax(1)
        reject = y_pred.max(1) <= theta
        y_pred_theta[reject] = (n_pos_classes - 1) + 1

        properties["accuracy"][idx] = accuracy_score(y_true, y_pred_theta)

        y_true_discovery = y_true == (n_pos_classes - 1) + 1
        y_pred_discovery = y_pred_theta == (n_pos_classes - 1) + 1
        properties["precision_discovery"][idx] = precision_score(
            y_true_discovery, y_pred_discovery
        )
        properties["recall_discovery"][idx] = recall_score(
            y_true_discovery, y_pred_discovery
        )
    return properties

def res_eval_loop(
    trainer,
    eval_encoder,
    counts_eval,
    encoder_eval_name,
    do_defensive: bool = False,
    debug: bool = False,
):
    model = trainer.model

    logging.info("Predictions computation ...")
    with torch.no_grad():
        # Below function integrates both inference methods for
        # mixture and simple statistics
        train_res = trainer.inference(
            trainer.test_loader,
            # trainer.train_loader,
            keys=[
                "qc_z1_all_probas",
                "y",
                "log_ratios",
                "qc_z1",
                "preds_is",
                "preds_plugin",
            ],
            n_samples=N_EVAL_SAMPLES,
            encoder_key=encoder_eval_name,
            counts=counts_eval,
        )
    y_pred = train_res["preds_plugin"].numpy()
    y_pred = y_pred / y_pred.sum(1, keepdims=True)

    y_pred_is = train_res["preds_is"].numpy()
    # y_pred_is = y_pred_is / y_pred_is.sum(1, keepdims=True)
    assert y_pred.shape == y_pred_is.shape, (y_pred.shape, y_pred_is.shape)

    y_true = train_res["y"].numpy()

    # Precision / Recall for discovery class
    # And accuracy
    logging.info("Precision, recall, auc ...")
    res_baseline = compute_reject_score(y_true=y_true, y_pred=y_pred)
    m_ap = res_baseline["precision_discovery"]
    m_recall = res_baseline["recall_discovery"]
    auc_pr = np.trapz(
        x=res_baseline["recall_discovery"], y=res_baseline["precision_discovery"],
    )

    res_baseline_is = compute_reject_score(y_true=y_true, y_pred=y_pred_is)
    m_ap_is = res_baseline_is["precision_discovery"]
    m_recall_is = res_baseline_is["recall_discovery"]
    auc_pr_is = np.trapz(
        x=res_baseline_is["recall_discovery"], y=res_baseline_is["precision_discovery"],
    )

    # Cubo / Iwelbo with 1e4 samples
    logging.info("Heldout CUBO/IWELBO computation ...")

    n_samples_total = 1e4
    if debug:
        n_samples_total = 200
    n_samples_per_pass = 100 if not do_defensive else counts_eval.sum()
    n_iter = int(n_samples_total / n_samples_per_pass)

    cubo_vals = []
    iwelbo_vals = []
    iwelbo_c_vals = []
    with torch.no_grad():
        i = 0
        for tensors in tqdm(trainer.test_loader):
            x, _ = tensors
            x = x.cuda()
            log_ratios_batch = []
            log_qc_batch = []
            for _ in tqdm(range(n_iter)):
                out = model.inference(
                    x,
                    temperature=0.5,
                    n_samples=n_samples_per_pass,
                    encoder_key=encoder_eval_name,
                    counts=counts_eval,
                )
                if do_defensive:
                    log_ratio = out["log_ratio"].cpu()
                else:
                    log_ratio = (
                        out["log_px_z"]
                        + out["log_pz2"]
                        + out["log_pc"]
                        + out["log_pz1_z2"]
                        - out["log_qz1_x"]
                        - out["log_qc_z1"]
                        - out["log_qz2_z1"]
                    ).cpu()
                log_ratios_batch.append(log_ratio)
                log_qc_batch.append(out["log_qc_z1"].cpu())

            i += 1
            if i == 20:
                break
            # Concatenation
            log_ratios_batch = torch.cat(log_ratios_batch, dim=1)
            log_qc_batch = torch.cat(log_qc_batch, dim=1)

            # Lower bounds
            # 1. Cubo
            # n_cat, n_samples, n_batch = log_ratios_batch.shape
            # cubo_val = torch.logsumexp(
            #     (2 * log_ratios_batch + log_qc_batch).view(n_cat * n_samples, n_batch),
            #     dim=0,
            #     keepdim=False,
            # ) - np.log(n_samples)

            # iwelbo_val = torch.logsumexp(
            #     (log_ratios_batch + log_qc_batch).view(n_cat * n_samples, n_batch),
            #     dim=0,
            #     keepdim=False,
            # ) - np.log(n_samples)
            # IWELBO C
            # # n_cat, n_samples, n_batch
            # qc_probs = log_qc_batch.permute([1, 2, 0]).exp()
            # qc_dist = Categorical(probs=qc_probs)
            # c_sampled = qc_dist.sample().unsqueeze(0)
            # # log_qc_samp = torch.gather(log_qc_batch, dim=0, index=c_sampled)
            # log_ratios_samp = torch.gather(log_ratios_batch, dim=0, index=c_sampled)
            # # Shape 1, n_samples, n_batch
            # iwelboc_val = torch.logsumexp(log_ratios_samp, dim=1) - np.log(n_samples)
            # iwelboc_val = iwelboc_val.squeeze()

            # cubo_vals.append(cubo_val.cpu())
            # iwelbo_vals.append(iwelbo_val.cpu())
            # iwelbo_c_vals.append(iwelboc_val.cpu())

            # RELAXED CASE
            n_samples, n_batch = log_ratios_batch.shape
            cubo_val = torch.logsumexp(
                2 * log_ratios_batch, dim=0, keepdim=False,
            ) - np.log(n_samples)

            iwelbo_val = torch.logsumexp(
                log_ratios_batch, dim=0, keepdim=False,
            ) - np.log(n_samples)

            cubo_vals.append(cubo_val.cpu())
            iwelbo_vals.append(iwelbo_val.cpu())
        cubo_vals = torch.cat(cubo_vals)
        iwelbo_vals = torch.cat(iwelbo_vals)
        # iwelbo_c_vals = torch.cat(iwelbo_c_vals)

    # Entropy
    where9 = train_res["y"] == 9
    probas9 = train_res["qc_z1_all_probas"].mean(0)[where9]
    entropy = (-probas9 * probas9.log()).sum(-1).mean(0)

    where_non9 = train_res["y"] != 9
    y_non9 = train_res["y"][where_non9]
    y_pred_non9 = y_pred[where_non9].argmax(1)
    m_accuracy = accuracy_score(y_non9, y_pred_non9)

    y_pred_non9_is = y_pred_is[where_non9].argmax(1)
    m_accuracy_is = accuracy_score(y_non9, y_pred_non9_is)

    # k_hat
    n_samples_total = 1e4
    if debug:
        n_samples_total = 200
    n_samples_per_pass = 25 if not do_defensive else counts_eval.sum()
    n_iter = int(n_samples_total / n_samples_per_pass)

    # a. Unsupervised case
    # log_ratios = []
    # qc_z = []
    # for _ in tqdm(range(n_iter)):
    #     with torch.no_grad():
    #         out = model.inference(
    #             X_SAMPLE,
    #             temperature=0.5,
    #             n_samples=n_samples_per_pass,
    #             encoder_key=encoder_eval_name,
    #             counts=counts_eval,
    #         )
    #     if do_defensive:
    #         log_ratio = out["log_ratio"].cpu()
    #     else:
    #         log_ratio = (
    #             out["log_px_z"]
    #             + out["log_pz2"]
    #             + out["log_pc"]
    #             + out["log_pz1_z2"]
    #             - out["log_qz1_x"]
    #             - out["log_qc_z1"]
    #             - out["log_qz2_z1"]
    #         ).cpu()
    #     qc_z_here = out["log_qc_z1"].cpu().exp()
    #     qc_z.append(qc_z_here)
    #     log_ratios.append(log_ratio)
    # # Concatenation over samples
    # log_ratios = torch.cat(log_ratios, 1)
    # qc_z = torch.cat(qc_z, 1)
    # log_ratios_sum = (log_ratios * qc_z).sum(0)  # Sum over labels
    # wi = torch.softmax(log_ratios_sum, 0)
    # _, khats = psislw(log_ratios_sum.T.clone())

    log_ratios = []
    for _ in tqdm(range(n_iter)):
        with torch.no_grad():
            out = model.inference(
                X_SAMPLE,
                temperature=0.5,
                n_samples=n_samples_per_pass,
                encoder_key=encoder_eval_name,
                counts=counts_eval,
            )
        if do_defensive:
            log_ratio = out["log_ratio"].cpu()
        else:
            log_ratio = (
                out["log_px_z"]
                + out["log_pz2"]
                + out["log_pc"]
                + out["log_pz1_z2"]
                - out["log_qz1_x"]
                - out["log_qc_z1"]
                - out["log_qz2_z1"]
            ).cpu()
        log_ratios.append(log_ratio)
    # Concatenation over samples
    log_ratios = torch.cat(log_ratios, 0)
    _, khats = psislw(log_ratios.T.clone())

    x_samp, y_samp = DATASET.train_dataset[:128]
    where_ = y_samp != 9
    x_samp = x_samp[where_].cuda()
    y_samp = y_samp[where_].cuda()
    log_ratios = []
    for _ in tqdm(range(n_iter)):
        with torch.no_grad():
            out = model.inference(
                x_samp,
                y_samp,
                temperature=0.5,
                n_samples=n_samples_per_pass,
                encoder_key=encoder_eval_name,
                counts=counts_eval,
            )
        if do_defensive:
            log_ratio = out["log_ratio"].cpu()
        else:
            log_ratio = (
                out["log_px_z"]
                + out["log_pz2"]
                + out["log_pc"]
                + out["log_pz1_z2"]
                - out["log_qz1_x"]
                # - out["log_qc_z1"]
                - out["log_qz2_z1"]
            ).cpu()
        log_ratios.append(log_ratio)
    # Concatenation over samples
    log_ratios = torch.cat(log_ratios, 0)
    _, khats_c_obs = psislw(log_ratios.T.clone())

    res = {
        "IWELBO": iwelbo_vals.mean().item(),
        # "IWELBOC": iwelbo_c_vals.mean().item(),
        "CUBO": cubo_vals.mean().item(),
        "KHAT": np.array(khats),
        "M_ACCURACY": m_accuracy,
        "MEAN_AP": m_ap,
        "MEAN_RECALL": m_recall,
        "KHATS_C_OBS": khats_c_obs,
        "M_ACCURACY_IS": m_accuracy_is,
        "MEAN_AP_IS": m_ap_is,
        "MEAN_RECALL_IS": m_recall_is,
        "AUC_IS": auc_pr_is,
        "AUC": auc_pr,
        "ENTROPY": entropy,
    }
    return res


import logging
import os
import sys

import yaml

from nlg_analysis.cfg import TrainConfig
from nlg_analysis.models import BERTModel, GPTModel, RNNModel
from nlg_analysis.utils import parse_arguments

logger = logging.getLogger()
logging.basicConfig(
    format="%(asctime)s : %(levelname)s : %(message)s",
    level=os.environ.get("LOGLEVEL", "INFO"),
)

def main():
    args = parse_arguments(sys.argv[1:])

    # Load config file
    logger.info("Loading config...")
    with open(args.config_file) as f:
        config = yaml.safe_load(f)
    train_cfg = TrainConfig.from_dict(config)

    # Setup models
    models = []
    if train_cfg.gpt_model:
        models.append(
            {
                "model": GPTModel(model_name=train_cfg.gpt_model_name),
                "output_path": train_cfg.gpt_output_path,
            }
        )
    if train_cfg.rnn_model:
        models.append(
            {"model": RNNModel(), "output_path": train_cfg.rnn_output_path}
        )
    if train_cfg.bert_model:
        models.append(
            {
                "model": BERTModel(model_name=train_cfg.bert_model_name),
                "output_path": train_cfg.bert_output_path,
            }
        )
    logger.info(
        "Train models: {0}".format(
            ", ".join([model["model"].approach() for model in models])
        )
    )

    # Train models
    for model in models:
        logger.info("Train model {0}".format(model["model"].approach()))
        model["model"].train(model["output_path"], train_cfg)

if __name__ == "__main__":
    main()


from django.urls import path
from . import views

app_name = 'quiz_urls'

urlpatterns = [
    path('', views.index_quiz, name='index_quiz'),
    path('<quiz_pk>/hard', views.quiz_questions, name='quiz_detail_hand'),
    path('<quiz_pk>/easy', views.quiz_questions_paginate, name='quiz_detail_easy'),
    path('<quiz_pk>/quiz_finish', views.quiz_finish, name='quiz_finish'),
    path('<quiz_pk>/restart_test', views.restart_test, name='restart_test'),
]


from django.shortcuts import render, HttpResponse
from booktest.models import BookInfo
from PIL import Image, ImageDraw, ImageFont
from django.utils.six import BytesIO
# Create your views here.

# 首页视图
def index(request):
	return render(request,'booktest/index.html')

# 向HTML文件中传递参数
def temp_var(request):
	dict1 = {'title':'haha'}
	list1 = [1,2,3,4]
	books = BookInfo.objects.all()[0]

	return render(request,'booktest/temp_var.html',{'dict1':dict1,'list1':list1,'books':books})

# 标签
def temp_tags(request):
	books = BookInfo.objects.all()
	return render(request,'booktest/temp_tags.html',{'books':books})

# 过滤器
def temp_filter(request):
	books = BookInfo.objects.all()
	return render(request,'booktest/temp_filter.html',{'books':books})

# 继承
def temp_inherit(request):
	return render(request,'booktest/temp_inherit.html')
def temp_inherit_son(request):
	books = BookInfo.objects.all()
	return render(request,'booktest/temp_inherit_son.html',{'books':books})

# 模板转义 不转义会引起xss攻击
def html_zhuan(request):
	return render(request,'booktest/html_zhuan.html',{'content':'
hello world
'})

# xss攻击
def xss_test(request):
    text = ''
    if request.method == 'POST':
        text = request.POST.get('text')
    return render(request, 'booktest/xss.html', {'text': text})

# 生成验证码
def verify_code(request):
	import random
	# 背景色(偏蓝)，宽，高
	bgcolor = (random.randrange(20, 100),random.randrange(20, 100), 255)
	width = 100
	height = 25

	# 创建画面对象
	im = Image.new('RGB', (width, height), bgcolor)
	# 创建画笔对象
	draw = ImageDraw.Draw(im)
	# 调用画笔的point()函数绘制噪点(绘制100个噪点)
	for i in range(1, 100):
		xy = (random.randrange(0,width), random.randrange(0, height))
		fill = (random.randrange(0, 255), 255, random.randrange(0, 255))
		draw.point(xy, fill=fill)
	# 定义验证码的备选值
	str1 = 'ABCD123EFGHIJK456LMNOPQRS789TUVWXYZ0'
	rand_str = ''
	for i in range(0,4):
		rand_str += str1[random.randrange(0, len(str1))]

	# 构造字体对象，
	font = ImageFont.truetype('static/font/MicroExtendFLF.ttf', 23)
	# 构造字体颜色（偏红）
	fontcolor = (255, random.randrange(0, 255), random.randrange(0, 255))

	# 绘制四个字
	draw.text((5, 2), rand_str[0], font=font, fill=fontcolor)
	draw.text((25, 2), rand_str[1], font=font, fill=fontcolor)
	draw.text((50, 2), rand_str[2], font=font, fill=fontcolor)
	draw.text((75, 2), rand_str[3], font=font, fill=fontcolor)
	# 释放画笔
	del draw
	# 存入session 用于做进一步的验证
	request.session['verifycode'] = rand_str

	buf = BytesIO()
	im.save(buf, 'png')
	return HttpResponse(buf.getvalue(), 'image/png')

# 验证码显示
def verify_show(request):
	return render(request, 'booktest/verify_show.html')

# 验证码验证
def verify_yz(request):
	# 获取用户输入
	yzm = request.POST.get('yzm')
	if yzm == '':
		response=HttpResponse('no')

	# 获取服务器session
	verifycode = request.session['verifycode'].lower()
	response=HttpResponse('no')
	if yzm.lower()==verifycode:
		# 登陆成功
		response=HttpResponse('ok')
	return response

# 反向解析
def fan1(request):
	return render(request,'booktest/fan1.html')

def fan2(request):
	return HttpResponse('fan2')


#!/usr/bin/env python3
"""html_utils.py

Tools to assist with graphyte HTML creation and other XML-related tasks.

"""

# imports
import logging
import os
import re
from subprocess import Popen, PIPE
from tempfile import mkstemp
from os import fdopen
from shutil import move, copy
import pprint
# import webbrowser

# info
__author__ = "Jorge Somavilla"

# initialize logger
logger = logging.getLogger('graphyte')

pp = pprint.PrettyPrinter(indent=4)

def yang_2_uml(gm):
    """Convert YANG module into UML format and store in
    graphyte module object (in_diagram_path attribute).

    :param gm: the graphyte module object
    :return: boolean result
    """
    yang_fname = os.path.basename(gm.in_diagram_path)
    work_yang_path = gm.work_dir + "/" + gm.in_diagram_name
    yang_fname_no_ext = os.path.splitext(yang_fname)[0]
    work_uml_path = gm.work_dir + "/" + yang_fname_no_ext + ".uml"
    copy(gm.in_diagram_path,work_yang_path)
    uml_no_option = "--uml-no=" + gm.pyang_uml_no
    #print ("\npyang --ignore-errors " + uml_no_option + " -f uml " + work_yang_path + " -o " + work_uml_path)
    p1 = Popen(["pyang", "--ignore-errors", uml_no_option, "-f", "uml", work_yang_path,
                "-o", work_uml_path], cwd=gm.run_dir, stdout=PIPE, stderr=PIPE)
    p1.communicate()  # wait for pyang execution
    result = False
    if p1.returncode == 0:
        if os.path.exists(work_uml_path):
            logger.info('         Successfully converted YANG to UML.' + '\r\n')
            result = True
        else:
            logger.info('         Failed to convert YANG to UML. Is pyang installed?' + '\r\n')
            result = True
    else:
        logger.error('              Couldn´t convert YANG to UML. Check ' + yang_fname + ' syntax?\r\n')
        result = False
        return result
    # remove pyang-generated title and footer
    remove_patterns = ['center footer', ' <size:20> UML Generated :', ' endfooter', 'Title ' + yang_fname_no_ext]
    tmp_fh, tmp_uml_path = mkstemp()  # temp file
    with fdopen(tmp_fh, 'w') as new_uml:
        with open(work_uml_path) as old_uml:
            for line in old_uml:
                if not any(p in line for p in remove_patterns):
                    new_uml.write(line)
    if os.path.exists(work_uml_path):
        os.remove(work_uml_path)
    copy(tmp_uml_path, work_uml_path)
    # store as reference diagram
    gm.in_diagram_path = work_uml_path
    return result

def uml_2_svg(gm):
    """Convert UML diagram into SVG format and store in
    graphyte module object (svg_path attribute).

    :param gm: the graphyte module object
    :return: None
    """
    # For pyang generated UML files:
    # Try to remove relative output path from UML file
    # '@startuml' line should be left without additional arguments
    logger.info('         Processing UML file...' + '\r\n')
    uml_fname = os.path.basename(gm.in_diagram_path)
    logger.info('             ' + uml_fname + '\r\n')
    tmp_fh, tmp_uml_path = mkstemp()  # temp file

    with fdopen(tmp_fh, 'w') as new_uml:
        with open(gm.in_diagram_path) as old_uml:
            for line in old_uml:
                new_uml.write(re.sub(r'@startuml.*', r'@startuml', line))
    work_uml_path = gm.work_dir + "/" + gm.in_diagram_name

    if os.path.exists(work_uml_path):
        os.remove(work_uml_path)
    copy(tmp_uml_path, work_uml_path)
    plantuml_out_file = gm.work_dir + "/" + os.path.splitext(gm.in_diagram_name)[0] + ".svg"

    # TODO: cath exception if plantuml not in place
    #print ("java -Xmx1024m -jar utils/plantuml.jar -v -tsvg " + work_uml_path + " -o " + gm.work_dir)
    p1 = Popen(["java","-Xmx1024m", "-jar", "utils/plantuml.jar", "-v", "-tsvg", work_uml_path,
                "-o", gm.work_dir], cwd=gm.run_dir, stdout=PIPE, stderr=PIPE)
    p1.communicate()  # wait for plantuml execution
    gm.svg_path = plantuml_out_file
    d = dict()
    d['modsvgpath'] = { os.path.basename(plantuml_out_file) : plantuml_out_file }
    #pp.pprint(d)
    logger.info('         ...done' + '\r\n')
    return d

def atag_2_gtag(svg_lines, i):
    """Figures out if <a tag should be replaced by <g tag
    in SVG link group, depending on link label used:

     do not replace by <g
     do not replace by <g
     replace by <g

    :param svg_lines: List of SVG lines
    :param i: current line
    :return: True if replace required, False otherwise.
    """
    while i < len(svg_lines):
        line = svg_lines[i]
        try:
            link = re.search('xlink:href="(.*?)"', line).group(1)
        except AttributeError:
            link = False
        if link:
            mod = re.search('mod:(.*)', link)
            lit = re.search('lit:(.*)', link)
            if mod:
                return False
            elif lit:
                return False
            else:
                return True
        i += 1
    return True

def guess_module(gm, name, fullname):
    """Finds the module linked with mod:module_name, looking into available
    modules in the model.

    If found, builds name of final html file for the module.

    If not found, issues a warning and ignores link.

    :param gm: the graphyte model
    :param name: the module specified by the user, stripped off extension,
    to be looked up among known modules.
    :param fullname: full link content as specified by the user.
    :return: "module" if name found, "fullname" if name not found.
    """
    if gm.menu_items:
        item_list = gm.menu_items.split(",")
        for item in item_list:
            item = item.strip()
            if name in item:
                item_no_sp = re.sub(r'\s+', r'_', item)
                module = gm.model_no_sp + "_" + item_no_sp + "_v" + gm.version + ".html"
                return module
    logger.warning('              Could not find linked module \"' + fullname + '\", ignoring.\r\n')
    return fullname

def process_svg(gm):
    """Process SVG so it can be embedded in HTML as
    supported by current web browsers.

    Plantuml, Draw.io, Visio specific sections are
    removed or modified to achieve browser support.

    :param gm: the graphyte module object
    :return: processed_svg, the processed SVG

    """

    # TODO: use xpath

    logger.info('         Processing SVG file...' + '\r\n')
    with open(gm.svg_path, 'r') as f:
        svg_fname = os.path.basename(gm.svg_path)
        logger.info('             ' + svg_fname + '\r\n')
        content = f.read()

        # plantuml svg
        if gm.diagram_is_uml():
            content = re.sub(r'\s+<(?!/text)', '<', content)
            content = re.sub(r'><', '>\n<', content)

        # draw.io svg
        is_drawio_file = False

        if "editor="www.draw.io"" in content \
                or "host="www.draw.io"" in content \
                or "host="scdp.cisco.com&quot" in content \
                or "host="app.diagrams.net&quot" in content \
                or re.search(r'<[^<>]+agent=[^>]+draw.io', content):
            is_drawio_file = True
            content = re.sub(r'width="(.*?)px" height="(.*?)px"'
                             , r'viewBox="0 0 \1 \2"', content)
            content = re.sub(r'><', '>\n<', content)

        processed_svg = ""
        a_tag = False
        svg_tag_level = 0
        svg_tag = False
        in_foreign_obj = False
        skip_line = False
        a_g_stack = []
        curr_tag = ""

        svg_lines = content.splitlines()
        i = 0
        while i < len(svg_lines):
            line = svg_lines[i]
            if re.match(r'^\s*<svg', line):
                svg_tag = True
                svg_tag_level += 1
                if svg_tag_level == 1:
                    line = re.sub(
                        r'<svg',
                        r'<svg id="svg" width="100%" preserveAspectRatio="xMinYMin slice"'
                        , line.rstrip()
                    )
            if svg_tag and svg_tag_level == 1:
                line = re.sub(r'id="(?!svg)"', r'', line.rstrip())
                line = re.sub(r'width="(?!%100)"', r'', line.rstrip())
                line = re.sub(r'height=".*?"', r'', line.rstrip())
                line = re.sub(
                    r'preserveAspectRatio="(?!xMinYMin slice)"'
                    , r'', line.rstrip())  # plantuml
                line = re.sub(r'style=".*?"', r'', line.rstrip())  # plantuml
            if re.match(r'^\s*<g', line):
                a_g_stack.append("g")
            if re.match(r'^\s*<a', line):
                a_tag = True
                if atag_2_gtag(svg_lines, i):
                    a_g_stack.append("g")
                    line = re.sub(r'<a', r'<g', line.rstrip())
                else:
                    a_g_stack.append("a")
            if a_tag:
                link = re.search('xlink:href="(.*?)"', line)
                if link:
                    link_str_old = link.group(1)
                    link_str_old_no_ext = os.path.splitext(link_str_old)[0]
                    mod = re.search('mod:(.*)', link_str_old_no_ext)
                    lit = re.search('lit:(.*)', link_str_old)
                    if mod:
                        link_str_new = guess_module(gm, mod.group(1), link_str_old)
                        line = line.replace(link_str_old, link_str_new)
                    elif lit:
                        link_str_new = lit.group(1)
                        line = line.replace(link_str_old, link_str_new)
                    else:
                        link_str_new = link_str_old.split("\\")[-1]
                        link_str_new = link_str_new.split("/")[-1]
                        gm.push_link(link_str_new)
                        line = line.replace(link_str_old, link_str_new)
                        line = re.sub(r'id=".*"', r'', line.rstrip())
                        line = re.sub(r'xlink:href="(.*?)"', r'class="wrapper" id="\1"', line.rstrip())
                        line = re.sub(r'xlink:actuate="onRequest"', r'', line.rstrip())  # plantuml
                        line = re.sub(r'xlink:show="new"', r'', line.rstrip())  # plantuml
                        line = re.sub(r'xlink:type="simple"', r'', line.rstrip())  # plantuml
            if re.match(r'.*>\s*$', line):
                svg_tag = False
                if a_tag:
                    a_tag = False
            if re.match(r'.*</(a|g)>.*$', line):
                if len(a_g_stack) > 0:
                    curr_tag = a_g_stack.pop()
                if curr_tag == "g":
                    line = re.sub(r'</a', r'</g', line.rstrip())
                curr_tag = ""
            if not is_drawio_file:
                if re.match(r'^\s*<foreignObject', line):
                    in_foreign_obj = True
                if re.match(r'^\s*</foreignObject>', line):
                    in_foreign_obj = False
                    line = ""
            if in_foreign_obj and not is_drawio_file:  # add here other conditions to skip line
                skip_line = True
            if not skip_line:
                processed_svg += (line.rstrip() + "\n")  # appends
            skip_line = False

            # increase loop index
            i += 1
    logger.info('         ...ok' + '\r\n')
    return processed_svg

def build_menu(gm):
    """Build navigation menu for final HTML module, which allows
    navigation between different modules in the graphyte model.

    Stores navigation menu on menu_tags attribute of graphyte module
    object.

    :param gm: the graphyte module object
    :return: None
    """
    logger.info('         Building navigation menu...' + '\r\n')
    if gm.menu_items:
        item_list = gm.menu_items.split(",")
        for item in item_list:
            item = item.strip()
            item_no_sp = re.sub(r'\s+', r'_', item)
            gm.menu_tags += "
" + item + "
"
    else:
        gm.menu_tags += "
" + gm.module + "
"
    logger.info('         ...ok' + '\r\n')
    return

def build_html(gm, processed_svg, file_script, xls_to_script):
    """Assembles final HTML module.

    Uses pre-cooked mod_template and embeds module specific templates,
    parameter lists, navigation menu, SVG diagram, title and viewer initial
    content.

    :param gm: the graphyte module object
    :param processed_svg: the SVG diagram adapted for browser support
    :param file_script: templates and template parameter table
     in JS


"""
This module contains a wrapper for the Faiss library by Facebook AI Research.
"""

from typing import List, Union, Optional, Dict, Any
import os
os.environ["TOKENIZERS_PARALLELISM"] = "false"

import torch
from transformers import AutoTokenizer, AutoModel
from datasets import Dataset

import pandas as pd

# FAISS library wrapper class
class FaissSearch:
    def __init__(self,
        model_name_or_path: str = 'facebook/bart-large',
        tokenizer_name_or_path: str = 'facebook/bart-large',
        device: str = 'cpu',
        ) -> None:
        r"""
        This function initializes the wrapper for the FAISS library, which is used to perform semantic search.

        .. attention::

            * If you use this class, please make sure to cite the following paper:

                .. code-block:: latex

                    @article{johnson2019billion,
                        title={Billion-scale similarity search with {GPUs}},
                        author={Johnson, Jeff and Douze, Matthijs and J{\'e}gou, Herv{\'e}},
                        journal={IEEE Transactions on Big Data},
                        volume={7},
                        number={3},
                        pages={535--547},
                        year={2019},
                        publisher={IEEE}
                    }

            * The code is based on the following GitHub repository:
                https://github.com/facebookresearch/faiss

        Arguments:
            model_name_or_path (str, optional): The name or path of the model to use. Defaults to 'facebook/bart-large'.
            tokenizer_name_or_path (str, optional): The name or path of the tokenizer to use. Defaults to 'facebook/bart-large'.
            device (str, optional): The device to use. Defaults to 'cpu'.

        Returns:
            None
        """

        # Set the device
        self.device = device

        # If the tokenizer is not specified, use the model name or path
        if tokenizer_name_or_path is None:
            tokenizer_name_or_path = model_name_or_path

        # Load the tokenizer
        self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path)

        # Load the model
        self.model = AutoModel.from_pretrained(model_name_or_path).to(self.device)

        # Set the model to evaluation mode (since we do not need the gradients)
        self.model.eval()

        # Initialize the dataset
        self.dataset = None

    # Auxiliary function to get the last hidden state
    def get_last_hidden_state(self,
        embeddings: torch.Tensor,
    ) -> torch.Tensor:
        """
        This function returns the last hidden state (e.g., [CLS] token's) of the input embeddings.

        Arguments:
            embeddings (torch.Tensor): The input embeddings.

        Returns:
            torch.Tensor: The last hidden state.
        """

        # Get the last hidden state
        last_hidden_state = embeddings.last_hidden_state

        # Return the last hidden state
        return last_hidden_state[:, 0, :]

    # Auxiliary function to get the mean pooling
    def get_mean_pooling(self,
        embeddings: torch.Tensor,
    ) -> torch.Tensor:
        """
        This function returns the mean pooling of the input embeddings.

        Arguments:
            embeddings (torch.Tensor): The input embeddings.

        Returns:
            torch.Tensor: The mean pooling.
        """

        # Get the mean pooling
        mean_pooling = embeddings.last_hidden_state.mean(dim=1)

        # Return the mean pooling
        return mean_pooling

    # Get the embeddings
    def get_embeddings(self,
        text: Union[str, List[str]],
        embedding_type: str = 'last_hidden_state',
        batch_size: int = 8,
        num_workers: int = 4,
    ) -> torch.Tensor:
        """
        This function returns the embeddings of the input text.

        Arguments:
            text (Union[str, List[str]]): The input text.
            embedding_type (str, optional): The type of embedding to use. Defaults to 'last_hidden_state'.
            batch_size (int, optional): The batch size to use. Defaults to 8.
            num_workers (int, optional): The number of workers to use. Defaults to 4.

        Returns:
            torch.Tensor: The embeddings.

        Raises:
            ValueError: If the embedding type is invalid.
        """

        # Check if the embedding type is valid
        if embedding_type not in ['last_hidden_state', 'mean_pooling']:
            raise ValueError(f'Invalid embedding type: {embedding_type}. Only "last_hidden_state" and "mean_pooling" are supported.')

        # Tokenize the input text
        encoded_text = self.tokenizer(
            text,
            padding=True,
            truncation=True,
            return_tensors='pt',
        )

        # Move the input text to the device
        encoded_text = encoded_text.to(self.device)

        # encoded_inputs = {k: v.to(self.device) for k, v in encoded_inputs.items()}

        # Get the embeddings
        with torch.no_grad():
            embeddings = self.model(**encoded_text)

        # Get the proper embedding type
        if embedding_type == 'last_hidden_state':
            # Get the last hidden state
            embeddings = self.get_last_hidden_state(embeddings)
        elif embedding_type == 'mean_pooling':
            # Get the mean pooling
            embeddings = self.get_mean_pooling(embeddings)

        # Return the embeddings
        return embeddings

    # Add FAISS index
    def add_faiss_index(self,
        column_name: str = 'embeddings',
        metric_type: Optional[int] = None,
        batch_size: int = 8,
        **kwargs,
    ) -> None:
        """
        This function adds a FAISS index to the dataset.

        Arguments:
            column_name (str, optional): The name of the column containing the embeddings. Defaults to 'embeddings'.
            index_type (str, optional): The index type to use. Defaults to 'Flat'.
            metric_type (str, optional): The metric type to use. Defaults to 'L2'.

        Returns:
            None

        Raises:
            ValueError: If the dataset is not initialized.
        """

        # Check if the dataset is initialized
        if self.dataset is None:
            raise ValueError('The dataset is not initialized. Please initialize the dataset first.')

        print('Adding FAISS index...')
        self.dataset.add_faiss_index(
            column_name,
            # metric_type=metric_type,
            # device=self.device,
            # batch_size=batch_size,
            faiss_verbose=True,
            # **kwargs,
        )

    def save_faiss_index(self,
        index_name: str,
        file_path: str,
    ) -> None:
        """
        This function saves the FAISS index to the specified file path.
            * This is a wrapper function for the `save_faiss_index` function in the `Dataset` class.

        Arguments:
            index_name (str): The name of the FAISS index  (e.g., "embeddings")
            file_path (str): The file path to save the FAISS index.

        Returns:
            None

        Raises:
            ValueError: If the dataset is not initialized.
        """

        # Check if the dataset is initialized
        if self.dataset is None:
            raise ValueError('The dataset is not initialized. Please initialize the dataset first.')

        print('Saving FAISS index...')
        self.dataset.save_faiss_index(index_name=index_name, file=file_path)

    def load_faiss_index(self,
        index_name: str,
        file_path: str,
        device: str = 'cpu',
    ) -> None:
        """
        This function loads the FAISS index from the specified file path.
            * This is a wrapper function for the `load_faiss_index` function in the `Dataset` class.

        Arguments:
            index_name (str): The name of the FAISS index  (e.g., "embeddings")
            file_path (str): The file path to load the FAISS index from.
            device (str, optional): The device to use ("cpu" or "cuda") (default: "cpu").

        Returns:
            None

        Raises:
            ValueError: If the dataset is not initialized.
        """

        # Check if the dataset is initialized
        if self.dataset is None:
            raise ValueError('The dataset is not initialized. Please initialize the dataset first.')

        print('Loading FAISS index...')
        self.dataset.load_faiss_index(index_name=index_name, file=file_path, device=device)

    # Initialize the corpus using a dictionary or pandas DataFrame or HuggingFace Datasets object
    def initialize_corpus(self,
        corpus: Union[Dict[str, List[str]], pd.DataFrame, Dataset],
        section: str = 'text',
        index_column_name: str = 'embeddings',
        embedding_type: str = 'last_hidden_state',
        batch_size: Optional[int] = None,
        num_workers: Optional[int] = None,
        save_path: Optional[str] = None,
    ) -> Dataset:
        """
        This function initializes a dataset using a dictionary or pandas DataFrame or HuggingFace Datasets object.

        Arguments:
            dataset_dict (Dict[str, List[str]]): The dataset dictionary.
            section (str): The section of the dataset to use whose embeddings will be used for semantic search (e.g., 'text', 'title', etc.) (default: 'text').
            index_column_name (str): The name of the column containing the embeddings (default: 'embeddings')
            embedding_type (str): The type of embedding to use (default: 'last_hidden_state').
            batch_size (int, optional): The batch size to use (default: 8).
            max_length (int, optional): The maximum length of the input sequences.
            num_workers (int, optional): The number of workers to use.
            save_path (Optional[str], optional): The path to save the dataset (default: None).

        Returns:
            Dataset: The dataset object (HuggingFace Datasets).

        Raises:
            ValueError: If the dataset is not a dictionary or pandas DataFrame or HuggingFace Datasets object.
        """

        # Create the dataset
        if isinstance(corpus, dict):
            self.dataset = Dataset.from_dict(corpus)
        elif isinstance(corpus, pd.DataFrame):
            self.dataset = Dataset.from_pandas(corpus)
        elif isinstance(corpus, Dataset):
            self.dataset = corpus
        else:
            raise ValueError('The dataset must be a dictionary or pandas DataFrame.')

        # Set the embedding_type
        self.embedding_type = embedding_type

        # Tokenize the dataset
        # self.dataset = self.dataset.map(
        #     lambda x: x[section],
        #     batched=True,
        #     batch_size=batch_size,
        #     num_proc=num_workers,
        # )

        # Map the section of the dataset to the embeddings
        self.dataset = self.dataset.map(
            lambda x: {
                index_column_name: self.get_embeddings(x[section], embedding_type=self.embedding_type).detach().cpu().numpy()[0]
                },
            # batched=True,
            batch_size=batch_size,
            num_proc=num_workers,
        )

        # Save the dataset
        if save_path is not None:
            self.dataset.to_json(save_path)

        # Add FAISS index
        self.add_faiss_index(
            column_name=index_column_name,
        )

        # Return the dataset
        return self.dataset

    # Initialize the dataset using a JSON file
    def load_dataset_from_json(self,
        json_path: str,
    ) -> Dataset:
        """
        This function loads a dataset from a JSON file.

        Arguments:
            json_path (str): The path to the JSON file.

        Returns:
            Dataset: The dataset.
        """

        # Load the dataset
        self.dataset = Dataset.from_json(json_path)

        # Return the dataset
        return self.dataset

    # Search for the most similar elements in the dataset, given a query
    def search(self,
        query: str,
        k: int = 1,
        index_column_name: str = 'embeddings',
    ) -> pd.DataFrame:
        """
        This function searches for the most similar elements in the dataset, given a query.

        Arguments:
            query (str): The query.
            k (int, optional): The number of elements to return  (default: 1).
            index_column_name (str, optional): The name of the column containing the embeddings (default: 'embeddings')

        Returns:
            pd.DataFrame: The most similar elements in the dataset (text, score, etc.), sorted by score.

        Remarks:
            The returned elements are dictionaries containing the text and the score.
        """

        # Get the embeddings of the query
        query_embeddings = self.get_embeddings([query], embedding_type=self.embedding_type).detach().cpu().numpy()

        # Search for the most similar elements in the dataset
        scores, similar_elts = self.dataset.get_nearest_examples(
            index_name=index_column_name,
            query=query_embeddings,
            k=k,
        )

        # Convert the results to a pandas DataFrame
        results_df = pd.DataFrame.from_dict(similar_elts)

        # Add the scores
        results_df['score'] = scores

        # Sort the results by score
        results_df.sort_values("score", ascending=True, inplace=True)

        # Return the most similar elements
        return results_df


import cv2
import mediapipe as mp
import math

video = cv2.VideoCapture('./estudos/cursos/udemy/Curso1/OpenCVBasic/ANEXO+polichinelos.mp4')
pose = mp.solutions.pose
Pose = pose.Pose(min_tracking_confidence=0.5, min_detection_confidence=0.5)
draw = mp.solutions.drawing_utils
count = 0
check = True

while True:
    check, img = video.read()

    videoRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    results = Pose.process(videoRGB)
    points = results.pose_landmarks
    draw.draw_landmarks(img, points, pose.POSE_CONNECTIONS)
    h, w, _ = img.shape

    if points:
        footRy = int(points.landmark[pose.PoseLandmark.RIGHT_FOOT_INDEX].y*h)
        footRx = int(points.landmark[pose.PoseLandmark.RIGHT_FOOT_INDEX].x*w)
        footLy = int(points.landmark[pose.PoseLandmark.LEFT_FOOT_INDEX].y*h)
        footLx = int(points.landmark[pose.PoseLandmark.LEFT_FOOT_INDEX].x*w)
        handRy = int(points.landmark[pose.PoseLandmark.RIGHT_INDEX].y*h)
        handRx = int(points.landmark[pose.PoseLandmark.RIGHT_INDEX].x*w)
        handLy = int(points.landmark[pose.PoseLandmark.LEFT_INDEX].y*h)
        handLx = int(points.landmark[pose.PoseLandmark.LEFT_INDEX].x*w)

        distHand = math.hypot(handRx-handLx, handRy-handLy)
        distFoot = math.hypot(footRx-footLx, footRy-footLy)

        if check == True and distHand <= 150 and distFoot >= 150:
            count += 1
            check = False
        if distHand > 150 and distFoot < 150:
            check = True

        text = f'Qtd: {count}'
        cv2.putText(img, text, (40,200), cv2.FONT_HERSHEY_SIMPLEX, 2, (0,0,255))

    cv2.imshow('Video', img)
    cv2.waitKey(40)


# -*- coding: utf-8 -*-
# 所有Handler基类

import copy

import tornado.web

class BaseHandler(tornado.web.RequestHandler):
    """所有Handler基类.

    重载了输出方法，
    正确输出 (200) 使用 write_json_f 方法，错误输出使用 write_error_f 方法
    """

    # 输出模板
    RSP = {
        "code": "200",
        "message": "OK",
        "relationships": {
            "author": "null"
        },
        "jsonapi": {
            "version": "null"
        }}

    # 输出JSON
    rsp = dict

    pagination = None

    _ARG_DEFAULT = object()

    def get_argument_t(self, name, default=_ARG_DEFAULT, type=None, strip=True):
        """重载get_argument方法，并判断参数类型是否匹配.

        输出时如果满足参数类型要求，则按类型输出，否则输出None
        """
        arg = self.get_argument(name, default, strip)
        if type == int:
            if not isinstance(arg, int):
                arg = int(arg) if arg.isdigit() else None
        return arg

    def get_argument_cardno(self, default=0):
        """获取cardno参数，并校验."""
        cardno = self.get_argument_t('cardno', default, int)
        if not cardno or cardno < 213120000 or cardno > 213259999:
            return self.write_error(4001)
        return cardno

    def write_json_f(self, data=None):
        """重载write方法，并finish.

        Args:
            data: 输出JSON的data字段，默认为空
        """
        self.write_json(data)
        self.finish()

    def write_json(self, data=None):
        """重载write方法.

        Args:
            data: 输出JSON的data字段，默认为空
        """
        self.rsp = copy.deepcopy(self.RSP)
        if self.pagination:
            self.rsp["pagination"] = self.pagination
        self._write_json(data)

    def _write_json(self, data=None):
        """输出准备."""
        if data:
            self.rsp["data"] = data
        self.rsp["relationships"]["author"] = self.INFO["author"]
        self.rsp["jsonapi"]["version"] = self.INFO["version"]
        self.write(self.rsp)

    def write_error_f(self, status_code):
        """重载write_error方法，并finish.

        Args:
            status_code: HTTP状态码
        """
        self.write_error(status_code)
        self.finish()

    def write_error(self, status_code, **kwargs):
        """重载write_error方法.

        Args:
            status_code: HTTP状态码
        """
        self.rsp = copy.deepcopy(self.RSP)
        self._transforms = []
        self._write_error(status_code)

    def _write_error(self, status_code):
        """write_error输出的准备.

        Args:
            status_code: HTTP状态码，根据此参数配置输出内容
        """
        status_code = str(status_code)
        status_code_b, status_code_s = status_code[:3], status_code[3:]

        self.rsp["code"] = status_code_b

        response = {
            "400": {
                "message": "Bad Request",
                "errors": {
                    '1': {"code": 4001,
                          "message": "Wrong Type Of Cardno"}
                }
            },
            "405": {
                "message": "Method Not Allowed"
            },
            "500": {
                "message": "Internal Server Error",
                "errors": {
                    '1': {"code": 5001,
                          "message": "Unknown Server Error"}
                }
            }
        }
        self.set_status(int(status_code_b))
        self.rsp["message"] = response.get(status_code_b, {}).get(
            "message", "Undefined Error")
        self.rsp["errors"] = response.get(status_code_b, {}).get(
            "errors", {}).get(status_code_s, {"code": status_code,
                                              "message": "Undefined Error"})

        self._write_json()


import sys
import matplotlib.pyplot as plt

def fib(n):
    fib.counter+=1
    if n == 0:
      return 0
    elif n == 1:
      return 1
    else:
      return fib(n-1) + fib(n-2)

fib.counter = 0
fib_mem.counter = 0

def plot(n, fun):
    x = []
    y = []
    for i in range(n):
        fun.counter = 0
        x.append(i)
        fun(i)
        y.append(fun.counter)
    plt.plot(x,y)
    plt.show()

def test(n):
    for i in range(n):
        assert(fib(i) == fib_mem(i))

if __name__ == '__main__':
    fib_mem(6)
    #test(18)
    #n = int(sys.argv[1])
    #plot(n, fib)
    #plot(n, fib_mem)


from __future__ import (absolute_import, unicode_literals, division,
                        print_function)

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import gridspec

from pyshell.util import ipydb
from pyshell.nosetests.postmortem import PostMortemScript
ipydb()

import glob
import os.path
import re

class SlopeDisplay(PostMortemScript):
    """Do the slope display."""

    module_ids = [ 'tests.test_zernike.test_zernike_slopes' ]

    filename_re = re.compile(r'([\w]+)_([\w]+)_([\d]+)_([\d-]+)')

    def parse_filename(self, filename):
        """docstring for parse_filename"""
        match = self.filename_re.match(filename)
        groups = match.groups()

        info = {
            'axes' : groups[0],
            'type' : groups[1],
            'n' : int(groups[2]),
            'm' : int(groups[3])
        }

        return info

    def do(self):
        """Actually do the hard work."""
        for key in self.test_data[self.module_ids[0]].keys():
            self.do_test(key)
        plt.show()

    def do_test(self, test):
        """Make displays for a specific test case."""
        filedata = {}
        filenames = self.test_data[self.module_ids[0]][test].keys()
        for filename in filenames:
            filedata[filename] = self.parse_filename(filename)

        row = { 'xs': 0, 'ys': 1 }
        col = { 'npy' : 1, 'pyt': 2, 'dif':3}
        gs = gridspec.GridSpec(2,5)
        figure = plt.figure(figsize=(8,4))

        for filename in filenames:
            info = filedata[filename]
            data = self.test_data[self.module_ids[0]][test][filename]
            col_pos = col.get(info['type'],0)
            row_pos = row.get(info['axes'],0)
            if info['axes'] == 'z':
                col_pos = 0
            axes = self.axes(figure, gs[row_pos,col_pos])
            self.label_axes(axes, "{:s}-{:s}-({:2d},{:2d})".format(
                info['axes'][0].capitalize(), info['type'], info['n'], info['m']
            ))
            axes.imshow(data)

    def axes(self, figure, subplotspec):
        """Return a setup axes from a subplotspec"""
        ax = figure.add_subplot(subplotspec)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
        ax.set_frame_on(False)
        return ax

    def label_axes(self, ax, label):
        """Label axes above."""
        ax.text(0.5, 1.0, label, transform=ax.transAxes, va='bottom', ha='center')

if __name__ == '__main__':
    SlopeDisplay.script()


from app import models, test
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from .database import engine

models.Base.metadata.create_all(bind=engine)
app = FastAPI()

origins = [
    "*"
]

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"]
)
app.include_router(test.router, tags=['Tests'], prefix='/api/tests')
@app.get("/api")
def root():
    return {"message": "welcome to this test fast api server"}


import os
import time
import pandas as pd
import numpy as np
from bifac import BiFac
from sklearn.metrics import normalized_mutual_info_score
from sklearn.metrics import f1_score

#preparation
logdir = "log/logdaisharin"
os.system("rm -rf log/logdaisharin")
df = pd.read_csv("data/edge_list.csv", header=None)
edge_list = [(i-1, j-1) for i, j in zip(df[0], df[1])]
weights = df[2]
ans = pd.read_csv("data/term_category.csv", header=None)[1]
N = ans.shape[0]
cnst = []
ncnst = 100
while len(cnst) < ncnst:
    n1 = np.random.randint(0, N)
    n2 = np.random.randint(0, N)
    if n1 == n2: continue
    if ans.ix[n1] == ans.ix[n2]:
        cnst.append((n1,n2))

#construct BiFac model
model = BiFac(K=8, edge_list=edge_list, weights=weights, learning_rate=1.0,
              lambda_r=0, constraint=cnst, lambda_c=0.)

#detect communities
start = time.time()
loss, z, theta1, theta2 = model.optimize(logdir=logdir, stop_threshold=10e-8,
                                        max_steps=5000)
sumtime = time.time() - start

com1, com2 = model.get_hard_community()

fvalue = f1_score(com1, ans)
nmi = normalized_mutual_info_score(com1, ans)

print("time", sumtime)
print("fvalue", fvalue)
print("nmi", nmi)


import pandas as pd
from mlxtend.frequent_patterns import apriori,association_rules

data=pd.read_csv("D:\\Data Science\\Assignment files\\Association Rules Assignment\\transactions_retail1.csv",header=None)
num_records=len(data)
data.describe()

#Imputation of NULL values#
data.isnull().sum()
data.shape
data.dropna().shape

data[1].fillna("HEART",inplace=True)
data[5].fillna("SET",inplace=True)
data[4].fillna("SET",inplace=True)
data[3].fillna("OF",inplace=True)
data[2].fillna("RED",inplace=True)

records=[]

for i in range(557):

    records.append([str(data.values[i,j]) for j in range (0,5)])

all_data_list= [i for item in records for i in item]

from collections import Counter
item_frequencies=Counter(all_data_list)

item_frequencies=sorted(item_frequencies.items(),key=lambda x:x[1])

records_series=pd.DataFrame(pd.Series(records))
records_series.columns=["transactions"]

# creating a dummy columns for the each item in each transactions ... Using column names as item name
X = records_series['transactions'].str.join(sep='*').str.get_dummies(sep='*')

frequent_itemsets = apriori(X,min_support=0.005, max_len=3,use_colnames = True)
frequent_itemsets.shape

# Most Frequent item sets based on support
frequent_itemsets.sort_values('support',ascending = False,inplace=True)
rules = association_rules(frequent_itemsets, metric="lift", min_threshold=1)
rules.shape

rules.head(10)
rules.sort_values('lift',ascending = False,inplace=True)


import configparser
import getopt
import sys

args = sys.argv[1:]
options = "hmo:f:"
long_options = ["help", "my_file", "output=", "config="]
default_ini = "sillypy.ini"
Config = configparser.ConfigParser()
Config.read(default_ini)
try:
    args, vals = getopt.getopt(args, options, long_options)
    for arg, val in args:
        if arg in ("-h", "--help"):
            print(
                "Usage: %s"
                % """
            Simple python script: takes arguments, prints out .ini file.
            -h, --help: this help
            -o, --output= : prints out value
            -f, --config=config.ini : use config.ini instead of default
            -m : returns own file name
"""
            )
            exit()
        elif arg in ("-m", "--my_file"):
            print("Displaying file_name:", sys.argv[0])
        elif arg in ("-o", "--output"):
            print(f"-o value is: {val}")
        elif arg in ("-f", "--config"):
            Config = configparser.ConfigParser()
            Config.read(val)
except getopt.error as err:
    print(str(err))
#
print(Config.sections())
for section in Config.sections():
    for option in Config[section]:
        print(option, Config.get(section, option))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Nov 26 20:09:34 2019

@author: alex
"""

from keras.models import Model
from keras.layers import Dense, GlobalAveragePooling2D, Dropout
from keras.applications.inception_v3 import InceptionV3, preprocess_input
from cfgTransfer import Config

import os
from scipy.io import wavfile
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from keras.layers import Conv2D, MaxPool2D, Flatten, LSTM
from keras.layers import TimeDistributed
from keras.models import Sequential
from keras.utils import to_categorical
from sklearn.utils.class_weight import compute_class_weight
from tqdm import tqdm
from python_speech_features import mfcc
import pickle
from keras.callbacks import ModelCheckpoint
import librosa
import time
#import MFCC

from PIL import Image
from keras.preprocessing.image import ImageDataGenerator

def pitchShift(data, sampling_rate, pitch_factor):
    return librosa.effects.pitch_shift(data, sampling_rate, pitch_factor)

def noiseInjection(data, noise_factor):
    noise = np.random.randn(len(data))
    augmented_data = data + noise_factor * noise
    # Cast back to same data type
    augmented_data = augmented_data.astype(type(data[0]))
    return augmented_data

def timeShift(data, sampling_rate, shift_max, shift_direction):
    shift = np.random.randint(sampling_rate * shift_max)
    if shift_direction == 'right':
        shift = -shift
    elif shift_direction == 'both':
        direction = np.random.randint(0, 2)
        if direction == 1:
            shift = -shift

    augmented_data = np.roll(data, shift)
    # Set to silence for heading/ tailing
    if shift > 0:
        augmented_data[:shift] = 0
    else:
        augmented_data[shift:] = 0
    return augmented_data

def envelope(y, rate, threshold):
    mask = []
    y = pd.Series(y).apply(np.abs)
    y_mean = y.rolling(window=int(rate/10), min_periods=1, center=True).mean() # if center != True, then it sets the value at the front of the window
    for mean in y_mean:
        if mean > threshold:
            mask.append(True)
        else:
            mask.append(False)
    return mask

def check_data():
    if os.path.isfile(config.p_path):
        print('Loading existing data for {} model'.format(config.mode))
        with open(config.p_path, 'rb') as handle:
            tmp = pickle.load(handle)
            return tmp
    else:
        return None

def build_rand_feat():
#    tmp = check_data()
#    if tmp:
#        return tmp.data[0], tmp.data[1]
    filePeaksPicked = {}
    for f in df.index:
        filePeaksPicked[f] = []
    x = []
    y = []
    _min, _max = float('inf'), -float('inf')
    totalNumPeaks = 0
    nextFile = False
    for _ in tqdm(range(n_samples)):
        rand_class = np.random.choice(class_dist.index, p=prob_dist)
        file = np.random.choice(df[df.label==rand_class].index)
        rate, wav = wavfile.read('cleanReducedLessTest/'+file)

        print("file = ", file)
        signal = wav
        peaks = []
        peakIndices = []
        numPeaks = 0
        peakValue = np.amax(signal)
        originalPeakValue = peakValue
        while peakValue > 0.5*originalPeakValue:
            peakValueIndex = np.argmax(signal)
            minWin = peakValueIndex - int((rate/10)*0.3)
            maxWin = peakValueIndex + int((rate/10)*0.7)
            signal = np.append(signal[:minWin], signal[maxWin:])
            if signal.shape[0] < int(rate/10):
                break
            if maxWin > (wav.shape[0]-(int((rate/10)*0.8))):
                print("Peak too close to end of file")
            elif minWin < 0:
                print("Peak too close to start of file")
            else:
                peaks.append(minWin)
                numPeaks += 1
                peakIndices.append(peakValueIndex)
                peakValue = np.amax(signal)
        print("totalNumPeaks = ", totalNumPeaks)
        totalNumPeaks += numPeaks
        print("numPeaks = ", numPeaks)
        print("totalNumPeaks = ", totalNumPeaks)
        while True:
            if numPeaks == len(filePeaksPicked[file]):
                nextFile = True
                break
            peakChoiceIndex = np.random.randint(numPeaks)
            peakChoice = peakIndices[peakChoiceIndex]
            if peakChoice in filePeaksPicked[file]:
                continue
            else:
                filePeaksPicked[file].append(peakChoice)
                break

        if nextFile == True:
            nextFile = False
            _ -= 1
            continue

        print(peakChoice)
        startIndex = peaks[peakChoiceIndex]
        sample = wav[startIndex:(startIndex + config.step)]

#        rand_index = np.random.randint(0, wav.shape[0] - config.step)
#        sample = wav[rand_index:rand_index + config.step]
        ## shift at most 3 half steps left or right

#        sample = pitchShift(sample, rate, (np.random.randint(4) - 1))
#        sample = timeShift(sample, rate, 0.02, 'both')
#        sample = noiseInjection(sample, np.random.uniform(0, 0.02))
#        mask = envelope(sample, rate, 0.0005)
        # Normalize before taking mfcc
#        maxValue = np.amax(sample)
#        minValue = np.amin(sample)
#        sample = (sample - minValue) / (maxValue - minValue)
        print("sample = ", sample)
        x_sample = mfcc(sample, rate,
                        numcep=config.nfeat, nfilt=config.nfilt,
                        nfft=config.nfft)
        minValueMfcc = np.amin(x_sample)
        maxValueMfcc = np.amax(x_sample)
        x_sample = (x_sample - minValueMfcc)/ (maxValueMfcc - minValueMfcc)
        print("sample = ", x_sample)
#        x_sample = MFCC.mfccNew(BigSample = sample, DESIRED_WINDOW_SIZE = 400, \
#                              WINDOW_STEP = config.step, ZeroPadWinSize = config.nfft, \
#                              SampleRate = rate, n_mels = config.nfilt, n_coeff = config.nfeat)
        #mfccNew(BigSample, DESIRED_WINDOW_SIZE, WINDOW_STEP, ZeroPadWinSize, SampleRate, n_mels, n_coeff):
        _min = min(np.amin(x_sample), _min)
        _max = max(np.amax(x_sample), _max)

        if config.mode == 'trans':
                x_sample = np.expand_dims(x_sample, axis=2)
                data = np.zeros((9,13,3))
                print("x_sample.shape = ", x_sample.shape)
                data[0:9, 0:13] = x_sample

                print("data shape = ", data.shape)
                print(data)
                img = Image.fromarray(data, 'RGB')
                img.save("Images/" + file + str(peakChoice) + '.png')

        x.append(x_sample)
        print("x_sample = ", x_sample)
        y.append(classes.index(rand_class))
    avgNumPeaks = totalNumPeaks / n_samples
    print("totalNumberPeaks = ", totalNumPeaks)
    print("Average number of peaks = ", avgNumPeaks)
#    config.min = _min
#    config.max = _max
    x, y = np.array(x), np.array(y)
#    print("x shape = ", x.shape)
#    x = (x - _min) / (_max - _min)
    print("x.shape = ", x.shape)
    print("y.shape = ", y.shape)

    if config.mode == 'conv':
        print("x.shape[0] = ", x.shape[0])
        print("x.shape[1] = ", x.shape[1])
        print("x.shape[2] = ", x.shape[2])
        x = x.reshape(x.shape[0], x.shape[1], x.shape[2], 1)
    elif config.mode == 'time':
#        x = x.reshape(x.shape[0], x.shape[1], x.shape[2])
        pass
    elif config.mode == 'trans':
#        x = np.expand_dims(x, axis=3)
#        zeroPadded = np.zeros((x.shape[0],299,299,3))
#        zeroPadded[:x.shape[0],:x.shape[1], :x.shape[2]] = x
#        x = zeroPadded.reshape(zeroPadded.shape[0], zeroPadded.shape[1],
#                               zeroPadded.shape[2], zeroPadded.shape[3])
        pass
    y = to_categorical(y, num_classes=3)

    config.data = (x, y)
    with open(config.p_path, 'wb') as handle:
        pickle.dump(config, handle, protocol=2)

    return x, y

def get_conv_model():
    model = Sequential()
    model.add(Conv2D(16, (3, 3), activation='relu', strides=(1,1),
                     padding='same', input_shape=input_shape))
    model.add(Conv2D(32, (3, 3), activation='relu', strides=(1,1),
                     padding='same'))
    model.add(Conv2D(64, (3, 3), activation='relu', strides=(1,1),
                     padding='same'))
    model.add(Conv2D(128, (3, 3), activation='relu', strides=(1,1),
                     padding='same'))
    model.add(MaxPool2D((2, 2)))
    model.add(Dropout(0.5))
    model.add(Flatten())
    model.add(Dense(128, activation='relu'))
    model.add(Dense(64, activation='relu'))
    model.add(Dense(3, activation='softmax'))
    model.summary()
    model.compile(loss='categorical_crossentropy',
                  optimizer='adam',
                  metrics=['acc'])
    return model

def get_recurrent_model():
    model = Sequential()
    model.add(LSTM(128, return_sequences=True,
                     input_shape=input_shape))
    model.add(LSTM(128, return_sequences=True))
    model.add(Dropout(0.5))
    model.add(TimeDistributed(Dense(64, activation='relu')))
    model.add(TimeDistributed(Dense(32, activation='relu')))
    model.add(TimeDistributed(Dense(16, activation='relu')))
    model.add(TimeDistributed(Dense(8, activation='relu')))
    model.add(Flatten())
    model.add(Dense(3, activation='softmax'))
    model.summary()
    model.compile(loss='categorical_crossentropy',
                  optimizer='adam',
                  metrics=['acc'])
    return model

df = pd.read_csv('materialsReduced.csv')
df.set_index('filename', inplace=True)

for f in df.index:
    rate, signal = wavfile.read('cleanReducedLessTest/'+f)
    df.at[f, 'length'] = signal.shape[0]/rate

classes = list(np.unique(df.label))
class_dist = df.groupby(['label'])['length'].mean()

#n_samples = int(1 * int(df['length'].sum()/0.1)) # number of samples dividing samples into 10th of seconds
n_samples = 3 * 140
prob_dist = class_dist / class_dist.sum()
choices = np.random.choice(class_dist.index, p=prob_dist)

fig, ax = plt.subplots()
ax.set_title('Class Distribution', y=1.08)
ax.pie(class_dist, labels=class_dist.index, autopct='%1.1f%%',
       shadow=False, startangle=90)
ax.axis('equal')
plt.show()

config = Config(mode='conv')

x, y = build_rand_feat()
y_flat = np.argmax(y, axis=1)

print("x.shape = ", x.shape)

if config.mode == 'conv':
    print('conv')
    input_shape = (x.shape[1], x.shape[2], 1)
    model = get_conv_model()

if config.mode == 'time':
    print('time')
    input_shape = (x.shape[1], x.shape[2])
    model = get_recurrent_model()

if config.mode == 'trans':
    CLASSES = 3
    #setup model
    base_model = InceptionV3(weights='imagenet', include_top=False)

    x = base_model.output
    x = GlobalAveragePooling2D(name='avg_pool')(x)
    x = Dropout(0.4)(x)
    predictions = Dense(CLASSES, activation='softmax')(x)
    model = Model(inputs=base_model.input, outputs=predictions)

    #transfer learning
    for layer in base_model.layers:
        layer.trainable = False

    model.compile(optimizer='rmsprop',
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    TRAIN_DIR = 'train'
    TEST_DIR = 'test'

    WIDTH = 299
    HEIGHT = 299
    BATCH_SIZE = 32

    # data prep
    train_datagen = ImageDataGenerator(
        preprocessing_function=preprocess_input,
        rotation_range=40,
        width_shift_range=0.2,
        height_shift_range=0.2,
        shear_range=0.2,
        zoom_range=0.2,
        horizontal_flip=True,
        fill_mode='nearest')

    validation_datagen = ImageDataGenerator(
        preprocessing_function=preprocess_input,
        rotation_range=40,
        width_shift_range=0.2,
        height_shift_range=0.2,
        shear_range=0.2,
        zoom_range=0.2,
        horizontal_flip=True,
        fill_mode='nearest')

    train_generator = train_datagen.flow_from_directory(
        TRAIN_DIR,
        target_size=(HEIGHT, WIDTH),
    		batch_size=BATCH_SIZE,
    		class_mode='categorical')

    validation_generator = validation_datagen.flow_from_directory(
        TEST_DIR,
        target_size=(HEIGHT, WIDTH),
        batch_size=BATCH_SIZE,
        class_mode='categorical')

    xSample, ySample = next(train_generator)
    print("xSample = ", xSample)
    print("ySample = ", ySample)

    EPOCHS = 5
    BATCH_SIZE = 32
    STEPS_PER_EPOCH = 320
    VALIDATION_STEPS = 64

    MODEL_FILE = 'filename.model'

    history = model.fit_generator(
        train_generator,
        epochs=EPOCHS,
        steps_per_epoch=STEPS_PER_EPOCH,
        validation_data=validation_generator,
        validation_steps=VALIDATION_STEPS)

    model.save(MODEL_FILE)

if config.mode != 'trans':
    class_weight = compute_class_weight('balanced',
                                        np.unique(y_flat),
                                        y_flat)

    checkpoint = ModelCheckpoint(config.model_path, monitor='val_acc',
                                 verbose=1, mode='max',
                                 save_best_only=True,
                                 save_weights_only=False,
                                 period=1)

    startTrainTime = time.time()

    model.fit(x, y, epochs=100, batch_size=32,
                  shuffle=True,
                  class_weight=class_weight,
                  validation_split=0.4,
                  callbacks=[checkpoint])

    finishTrainTime = time.time()
    totalTrainTime = finishTrainTime - startTrainTime
    print(totalTrainTime)

    model.save(config.model_path)

    #model_json = model.to_json()
    #with open("Data/model.json", "w") as json_file:
    #    json_file.write(simplejson.dumps(simplejson.loads(model_json), indent=4))
    #
    #model.save_weights("Data/model.h5")
    #print("Saved model to disk")

    model.save('Data/model_' + config.mode + 'ReducedLessTest.h5')
    print("Saved model to disk")
    print(config.min)
    print(config.max)