import numpy as np

from .._fiff.constants import FIFF
from .._fiff.pick import pick_info
from ..baseline import _log_rescale, rescale
from ..epochs import Epochs
from ..event import make_fixed_length_events
from ..evoked import EvokedArray
from ..fixes import _safe_svd
from ..label import BiHemiLabel, Label
from ..parallel import parallel_func
from ..source_estimate import _make_stc
from ..time_frequency.multitaper import (
    _compute_mt_params,
    _mt_spectra,
    _psd_from_mt,
    _psd_from_mt_adaptive,
)
from ..time_frequency.tfr import cwt, morlet
from ..utils import ProgressBar, _check_option, _pl, _validate_type, logger, verbose
from .inverse import (
    INVERSE_METHODS,
    _assemble_kernel,
    _check_or_prepare,
    _check_ori,
    _pick_channels_inverse_operator,
    _subject_from_inverse,
    combine_xyz,
)

def _restrict_K_to_lbls(labels, K, noise_norm, vertno, pick_ori):
    """Use labels to choose desired sources in the kernel."""
    verts_to_use = [[], []]
    # create mask for K by compiling original vertices from vertno in labels
    for ii in range(len(labels)):
        lab = labels[ii]
        # handle BiHemi labels; ok so long as no overlap w/ single hemi labels
        if lab.hemi == "both":
            l_verts = np.intersect1d(vertno[0], lab.lh.vertices)
            r_verts = np.intersect1d(vertno[1], lab.rh.vertices)  # output sorted
            verts_to_use[0] += list(l_verts)
            verts_to_use[1] += list(r_verts)
        else:
            hidx = 0 if lab.hemi == "lh" else 1
            verts = np.intersect1d(vertno[hidx], lab.vertices)
            verts_to_use[hidx] += list(verts)

    # check that we don't have overlapping vertices in our labels
    for ii in range(2):
        if len(np.unique(verts_to_use[ii])) != len(verts_to_use[ii]):
            raise RuntimeError(
                "Labels cannot have overlapping vertices. "
                "Please select labels with unique vertices "
                "and try again."
            )

    # turn original vertex numbers from vertno into indices for K
    K_mask = np.searchsorted(vertno[0], verts_to_use[0])
    r_kmask = np.searchsorted(vertno[1], verts_to_use[1]) + len(vertno[0])
    K_mask = np.hstack((K_mask, r_kmask))

    # record which original vertices are at each index in out_K
    hemis = ("lh", "rh")
    ki_keys = [
        (hemis[hi], verts_to_use[hi][ii])
        for hi in range(2)
        for ii in range(len(verts_to_use[hi]))
    ]
    ki_vals = list(range(len(K_mask)))
    k_idxs = dict(zip(ki_keys, ki_vals))

    # mask K, handling the orientation issue
    len_allverts = len(vertno[0]) + len(vertno[1])
    if len(K) == len_allverts:
        assert pick_ori == "normal"
        out_K = K[K_mask]
    else:
        # here, K = [x0, y0, z0, x1, y1, z1 ...]
        # we need to drop x, y and z of unused vertices
        assert not pick_ori == "normal", pick_ori
        assert len(K) == 3 * len_allverts, (len(K), len_allverts)
        out_len = len(K_mask) * 3
        out_K = K[0:out_len]  # get the correct-shaped array
        for di in range(3):
            K_pick = K[di::3]
            out_K[di::3] = K_pick[K_mask]  # set correct values for out

    out_vertno = verts_to_use
    if noise_norm is not None:
        out_nn = noise_norm[K_mask]
    else:
        out_nn = None

    return out_K, out_nn, out_vertno, k_idxs

def _prepare_source_params(
    inst,
    inverse_operator,
    label=None,
    lambda2=1.0 / 9.0,
    method="dSPM",
    nave=1,
    pca=True,
    pick_ori="normal",
    prepared=False,
    method_params=None,
    use_cps=True,
):
    """Prepare inverse operator and params for spectral / TFR analysis."""
    inv = _check_or_prepare(
        inverse_operator, nave, lambda2, method, method_params, prepared
    )

    #
    #   Pick the correct channels from the data
    #
    sel = _pick_channels_inverse_operator(inst.ch_names, inv)
    logger.info("Picked %d channels from the data" % len(sel))
    logger.info("Computing inverse...")
    #
    #   Simple matrix multiplication followed by combination of the
    #   three current components
    #
    #   This does all the data transformations to compute the weights for the
    #   eigenleads
    #
    # K shape: (3 x n_sources, n_channels) or (n_sources, n_channels)
    # noise_norm shape: (n_sources, 1)
    # vertno: [lh_verts, rh_verts]

    k_idxs = None
    if not isinstance(label, (Label, BiHemiLabel)):
        whole_K, whole_noise_norm, whole_vertno, _ = _assemble_kernel(
            inv, None, method, pick_ori, use_cps=use_cps
        )
        if isinstance(label, list):
            K, noise_norm, vertno, k_idxs = _restrict_K_to_lbls(
                label, whole_K, whole_noise_norm, whole_vertno, pick_ori
            )
        else:
            assert not label
            K, noise_norm, vertno = whole_K, whole_noise_norm, whole_vertno
    elif isinstance(label, (Label, BiHemiLabel)):
        K, noise_norm, vertno, _ = _assemble_kernel(
            inv, label, method, pick_ori, use_cps=use_cps
        )

    if pca:
        U, s, Vh = _safe_svd(K, full_matrices=False)
        rank = np.sum(s > 1e-8 * s[0])
        K = s[:rank] * U[:, :rank]
        Vh = Vh[:rank]
        logger.info("Reducing data rank %d -> %d" % (len(s), rank))
    else:
        Vh = None
    is_free_ori = inverse_operator["source_ori"] == FIFF.FIFFV_MNE_FREE_ORI

    return K, sel, Vh, vertno, is_free_ori, noise_norm, k_idxs

@verbose
def source_band_induced_power(
    epochs,
    inverse_operator,
    bands,
    label=None,
    lambda2=1.0 / 9.0,
    method="dSPM",
    nave=1,
    n_cycles=5,
    df=1,
    use_fft=False,
    decim=1,
    baseline=None,
    baseline_mode="logratio",
    pca=True,
    n_jobs=None,
    prepared=False,
    method_params=None,
    use_cps=True,
    *,
    verbose=None,
):
    """Compute source space induced power in given frequency bands.

    Parameters
    ----------
    epochs : instance of Epochs
        The epochs.
    inverse_operator : instance of InverseOperator
        The inverse operator.
    bands : dict
        Example : bands = dict(alpha=[8, 9]).
    label : Label | list of Label
        Restricts the source estimates to a given label or list of labels. If
        labels are provided in a list, power will be averaged over vertices.
    lambda2 : float
        The regularization parameter of the minimum norm.
    method : "MNE" | "dSPM" | "sLORETA" | "eLORETA"
        Use minimum norm, dSPM (default), sLORETA, or eLORETA.
    nave : int
        The number of averages used to scale the noise covariance matrix.
    n_cycles : float | array of float
        Number of cycles. Fixed number or one per frequency.
    df : float
        Delta frequency within bands.
    use_fft : bool
        Do convolutions in time or frequency domain with FFT.
    decim : int
        Temporal decimation factor.
    baseline : None (default) or tuple, shape (2,)
        The time interval to apply baseline correction. If None do not apply
        it. If baseline is (a, b) the interval is between "a (s)" and "b (s)".
        If a is None the beginning of the data is used and if b is None then b
        is set to the end of the interval. If baseline is equal to (None, None)
        all the time interval is used.
    baseline_mode : 'mean' | 'ratio' | 'logratio' | 'percent' | 'zscore' | 'zlogratio'
        Perform baseline correction by

        - subtracting the mean of baseline values ('mean')
        - dividing by the mean of baseline values ('ratio')
        - dividing by the mean of baseline values and taking the log
          ('logratio')
        - subtracting the mean of baseline values followed by dividing by
          the mean of baseline values ('percent')
        - subtracting the mean of baseline values and dividing by the
          standard deviation of baseline values ('zscore')
        - dividing by the mean of baseline values, taking the log, and
          dividing by the standard deviation of log baseline values
          ('zlogratio')

    pca : bool
        If True, the true dimension of data is estimated before running
        the time-frequency transforms. It reduces the computation times
        e.g. with a dataset that was maxfiltered (true dim is 64).
    %(n_jobs)s
    prepared : bool
        If True, do not call :func:`prepare_inverse_operator`.
    method_params : dict | None
        Additional options for eLORETA. See Notes of :func:`apply_inverse`.

        .. versionadded:: 0.16
    %(use_cps_restricted)s

        .. versionadded:: 0.20
    %(verbose)s

    Returns
    -------
    stcs : dict of SourceEstimate (or VolSourceEstimate)
        The estimated source space induced power estimates in shape
        (n_vertices, n_frequencies, n_samples) if label=None or label=label.
        For lists of one or more labels, the induced power estimate has shape
        (n_labels, n_frequencies, n_samples).
    """  # noqa: E501
    _check_option("method", method, INVERSE_METHODS)

    freqs = np.concatenate(
        [np.arange(band[0], band[1] + df / 2.0, df) for _, band in bands.items()]
    )

    powers, _, vertno = _source_induced_power(
        epochs,
        inverse_operator,
        freqs,
        label=label,
        lambda2=lambda2,
        method=method,
        nave=nave,
        n_cycles=n_cycles,
        decim=decim,
        use_fft=use_fft,
        pca=pca,
        n_jobs=n_jobs,
        with_plv=False,
        prepared=prepared,
        method_params=method_params,
        use_cps=use_cps,
    )

    Fs = epochs.info["sfreq"]  # sampling in Hz
    stcs = dict()

    subject = _subject_from_inverse(inverse_operator)
    _log_rescale(baseline, baseline_mode)  # for early failure
    for name, band in bands.items():
        idx = [k for k, f in enumerate(freqs) if band[0] <= f <= band[1]]

        # average power in band + mean over epochs
        power = np.mean(powers[:, idx, :], axis=1)

        # Run baseline correction
        power = rescale(
            power,
            epochs.times[::decim],
            baseline,
            baseline_mode,
            copy=False,
            verbose=False,
        )

        tmin = epochs.times[0]
        tstep = float(decim) / Fs
        stc = _make_stc(
            power,
            vertices=vertno,
            tmin=tmin,
            tstep=tstep,
            subject=subject,
            src_type=inverse_operator["src"].kind,
        )
        stcs[name] = stc

        logger.info("[done]")

    return stcs

def _prepare_tfr(data, decim, pick_ori, Ws, K, source_ori):
    """Prepare TFR source localization."""
    n_times = data[:, :, ::decim].shape[2]
    n_freqs = len(Ws)
    n_sources = K.shape[0]
    is_free_ori = False
    if source_ori == FIFF.FIFFV_MNE_FREE_ORI and pick_ori is None:
        is_free_ori = True
        n_sources //= 3

    shape = (n_sources, n_freqs, n_times)
    return shape, is_free_ori

@verbose
def _compute_pow_plv(
    data,
    K,
    sel,
    Ws,
    source_ori,
    use_fft,
    Vh,
    with_power,
    with_plv,
    pick_ori,
    decim,
    noise_norm=None,
    verbose=None,
):
    """Aux function for induced power and PLV."""
    shape, is_free_ori = _prepare_tfr(data, decim, pick_ori, Ws, K, source_ori)
    power = np.zeros(shape, dtype=np.float64)  # power or raw TFR
    # phase lock
    plv = np.zeros(shape, dtype=np.complex128) if with_plv else None

    for epoch in data:
        epoch = epoch[sel]  # keep only selected channels

        if Vh is not None:
            epoch = np.dot(Vh, epoch)  # reducing data rank

        power_e, plv_e = _single_epoch_tfr(
            data=epoch,
            is_free_ori=is_free_ori,
            K=K,
            Ws=Ws,
            use_fft=use_fft,
            decim=decim,
            shape=shape,
            with_plv=with_plv,
            with_power=with_power,
        )

        power += power_e
        if with_plv:
            plv += plv_e

    if noise_norm is not None:
        power *= noise_norm[:, :, np.newaxis] ** 2

    return power, plv

def _single_epoch_tfr(
    data, is_free_ori, K, Ws, use_fft, decim, shape, with_plv, with_power
):
    """Compute single trial TFRs, either ITC, power or raw TFR."""
    tfr_e = np.zeros(shape, dtype=np.float64)  # power or raw TFR
    # phase lock
    plv_e = np.zeros(shape, dtype=np.complex128) if with_plv else None
    n_sources, _, n_times = shape
    for f, w in enumerate(Ws):
        tfr_ = cwt(data, [w], use_fft=use_fft, decim=decim)
        tfr_ = np.asfortranarray(tfr_.reshape(len(data), -1))

        # phase lock and power at freq f
        if with_plv:
            plv_f = np.zeros((n_sources, n_times), dtype=np.complex128)

        tfr_f = np.zeros((n_sources, n_times), dtype=np.float64)

        for k, t in enumerate([np.real(tfr_), np.imag(tfr_)]):
            sol = np.dot(K, t)

            sol_pick_normal = sol
            if is_free_ori:
                sol_pick_normal = sol[2::3]

            if with_plv:
                if k == 0:  # real
                    plv_f += sol_pick_normal
                else:  # imag
                    plv_f += 1j * sol_pick_normal

            if is_free_ori:
                logger.debug("combining the current components...")
                sol = combine_xyz(sol, square=with_power)
            elif with_power:
                sol *= sol
            tfr_f += sol
            del sol

        tfr_e[:, f, :] += tfr_f
        del tfr_f

        if with_plv:
            plv_f /= np.abs(plv_f)
            plv_e[:, f, :] += plv_f
            del plv_f

    return tfr_e, plv_e

def _get_label_power(power, labels, vertno, k_idxs):
    """Average power across vertices in labels."""
    (_, ps1, ps2) = power.shape
    # construct out array with correct shape
    out_power = np.zeros(shape=(len(labels), ps1, ps2))

    # for each label, compile list of vertices we want
    for li in np.arange(len(labels)):
        lab = labels[li]
        hemis = ("lh", "rh")
        all_vnums = [[], []]
        if lab.hemi == "both":
            all_vnums[0] = np.intersect1d(lab.lh.vertices, vertno[0])
            all_vnums[1] = np.intersect1d(lab.rh.vertices, vertno[1])
        else:
            assert lab.hemi == "lh" or lab.hemi == "rh"
            h_id = 0 if lab.hemi == "lh" else 1
            all_vnums[h_id] = np.intersect1d(vertno[h_id], lab.vertices)

        verts = [(hemis[hi], vn) for hi in range(2) for vn in all_vnums[hi]]

        # restrict power to relevant vertices in label
        lab_mask = np.array([False] * len(power))
        for vert in verts:
            lab_mask[k_idxs[vert]] = True  # k_idxs[vert] gives power row index
        lab_power = power[lab_mask]  # only pass through rows we want
        assert lab_power.shape == (len(verts), ps1, ps2)

        # set correct out values for label
        out_power[li, :, :] = np.mean(lab_power, axis=0)

    assert out_power.shape == (len(labels), ps1, ps2)
    return out_power

@verbose
def _source_induced_power(
    epochs,
    inverse_operator,
    freqs,
    label=None,
    lambda2=1.0 / 9.0,
    method="dSPM",
    nave=1,
    n_cycles=5,
    decim=1,
    use_fft=False,
    pca=True,
    pick_ori="normal",
    n_jobs=None,
    with_plv=True,
    zero_mean=False,
    prepared=False,
    method_params=None,
    use_cps=True,
    verbose=None,
):
    """Aux function for source induced power."""
    if label:
        _validate_type(
            label,
            types=(Label, BiHemiLabel, list, tuple, None),
            type_name=("Label or BiHemiLabel", "list of labels", "None"),
        )
        if isinstance(label, (list, tuple)):
            for item in label:
                _validate_type(
                    item,
                    types=(Label, BiHemiLabel),
                    type_name=("Label or BiHemiLabel"),
                )
            if len(label) > 1 and with_plv:
                raise RuntimeError(
                    "Phase-locking value cannot be calculated "
                    "when averaging induced power within "
                    "labels. Please set `with_plv` to False, pass a "
                    "single `label=label`, or set `label=None`."
                )

    epochs_data = epochs.get_data(copy=False)
    K, sel, Vh, vertno, is_free_ori, noise_norm, k_id = _prepare_source_params(
        inst=epochs,
        inverse_operator=inverse_operator,
        label=label,
        lambda2=lambda2,
        method=method,
        nave=nave,
        pca=pca,
        pick_ori=pick_ori,
        prepared=prepared,
        method_params=method_params,
        use_cps=use_cps,
    )

    inv = inverse_operator
    parallel, my_compute_source_tfrs, n_jobs = parallel_func(
        _compute_pow_plv, n_jobs, max_jobs=len(epochs_data)
    )
    Fs = epochs.info["sfreq"]  # sampling in Hz

    logger.info("Computing source power ...")

    Ws = morlet(Fs, freqs, n_cycles=n_cycles, zero_mean=zero_mean)

    out = parallel(
        my_compute_source_tfrs(
            data=data,
            K=K,
            sel=sel,
            Ws=Ws,
            source_ori=inv["source_ori"],
            use_fft=use_fft,
            Vh=Vh,
            with_plv=with_plv,
            with_power=True,
            pick_ori=pick_ori,
            decim=decim,
            noise_norm=noise_norm,
        )
        for data in np.array_split(epochs_data, n_jobs)
    )
    power = sum(o[0] for o in out)  # power shape: (n_verts, n_freqs, n_samps)
    power /= len(epochs_data)  # average power over epochs

    if isinstance(label, (Label, BiHemiLabel)):
        logger.info(
            f"Outputting power for {len(power)} vertices in label {label.name}."
        )
    elif isinstance(label, list):
        power = _get_label_power(power, label, vertno, k_id)
        logger.info(
            "Averaging induced power across vertices within labels "
            f"for {len(label)} label{_pl(label)}."
        )
    else:
        assert not label

    if with_plv:
        plv = sum(o[1] for o in out)
        plv = np.abs(plv)
        plv /= len(epochs_data)  # average power over epochs
    else:
        plv = None

    return power, plv, vertno

@verbose
def source_induced_power(
    epochs,
    inverse_operator,
    freqs,
    label=None,
    lambda2=1.0 / 9.0,
    method="dSPM",
    nave=1,
    n_cycles=5,
    decim=1,
    use_fft=False,
    pick_ori=None,
    baseline=None,
    baseline_mode="logratio",
    pca=True,
    n_jobs=None,
    *,
    return_plv=True,
    zero_mean=False,
    prepared=False,
    method_params=None,
    use_cps=True,
    verbose=None,
):
    """Compute induced power and phase lock.

    Computation can optionally be restricted in a label.

    Parameters
    ----------
    epochs : instance of Epochs
        The epochs.
    inverse_operator : instance of InverseOperator
        The inverse operator.
    freqs : array
        Array of frequencies of interest.
    label : Label | list of Label
        Restricts the source estimates to a given label or list of labels. If
        labels are provided in a list, power will be averaged over vertices within each
        label.
    lambda2 : float
        The regularization parameter of the minimum norm.
    method : "MNE" | "dSPM" | "sLORETA" | "eLORETA"
        Use minimum norm, dSPM (default), sLORETA, or eLORETA.
    nave : int
        The number of averages used to scale the noise covariance matrix.
    n_cycles : float | array of float
        Number of cycles. Fixed number or one per frequency.
    decim : int
        Temporal decimation factor.
    use_fft : bool
        Do convolutions in time or frequency domain with FFT.
    pick_ori : None | "normal"
        If "normal", rather than pooling the orientations by taking the norm,
        only the radial component is kept. This is only implemented
        when working with loose orientations.
    baseline : None (default) or tuple of length 2
        The time interval to apply baseline correction.
        If None do not apply it. If baseline is (a, b)
        the interval is between "a (s)" and "b (s)".
        If a is None the beginning of the data is used
        and if b is None then b is set to the end of the interval.
        If baseline is equal to (None, None) all the time
        interval is used.
    baseline_mode : 'mean' | 'ratio' | 'logratio' | 'percent' | 'zscore' | 'zlogratio'
        Perform baseline correction by

        - subtracting the mean of baseline values ('mean')
        - dividing by the mean of baseline values ('ratio')
        - dividing by the mean of baseline values and taking the log
          ('logratio')
        - subtracting the mean of baseline values followed by dividing by
          the mean of baseline values ('percent')
        - subtracting the mean of baseline values and dividing by the
          standard deviation of baseline values ('zscore')
        - dividing by the mean of baseline values, taking the log, and
          dividing by the standard deviation of log baseline values
          ('zlogratio')

    pca : bool
        If True, the true dimension of data is estimated before running
        the time-frequency transforms. It reduces the computation times
        e.g. with a dataset that was maxfiltered (true dim is 64).
    %(n_jobs)s
    return_plv : bool
        If True, return the phase-locking value array. Else, only return power.

        .. versionadded:: 1.6
    zero_mean : bool
        Make sure the wavelets are zero mean.
    prepared : bool
        If True, do not call :func:`prepare_inverse_operator`.
    method_params : dict | None
        Additional options for eLORETA. See Notes of :func:`apply_inverse`.
    %(use_cps_restricted)s

        .. versionadded:: 0.20
    %(verbose)s

    Returns
    -------
    power : array
        The induced power array with shape (n_sources, n_freqs, n_samples) if
        label=None or label=label. For lists of one or more labels, the induced
        power estimate has shape (n_labels, n_frequencies, n_samples).
    plv : array
        The phase-locking value array with shape (n_sources, n_freqs,
        n_samples). Only returned if ``return_plv=True``.
    """  # noqa: E501
    _check_option("method", method, INVERSE_METHODS)
    _check_ori(pick_ori, inverse_operator["source_ori"], inverse_operator["src"])

    power, plv, vertno = _source_induced_power(
        epochs,
        inverse_operator,
        freqs,
        label=label,
        lambda2=lambda2,
        method=method,
        nave=nave,
        n_cycles=n_cycles,
        decim=decim,
        use_fft=use_fft,
        pick_ori=pick_ori,
        pca=pca,
        n_jobs=n_jobs,
        with_plv=return_plv,
        method_params=method_params,
        zero_mean=zero_mean,
        prepared=prepared,
        use_cps=use_cps,
    )

    # Run baseline correction
    power = rescale(power, epochs.times[::decim], baseline, baseline_mode, copy=False)

    outs = (power, plv) if return_plv else power
    return outs

@verbose
def compute_source_psd(
    raw,
    inverse_operator,
    lambda2=1.0 / 9.0,
    method="dSPM",
    tmin=0.0,
    tmax=None,
    fmin=0.0,
    fmax=200.0,
    n_fft=2048,
    overlap=0.5,
    pick_ori=None,
    label=None,
    nave=1,
    pca=True,
    prepared=False,
    method_params=None,
    inv_split=None,
    bandwidth="hann",
    adaptive=False,
    low_bias=False,
    n_jobs=None,
    return_sensor=False,
    dB=False,
    *,
    verbose=None,
):
    """Compute source power spectral density (PSD).

    Parameters
    ----------
    raw : instance of Raw
        The raw data.
    inverse_operator : instance of InverseOperator
        The inverse operator.
    lambda2 : float
        The regularization parameter.
    method : "MNE" | "dSPM" | "sLORETA"
        Use minimum norm, dSPM (default), sLORETA, or eLORETA.
    tmin : float
        The beginning of the time interval of interest (in seconds).
        Use 0. for the beginning of the file.
    tmax : float | None
        The end of the time interval of interest (in seconds). If None
        stop at the end of the file.
    fmin : float
        The lower frequency of interest.
    fmax : float
        The upper frequency of interest.
    n_fft : int
        Window size for the FFT. Should be a power of 2.
    overlap : float
        The overlap fraction between windows. Should be between 0 and 1.
        0 means no overlap.
    pick_ori : None | "normal"
        If "normal", rather than pooling the orientations by taking the norm,
        only the radial component is kept. This is only implemented
        when working with loose orientations.
    label : Label
        Restricts the source estimates to a given label.
    nave : int
        The number of averages used to scale the noise covariance matrix.
    pca : bool
        If True, the true dimension of data is estimated before running
        the time-frequency transforms. It reduces the computation times
        e.g. with a dataset that was maxfiltered (true dim is 64).
    prepared : bool
        If True, do not call :func:`prepare_inverse_operator`.
    method_params : dict | None
        Additional options for eLORETA. See Notes of :func:`apply_inverse`.

        .. versionadded:: 0.16
    inv_split : int or None
        Split inverse operator into inv_split parts in order to save memory.

        .. versionadded:: 0.17
    bandwidth : float | str
        The bandwidth of the multi taper windowing function in Hz.
        Can also be a string (e.g., 'hann') to use a single window.

        For backward compatibility, the default is 'hann'.

        .. versionadded:: 0.17
    adaptive : bool
        Use adaptive weights to combine the tapered spectra into PSD
        (slow, use n_jobs >> 1 to speed up computation).

        .. versionadded:: 0.17
    low_bias : bool
        Only use tapers with more than 90%% spectral concentration within
        bandwidth.

        .. versionadded:: 0.17
    %(n_jobs)s
        It is only used if adaptive=True.

        .. versionadded:: 0.17
    return_sensor : bool
        If True, return the sensor PSDs as an EvokedArray.

        .. versionadded:: 0.17
    dB : bool
        If True (default False), return output it decibels.

        .. versionadded:: 0.17
    %(verbose)s

    Returns
    -------
    stc_psd : instance of SourceEstimate | VolSourceEstimate
        The PSD of each of the sources.
    sensor_psd : instance of EvokedArray
        The PSD of each sensor. Only returned if ``return_sensor`` is True.

    See Also
    --------
    compute_source_psd_epochs

    Notes
    -----
    Each window is multiplied by a window before processing, so
    using a non-zero overlap is recommended.

    This function is different from :func:`compute_source_psd_epochs` in that:

    1. ``bandwidth='hann'`` by default, skipping multitaper estimation
    2. For convenience it wraps
       :func:`mne.make_fixed_length_events` and :class:`mne.Epochs`.

    Otherwise the two should produce identical results.
    """
    tmin = 0.0 if tmin is None else float(tmin)
    overlap = float(overlap)
    if not 0 <= overlap < 1:
        raise ValueError(
            "Overlap must be at least 0 and less than 1, got %s" % (overlap,)
        )
    n_fft = int(n_fft)
    duration = ((1.0 - overlap) * n_fft) / raw.info["sfreq"]
    events = make_fixed_length_events(raw, 1, tmin, tmax, duration)
    epochs = Epochs(raw, events, 1, 0, (n_fft - 1) / raw.info["sfreq"], baseline=None)
    out = compute_source_psd_epochs(
        epochs,
        inverse_operator,
        lambda2,
        method,
        fmin,
        fmax,
        pick_ori,
        label,
        nave,
        pca,
        inv_split,
        bandwidth,
        adaptive,
        low_bias,
        True,
        n_jobs,
        prepared,
        method_params,
        return_sensor=True,
    )
    source_data = 0.0
    sensor_data = 0.0
    count = 0
    for stc, evoked in out:
        source_data += stc.data
        sensor_data += evoked.data
        count += 1
    assert count > 0  # should be guaranteed by make_fixed_length_events
    sensor_data /= count
    source_data /= count
    if dB:
        np.log10(sensor_data, out=sensor_data)
        sensor_data *= 10.0
        np.log10(source_data, out=source_data)
        source_data *= 10.0
    evoked.data = sensor_data
    evoked.nave = count
    stc.data = source_data
    out = stc
    if return_sensor:
        out = (out, evoked)
    return out

def _compute_source_psd_epochs(
    epochs,
    inverse_operator,
    lambda2=1.0 / 9.0,
    method="dSPM",
    fmin=0.0,
    fmax=200.0,
    pick_ori=None,
    label=None,
    nave=1,
    pca=True,
    inv_split=None,
    bandwidth=4.0,
    adaptive=False,
    low_bias=True,
    n_jobs=None,
    prepared=False,
    method_params=None,
    return_sensor=False,
    use_cps=True,
):
    """Generate compute_source_psd_epochs."""
    logger.info("Considering frequencies %g ... %g Hz" % (fmin, fmax))

    if label:
        # TODO: add multi-label support
        # since `_prepare_source_params` can handle a list of labels now,
        # multi-label support should be within reach for psd calc as well
        _validate_type(
            label,
            types=(Label, BiHemiLabel, None),
            type_name=("Label or BiHemiLabel", "None"),
        )

    K, sel, Vh, vertno, is_free_ori, noise_norm, _ = _prepare_source_params(
        inst=epochs,
        inverse_operator=inverse_operator,
        label=label,
        lambda2=lambda2,
        method=method,
        nave=nave,
        pca=pca,
        pick_ori=pick_ori,
        prepared=prepared,
        method_params=method_params,
        use_cps=use_cps,
    )
    # Simplify code with a tiny (rel. to other computations) penalty for eye
    # mult
    Vh = np.eye(K.shape[1]) if Vh is None else Vh

    # split the inverse operator
    if inv_split is not None:
        K_split = np.array_split(K, inv_split)
    else:
        K_split = [K]

    # compute DPSS windows
    n_times = len(epochs.times)
    sfreq = epochs.info["sfreq"]

    dpss, eigvals, adaptive = _compute_mt_params(
        n_times, sfreq, bandwidth, low_bias, adaptive, verbose=False
    )

    n_tapers = len(dpss)
    try:
        n_epochs = len(epochs)
    except RuntimeError:
        n_epochs = len(epochs.events)
        extra = "on at most %d epochs" % (n_epochs,)
    else:
        extra = "on %d epochs" % (n_epochs,)
    if isinstance(bandwidth, str):
        bandwidth = "%s windowing" % (bandwidth,)
    else:
        bandwidth = "%d tapers with bandwidth %0.1f Hz" % (n_tapers, bandwidth)
    logger.info("Using %s %s" % (bandwidth, extra))

    if adaptive:
        parallel, my_psd_from_mt_adaptive, n_jobs = parallel_func(
            _psd_from_mt_adaptive, n_jobs
        )
    else:
        weights = np.sqrt(eigvals)[np.newaxis, :, np.newaxis]

    subject = _subject_from_inverse(inverse_operator)
    iter_epochs = ProgressBar(epochs, max_value=n_epochs)
    evoked_info = pick_info(epochs.info, sel, verbose=False)
    for k, e in enumerate(iter_epochs):
        data = np.dot(Vh, e[sel])  # reducing data rank

        # compute tapered spectra in sensor space
        x_mt, freqs = _mt_spectra(data, dpss, sfreq)

        if k == 0:
            freq_mask = (freqs >= fmin) & (freqs <= fmax)
            fstep = np.mean(np.diff(freqs))
            with evoked_info._unlock():
                evoked_info["sfreq"] = 1.0 / fstep
        freqs = freqs[freq_mask]

        # sensor space PSD
        x_mt_sensor = np.empty(
            (len(sel), x_mt.shape[1], x_mt.shape[2]), dtype=x_mt.dtype
        )
        for i in range(n_tapers):
            x_mt_sensor[:, i, :] = np.dot(Vh.T, x_mt[:, i, :])
        if adaptive:
            out = parallel(
                my_psd_from_mt_adaptive(x, eigvals, freq_mask)
                for x in np.array_split(x_mt_sensor, min(n_jobs, len(x_mt_sensor)))
            )
            sensor_psd = np.concatenate(out)
        else:
            x_mt_sensor = x_mt_sensor[:, :, freq_mask]
            sensor_psd = _psd_from_mt(x_mt_sensor, weights)

        # allocate space for output
        psd = np.empty((K.shape[0], np.sum(freq_mask)))

        # Optionally, we split the inverse operator into parts to save memory.
        # Without splitting the tapered spectra in source space have size
        # (n_vertices x n_tapers x n_times / 2)
        pos = 0
        for K_part in K_split:
            # allocate space for tapered spectra in source space
            x_mt_src = np.empty(
                (K_part.shape[0], x_mt.shape[1], x_mt.shape[2]), dtype=x_mt.dtype
            )

            # apply inverse to each taper (faster than equiv einsum)
            for i in range(n_tapers):
                x_mt_src[:, i, :] = np.dot(K_part, x_mt[:, i, :])

            # compute the psd
            if adaptive:
                out = parallel(
                    my_psd_from_mt_adaptive(x, eigvals, freq_mask)
                    for x in np.array_split(x_mt_src, min(n_jobs, len(x_mt_src)))
                )
                this_psd = np.concatenate(out)
            else:
                x_mt_src = x_mt_src[:, :, freq_mask]
                this_psd = _psd_from_mt(x_mt_src, weights)

            psd[pos : pos + K_part.shape[0], :] = this_psd
            pos += K_part.shape[0]

        # combine orientations
        if is_free_ori and pick_ori is None:
            psd = combine_xyz(psd, square=False)

        if noise_norm is not None:
            psd *= noise_norm**2

        out = _make_stc(
            psd,
            tmin=freqs[0],
            tstep=fstep,
            vertices=vertno,
            subject=subject,
            src_type=inverse_operator["src"].kind,
        )

        if return_sensor:
            comment = "Epoch %d PSD" % (k,)
            out = (
                out,
                EvokedArray(sensor_psd, evoked_info.copy(), freqs[0], comment, nave),
            )

        # we return a generator object for "stream processing"
        yield out

    iter_epochs.update(n_epochs)  # in case some were skipped

@verbose
def compute_source_psd_epochs(
    epochs,
    inverse_operator,
    lambda2=1.0 / 9.0,
    method="dSPM",
    fmin=0.0,
    fmax=200.0,
    pick_ori=None,
    label=None,
    nave=1,
    pca=True,
    inv_split=None,
    bandwidth=4.0,
    adaptive=False,
    low_bias=True,
    return_generator=False,
    n_jobs=None,
    prepared=False,
    method_params=None,
    return_sensor=False,
    use_cps=True,
    verbose=None,
):
    """Compute source power spectral density (PSD) from Epochs.

    This uses the multi-taper method to compute the PSD for each epoch.

    Parameters
    ----------
    epochs : instance of Epochs
        The raw data.
    inverse_operator : instance of InverseOperator
        The inverse operator.
    lambda2 : float
        The regularization parameter.
    method : "MNE" | "dSPM" | "sLORETA" | "eLORETA"
        Use minimum norm, dSPM (default), sLORETA, or eLORETA.
    fmin : float
        The lower frequency of interest.
    fmax : float
        The upper frequency of interest.
    pick_ori : None | "normal"
        If "normal", rather than pooling the orientations by taking the norm,
        only the radial component is kept. This is only implemented
        when working with loose orientations.
    label : Label
        Restricts the source estimates to a given label.
    nave : int
        The number of averages used to scale the noise covariance matrix.
    pca : bool
        If True, the true dimension of data is estimated before running
        the time-frequency transforms. It reduces the computation times
        e.g. with a dataset that was maxfiltered (true dim is 64).
    inv_split : int or None
        Split inverse operator into inv_split parts in order to save memory.
    bandwidth : float | str
        The bandwidth of the multi taper windowing function in Hz.
        Can also be a string (e.g., 'hann') to use a single window.
    adaptive : bool
        Use adaptive weights to combine the tapered spectra into PSD
        (slow, use n_jobs >> 1 to speed up computation).
    low_bias : bool
        Only use tapers with more than 90%% spectral concentration within
        bandwidth.
    return_generator : bool
        Return a generator object instead of a list. This allows iterating
        over the stcs without having to keep them all in memory.
    %(n_jobs)s
        It is only used if adaptive=True.
    prepared : bool
        If True, do not call :func:`prepare_inverse_operator`.
    method_params : dict | None
        Additional options for eLORETA. See Notes of :func:`apply_inverse`.

        .. versionadded:: 0.16
    return_sensor : bool
        If True, also return the sensor PSD for each epoch as an EvokedArray.

        .. versionadded:: 0.17
    %(use_cps_restricted)s

        .. versionadded:: 0.20
    %(verbose)s

    Returns
    -------
    out : list (or generator object)
        A list (or generator) for the source space PSD (and optionally the
        sensor PSD) for each epoch.

    See Also
    --------
    compute_source_psd
    """
    # use an auxiliary function so we can either return a generator or a list
    stcs_gen = _compute_source_psd_epochs(
        epochs,
        inverse_operator,
        lambda2=lambda2,
        method=method,
        fmin=fmin,
        fmax=fmax,
        pick_ori=pick_ori,
        label=label,
        nave=nave,
        pca=pca,
        inv_split=inv_split,
        bandwidth=bandwidth,
        adaptive=adaptive,
        low_bias=low_bias,
        n_jobs=n_jobs,
        prepared=prepared,
        method_params=method_params,
        return_sensor=return_sensor,
        use_cps=use_cps,
    )

    if return_generator:
        # return generator object
        return stcs_gen
    else:
        # return a list
        stcs = list()
        for stc in stcs_gen:
            stcs.append(stc)

        return stcs


from flask import Flask, jsonify, request

app = Flask(__name__)

pecas = [
    {
        'id': 1,
        'peca' : 'bateria',
        'descricao':'bateria moura'
    },
    {
        'id': 2,
        'peca' : 'bubina',
        'descricao':'bubina generica'
    },
    {
        'id': 3,
        'peca' : 'filtro de oleo',
        'descricao':'filtro de oleo generico'
    }
]

@app.route('/pecas', methods=['GET'])
def obter_pecas():
    return jsonify(pecas)

@app.route('/pecas/<int:id>', methods=['GET'])
def obter_pecas_id(id):
    for peca in pecas:
      if peca.get('id') == id:
            return jsonify(peca)

@app.route('/pecas/<int:id>', methods=['PUT'])
def editar_peca_id(id):
    peca_dif = request.get_json()
    for indice,peca in enumerate(pecas):
        if peca.get('id') == id:
            pecas[indice].update(peca_dif)
            return jsonify(pecas[indice])

@app.route('/pecas', methods=['POST'])
def incluir_nova_peca():
    nova_peca = request.get_json()
    pecas.append(nova_peca)

    return jsonify(pecas)

@app.route('/pecas/<int:id>', methods=['DELETE'])
def excluir_peca(id):
    for indice, peca in enumerate(pecas):
        if peca.get('id') == id:
            del pecas[indice]
        return jsonify(pecas)

app.run(port=5000, host='localhost', debug=True)


import pickle
from numpy import dot
from numpy.linalg import norm
from scipy.spatial import distance

with open('./anchor_embeddings.pkl', 'rb') as handler:
    anchor_embeddings = pickle.load(handler)

with open('./positive_embeddings.pkl', 'rb') as handler:
    positive_embeddings = pickle.load(handler)

with open('./negative_embeddings.pkl', 'rb') as handler:
    negative_embeddings = pickle.load(handler)

def cos_sim(a, b):
    return dot(a, b)/(norm(a)*norm(b))

# print(len(anchor_embeddings))
similarity_scores = []
for i in range(len(anchor_embeddings)):
    anchor_vector = anchor_embeddings[i]
    positive_vector = positive_embeddings[i]
    negative_vector = negative_embeddings[i]
    # print(type(anchor_vector), anchor_vector.shape)
    pos_cos_sim = 1 - distance.cosine(anchor_vector, positive_vector)
    # pos_cos_sim = 1 - distance.euclidean(anchor_vector, positive_vector)
    # pos_cos_sim = cos_sim(anchor_vector, positive_vector)

    neg_cos_sim = 1 - distance.cosine(anchor_vector, negative_vector)
    # neg_cos_sim = 1 - distance.euclidean(anchor_vector, negative_vector)
    # neg_cos_sim = cos_sim(anchor_vector, negative_vector)
    print(pos_cos_sim, neg_cos_sim)
    similarity_scores.append( (pos_cos_sim, neg_cos_sim) )
    # break

with open('./similarity_scores.pkl', 'wb') as handler:
    pickle.dump(similarity_scores, handler)


#!/usr/bin/python3
# coding=utf-8
import sys
sys.path.append("../../")
import tensorflow as tf
import time
from input_data import Load_data
from model import QueryModel
from model import Decoder
from model import Hidden_init
from metric import *
from sklearn.model_selection import LeaveOneOut, KFold

import sys
import random

cities = [
'osaka',
#'glas',
#'edin',
#'toro',
#'melb',
#'weeplaces_poi_25_length_3-15-numtraj_765',
#'weeplaces_poi_50_length_3-15-numtraj_2134',
#'weeplaces_poi_100_length_3-15-numtraj_4497',
#'weeplaces_poi_200_length_3-15-numtraj_7790',
#'weeplaces_poi_400_length_3-15-numtraj_12288',
]
if len(sys.argv) > 1:
    id = int(sys.argv[1]) % len(cities)
else:
    id = random.randint(0, len(cities) - 1)
city = cities[id]
# city = 'Glas'
# city = 'Edin'
# city = 'Toro'
tf.keras.backend.set_floatx('float64')# float desgin

pretrain_batch_size = 32
batch_size = 32
k = 256
dec_units = 256

data = Load_data(city)
poi_embedding, poi_size = data.self_embedding()
data_size = data.data_size()

query = QueryModel(poi_embedding, k)
decoder = Decoder(poi_embedding, poi_size, dec_units)
h_state = Hidden_init(dec_units)

loss_object = tf.keras.losses.SparseCategoricalCrossentropy(
    from_logits=True, reduction='none')

def loss_function(real, pred, real2, pred2):
    mask = tf.math.logical_not(tf.math.equal(real, 0))
    loss_ = loss_object(real, pred)
    mask = tf.cast(mask, dtype=loss_.dtype)
    loss_ *= mask

    loss = loss_object(real2, pred2)

    return tf.reduce_mean(loss_) + 1 * tf.reduce_mean(loss)

def pre_loss_function(sim):
    real = np.eye(pretrain_batch_size)
    loss = tf.keras.losses.categorical_crossentropy(real, sim)

    return tf.reduce_mean(loss)

# --------------------------- pre-train_step ------------------------------
def pre_train_step(pre_que, sample1, sample2, lr=0.0005):
    pre_loss = 0
    optimizer = tf.keras.optimizers.Adam(lr=lr)

    with tf.GradientTape() as tape:
        query_out = query(pre_que)
        dec_input1 = sample1[:, 0]
        dec_input2 = sample2[:, 0]
        dec_hidden1 = h_state(query_out)
        dec_hidden2 = h_state(query_out)
        for t in range(1, sample1.shape[1]):
            output1, dec_hidden1 = decoder.pre_train(dec_input1, query_out, dec_hidden1)
            output2, dec_hidden2 = decoder.pre_train(dec_input2, query_out, dec_hidden2)
            dec_input1 = sample1[:, t]
            dec_input2 = sample2[:, t]
        sim = tf.matmul(output1, tf.transpose(output2))
        sim = tf.math.softmax(sim)
        pre_loss += pre_loss_function(sim)

    batch_loss = pre_loss
    variables = decoder.trainable_variables + query.trainable_variables + h_state.trainable_variables
    gradients = tape.gradient(pre_loss, variables)
    optimizer.apply_gradients(zip(gradients, variables))
    return batch_loss

exps = tf.keras.optimizers.schedules.ExponentialDecay(
                                                    0.1,
                                                    decay_steps=5,
                                                    decay_rate=0.9,
                                                    staircase=False)

# --------------------------- train_step ---------------------------------
def train_step(que, traj,lr = 0.1):
    loss = 0
    optimizer = tf.keras.optimizers.Adam(lr=lr)

    with tf.GradientTape() as tape:
        query_out = query(que)
        dec_input = traj[:, 0]
        dec_hidden=h_state(query_out)

        for t in range(1, traj.shape[1]):
            predictions, predictions2, dec_hidden = decoder(dec_input, query_out, dec_hidden)
            loss += loss_function(traj[:, t], predictions, que[:, 2], predictions2)
            dec_input = tf.argmax(tf.nn.softmax(predictions), 1)

    batch_loss = loss
    variables = query.trainable_variables + decoder.trainable_variables+h_state.trainable_variables
    gradients = tape.gradient(loss, variables)
    optimizer.apply_gradients(zip(gradients, variables))
    return batch_loss

# ------------------------------- evaluate ------------------------------
def evaluate(que, traj):
    predict_traj = []
    realnum_poi = 0
    query_out = query(que)
    dec_input = traj[:, 0]

    for poi in tf.squeeze(traj):
        if (poi == 0):
            break
        realnum_poi += 1
    realnum_poi = realnum_poi - 2

    start_poi = traj[:, 0]
    start_poi = tf.cast(start_poi,dtype=tf.int32)
    end_poi = traj[:, realnum_poi + 1]
    predict_traj.append(start_poi)
    dec_hidden = h_state(query_out)
    table = np.ones([poi_size],dtype=np.float64)

    table[start_poi.numpy()] = 0.
    table[end_poi.numpy()] = 0.
    table[0] = 0.

    for t in range(realnum_poi):
        decoder.set_dropout()
        predictions, _, dec_hidden = decoder(dec_input, query_out,dec_hidden)
        mask = tf.expand_dims(table,axis=0)
        dec_input = tf.argmax(tf.nn.softmax(predictions * mask), 1)
        predict_traj.append(dec_input)

    predict_traj.append(end_poi)
    real_traj = tf.squeeze(traj)[0:realnum_poi + 1].numpy()
    real_traj = np.append(real_traj, end_poi)
    predict_traj = [i.numpy().tolist() for i in predict_traj]
    predict_traj = [i[0] for i in predict_traj]

    return real_traj, predict_traj

if __name__ == '__main__':
    import pandas as pd
    import datetime
    from BERT_Trip.util import evaluate_results, save_results
    total_test_f1 = []
    total_test_pairs_f1 = []

    max_f1 = []
    max_pf1 = []
    fold = 0
    df = pd.read_csv('./train_data/'+city+'-query.csv', header = None, sep= ' ', names = ['startPOI', 'startTime', 'endPOI', 'endTime'])
    seed = int(datetime.datetime.now().timestamp())
    random.seed(seed)
    np.random.seed(seed)
    loo = KFold(n_splits = 5, shuffle = True, random_state = seed)
    for train_index, test_index in loo.split(df):
        decoder.reset_variable()
        query.reset_variable()
        h_state.reset_variable()

        pre_dataset_train, _, pre_steps_train, _ = data.load_dataset_kfold(train_index, test_index, pretrain_batch_size)
        # -------------------------- pre-train ----------------------------
        PRE_EPOCHES = 5
        EPOCHS = 20
        avg = []
        for epoch in range(PRE_EPOCHES):
            start = time.time()
            pre_loss = 0
            #print("pretrain epoch", epoch)
            for (batch, (que, traj)) in enumerate(pre_dataset_train.take(pre_steps_train)):
                total_batch_loss = 0
                pre_que, sample1, sample2 = data.load_pretrain_dataset(que, traj)
                pre_batch_loss = pre_train_step(pre_que, sample1, sample2)
                pre_loss += pre_batch_loss
            #print('Epoch {} Loss {:.4f}'.format(epoch + 1, pre_loss / pre_steps_train))
            avg.append(time.time() - start)
            print(f'{city}: Pretain Time taken for 1 epoch {avg[-1]:.3f} sec avg: {np.mean(avg):.3f}')

        # ------------------------------ train -----------------------------------
        dataset_train, dataset_val, steps_train, steps_val = data.load_dataset_kfold(train_index, test_index, pretrain_batch_size)
        start1 = time.time()
        best_test_f1 = -1
        best_result_list = []
        res = {}
        avg = []
        for epoch in range(EPOCHS):
            start = time.time()
            total_loss = 0
            lr = 0.1
            #print(f"train epoch: {epoch}")
            for (batch, (que, traj)) in enumerate(dataset_train.take(steps_train)):
                batch_loss = train_step(que, traj, lr)
                total_loss += batch_loss

            avg.append(time.time() - start)
            print(f'{city}: Time taken for 1 epoch {avg[-1]:.3f} sec avg: {np.mean(avg):.3f}')
            result_list = []
            for (batch, (que, traj)) in enumerate(dataset_val.take(steps_val)):
                for i in range(len(traj)):
                    traj1 = traj[i].numpy()
                    indexs = np.where(traj1 == 0)
                    traj1 = np.delete(traj1,indexs)
                    traj1 = tf.convert_to_tensor(traj1)
                    que1 = tf.expand_dims(que[i], 0)
                    traj1 = tf.expand_dims(traj1, 0)
                    real_traj, predict_traj = evaluate(que1, traj1)

                    result_list.append({'expected': real_traj.tolist(), 'predict': predict_traj})
            e = evaluate_results(result_list)['f1']
            if e > best_test_f1:
                best_test_f1 = e
                best_result_list = result_list

        save_results(
            dataset = city,
            method = 'SelfTrip',
            train_size = len(train_index),
            test_size =  len(test_index),
            fold = fold,
            seed = seed,
            results = result_list,
        )
        print("best f1: ", best_test_f1)
        print('\n--' * 2 + 'Time take {} sec'.format(time.time() - start1))
        print('---- finish' + ' index = ' + str(fold) + '-' * 5 + '\n')
        fold = fold + 1


# coding=utf-8

import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import os
import uuid
import json
from django.shortcuts import render, render_to_response
from django.http import HttpResponse
from cms.models import Picture, Classification
from django.views.decorators.csrf import csrf_exempt
from PIL import Image

basePath = os.path.dirname(os.path.dirname(__file__))

@csrf_exempt
def getPicturePosition(req):
	"""
	获取所属栏目
	:param req:
	:return:
	"""
	classification = Classification.objects.all().order_by("classificationIdLevel")    #根据栏目级别排序
	List = []
	for obj in classification:
		classificationList = {}
		classificationList["classificationId"] = obj.classificationId
		classificationList["classificationName"] = obj.classificationName
		classificationList["classificationIdLevel"] = obj.classificationIdLevel
		List.append(classificationList)
	return HttpResponse(json.dumps(List))

@csrf_exempt
def savePicture(req):
	"""
	添加图片
	:param req:
	:return:
	"""
	# picture = Picture.objects.filter(pictureId = req.POST["pictureId"])
	picturePosition = req.POST["picturePosition"]          #所属栏目
	pictureName = req.POST["pictureName"]                  #图片名称
	isShow = req.POST["isShow"]                            #是否启用
	if isShow == "true":
		isShow = 1
	else:
		isShow = 0
	#保存图片
	reqfile = req.FILES.get("pictureUrl", False)
	if reqfile != False:
		pictureUrl = "cms/static/img/classPic/"+pictureName
		picturePath = os.path.join(basePath, pictureUrl)
		img = Image.open(reqfile)
		img.thumbnail((700, 700), Image.ANTIALIAS)         #对图片进行等比缩放
		img.save(picturePath, "png")                      #保存图片
	else:
		pictureUrl = ""
	#修改图片
	#添加图片
	pictureId = uuid.uuid1()
	picture = Picture(pictureId=pictureId, pictureName=pictureName, picturePath="/"+pictureUrl, isShow=isShow, classificationId_id=picturePosition)
	picture.save()
	return HttpResponse(1)


from django.shortcuts import render

from django.core.exceptions import ValidationError
from django.db.models import Q, ProtectedError

from rest_framework import status
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework.permissions import IsAuthenticated, AllowAny

from recruit.models import JobOpening, Company
from recruit.serializers import JobOpeningSerializer, UpdateJobOpeningSerializer, CompanySerializer

class JobOpeningView(APIView):
    permission_classes = [AllowAny, ]

    def get(self, request, jo_pk):
        job_opening = JobOpening.objects.get(pk=jo_pk)
        serializer = JobOpeningSerializer(job_opening)

        other_opening = JobOpening.objects.filter(company=job_opening.company)
        other_opening_pk = [x.pk for x in other_opening]
        other_opening_pk.pop(other_opening_pk.index(jo_pk))

        data = serializer.data
        data["other_opening"] = other_opening_pk

        return Response({"message":"success get!", "data": data}, status=status.HTTP_200_OK)

    def put(self, request, jo_pk):
        data = request.data
        if data.get("company"):
            raise ValidationError("company cannot be changed.")

        serializer = UpdateJobOpeningSerializer(data=data, partial=True)
        job_opening = JobOpening.objects.get(pk=jo_pk)

        if serializer.is_valid():
            # serializer에 의해 company가 제외된다.
            print("serial data:", serializer.validated_data)
            obj = serializer.update(job_opening, serializer.validated_data)
            result = UpdateJobOpeningSerializer(obj)
            return Response({"message": "success update!", "data": result.data}, status=status.HTTP_200_OK)

        return Response({"message": "fail"}, status=status.HTTP_400_BAD_REQUEST)

    def delete(self, request, jo_pk):
        job_opening = JobOpening.objects.get(pk=jo_pk)
        serializer = JobOpeningSerializer(job_opening)
        job_opening.delete()

        return Response({"message":"success delete!", "data": serializer.data}, status=status.HTTP_200_OK)

class JobOpeningCreateView(APIView):
    permission_classes = [AllowAny, ]

    def post(self, request):
        data = request.data
        serializer = JobOpeningSerializer(data=data)
        print(serializer.is_valid())
        print(serializer.errors)

        if serializer.is_valid():
            # serializer.data 는 Foreinkey의 pk만, serializer.validated_data는 object가 나옴
            obj = serializer.create(serializer.validated_data)
            result = JobOpeningSerializer(obj)
            return Response({"message": "success create!", "data": result.data }, status=status.HTTP_201_CREATED)

        return Response({"message": "fail", "error": serializer.errors }, status=status.HTTP_400_BAD_REQUEST)

class JobOpeningListView(APIView):
    permission_classes = [AllowAny, ]

    def get(self, request):
        search_query = request.GET.get('search', None)

        if search_query:
            print(search_query)

            try:
                results = JobOpening.objects.filter(
                    Q(company__name__icontains=search_query) |
                    Q(position__icontains=search_query) |
                    Q(content__icontains=search_query) |
                    Q(tech__icontains=search_query)
                    )
            except Company.DoesNotExist as e:
                results = JobOpening.objects.filter(
                    Q(position__icontains=search_query) |
                    Q(content__icontains=search_query) |
                    Q(tech__icontains=search_query)
                    )
            finally:
                serializer = JobOpeningSerializer(results, many=True)
                for data in serializer.data:
                    data["company_name"] = Company.objects.get(pk=data.get("id")).name

                print("results:",results)
                return Response({"message": "success get!", "data": serializer.data}, status=status.HTTP_200_OK)
        else:
            result = JobOpening.objects.all()
            serializer = JobOpeningSerializer(result, many=True)

            return Response({"message": "success get!", "data": serializer.data}, status=status.HTTP_200_OK)

class ApplyView(APIView):
    permission_classes = [AllowAny, ]

    def post(self, request, jo_pk):
        user = request.user
        job_opening = JobOpening.objects.get(pk=jo_pk)

        if job_opening in user.applyed.all():
            return Response({"message" : "A Job Opening could be applied only once."}, status=status.HTTP_400_BAD_REQUEST)

        user.applyed.add(job_opening)
        serializer = JobOpeningSerializer(job_opening)

        return Response({"message": "success to apply", "data":serializer.data}, status=status.HTTP_200_OK)


import cv2 as cv
import numpy as np

Part28 = "Face and eye detection"
# all the idea here that we are going to apply haar
# haar is basically a method which applies different things on the picture to detect something

# for eye detection, you won't get a wrong answer if you didn't crop the face, it will gave you the same result

#image = cv.imread('C:/Users/LENOVO/Desktop/Pictures and videos/Me.jpg')

image = cv.imread('C:/Users/LENOVO/Desktop/Me.jpg')
gray = cv.cvtColor(image,cv.COLOR_BGR2GRAY)

# actually I didn't see any difference when I used haarcascade_frontalface_alt2, it gave me the same result

face_classifier = cv.CascadeClassifier('haarcascade_frontalface_default.xml')

eye_classifier = cv.CascadeClassifier('haarcascade_eye.xml')
eye = eye_classifier.detectMultiScale(gray,1.3,5)

# faces is a list holds four lines which borders the face

faces = face_classifier.detectMultiScale(gray,1.3,5)

for x,y,w,h in faces:
    cv.rectangle(image,(x,y),(x+w,y+h),(0,255,0),3)
    cv.imshow("window",image)

for x,y,w,h in eye:
    cv.rectangle(image,(x,y),(x+w,y+h),(0,255,0),3)
    cv.imshow("window",image)

cv.waitKey()

Learning_Codes_and_Projects/Face and eye detection.py


import numpy as np
import matplotlib.pyplot as plt
from scipy import optimize
# unfinished 18/02

def gaussian(height, center_x, center_y, width_x, width_y):
    """Returns a gaussian function with the given parameters"""
    width_x = float(width_x)
    width_y = float(width_y)
    return lambda x, y: height*np.exp(
        -(((center_x-x)/width_x)**2+((center_y-y)/width_y)**2)/2)

def moments(data):
    """Returns (height, x, y, width_x, width_y)
    the gaussian parameters of a 2D distribution by calculating its
    moments """
    total = data.sum()
    X, Y = np.indices(data.shape)
    x = (X*data).sum()/total
    y = (Y*data).sum()/total
    col = data[:, int(y)]
    width_x = np.sqrt(np.abs((np.arange(col.size)-y)**2*col).sum()/col.sum())
    row = data[int(x), :]
    width_y = np.sqrt(np.abs((np.arange(row.size)-x)**2*row).sum()/row.sum())
    height = data.max()
    return height, x, y, width_x, width_y

def fitgaussian(data):
    """Returns (height, x, y, width_x, width_y)
    the gaussian parameters of a 2D distribution found by a fit"""
    params = moments(data)
    def errorfunction(p): return np.ravel(gaussian(*p)(*np.indices(data.shape)) -
                                          data)
    p, success = optimize.leastsq(errorfunction, params)
    return p

def gaussian_fitting(galaxies_points, img):
    for counter in range(5):
        galaxy = galaxies_points[counter]
        x_min, x_max = 1e9, 0
        y_min, y_max = 1e9, 0
        for point in galaxy:
            if point[0] < x_min:
                x_min = point[0]
            elif point[0] > x_max:
                x_max = point[0]
            if point[1] < y_min:
                y_min = point[1]
            elif point[1] > y_max:
                y_max = point[1]
        buffer = 0
        x_min = max(int(x_min-buffer), 0)
        # x_max = min(int(x_max+buffer), img.shape[1] - 1)
        y_min = max(int(y_min-buffer), 0)
        # y_max = min(int(y_max+buffer), img.shape[0] - 1)
        # print(x_min, x_max, y_min, y_max)
        if x_min == x_max or y_min == y_max:
            print(f'weird stuff {counter}')
            continue

        x = np.arange(x_min, x_max, 1)
        y = np.arange(y_min, y_max, 1)
        x, y = np.meshgrid(y, x)
        img_slice = img[x_min:x_max, y_min:y_max]
        params = fitgaussian(img_slice)
        fit = gaussian(*params)
        fig, ax = plt.subplots()
        plt.imshow(img_slice, origin='upper')
        plt.contour(fit(*np.indices(img_slice.shape)))
        # ax.contour(x, y, twoD_Gaussian((x, y), *popt).reshape(7, 12), 1, colors='w')

if __name__ == '__main__':
    img = np.load('testData_noisy.npy')
    galaxies_points = np.load('galaxies_points.npy', allow_pickle=True)
    # print(img.shape)
    # print(len(galaxies_points))
    gaussian_fitting(galaxies_points, img)
    plt.show()


from __future__ import unicode_literals

import frappe, os, copy, json, re
from frappe import _

from frappe.modules import get_doc_path
from jinja2 import TemplateNotFound
from frappe.utils import cint, strip_html
from frappe.utils.pdf import get_pdf,cleanup
from PyPDF2 import PdfFileWriter, PdfFileReader

no_cache = 1
no_sitemap = 1

base_template_path = "templates/www/printview.html"
standard_format = "templates/print_formats/standard.html"

@frappe.whitelist()
def download_multi_pdf(doctype, name, format=None):
	"""
	Concatenate multiple docs as PDF .

	Returns a PDF compiled by concatenating multiple documents. The documents
	can be from a single DocType or multiple DocTypes

	Note: The design may seem a little weird, but it exists exists to
		ensure backward compatibility. The correct way to use this function is to
		pass a dict to doctype as described below

	NEW FUNCTIONALITY
	=================
	Parameters:
	doctype (dict):
		key (string): DocType name
		value (list): of strings of doc names which need to be concatenated and printed
	name (string):
		name of the pdf which is generated
	format:
		Print Format to be used

	Returns:
	PDF: A PDF generated by the concatenation of the mentioned input docs

	OLD FUNCTIONALITY - soon to be deprecated
	=========================================
	Parameters:
	doctype (string):
		name of the DocType to which the docs belong which need to be printed
	name (string or list):
		If string the name of the doc which needs to be printed
		If list the list of strings of doc names which needs to be printed
	format:
		Print Format to be used

	Returns:
	PDF: A PDF generated by the concatenation of the mentioned input docs
	"""

	import json
	output = PdfFileWriter()

	if not isinstance(doctype, dict):
		result = json.loads(name)

		# Concatenating pdf files
		for i, ss in enumerate(result):
			output = frappe.get_print(doctype, ss, format, as_pdf = True, output = output)
		frappe.local.response.filename = "{doctype}.pdf".format(doctype=doctype.replace(" ", "-").replace("/", "-"))
	else:
		for doctype_name in doctype:
			for doc_name in doctype[doctype_name]:
				try:
					output = frappe.get_print(doctype_name, doc_name, format, as_pdf = True, output = output)
				except Exception:
					frappe.log_error("Permission Error on doc {} of doctype {}".format(doc_name, doctype_name))
		frappe.local.response.filename = "{}.pdf".format(name)

	frappe.local.response.filecontent = read_multi_pdf(output)
	frappe.local.response.type = "download"

def read_multi_pdf(output):
	# Get the content of the merged pdf files
	fname = os.path.join("/tmp", "frappe-pdf-{0}.pdf".format(frappe.generate_hash()))
	output.write(open(fname,"wb"))

	with open(fname, "rb") as fileobj:
		filedata = fileobj.read()

	return filedata

@frappe.whitelist()
def download_pdf(doctype, name, format=None, doc=None, no_letterhead=0):
	html = frappe.get_print(doctype, name, format, doc=doc, no_letterhead=no_letterhead)
	frappe.local.response.filename = "{name}.pdf".format(name=name.replace(" ", "-").replace("/", "-"))
	frappe.local.response.filecontent = get_pdf(html)
	frappe.local.response.type = "download"

@frappe.whitelist()
def report_to_pdf(html, orientation="Landscape"):
	frappe.local.response.filename = "report.pdf"
	frappe.local.response.filecontent = get_pdf(html, {"orientation": orientation})
	frappe.local.response.type = "download"

@frappe.whitelist()
def print_by_server(doctype, name, print_format=None, doc=None, no_letterhead=0):
	print_settings = frappe.get_doc("Print Settings")
	try:
		import cups
	except ModuleNotFoundError:
		frappe.throw("You need to install pycups to use this feature!")
		return
	try:
		cups.setServer(print_settings.server_ip)
		cups.setPort(print_settings.port)
		conn = cups.Connection()
		output = PdfFileWriter()
		output = frappe.get_print(doctype, name, print_format, doc=doc, no_letterhead=no_letterhead, as_pdf = True, output = output)
		file = os.path.join("/", "tmp", "frappe-pdf-{0}.pdf".format(frappe.generate_hash()))
		output.write(open(file,"wb"))
		conn.printFile(print_settings.printer_name,file , name, {})
	except IOError as e:
		if ("ContentNotFoundError" in e.message
			or "ContentOperationNotPermittedError" in e.message
			or "UnknownContentError" in e.message
			or "RemoteHostClosedError" in e.message):
			frappe.throw(_("PDF generation failed"))
	except cups.IPPError:
		frappe.throw(_("Printing failed"))
	finally:
		cleanup(file,{})


__author__ = 'sameer'
from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.internet import reactor
from twisted.python import log
from pprint import pprint
import treq
from decimal import Decimal
from bs4 import BeautifulSoup
from datetime import datetime, timedelta
import json

class Yahoo():
    def __init__(self, **kwargs):
        self.yahoo_uri = "http://finance.yahoo.com/"
        self.oanda_uri = "http://www.oanda.com/currency/"

    @inlineCallbacks
    def getOrderBook(self, ticker):
        payout, denominated = ticker.split('/')
        url =  self.yahoo_uri + "q"
        params = {'s': "%s%s=X" % (payout, denominated)}
        response = yield treq.get(url, params=params)
        content = yield response.content()
        soup = BeautifulSoup(content)
        bid = Decimal(soup.find(id="yfs_b00_%s%s=x" % (payout.lower(), denominated.lower())).text.replace(',', ''))
        ask = Decimal(soup.find(id="yfs_a00_%s%s=x" % (payout.lower(), denominated.lower())).text.replace(',', ''))
        book = {'contract': ticker,
                'bids': [{'price': bid, 'quantity': 0}],
                'asks': [{'price': ask, 'quantity': 0}]}
        returnValue(book)

    @inlineCallbacks
    def getOHLCVHistory(self, ticker, period="day", start_datetime=None, end_datetime=None):
        payout, denominated = ticker.split('/')

        url = self.oanda_uri + "historical-rates/update"
        params = {'date_fmt': 'us',
                  'start_date': start_datetime.strftime('%Y-%m-%d'),
                  'end_date': end_datetime.strftime('%Y-%m-%d'),
                  'period': "daily",
                  'quote_currency': payout,
                  'base_currency_0': denominated,
                  'rate': 0,
                  'view': 'table',
                  'display': 'absolute',
                  'price': 'bid',
                  'data_range': 'd90'}
        headers = {'X-Requested-With': 'XMLHttpRequest',
                   'X-Prototype-Version': '1.7',
                   'Referer': 'http://www.oanda.com/currency/historical-rates/'}
        try:
            response = yield treq.get(url, params=params, headers=headers)
        except Exception as e:
            pass
        content = yield response.content()
        parsed = json.loads(content)
        data = parsed['widget'][0]['data']
        ohlcv_history = {}

        for row in data:
            date = datetime.fromtimestamp(row[0]/1e3)
            price = float(row[1])
            epoch = datetime.utcfromtimestamp(0)
            open_timestamp = int((date - epoch).total_seconds() * 1e6)
            tomorrow_date = date + timedelta(days=1)
            close_timestamp = int((tomorrow_date - epoch).total_seconds() * 1e6) - 1

            ohlcv = {
                'contract': ticker,
                'open': price,
                'high': price,
                'low': price,
                'close': price,
                'volume': 0,
                'vwap': price,
                'open_timestamp': open_timestamp,
                'close_timestamp': close_timestamp,
                'period': period
            }
            ohlcv_history[open_timestamp] = ohlcv

        returnValue(ohlcv_history)

if __name__ == "__main__":
    yahoo = Yahoo()
    #d = yahoo.getOrderBook('USD/MXN')
    #d.addCallback(pprint).addErrback(log.err)
    now = datetime.utcnow()
    d2 = yahoo.getOHLCVHistory('USD/HUF', start_datetime=now-timedelta(days=30), end_datetime=now)
    d2.addCallback(pprint).addErrback(log.err)

    reactor.run()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import codecs
from collections import OrderedDict
import fnmatch
import json
import os
import sys

# error counter
counterr = 0

# Need the json path
if len(sys.argv) < 2:
    dir = "json"
else:
    dir = sys.argv[1]

# collect all the JSON files
json_files = [
    os.path.join(dirpath, f)
    for dirpath, dirnames, files in os.walk(dir)
    for f in fnmatch.filter(files, '*.txt')
]

for files in json_files:
    update = False
    f = os.path.splitext(os.path.basename(files))[0]
    with codecs.open(files, mode='r', encoding='utf-8') as json_file:
        print("Opening {}".format(files))
        djson = json.load(json_file, object_pairs_hook=OrderedDict)
        for entry in djson:
            for data in entry:
                if data.startswith('tr_'):
                    continue
                elif data.startswith('jp_'):
                    continue
                elif data == 'title_id':
                    continue
                elif data.endswith('_id'):
                    entry[data] = 0
                    update = True
                elif entry[data] != "":
                    entry[data] = None
                    update = True

    if (update):
        print("Updating {}".format(files))
        with codecs.open(files, mode='w+', encoding='utf-8') as json_file:
            json.dump(
                djson, json_file, ensure_ascii=False,
                indent="\t", sort_keys=False)
            json_file.write("\n")

if counterr > 0:
    sys.exit("Issues found")


import matplotlib.pyplot as plt
import numpy as np
from math import sqrt
import sys

def fpga_plot(casename):
    filename_arch = '../benchmarks/' + casename + '.arch'
    filename_fp = '../outputs/' + casename + '.floorplan'
    fa = open(filename_arch, 'r')
    line = fa.readline()
    line = line.split()
    r, c, s, d = int(line[0]), int(line[1]), int(line[2]), int(line[3])
    fig = plt.figure(figsize=(20, 20))
    ax = fig.add_subplot(111)

    for i in range(s, c, d):
        for j in range(0, r, 3):
            mux = plt.Rectangle((i, j), 1, 3, edgecolor='white',
                                facecolor='lightblue', fill=True, linewidth=2)
            ax.add_patch(mux)

    fp = open(filename_fp, 'r')
    for line in fp.readlines():
        line = line.split()
        if len(line) == 1:
            break
        name, x, y, w, h = int(line[0]), int(line[1]), int(
            line[2]), int(line[3]), int(line[4])
        module = plt.Rectangle((x, y), w, h, edgecolor='grey',
                               facecolor='pink', fill=True, linewidth=2, alpha=0.8)
        ax.add_patch(module)
        ax.text(x, y+1, "M"+line[0], fontsize=10)
    fp.close()

    fixedline = plt.Rectangle((0, 0), c, r, edgecolor='red',
                              facecolor='green', fill=False, linewidth=2, linestyle='dashed')
    ax.add_patch(fixedline)
    ax.text(0, -5, "Fixed-outline(W, H): (" +
            str(c)+", "+str(r)+")", fontsize=20)

    plt.plot(range(r), alpha=0)
    plt.axis('off')
    plt.savefig('../outputs/'+casename + '.png')
    print(casename + '.png '+'is saved.')
    plt.show()

if __name__ == '__main__':
    casename = sys.argv[1]
    fpga_plot(casename)


import requests
import logging
import jq
from deckster.common.core import update_key_image, update_label_display

def main(deck, key, pressed):
    logger = logging.getLogger("deckster")
    args = key.args
    url = args["url"]
    if "json_data" in args:
        json_data = args["json_data"]
    else:
        json_data = None
    if "headers" in args:
        headers = args["headers"]
    else:
        headers = None

    try:
        logger.debug(f"Trying GET request for {url} with parameters: {json_data} and headers: {headers}.")
        res = requests.get(url, params = json_data, headers = headers)
    except Exception as e:
        logger.error(f"Request to {url} failed: {e}")

    if not res.status_code in args["status_codes"]:
        logger.error(f"Status code returned {res.status_code} not in expected codes.")
        return

    d = jq.compile(args["json_parse"]).input(res.json()).first()
    logger.info(f"Parsed result: '{d}' from {url}")
    if "send_to_display" in key.args or "send_to_label" in key.args:
            to_label = "send_to_label" in key.args
            logger.info(f"Sending GET result to {'label' if to_label else 'display'} for key {key.key}.")
            if to_label:
                key.label = str(d)
            else:
                key.display = str(d)
            update_label_display(key, True if "send_to_label" in key.args else False)
            update_key_image(deck, key, pressed)


import os
import pytest
from moto import mock_s3

import boto3
from graphql_relay import to_global_id
from django.conf import settings
from django.contrib.auth import get_user_model
from django.http.response import HttpResponse
from django.core.files import File
from django_s3_storage.storage import S3Storage

from creator.studies.models import Membership
from creator.data_reviews.factories import DataReviewFactory
from creator.ingest_runs.factories import ValidationResultsetFactory

User = get_user_model()

@pytest.mark.parametrize(
    "user_group,allowed",
    [
        ("Administrators", True),
        ("Services", False),
        ("Developers", True),
        ("Investigators", True),
        ("Bioinformatics", True),
        (None, False),
    ],
)
def test_local_download_and_delete(
    clients,
    db,
    mocker,
    data_review,
    user_group,
    allowed,
):
    """
    Test download of validation report and results files
    Test download after local file is deleted
    """
    mock_resp = mocker.patch("creator.ingest_runs.views.HttpResponse")
    mock_resp.return_value = HttpResponse(open("tests/data/data.csv"))

    if user_group:
        user = User.objects.filter(groups__name=user_group).first()
        Membership(collaborator=user, study=data_review.study).save()

    # Upload file and save validation resultset
    file_field = data_review.validation_resultset.report_file
    file_field.save("data.csv", File(open("tests/data/data.csv")))

    # Download file
    client = clients.get(user_group)
    download_url = data_review.validation_resultset.report_path
    resp = client.get(download_url)

    if allowed:
        assert resp.status_code == 200
        assert resp.get("Content-Disposition") == (
            f"attachment; filename*=UTF-8''"
            f"{file_field.name.split('/')[-1]}"
        )
        assert resp.content == b"aaa,bbb,ccc\nddd,eee,fff\n"
        # Delete file from storage system
        file_field.storage.delete(file_field.name)

        # Try to download
        resp = client.get(download_url)
        assert resp.status_code == 404
        assert b"file does not exist" in resp.content
    else:
        resp.status_code == 401

@mock_s3
def test_s3_download_and_delete(
    clients, db, mocker, tmp_uploads_s3, settings, data_review
):
    """
    Test s3 download of validation report and results files
    Test download after s3 file is deleted
    """
    mock_resp = mocker.patch("creator.ingest_runs.views.HttpResponse")
    mock_resp.return_value = HttpResponse(open("tests/data/data.csv"))

    # Upload file
    settings.DEFAULT_FILE_STORAGE = "django_s3_storage.storage.S3Storage"
    file_field = data_review.validation_resultset.report_file
    tmp_uploads_s3(bucket_name=data_review.study.bucket)
    file_field.storage = S3Storage(aws_s3_bucket_name=data_review.study.bucket)
    file_field.save("data.csv", File(open("tests/data/data.csv")))
    assert file_field.storage.exists(file_field.name)

    # Download file
    client = clients.get("Administrators")
    download_url = data_review.validation_resultset.report_path
    resp = client.get(download_url)

    resp.status_code == 200
    assert resp.get("Content-Disposition") == (
        f"attachment; filename*=UTF-8''" f"{file_field.name.split('/')[-1]}"
    )
    assert resp.content == b"aaa,bbb,ccc\nddd,eee,fff\n"
    # Delete file from storage system
    file_field.storage.delete(file_field.name)

    # Try to download
    resp = client.get(download_url)
    assert resp.status_code == 404
    assert b"file does not exist" in resp.content

def test_no_validation_result(clients, db, data_review):
    """
    Test download failures

    Data review not found
    Data review validation results don't exist
    """
    client = clients.get("Administrators")

    # Data review not found
    resp = client.get("/download/data_review/foo/validation/report")
    assert resp.status_code == 404
    assert b"No data review exists" in resp.content

    # Data review validation result set doesn't exist yet
    dr = DataReviewFactory()
    download_urls = [
        f"/download/data_review/{dr.kf_id}/validation/{ft}"
        for ft in ["report", "results"]
    ]
    for url in download_urls:
        resp = client.get(url)
        assert resp.status_code == 404
        assert b"Validation results not yet available" in resp.content

def test_file_not_found(clients, db, mocker, tmpdir, data_review):
    """
    Test validation report/results files have not been uploaded
    """
    client = clients.get("Administrators")

    for ft in ["report", "results"]:
        # Validation result set has no file uploaded yet
        download_url = (
            f"/download/data_review/{data_review.kf_id}/validation/{ft}"
        )
        resp = client.get(download_url)
        assert resp.status_code == 404
        assert b"file does not exist" in resp.content


from   qm.executable import *
import os
import os.path
import qm
import StringIO
import re
import sys
if sys.platform != "win32":
    import resource

########################################################################
# Classes
########################################################################

class CompilerExecutable(RedirectedExecutable):
    """A 'CompilerExecutable' is a 'Compiler' that is being run."""

    def _InitializeChild(self):
        """Initialize the child process.

        After 'fork' is called this method is invoked to give the
        child a chance to initialize itself.  '_InitializeParent' will
        already have been called in the parent process."""

        # Disable compiler core dumps.
        if sys.platform != "win32":
            resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
        # Do whatever the base class version would otherwise do.
        RedirectedExecutable._InitializeChild(self)

    def _StdinPipe(self):
        """Return a pipe to which to redirect the standard input.

        returns -- A pipe, or 'None' if the standard input should be
        closed in the child."""

        # The compiler should not need the standard input.
        return None

    def _StderrPipe(self):
        """Return a pipe to which to redirect the standard input.

        returns -- A pipe, or 'None'.  If 'None' is returned, but
        '_StdoutPipe' returns a pipe, then the standard error and
        standard output will both be redirected to that pipe.  However,
        if '_StdoutPipe' also returns 'None', then the standard error
        will be closed in the child."""

        # The standard output and standard error are combined.
        return None

class Compiler:
    """A 'Compiler' compiles and links source files."""

    MODE_PREPROCESS = 'preprocess'
    """Preprocess the source files, but do not compile them."""

    MODE_COMPILE = 'compile'
    """Compile the source files, but do not assemble them."""

    MODE_ASSEMBLE = 'assemble'
    """Compile the source files, but do not link them."""

    MODE_LINK = 'link'
    """Compile and link the source files."""

    modes = [ MODE_COMPILE, MODE_ASSEMBLE, MODE_LINK, MODE_PREPROCESS ]
    """The available compilation modes."""

    def __init__(self, path, options=None, ldflags=None):
        """Construct a new 'Compiler'.

        'path' -- A string giving the location of the compiler
        executable.

        'options' -- A list of strings indicating options to the
        compiler, or 'None' if there are no options.

        'ldflags' -- A list of strings indicating ld flags to the
        compiler, or 'None' if there are no flags."""

        self._path = path
        self.SetOptions(options or [])
        self.SetLDFlags(ldflags or [])

    def Compile(self, mode, files, dir, options = [], ldflags = [],
                output = None, timeout = -1):
        """Compile the 'files'.

        'mode' -- The compilation mode (one of the 'Compiler.modes')
        that should be used to compile the 'files'.

        'files' -- A sequence of strings giving the names of source
        files (including, in general, assembly files, object files,
        and libraries) that should be compiled.

        'dir' -- The directory in which to run the compiler.

        'options' -- A sequence of strings indicating additional
        options that should be provided to the compiler.

        'ldflags' -- A sequence of strings indicating additional
        linker flags that should be provided to the compiler, if
        linking is done.

        'output' -- The name of the file should be created by the
        compilation.  If 'None', the compiler will use a default
        value.

        'timeout' -- The maximum number of seconds the compiler is
        permitted to run.  If 'timeout' is -1, the compiler is
        permitted to run forever.

        returns -- A tuple '(status, output)'.  The 'status' is the
        exit status returned by the compiler, as indicated by
        'waitpid'.  The 'output' is a string containing the standard
        outpt and standard errror generated by the compiler."""

        # Get the command to use.
        command = self.GetCompilationCommand(mode, files, options,
                                             ldflags, output)
        # Invoke the compiler.
        return self.ExecuteCommand(dir, command, timeout)

    def ExecuteCommand(self, dir, command, timeout = -1):
        """Execute 'command' in 'dir'.

        'dir' -- The directory in which to execute the command.

        'command' --  A sequence of strings, as returned by
        'GetCompilationCommand'.

        'timeout' -- The maximum number of seconds the compiler is
        permitted to run.  If 'timeout' is -1, the compiler is
        permitted to run forever.

        returns -- A tuple '(status, output)'.  The 'status' is the
        exit status returned by the compiler, as indicated by
        'waitpid'.  The 'output' is a string containing the standard
        output and standard errror generated by the compiler."""

        # Invoke the compiler.
        executable = CompilerExecutable(timeout)
        status = executable.Run(command, dir = dir)
        # Return all of the information.
        return (status, executable.stdout)

    def GetCompilationCommand(self, mode, files, options=[],
                              ldflags = [], output=None):
        """Return the appropriate command for compiling 'files'.

        'mode' -- The compilation mode (one of the 'Compiler.modes')
        that should be used to compile the 'files'.

        'files' -- A sequence of strings giving the names of source
        files (including, in general, assembly files, object files,
        and libraries) that should be compiled.

        'options' -- A sequence of strings indicating additional
        options that should be provided to the compiler.

        'ldflags' -- A sequence of strings indicating additional
        linker flags that should be provided to the compiler, if
        linking is done.

        'output' -- The name of the file should be created by the
        compilation.  If 'None', the compiler will use a default
        value.  (In some cases there may be multiple outputs.  For
        example, when generating multiple object files from multiple
        source files, the compiler will create a variety of objects.)

        returns -- A sequence of strings indicating the arguments,
        including 'argv[0]', for the compilation command."""

        # Start with the path to the compiler.
        command = [self.GetPath()]
        # Add switches indicating the compilation mode, if appropriate.
        command += self._GetModeSwitches(mode)
        # Add the options that should be used with every compilation.
        command += self._options
        # Add the options that apply to this compilation.
        command += options
        # Set the output file.
        if output:
            command += ["-o", output]
        # Add the input files.
        command += files
        if mode == Compiler.MODE_LINK:
            command += ldflags
            command += self.GetLDFlags()

        return command

    def ParseOutput(self, output, ignore_regexps = ()):
        """Turn the 'output' into a sqeuence of 'Diagnostic's.

        'output' -- A string containing the compiler's output.

        'ignore_regexps' -- A sequence of regular expressions.  If a
        diagnostic message matches one of these regular expressions,
        it will be ignored.

        returns -- A list of 'Diagnostic's corresponding to the
        messages indicated in 'output', in the order that they were
        emitted."""

        raise NotImplementedError

    def GetPath(self):
        """Return the location of the executable.

        returns -- A string giving the location of the executable.
        This location is the one that was specified as the 'path'
        argument to '__init__'."""

        return self._path

    def GetOptions(self):
        """Return the list of compilation options.

        returns -- A list of strings giving the compilation options
        specified when the 'Compiler' was constructed."""

        return self._options

    def SetOptions(self, options):
        """Reset the list of compiler options.

        'options' -- A list of strings indicating options to the
        compiler, or 'None' if there are no options."""

        self._options = options

    def GetLDFlags(self):
        """Return the list of link options.

        returns -- A list of strings giving the link options
        specified when the 'Compiler' was constructed."""

        return self._ldflags

    def SetLDFlags(self, ldflags):
        """Reset the list of link options.

        'ldflags' -- A list of strings indicating options to the
        linker, or 'None' if there are no flags."""

        self._ldflags = ldflags

    def GetExecutableExtension(self):
        """Return the extension for executables.

        returns -- The extension (including leading '.', if
        applicable) for executable files created by this compiler."""

        if sys.platform == "win32":
            return ".exe"
        else:
            return ""

    def GetObjectExtension(self):
        """Return the extension for object files.

        returns -- The extension (including leading '.', if
        applicable) for object files created by this compiler."""

        if sys.platform == "win32":
            return ".obj"
        else:
            return ".o"

    def _GetModeSwitches(self, mode):
        """Return the compilation switches for the compilation 'mode'.

        'mode' -- The compilation mode (one of 'Compiler.modes').

        returns -- A sequence of strings indicating the switches that
        are used to indicate the compilation mode."""

        if mode == self.MODE_PREPROCESS:
            return ["-E"]
        elif mode == self.MODE_COMPILE:
            return ["-S"]
        elif mode == self.MODE_ASSEMBLE:
            return ["-c"]

        # Other modes require no special option.
        return []

class SourcePosition:
    """A 'SourcePosition' indicates a location in source code.

    A 'SourcePosition' consists of:

    - A file name.  The file name is a string.  It may be an absolute
      or relative path.  If no file name is available, the file name
      is the empty string.

    - A line number, indexed from one.  If no line number is
      available, the line number is zero.

    - A column number, indexed from one.  If no column number is
      available, the column nubmer is zero."""

    def __init__(self, file, line, column):
        """Construct a new 'SourcePosition'.

        'file' -- The file name.

        'line' -- The line number, indexed from one.  If no line numer
        is availble, use zero for this parameter.

        'column' -- The column number, indexed from one.  If no column
        number is available, use zero for this parameter."""

        self.file = file
        self.line = line
        self.column = column

    def __str__(self):
        """Return a textual representation of this 'SourcePosition'.

        returns -- A string representing this 'SourcePosition'"""

        result = ''
        if self.file:
            result = result + '"%s"' % os.path.split(self.file)[0]
        if self.line:
            if self.file:
                result = result + ', '
            result = result + 'line %d' % self.line
        if self.column:
            result = result + ': %d' % self.column

        return result

class Diagnostic:
    """A 'Diagnostic' is a message issued by a compiler.

    Each 'Diagnostic' has the following attributes:

    - The source position that the compiler associates with the
      diagnostic.

    - The severity of the diagnostic.

    - The message issued by the compiler.

    A 'Diagnostic' may either be an actual diagnostic emitted by a
    compiler, or it may be the pattern for a diagnostic that might be
    emitted.  In the latter case, the message is a regular expression
    indicating the message that should be emitted."""

    def __init__(self, source_position, severity, message):
        """Construct a new 'Diagnostic'.

        'source_position' -- A 'SourcePosition' indicating where the
        diagnostic was issued.  For an expected diagnostic, 'None'
        indicates that the position does not matter.

        'severity' -- A string indicating the severity of the
        diagnostic.  For an expected diagnostic, 'None' indicates
        that the severity does not matter.

        'message' -- For an emitted diagnostic, a string indicating
        the message produced by the compiler.  For an expected
        diagnostic, a string giving a regular expression indicating
        the message that might be emitted.  For an expected
        diagnostic, 'None' indicates that the message does not
        matter."""

        self.source_position = source_position
        self.severity = severity
        self.message = message

    def __str__(self):
        """Return an informal representation of this 'Diagnostic'.

        returns -- A string representing this 'Diagnostic'."""

        if self.source_position:
            source_position_string = str(self.source_position)
        else:
            source_position_string = "<no source position>"

        if self.severity:
            severity_string = self.severity
        else:
            severity_string = "<no severity>"

        if self.message:
            message_string = self.message
        else:
            message_string = "<no message>"

        return '%s: %s: %s' % (source_position_string,
                               severity_string,
                               message_string)

########################################################################
# Compilers
########################################################################

class GCC(Compiler):
    """A 'GCC' is a GNU Compiler Collection compiler."""

    _severities = [ 'warning', 'error' ]
    """The diagnostic severities generated by the compiler.  Order
    matters; the order given here is the order that the
    '_severity_regexps' will be tried."""

    _severity_regexps = {
        'warning' :
          re.compile('^(?P<file>[^:]*):((?P<line>[^:]*):)?'
                     '(\s*(?P<column>[0-9]+):)? '
                     'warning: (?P<message>.*)$'),
        'error':
          re.compile('^(?P<file>[^:]*):((?P<line>[^:]*):)?'
                     '(\s*(?P<column>[0-9]+):)? '
                     '(?P<message>.*)$')
        }
    """A map from severities to compiled regular expressions.  If the
    regular expression matches a line in the compiler output, then that
    line indicates a diagnostic with the indicated severity."""

    _internal_error_regexp = re.compile('Internal (compiler )?error')
    """A compiled regular expression.  When an error message is matched
    by this regular expression, the error message indicates an
    internal error in the compiler."""

    MODE_PRECOMPILE = "precompile"
    """Precompile a header file."""

    modes = Compiler.modes + [MODE_PRECOMPILE]

    def ParseOutput(self, output, ignore_regexps = ()):
        """Return the 'Diagnostic's indicated in the 'output'.

        'output' -- A string giving the output from the compiler.

        'ignore_regexps' -- A sequence of regular expressions.  If a
        diagnostic message matches one of these regular expressions,
        it will be ignored.

        returns -- A list of 'Diagnostic's corresponding to the
        messages indicated in 'output', in the order that they were
        emitted."""

        # Assume there were no diagnostics.
        diagnostics = []
        # Create a file object containing the 'output'.
        f = StringIO.StringIO(output)
        # Reall all of the output, line by line.
        for line in f.readlines():
            for severity in self._severities:
                match = self._severity_regexps[severity].match(line)
                # If it does not look like an error message, skip it.
                if not match:
                    continue

                # Some error messages are ignored.
                ignore = 0
                for ignore_regexp in ignore_regexps:
                    if ignore_regexp.match(match.group()):
                        ignore = 1
                        break
                if ignore:
                    continue

                # An internal error is an error that indicates that
                # the compiler crashed.
                message = match.group('message')
                if (severity == 'error'
                    and self._internal_error_regexp.search(message)):
                    severity = 'internal_error'

                # If there is no line number, then we will not be
                # able to convert it to an integer.
                try:
                    line_number = int(match.group('line'))
                except:
                    line_number = 0

                # See if there is a column number.
                try:
                    column_number = int(match.group('column'))
                except:
                    column_number = 0

                source_position = SourcePosition(match.group('file'),
                                                 line_number,
                                                 column_number)
                diagnostic = Diagnostic(source_position,
                                        severity,
                                        message)
                diagnostics.append(diagnostic)
                break

        return diagnostics

class EDG(Compiler):
    """An 'EDG' is an Edison Design Group compiler."""

    __diagnostic_regexp = re.compile('^"(?P<file>.*)", line (?P<line>.*): '
                                     '(?P<severity>.*): (?P<message>.*)$')

    def ParseOutput(self, output, ignore_regexps = ()):
        """Return the 'Diagnostic's indicated in the 'output'.

        'output' -- A string giving the output from the compiler.

        'ignore_regexps' -- A sequence of regular expressions.  If a
        diagnostic message matches one of these regular expressions,
        it will be ignored.

        returns -- A list of 'Diagnostic's corresponding to the
        messages indicated in 'output', in the order that they were
        emitted."""

        # Assume there were no diagnostics.
        diagnostics = []
        # Create a file object containing the 'output'.
        f = StringIO.StringIO(output)
        # Reall all of the output, line by line.
        for line in f.readlines():
            match = self.__diagnostic_regexp.match(line)
            if match:
                source_position = SourcePosition(match.group('file'),
                                                 int(match.group('line')),
                                                 0)
                diagnostic = Diagnostic(source_position,
                                        match.group('severity'),
                                        match.group('message'))
                diagnostics.append(diagnostic)

        return diagnostics