# pylint: disable=abstract-method

"""
    Functions and objects to help with blocked/chunked time integration.

    These include:
        1. Sparse matrix classes for banded systems
        2. General sparse matrix classes
        3. Specialized solver routines working with banded systems
"""

import warnings
from math import prod, log2, floor

import torch
from torch.nn.functional import pad
import numpy as np

from pyoptmat.utility import mbmm

def newton_raphson_chunk(
    fn, x0, solver, rtol=1e-6, atol=1e-10, miter=100, throw_on_fail=False
):
    """
    Solve a nonlinear system with Newton's method with a tensor for a
    BackwardEuler type chunking operator context manager.

    Args:
      fn (function):        function that returns R, J, and the solver context
      x0 (torch.tensor):    starting point
      solver (ChunkTimeOperatorSolverContext): solver context

    Keyword Args:
      rtol (float):         nonlinear relative tolerance
      atol (float):         nonlinear absolute tolerance
      miter (int):          maximum number of nonlinear iterations
      throw_on_fail (bool): throw exception if solve fails

    Returns:
      torch.tensor:         solution to system of equations
    """
    x = x0
    R, J = fn(x)

    nR = torch.norm(R, dim=-1)
    nR0 = nR
    i = 0

    while (i < miter) and torch.any(nR > atol) and torch.any(nR / nR0 > rtol):
        x -= solver.solve(J, R)
        R, J = fn(x)
        nR = torch.norm(R, dim=-1)
        i += 1

    if i == miter:
        if throw_on_fail:
            raise RuntimeError("Implicit solve did not succeed.")
        warnings.warn("Implicit solve did not succeed.  Results may be inaccurate...")

    return x

class BidiagonalOperator(torch.nn.Module):
    """
    An object working with a Batched block diagonal operator of the type

    .. math::

        \\begin{bmatrix}
        A_1 & 0 & 0 & 0 & \\cdots  & 0\\\\
        B_1 & A_2 & 0 & 0 & \\cdots & 0\\\\
        0 & B_2 & A_3 & 0 & \\cdots & 0\\\\
        \\vdots & \\vdots & \\ddots & \\ddots & \\ddots  & \\vdots \\\\
        0 & 0 & 0 & B_{n-2} & A_{n-1} & 0\\\\
        0 & 0 & 0 & 0 & B_{n-1} & A_n
        \\end{bmatrix}

    that is, a blocked banded system with the main
    diagonal and the first lower diagonal filled

    We use the following sizes:
        - nblk:   number of blocks in the square matrix
        - sblk:   size of each block
        - sbat:   batch size

    Args:
        A (torch.tensor): tensor of shape (nblk,sbat,sblk,sblk)
            storing the nblk main diagonal blocks
        B (torch.tensor): tensor of shape (nblk-1,sbat,sblk,sblk)
            storing the nblk-1 off diagonal blocks
    """

    def __init__(self, A, B, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.A = A
        self.B = B
        self.nblk = A.shape[0]
        self.sbat = A.shape[1]
        self.sblk = A.shape[2]

    @property
    def dtype(self):
        """
        dtype, which is just the dtype of self.diag
        """
        return self.A.dtype

    @property
    def device(self):
        """
        device, which is just the device of self.diag
        """
        return self.A.device

    @property
    def n(self):
        """
        Size of the unbatched square matrix
        """
        return self.nblk * self.sblk

    @property
    def shape(self):
        """
        Logical shape of the dense array
        """
        return (self.sbat, self.n, self.n)

class LUFactorization(BidiagonalOperator):
    """A factorization that uses the LU decomposition of A

    Args:
        A (torch.tensor): tensor of shape (nblk,sbat,sblk,sblk) with the main diagonal
        B (torch.tensor): tensor of shape (nblk-1,sbat,sblk,sblk) with the off diagonal
    """

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self._setup_factorization()

    def _setup_factorization(self):
        """
        Form the factorization...

        Args:
            diag (torch.tensor): diagonal blocks of shape (nblk, sbat, sblk, sblk)
        """
        self.lu, self.pivots, _ = torch.linalg.lu_factor_ex(self.A)

    def forward(self, v):
        """
        Run the solve using the linear algebra type interface
        with the number of blocks and block size squeezed

        Args:
            v (torch.tensor): tensor of shape (sbat, sblk*nblk)
        """
        return (
            self.matvec(
                v.reshape((self.sbat, self.nblk, self.sblk))
                .transpose(0, 1)
                .unsqueeze(-1)
                .contiguous()
            )
            .squeeze(-1)
            .transpose(0, 1)
            .flatten(start_dim=1)
        )

def thomas_solve(lu, pivots, B, v):
    """Simple function implementing a Thomas solve

    Solves in place of v

    Args:
        lu (torch.tensor): factorized diagonal blocks, (nblk,sbat,sblk,sblk)
        pivots (torch.tensor): pivots for factorization
        B (torch.tensor): lower diagonal blocks (nblk-1,sbat,sblk,sblk)
        v (torch.tensor): right hand side (nblk,sbat,sblk)
    """
    i = 0
    v[i] = torch.linalg.lu_solve(lu[i], pivots[i], v[i])
    for i in range(1, lu.shape[0]):
        v[i] = torch.linalg.lu_solve(
            lu[i], pivots[i], v[i] - torch.bmm(B[i - 1], v[i - 1].clone())
        )

    return v

class BidiagonalThomasFactorization(LUFactorization):
    """
    Manages the data needed to solve our bidiagonal system via Thomas
    factorization

    Args:
        A (torch.tensor): tensor of shape (nblk,sbat,sblk,sblk) with the main diagonal
        B (torch.tensor): tensor of shape (nblk-1,sbat,sblk,sblk) with the off diagonal
    """

    def matvec(self, v):
        """
        Complete the backsolve for a given right hand side

        Args:
            v (torch.tensor): tensor of shape (nblk, sbat, sblk, 1)
        """
        return thomas_solve(self.lu, self.pivots, self.B, v)

class BidiagonalPCRFactorization(LUFactorization):
    """
    Manages the data needed to solve our bidiagonal system via parallel cyclic reduction

    Args:
        A (torch.tensor): tensor of shape (nblk,sbat,sblk,sblk) with the main diagonal
        B (torch.tensor): tensor of shape (nblk-1,sbat,sblk,sblk) with the off diagonal
    """

    def matvec(self, v):
        """
        Complete the backsolve for a given right hand side

        Args:
            v (torch.tensor): tensor of shape (nblk, sbat, sblk, 1)
        """
        # We could do this in place if it wasn't for the pad
        self.B = pad(self.B, (0, 0, 0, 0, 0, 0, 1, 0))

        # Now figure out how many powers of 2 we need to complete our matrix
        for s, e in zip(*self._pow2(self.nblk)):
            self.B[s + 1 : e], v[s + 1 : e] = self._solve_block(
                self.lu[s:e], self.pivots[s:e], self.B[s:e], v[s:e]
            )

        # To retain consistent sizes
        self.B = self.B[1:]

        return torch.linalg.lu_solve(self.lu, self.pivots, v)

    def _solve_block(self, lu, pivots, B, v):
        """Solve a subsection of the matrix via PCR

        Args:
            lu (torch.tensor): (ncurr,sbat,sblk,sblk)
            pivots (torch.tensor): (ncurr,sbat,sblk)
            B (torch.tensor): (ncurr,sbat,sblk,sblk)
            v (torch.tensor): (ncurr,sbat,sblk,1)
        """
        # Number of iterations required to reduce this block
        niter = lu.shape[0].bit_length() - 1

        # Add the extra working dimension to the start of everything
        lu = lu.unsqueeze(0)
        pivots = pivots.unsqueeze(0)
        B = B.unsqueeze(0)
        v = v.unsqueeze(0)

        # Actually start reduction!
        for i in range(niter):
            # Reduce RHS
            v[:, 1:] -= mbmm(
                B[:, 1:], torch.linalg.lu_solve(lu[:, :-1], pivots[:, :-1], v[:, :-1])
            )

            # Reduce off diagonal coefficients
            B[:, 2:] = -mbmm(
                B[:, 2:],
                torch.linalg.lu_solve(lu[:, 1:-1], pivots[:, 1:-1], B[:, 1:-1]),
            )

            # Shuffle dimensions
            v = self._cyclic_shift(v, i)
            B = self._cyclic_shift(B, i)
            lu = self._cyclic_shift(lu, i)
            pivots = self._cyclic_shift(pivots, i)

        return B.squeeze(1)[1:], v.squeeze(1)[1:]

    @staticmethod
    def _pow2(n):
        """Calculate submatrix sizes

        Args:
            n (int): number of blocks

        Returns:
            two lists, one giving start block indices and the
            second giving end block indices.

        The first (start,end) pair is the largest power of 2 that fits in
        n.  Subsequent pairs are the largest power of 2 that fit in the remainder
        *with one overlap between the next increment and the previous*.
        """

        def sz(n):
            return 2 ** floor(log2(n))

        start = [0]
        end = [sz(n)]
        n -= end[-1]

        while n > 0:
            cz = sz(n + 1)
            start.append(end[-1] - 1)
            end.append(start[-1] + cz)
            n -= cz - 1

        return start, end

    @staticmethod
    def _cyclic_shift(A, n):
        """Provide a view of the input with a cyclic shift applied

        Args:
            A (torch.tensor): input tensor
            n (int): number of cyclic shifts
        """
        return A.as_strided(
            (A.shape[0] * 2, A.shape[1] // 2) + A.shape[2:],
            (prod(A.shape[2:]), 2 ** (n + 1) * prod(A.shape[2:])) + A.stride()[2:],
        )

class BidiagonalHybridFactorization(BidiagonalPCRFactorization):
    """A factorization approach that switches from PCR to Thomas

    Specifically, this class uses PCR until the PCR chunk size is
    smaller than user provided minimum chunk size.  Then it switches
    to Thomas.

    Args:
        A (torch.tensor): tensor of shape (nblk,sbat,sblk,sblk) with the main diagonal
        B (torch.tensor): tensor of shape (nblk-1,sbat,sblk,sblk) with the off diagonal

    Keyword Args:
        min_size (int): minimum block size, default is zero
    """

    def __init__(self, *args, min_size=0, **kwargs):
        super().__init__(*args, **kwargs)

        # I use < below...
        self.min_size = min_size + 1

    def matvec(self, v):
        """
        Complete the backsolve for a given right hand side

        Args:
            v (torch.tensor): tensor of shape (nblk, sbat, sblk, 1)
        """
        # We could do this in place if it wasn't for the pad
        self.B = pad(self.B, (0, 0, 0, 0, 0, 0, 1, 0))

        # Get the PCR blocks to actually use
        start, end, last = self._pcr_blocks()

        # Do PCR
        for s, e in zip(start, end):
            self.B[s + 1 : e], v[s + 1 : e] = self._solve_block(
                self.lu[s:e], self.pivots[s:e], self.B[s:e], v[s:e]
            )

        # To retain consistent sizes
        self.B = self.B[1:]

        # We still need to solve the first block even if last is 0

        # The actual LU solve for the solution
        v[:last] = torch.linalg.lu_solve(self.lu[:last], self.pivots[:last], v[:last])

        # Now take over for Thomas
        for i in range(last, self.nblk):
            # The .clone() here should not be necessary, but for whatever
            # reason torch autograd give the usual "in place" complaint
            # without it...
            v[i] = torch.linalg.lu_solve(
                self.lu[i],
                self.pivots[i],
                v[i] - torch.bmm(self.B[i - 1], v[i - 1].clone()),
            )

        return v

    def _pcr_blocks(self):
        """Figure out the PCR blocks we are actually going to use"""
        # Figure out which blocks we're going to use
        start, end = self._pow2(self.nblk)
        # These are sorted...
        blk_size = [e - s for e, s in zip(end, start)]
        if blk_size[0] < self.min_size:
            return [], [], 1

        ilast = [i for i, j in enumerate(blk_size) if j < self.min_size]
        if len(ilast) == 0:
            ilast = len(start)
        else:
            ilast = ilast[0]

        start = start[:ilast]
        end = end[:ilast]

        return start, end, end[-1]

class BidiagonalForwardOperator(BidiagonalOperator):
    """
    A batched block banded matrix of the form:

    .. math::

        \\begin{bmatrix}
        A_1 & 0 & 0 & 0 & \\cdots  & 0\\\\
        B_1 & A_2 & 0 & 0 & \\cdots & 0\\\\
        0 & B_2 & A_3 & 0 & \\cdots & 0\\\\
        \\vdots & \\vdots & \\ddots & \\ddots & \\ddots  & \\vdots \\\\
        0 & 0 & 0 & B_{n-2} & A_{n-1} & 0\\\\
        0 & 0 & 0 & 0 & B_{n-1} & A_n
        \\end{bmatrix}

    that is, a blocked banded system with the main
    diagonal and first lower block diagonal filled

    We use the following sizes:
        - nblk: number of blocks in the square matrix
        - sblk: size of each block
        - sbat: batch size

    Args:
        A (torch.tensor): tensor of shape (nblk,sbat,sblk,sblk)
            storing the nblk diagonal blocks
        B (torch.tensor): tensor of shape (nblk-1,sbat,sblk,sblk)
            storing the nblk-1 off diagonal blocks
    """

    def __init__(self, *args, inverse_operator=BidiagonalThomasFactorization, **kwargs):
        super().__init__(*args, **kwargs)
        self.inverse_operator = inverse_operator

    def to_diag(self):
        """
        Convert to a SquareBatchedBlockDiagonalMatrix, for testing
        or legacy purposes
        """
        return SquareBatchedBlockDiagonalMatrix([self.A, self.B], [0, -1])

    def forward(self, v):
        """
        :math:`A \\cdot v` in an efficient manner

        Args:
            v (torch.tensor):   batch of vectors
        """
        # Reshaped v
        vp = (
            v.view(self.sbat, self.nblk, self.sblk)
            .transpose(0, 1)
            .reshape(self.sbat * self.nblk, self.sblk)
            .unsqueeze(-1)
        )

        # Actual calculation
        b = torch.bmm(self.A.view(-1, self.sblk, self.sblk), vp)
        b[self.sbat :] += torch.bmm(
            self.B.view(-1, self.sblk, self.sblk), vp[: -self.sbat]
        )

        return (
            b.squeeze(-1)
            .view(self.nblk, self.sbat, self.sblk)
            .transpose(1, 0)
            .flatten(start_dim=1)
        )

    def inverse(self):
        """
        Return an inverse operator
        """
        return self.inverse_operator(self.A, self.B)

class ChunkTimeOperatorSolverContext:
    """
    Context manager for solving sparse chunked time systems

    Args:
        solve_method:   one of "dense" or "direct"
    """

    def __init__(self, solve_method):

        if solve_method not in ["dense", "direct"]:
            raise ValueError("Solve method must be one of dense or direct")
        self.solve_method = solve_method

    def solve(self, J, R):
        """
        Actually solve Jx = R

        Args:
            J (BidiagonalForwardOperator):  matrix operator
            R (torch.tensor):       right hand side
        """
        if self.solve_method == "dense":
            return self.solve_dense(J, R)
        if self.solve_method == "direct":
            return self.solve_direct(J, R)
        raise RuntimeError("Unknown solver method...")

    def solve_dense(self, J, R):
        """
        Highly inefficient solve where we first convert to a dense tensor

        Args:
            J (BidiagonalForwardOperator):  matrix operator
            R (torch.tensor):       right hand side
        """
        return torch.linalg.solve_ex(J.to_diag().to_dense(), R)[0]

    def solve_direct(self, J, R):
        """
        Solve with a direct factorization

        Args:
            J (BidiagonalForwardOperator):  matrix operator
            R (torch.tensor):       right hand side
        """
        M = J.inverse()
        return M(R)

class SquareBatchedBlockDiagonalMatrix:
    """
    A batched block diagonal matrix of the type

    .. math::

        \\begin{bmatrix}
        A_1 & B_1 & 0 & 0\\\\
        C_1 & A_2 & B_2 & 0 \\\\
        0 & C_2 & A_3 & B_3\\\\
        0 & 0 & C_3 & A_4
        \\end{bmatrix}

    where the matrix has diagonal blocks of non-zeros and
    can have arbitrary numbers of filled diagonals

    Additionally, this matrix is batched.

    We use the following sizes:
        - nblk: number of blocks in the each direction
        - sblk: size of each block
        - sbat: batch size

    Args:
        data (list of tensors):     list of tensors of length ndiag.
                                    Each tensor
                                    has shape :code:`(nblk-abs(d),sbat,sblk,sblk)`
                                    where d is the diagonal number
                                    provided in the next input
        diags (list of ints):       list of ints of length ndiag.
                                    Each entry gives the diagonal
                                    for the data in the corresponding
                                    tensor.  These values d can
                                    range from -(n-1) to (n-1)
    """

    def __init__(self, data, diags):
        # We will want this in order later
        iargs = np.argsort(diags)

        self.data = [data[i] for i in iargs]
        self.diags = [diags[i] for i in iargs]

        self.nblk = self.data[0].shape[0] + abs(self.diags[0])
        self.sbat = self.data[0].shape[1]
        self.sblk = self.data[0].shape[-1]

    @property
    def dtype(self):
        """
        dtype, as reported by the first entry in self.data
        """
        return self.data[0].dtype

    @property
    def device(self):
        """
        device, as reported by the first entry in self.device
        """
        return self.data[0].device

    @property
    def n(self):
        """
        Size of the unbatched square matrix
        """
        return self.nblk * self.sblk

    @property
    def shape(self):
        """
        Logical shape of the dense array
        """
        return (self.sbat, self.n, self.n)

    @property
    def nnz(self):
        """
        Number of logical non-zeros (not counting the batch dimension)
        """
        return sum(
            self.data[i].shape[0] * self.sblk * self.sblk
            for i in range(len(self.diags))
        )

    def to_dense(self):
        """
        Convert the representation to a dense tensor
        """
        A = torch.zeros(*self.shape, dtype=self.dtype, device=self.device)

        # There may be a more clever way than for loops, but for now
        for d, data in zip(self.diags, self.data):
            for k in range(self.nblk - abs(d)):
                if d <= 0:
                    i = k - d
                    j = k
                else:
                    i = k
                    j = k + d
                A[
                    :,
                    i * self.sblk : (i + 1) * self.sblk,
                    j * self.sblk : (j + 1) * self.sblk,
                ] = data[k]

        return A

    def to_batched_coo(self):
        """
        Convert to a torch sparse batched COO tensor

        This is done in a weird way.  torch recognizes "batch" dimensions at
        the start of the tensor and "dense" dimensions at the end (with "sparse"
        dimensions in between).  batch dimensions can/do have difference indices,
        dense dimensions all share the same indices.  We have the latter situation
        so this is setup as a tensor with no "batch" dimensions, 2 "sparse" dimensions,
        and 1 "dense" dimension.  So it will be the transpose of the shape of the
        to_dense function.
        """
        inds = torch.zeros(2, self.nnz)
        data = torch.zeros(self.nnz, self.sbat, dtype=self.dtype, device=self.device)

        # Order doesn't matter, nice!
        c = 0
        chunk = self.sblk * self.sblk
        for d, bdata in zip(self.diags, self.data):
            for i in range(bdata.shape[0]):
                data[c : c + chunk] = bdata[i].flatten(start_dim=1).t()

                offset = (i + abs(d)) * self.sblk

                if d < 0:
                    roffset = offset
                    coffset = i * self.sblk
                else:
                    roffset = i * self.sblk
                    coffset = offset

                inds[0, c : c + chunk] = (
                    torch.repeat_interleave(
                        torch.arange(
                            0, self.sblk, dtype=torch.int64, device=self.device
                        ).unsqueeze(-1),
                        self.sblk,
                        -1,
                    ).flatten()
                    + roffset
                )
                inds[1, c : c + chunk] = (
                    torch.repeat_interleave(
                        torch.arange(
                            0, self.sblk, dtype=torch.int64, device=self.device
                        ).unsqueeze(0),
                        self.sblk,
                        0,
                    ).flatten()
                    + coffset
                )

                c += chunk

        return torch.sparse_coo_tensor(
            inds,
            data,
            dtype=self.dtype,
            device=self.device,
            size=(self.n, self.n, self.sbat),
        ).coalesce()

    def to_unrolled_csr(self):
        """
        Return a list of CSR tensors with length equal to the batch size

        """
        coo = self.to_batched_coo()
        return [
            torch.sparse_coo_tensor(coo.indices(), coo.values()[:, i]).to_sparse_csr()
            for i in range(self.sbat)
        ]


import os
import logging
log = logging.getLogger('zen.ExampleDeviceFacade')

from zope.interface import implements

from Products.Zuul.facades import ZuulFacade
from Products.Zuul.utils import ZuulMessageFactory as _t

from .interfaces import IExampleDeviceFacade
from .interfaces import ImyAppFacade

# The ZuulFacade class name, ExampleDeviceFacade, must match the name specified
#  in the factory field of an adapter stanza in configure.zcml
#  ie. ExampleDeviceFacade
#
# The implements line must match that specified in the same adapter's "provides" field
#    and in interfaces.py, must match the IFacade class defined there ( ie. IExampleDeviceFacade )

class ExampleDeviceFacade(ZuulFacade):
    implements(IExampleDeviceFacade)

# The method name add_ExampleDevice and its parameters must match those defined in
#    interfaces.py and routers.py
# It is the following method that ACTUALLY does the work of adding a device

    def add_ExampleDevice(self, deviceIp, community, comment):
        """Add a device of class ExampleDevice """

        deviceRoot = self._dmd.getDmdRoot("Devices")
        device = deviceRoot.findDeviceByIdExact(deviceIp)
        if device:
            return False, _t("A device named %s already exists." % deviceIp)

        zProperties = {
            'zSnmpCommunity': community,
            'zPythonClass': 'ZenPacks.skills1st.MenuExamples.ExampleDevice',
            }

        perfConf = self._dmd.Monitors.getPerformanceMonitor('localhost')

        # addDeviceCreationJob is a method defined in $ZENHOME/Products/ZenModel/PerformanceConf.py
        # Parameters here are not exhustive. discoverProto='snmp' ensures device is modeled as well
        #   as discovered into the Zope database

        jobStatus = perfConf.addDeviceCreationJob(
            deviceName=deviceIp,
            devicePath='/Example/TestClass',
            discoverProto='snmp',
            comments=comment,
            zProperties=zProperties)

        return True, jobStatus.id

class myAppFacade(ZuulFacade):
    implements(ImyAppFacade)

    # Note that the the facade function, myFacadeFunc has 3 parameters
    #  The object is passed in addition to the comment and rackSlot

    def myFacadeFunc(self, ob, comments, rackSlot):
        """ Modifies comments and rackSlot attributes for a device """

        ob.comments = comments
        ob.rackSlot = rackSlot

        return True, _t(" Comments and rackSlot attributes set for device %s" % (ob.id))


'''
This module defines the generic functions and abstract class used to manage
file access in CRDS.
'''
from collections.abc import Mapping
import functools
import warnings
import re

# ================================================================================================

import datetime
from astropy.time import Time

# ================================================================================================

# from astropy.utils.exceptions import AstropyUserWarning    # deferred

# ================================================================================================

from crds.core import exceptions, config, log, utils, timestamp

# ================================================================================================

DUPLICATES_OK = ["COMMENT", "HISTORY", "NAXIS","EXTNAME","EXTVER"]
APPEND_KEYS = ["COMMENT", "HISTORY"]

# ===========================================================================

# The point of hijack_warnings is to remap warnings from other packages to CRDS
# so that they are counted and logged and visible in web output.
# NOTE:  hijack_warnings needs to be nestable
# XXX: hijack_warnings is non-reentrant and FAILS with THREADS

def hijack_warnings(func):
    """Decorator that redirects warning messages to CRDS warnings."""
    @functools.wraps(func)
    def wrapper(*args, **keys):
        """Reassign warnings to CRDS warnings prior to executing `func`,  restore
        warnings state afterwards and return result of `func`.
        """
        # warnings.resetwarnings()
        from astropy.utils.exceptions import AstropyUserWarning
        with warnings.catch_warnings():
            old_showwarning = warnings.showwarning
            warnings.showwarning = hijacked_showwarning
            warnings.simplefilter("always", AstropyUserWarning)
            try:
                from stdatamodels.validate import ValidationWarning
            except:
                log.verbose_warning(
                    "stdatamodels ValidationWarning import failed.  "
                    "Not a problem for HST.",
                    verbosity=70)
            else:
                warnings.filterwarnings("always", r".*", ValidationWarning, r".*jwst.*")
                if not config.ALLOW_SCHEMA_VIOLATIONS:
                    warnings.filterwarnings("error", r".*is not one of.*", ValidationWarning, r".*jwst.*")
            try:
                result = func(*args, **keys)
            finally:
                warnings.showwarning = old_showwarning
        return result
    return wrapper

def hijacked_showwarning(message, category, filename, lineno, *args, **keys):
    """Map the warnings.showwarning plugin function parameters onto log.warning."""
    try:
        scat = str(category).split(".")[-1].split("'")[0]
    except Exception:
        scat = category
    try:
        sfile = str(filename).split(".egg")[-1].split("site-packages")[-1].replace("/",".").replace(".py", "")
        while sfile.startswith(("/",".")):
            sfile = sfile[1:]
    except Exception:
        sfile = filename
    message = str(message).replace("\n","")
    log.warning(scat, ":", sfile, ":", message)

# ----------------------------------------------------------------------------------------------
#
# Cross-strapping is used to support different synonyms for the same keyword:
#
#   FITS
#   JWST data model dotted path
#   Ad hoc names appearing in un-modeled reference files
#
# It's complicated because of inconsistent support for data model nomenclature both
# within and between projects;  it's currently a JWST-only concept but is implemented
# in a pluggable way.
#
# For "un-modeled" references or if the decision is made to drop datamodels use, each
# observatories locator module can define CROSS_STRAPPED_KEYWORDS.  See an example, jwst.
#
# For projects supporting the datamodels,  the schema is used to define the correspondence
# between data model dotted paths and the keyword of the underlying file format.
#
def cross_strap_header(header):
    """Set up keyword equivalencies in a copy of `header`.  Ensure both FITS
    and datamodel dotted path variants are defined for each keyword.
    Also add variations defined by observatory locator module
    CROSS_STRAPPED_KEYWORDS.
    """
    crossed = dict(header)
    try:
        locator = utils.header_to_locator(header)
    except Exception:
        log.verbose_warning(
            "Cannot identify observatory from header. Skipping keyword aliasing")
        return crossed
    equivalency_pairs = locator.get_cross_strapped_pairs(header)
    for pair in equivalency_pairs:
        _cross_strap_pair(crossed, pair)
    return crossed

def equivalence_dict_to_pairs(equivalent_keywords_dict):
    """Convert a dictionary mapping master keywords to equivalents to
    a list of keyword pairs that should be cross-strapped.
    """
    pairs = []
    log.verbose("Explicitly cross_strapped_keywords:",
                log.PP(equivalent_keywords_dict), verbosity=90)
    for master, slaves in equivalent_keywords_dict.items():
        for slave in slaves:
            if master != slave:
                pairs.append((master, slave))
                pairs.append((slave, master))
    return pairs

def _cross_strap_pair(header, keyword_pair):
    """Mutate `header` using (master, slave) `keyword_pair` so that slave
    duplicates master's value under slave's name IFF master is defined
    in header and slave is not.
    """
    master_key, slave_key = keyword_pair
    master_val = header.get(master_key, "UNDEFINED")
    slave_val =  header.get(slave_key, "UNDEFINED")
    if slave_val == "UNDEFINED" and master_val != "UNDEFINED":
        header[slave_key] = master_val

# ----------------------------------------------------------------------------------------------

def convert_to_eval_header(header):
    """To support using file headers in eval expressions,  two changes need to be made:

    1. JWST data model keywords, dotted paths, need to be translated to underscored paths.
    This makes them valid identifiers instead of non-existent nested objects when evaled.

    2. Numerial values that CRDS has conditioned into strings need to be converted back to
    numerical values so they can be used in arithmetic expressions.
    """
    header = _destringize_numbers(header)
    header = _convert_dotted_paths(header)
    return header

def _destringize_numbers(header):
    """Convert string values in `header` that work as numbers back into
    ints and floats.
    """
    with_numbers = {}
    for key, val in header.items():
        try:
            val = int(val)
        except:
            try:
                val = float(val)
            except:
                pass
        with_numbers[key] = val
    return with_numbers

def _convert_dotted_paths(header):
    """Convert header dotted-path keys into valid Python identifiers
    (for eval()) by using underscores instead of periods and add to
    existing contents of `header`.
    """
    cleaned = dict(header)
    for key, val in header.items():
        clean = re.sub(r"([A-Za-z][A-Za-z0-9_]*)\.", r"\1_", key)
        cleaned[clean] = val
    return cleaned

# ----------------------------------------------------------------------------------------------

def ensure_keys_defined(header, needed_keys=(), define_as="UNDEFINED"):
    """Define any keywords from `needed_keys` which are missing in `header`,  or defined as 'UNDEFINED',
    as `default`.

    Normally this defines missing keys as UNDEFINED.

    It can be used to redefine UNDEFINED as something else,  like N/A.
    """
    header = dict(header)
    for key in needed_keys:
        if key not in header or header[key] in ["UNDEFINED", None]:
            header[key] = define_as
    return header

# ================================================================================================

class AbstractFile:

    format = "ABSTRACT"

    def __init__(self, filepath, original_name=None, observatory=None):
        """Create an AbstractFile.

        Required for use:

        filepath          working path for this file

        Retained for debug:

        original_name     abstract,  possibly more readable name to infer type and observatory w/o io
        observatory       observatory this file belongs to
        """
        # This __init__ should be kept fast
        self.filepath = filepath
        self.original_name = original_name
        self.observatory = observatory
        self.array_formats = {}

    def _unsupported_file_op_error(self, method):
        return exceptions.UnsupportedFileOpError(
            "Method", repr(method), "is not supported for file format", repr(self.format))

    def add_checksum(self):
        """Add checksum to`self.filepath`."""
        raise self._unsupported_file_op_error("add_checksum")

    def remove_checksum(self):
        """Remove checksum from`self.filepath`."""
        raise self._unsupported_file_op_error("remove_checksum")

    def verify_checksum(self):
        """Verify checksum in `self.filepath`."""
        raise self._unsupported_file_op_error("verify_checksum")

    def get_format(self):
        """Return a string describing the structure of file at `filepath`,  intended
        for file overview describing generic array structure.
        """
        raise self._unsupported_file_op_error("get_format")

    def get_array_properties(self, array_name, keytype="A"):
        """Return a basic properties dictionary for array named `array_name`."""
        raise self._unsupported_file_op_error("get_array_properties")

    def get_array(self, array_name):
        """Return the array object corresponding to array selected by `array_id_info`."""
        raise self._unsupported_file_op_error("get_array")

    # ----------------------------------------------------------------------------------------------

    def getval(self, key, **keys):
        """Return a single metadata value from `key` of file at `filepath`."""
        return self.get_header((key,), **keys)[key]

    def setval(self, key, value):
        """Set the value of a single metadata key,  nominally in the 'primary header'."""
        raise self._unsupported_file_op_error("setval")

    # ----------------------------------------------------------------------------------------------

    def get_header(self, needed_keys, **keys):
        """Return dictionary of metadata for this file,  e.g. FITS primary header
         dictionary featuring keywords `needed_keys`.
         """
        raw_header = self.get_raw_header(needed_keys, **keys)
        reduced_header = self._reduce_header(raw_header, needed_keys)
        crossed_header = cross_strap_header(reduced_header)
        crossed_header["FILE_FORMAT"] = \
            self.__class__.__name__[:-len("File")].upper()
        return crossed_header

    def get_raw_header(self, needed_keys, **keys):
        """Return the metadata dictionary associated with this file,  nominally a dict
        describing the FITS header, ASDF tree, or JSON or YAML contents.
        """
        raise self._unsupported_file_op_error("get_raw_header")
    # ----------------------------------------------------------------------------------------------

    def _reduce_header(self, old_header, needed_keys=()):
        """Limit `header` to `needed_keys`,  converting all keys to upper case
        and making note of any significant duplicates, and adding any missing
        `needed_keys` as UNDEFINED.

        To detect duplicates,  use an item list for `old_header`,  not a dict.
        """
        needed_keys = tuple(key.upper() for key in needed_keys)
        header = {}
        if isinstance(old_header, dict):
            old_header = old_header.items()
        for (key, value) in old_header:
            key = str(key).upper()
            value = str(value)
            if (not needed_keys) or key in needed_keys:
                if key in header and header[key] != value:
                    if key not in DUPLICATES_OK:
                        log.verbose_warning("Duplicate key", repr(key), "in", repr(self.filepath),
                                            "using", repr(header[key]), "not", repr(value), verbosity=70)
                        continue
                    elif key in APPEND_KEYS:
                        header[key] += "\n" + value
                else:
                    header[key] = value
        return ensure_keys_defined(header, needed_keys)

    # ----------------------------------------------------------------------------------------------

    def to_simple_types(self, tree):
        """Convert a tree structure to a flat dictionary of simple types with dotted path tree keys."""
        result = dict()
        for key in tree:
            if not isinstance(key, str):  # skip non-string keys
                continue
            value = tree[key]
            if isinstance(value, Mapping):
                nested = self.to_simple_types(value)
                for nested_key, nested_value in nested.items():
                    result[str(key.upper() + "." + nested_key)] = nested_value
            else:
                result[str(key.upper())] = self._simple_type(value)
        return result

    def _simple_type(self, value):
        """Convert ASDF values to simple strings, where applicable,  exempting potentially large values."""
        if isinstance(value, (str, int, float, complex)):
            rval = str(value)
        elif isinstance(value, (list, tuple)):
            rval = tuple(self._simple_type(val) for val in value)
        elif isinstance(value, (datetime.datetime, Time)):
            rval = timestamp.reformat_date(value).replace(" ", "T")
        else:
            rval = "SUPRESSED_NONSTD_TYPE: " + repr(str(value.__class__.__name__))
        return rval

    def get_asdf_standard_version(self):
        """
        Return the ASDF Standard version associated with this file as a string,
        or `None` if the file is neither an ASDF file nor contains an embedded
        ASDF file.
        """
        return None


from typing import List

class Solution:
    def exclusiveTime(self, n: int, logs: List[str]) -> List[int]:
        """
        Stack

        https://leetcode.com/problems/exclusive-time-of-functions/solution/
        https://leetcode.com/problems/exclusive-time-of-functions/discuss/105100/Python-Straightforward-with-Explanation
        """
        exclusive_times = [0] * n

        stack = []  # keep track of the "current function"
        prev_t = 0

        for log in logs:
            index, op, t = log.split(':')
            index = int(index)
            t = int(t)

            # Don't have to do like this
            # We don't care if the index of function is equal to current one
            # We just settle the time
            # if stack and stack[-1][0] == index:
            #     exclusive_times

            if op == 'start':
                if stack:
                    exclusive_times[stack[-1]] += t - prev_t
                stack.append(index)
                prev_t = t
            else:
                exclusive_times[stack.pop()] += t - prev_t + 1
                prev_t = t + 1

        return exclusive_times

# Runtime: 80 ms, faster than 30.60% of Python3 online submissions for Exclusive Time of Functions.
# Memory Usage: 14.5 MB, less than 100.00% of Python3 online submissions for Exclusive Time of Functions.


# -*- coding: utf-8 -*-
"""
Created on Thu Sep 15 11:16:36 2016

@author: admin
"""
from infodens.feature_extractor.feature_extractor import featid, Feature_extractor
from scipy import sparse
import argparse

class Lexical_features(Feature_extractor):

    def computeDensity(self, taggedSentences, jnrv):

        densities = []
        for sent in taggedSentences:
            if len(sent) is 0:
                densities.append(0)
            else:
                jnrvList = [tagPOS for tagPOS in sent if tagPOS in jnrv]
                densities.append(float(len(sent) - len(jnrvList)) / len(sent))

        return sparse.lil_matrix(densities).transpose()

    def parsePOSArgs(self, args):

        parser = argparse.ArgumentParser(description='Lexical density args')
        parser.add_argument("-pos_train", help="Path for POS tagged train sentences.",
                            type=str, default="")
        parser.add_argument("-pos_test", help="Path for POS tagged test sentences.",
                            type=str, default="")
        parser.add_argument("-pos_tags", help="Comma separated list of POS tags.",
                            type=str, default="")

        argsOut = parser.parse_args(args.split())
        return argsOut.pos_train, argsOut.pos_test, argsOut.pos_tags.split(",")

    @featid(3)
    def lexicalDensity(self, argString, preprocessReq=0):
        trainPOS, testPOS, jnrv = self.parsePOSArgs(argString)

        if preprocessReq:
            # Request all preprocessing functions to be prepared
            self.preprocessor.getPOStagged(trainPOS)
            self.testPreprocessor.getPOStagged(testPOS)
            return 1

        '''
        The frequency of tokens that are not nouns, adjectives, adverbs or verbs.
        This is computed by dividing the number of tokens tagged with POS tags
        that do not start with J, N, R or V by the number of tokens in the chunk
        '''
        trainLexDens = self.computeDensity(self.preprocessor.getPOStagged(trainPOS), jnrv)
        testLexDens = self.computeDensity(self.testPreprocessor.getPOStagged(testPOS), jnrv)

        return trainLexDens, testLexDens, "Lexical density"

    def getLexicalRichness(self, sents):
        sentRichness = []
        for sentence in sents:
            if len(sentence) is 0:
                sentRichness.append(0)
            else:
                sentRichness.append(float(len(set(sentence)))/len(sentence))
        return sparse.lil_matrix(sentRichness).transpose()

    @featid(11)
    def lexicalRichness(self, argString, preprocessReq=0):
        '''
        The ratio of unique tokens in the sentence over the sentence length.
        '''

        if preprocessReq:
            # Request all preprocessing functions to be prepared
            self.testPreprocessor.gettokenizeSents()
            self.preprocessor.gettokenizeSents()
            return 1

        #TODO : Lemmatize tokens?
        trainSentRichness = self.getLexicalRichness(self.preprocessor.gettokenizeSents())
        testSentRichness = self.getLexicalRichness(self.testPreprocessor.gettokenizeSents())

        return trainSentRichness, testSentRichness, "Type token ratio"

    def getLexicalToTokens(self, sents, lexicalTags):

        lexicalTokensRatio = []
        for sentence in sents:
            lexicalCount = 0
            for tagPOS in sentence:
                if tagPOS in lexicalTags:
                    lexicalCount += 1
            if len(sentence) is 0:
                lexicalTokensRatio.append(0)
            else:
                lexicalTokensRatio.append(float(lexicalCount) / len(sentence))

        return sparse.lil_matrix(lexicalTokensRatio).transpose()

    @featid(12)
    def lexicalToTokens(self, argString, preprocessReq=0):
        '''
        The ratio of lexical words to tokens in the sentence.
        '''
        trainPOS, testPOS, lexicalTags = self.parsePOSArgs(argString)

        if preprocessReq:
            # Request all preprocessing functions to be prepared
            self.preprocessor.getPOStagged(trainPOS)
            self.testPreprocessor.getPOStagged(testPOS)
            return 1

        trainLexTokRatio = self.getLexicalToTokens(self.preprocessor.getPOStagged(trainPOS), lexicalTags)
        testLexTokRatio = self.getLexicalToTokens(self.testPreprocessor.getPOStagged(testPOS), lexicalTags)

        return trainLexTokRatio, testLexTokRatio, "Ratio of lexical words to tokens"


from flask import Flask, request, jsonify
from flask_cors import CORS, cross_origin
import util

app = Flask(__name__)
CORS(app, resources={r"/api/*": {"origins": "*"}})
app.config['CORS HEADERS'] = 'Content-Type'

@app.route('/', methods=['GET'])
def index():
    return ""

@app.route('/classify_image', methods=['GET', 'POST'])
#@cross_origin()
def classify_image():
    image_data = request.form['image_data']

    response = jsonify(util.classify_image(image_data))

    response.headers.add('Access-Control-Allow-Origin', '*')

    return response

if __name__ == "__main__":
    print("Starting Python Flask Server For Sports Celebrity Image Classification")
    util.load_saved_artifacts()
    app.run(port=5000)


import argparse

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--local_rank",
                        type=int,
                        default=-1,
                        help="local_rank for distributed training on gpus")
    args = parser.parse_args()

    print(f"hello from local rank: {args.local_rank}")


#!/usr/bin/python
import pathlib

from bs4 import BeautifulSoup
import os.path
import os
import sys
import requests
import argparse
import json
import time

# whatever im gonna make it a 4chan scraper now

class FourchanDL:
    def __init__(self, args, config):
        self._args = args
        self._config = config
        self._url = args.url
        self._soup = self.prepare_soup()
        self._dl_count = 0
        self._skip_count = 0
        self._name = args.name
        self._format = self.get_format()

    def prepare_soup(self):
        """
        Gets the bs4 object from the url
        :return: beautifulsoup object of the site
        :rtype: beautifulsoup
        """

        html_get = requests.get(self._url)
        if not html_get.ok:
            sys.exit("Invalid url")
        return BeautifulSoup(html_get.text, 'html.parser')

    def get_format(self):
        """
        Gets the output format according to the priorities
        :return: output format
        :rtype: string
        """

        if self._args.format is not None:
            # 1. -f argument
            form = self._args.format
            if self._args.set_default_format is True:
                self._config['format'] = self._args.format
                print("Set", self._args.format, "as default format")
        else:
            # 2. default format
            form = self._config['format']
        return form

    def process_format(self, post):
        """
        Performs substitution on the format
        :return: absolute path of the current image being exported
        :rtype: string
        """

        # get the original file extension
        extension = "." + post.find(class_="fileText").a.text.split('.')[-1]

        # get some variables
        post_filename = post.find(class_="fileText").a.text.replace(extension, '')
        post_id = post.get('id')[2:]
        post_count = str(self._dl_count + self._skip_count + 1)
        op = self._soup.find(class_='opContainer')
        op_id = op.get('id')[2:]
        op_subject = op.find(class_="subject").text.replace('/', ' ')

        # substitutions
        path_format = self._format
        path_format = path_format.replace("%filename", post_filename)
        path_format = path_format.replace("%id", post_id)
        path_format = path_format.replace("%count", post_count)
        path_format = path_format.replace("%opid", op_id)
        path_format = path_format.replace("%opsubject", op_subject)
        if "%name" in path_format:
            try:
                path_format = path_format.replace("%name", self._name)
            except TypeError:
                sys.exit("Name must be set (use -n)")

        path_format = path_format + extension
        return os.path.expanduser(path_format)

    def download_post(self, post):
        """
        Downloads an image
        :return: True if the image was downloaded, False if it already exists
        :rtype: bool
        """

        img_path = self.process_format(post)
        img_link = "http:" + post.find(class_="fileThumb").get('href')

        # will make a folder if it doesn't exist
        os.makedirs('/'.join(img_path.split('/')[:-1]) + '/', exist_ok=True)

        # archived posts will have a gif that messes everything up
        if img_link.split('/')[-1] == "archived.gif":
            return False

        # download:
        if not os.path.exists(img_path):
            img_get = requests.get(img_link)
            if img_get.ok:
                with open(img_path, 'wb') as file:
                    file.write(img_get.content)
                return True
        return False

    def print_stats(self):
        if self._dl_count != 0:
            print("Downloaded", self._dl_count, "images\nSkipped", self._skip_count, "images")
        elif not self._args.quiet:
            print("Nothing to download\nSkipped", self._skip_count, "images")

    def run(self):
        for post in self._soup.find_all(class_='postContainer'):
            if post.find('img') is not None:
                download_successful = self.download_post(post)
                if download_successful:
                    if not self._args.quiet:
                        print('Downloaded post', post.get('id')[2:], 'to', self.process_format(post))
                    self._dl_count += 1
                else:
                    if not self._args.quiet:
                        print('Skipped post', post.get('id')[2:], '-', self.process_format(post))
                    self._skip_count += 1

def get_config_path():
    if sys.platform == "win32":
        # untested!
        return str(pathlib.Path.home()) + "/" + "AppData/Roaming/4chan-dl/"
    else:
        return os.path.expanduser("~/.config/4chan-dl/")

def load_config():
    config_path = get_config_path()
    config_file = config_path + "config.json"
    config = {}
    if os.path.isfile(config_file):
        with open(config_file, 'r') as file:
            config = json.load(file)
    return config

def export_config(config):
    config_path = get_config_path()
    config_file = config_path + "config.json"
    if not os.path.exists(config_path):
        os.makedirs(config_path)
    with open(config_file, 'w') as file:
        json.dump(config, file)

def get_args():
    parser = argparse.ArgumentParser(description="4chan image downloader")
    parser.add_argument("-f", "--format", type=str, help="File naming")
    parser.add_argument("-n", "--name", type=str, help="Set the name variable")
    parser.add_argument("-q", "--quiet", action=argparse.BooleanOptionalAction, help="Less verbose")
    parser.add_argument("-w", "--watch", type=int, default=0, help="Time between retries. 0 will run the program once")
    parser.add_argument("--set-default-format", action=argparse.BooleanOptionalAction,
                        help="Set the current directory format as default")
    parser.add_argument("url", type=str, help="Thread URL")

    return parser.parse_args()

def main():
    args = get_args()
    default_config = {
        "format": "%filename"
    }

    config = load_config()
    if config == {}:
        print("No config file found. Creating default config")
        config = default_config

    while True:
        dl = FourchanDL(args, config)

        dl.run()
        dl.print_stats()
        if args.watch == 0:
            break
        time.sleep(args.watch)

    export_config(config)

if __name__ == "__main__":
    main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division, print_function

import numpy as np
from scipy.interpolate import interp1d
from scipy.signal import medfilt
import astropy.units as u
import astropy.constants as c

# Plotting
import matplotlib; matplotlib.use('TkAgg')
import matplotlib.pyplot as pl
import seaborn; seaborn.set_style('ticks')
import lineid_plot

import matplotlib as mpl
from matplotlib.ticker import FormatStrFormatter
params = {
   'axes.labelsize': 24,
   'font.size': 24,
   'legend.fontsize': 24,
   'xtick.labelsize': 24,
   'ytick.labelsize': 24,
   'text.usetex': False,
   'figure.figsize': [16, 16/1.618]
   }
mpl.rcParams.update(params)

def main():
    # Small script to plot SED, photometry and spectrum

    # Load data
    spectrum = np.genfromtxt("../data/spectroscopy/stitched_spectrum_bin5.dat")
    wl_spec = spectrum[:, 0] #* u.AA
    flux_spec = spectrum[:, 1] #* (u.erg/ u.s / u.cm**2 / u.AA)
    error_spec = spectrum[:, 2] #* (u.erg/(u.s * u.cm**2 * u.AA))
    f = interp1d(wl_spec, flux_spec, fill_value = np.nan)
    g = interp1d(wl_spec, error_spec, fill_value = np.nan)
    x = np.arange(min(wl_spec), max(wl_spec), np.median(np.diff(wl_spec)))# * u.AA
    flux_spec = f(x) * (u.erg/(u.s * u.cm**2 * u.AA))
    error_spec = g(x) * (u.erg/(u.s * u.cm**2 * u.AA))
    wl_spec = x * u.AA

    SED = np.genfromtxt("../data/SED.dat")
    wl_SED = SED[:, 0]
    AB_SED = SED[:, 1]
    phot = np.genfromtxt("../data/photometry.dat")
    mag = phot[:, 0]
    magmask = (mag != -99)
    mag = phot[:, 0][magmask]
    magerr = phot[:, 1][magmask]
    wl_phot = phot[:, 2][magmask]
    wl_plot_wid = phot[:, 3][magmask]/2.35

    AB = flux_spec.to(u.ABmag, u.spectral_density(wl_spec))
    flux_jan = flux_spec.to(u.Jy, u.spectral_density(wl_spec))*1e6
    err_jan = error_spec.to(u.Jy, u.spectral_density(wl_spec))*1e6

    ax1 = pl.gca()

    ax2 = ax1.twinx()
    # pl.errorbar(wl_spec, AB, yerr=[AB - AB_err_lo, AB_err_hi - AB], fmt=".k", capsize=0, elinewidth=0.5, ms=3, alpha=0.3, label="X-shooter spectrum", zorder=1)
    ax1.errorbar(wl_phot, mag, xerr=wl_plot_wid, yerr=magerr, fmt=".", capsize=0, ms=20, color = "#4C72B0", label = "Photometric points", zorder=3, mec="black", mew = 1)
    # pl.errorbar(wl_spec, AB, fmt=".k", capsize=0, elinewidth=0.5, ms=3, alpha=0.3)
    ax1.plot(0, 0, color="black", linestyle="steps-mid", label="X-shooter spectrum", zorder=1, alpha=0.3)
    ax1.plot(0, 0, label="X-shooter error spectrum", linestyle="dashed", lw=1, zorder=1)
    ax1.plot(wl_SED, AB_SED, linestyle="steps-mid", lw=2.0, color = "#C44E52", label = "SED fit", zorder=2)
    # pl.axhline(0, linestyle="dashed", color = "black", lw = 0.4)

    ax1.legend(loc=2)
    # scale = np.median(AB[~np.isnan(AB)])

    # ax1 = pl.gca()
    ax1.invert_yaxis()
    ax1.set_xlabel(r"Observed wavelength [$\mathrm{\AA}$]")
    ax1.set_ylabel(r'Brightness [AB mag]')
    ax1.set_ylim(26.1, 17.5)

    # ax2 = ax1.twinx()
    ax2.plot(wl_spec[::1], medfilt(flux_jan[::1], 1), color="black", linestyle="steps-mid", lw=1, alpha=0.3, label="X-shooter spectrum", zorder=1)
    ax2.plot(wl_spec[::1], medfilt(err_jan[::1], 31), linestyle="dashed", lw=1, zorder=1)
    mn, mx = 26.1, 17.5#ax2.get_ylim()
    print(mn)
    ax2.set_ylim(10**((23.9 - mn)/2.5), 10**((23.9 - mx)/2.5))
    ax2.semilogy()
    ax2.set_ylabel(r'Flux density [$\mu$Jy]')

    # pl.ylim(-1 * scale, 10 * scale)

    ax1.yaxis.set_label_position("right")
    ax1.yaxis.tick_right()
    ax2.yaxis.set_label_position("left")
    ax2.yaxis.tick_left()

    ax1.yaxis.set_major_formatter(FormatStrFormatter('%.0f'))
    ax2.yaxis.set_major_formatter(FormatStrFormatter('%.1f'))

    #Overplot lines
    linelist = np.array([1216, 3728.09, 4862.68, 4960.30, 5008.24, 6564.61]) * (1 + 2.210)
    linenames = [r"$\mathrm{Ly\alpha}$", r"$\mathrm{[OII]}$", r"$\mathrm{H\beta}$", r"$\mathrm{[OIII]}$", r"$\mathrm{[OIII]}$", r"$\mathrm{H\alpha}$"]
    val = []
    for p in range(len(linelist)):
        xcoord = linelist[p]
        mask = (wl_SED> xcoord - 1) & (wl_SED< xcoord + 1)
        y_val = np.mean(AB_SED[mask])
        val.append(y_val - 0.5)
    val[3] -= 1
    lineid_plot.plot_line_ids(wl_SED, AB_SED, linelist, linenames, arrow_tip=val, ax=ax1, max_iter=0)
    # for i in ax1.lines:
    #     if '$' in i.get_label():
    #         i.set_alpha(0.3)
    a = ax1.findobj(mpl.text.Annotation)
    for i in a:
        if '$' in i.get_label():
            i.set_size(16)

    pl.semilogx()
    pl.xlim(3000, 24000)

    pos = [4000, 6000, 20000]
    ax1.set_xticks(pos, pos)
    # ax2.set_xticks(pos, pos)
    ax1.xaxis.set_minor_formatter(FormatStrFormatter('%i'))
    ax2.xaxis.set_minor_formatter(FormatStrFormatter('%i'))
    ax1.xaxis.set_major_formatter(FormatStrFormatter('%i'))
    ax2.xaxis.set_major_formatter(FormatStrFormatter('%i'))

    pl.tight_layout()
    pl.savefig("../figures/SEDspecphot.pdf")
    pl.show()

if __name__ == '__main__':
    main()


#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# Author: kerlomz <kerlomz@gmail.com>
import yaml
import json

class ModelConfig:

    def __init__(self, model_conf: str):
        self.model_conf = model_conf
        self.system = None
        self.device = None
        self.device_usage = None
        self.charset = None
        self.split_char = None
        self.gen_charset = None
        self.char_exclude = None
        self.model_name = None
        self.model_type = None
        self.image_height = None
        self.image_width = None
        self.image_channel = None
        self.padding = None
        self.lower_padding = None
        self.resize = None
        self.binaryzation = None
        self.smooth = None
        self.blur = None
        self.replace_transparent = None
        self.model_site = None
        self.version = None
        self.color_engine = None
        self.cf_model = self.read_conf
        self.model_exists = False
        self.assignment()

    def assignment(self):

        system = self.cf_model.get('System')
        self.device = system.get('Device') if system else None
        self.device = self.device if self.device else "cpu:0"
        self.device_usage = system.get('DeviceUsage') if system else None
        self.device_usage = self.device_usage if self.device_usage else 0.02
        self.charset = self.cf_model['Model'].get('CharSet')
        self.char_exclude = self.cf_model['Model'].get('CharExclude')
        self.model_name = self.cf_model['Model'].get('ModelName')
        self.model_type = self.cf_model['Model'].get('ModelType')
        self.model_site = self.cf_model['Model'].get('Sites')
        self.model_site = self.model_site if self.model_site else []
        self.version = self.cf_model['Model'].get('Version')
        self.version = self.version if self.version else 1.0
        self.split_char = self.cf_model['Model'].get('SplitChar')
        self.split_char = '' if not self.split_char else self.split_char

        self.image_height = self.cf_model['Model'].get('ImageHeight')
        self.image_width = self.cf_model['Model'].get('ImageWidth')
        self.image_channel = self.cf_model['Model'].get('ImageChannel')
        self.image_channel = self.image_channel if self.image_channel else 1
        self.binaryzation = self.cf_model['Pretreatment'].get('Binaryzation')
        self.resize = self.cf_model['Pretreatment'].get('Resize')
        self.resize = self.resize if self.resize else [self.image_width, self.image_height]
        self.replace_transparent = self.cf_model['Pretreatment'].get('ReplaceTransparent')

    @property
    def read_conf(self):
        with open(self.model_conf, 'r', encoding="utf-8") as sys_fp:
            sys_stream = sys_fp.read()
            return yaml.load(sys_stream, Loader=yaml.SafeLoader)

    def convert(self):
        with open("../model.template", encoding="utf8") as f:
            lines = f.readlines()
            bc = "".join(lines)
            model = bc.format(
                MemoryUsage=0.7,
                CNNNetwork='CNNX',
                RecurrentNetwork='GRU',
                UnitsNum=64,
                Optimizer='Adam',
                LossFunction='CTC',
                Decoder='CTC',
                ModelName=self.model_name,
                ModelField='Image',
                ModelScene='Classification',
                Category=self.charset,
                Resize=json.dumps(self.resize),
                ImageChannel=self.image_channel,
                ImageWidth=self.image_width,
                ImageHeight=self.image_height,
                MaxLabelNum=4,
                AutoPadding=False,
                OutputSplit="",
                LabelFrom="FileName",
                ExtractRegex=".*?(?=_)",
                LabelSplit='null',
                DatasetTrainsPath="",
                DatasetValidationPath="",
                SourceTrainPath="",
                SourceValidationPath="",
                ValidationSetNum="300",
                SavedSteps="500",
                ValidationSteps="500",
                EndAcc="0.98",
                EndCost="0.05",
                EndEpochs="2",
                BatchSize="64",
                ValidationBatchSize="300",
                LearningRate="0.001",
                DA_Binaryzation="-1",
                DA_MedianBlur="-1",
                DA_GaussianBlur="-1",
                DA_EqualizeHist="False",
                DA_Laplace="False",
                DA_WarpPerspective="False",
                DA_Rotate="-1",
                DA_PepperNoise="-1",
                DA_Brightness="False",
                DA_Saturation="False",
                DA_Hue="False",
                DA_Gamma="False",
                DA_ChannelSwap="False",
                DA_RandomBlank="-1",
                DA_RandomTransition="-1",
                Pre_Binaryzation="-1",
                Pre_ReplaceTransparent="False",
                Pre_HorizontalStitching="False",
                Pre_ConcatFrames="-1",
                Pre_BlendFrames="-1",
                DA_RandomCaptcha="",
                Pre_ExecuteMap="",
            )
        open(self.model_conf.replace(".yaml", "_2.0.yaml"), "w", encoding="utf8").write(model)

if __name__ == '__main__':
    ModelConfig(model_conf="model.yaml").convert()


from .models import Event
import names
import time

def create_event_schedule():
    event = Event(
        name=names.get_first_name(), description=names.get_last_name()
    )
    time.sleep(5)
    print("here")
    event.save()


from future.utils import iteritems

from recipe_engine import recipe_test_api

from .api import EnsureFile

class CIPDTestApi(recipe_test_api.RecipeTestApi):

  EnsureFile = EnsureFile

  def make_resolved_package(self, v):
    return v.replace('${platform}', 'resolved-platform')

  def make_resolved_version(self, v):
    if not v:
      return '40-chars-fake-of-the-package-instance_id'
    if len(v) == 40:
      return v
    # Truncate or pad to 40 chars.
    prefix = 'resolved-instance_id-of-'
    if len(v) + len(prefix) >= 40:
      return '%s%s' % (prefix, v[:40-len(prefix)])
    return '%s%s%s' % (prefix, v, '-' * (40 - len(prefix) - len(v)))

  def make_pin(self, package_name, version=None):
    return {
        'package': self.make_resolved_package(package_name),
        'instance_id': self.make_resolved_version(version),
    }

  def _resultify(self, result, error=None, retcode=None):
    dic = {'result': result}
    if error:
      dic['error'] = error
    return self.m.json.output(dic, retcode=retcode)

  def example_error(self, error, retcode=None):
    return self._resultify(
        result=None,
        error=error,
        retcode=1 if retcode is None else retcode)

  def example_acl_check(self, package_path, check=True):
    return self._resultify(check)

  def example_build(self, package_name, version=None):
    return self._resultify(self.make_pin(package_name, version))

  example_register = example_build
  example_pkg_fetch = example_build
  example_pkg_deploy = example_build

  def example_ensure(self, ensure_file):
    return self._resultify({
        subdir or '': [self.make_pin(name, version)
                       for name, version in sorted(packages)]
        for subdir, packages in iteritems(ensure_file.packages)
    })

  def example_ensure_file_resolve(self, ensure_file):
    return self._resultify({
        subdir or '': [{
            'package': self.make_resolved_package(name),
            'pin': self.make_pin(name, version)}
            for name, version in sorted(packages)]
        for subdir, packages in iteritems(ensure_file.packages)
    })

  def example_set_tag(self, package_name, version):
    return self._resultify([{
        'package': package_name,
        'pin': self.make_pin(package_name, version)
    }])

  def example_set_metadata(self, package_name, version):
    return self._resultify([{
        'package': package_name,
        'pin': self.make_pin(package_name, version)
    }])

  def example_set_ref(self, package_name, version):
    return self._resultify({'': [{
        'package': package_name,
        'pin': self.make_pin(package_name, version)
    }]})

  def example_search(self, package_name, instances=None):
    if instances is None:
      # Return one instance by default.
      return self._resultify([self.make_pin(package_name)])
    if isinstance(instances, int):
      instances = ['instance_id_%i' % (i+1) for i in range(instances)]
    return self._resultify([self.make_pin(package_name, instance)
                           for instance in instances])

  def example_describe(self, package_name, version=None,
                       test_data_refs=None, test_data_tags=None,
                       user='user:44-blablbla@developer.gserviceaccount.com',
                       tstamp=1446574210):
    assert not test_data_tags or all(':' in tag for tag in test_data_tags)

    if test_data_tags is None:
      test_data_tags = [
        'buildbot_build:some.waterfall/builder/1234',
        'git_repository:https://chromium.googlesource.com/some/repo',
        'git_revision:397a2597cdc237f3026e6143b683be4b9ab60540',
      ]

    if test_data_refs is None:
      test_data_refs = ['latest']

    # If user explicitly put empty tags/refs (i.e. ())
    if not test_data_refs and not test_data_tags:
      # quick and dirty version differentiation
      if ':' in version:
        return self._resultify(None, error='no such tag', retcode=1)
      if len(version) == 44 or len(version) == 40:
        return self._resultify(None, error='no such instance', retcode=1)
      return self._resultify(None, error='no such ref', retcode=1)

    return self._resultify({
        'pin': self.make_pin(package_name, version),
        'registered_by': user,
        'registered_ts': tstamp,
        'refs': [
          {
            'ref': ref,
            'modified_by': user,
            'modified_ts': tstamp,
            'instance_id': self.make_resolved_version(ref),
          }
          for ref in test_data_refs
        ],
        'tags': [
          {
            'tag': tag,
            'registered_by': user,
            'registered_ts': tstamp,
          }
          for tag in test_data_tags
        ],
    })

  def example_instances(self, package_name, limit=None,
                        user='user:44-blablbla@developer.gserviceaccount.com',
                        tstamp=1446574210):
    # Return two instances by default.
    limit = limit or 2
    instances =[]
    for i in range(limit):
      instance = {
          'pin': self.make_pin(package_name, 'instance_id_%i' % (i+1)),
          'registered_by': user,
          'registered_ts': tstamp-i-1,
      }
      # Add "latest" ref to the first instance
      if i == 0:
        instance['refs'] = ['latest']
      instances.append(instance)
    return self._resultify({'instances': instances})


import io

import mock
from pytest import raises

from paasta_tools.cli.fsm import autosuggest

class TestGetSmartstackProxyPortFromFile:
    @mock.patch("paasta_tools.cli.fsm.autosuggest.read_etc_services", autospec=True)
    def test_multiple_stanzas_per_file(self, mock_read_etc_services):
        with mock.patch("builtins.open", autospec=True):
            with mock.patch(
                "paasta_tools.cli.fsm.autosuggest.yaml", autospec=True
            ) as mock_yaml:
                mock_yaml.safe_load.return_value = {
                    "main": {"proxy_port": 1},
                    "foo": {"proxy_port": 2},
                }
                actual = autosuggest._get_smartstack_proxy_ports_from_file(
                    "fake_root", "smartstack.yaml"
                )
                assert actual == {1, 2}

# Shamelessly copied from TestSuggestPort
class TestSuggestSmartstackProxyPort:
    @mock.patch("paasta_tools.cli.fsm.autosuggest.read_etc_services", autospec=True)
    def test_suggest_smartstack_proxy_port(self, mock_read_etc_services):
        yelpsoa_config_root = "fake_yelpsoa_config_root"
        walk_return = [
            ("fake_root1", "fake_dir1", ["smartstack.yaml"]),
            ("fake_root2", "fake_dir2", ["smartstack.yaml"]),
            ("fake_root3", "fake_dir3", ["smartstack.yaml"]),
        ]
        mock_walk = mock.Mock(return_value=walk_return)

        # See http://www.voidspace.org.uk/python/mock/examples.html#multiple-calls-with-different-effects
        get_smartstack_proxy_ports_from_file_returns = [
            {20001, 20003},
            {20002},
            {55555},  # bogus out-of-range value
        ]

        def get_smarstack_proxy_ports_from_file_side_effect(*args):
            return get_smartstack_proxy_ports_from_file_returns.pop(0)

        mock_get_smartstack_proxy_ports_from_file = mock.Mock(
            side_effect=get_smarstack_proxy_ports_from_file_side_effect
        )
        with mock.patch("os.walk", mock_walk, autospec=None):
            with mock.patch(
                "paasta_tools.cli.fsm.autosuggest._get_smartstack_proxy_ports_from_file",
                mock_get_smartstack_proxy_ports_from_file,
                autospec=None,
            ):
                actual = autosuggest.suggest_smartstack_proxy_port(
                    yelpsoa_config_root, range_min=20001, range_max=20004
                )
        # Sanity check: our mock was called once for each legit port file in
        # walk_return
        assert mock_get_smartstack_proxy_ports_from_file.call_count == 3

        # What we came here for: the actual output of the function under test
        assert actual == 20004  # The only available integer in [20001, 20004]

    @mock.patch("paasta_tools.cli.fsm.autosuggest.read_etc_services", autospec=True)
    def test_suggest_smartstack_proxy_port_too_many_services(
        self, mock_read_etc_services
    ):
        """If all the ports are taken, we should raise an error"""
        yelpsoa_config_root = "fake_yelpsoa_config_root"
        walk_return = [
            ("fake_root1", "fake_dir1", ["smartstack.yaml"]),
            ("fake_root2", "fake_dir2", ["smartstack.yaml"]),
            ("fake_root3", "fake_dir3", ["smartstack.yaml"]),
        ]
        mock_walk = mock.Mock(return_value=walk_return)

        # See http://www.voidspace.org.uk/python/mock/examples.html#multiple-calls-with-different-effects
        get_smartstack_proxy_ports_from_file_returns = [
            {20001, 20003},
            {20002},
            {55555},  # bogus out-of-range value
        ]

        def get_smarstack_proxy_ports_from_file_side_effect(*args):
            return get_smartstack_proxy_ports_from_file_returns.pop(0)

        mock_get_smartstack_proxy_ports_from_file = mock.Mock(
            side_effect=get_smarstack_proxy_ports_from_file_side_effect
        )
        with mock.patch("os.walk", mock_walk, autospec=None):
            with mock.patch(
                "paasta_tools.cli.fsm.autosuggest._get_smartstack_proxy_ports_from_file",
                mock_get_smartstack_proxy_ports_from_file,
                autospec=None,
            ):
                with raises(Exception) as exc:
                    autosuggest.suggest_smartstack_proxy_port(
                        yelpsoa_config_root, range_min=20001, range_max=20003
                    )
                assert (
                    "There are no more ports available in the range [20001, 20003]"
                    == str(exc.value)
                )

@mock.patch("paasta_tools.cli.fsm.autosuggest.read_etc_services", autospec=True)
def test_get_inuse_ports_from_etc_services_parses_correctly(mock_read_etc_services):
    input_services = """
# by IANA and used in the real-world or are needed by a debian package.
# If you need a huge list of used numbers please install the nmap package.

tcpmux		1/tcp				# TCP port service multiplexer
echo		7/tcp
echo		7/udp
discard		9/tcp		sink null
discard		9/udp		sink null
systat		11/tcp		users
daytime		13/tcp
"""
    mock_read_etc_services.return_value = io.StringIO(input_services)
    actual = autosuggest.get_inuse_ports_from_etc_services()
    expected = {1, 7, 9, 11, 13}
    assert actual == expected


from rest_framework.response import Response
from rest_framework.decorators import api_view
from django.contrib.auth.models import User
from django.http import Http404
from rest_framework import status
from base.models import Transaction
from .serializers import TransactionSerilizer, UserSerailizer
from django.db.models import Sum
from django.db.models.functions import Coalesce
from django.db.models import DecimalField
import json
from django.db import IntegrityError

@api_view(['POST'])
def add_user(request):
    """
    Add new user  to the system
    {"user":"bob"}
    """
    payload = request.data
    try:
        t = User(username=payload.get('user').lower())
        t.save()
        serializer = UserSerailizer(t)
        return Response(serializer.data, status=status.HTTP_201_CREATED)

    except IntegrityError as e:

        if 'unique constraint failed' in e.args[0].lower(): # or e.args[0] from Django 1.10
            return Response('User already exists', status=status.HTTP_422_UNPROCESSABLE_ENTITY)

    except Exception as ex:
        print(ex)
        return Response("Bad request", status=status.HTTP_400_BAD_REQUEST)

@api_view(['POST'])
def add_transactions(request):
    # ----- YAML below for Swagger -----
    """
    Add transaction of borroing and lending

    {"lender":"bob","borrower":"adam","amount":5.25, "expiration":"2022-11-10"}
    """

    payload = request.data
    borrower =  request.data.get('borrower')
    lender =  request.data.get('lender')
    amount =  float(request.data.get('amount'))
    expiration =  request.data.get('expiration')
    borrower_obj = User.objects.filter(username=borrower)[0]
    lender_obj = User.objects.filter(username=lender)[0]

    try:
        t = Transaction(borrower=borrower_obj, lender=lender_obj, amount= amount, expiry=expiration)
        t.save()
        serializer = TransactionSerilizer(t)
        return Response(serializer.data, status=status.HTTP_201_CREATED)

    except Exception as ex:
        print(ex)
        return Response('Unknown Error Occured', status=status.HTTP_400_BAD_REQUEST)

@api_view(['GET'])
def settle_up(request):
    # ----- YAML below for Swagger -----
    """
    Fetch transaction details of the users who borrowed or lending details

    {"lender":"bob","borrower":"adam","amount":5.25, "expiration":"2022-11-10"}
    """
    try:
        payload = json.loads(request.query_params.get('payload'))

        #  ToDo: Following section can be improvide in better way
        # Fetch users from db as per requested list
        users = User.objects.filter(username__in=[ user.lower() for user in payload.get("users")]).order_by('username')

        # Compute how many amount is owes the user
        owes = [ { "name": user.username.title(), "owes":  {User.objects.get(pk=lender.get('lender')).username: lender.get('amount') \
                                                            for lender in \
                                                            Transaction.objects.filter(borrower__username=user.username).values( 'lender').annotate(amount=Sum('amount'))} }  \
                for user in users ]

        # Compute how many amount is owed_by the user
        owedby = [ { "name": user.username.title(), "owed_by":  { User.objects.get(pk=borrower.get('borrower')).username: borrower.get('amount') \
                                                                 for borrower in Transaction.objects.filter(lender__username=user.username).values( 'borrower').annotate(amount=Sum('amount'))} }  \
                  for user in users ]

        # Balaance calculation
        balance_owes = [   Transaction.objects.filter(borrower__username=user.username).aggregate(balance_owes=Coalesce(Sum('amount'), 0.0, output_field=DecimalField()))  for user in users]
        balance_owedby = [   Transaction.objects.filter(lender__username=user.username).aggregate(balance_owedby=Coalesce(Sum('amount'), 0.0, output_field=DecimalField()))  for user in users]
        balance = [ {'balance':item[1].get('balance_owedby') - item[0].get('balance_owes')} for item in list(zip( balance_owes, balance_owedby))]

        # Merging for final results
        results = [ {**owed, **owedby, **balance} for owed, owedby, balance in list(zip(owes,owedby,balance))]

        return Response(results, status=status.HTTP_201_CREATED)

    except ValueError as ex:
        print(ex)
        return Response("Bad incoming request", status=status.HTTP_400_BAD_REQUEST)

    except Exception as ex:
        print(ex)
        return Response("Unkown error occured", status=status.HTTP_400_BAD_REQUEST)


from rest_framework import serializers

from projects.models import (
    Project,
    ProjectVolunteers,
    ProjectVolunteersRegistration,
    ProjectAttendees,
    ProjectAttendeesRegistration,
    ProjectDiscussion,
    ProjectAnswerDiscussion,
    ProjectHub,
)

class ProjectVolunteersRegistrationSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = ProjectVolunteersRegistration
        fields = ('url', 'profile', 'project_volunteers', 'project_volunteers_ref')

    def create(self, validated_data):
        project_volunteers = ProjectVolunteers.objects.get(pk=validated_data['project_volunteers_ref'])
        registration = ProjectVolunteersRegistration.objects.create(
            project_volunteers=project_volunteers,
            **validated_data
        )
        count = ProjectVolunteersRegistration.objects.filter(
            project_volunteers=project_volunteers
        ).count()
        project_volunteers.registered = count
        project_volunteers.save()
        return registration

class ProjectVolunteersSerializer(serializers.HyperlinkedModelSerializer):
    volunteers_registration = ProjectVolunteersRegistrationSerializer(many=True, read_only=True)

    class Meta:
        model = ProjectVolunteers
        fields = (
            'url',
            'id',
            'project',
            'role',
            'description',
            'seats',
            'registered',
            'minimum_registration',
            'volunteers_registration',
        )
        read_only_fields = ('registered', 'project', 'id')

class ProjectAttendeesRegistrationSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = ProjectAttendeesRegistration
        fields = ('url', 'profile', 'project_attendees', 'project_attendees_ref')

    def create(self, validated_data):
        project_attendees = ProjectAttendees.objects.get(pk=validated_data['project_attendees_ref'])
        registration = ProjectAttendeesRegistration.objects.create(project_attendees=project_attendees, **validated_data)
        count = ProjectAttendeesRegistration.objects.filter(project_attendees=project_attendees).count()
        project_attendees.registered = count
        project_attendees.save()
        return registration

class ProjectAttendeesSerializer(serializers.HyperlinkedModelSerializer):
    attendees_registration = ProjectAttendeesRegistrationSerializer(many=True, read_only=True)

    class Meta:
        model = ProjectAttendees
        fields = (
            'url',
            'id',
            'project',
            'seats',
            'registered',
            'attendees_registration',
            'minimum_registration',
        )
        read_only_fields = ('registered', 'project',)

class ProjectAnswerDiscussionSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = ProjectAnswerDiscussion
        fields = ('url', 'id', 'discussion_ref', 'discussion', 'text', 'profile', 'created', 'updated')
        read_only_fields = ('discussion', 'profile')

    def create(self, validated_data):
        project_discussion = ProjectDiscussion.objects.get(pk=validated_data['discussion_ref'])
        answer = ProjectAnswerDiscussion.objects.create(discussion=project_discussion, **validated_data)
        return answer

class ProjectDiscussionSerializer(serializers.HyperlinkedModelSerializer):
    answer_discussion_project = ProjectAnswerDiscussionSerializer(many=True, read_only=True)

    class Meta:
        model = ProjectDiscussion
        fields = (
            'url',
            'id',
            'project',
            'project_ref',
            'title',
            'text',
            'profile',
            'created',
            'updated',
            'answer_discussion_project',
        )
        read_only_fields = ('profile', 'project', 'id')

    def create(self, validated_data):
        project = Project.objects.get(pk=validated_data['project_ref'])
        new_discussion = ProjectDiscussion.objects.create(project=project, **validated_data)
        return new_discussion

class ProjectSerializer(serializers.HyperlinkedModelSerializer):
    attendees = ProjectAttendeesSerializer()
    volunteers = ProjectVolunteersSerializer(many=True)
    discussion_project = ProjectDiscussionSerializer(many=True, read_only=True)
    ### cause of the error :
    #serializers.HyperlinkedRelatedField(
    #    many=True,
    #    view_name='discussion_project',
    #    read_only=True
    #)

    class Meta:
        model = Project
        fields = ('url', 'id', 'name', 'start',
                  'end', 'description', 'category',
                  'sub_category', 'oth_category', 'oth_sub_cat','place_name', 'number', 'street',
                  'postal_code', 'city', 'organizer', 'created',
                  'updated', 'project_type', 'attendees',
                  'volunteers', 'discussion_project')
        read_only_fields = ('organizer', 'id')

    def create(self, validated_data):
        attendees_data = validated_data.pop('attendees')
        volunteers_data = validated_data.pop('volunteers')
        new_project = Project.objects.create(**validated_data)
        if validated_data['project_type'] == 'CO':
            ProjectAttendees.objects.create(project=new_project, **attendees_data)
        elif validated_data['project_type'] == 'CP':
            for volunteer_data in volunteers_data:
                ProjectVolunteers.objects.create(project=new_project, **volunteer_data)
        else:
            ProjectAttendees.objects.create(project=new_project, **attendees_data)
            for volunteer_data in volunteers_data:
                ProjectVolunteers.objects.create(project=new_project, **volunteer_data)
        return new_project

class ProjectShortSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = Project
        fields = ('url', 'id', 'name', 'start', 'created', 'updated',)

class ProjectHubSerializer(serializers.HyperlinkedModelSerializer):
    project = ProjectSerializer()

    class Meta:
        model = ProjectHub
        fields = ('project', 'distance_km', 'lat', 'lng')


#!/usr/bin/env python

import numpy as np
import pandas as pd
from pandas.plotting import scatter_matrix
import os, sys
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from matplotlib import cm as cm

def plot_corr_mat(df):
    labels=list(df)
    labels.remove('Weight')
    labels.remove('y')

    corr_matrix = df[labels].corr()

    jump_x = corr_matrix.shape[0]*1.0/len(labels)
    jump_y = corr_matrix.shape[1]*1.0/len(labels)
    fig = plt.figure(figsize=(10,10))
    ax1 = fig.add_subplot(111)
    cmap = cm.get_cmap('jet', 30)
    cax = ax1.imshow(corr_matrix,
                     interpolation="nearest",
                     cmap=cmap)
    ax1.set_xticklabels(labels,fontsize=16, rotation='vertical')
    ax1.set_yticklabels(labels,fontsize=16)
    ax1.set_xticks(np.arange(0,corr_matrix.shape[0], jump_x))
    ax1.set_yticks(np.arange(0,corr_matrix.shape[1], jump_y))
    ax1.tick_params('both', length=0, width=0, which='major')
    ax1.tick_params('both', length=0, width=0, which='minor')
    ax1.set_aspect('auto')
    fig.colorbar(cax)

    x_positions = np.linspace(start=0, stop=len(labels), num=len(labels), endpoint=False)
    y_positions = np.linspace(start=0, stop=len(labels), num=len(labels), endpoint=False)
    for y_index, y in enumerate(y_positions):
        for x_index, x in enumerate(x_positions):
            x_feat = labels[x_index]
            y_feat = labels[y_index]
            c = "{0:.2f}".format(df[x_feat].corr(df[y_feat]))
            ax1.text(x, y, str(c), {'size': 12}, color='black', ha='center', va='center')

    return fig

def compare_weights(df):
    #1D projections (weighted VS unweighted)
    fig, axs = plt.subplots(nrows=5, ncols=3, figsize=(25,20))
    labels=list(df)
    labels.remove('Weight')
    idx=0
    for row in range(5):
        for col in range(3):
            label=labels[idx]
            ax = axs[row,col];
            _ = ax.hist(df[label], color='blue', alpha=0.3, normed=True, weights=None);
            _ = ax.hist(df[label], color='red', alpha=0.3, normed=True, weights=df.Weight.values);
            ax.set_xlabel(label);
            if "x" in label or "y" in label:
                ax.set_yscale('log');
            patch0 = mpatches.Patch(color='blue',
                                    label='Unweighted',
                                    alpha=0.3);
            patch1 = mpatches.Patch(color='red',
                                    label='Weighted',
                                    alpha=0.3);
            ax.legend(loc='best', handles=[patch0,patch1]);
            idx += 1
    plt.minorticks_on()
    plt.tight_layout()

    return fig

def marketwise_proj(df):
    fig, axs = plt.subplots(nrows=7, ncols=2, figsize=(20,30))
    labels=list(df)
    labels.remove('Weight')
    labels.remove('Market')
    idx=0
    for row in range(7):
        for col in range(2):
            label=labels[idx]
            ax = axs[row,col];
            if "x" in label or "y" in label:
                ax.set_yscale('log');
                bins = 10
                _,bins,_ = ax.hist(df.query("Market==1")[label], bins=bins, color='blue', alpha=0.3, normed=True, weights=df.query("Market==1").Weight.values);
                _,bins,_ = ax.hist(df.query("Market==2")[label], bins=bins, color='red', alpha=0.3, normed=True, weights=df.query("Market==2").Weight.values);
                _,bins,_ = ax.hist(df.query("Market==3")[label], bins=bins, color='black', alpha=0.3, normed=True, weights=df.query("Market==3").Weight.values);
                _,bins,_ = ax.hist(df.query("Market==4")[label], bins=bins, color='green', alpha=0.3, normed=True, weights=df.query("Market==4").Weight.values);
            else:
                _ = ax.hist(df.query("Market==1")[label], color='blue', alpha=0.3, normed=True, weights=df.query("Market==1").Weight.values);
                _ = ax.hist(df.query("Market==2")[label], color='red', alpha=0.3, normed=True, weights=df.query("Market==2").Weight.values);
                _ = ax.hist(df.query("Market==3")[label], color='black', alpha=0.3, normed=True, weights=df.query("Market==3").Weight.values);
                _ = ax.hist(df.query("Market==4")[label], color='green', alpha=0.3, normed=True, weights=df.query("Market==4").Weight.values);
            ax.set_xlabel(label);
            patch1 = mpatches.Patch(color='blue',
                                    label='Market 1',
                                    alpha=0.3);
            patch2 = mpatches.Patch(color='red',
                                    label='Market 2',
                                    alpha=0.3);
            patch3 = mpatches.Patch(color='black',
                                    label='Market 3',
                                    alpha=0.3);
            patch4 = mpatches.Patch(color='green',
                                    label='Market 4',
                                    alpha=0.3);
            ax.legend(loc='best', handles=[patch1,patch2,patch3,patch4]);
            idx += 1
    plt.minorticks_on()
    plt.tight_layout()

    return fig

def scatter_plots(df):
    fig, axs = plt.subplots(nrows=7, ncols=2, figsize=(20,30))
    labels=list(df)
    if 'Weight' in labels:
        labels.remove('Weight')
    labels.remove('y')
    idx=0
    for row in range(7):
        for col in range(2):
            label=labels[idx]
            ax = axs[row,col];
            ax.scatter(df[label],df["y"])
            ax.set_xlabel(label);
            ax.set_ylabel("y");
            c = "correlation = {0:.4f}".format(df[label].corr(df["y"]))
            ax.text(0.5, 0.85, str(c), {'size': 20}, transform=ax.transAxes, color='black', ha='right', va='bottom');
            idx += 1
    plt.minorticks_on()
    plt.tight_layout()

    return fig

def time_series(df):
    fig, axs = plt.subplots(nrows=6, ncols=2, figsize=(20,30))
    labels=list(df)
    if 'Weight' in labels:
        labels.remove('Weight')
    labels.remove('Stock')
    labels.remove('Day')
    labels.remove('Market')
    idx=0
    for row in range(6):
        for col in range(2):
            label=labels[idx]
            ax = axs[row,col];
            ax.plot(df.query("Market==1")["Day"], df.query("Market==1")[label], color='blue')
            ax.plot(df.query("Market==2")["Day"], df.query("Market==2")[label], color='red')
            ax.plot(df.query("Market==3")["Day"], df.query("Market==3")[label], color='black')
            ax.plot(df.query("Market==4")["Day"], df.query("Market==4")[label], color='green')
            ax.set_xlabel("time");
            ax.set_ylabel(label);
            patch1 = mpatches.Patch(color='blue',
                                    label='Market 1',
                                    alpha=0.3);
            patch2 = mpatches.Patch(color='red',
                                    label='Market 2',
                                    alpha=0.3);
            patch3 = mpatches.Patch(color='black',
                                    label='Market 3',
                                    alpha=0.3);
            patch4 = mpatches.Patch(color='green',
                                    label='Market 4',
                                    alpha=0.3);
            ax.legend(loc='best', handles=[patch1,patch2,patch3,patch4]);
            idx += 1
    plt.minorticks_on()
    plt.tight_layout()

    return fig


import scrapy
import csv
from bs4 import BeautifulSoup
import spacy
import re
import regex
import string
from shop_scraper.items import Product
from urllib.parse import urlparse

class ProductsSpiderSpider(scrapy.Spider):
    name = 'products_spider'
    allowed_domains = []

    def __init__(self, *args, **kwargs):
        super(ProductsSpiderSpider, self).__init__(*args, **kwargs)
        with open('../result_without_async.csv', 'r', newline='', encoding='utf-8') as csvfile:
            csvreader = csv.DictReader(csvfile)
            for row in csvreader:
                if row['IsAccessible'].strip().lower() == 'true':
                    url = row['URL']
                    domain = urlparse(url).netloc
                    self.allowed_domains.append(domain)
        self.currency_pattern = re.compile(
            r'\b(?:\$|€|£|JPY|USD|EUR|GBP|INR|AUD|CAD|SGD|CNY|JPY)\b', re.IGNORECASE)
        self.currency_pattern2 = regex.compile(
            r'(?:\p{Sc}|\b(?:USD|EUR|GBP|INR|AUD|CAD|SGD|CNY|JPY)\b)\s*\d+(?:,\d{3})*(?:\.\d+)?', re.IGNORECASE)
        self.allowed_chars = set(string.ascii_letters + " .,;:-_/")
        self.non_content_keywords = [
            'header', 'footer', 'nav', 'sidebar', 'advertisement', ]
        # Load Spacy model
        self.nlp = spacy.load('../v5_product_ner_model')

    def start_requests(self):
        # Read the CSV file and start requests
        with open('../result_without_async.csv', 'r', newline='', encoding='utf-8') as csvfile:
            csvreader = csv.DictReader(csvfile)
            for row in csvreader:
                if row['IsAccessible'].strip().lower() == 'true':
                    yield scrapy.Request(row['URL'].strip(), self.parse)

    def parse(self, response):
        soup = BeautifulSoup(response.body, 'lxml')
        for script in soup(['script', 'style', 'nav', 'header', 'footer', 'form', 'img']):
            script.extract()

        non_content_attributes = [
            ('class', 'header'),
            ('class', 'footer'),
            ('class', 'nav'),
            ('class', 'sidebar'),
            ('data-section-id', 'sidebar'),
            ('data-section-id', 'header'),
            ('data-section-id', 'footer'),
            ('data-section-id', 'nav'),
            ('id', 'sidebar'),
            ('id', 'header'),
            ('id', 'footer'),
            ('id', 'nav'),
        ]

        for attr, value in non_content_attributes:
            for element in soup.find_all(attrs={attr: value}):
                element.extract()
        for tag in soup.find_all(self.has_non_content_attribute):
            tag.extract()
        text = soup.get_text()
        text = ' '.join(text.split())

        doc = self.nlp(text)
        for ent in doc.ents:
            product = ent.text
            if self.currency_pattern.search(product) or self.currency_pattern2.search(product):
                continue
            if any(char not in self.allowed_chars for char in product):
                continue
            if '.' in product:
                product = product.split(".")[0]
            item = Product()
            item['product'] = product.lower()
            item['url'] = response.url
            yield item
            links = response.css('a::attr(href)').extract()
            for link in links:
                yield response.follow(link, self.parse)

    def has_non_content_attribute(self, tag):
        for attribute in tag.attrs.values():
            if any(keyword in str(attribute).lower() for keyword in self.non_content_keywords):
                return True
        return False


"""Module for showing and filtering client messages."""
import sys

from PyQt5 import QtCore, QtWidgets, QtSql

import constants

class MyWindow(QtWidgets.QWidget):
    def __init__(self, parent = None):
        QtWidgets.QWidget.__init__(self, parent,
            flags = QtCore.Qt.Window)
        self.setWindowTitle("Filtering and sorting messages")

        con = QtSql.QSqlDatabase.addDatabase('QSQLITE')
        con.setDatabaseName("/".join((constants.ARTIFACTS_FOLDER_NAME, constants.DB_CLIENT)))
        con.open()

##        self.model = QtSql.QSqlQueryModel(parent = self)
##        self.model.setQuery('select * from messages')

        self.model = QtSql.QSqlTableModel(parent = self)
        self.model.setTable('messages')
        self.model.select()

        self.model.setHeaderData(1, QtCore.Qt.Horizontal, 'From_client')
        self.model.setHeaderData(2, QtCore.Qt.Horizontal, 'To_client')
        self.model.setHeaderData(3, QtCore.Qt.Horizontal, 'Message')

        vbox = QtWidgets.QVBoxLayout()
        hbox = QtWidgets.QHBoxLayout()
        lblSort = QtWidgets.QLabel("S&ort by field")
        hbox.addWidget(lblSort)

        self.cboSort = QtWidgets.QComboBox()

        message_table = con.record("messages")
        field_count = message_table.count()
        for field in range(0, field_count):
            field_name = message_table.field(field).name()
            if field_name != "gid":
                self.cboSort.addItem(field_name)
        lblSort.setBuddy(self.cboSort)
        hbox.addWidget(self.cboSort)

        self.chkDesc = QtWidgets.QCheckBox("&Descendingly")
        hbox.addWidget(self.chkDesc, alignment=QtCore.Qt.AlignRight)
        vbox.addLayout(hbox)
        hbox = QtWidgets.QHBoxLayout()

        lblFilterSender = QtWidgets.QLabel("Filter by &sender")
        hbox.addWidget(lblFilterSender)
        self.txtFilterSender = QtWidgets.QLineEdit()
        lblFilterSender.setBuddy(self.txtFilterSender)
        hbox.addWidget(self.txtFilterSender)
        vbox.addLayout(hbox)

        hbox = QtWidgets.QHBoxLayout()
        lblFilterReceiver = QtWidgets.QLabel("Filter by &recipient")
        hbox.addWidget(lblFilterReceiver)
        self.txtFilterReceiver = QtWidgets.QLineEdit()
        lblFilterReceiver.setBuddy(self.txtFilterReceiver)
        hbox.addWidget(self.txtFilterReceiver)
        vbox.addLayout(hbox)
        btnRefresh = QtWidgets.QPushButton("Re&fresh")
        btnRefresh.clicked.connect(self._refreshData)
        vbox.addWidget(btnRefresh)

        self.tblMain = QtWidgets.QTableView()
        self.tblMain.setModel(self.model)
        self.tblMain.hideColumn(0)
        self.tblMain.setColumnWidth(1, 70)
        self.tblMain.setColumnWidth(2, 70)
        self.tblMain.setColumnWidth(3, 360)
        vbox.addWidget(self.tblMain)
        self.setLayout(vbox)
        self.resize(500, 900)

    """
    def _refreshData(self):
        s = ''
        sender = self.txtFilterSender.text()
        receiver = self.txtFilterReceiver.text()
        if sender and not receiver:
            s += "where from_client like '%" + sender + "%'"
        elif receiver and not sender:
            s += "where to_client like '%" + receiver + "%'"
        elif sender and receiver:
            s += "where from_client like '%" + sender + "%' and to_client like '%" + receiver + "%'"
        s += " order by " + self.cboSort.currentText()
        if self.chkDesc.isChecked():
            s += " desc"
        self.model.setQuery("select * from messages " + s)

    """
    def _refreshData(self):
        self.model.setFilter('')
        sender = self.txtFilterSender.text()
        receiver = self.txtFilterReceiver.text()
        if sender and not receiver:
            self.model.setFilter("from_client like '%" + sender + "%'")
        elif receiver and not sender:
            self.model.setFilter("to_client like '%" + receiver + "%'")
        elif sender and receiver:
            self.model.setFilter("from_client like '%" + sender + "%' and to_client like '%" + receiver + "%'")
        index = self.cboSort.currentIndex()
        self.model.setSort(index+1, QtCore.Qt.AscendingOrder)
        """
        if self.cboSort.currentText() == "from_client":
            self.model.setSort(1, QtCore.Qt.AscendingOrder)
        elif self.cboSort.currentText() == "to_client":
            self.model.setSort(2, QtCore.Qt.AscendingOrder)
        elif self.cboSort.currentText() == "message":
            self.model.setSort(3, QtCore.Qt.AscendingOrder)
         """
        if self.chkDesc.isChecked():
            self.model.setSort(index + 1, QtCore.Qt.DescendingOrder)
        self.model.select()

app = QtWidgets.QApplication(sys.argv)
window = MyWindow()
window.show()
sys.exit(app.exec_())