import datetime
import importlib
import logging
import os
import os.path
import subprocess
import sys

__all__ = ('citation', 'get_config', 'get_include', 'smoke')

from . import _version
__version__ = _version.get_versions()['version']

class Formatter(logging.Formatter):
    def format(self, record):
        # Get the full message, #1688 shows we can not rely
        # in record.msg.
        #
        msg = record.getMessage()
        if record.levelno > logging.INFO:
            msg = f"{record.levelname}: {msg}"
        return msg

log = logging.getLogger('sherpa')
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(Formatter())
log.addHandler(handler)
log.setLevel(logging.INFO)

dbg = log.debug

del Formatter, log, handler

DEFAULT_CITATION = """Please review the Zenodo Sherpa page at
       https://doi.org/10.5281/zenodo.593753
to identify the latest release of Sherpa. The Zenodo page
https://help.zenodo.org/#versioning provides information on how to
cite a specific version.

If you want a general-purpose citation then please use either of the
following, kindly provided by the SAO/NASA Astrophysics Data System
service:

@INPROCEEDINGS{2001SPIE.4477...76F,
       author = {{Freeman}, Peter and {Doe}, Stephen and {Siemiginowska}, Aneta},
        title = "{Sherpa: a mission-independent data analysis application}",
     keywords = {Astrophysics},
    booktitle = {Astronomical Data Analysis},
         year = 2001,
       editor = {{Starck}, Jean-Luc and {Murtagh}, Fionn D.},
       series = {Society of Photo-Optical Instrumentation Engineers (SPIE) Conference Series},
       volume = {4477},
        month = nov,
        pages = {76-87},
          doi = {10.1117/12.447161},
archivePrefix = {arXiv},
       eprint = {astro-ph/0108426},
 primaryClass = {astro-ph},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2001SPIE.4477...76F},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@INPROCEEDINGS{2007ASPC..376..543D,
   author = {{Doe}, S. and {Nguyen}, D. and {Stawarz}, C. and {Refsdal}, B. and
        {Siemiginowska}, A. and {Burke}, D. and {Evans}, I. and {Evans}, J. and
        {McDowell}, J. and {Houck}, J. and {Nowak}, M.},
    title = "{Developing Sherpa with Python}",
booktitle = {Astronomical Data Analysis Software and Systems XVI},
     year = 2007,
   series = {Astronomical Society of the Pacific Conference Series},
   volume = 376,
   editor = {{Shaw}, R.~A. and {Hill}, F. and {Bell}, D.~J.},
    month = oct,
    pages = {543},
   adsurl = {http://adsabs.harvard.edu/abs/2007ASPC..376..543D},
  adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}
"""

def _make_citation(version, title, date, authors, idval, latest=True):

    year = date.year
    month = date.strftime('%b').lower()
    nicedate = date.strftime('%B %d, %Y')

    authnames = ' and\n                  '.join(authors)

    if latest:
        out = "The latest release of Sherpa is {}\n".format(title)
        out += "released on {}.".format(nicedate)
    else:
        out = "Sherpa {} was released on {}.".format(version, nicedate)

    out += """

@software{{sherpa_{year}_{idval},
  author       = {{{authnames}}},
  title        = {{{title}}},
  month        = {month},
  year         = {year},
  publisher    = {{Zenodo}},
  version      = {{{version}}},
  doi          = {{10.5281/zenodo.{idval}}},
  url          = {{https://doi.org/10.5281/zenodo.{idval}}}
}}

""".format(authnames=authnames, idval=idval, title=title,
           version=version, year=year, month=month)

    out += DEFAULT_CITATION
    return out

def _get_citation_hardcoded(version):
    """Retrieve the citation information.

    Parameters
    ----------
    version : str
        The version to retrieve the citation before. It is expected
        to be '4.8.0' or higher.

    Returns
    -------
    citation : str or None
        Citation information if known, otherwise None.

    Notes
    -----
    The entries can be created with the script:
    scripts/make_zenodo_release.py

    """

    def todate(year, mnum, dnum):
        return datetime.datetime(year, mnum, dnum)

    cite = {}

    def add(version, **kwargs):
        assert version not in cite
        cite[version] = dict(**kwargs)
        cite[version]['version'] = version

    add(version='4.15.1', title='sherpa/sherpa: Sherpa 4.15.1',
        date=todate(2023, 5, 18),
        authors=['Doug Burke', 'Omar Laurino', 'wmclaugh', 'Marie-Terrell',
                 'dtnguyen2', 'Hans Moritz Günther', 'Aneta Siemiginowska',
                 'Jamie Budynkiewicz', 'Harlan Cheer', 'Tom Aldcroft',
                 'Christoph Deil', 'Brigitta Sipőcz', 'Johannes Buchner',
                 'Axel Donath', 'Iva Laginja', 'Katrin Leinweber',
                 'nplee', 'Todd'],
        idval='7948720')
    add(version='4.15.0', title='sherpa/sherpa: Sherpa 4.15.0',
        date=todate(2022, 10, 11),
        authors=['Doug Burke', 'Omar Laurino', 'wmclaugh', 'dtnguyen2',
                 'Hans Moritz Günther', 'Marie-Terrell', 'Aneta Siemiginowska',
                 'Jamie Budynkiewicz', 'Harlan Cheer', 'Tom Aldcroft',
                 'Christoph Deil', 'Brigitta Sipőcz', 'Johannes Buchner',
                 'Axel Donath', 'Iva Laginja', 'Katrin Leinweber',
                 'nplee', 'Todd'],
        idval='7186379')
    add(version='4.14.1', title='sherpa/sherpa: Sherpa 4.14.1',
        date=todate(2022, 5, 20),
        authors=['Doug Burke', 'Omar Laurino', 'wmclaugh', 'dtnguyen2',
                 'Hans Moritz Günther', 'Marie-Terrell', 'Aneta Siemiginowska',
                 'Jamie Budynkiewicz', 'Tom Aldcroft', 'Christoph Deil',
                 'Harlan Cheer', 'Brigitta Sipőcz', 'Johannes Buchner',
                 'Axel Donath', 'Iva Laginja', 'Katrin Leinweber',
                 'nplee', 'Todd'],
        idval='6567264')
    add(version='4.14.0', title='sherpa/sherpa: Sherpa 4.14.0',
        date=todate(2021, 10, 7),
        authors=['Doug Burke', 'Omar Laurino', 'wmclaugh', 'dtnguyen2',
                 'Hans Moritz Günther', 'Marie-Terrell', 'Aneta Siemiginowska',
                 'Jamie Budynkiewicz', 'Tom Aldcroft', 'Christoph Deil',
                 'Brigitta Sipőcz', 'Johannes Buchner', 'Iva Laginja',
                 'Katrin Leinweber', 'nplee', 'Todd'],
        idval='5554957')

    add(version='4.13.1', title='sherpa/sherpa: Sherpa 4.13.1',
        date=todate(2021, 5, 18),
        authors=['Doug Burke', 'Omar Laurino', 'wmclaugh', 'dtnguyen2',
                 'Marie-Terrell', 'Hans Moritz Günther', 'Aneta Siemiginowska',
                 'Jamie Budynkiewicz', 'Tom Aldcroft', 'Christoph Deil',
                 'Brigitta Sipőcz', 'Johannes Buchner', 'Iva Laginja',
                 'Katrin Leinweber', 'nplee', 'Todd'],
        idval='4770623')
    add(version='4.13.0', title='sherpa/sherpa: Sherpa 4.13.0',
        date=todate(2021, 1, 8),
        authors=['Doug Burke', 'Omar Laurino', 'wmclaugh', 'dtnguyen2',
                 'Marie-Terrell', 'Hans Moritz Günther', 'Aneta Siemiginowska',
                 'Jamie Budynkiewicz', 'Tom Aldcroft', 'Christoph Deil',
                 'Brigitta Sipőcz', 'Johannes Buchner', 'Iva Laginja',
                 'Katrin Leinweber', 'nplee', 'Todd'],
        idval='4428938')

    add(version='4.12.2', title='sherpa/sherpa: Sherpa 4.12.2',
        date=todate(2020, 10, 27),
        authors=['Doug Burke', 'Omar Laurino', 'wmclaugh', 'dtnguyen2',
                 'Hans Moritz Günther', 'Marie-Terrell', 'Aneta Siemiginowska',
                 'Jamie Budynkiewicz', 'Tom Aldcroft', 'Christoph Deil',
                 'Brigitta Sipőcz', 'Johannes Buchner', 'Iva Laginja',
                 'Katrin Leinweber', 'nplee', 'Todd'],
        idval='4141888')
    add(version='4.12.1', title='sherpa/sherpa: Sherpa 4.12.1',
        date=todate(2020, 7, 14),
        authors=['Doug Burke', 'Omar Laurino', 'wmclaugh', 'dtnguyen2',
                 'Marie-Terrell', 'Hans Moritz Günther', 'Jamie Budynkiewicz',
                 'Aneta Siemiginowska', 'Tom Aldcroft', 'Christoph Deil',
                 'Brigitta Sipőcz', 'Katrin Leinweber', 'Todd'],
        idval='3944985')
    add(version='4.12.0', title='sherpa/sherpa: Sherpa 4.12.0',
        date=todate(2020, 1, 30),
        authors=['Doug Burke', 'Omar Laurino', 'dtnguyen2', 'wmclaugh',
                 'Jamie Budynkiewicz', 'Tom Aldcroft', 'Aneta Siemiginowska',
                 'Christoph Deil', 'Marie-Terrell', 'Brigitta Sipőcz',
                 'Hans Moritz Günther', 'Todd', 'Katrin Leinweber'],
        idval='3631574')

    add(version='4.11.1', title='sherpa/sherpa: Sherpa 4.11.1',
        date=todate(2019, 8, 1),
        authors=['Doug Burke', 'Omar Laurino', 'dtnguyen2',
                 'Jamie Budynkiewicz', 'Tom Aldcroft', 'Aneta Siemiginowska',
                 'Christoph Deil', 'wmclaugh', 'Brigitta Sipocz',
                 'Katrin Leinweber'],
        idval='3358134')
    add(version='4.11.0', title='sherpa/sherpa: Sherpa 4.11.0',
        date=todate(2019, 2, 20),
        authors=['Doug Burke', 'Omar Laurino', 'dtnguyen2',
                 'Jamie Budynkiewicz', 'Tom Aldcroft', 'Aneta Siemiginowska',
                 'Christoph Deil', 'wmclaugh', 'Brigitta Sipocz',
                 'Katrin Leinweber'],
        idval='2573885')

    add(version='4.10.2', title='sherpa/sherpa: Sherpa 4.10.2',
        date=todate(2018, 12, 14),
        authors=['Doug Burke', 'Omar Laurino', 'dtnguyen2',
                 'Jamie Budynkiewicz', 'Tom Aldcroft', 'Aneta Siemiginowska',
                 'Christoph Deil', 'wmclaugh', 'Brigitta Sipocz',
                 'Katrin Leinweber'],
        idval='2275738')
    add(version='4.10.1', title='sherpa/sherpa: Sherpa 4.10.1',
        date=todate(2018, 10, 16),
        authors=['Doug Burke', 'Omar Laurino', 'dtnguyen2',
                 'Jamie Budynkiewicz', 'Tom Aldcroft', 'Aneta Siemiginowska',
                 'wmclaugh', 'Brigitta Sipocz', 'Christoph Deil',
                 'Katrin Leinweber'],
        idval='1463962')
    add(version='4.10.0', title='sherpa/sherpa: Sherpa 4.10.0',
        date=todate(2018, 5, 11),
        authors=['Doug Burke', 'Omar Laurino', 'dtnguyen2',
                 'Jamie Budynkiewicz', 'Tom Aldcroft', 'Aneta Siemiginowska',
                 'wmclaugh', 'Christoph Deil', 'Brigitta Sipocz'],
        idval='1245678')

    add(version='4.9.1', title='sherpa/sherpa: Sherpa 4.9.1',
        date=todate(2017, 8, 3),
        authors=['Doug Burke', 'Omar Laurino', 'dtnguyen2',
                 'Jamie Budynkiewicz', 'Tom Aldcroft', 'Aneta Siemiginowska',
                 'wmclaugh', 'Christoph Deil', 'Brigitta Sipocz'],
        idval='838686')
    add(version='4.9.0', title='sherpa/sherpa: Sherpa 4.9.0',
        date=todate(2017, 1, 27),
        authors=['Doug Burke', 'Omar Laurino', 'dtnguyen2',
                 'Jamie Budynkiewicz', 'Tom Aldcroft', 'Aneta Siemiginowska',
                 'wmclaugh', 'Christoph Deil', 'Brigitta Sipocz'],
        idval='260416')

    add(version='4.8.2', title='sherpa/sherpa: Sherpa 4.8.2',
        date=todate(2016, 9, 23),
        authors=['Doug Burke', 'Omar Laurino', 'dtnguyen2',
                 'Jamie Budynkiewicz', 'Tom Aldcroft', 'Aneta Siemiginowska',
                 'Christoph Deil', 'wmclaugh', 'Brigitta Sipocz'],
        idval='154744')
    add(version='4.8.1', title='sherpa: Sherpa 4.8.1',
        date=todate(2016, 4, 15),
        authors=['Doug Burke', 'Omar Laurino', 'dtnguyen2', 'Tom Aldcroft',
                 'Jamie Budynkiewicz', 'Aneta Siemiginowska', 'Christoph Deil',
                 'Brigitta Sipocz'],
        idval='49832')
    add(version='4.8.0', title='sherpa: Sherpa 4.8.0',
        date=todate(2016, 1, 27),
        authors=['Doug Burke', 'Omar Laurino', 'dtnguyen2', 'Tom Aldcroft',
                 'Aneta Siemiginowska'],
        idval='45243')

    kwargs = cite.get(version, None)
    if kwargs is None:
        return None

    return _make_citation(**kwargs, latest=False)

def _download_json(url):
    """Return the JSON data or None.

    Parameters
    ----------
    url : str
        The URL to process.

    Returns
    -------
    response : dict
        The data in JSON as 'success' or an error message as 'failed'.

    """

    import json
    import ssl
    from urllib.request import Request, urlopen
    from urllib.error import HTTPError

    # Need to set the Content-Type so it looks like I need
    # a request. Darn it. Even though https://developers.zenodo.org/#list36
    # claims you can set the content-type, it just seems to
    # return JSON.
    #
    # req = Request(url,
    #               headers={'Content-Type': 'application/x-bibtex'})
    #
    req = Request(url)

    # This is not ideal, but given the way that we package
    # CIAO we can end up with issues evaluating SSL certificates,
    # so as we are not sending secure information we drop
    # the SSL checks.
    #
    context = ssl._create_unverified_context()

    try:
        res = urlopen(req, context=context)
    except HTTPError as he:
        dbg("Unable to access {}: {}".format(url, he))
        return {'failed': 'Unable to access the Zenodo site.'}

    try:
        jsdata = json.load(res)
    except UnicodeDecodeError as ue:
        dbg("Unable to decode JSON from Zenodo: {}".format(ue))
        return {'failed': 'Unable to understand the response from Zenodo.'}

    return {'success': jsdata}

def _make_zenodo_citation(jsdata, latest=True):
    """Convert Zenodo record to a citation.

    Parameters
    ----------
    jsdata : object
        A Zenodo record.

    Returns
    -------
    response : dict
        Success in 'success' (dict) or failure message in 'failed'.

    """

    try:
        created = jsdata['created']
        mdata = jsdata['metadata']
        idval = jsdata['id']
    except KeyError as ke:
        dbg("Unable to find metadata: {}".format(ke))
        return {'failed': 'Unable to parse the Zenodo response.'}

    created = created.split('T')[0]
    isoformat = '%Y-%m-%d'
    try:
        date = datetime.datetime.strptime(created, isoformat)
    except ValueError:
        dbg("Unable to convert created: '{}'".format(created))
        return {'failed': 'Unable to parse the Zenodo response.'}

    try:
        version = mdata['version']
        title = mdata['title']
        creators = mdata['creators']
    except KeyError as ke:
        dbg("Unable to find metadata: {}".format(ke))
        return {'failed': 'Unable to parse the Zenodo response.'}

    authors = [c['name'] for c in creators]
    out = _make_citation(version=version, title=title, date=date,
                         authors=authors, idval=idval,
                         latest=latest)
    return {'success': out}

def _get_citation_version():
    """What version of Sherpa are we using?

    Returns
    -------
    version : str
        The Sherpa version.
    """

    out = 'You are using Sherpa {}'.format(__version__)
    if '+' in __version__:
        out += " (it is not a released version)"

    out += ".\n\n"
    return out

def _get_citation_zenodo_failure(failed):
    """Standard response when there's a problem.

    Returns
    -------
    text : str
        The failure message.

    """
    out = 'There was a problem retrieving the data from Zenodo:\n'
    out += failed
    out += '\n\n'
    out += DEFAULT_CITATION
    return out

def _get_citation_zenodo_latest():
    """Query Zenodo for the latest release.

    Returns
    -------
    citation : str
        Citation information. It will include information
        on any failure.
    """

    # Can we retrieve the information from Zenodo?
    #
    # We do not access the DOI but the Zenodo API
    # url = 'https://doi.org/10.5281/zenodo.593753'
    url = 'https://zenodo.org/api/records/593753'
    jsdata = _download_json(url)
    if 'failed' in jsdata:
        return _get_citation_zenodo_failure(jsdata['failed'])

    out = _make_zenodo_citation(jsdata['success'])
    if 'failed' in out:
        return _get_citation_zenodo_failure(out['failed'])

    return out['success']

def _zenodo_missing(version):
    """

    Parameters
    ----------
    version : str
        The version we are looking for.

    Returns
    -------
    message : str
        The "unable to find" message.
    """

    return 'Zenodo has no information for version {}.'.format(version)

def _parse_zenodo_data(jsdata, version):
    """Extract data for the given version from a Zenodo query.

    Parameters
    ----------
    jsdata : json
        The response from Zenodo.
    version : str
        The version we are looking for.

    Returns
    -------
    response : dict
        Success in 'success' (dict) or failure message in 'failed'.
    """

    try:
        hits = jsdata['hits']['hits']
    except KeyError:
        dbg("Unable to find hits/hits")
        return {'failed': 'Unable to parse the Zenodo response.'}

    data = None
    try:
        for hit in hits:
            if hit['metadata']['version'] == version:
                data = hit
                break

    except KeyError:
        dbg("Record missing version")
        return {'failed': 'Unable to parse the Zenodo response.'}

    except TypeError:
        dbg('hits/hits is not iterable!')
        return {'failed': 'Unable to parse the Zenodo response.'}

    if data is None:
        dbg('Version {} not found'.format(version))
        return {'failed': _zenodo_missing(version)}

    return {'success': data}

def _download_zenodo_data(version):
    """Query Zenodo for the specific release.

    We have to deal with pagination in the Zenodo response.

    Parameters
    ----------
    version : str
        The release number (e.g. '4.12.2').

    Returns
    -------
    response : dict
        The data in JSON as 'success' or an error message as 'failed'.

    """

    # We could set the size parameter to something very large, to
    # get all responses with one call, but instead we use pagination.
    # Zenodo helpfully provides a links/next record with the
    # next URL, but it seems to be missing the all_versions=True
    # option, which makes it less-than-useful, hence the addition
    # of it below. The alternative would be to manually track the
    # page counter and add '&page=n' to the call.
    #
    url = 'https://zenodo.org/api/records/?q=conceptrecid:"593753"&all_versions=True&sort=mostrecent'
    missing = _zenodo_missing(version)

    while True:
        dbg("Zenodo query: {}".format(url))
        jsdata = _download_json(url)

        # If the query fails then we error out
        #
        if 'failed' in jsdata:
            return jsdata

        jsdata = jsdata['success']
        data = _parse_zenodo_data(jsdata, version)
        if 'success' in data:
            return data

        # There are two failures we care about:
        #  - we can parse the information but have not been able to
        #    find the version
        #  - any other reason
        #
        # If the former then we look for the links/next entry to
        # look at the next page of the response. If it doesn't
        # exist we assume we are on the last page and so can error
        # out.
        #
        # If the latter then we error out rather than trying anything
        # else.
        #
        if data['failed'] != missing:
            return data

        try:
            url = jsdata['links']['next']

            # Add in the necessary all_versions tag: see
            # https://github.com/zenodo/zenodo/issues/1662
            #
            if 'all_versions=True' not in url:
                url += '&all_versions=True'

        except KeyError:
            # There is no links/next field so assume we've checked all
            # pages
            return data

def _get_citation_zenodo_version(version):
    """Query Zenodo for the specific release.

    As this has to return all Sherpa records it is slow.

    Parameters
    ----------
    version : str
        The release number (e.g. '4.12.2').

    Returns
    -------
    citation : str
        Citation information.
    """

    jsdata = _download_zenodo_data(version)
    if 'failed' in jsdata:
        return _get_citation_zenodo_failure(jsdata['failed'])

    out = _make_zenodo_citation(jsdata['success'], latest=False)
    if 'failed' in out:
        return _get_citation_zenodo_failure(out['failed'])

    return out['success']

def _get_citation(version='current'):
    """Retrieve the citation information.

    Parameters
    ----------
    version : str, optional
        The version to retrieve the citation for. The supported values
        are limited to 'current', to return the citation for the
        installed version of Sherpa, 'latest' which will return the
        latest release, and the current set of releases available on
        Zenodo (this goes back to '4.8.0').

    Returns
    -------
    citation : str
        Citation information.

    """

    vstr = _get_citation_version()

    if version == 'latest':
        return vstr + _get_citation_zenodo_latest()

    if version == 'current':
        # Replace with the version (excluding any extraneous
        # information). Note that development versions close to a
        # release will have a version which has no release, but
        # I believe it is okay to say "Hey, Zenodo has no info
        # on this", rather than to try and fall back to the
        # previous version (since there's no good Zenodo reference
        # in this case).
        #
        version = __version__.split('+')[0]

    # We could include a check on the version number (e.g. a.b.c
    # where all elements are an integer), but do we want to
    # require this naming scheme?

    # If we know this version there's no need to call Zenodo
    #
    out = _get_citation_hardcoded(version)
    if out is not None:
        return vstr + out

    # In case the hardcoded list hasn't been updated.
    #
    return vstr + _get_citation_zenodo_version(version)

def citation(version='current', filename=None, clobber=False):
    """Return citatation information for Sherpa.

    The citation information is taken from Zenodo [1]_, using the
    Sherpa "latest release" identifier [2]_, and so requires an
    internet connection. The message is displayed on screen, using
    pagination, or can be written to a file.

    Parameters
    ----------
    version : str, optional
        The version to retrieve the citation for. The supported values
        are limited to 'current', to return the citation for the
        installed version of Sherpa, 'latest' which will return the
        latest release, and the current set of releases available on
        Zenodo (this goes back to '4.8.0').
    filename : str or StringIO or None, optional
        If not None, write the output to the given file or filelike
        object.
    clobber : bool, optional
        If filename is a string, then - when clobber is set - refuse
        to overwrite the file if it already exists.

    Notes
    -----
    If there is no internet connection, or there was a problem in
    downloading the data, or the Zenodo API has started to return
    different informatioon than expected, then the code will return
    information on why the call failed and other citation options.

    Zenodo only lets you perform a limited number of calls in
    a set time span, so if you call this routine too many times
    then it may start to fail.

    If a specific version is given then a hard-coded list of versions
    is checked, and if it matches then this information is used,
    rather than requiring a call to Zenodo.

    References
    ----------

    .. [1] https://zenodo.org/

    .. [2] https://doi.org/10.5281/zenodo.593753

    Examples
    --------

    Display the citation information for the current release on
    Zenodo. The information is paged to the display:

    >>> import sherpa
    >>> sherpa.citation()

    Write out the citation information for Sherpa 4.12.1 to the file
    ``cite.txt``:

    >>> sherpa.citation('4.12.1', outfile='cite.txt')

    Display the information for the latest release:

    >>> sherpa.citation('latest')

    """

    from sherpa.utils import send_to_pager
    cite = _get_citation(version=version)
    send_to_pager(cite, filename=filename, clobber=clobber)

def get_include():
    "Get the root path for installed Sherpa header files"

    return os.path.join(os.path.dirname(__file__), 'include')

def get_config():
    "Get the path for the installed Sherpa configuration file"

    filename = "sherpa-standalone.rc"

    # The behavior depends on whether the NOSHERPARC
    # environment variable is set.
    #
    if 'NOSHERPARC' not in os.environ:

        # If SHERPARC is set, try that
        #
        if 'SHERPARC' in os.environ:
            config = os.environ.get('SHERPARC')
            if os.path.isfile(config):
                return config

        # Can we use the HOME directory?
        #
        home_dir = os.environ.get('HOME')
        if home_dir is not None:
            config = os.path.join(home_dir, f'.{filename}')
            if os.path.isfile(config):
                return config

    # Fall back to the system config file
    return os.path.join(os.path.dirname(__file__), filename)

def smoke(verbosity=0, require_failure=False, fits=None, xspec=False, ds9=False):
    """Run Sherpa's "smoke" test.

    The smoke test is a simple test that ensures the Sherpa
    installation is functioning. It is not a complete test suite, but
    it fails if obvious issues are found.

    Parameters
    ----------
    verbosity : int, optional
        The level of verbosity of this test
    require_failure : boolean, optional
        For debugging purposes, the smoke test may be required to
        always fail. Defaults to False.
    fits : str or None, optional
        Require a fits module with this name to be present before
        running the smoke test. This option makes sure that when the
        smoke test is run the required modules are present.  Note that
        tests requiring fits may still run if any fits backend is
        available, and they might still fail on their own.
    xspec : boolean, optional
        Require xspec module when running tests. Tests requiring xspec
        may still run if the xspec module is present.
    ds9 : boolean, optional
        Requires DS9 when running tests.

    Raises
    ------
    SystemExit
        Raised if any errors are found during the tests.

    """
    from sherpa.astro.utils import smoke
    smoke.run(verbosity=verbosity, require_failure=require_failure, fits=fits, xspec=xspec, ds9=ds9)

def _smoke_cli(verbosity=0, require_failure=False, fits=None, xspec=False, ds9=False):
    from optparse import OptionParser
    parser = OptionParser()
    parser.add_option("-v", "--verbosity", dest="verbosity",
                      help="verbosity level")
    parser.add_option("-0", "--require-failure", dest="require_failure", action="store_true",
                      help="require smoke test to fail (for debugging)")
    parser.add_option("-f", "--fits-module", dest="fits", action="store",
                      help="require a specific fits module to be present")
    parser.add_option("-x", "--require-xspec", dest="xspec", action="store_true",
                      help="require xspec module")
    parser.add_option("-d", "--require-ds9", dest="ds9", action="store_true",
                      help="require DS9")

    options, _ = parser.parse_args()

    xspec = options.xspec or xspec
    verbosity = options.verbosity or verbosity
    require_failure = options.require_failure or require_failure
    fits = options.fits or fits
    ds9 = options.ds9 or ds9

    smoke(verbosity=verbosity, require_failure=require_failure, fits=fits, xspec=xspec, ds9=ds9)

def _install_test_deps() -> list[str]:
    """Check the needed modules for running the tests are installed.

    Since pytest has historically had issues with being able to use
    packages installed during this check, the list of such modules is
    returned so it can be passed to pytest.main().

    Returns
    -------
    plugins : list of module
        The pytest modules that were added by this call
        (may be empty).

    """

    def install(package_name):
        try:
            subprocess.call([sys.executable, '-m', 'pip', 'install', package_name],
                            stdout=sys.stdout, stderr=sys.stderr)
        except:
            print("""Cannot import pip or install packages with it.

            You need pytest in order to run the tests.
            If you downloaded the source code, please run 'pip install -r test_requirements.txt'
            from the source directory first.
            """)
            raise

    # packages are stored as a dictionary with keys
    #   name:  package name, as used by pip
    #   check: module to check it is installed
    #          (defaults to name if not given)
    #   constraint: the constraint to use with pip
    #               (defaults to name if not given)
    #
    # So we can have
    #
    #  {'name': 'pycheck'}
    #  {'name': 'pycheck', 'constraint': 'pytest>=5.0,!=5.2.3'}
    #  {'name': 'pytest-doctestplus',
    #   'check': 'pytest_doctestplus.output_checker'}
    #
    # Note that for pytest plugins, the module name is assumed to be
    # the name field with hyphens replaced by underscores. If necessary
    # this could be updated to be another field in the package
    # dictionary.
    #
    # The reason for the "check" field is that mid 2023 it was found
    # that
    #
    #    importlib.import_module("pytest_doctestplus")
    #
    # would work even when not installed, hence the need to import a
    # module within the package as a check. This has been left in even
    # though the current list of required plugins is empty.
    #
    deps: list[dict[str, str]] = [{'name': 'pytest',
                                   'constraint': 'pytest>=5.0,!=5.2.3'}]
    pytest_plugins: list[dict[str, str]] = []

    def get(dep: dict[str, str]) -> tuple[str, str, str]:
        name = dep['name']
        constraint = dep.get('constraint', name)
        check = dep.get('check', name)
        return name, constraint, check

    installed_plugins = []
    for dep in deps:
        name, constraint, check = get(dep)
        try:
            importlib.import_module(check)
        except ImportError:
            install(constraint)

    for dep in pytest_plugins:
        name, constraint, check = get(dep)
        try:
            importlib.import_module(check)
        except ImportError:
            install(constraint)
            installed_plugins.append(name.replace('-', '_'))

    return installed_plugins

def clitest() -> None:
    """The sherpa_test endpoint."""

    plugins = _install_test_deps()
    import pytest

    # Add in command-line arguments to allow configuring the Sherpa tests
    args = [os.path.dirname(__file__), '-rs'] + sys.argv[1:]

    # passing the plugins that have been installed "now".
    errno = pytest.main(args, plugins=plugins)
    sys.exit(errno)


"""
File that contains the functions to transform the data in dataset for the regressors, cnn and rnn
"""
import math
import numpy as np
import pandas as pd
from filterpy.kalman import KalmanFilter
import matplotlib.pyplot as plt

from Configuration import dataset_config

def create_kalman_filter(kalman_filter_par):
    """
    create the kalman filter object
    :param kalman_filter_par: parameters to be used
    :return: the kalman filter object
    """
    kalman_filter = KalmanFilter(dim_x=1, dim_z=1)
    kalman_filter.F = np.array([[kalman_filter_par['A']]])
    kalman_filter.H = np.array([[kalman_filter_par['C']]])
    kalman_filter.R = np.array([[kalman_filter_par['R']]])
    kalman_filter.Q = np.array([[kalman_filter_par['Q']]])

    return kalman_filter

def get_index_taglio(tele):
    """
    Get the index on which the tag has been moved from one position to another from the camera data
    :param tele: array of data of the camera
    :return: array of index
    """
    indextaglio = [100]
    for i in range(len(tele[2]) - 1):  # len(tele[2]) = lunghezza valori nella x
        if (abs(tele[2][i] - tele[2][i + 1]) > 0.05) | (abs(tele[3][i] - tele[3][i + 1]) > 0.05):
            indextaglio.append(i)

    indextaglio.append(len(tele[2]) - 1)

    return indextaglio

def get_index_taglio_reader(time):
    """
    Get the index on which the tag has been moved from one position to another from the readers data
    :param time: array of timestamp on which the reader receives the beacon
    :return: array of index
    """
    indextaglio_reader = []

    for j in range(5):
        indextaglio_reader.append([0])
        for i in range(len(time[j]) - 1):
            if abs(time[j][i] - time[j][i + 1]) > 5:  # 5 secondi
                indextaglio_reader[j].append(i)
        indextaglio_reader[j].append(len(time[j]) - 1)

    # for j in range(2):
    #     for i in range(len(indextaglio_reader)):
    #         indextaglio_reader.append(indextaglio_reader[i])

    return indextaglio_reader

def fixReader(dati, time, tele):
    """
    Function for cutting data from the 5 readers, aligning the data collected by the cameras and the data collected by the readers over time over time
    :param dati: data from the readers
    :param time: time from the readers
    :param tele: data from the cameras
    :return: extended reader data, data from the cameras and the new index of change position
    """
    newData = []
    newTime = []

    indextaglio = get_index_taglio(tele)
    indextaglio_reader = get_index_taglio_reader(time)

    num_tagli = 0
    for i in range(len(indextaglio_reader[0]) - 1):
        if indextaglio_reader[0][i + 1] - indextaglio_reader[0][i] >= 1:
            num_tagli += 1

    print(num_tagli)

    # fig, ax = plt.subplots()
    # debug_index_taglio_dict = {}
    # for j in range(0):
    #     debug_index_taglio = []
    #     for i in range(len(time[j]) - 1):
    #         elem = time[j][i + 1] - time[j][i]
    #         debug_index_taglio.append(elem)
    #     x_axis = list(range(len(debug_index_taglio)))
    #     ax.plot(x_axis, debug_index_taglio)

    # df = pd.DataFrame(debug_index_taglio_dict)
    # df.plot.line()

    if len(indextaglio) < len(indextaglio_reader[0]):
        for index in indextaglio_reader:
            index.remove(index[0])

    for i in range(len(dati)):
        newData.append([])
        newTime.append([])
        for j in range(len(indextaglio_reader[i]) - 1):
            tot_data_inserted = 0
            dim_data_reader = indextaglio_reader[i][j + 1] - indextaglio_reader[i][j]  # dimensione dei dati raccolti

            # nel punto j-esimo dal BLE
            dim_data_tele = indextaglio[j + 1] - indextaglio[j]  # dimensione dei dati raccolti nel punto j-esimo
            # dalle tele
            fattore = math.trunc(dim_data_tele / dim_data_reader)

            for t in range(dim_data_reader + 1):
                for x in range(fattore + 1):
                    if tot_data_inserted < dim_data_tele:
                        newData[i].append(dati[i][indextaglio_reader[i][j] + t + 1])
                        newTime[i].append(time[i % 5][indextaglio_reader[i][j] + t + 1])
                        tot_data_inserted += 1

    return newData, newTime, indextaglio

def transform_in_dataframe(data, index):
    """
    transform the dataset in a list o dataframe in witch each dataframe represent a position
    :param data: data from the readers
    :param index: index of change position
    :return: list of dataframe with five columns corresponding to the RSSI value of the readers
    """
    reader_position_list = {}
    for index_row, row in index[0].iterrows():
        reader_position_list[f'pos{index_row}'] = pd.DataFrame()

    for i in range(len(data)):
        for index_row, row in index[i].iterrows():
            position_reader_data = data[i][row['start']: row['end']]
            position_reader_df = pd.DataFrame({f'reader{i}': position_reader_data})

            reader_position_list[f'pos{index_row}'] = pd.concat(
                [
                    reader_position_list[f'pos{index_row}'],
                    position_reader_df
                ], axis=1)

    for position in reader_position_list.keys():
        df = reader_position_list[position]
        first_cut_number = int(df.shape[0] / dataset_config.INITIAL_CUT)
        df = df.iloc[first_cut_number:]
        final_cut_number = int(df.shape[0] / dataset_config.FINAL_CUT)
        reader_position_list[position] = df.iloc[:-final_cut_number]

    for position in reader_position_list.keys():
        df = reader_position_list[position]
        reader_position_list[position] = df.loc[(df.shift() != df).any(axis=1)]

    return reader_position_list

def cutReader(dati, tele, ind):
    """
    remove the first one third and last one third of each position and remove the duplicates
    :param dati: data from the readers
    :param tele: data from the comeras
    :param ind: index of change position
    :return: cutted data from reader and cameras and new list of indexes of change position
    """
    newData = [[] for _ in range(len(dati))]
    newTele = [[] for _ in range(2)]
    new_data = [0 for _ in range(len(dati))]
    old_data = [0 for _ in range(len(dati))]

    new_ind = [0]
    for j in range(len(ind) - 1):
        dim = ind[j + 1] - ind[j]
        offset = math.trunc(dim / 3)
        # print(offset)
        lun = offset
        for t in range(lun):
            for i in range(len(dati)):
                if ind[j] + offset + t < len(dati[i]):
                    new_data[i] = dati[i][ind[j] + offset + t]

            # np.isnan, is for mean and std dataset
            if not np.array_equal(new_data, old_data) and not np.isnan(np.min(new_data)):
                for i in range(len(dati)):
                    newData[i].append(dati[i][ind[j] + offset + t])
                for i in range(len(new_data)):
                    old_data[i] = new_data[i]
                newTele[0].append(tele[2][ind[j] + offset + t])
                newTele[1].append(tele[3][ind[j] + offset + t])
        new_ind.append(len(newData[0]))

    # for i in range (5):
    # print(len(newData[i]))
    return newData, newTele, new_ind

def apply_kalman_filter(no_kalman_data, kalman_filter_par):
    """
    Apply  the kalman filter on the raw data
    :param no_kalman_data: raw data from the readers
    :param kalman_filter_par: kalman filter parameters to be used
    :return: Data from readers filtered with the kalman filter
    """
    kalman_data = []

    kalman_filter = create_kalman_filter(kalman_filter_par)

    for i, raw_data_reader in enumerate(no_kalman_data):
        # reset the filter
        kalman_filter.x = np.array([kalman_filter_par['x']])
        kalman_filter.P = np.array([[kalman_filter_par['P']]])

        kalman_data.append([])
        for raw_value in raw_data_reader:
            kalman_filter.predict()
            kalman_filter.update(raw_value)
            kalman_data[i].append(kalman_filter.x[0])

    return kalman_data

def positivize_rssi(data_readers, min_RSSI):
    """
    For RNN process the RSSI by adding the min value to get all positive value
    :param data_readers: data from readers
    :param min_RSSI: min value to add
    :return: the data from reader in positive value
    """
    normalized_RSSI = [[] for _ in data_readers]

    for r, data_reader in enumerate(data_readers):
        for RSSI_value in data_reader:
            normalized_RSSI[r].append(RSSI_value + abs(min_RSSI))

    return normalized_RSSI


# coding=utf-8
"""Further reading form."""

from django import forms
from django.utils.translation import ugettext_lazy as _

from crispy_forms.helper import FormHelper
from crispy_forms.layout import (
    Layout,
    Fieldset,
    Field,
    Submit,
)
from tinymce.widgets import TinyMCE

from modeltranslation.forms import TranslationModelForm
from lesson.models.further_reading import FurtherReading

class FurtherReadingForm(TranslationModelForm):
    """Form for creating further reading item."""

    text = forms.CharField(widget=TinyMCE(attrs={'cols': 80, 'rows': 30}))

    class Meta:
        model = FurtherReading
        fields = (
            'text',
        )

    def __init__(self, *args, **kwargs):
        self.worksheet = kwargs.pop('worksheet')
        self.helper = FormHelper()
        layout = Layout(
            Fieldset(
                _('Worksheet {}: further more item details').format(
                    self.worksheet),
                Field('text', css_class='form_control'),
                css_id='project-form'
            )
        )

        self.helper.layout = layout
        self.helper.html5_required = False

        super(FurtherReadingForm, self).__init__(*args, **kwargs)

        self.helper.add_input(Submit('submit', 'Submit'))

    def save(self, commit=True):
        instance = super(FurtherReadingForm, self).save(commit=False)
        instance.worksheet = self.worksheet
        instance.save()
        return instance


import cv2
import pyzed.camera as zcam
import pyzed.types as tp
import pyzed.core as core
import pyzed.defines as sl
import math
import numpy as np
import sys
import imutils

def main():
    i = 0
    image = core.PyMat()
    depth = core.PyMat()
    point_cloud = core.PyMat()

    print("Running...")
    init_params = zcam.PyInitParameters()
    init_params.depth_mode = sl.PyDEPTH_MODE.PyDEPTH_MODE_PERFORMANCE  # Use PERFORMANCE depth mode
    init_params.coordinate_units = sl.PyUNIT.PyUNIT_MILLIMETER  # Use milliliter units (for depth measurements)

    cam = zcam.PyZEDCamera()
    if not cam.is_opened():
        print("Opening ZED Camera...")
    status = cam.open(init_params)

    if status != tp.PyERROR_CODE.PySUCCESS:
        print(repr(status))
        exit()

    runtime = zcam.PyRuntimeParameters()
    runtime.sensing_mode = sl.PySENSING_MODE.PySENSING_MODE_STANDARD  # Use STANDARD sensing mode

    mat = core.PyMat()

    key = ''
    while key != 113:  # for 'q' key
        err = cam.grab(runtime)
        if err == tp.PyERROR_CODE.PySUCCESS:
            #cam.retrieve_image(mat, sl.PyVIEW.PyVIEW_LEFT)
           # cam.retrieve_image(mat, sl.PyVIEW.PyVIEW_DEPTH)
            #cam.retrieve_image(mat, sl.PyVIEW.PyVIEW_LEFT)

            cam.retrieve_image(mat, sl.PyVIEW.PyVIEW_NORMALS)
            cam.retrieve_measure(depth, sl.PyMEASURE.PyMEASURE_DEPTH)
            cam.retrieve_measure(point_cloud, sl.PyMEASURE.PyMEASURE_XYZRGBA)

            #euklinda distanca za mirit udaljenost objekta

            x = round(mat.get_width() / 2)
            y = round(mat.get_height() / 2)

            err, point_cloud_value = point_cloud.get_value(x, y)
            distance = math.sqrt(point_cloud_value[0] * point_cloud_value[0] +
                                 point_cloud_value[1] * point_cloud_value[1] +
                                 point_cloud_value[2] * point_cloud_value[2])

            if not np.isnan(distance) and not np.isinf(distance):
                distance = round(distance)
                print("Distance to Camera at ({0}, {1}): {2} mm\n".format(x, y, distance))
                # Increment the loop
                i = i + 1
            else:
                print("Can't estimate distance at this position, move the camera\n")
            sys.stdout.flush()
            #cam.retrieve_image(mat, sl.PyVIEW.PyVIEW_LEFT_UNRECTIFIED)

            cv2.imshow("ZED", mat.get_data())
            key = cv2.waitKey(5)
        else:
            key = cv2.waitKey(5)
    cv2.destroyAllWindows()

    cam.close()

main()


# python _52products.iterTools.py

#we know that with sets we can do union, intersection, difference, isSubset, issuperset, isDisjoint

#can we do te same with lists
#lets try

'''
myList1 = [1,2]
myList2 = [3]

print(myList1.union(myList2))
'''

#no it is not possible
#but python gives all the pairs we can form between the two lists
#it uses the product method from itertools

from itertools import product

myList1 = [1,2]
myList2 = [3]
result1 = product(myList1, myList2)
print(list(result1))

#if we want multiple element instead of pairs we can also do by
result2 = product(myList1, myList2,repeat = 2)
print(list(result2))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Time    : 2019/2/22 10:01
# @User    : zhunishengrikuaile
# @File    : Numbers.py
# @Email   : binary@shujian.org
# @MyBlog  : WWW.SHUJIAN.ORG
# @NetName : 書劍
# @Software: 百度识图Api封装
# Numbers
# 数字识别

import os
import base64
import requests
from bin.AccessToken.AccessToken import AccessToken
from config.config import LOCALHOST_PATH, URL_LIST_URL

ACCESS_TOKEN = AccessToken().getToken()['access_token']

NUMBERS_URL = URL_LIST_URL['NUMBERS'] + '?access_token={}'.format(ACCESS_TOKEN)

class NumbersSuper(object):
    pass

class Numbers(NumbersSuper):

    def __init__(self, image=None, recognize_granularity='small', detect_direction=True):
        self.HEADER = {
            'Content-Type': 'application/x-www-form-urlencoded',
        }
        self.IMAGE_CONFIG = {
            'recognize_granularity': recognize_granularity,
            'detect_direction': detect_direction
        }

        if image is not None:
            imagepath = os.path.exists(LOCALHOST_PATH['PATH'] + image)
            if imagepath == True:
                images = LOCALHOST_PATH['PATH'] + image
                with open(images, 'rb') as images:
                    self.IMAGE_CONFIG['image'] = base64.b64encode(images.read())

    def postNumbers(self):
        if self.IMAGE_CONFIG.get('image', None) == None:
            return 'image参数不能为空!'
        numbers = requests.post(url=NUMBERS_URL, headers=self.HEADER,
                                data=self.IMAGE_CONFIG)
        return numbers.json()


from django.http import HttpResponse

from django.template import loader
from django.template.loader import get_template
# def index(request):
#     latest_question_list = ['a','b','c']
#     template = loader.get_template('product/index.html')
#     context = {
#         'latest_question_list': 'string',
#     }
#     return HttpResponse(template.render(context, request))

from my_backend.load_search import find_file
from django.shortcuts import render
from django.http import HttpResponseRedirect

from product.forms import NameForm
from product.forms import SelectForm
import os,sys
import config
# from tests.unit_tests import run_all_tests
INDEX = '/product/templates/product/index.html'
def name(request):
    # if this is a POST request we need to process the form data
    if request.method == 'POST':
        form = NameForm(request.POST)
        result = str(form["your_product"].value())
        q_param = form["your_product"].value()
        resultt , search_r = find_file(q_param)
        # check whether it's valid:
        if form.is_valid():
            # print(form["your_product"].value())
            return render(request, "selected.html" , {'resultt':resultt,'search_results' : search_r})
    else:
        form = NameForm()

    return render(request, 'selected.html', {'form': form})

def results(request):
    return render(request, 'results.html')

def index(request):
    # TEMPLATE_PATH =os.path.join(os.getcwd(), INDEX)
    print("os.getcwd... ",os.getcwd())
    TEMPLATE_PATH = os.getcwd() + INDEX
    template = get_template(TEMPLATE_PATH)
    context = {
        'key': 'value',
    }
    return HttpResponse(template.render(context, request))

def selected(request):
    # if this is a POST request we need to process the form data
    if request.method == 'POST':
        form = SelectForm(request.POST)
        resultt = (form["selected_choice"].value())
        print(form["selected_choice"].value())
        # check whether it's valid:
        if form.is_valid():
            print("debug_inside")
        else:
            print("debug !form is_valid")

        return render(request, "selected.html" , {'form' : form,'resultt':resultt})
# def test(request):
#     if request.method == 'GET':
#         unit_tests.run_all_tests()


import io
import os
import cv2
import flask
import tensorflow as tf
from tensorflow.keras.models import load_model
import numpy as np
from PIL import Image
import imagehash
import subprocess

app = flask.Flask(__name__)

# tf.config.gpu.set_per_process_memory_fraction(0.65)

model = None

IMG_SIZE = 150

def do_load_model():
    with tf.device('/cpu:0'):
        global model
        model = load_model('model.h5')

do_load_model()

def get_extension(format):
    if format == 'JPEG':
        return "jpg"
    if format == 'PNG':
        return "png"
    return "DUNNO"

def find_duplicate(hash):
    DIR = "./data/combined"

    level = []
    files = []

    for folder in os.listdir(DIR):
        for img in os.listdir(DIR + "/" + folder):
            files.append(img)
            level.append(folder)

    if hash in files:
        return int(level[files.index(hash)])

    return 0

def to_pimg(file):
    in_memory_file = io.BytesIO()
    file.save(in_memory_file)
    return Image.open(in_memory_file)

def load_mem(file):
    in_memory_file = io.BytesIO()
    file.save(in_memory_file)
    return in_memory_file

training = False
proc = None

import os

@app.route("/retrain", methods=["GET"])
def retrain():
    global training
    if not training:
        training = True
        global proc
        dirpath = os.getcwd()
        proc = subprocess.Popen([dirpath + "/venv/Scripts/python.exe", "train.py"], shell=True,
                                stdin=None, stdout=None, stderr=None, close_fds=True)
        return flask.jsonify({"success": True})
    else:
        return flask.jsonify({"success": False})

@app.route("/forcetrain", methods=["GET"])
def forcetrain():
    global training
    training = False
    return flask.jsonify({"success": True})

@app.route("/reload", methods=["GET"])
def reload():
    global training
    training = False
    do_load_model()
    return flask.jsonify({"success": True})

@app.route("/predict", methods=["POST"])
def predict():
    res = {"success": False}
    if 'photo' in flask.request.files:
        photo = flask.request.files['photo']
        mf = load_mem(photo)
        pimg = Image.open(mf)
        hash = imagehash.average_hash(pimg)
        duplicate = find_duplicate(str(hash) + "." + get_extension(pimg.format))
        if duplicate > 0:
            res['predict'] = duplicate
            res['success'] = True
            res['fromCache'] = True
        else:
            X = []
            mf.seek(0)
            data = np.asarray(bytearray(mf.read()), dtype=np.uint8)
            img = cv2.imdecode(data, cv2.IMREAD_COLOR)
            img = cv2.resize(img, (IMG_SIZE, IMG_SIZE))
            X.append(np.array(img))

            X = np.array(X)
            X = X / 255
            with tf.device('/cpu:0'):
                pred = model.predict_classes(X)[0]
                res['predict'] = int(pred) + 1
            res['fromCache'] = False
            res['success'] = True

    return flask.jsonify(res)

@app.route("/add", methods=["POST"])
def add_to_model():
    res = {"success": False, "move": False}

    if 'rank' in flask.request.form and 'photo' in flask.request.files:
        photo = flask.request.files['photo']
        rank = str(flask.request.form['rank'])
        pimg = to_pimg(photo)
        hash = imagehash.average_hash(pimg)
        res['hash'] = str(hash)
        duplicate = find_duplicate(str(hash) + "." + get_extension(pimg.format))
        if duplicate > 0:
            res['move'] = True
            res['from'] = duplicate
            os.rename('./data/combined/' + str(duplicate) + "/" + str(hash) + "." + get_extension(pimg.format),
                      './data/combined/' + rank + "/" + str(hash) + "." + get_extension(pimg.format))
        else:
            pimg.save('./data/combined/' + rank + "/" + str(hash) + "." + get_extension(pimg.format))

        res['success'] = True
    return flask.jsonify(res)

app.run(host="0.0.0.0")


import json
from bvss import Vuln

tests = {'web': 'tests/web.json',
         'smart conract': 'tests/smart_contract.json',
         'blockchain': 'tests/blockchain.json',
         'hardware': 'tests/hardware.json'}
for key in tests:
    print('------------------------{}------------------------'.format(key))
    with open(tests[key]) as f:
        vulns = json.load(f)

    for vuln in vulns:
        item = Vuln(vuln)
        print("Desc: {}\nScore: {}\nBVSS: {}\n".format(item.desc, item.calc_score(), item.calc_bvss()))


import numpy as np

import gym
from gym import spaces

class NasEnvEmb(gym.Env):

    def __init__(self, space, evaluator, structure):

        self.space = space
        self.structure = structure
        self.evaluator = evaluator

        self.dim_ob_sp = len(structure.get_hash(0, 0))
        self.observation_space = spaces.Box(low=0, high=1,
            shape=(self.dim_ob_sp,), dtype=np.float32)
        self.action_space = spaces.Discrete(self.structure.max_num_ops)

        self._state = np.array([0 for i in range(self.dim_ob_sp)])
        self.action_buffer = []
        self.num_timesteps = self.structure.num_nodes

    def step(self, action, index, rank=None):

        self.action_buffer.append(action)

        if len(self.action_buffer) < self.num_timesteps:
            terminal = False
            reward = 0
            action_hash = self.structure.get_hash(len(self.action_buffer)-1, action)
            self._state = np.array([float(e) for e in action_hash])
            return self._state, reward, terminal, {}

        conv_action = np.array(self.action_buffer) / self.structure.max_num_ops

        terminal = True
        self.action_buffer = []
        self._state = np.array([0 for i in range(self.dim_ob_sp)])

        cfg = self.space.copy()
        cfg['arch_seq'] = list(conv_action)
        cfg['w'] = index
        if rank != None:
            cfg['rank'] = rank

        self.evaluator.add_eval(cfg)

        # ob, reward, terminal
        return self._state, None, terminal, {}

    def get_rewards_ready(self):
        return self.evaluator.get_finished_evals()

    def reset(self):
        self.__init__(self.space, self.evaluator, self.structure)
        return self._state


import argparse
import collections
import cv2
import numpy as np
import os
import pdb
import random
import time

import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.optim as optim
import torch.utils.data
from torch.autograd import Variable
import torch.nn.functional as F
from torch.utils.tensorboard import SummaryWriter
from torch.cuda.amp import GradScaler
torch.backends.cudnn.benchmark=True
autocast = torch.cuda.amp.autocast

from utils.flowlib import flow_to_image
from utils.io import add_image
from models import *
from utils.multiscaleloss import realEPE

parser = argparse.ArgumentParser(description='VCNPlus')
parser.add_argument('--maxdisp', type=int ,default=256,
                    help='maxium disparity, out of range pixels will be masked out. Only affect the coarsest cost volume size (default 256)')
parser.add_argument('--fac', type=float ,default=1,
                    help='controls the shape of search grid. Only affect the coarsest cost volume size (default 1)')
parser.add_argument('--logname', default='exp-1',
                    help='name of the log file (default exp-1)')
parser.add_argument('--database',
                    help='path to the database (required)')
parser.add_argument('--loadmodel', default=None,
                    help='path of the pre-trained model (default None)')
parser.add_argument('--loadflow', default=None,
                    help='path of the pre-trained flow model (default None)')
parser.add_argument('--savemodel',
                    help='path to save the model (required)')
parser.add_argument('--retrain', default='true',
                    help='whether to reset moving mean / other hyperparameters (default true)')
parser.add_argument('--stage', default='expansion',
                    help='one of {chairs, things, 2015train, 2015trainval, sinteltrain, sinteltrainval, expansion, expansion2015train, expansion2015tv} (deafult expansion)')
parser.add_argument('--nproc', type=int, default=1,
                    help='number of process to use (default 1)')
parser.add_argument('--ngpus', type=int, default=1,
                    help='(deprecated) number of gpus to use before ddp (default 1)')
parser.add_argument('--itersave', default='./',
                    help='a dir to save iteration counts (default ./)')
parser.add_argument('--niter', type=int ,default=300000,
                    help='maximum iteration (default 300k)')
parser.add_argument('--local_rank', type=int, default=0)
args = parser.parse_args()

# distributed dataparallel
torch.cuda.set_device(args.local_rank)
world_size = args.nproc
torch.distributed.init_process_group(
    'nccl',
    init_method='env://',
    world_size=world_size,
    rank=args.local_rank,
)

# fix random seed
torch.manual_seed(1)
def _init_fn(worker_id):
    np.random.seed()
    random.seed()
torch.manual_seed(8)  # do it again
torch.cuda.manual_seed(1)

## set hyperparameters for training
ngpus = args.ngpus
worker_mul = int(1)

if 'expansion' in args.stage:
    datashape = [320,640]
    batch_size = 6*ngpus
elif 'seg' in args.stage:
    datashape = [320,640]
    batch_size = 6*ngpus
elif args.stage == 'chairs' or args.stage == 'things':
    datashape = [320,448]
    batch_size = 4*ngpus
elif '2015' in args.stage:
    datashape = [256,768]
    batch_size = 4*ngpus
elif 'sintel' in args.stage:
    datashape = [320,576]
    batch_size = 4*ngpus
elif args.stage == 'rob':
    datashape = [320,640]
    batch_size = 3*ngpus
else:
    print('error')
    exit(0)

## dataloader
## expansion datasets
if 'expansion' in args.stage:
    from dataloader import exploader as dd
    if '2015' in args.stage:
        if 'train' in args.stage:
            from dataloader import kitti15list_train as lk15
        elif 'tv' in args.stage:
            from dataloader import kitti15list as lk15
        iml0, iml1, flowl0 = lk15.dataloader('%s/kitti_scene/training/'%args.database)
        disp0 = [i.replace('flow_occ','disp_occ_0') for i in flowl0]
        disp1 = [i.replace('flow_occ','disp_occ_1') for i in flowl0]
        calib = [i.replace('flow_occ','calib')[:-7]+'.txt' for i in flowl0]
        loader_kitti15_sc = dd.myImageFloder(iml0,iml1,flowl0, shape=datashape, scale=1, order=0,prob=0.5,disp0=disp0, disp1=disp1, calib=calib)
    else:
        from dataloader import sceneflowlist as lsf
        iml0, iml1, flowl0, disp0, dispc, calib = lsf.dataloader('%s/Driving/'%args.database, level=6)
        loader_driving_sc = dd.myImageFloder(iml0,iml1,flowl0, shape=datashape, disp0=disp0,disp1=dispc,calib=calib)
        iml0, iml1, flowl0, disp0, dispc, calib = lsf.dataloader('%s/Monkaa/'%args.database, level=4)
        loader_monkaa_sc = dd.myImageFloder(iml0,iml1,flowl0, shape=datashape, disp0=disp0,disp1=dispc,calib=calib)
        iml0, iml1, flowl0, disp0, dispc, calib = lsf.dataloader('/ssd1/gengshay/FlyingThings3D/', level=6)
        loader_things_sc = dd.myImageFloder(iml0,iml1,flowl0, shape=datashape, disp0=disp0,disp1=dispc,calib=calib)
        # kitti
        from dataloader import kitti15list_train as lk15
        iml0, iml1, flowl0 = lk15.dataloader('%s/kitti_scene/training/'%args.database)
        disp0 = [i.replace('flow_occ','disp_occ_0') for i in flowl0]
        disp1 = [i.replace('flow_occ','disp_occ_1') for i in flowl0]
        calib = [i.replace('flow_occ','calib')[:-7]+'.txt' for i in flowl0]
        loader_kitti15_sc = dd.myImageFloder(iml0,iml1,flowl0, shape=datashape, scale=1, order=0,prob=0.5,disp0=disp0, disp1=disp1, calib=calib)
        # sintel
        from dataloader import sintellist_train as ls
        iml0, iml1, flowl0 = ls.dataloader('%s/rob_flow/training/'%args.database)
        disp0 = []; disp1 = []; calib = []
        for impath in iml0:
            passname = impath.split('/')[-1].split('_')[-4]
            seqname1 = impath.split('/')[-1].split('_')[-3]
            seqname2 = impath.split('/')[-1].split('_')[-2]
            framename = int(impath.split('/')[-1].split('_')[-1].split('.')[0])
            disp0.append('%s/Sintel/disparities/%s_%s/frame_%04d.png'%(impath.rsplit('/',2)[0], seqname1, seqname2,framename+1))
            disp1.append('%s/Sintel/disparities/%s_%s/frame_%04d.png'%(impath.rsplit('/',2)[0], seqname1, seqname2,framename+2))
            calib.append('%s/Sintel/camdata_left/%s_%s/frame_%04d.cam'%(impath.rsplit('/',2)[0], seqname1,seqname2,framename+1))
        loader_sintel_sc = dd.myImageFloder(iml0,iml1,flowl0, shape=datashape, scale=1, order=1, noise=0, disp0=disp0, disp1=disp1, calib=calib)
elif 'seg' in args.stage:
    from dataloader import exploader as dd
    from dataloader import sceneflowlist as lsf
    iml0, iml1, flowl0, disp0, dispc, calib = lsf.dataloader('%s/Driving/'%args.database, level=6)
    loader_driving_sc = dd.myImageFloder(iml0,iml1,flowl0, shape=datashape, disp0=disp0,disp1=dispc,calib=calib)
    iml0, iml1, flowl0, disp0, dispc, calib = lsf.dataloader('%s/Monkaa/'%args.database, level=4)
    loader_monkaa_sc = dd.myImageFloder(iml0,iml1,flowl0, shape=datashape, disp0=disp0,disp1=dispc,calib=calib)
    iml0, iml1, flowl0, disp0, dispc, calib = lsf.dataloader('%s/FlyingThings3D/', level=6)
    loader_things_sc = dd.myImageFloder(iml0,iml1,flowl0, shape=datashape, disp0=disp0,disp1=dispc,calib=calib)
    # kitti
    from dataloader import kitti15list as lk15
    iml0, iml1, flowl0 = lk15.dataloader('%s/kitti_scene/training/'%args.database)
    disp0 = [i.replace('flow_occ','disp_occ_0') for i in flowl0]
    disp1 = [i.replace('flow_occ','disp_occ_1') for i in flowl0]
    calib = [i.replace('flow_occ','calib')[:-7]+'.txt' for i in flowl0]
    # dense disp
    disp0 = [i.replace('disp_occ_0','disp_occ_0_ganet') for i in disp0]
    loader_kitti15_sc = dd.myImageFloder(iml0,iml1,flowl0, shape=datashape, scale=1, order=0,prob=0.5,disp0=disp0, disp1=disp1, calib=calib)
else: # flow
    from dataloader import robloader as dr
    if args.stage == 'chairs' or 'sintel' in args.stage or args.stage=='rob':
        # flying chairs
        from dataloader import chairslist as lc
        iml0, iml1, flowl0 = lc.dataloader('%s/FlyingChairs_release/data/'%args.database)
        with open('misc/order.txt','r') as f:
            order = [int(i) for i in f.readline().split(' ')]
        with open('misc/FlyingChairs_train_val.txt', 'r') as f:
            split = [int(i) for i in f.readlines()]
        iml0 = [iml0[i] for i in order     if split[i]==1]
        iml1 = [iml1[i] for i in order     if split[i]==1]
        flowl0 = [flowl0[i] for i in order if split[i]==1]
        loader_chairs = dr.myImageFloder(iml0,iml1,flowl0, shape = datashape)
    if args.stage == 'things' or 'sintel' in args.stage or args.stage=='rob':
        # flything things
        from dataloader import thingslist as lt
        iml0, iml1, flowl0 = lt.dataloader('/ssd0/gengshay/FlyingThings3D_subset/train/')
        loader_things = dr.myImageFloder(iml0,iml1,flowl0,shape = datashape,scale=1, order=1)

    # fine-tuning datasets
    if args.stage == '2015train' or args.stage=='rob':
        from dataloader import kitti15list_train as lk15
    else:
        from dataloader import kitti15list as lk15
    if args.stage == 'sinteltrain' or args.stage=='rob':
        from dataloader import sintellist_train as ls
    else:
        from dataloader import sintellist as ls
    from dataloader import kitti12list as lk12
    from dataloader import hd1klist_train as lh

    if 'sintel' in args.stage:
        iml0, iml1, flowl0 = lk15.dataloader('%s/kitti_scene/training/'%args.database)
        loader_kitti15 = dr.myImageFloder(iml0,iml1,flowl0, shape=datashape, scale=1, order=0, noise=0)  # SINTEL
        iml0, iml1, flowl0 = lh.dataloader('%s/rob_flow/training/'%args.database)
        loader_hd1k = dr.myImageFloder(iml0,iml1,flowl0,shape=datashape, scale=0.5,order=0, noise=0)
        iml0, iml1, flowl0 = ls.dataloader('%s/rob_flow/training/'%args.database)
        loader_sintel = dr.myImageFloder(iml0,iml1,flowl0, shape=datashape, scale=1, order=1, noise=0)
        #loader_sintel = dr.myImageFloder(iml0,iml1,flowl0, shape=datashape, scale=1, order=1, noise=0, scale_aug=[0.2,0.])
    if '2015' in args.stage:
        iml0, iml1, flowl0 = lk12.dataloader('%s/data_stereo_flow/training/'%args.database)
        #loader_kitti12 = dr.myImageFloder(iml0,iml1,flowl0, shape=datashape, scale=1, order=0, prob=0.5, scale_aug=[0.2,0.])
        loader_kitti12 = dr.myImageFloder(iml0,iml1,flowl0, shape=datashape, scale=1, order=0, prob=0.5)
        iml0, iml1, flowl0 = lk15.dataloader('%s/kitti_scene/training/'%args.database)
        #loader_kitti15 = dr.myImageFloder(iml0,iml1,flowl0, shape=datashape, scale=1, order=0, prob=0.5, scale_aug=[0.2,0.])  # KITTI
        loader_kitti15 = dr.myImageFloder(iml0,iml1,flowl0, shape=datashape, scale=1, order=0, prob=0.5)  # KITTI
    if args.stage=='rob':
        #from dataloader import kitti15list as lk15
        #from dataloader import sintellist as ls
        #from dataloader import viperlist as lv
        from dataloader import kitti15list_train as lk15
        from dataloader import sintellist_train as ls
        from dataloader import viperlist_train as lv
        from dataloader import hd1klist as lh
        iml0, iml1, flowl0 = lk12.dataloader('%s/data_stereo_flow/training/'%args.database)
        loader_kitti12 = dr.myImageFloder(iml0,iml1,flowl0, shape=datashape, scale=1, order=0, prob=0.5)
        iml0, iml1, flowl0 = lk15.dataloader('%s/kitti_scene/training/'%args.database)
        loader_kitti15 = dr.myImageFloder(iml0,iml1,flowl0, shape=datashape, scale=1, order=0, prob=0.5)  # KITTI
        iml0, iml1, flowl0 = lh.dataloader('%s/rob_flow/training/'%args.database)
        loader_hd1k = dr.myImageFloder(iml0,iml1,flowl0,shape=datashape, scale=0.5,order=1, noise=0)
        iml0, iml1, flowl0 = ls.dataloader('%s/rob_flow/training/'%args.database)
        loader_sintel = dr.myImageFloder(iml0,iml1,flowl0, shape=datashape, scale=1, order=1, noise=0)
        from dataloader import sceneflowlist as lsf
        iml0, iml1, flowl0, disp0, dispc, calib = lsf.dataloader('%s/Driving/'%args.database, level=6)
        loader_driving = dr.myImageFloder(iml0,iml1,flowl0, shape=datashape, scale=1, order=1)
        iml0, iml1, flowl0, disp0, dispc, calib = lsf.dataloader('%s/Monkaa/'%args.database, level=4)
        loader_monkaa = dr.myImageFloder(iml0,iml1,flowl0, shape=datashape, scale=1, order=1)
        iml0, iml1, flowl0 = lv.dataloader('%s/rob_flow/training/'%args.database)
        loader_viper = dr.myImageFloder(iml0,iml1,flowl0, shape = datashape, scale=1, order=1, scale_aug=[0.8,-0.2])

## aggregate datasets
if 'expansion' in args.stage:
    if '2015' in args.stage:
        data_inuse = torch.utils.data.ConcatDataset([loader_kitti15_sc]*10000)
    else:
        #data_inuse = torch.utils.data.ConcatDataset([loader_driving_sc]*200+[loader_monkaa_sc]*100+[loader_things_sc]*40 + [loader_kitti15_sc]*22000 + [loader_sintel_sc]*2200)
        #data_inuse = torch.utils.data.ConcatDataset([loader_driving_sc]*200+[loader_monkaa_sc]*100+[loader_things_sc]*40 + [loader_gtav_sc]*700)  # no kitti sintel
        data_inuse = torch.utils.data.ConcatDataset([loader_driving_sc]*200+[loader_monkaa_sc]*100+[loader_things_sc]*40)  # no kitti sintel
    for i in data_inuse.datasets:
        i.black = False
        i.cover = True
    baselr = 1e-3
    num_steps = 7e4
elif 'seg' in args.stage:
    if args.stage=='segsf':
        #data_inuse = torch.utils.data.ConcatDataset([loader_driving_sc]*200+[loader_things_sc]*40)
        data_inuse = torch.utils.data.ConcatDataset([loader_driving_sc]*200+[loader_monkaa_sc]*100+[loader_things_sc]*40)
    elif args.stage=='segkitti':
        #data_inuse = torch.utils.data.ConcatDataset([loader_kitti15_sc]*22000)
        #data_inuse = torch.utils.data.ConcatDataset([loader_driving_sc]*200+[loader_things_sc]*40 + [loader_kitti15_sc]*22000)
        data_inuse = torch.utils.data.ConcatDataset([loader_driving_sc]*200+[loader_monkaa_sc]*100+[loader_things_sc]*40 + [loader_kitti15_sc]*22000)
    for i in data_inuse.datasets:
        i.black = False
        i.cover = True
    baselr=5e-4
    num_steps = 7e4
elif args.stage=='chairs':
    data_inuse = torch.utils.data.ConcatDataset([loader_chairs]*100)
    baselr = 1e-3
    num_steps = 7e4
elif args.stage=='things':
    data_inuse = torch.utils.data.ConcatDataset([loader_things]*100)
    baselr = 1e-3
    num_steps = 7e4
elif '2015' in args.stage:
    data_inuse = torch.utils.data.ConcatDataset([loader_kitti15]*50+[loader_kitti12]*50)
    for i in data_inuse.datasets:
        i.black = True
        i.cover = True
elif 'sintel' in args.stage:
    data_inuse = torch.utils.data.ConcatDataset([loader_kitti15]*200*6+[loader_hd1k]*40*6 + [loader_sintel]*150*6 + [loader_chairs]*2*6 + [loader_things]*6)
    for i in data_inuse.datasets:
        i.black = True
        i.cover = True
    baselr = 1e-4
elif args.stage=='rob':
    data_inuse = torch.utils.data.ConcatDataset([loader_kitti12]*2700+[loader_kitti15]*2700 + [loader_sintel]*600 + [loader_chairs]*12 + [loader_things]*6 + [loader_hd1k]*900 + [loader_driving]*50 + [loader_monkaa]*25+[loader_viper]*70)
    #data_inuse = torch.utils.data.ConcatDataset([loader_chairs]*12 + [loader_things]*6 + [loader_driving]*50 + [loader_monkaa]*25+ [loader_viper]*70)  # noks
    for i in data_inuse.datasets:
        i.black = True
        i.cover = True
    baselr = 1e-3
    num_steps = 7e4
else:
    print('error')
    exit(0)

print('Total iterations: %d'%(len(data_inuse)//batch_size))
print('Max iterations: %d'  %(args.niter))

from models.VCNplus  import VCN
model = VCN([batch_size//ngpus]+data_inuse.datasets[0].shape[::-1],
md=[int(4*(args.maxdisp/256)), 4,4,4,4], fac=args.fac, exp_unc= args.loadmodel is None or not ('kitti' in args.loadmodel))

# sync bn and dataparallel
model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)
device = torch.device('cuda:{}'.format(args.local_rank))
model = model.to(device)
model = torch.nn.parallel.DistributedDataParallel(
    model,
    device_ids=[args.local_rank],
    output_device=args.local_rank,
    find_unused_parameters=True
)

total_iters = 0
mean_L=[[0.33,0.33,0.33]]
mean_R=[[0.33,0.33,0.33]]
if args.loadmodel is not None:
    pretrained_dict = torch.load(args.loadmodel,map_location='cpu')
    pretrained_dict['state_dict'] =  {k:v for k,v in pretrained_dict['state_dict'].items()}
    #pretrained_dict['state_dict'] =  {k:v for k,v in pretrained_dict['state_dict'].items() if 'fgnet' not in k and 'det' not in k}
    model.load_state_dict(pretrained_dict['state_dict'],strict=False)

    if args.retrain == 'true':
        print('re-training')
        if 'expansion' in args.stage or 'depth' in args.stage or 'seg' in args.stage:
            print('resuming mean from %d'%total_iters)
            mean_L=pretrained_dict['mean_L']
            mean_R=pretrained_dict['mean_R']
    else:
        with open('%s/iter_counts-%d.txt'%(args.itersave, int(args.logname.split('-')[-1])), 'r') as f:
            total_iters = int(f.readline())
        print('resuming from %d'%total_iters)
        mean_L=pretrained_dict['mean_L']
        mean_R=pretrained_dict['mean_R']

if args.loadflow is not None:
    pretrained_dict = torch.load(args.loadflow,map_location='cpu')
    pretrained_dict['state_dict'] =  {k:v for k,v in pretrained_dict['state_dict'].items() if 'f_modules' in k or 'p_modules' in k or 'oor_modules' in k or 'fuse_modules' in k}
    model.load_state_dict(pretrained_dict['state_dict'],strict=False)

mix_precision = False
print('Number of model parameters: {}'.format(sum([p.data.nelement() for p in model.parameters()])))
optimizer = optim.AdamW(model.parameters(), lr=baselr, weight_decay=0.0001, eps=1e-8)
scheduler = optim.lr_scheduler.OneCycleLR(optimizer, baselr, int(num_steps+100),
        pct_start=0.05, cycle_momentum=False, anneal_strategy='linear')
if args.local_rank==0: log = SummaryWriter('%s/%s'%(args.savemodel,args.logname), comment = args.logname)
scaler = GradScaler(enabled=mix_precision)

def train(imgL,imgR,flowl0,imgAux,intr, imgoL, imgoR, occp, RT01):
        model.train()
        imgL   = Variable(torch.FloatTensor(imgL))
        imgR   = Variable(torch.FloatTensor(imgR))
        flowl0 = Variable(torch.FloatTensor(flowl0))

        imgL, imgR, flowl0 = imgL.cuda(device), imgR.cuda(device), flowl0.cuda(device)
        # mask: valid flow GT & within pre-defined range
        mask = (flowl0[:,:,:,2] == 1) & (flowl0[:,:,:,0].abs() < args.maxdisp) & (flowl0[:,:,:,1].abs() < (args.maxdisp//args.fac))
        if not imgAux is None:
            imgAux = imgAux.cuda(device)
            imgoL, imgoR = imgoL.float().cuda(device), imgoR.float().cuda(device)
            # mask: + 0.01<depth<100, imgAux: depth, d1,d2,d2,flow3d
            mask = mask & (imgAux[:,:,:,0] < 100) & (imgAux[:,:,:,0] > 0.01)
            exp_flag = True
        else:
            exp_flag = False
        if 'expansion' in args.stage:
            exp_flag = 1 # expanson
        elif 'seg' in args.stage:
            exp_flag = 2 # segmentation
        else:
            exp_flag = 0 # flow
        mask.detach_();

        # rearrange inputs
        groups = []
        for i in range(ngpus):
            groups.append(imgL[i*batch_size//ngpus:(i+1)*batch_size//ngpus])
            groups.append(imgR[i*batch_size//ngpus:(i+1)*batch_size//ngpus])

        # forward-backward
        optimizer.zero_grad()
        disp_input = None
        #disp_input = 1./torch.clamp(imgAux[:,:,:,0],1,100)[:,np.newaxis]
        with autocast(enabled=mix_precision):
            output = model(torch.cat(groups,0), [flowl0,mask,imgAux,intr, imgoL, imgoR, occp, RT01, exp_flag],disp_input=disp_input)
            loss = output[-3].mean()

        scaler.scale(loss).backward()
        scaler.unscale_(optimizer)
        grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), 100.)
        scaler.step(optimizer)
        scheduler.step()
#        for param_group in optimizer.param_groups:
#            print(param_group['lr'])
        scaler.update()

#        loss.backward()
#        #torch.nn.utils.clip_grad_norm_(model.parameters(), 100.0)
#        optimizer.step()

        # for debugging
        if np.isnan(np.asarray(model.module.dc2_conv7.weight.max().detach().cpu())):
            pdb.set_trace()
            pass

        vis = {}
        vis['output2'] = output[0].detach().cpu().numpy()
        vis['output3'] = output[1].detach().cpu().numpy()
        vis['output4'] = output[2].detach().cpu().numpy()
        vis['output5'] = output[3].detach().cpu().numpy()
        vis['output6'] = output[4].detach().cpu().numpy()
        if 'expansion' in args.stage:
            vis['mid'] = output[6][0].detach().cpu().numpy()
            vis['exp'] = output[7][0].detach().cpu().numpy()
        elif 'seg' in args.stage:
            vis['fg'] = output[6][0].detach().cpu().numpy()
            vis['fg_gt'] = output[7][0].detach().cpu().numpy()
        vis['gt'] = flowl0[:,:,:,:].detach().cpu().numpy()
        if mask.sum():
            vis['AEPE'] = realEPE(output[0].detach(), flowl0.permute(0,3,1,2).detach(),mask,sparse=False)
        vis['mask'] = mask
        vis['grad_norm'] = grad_norm
        return loss.data,vis

# get global counts
with open('%s/iter_counts-%d.txt'%(args.itersave, int(args.logname.split('-')[-1])), 'w') as f:
    f.write('%d'%total_iters)

def main():
    sampler = torch.utils.data.distributed.DistributedSampler(
        data_inuse,
        num_replicas=args.nproc,
        rank=args.local_rank,
    )
    TrainImgLoader = torch.utils.data.DataLoader(
         data_inuse,
         batch_size= batch_size, num_workers=int(worker_mul*batch_size), drop_last=True, worker_init_fn=_init_fn, pin_memory=True,sampler=sampler)
    start_full_time = time.time()
    global total_iters

    # training loop
    for batch_idx, databatch in enumerate(TrainImgLoader):
        if 'expansion' in args.stage or 'seg' in args.stage:
            imgL_crop, imgR_crop, flowl0,imgAux,intr, imgoL, imgoR, occp, RT01  = databatch
            intr = [t.float() for t in intr]
        else:
            imgL_crop, imgR_crop, flowl0 = databatch
            imgAux,intr, imgoL, imgoR, occp, RT01 = None,None,None,None,None,None
        if total_iters < 1000 and not ('expansion' in args.stage or 'seg' in args.stage):
            # subtract mean
            mean_L.append( np.asarray(imgL_crop.mean(0).mean(1).mean(1)) )
            mean_R.append( np.asarray(imgR_crop.mean(0).mean(1).mean(1)) )
        imgL_crop -= torch.from_numpy(np.asarray(mean_L).mean(0)[np.newaxis,:,np.newaxis, np.newaxis]).float()
        imgR_crop -= torch.from_numpy(np.asarray(mean_R).mean(0)[np.newaxis,:,np.newaxis, np.newaxis]).float()

        start_time = time.time()
        loss,vis = train(imgL_crop,imgR_crop, flowl0, imgAux,intr, imgoL, imgoR, occp, RT01)

        if args.local_rank==0:
            print('Iter %d training loss = %.3f , time = %.2f' %(batch_idx, loss, time.time() - start_time))
            if total_iters %10 == 0:
                log.add_scalar('train/loss_batch',loss, total_iters)
                log.add_scalar('train/aepe_batch',vis['AEPE'], total_iters)
                log.add_scalar('train/grad_norm',vis['grad_norm'], total_iters)
            if total_iters %100 == 0:
                #torch.cuda.empty_cache()
                add_image(log,'train/left', imgL_crop[0:1],total_iters)
                add_image(log,'train/right',imgR_crop[0:1],total_iters)
                if len(np.asarray(vis['gt']))>0:
                    log.add_histogram('train/gt_hist',np.asarray(vis['gt']).reshape(-1,3)[np.asarray(vis['gt'])[:,:,:,-1].flatten().astype(bool)][:,:2], total_iters)
                gu = vis['gt'][0,:,:,0]; gv = vis['gt'][0,:,:,1]
                gu = gu*np.asarray(vis['mask'][0].float().cpu());  gv = gv*np.asarray(vis['mask'][0].float().cpu())
                mask = vis['mask'][0].float().cpu()
                add_image(log,'train/gt0', flow_to_image(np.concatenate((gu[:,:,np.newaxis],gv[:,:,np.newaxis],mask[:,:,np.newaxis]),-1))[np.newaxis],total_iters)
                add_image(log,'train/output2',flow_to_image(vis['output2'][0].transpose((1,2,0)))[np.newaxis],total_iters)
                add_image(log,'train/output3',flow_to_image(vis['output3'][0].transpose((1,2,0)))[np.newaxis],total_iters)
                add_image(log,'train/output4',flow_to_image(vis['output4'][0].transpose((1,2,0)))[np.newaxis],total_iters)
                add_image(log,'train/output5',flow_to_image(vis['output5'][0].transpose((1,2,0)))[np.newaxis],total_iters)
                add_image(log,'train/output6',flow_to_image(vis['output6'][0].transpose((1,2,0)))[np.newaxis],total_iters)
                if 'expansion' in args.stage:
                    add_image(log,'train/mid_gt',(1+imgAux[:1,:,:,6]/imgAux[:1,:,:,0]).log() ,total_iters)
                    add_image(log,'train/mid',vis['mid'][np.newaxis],total_iters)
                    add_image(log,'train/exp',vis['exp'][np.newaxis],total_iters)
                if 'seg' in args.stage:
                    add_image(log,'train/fg_gt',vis['fg_gt'][np.newaxis],total_iters)
                    add_image(log,'train/fg_pred',vis['fg'][np.newaxis],total_iters)
        total_iters += 1
        # get global counts
        with open('%s/iter_counts-%d.txt'%(args.itersave,int(args.logname.split('-')[-1])), 'w') as f:
            f.write('%d'%total_iters)
#        torch.cuda.empty_cache()

        if (total_iters + 1)%2000==0:
            if args.local_rank==0:
                #SAVE
                savefilename = args.savemodel+'/'+args.logname+'/finetune_'+str(total_iters)+'.pth'
                save_dict = model.state_dict()
                save_dict = collections.OrderedDict({k:v for k,v in save_dict.items() if ('reg_modules' not in k or 'conv1' in k) and ('grid' not in k) and ('flow_reg' not in k) and ('midas' not in k)  })
                torch.save({
                    'iters': total_iters,
                    'state_dict': save_dict,
                    'mean_L': mean_L,
                    'mean_R': mean_R,
                }, savefilename)

    print('full finetune time = %.2f HR' %((time.time() - start_full_time)/3600))

if __name__ == '__main__':
    main()


'''
    app.py
    Zack Dong, Ali Ramazani, Muno Siyakurima

    A tiny Flask application that provides a website with an accompanying
    API (which is also tiny) to support that website.

'''
import sys
import flask
import api

########### Initializing Flask ###########
# Note that this stuff has to be up here at the top, because otherwise
# the @app.route lines would raise a "name not defined" exception.
app = flask.Flask(__name__, static_folder='static', template_folder='templates')
app.register_blueprint(api.api, url_prefix='/api')

########### The website routes ###########
@app.route('/')
def get_main_page():
    ''' This is the only route intended for human users '''
    return flask.render_template('index.html')

@app.route('/mockup1.html')
def get_mockup1():
    ''' This is the only route intended for human users '''
    return flask.render_template('mockups/mockup1.html')

@app.route('/mockup2.html')
def get_mockup2():
    ''' This is the only route intended for human users '''
    return flask.render_template('mockups/mockup2.html')

@app.route('/mockup3.html')
def get_mockup3():
    ''' This is the only route intended for human users '''
    return flask.render_template('mockups/mockup3.html')

@app.route('/mockup4.html')
def get_team_page():
    ''' This is the only route intended for human users '''
    return flask.render_template('mockups/mockup4.html')

########### Running the website server ###########
if __name__ == '__main__':
    if len(sys.argv) != 3:
        print('Usage: {0} host port'.format(sys.argv[0]), file=sys.stderr)
        exit()

    host = sys.argv[1]
    port = int(sys.argv[2])
    app.run(host=host, port=port, debug=True)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Torrez, Milton N.

import json
import requests
from bs4 import BeautifulSoup
import datetime
import time
import ast
from random import randint
from time import sleep

base_url = "https://www.ambito.com/contenidos/dolar.html"

headers = {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36"
}

def get_current_month_and_year():
    """Returns the current month and year in the format year-month."""
    now = datetime.datetime.now()
    month = now.month
    year = now.year
    return f"{year}-{month:02d}"

def get_current_date():
    """Returns the current date in the format dd/mm/yy."""
    today = datetime.date.today()
    return f"{today.day}/{today.month}/{today.year % 100}"

def get_current_time():
    """Returns the current time in the format HH:MM:SS."""
    now = datetime.datetime.now()
    return f"{now.hour:02d}:{now.minute:02d}:{now.second:02d}"

def pprint_data(d):
    """Prints nicely a dict."""
    data = json.dumps(d, indent=4, ensure_ascii=False)
    print(data)

def get_coti_data(d):
    c = {}
    c["name"] = d
    api_base_url = f"https://mercados.ambito.com//dolar/{d}/variacion"
    response = requests.get(api_base_url, headers=headers)
    c["cotization"] = response.json()
    return c

def process_data(d):
    data = {}
    #  DEBUG INFO
    data["debug_info"] = {}
    data["debug_info"]["running_date"] = get_current_date()
    data["debug_info"]["running_time"] = get_current_time()
    data["data"] = {}
    data["data"]["coti"] = []

    data_boxes = d.findAll("div", {"class": "indicador"})
    ref_names = []

    for i in data_boxes:
        for j in i["data-indice"].split("/"):
            if j != "dolar" and j != "":
                ref_names.append(j)

    for i in ref_names:
        data["data"]["coti"].append(get_coti_data(i))
        sleep(randint(1, 10))

    #  Append data to file.
    append_data_to_json_file(data, get_current_month_and_year())

def get_data():
    """Gets page content."""
    response = requests.get(base_url, headers=headers)
    return response.content

def append_data_to_json_file(new_data, filename):
    """Appends data to a JSON file, creating the file if it does not exist.

    Args:
    new_data: The data to append to the file.
    filename: The path to the JSON file.
    """

    try:
        # Open the file in reading and writing mode.
        with open(f"data/ambito/{filename}.json", "r+", encoding="utf-8") as f:
            # Load the existing data from the file.
            existing_data = json.load(f)

            # Append the new data to the existing data.
            existing_data.append(new_data)

            # Seek to the beginning of the file.
            f.seek(0)

            # Write the updated data to the file.
            json.dump(existing_data, f)
    except FileNotFoundError:
        # Create the file if it does not exist.
        with open(f"data/ambito/{filename}.json", "w") as f:
            json.dump([new_data], f)

def ambito_main():
    soup = BeautifulSoup(get_data(), "html.parser")
    process_data(soup)

if __name__ == "__main__":
    ambito_main()


from scipy.spatial import distance

def getRelationDistance(entityData, x=650, y=740):
    location = entityData.info.location
    loc = location.split()
    x1 = float(loc[0])
    y1 = float(loc[1])
    p = (x1, y1)
    xFixed = getDistanceDictionaryX(p[0], x, y)
    yFixed = getDistanceDictionaryY(p[1], x, y)
    print(xFixed)
    return 0

def getDistanceDictionaryX(coord, x, y):
    point1 = (x, y)
    xFixed = {point1: []}
    for i in range(1079):
        point2 = (coord, i)
        d = distance.euclidean(point1, point2)
        xFixed[point1].append([point2, d])
    return xFixed

def getDistanceDictionaryY(coord, x, y):
    point1 = (x, y)
    yFixed = {point1: []}
    for i in range(1079):
        point2 = (i, coord)
        d = distance.euclidean(point1, point2)
        yFixed[point1].append([point2, d])
    return yFixed


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jul  1 15:11:31 2019

@author: quanghuy
"""
import numpy as np
import imageio
import matplotlib.pyplot as plt
np.random.seed(1)

# filename structure
path = '/Users/quanghuy/Documents/Massp/Class/20190701/YALE/centered/' # path to the database
ids = range(1, 16) # 15 persons
states = ['centerlight', 'glasses', 'happy', 'leftlight',
          'noglasses', 'normal', 'rightlight','sad',
          'sleepy', 'surprised', 'wink' ]
prefix = 'subject'
surfix = '.pgm'

# data dimension
h = 231 # hight
w = 195 # width
D = h * w
N = len(states)*15
K = 100

# collect all data
X = np.zeros((D, N))
cnt = 0
for person_id in range(1, 15):
    for state in states:
        fn = path + prefix + str(person_id).zfill(2) + '.' + state + surfix
        X[:, cnt] = imageio.imread(fn).reshape(D)
        cnt += 1

# Doing PCA, note that each row is a datapoint
from sklearn.decomposition import PCA
pca = PCA(n_components=K) # K = 100
pca.fit(X.T) #training data

# projection matrix
U = pca.components_.T

for i in range(U.shape[1]): #show each of Xi
    plt.axis('off')
    f1 = plt.imshow(U[:, i].reshape(h,w), interpolation='nearest')
    # interpolation == 'nearest' means pixel color = nearest coordinates
    f1.axes.get_xaxis().set_visible(False)
    f1.axes.get_yaxis().set_visible(False)
#     f2 = plt.imshow(, interpolation='nearest' )
    plt.gray()
    fn = path + 'store/eigenface' + str(i).zfill(2) + '.png'
    plt.savefig(fn, bbox_inches='tight', pad_inches=0)
#    plt.show()

# See reconstruction of first 6 persons

for person_id in [2]:
    for state in ['centerlight']:
        fn = path + prefix + str(person_id).zfill(2) + '.' + state + surfix
        im = imageio.imread(fn)
        plt.axis('off')
#       plt.imshow(im, interpolation='nearest' )
        f1 = plt.imshow(im, interpolation='nearest')
        f1.axes.get_xaxis().set_visible(False)
        f1.axes.get_yaxis().set_visible(False)
        plt.gray()
        fn = path + 'store/ori' + str(person_id).zfill(2) + '.png'
        # show the the original image
        plt.savefig(fn, bbox_inches='tight', pad_inches=0)
        plt.show()
        # reshape and subtract mean, don't forget
        x = im.reshape(D, 1) - pca.mean_.reshape(D, 1)
        # x is the X-Xo = X1*z1+X2*z2...Xk*zk
        # encode
        z = U.T.dot(x)
        # z is the coordinates matrix
        #decode
        error = []
        for index in [1,10,20,50,100]:
            x_tilde = U[:,:index].dot(z[:index,:]) + pca.mean_.reshape(D, 1)
            # x_tilde = Xo + X1*z1 + ... X_index*z_index
    # reshape to orginal dim
            im_tilde = x_tilde.reshape(h,w)
            plt.axis('off')
    #       plt.imshow(im_tilde, interpolation='nearest' )
            f1 = plt.imshow(im_tilde, interpolation='nearest')
            f1.axes.get_xaxis().set_visible(False)
            f1.axes.get_yaxis().set_visible(False)
            plt.gray()
            fn = path + 'store/res' + str(person_id).zfill(2) + '.png'
            plt.savefig(fn, bbox_inches='tight', pad_inches=0)
            # show the im_tilde
            plt.show()
            loss = 0.0
            for i in range(D):
                loss = loss + float((x_tilde[i]-x[i]))**2
            print(index)
            print(loss**0.5)
            error.append(loss**0.5)
        print(error)
        plt.plot([1,10,20,50,100],error)


import random
from SkillService import SkillService
from Model import Skill
import simplejson as json

def getImages(event, context):
    skillService = SkillService()
    try:
        primaryCategory = json.loads(event.get("body")).get("primary_category")
        secondary_category = json.loads(event.get("body")).get("secondary_category")
        tertiary_category = json.loads(event.get("body")).get("tertiary_category")

        # images = None
        Skill.validateImageTags(primaryCategory, secondary_category, skillService.getImageTags())

        if primaryCategory and secondary_category and tertiary_category:
            # primaryCategory, secondary_category, tertiary_category
            images = skillService.getMatchingImages(primaryCategory, secondary_category, tertiary_category)
            random.shuffle(images)

        elif primaryCategory and secondary_category:
            # primaryCategory, secondary_category
            images = skillService.getMatchingImages(primaryCategory, secondary_category, None)
            random.shuffle(images)

        else:
            # mystery (nothing else)
            tags = skillService.getImageTags()
            random.shuffle(tags)
            images = skillService.getMatchingImages(tags[0], tags[1], None)
            random.shuffle(images)

        response = {
            'statusCode': 200,
            'body': json.dumps(images, use_decimal=True)
        }
        return response

    except NameError as tagNotFoundException:
        response = {
            "errorType": "Not Found Error",
            "errorMessage": str(tagNotFoundException)
        }
        statusCode = 400
        return {
            'statusCode': statusCode,
            'body': json.dumps(response)
        }

def getMysteryImage(events, context):
    skillService = SkillService()
    tags = skillService.getImageTags()

    random.shuffle(tags.get('primary_categories'))
    random.shuffle(tags.get('secondary_categories'))
    images = skillService.getMatchingImages(tags.get('primary_categories')[0], tags.get('secondary_categories')[0], None)
    random.shuffle(images)

    response = {
        'statusCode': 200,
        'body': json.dumps(images, use_decimal=True)
    }
    return response


import os
from flask import Flask, render_template, request, jsonify

from modules import auth, compute, db_manager

app = Flask(__name__)

app.secret_key = os.environ['SESSION_SECRET_KEY']
auth.google_signin_key = os.environ['GOOGLE_SIGNIN_KEY']

@app.route('/')
def root():
    return render_template('index.html', gsignin_key=auth.google_signin_key)

@app.route('/get-user-data', methods=['GET'])
def get_user_data():
    user_info = auth.authenticate(request)
    if user_info:
        return render_template('partials/user_info.html', user_info=user_info)
    else:
        return jsonify({'message': 'error'}), 401

@app.route('/sign-out', methods=['GET'])
def sign_out():
    sign_out_request = auth.sing_out()

    if sign_out_request:
        return jsonify({'message': 'success'})
    else:
        return jsonify({'message': 'error'}), 400

@app.route('/zones', methods=['GET'])
def get_zones():
    user_info = auth.authenticate(request)
    if user_info:
        # zones = compute.list_zones()
        # Uncomment in case I want to check for all the zones
        zones = db_manager.get_available_zones()
        zone_count = len(zones)
        return jsonify({'zones': zones, 'zone_count': zone_count})
    else:
        return jsonify({'message': 'error'}), 401

@app.route('/instances/<zone>/<id>', methods=['GET'])
def get_instance(zone, id):
    user_info = auth.authenticate(request)
    if user_info:
        instance = compute.get_instance(zone, id)

        if instance:
            return render_template(
                'partials/instance.html', instance=instance, zone=zone)
        else:
            return jsonify({'message': 'empty zone'})
    else:
        return jsonify({'message': 'error'}), 401

@app.route('/instances/<zone>', methods=['GET'])
def get_instances(zone):
    user_info = auth.authenticate(request)
    if user_info:
        instances = compute.list_instances(zone)

        if instances:
            return render_template(
                'partials/instances.html', instances=instances, zone=zone)
        else:
            return jsonify({'message': 'empty zone'})
    else:
        return jsonify({'message': 'error'}), 401

@app.route('/instances/<action>', methods=['POST'])
def update_instance(action):
    user_info = auth.authenticate(request)
    if user_info:
        vm = request.form['instance']
        zone = request.form['zone']

        if action == 'start':
            return jsonify(compute.start_vm(zone, vm))
        else:
            return jsonify(compute.stop_vm(zone, vm))

    else:
        return jsonify({'message': 'error'}), 401

@app.route('/operations/<zone>/<operation>', methods=['GET'])
def get_operation(zone, operation):
    user_info = auth.authenticate(request)
    if user_info:
        return jsonify(compute.get_op(zone, operation))
    else:
        return jsonify({'message': 'error'}), 401

if __name__ == '__main__':
    app.run(host='127.0.0.1', port=8080, debug=True)


from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from flask import Blueprint
from flask_restplus import Api
import os

db = SQLAlchemy()

def create_app():
    project_root = os.path.abspath(os.path.dirname(__file__))
    app = Flask("Spending-tracker")
    app.config["SQLALCHEMY_DATABASE_URI"] = f"sqlite:///{os.path.join(project_root, 'spending_tracker.db')}"
    app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
    db.init_app(app)
    return app

blueprint = Blueprint('api', __name__)
api = Api(
    blueprint,
    title='Spending Tracker',
    version='0.1',
    description='Track your spending with this app',
    doc='/api/doc',
    default_mediatype='json'
)


import datetime as dt

class Employee:

    raise_amountGlobal = 15.003 # that is a global variable
    gmail = '@gmail.com'
    num_of_Employee = 0

    def __init__(self,firstName,lastName,salary,gender=None): # None helps to declare a optional parameter, not mandatory
        self.first = firstName  # self represents this class, the instance of this class
        self.last = lastName
        self.pay = salary
        self.email = firstName + '.' + lastName + self.gmail # always use self to access the instances of that file
        self.gen = gender
        Employee.num_of_Employee += 1 # this will help increment the number of employees.

    def apply_raiseLocal(self):
        raise_amountLocal = 10.002
        new_pay = int(self.pay + raise_amountLocal) # raise amount is a local variable. It does not work with self,. raise amount will only work for this function
        return new_pay

    def apply_raiseGlobal(self): # the apply function is a regular method
        new_pay = int(self.pay + self.raise_amountGlobal)
        return new_pay

    @staticmethod
    def is_workDay(day):
        if day.weekday == 5 or day.weekday == 6:
            return False
        else:
            return True

    @classmethod # that is a class method, its different from the other methods
    def set_raiseAmount(cls,amount): # a regular we need an object of that function to call it but a class function we can call it directly
        cls.raise_amountGlobal = amount

# print(f'Number of employees is: {Employee.num_of_Employee}')
emp_1 = Employee('Ishe','Murwira',20000)                  # we created an object for the class and call it an instant of the class. The object we created emp1 is still called the instant of the class
emp_2 = Employee('John','Jamerson',25000,'Male')
emp_3 = Employee ('Susan','Rusike',15000,'Female')
# print(f'Number of employees is: {Employee.num_of_Employee}')

# print(emp_1.apply_raiseGlobal())
# print(emp_1.apply_raiseLocal())

# print(Employee.raise_amountGlobal)
# Employee.set_raiseAmount(50)
# print(Employee.raise_amountGlobal)

myDate = dt.date(2021,6,24)
workDay = Employee.is_workDay(myDate)

print(workDay)


import streamlit as st
from models.lineal_model import LinealModel
from models.polynomial_regression import PolynomialRegression
from models.read_csv_model import ReadCsvModel

import pandas as pd

class SimpleRegressionPage:
    def __init__(self):
        model = st.selectbox("Simple Models", ["Linear Regression", "Polynomial Regression"])
        dataset = st.selectbox("DataSet", ["Salary Vs Experience", "Position Salaries", "Polynomial Example",  "Otro Conjunto de Datos"])

        real_name_dataset = {
            "Salary Vs Experience" : "Salary_Data",
            "Position Salaries"    : "Position_Salaries",
            "Polynomial Example"   : "polynomial-regression",
        }

        if dataset == "Otro Conjunto de Datos":
            dataset = ReadCsvModel( st.file_uploader("Otro Documento"), None)
        dataset = ReadCsvModel(real_name_dataset[dataset], "simple_regression")

        if dataset is not None:
            cols = dataset.columns.tolist()

            col1, col2 = st.columns(2)
            x_col = col1.selectbox('Columnas independientes (X)', cols)
            y_col = col2.selectbox('Columna dependiente (y)', cols)

        if model == "Polynomial Regression":
            grado = st.slider("Grado Polinomico", 0, 10, 5, 1)

        # Models
        if model == "Linear Regression":
            LinealModel(dataset, x_col, y_col)
        elif model == "Polynomial Regression":
            PolynomialRegression(dataset, x_col, y_col, grado)