"""python library for accessing the openHAB REST API."""

#
# Georges Toth (c) 2016-present <georges@trypill.org>
#
# python-openhab is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# python-openhab is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with python-openhab.  If not, see <http://www.gnu.org/licenses/>.
#

# pylint: disable=bad-indentation

import datetime
import logging
import re
import typing

import dateutil.parser

import openhab.command_types
import openhab.exceptions

__author__ = 'Georges Toth <georges@trypill.org>'
__license__ = 'AGPLv3+'

class Item:
  """Base item class."""

  types: typing.Sequence[typing.Type[openhab.command_types.CommandType]] = []
  state_types: typing.Sequence[typing.Type[openhab.command_types.CommandType]] = []
  command_event_types: typing.Sequence[typing.Type[openhab.command_types.CommandType]] = []
  state_event_types: typing.Sequence[typing.Type[openhab.command_types.CommandType]] = []
  state_changed_event_types: typing.Sequence[typing.Type[openhab.command_types.CommandType]] = []

  TYPENAME = 'unknown'

  def __init__(self, openhab_conn: 'openhab.client.OpenHAB', json_data: dict) -> None:
    """Constructor.

    Args:
      openhab_conn (openhab.OpenHAB): openHAB object.
      json_data (dic): A dict converted from the JSON data returned by the openHAB
                       server.
    """
    self.openhab = openhab_conn
    self.type_: typing.Optional[str] = None
    self.quantityType: typing.Optional[str] = None
    self.editable = None
    self.label = ''
    self.category = ''
    self.tags = ''
    self.groupNames = ''
    self.group = False
    self.name = ''
    self._state = None  # type: typing.Optional[typing.Any]
    self._unitOfMeasure = ''
    self._raw_state = None  # type: typing.Optional[typing.Any]  # raw state as returned by the server
    self._members = {}  # type: typing.Dict[str, typing.Any] #  group members (key = item name), for none-group items it's empty
    self.function_name: typing.Optional[str] = None
    self.function_params: typing.Optional[typing.Sequence[str]] = None

    self.logger = logging.getLogger(__name__)

    self.init_from_json(json_data)

  def init_from_json(self, json_data: dict) -> None:
    """Initialize this object from a json configuration as fetched from openHAB.

    Args:
      json_data (dict): A dict converted from the JSON data returned by the openHAB
                        server.
    """
    self.name = json_data['name']
    if json_data['type'] == 'Group':
      self.group = True
      if 'groupType' in json_data:
        self.type_ = json_data['groupType']

      if 'function' in json_data:
        self.function_name = json_data['function']['name']

        if 'params' in json_data['function']:
          self.function_params = json_data['function']['params']

      # init members
      for i in json_data['members']:
        self.members[i['name']] = self.openhab.json_to_item(i)

    else:
      self.type_ = json_data.get('type', None)

      if self.type_ is None:
        raise openhab.exceptions.InvalidReturnException('Item did not return a type attribute.')

      parts = self.type_.split(':')
      if len(parts) == 2:
        self.quantityType = parts[1]

    if 'editable' in json_data:
      self.editable = json_data['editable']
    if 'label' in json_data:
      self.label = json_data['label']
    if 'category' in json_data:
      self.category = json_data['category']
    if 'tags' in json_data:
      self.tags = json_data['tags']
    if 'groupNames' in json_data:
      self.groupNames = json_data['groupNames']

    self._raw_state = json_data['state']

    if self.is_undefined(self._raw_state):
      self._state = None
    else:
      self._state, self._unitOfMeasure = self._parse_rest(self._raw_state)

  @property
  def state(self) -> typing.Any:
    """The state property represents the current state of the item.

    The state is automatically refreshed from openHAB on reading it.
    Updating the value via this property send an update to the event bus.
    """
    json_data = self.openhab.get_item_raw(self.name)
    self.init_from_json(json_data)

    return self._state

  @state.setter
  def state(self, value: typing.Any) -> None:
    self.update(value)

  @property
  def unit_of_measure(self) -> str:
    """Return the unit of measure. Returns an empty string if there is none defined."""
    return self._unitOfMeasure

  @property
  def members(self) -> typing.Dict[str, typing.Any]:
    """If item is a type of Group, it will return all member items for this group.

    For none group item empty dictionary will be returned.

    Returns:
      dict: Returns a dict with item names as key and `Item` class instances as value.

    """
    return self._members

  def _validate_value(self, value: typing.Union[str, typing.Type[openhab.command_types.CommandType]]) -> None:
    """Private method for verifying the new value before modifying the state of the item."""
    if self.type_ == 'String':
      if not isinstance(value, (str, bytes)):
        raise ValueError
    elif self.types:
      validation = False

      for type_ in self.types:
        try:
          type_.validate(value)
        except ValueError:
          pass
        else:
          validation = True

      if not validation:
        raise ValueError(f'Invalid value "{value}"')
    else:
      raise ValueError

  def _parse_rest(self, value: str) -> typing.Tuple[str, str]:
    """Parse a REST result into a native object."""
    return value, ''

  def _rest_format(self, value: str) -> typing.Union[str, bytes]:
    """Format a value before submitting to openHAB."""
    _value = value  # type: typing.Union[str, bytes]

    # Only ascii encoding is supported by default. If non-ascii characters were provided, convert them to bytes.
    try:
      _ = value.encode('ascii')
    except UnicodeError:
      _value = value.encode('utf-8')

    return _value

  def is_undefined(self, value: str) -> bool:
    """Check if value is undefined."""
    for aStateType in self.state_types:
      if not aStateType.is_undefined(value):
        return False

    return True

  def __str__(self) -> str:
    """String representation."""
    state = self._state
    if self._unitOfMeasure and not isinstance(self._state, tuple):
        state = f'{self._state} {self._unitOfMeasure}'
    return f'<{self.type_} - {self.name} : {state}>'

  def _update(self, value: typing.Any) -> None:
    """Updates the state of an item, input validation is expected to be already done.

    Args:
      value (object): The value to update the item with. The type of the value depends
                      on the item type and is checked accordingly.
    """
    # noinspection PyTypeChecker
    self.openhab.req_put(f'/items/{self.name}/state', data=value)

  def update(self, value: typing.Any) -> None:
    """Updates the state of an item.

    Args:
      value (object): The value to update the item with. The type of the value depends
                      on the item type and is checked accordingly.
    """
    self._validate_value(value)

    v = self._rest_format(value)

    self._state = value

    self._update(v)

  def command(self, value: typing.Any) -> None:
    """Sends the given value as command to the event bus.

    Args:
      value (object): The value to send as command to the event bus. The type of the
                      value depends on the item type and is checked accordingly.
    """
    self._validate_value(value)

    v = self._rest_format(value)

    self._state = value

    self.openhab.req_post(f'/items/{self.name}', data=v)

  def update_state_null(self) -> None:
    """Update the state of the item to *NULL*."""
    self._update('NULL')

  def update_state_undef(self) -> None:
    """Update the state of the item to *UNDEF*."""
    self._update('UNDEF')

  def is_state_null(self) -> bool:
    """If the item state is None, use this method for checking if the remote value is NULL."""
    if self.state is None:
      # we need to query the current remote state as else this method will not work correctly if called after
      # either update_state method
      if self._raw_state is None:
        # This should never happen
        raise ValueError('Invalid internal (raw) state.')

      return self._raw_state == 'NULL'

    return False

  def is_state_undef(self) -> bool:
    """If the item state is None, use this method for checking if the remote value is UNDEF."""
    if self.state is None:
      # we need to query the current remote state as else this method will not work correctly if called after
      # either update_state method
      if self._raw_state is None:
        # This should never happen
        raise ValueError('Invalid internal (raw) state.')

      return self._raw_state == 'UNDEF'

    return False

class GroupItem(Item):
  """String item type."""

  TYPENAME = 'Group'
  types: typing.List[typing.Type[openhab.command_types.CommandType]] = []
  state_types: typing.List[typing.Type[openhab.command_types.CommandType]] = []

class StringItem(Item):
  """String item type."""

  TYPENAME = 'String'
  types = [openhab.command_types.StringType]
  state_types = types

class DateTimeItem(Item):
  """DateTime item type."""

  TYPENAME = 'DateTime'
  types = [openhab.command_types.DateTimeType]
  state_types = types

  def __gt__(self, other: datetime.datetime) -> bool:
    """Greater than comparison."""
    if self._state is None or not isinstance(other, datetime.datetime):
      raise NotImplementedError('You can only compare two DateTimeItem objects.')

    return self._state > other

  def __ge__(self, other: datetime.datetime) -> bool:
    """Greater or equal comparison."""
    if self._state is None or not isinstance(other, datetime.datetime):
      raise NotImplementedError('You can only compare two DateTimeItem objects.')

    return self._state >= other

  def __lt__(self, other: object) -> bool:
    """Less than comparison."""
    if not isinstance(other, datetime.datetime):
      raise NotImplementedError('You can only compare two DateTimeItem objects.')

    return not self.__gt__(other)

  def __le__(self, other: object) -> bool:
    """Less or equal comparison."""
    if self._state is None or not isinstance(other, datetime.datetime):
      raise NotImplementedError('You can only compare two DateTimeItem objects.')

    return self._state <= other

  def __eq__(self, other: object) -> bool:
    """Equality comparison."""
    if not isinstance(other, datetime.datetime):
      raise NotImplementedError('You can only compare two DateTimeItem objects.')

    return self._state == other

  def __ne__(self, other: object) -> bool:
    """Not equal comparison."""
    if not isinstance(other, datetime.datetime):
      raise NotImplementedError('You can only compare two DateTimeItem objects.')

    return not self.__eq__(other)

  def _parse_rest(self, value: str) -> typing.Tuple[datetime.datetime, str]:  # type: ignore[override]
    """Parse a REST result into a native object.

    Args:
      value (str): A string argument to be converted into a datetime.datetime object.

    Returns:
      datetime.datetime: The datetime.datetime object as converted from the string
                         parameter.
    """
    return dateutil.parser.parse(value), ''

  def _rest_format(self, value: datetime.datetime) -> str:  # type: ignore[override]
    """Format a value before submitting to openHAB.

    Args:
      value (datetime.datetime): A datetime.datetime argument to be converted
                                 into a string.

    Returns:
      str: The string as converted from the datetime.datetime parameter.
    """
    # openHAB supports only up to milliseconds as of this writing
    return value.isoformat(timespec='milliseconds')

class PlayerItem(Item):
  """PlayerItem item type."""

  TYPENAME = 'Player'
  types = [openhab.command_types.PlayPauseType, openhab.command_types.NextPrevious, openhab.command_types.RewindFastforward]
  state_types = [openhab.command_types.PlayPauseType, openhab.command_types.RewindFastforward]

  def play(self) -> None:
    """Send the command PLAY."""
    self.command(openhab.command_types.PlayPauseType.PLAY)

  def pause(self) -> None:
    """Send the command PAUSE."""
    self.command(openhab.command_types.PlayPauseType.PAUSE)

  def next(self) -> None:
    """Send the command NEXT."""
    self.command(openhab.command_types.NextPrevious.NEXT)

  def previous(self) -> None:
    """Send the command PREVIOUS."""
    self.command(openhab.command_types.NextPrevious.PREVIOUS)

  def fastforward(self) -> None:
    """Send the command FASTFORWARD."""
    self.command(openhab.command_types.RewindFastforward.FASTFORWARD)

  def rewind(self) -> None:
    """Send the command REWIND."""
    self.command(openhab.command_types.RewindFastforward.REWIND)

class SwitchItem(Item):
  """SwitchItem item type."""

  TYPENAME = 'Switch'
  types = [openhab.command_types.OnOffType]
  state_types = types

  def on(self) -> None:
    """Set the state of the switch to ON."""
    self.command(openhab.command_types.OnOffType.ON)

  def off(self) -> None:
    """Set the state of the switch to OFF."""
    self.command(openhab.command_types.OnOffType.OFF)

  def toggle(self) -> None:
    """Toggle the state of the switch to OFF to ON and vice versa."""
    if self.state == openhab.command_types.OnOffType.ON:
      self.off()
    elif self.state == openhab.command_types.OnOffType.OFF:
      self.on()

class NumberItem(Item):
  """NumberItem item type."""

  TYPENAME = 'Number'
  types = [openhab.command_types.DecimalType]
  state_types = types

  def _parse_rest(self, value: str) -> typing.Tuple[typing.Union[float, None], str]:  # type: ignore[override]
    """Parse a REST result into a native object.

    Args:
      value (str): A string argument to be converted into a float object.

    Returns:
      float: The float object as converted from the string parameter.
      str: The unit Of Measure or empty string
    """
    if value in ('UNDEF', 'NULL'):
      return None, ''
    # m = re.match(r'''^(-?[0-9.]+)''', value)
    try:
      m = re.match(r'(-?[0-9.]+(?:[eE]-?[0-9]+)?)\s?(.*)?$', value)

      if m:
        value = m.group(1)
        unit_of_measure = m.group(2)

        return float(value), unit_of_measure

      return float(value), ''

    except (ArithmeticError, ValueError) as exc:
      self.logger.error('Error in parsing new value "%s" for "%s" - "%s"', value, self.name, exc)

    raise ValueError(f'{self.__class__}: unable to parse value "{value}"')

  def _rest_format(self, value: typing.Union[float, typing.Tuple[float, str], str]) -> typing.Union[str, bytes]:
    """Format a value before submitting to openHAB.

    Args:
      value: Either a float, a tuple of (float, str), or string; in the first two cases we have to cast it to a string.

    Returns:
      str or bytes: A string or bytes as converted from the value parameter.
    """
    if isinstance(value, tuple) and len(value) == 2:
        return super()._rest_format(f'{value[0]:G} {value[1]}')
    if not isinstance(value, str):
        return super()._rest_format(f'{value:G}')
    return super()._rest_format(value)

class ContactItem(Item):
  """Contact item type."""

  TYPENAME = 'Contact'
  types = [openhab.command_types.OpenCloseType]
  state_types = types

  def command(self, *args: typing.Any, **kwargs: typing.Any) -> None:
    """This overrides the `Item` command method.

    Note: Commands are not accepted for items of type contact.
    """
    raise ValueError(f'This item ({self.__class__}) only supports updates, not commands!')

  def open(self) -> None:
    """Set the state of the contact item to OPEN."""
    self.state = openhab.command_types.OpenCloseType.OPEN

  def closed(self) -> None:
    """Set the state of the contact item to CLOSED."""
    self.state = openhab.command_types.OpenCloseType.CLOSED

class DimmerItem(Item):
  """DimmerItem item type."""

  TYPENAME = 'Dimmer'
  types = [openhab.command_types.OnOffType, openhab.command_types.PercentType, openhab.command_types.IncreaseDecreaseType]
  state_types = [openhab.command_types.PercentType]

  def _parse_rest(self, value: str) -> typing.Tuple[float, str]:  # type: ignore[override]
    """Parse a REST result into a native object.

    Args:
      value (str): A string argument to be converted into a int object.

    Returns:
      float: The int object as converted from the string parameter.
      str: Possible UoM
    """
    return float(value), ''

  def _rest_format(self, value: typing.Union[str, int]) -> str:
    """Format a value before submitting to OpenHAB.

    Args:
      value: Either a string or an integer; in the latter case we have to cast it to a string.

    Returns:
      str: The string as possibly converted from the parameter.
    """
    if not isinstance(value, str):
      return str(value)

    return value

  def on(self) -> None:
    """Set the state of the dimmer to ON."""
    self.command(openhab.command_types.OnOffType.ON)

  def off(self) -> None:
    """Set the state of the dimmer to OFF."""
    self.command(openhab.command_types.OnOffType.OFF)

  def increase(self) -> None:
    """Increase the state of the dimmer."""
    self.command(openhab.command_types.IncreaseDecreaseType.INCREASE)

  def decrease(self) -> None:
    """Decrease the state of the dimmer."""
    self.command(openhab.command_types.IncreaseDecreaseType.DECREASE)

class ColorItem(DimmerItem):
  """ColorItem item type."""

  TYPENAME = 'Color'
  types = [openhab.command_types.OnOffType, openhab.command_types.PercentType, openhab.command_types.IncreaseDecreaseType, openhab.command_types.ColorType]
  state_types = [openhab.command_types.ColorType]

  def _parse_rest(self, value: str) -> typing.Tuple[typing.Optional[typing.Tuple[float, float, float]], str]:  # type: ignore[override]
    """Parse a REST result into a native object.

    Args:
      value (str): A string argument to be converted into a str object.

    Returns:
      HSB components
      Optional UoM
    """
    result = openhab.command_types.ColorType.parse(value)
    return result, ''

  def _rest_format(self, value: typing.Union[typing.Tuple[int, int, float], str, int]) -> str:
    """Format a value before submitting to openHAB.

    Args:
      value: Either a string, an integer or a tuple of HSB components (int, int, float); in the latter two cases we have to cast it to a string.

    Returns:
      str: The string as possibly converted from the parameter.
    """
    if isinstance(value, tuple):
      if len(value) == 3:
        return f'{value[0]},{value[1]},{value[2]}'

    if not isinstance(value, str):
      return str(value)

    return value

class RollershutterItem(Item):
  """RollershutterItem item type."""

  TYPENAME = 'Rollershutter'
  types = [openhab.command_types.UpDownType, openhab.command_types.PercentType, openhab.command_types.StopMoveType]
  state_types = [openhab.command_types.PercentType]

  def _parse_rest(self, value: str) -> typing.Tuple[int, str]:  # type: ignore[override]
    """Parse a REST result into a native object.

    Args:
      value (str): A string argument to be converted into a int object.

    Returns:
      int: The int object as converted from the string parameter.
      str: Possible UoM
    """
    return int(float(value)), ''

  def _rest_format(self, value: typing.Union[str, int]) -> str:
    """Format a value before submitting to openHAB.

    Args:
      value: Either a string or an integer; in the latter case we have to cast it to a string.

    Returns:
      str: The string as possibly converted from the parameter.
    """
    if not isinstance(value, str):
      return str(value)

    return value

  def up(self) -> None:
    """Set the state of the dimmer to ON."""
    self.command(openhab.command_types.UpDownType.UP)

  def down(self) -> None:
    """Set the state of the dimmer to OFF."""
    self.command(openhab.command_types.UpDownType.DOWN)

  def stop(self) -> None:
    """Set the state of the dimmer to OFF."""
    self.command(openhab.command_types.StopMoveType.STOP)


"""Sphinx documentation configuration file."""
from datetime import datetime
import os
from pathlib import Path
import re
import shutil

from ansys_sphinx_theme import ansys_favicon, get_version_match, pyansys_logo_black
import jupytext

from ansys.grantami.recordlists import __version__

# Project information
project = "ansys-grantami-recordlists"
project_copyright = f"(c) {datetime.now().year} ANSYS, Inc. All rights reserved"
author = "ANSYS, Inc."
release = version = __version__

# Select desired logo, theme, and declare the html title
html_logo = pyansys_logo_black
html_theme = "ansys_sphinx_theme"
html_short_title = html_title = "PyGranta RecordLists"
html_favicon = ansys_favicon

cname = os.getenv("DOCUMENTATION_CNAME", "recordlists.grantami.docs.pyansys.com")
"""The canonical name of the webpage hosting the documentation."""

# specify the location of your github repo
html_theme_options = {
    "github_url": "https://github.com/ansys/grantami-recordlists",
    "show_prev_next": False,
    "show_breadcrumbs": True,
    "additional_breadcrumbs": [
        ("PyAnsys", "https://docs.pyansys.com/"),
        ("PyGranta", "https://grantami.docs.pyansys.com/"),
    ],
    "switcher": {
        "json_url": f"https://{cname}/versions.json",
        "version_match": get_version_match(__version__),
    },
    "check_switcher": False,
}

# Sphinx extensions
extensions = [
    "numpydoc",
    "sphinx.ext.intersphinx",
    "sphinx_copybutton",
    "sphinx_jinja",
    "nbsphinx",
]

# sphinx
add_module_names = False

# sphinx.ext.autodoc
autodoc_typehints = "description"  # Remove typehints from signatures in docs
autodoc_typehints_description_target = "documented"
autodoc_member_order = "bysource"

# Intersphinx mapping
intersphinx_mapping = {
    "python": ("https://docs.python.org/3", None),
    "openapi-common": ("https://openapi.docs.pyansys.com", None),
    "requests": ("https://requests.readthedocs.io/en/latest/", None),
    # kept here as an example
    # "scipy": ("https://docs.scipy.org/doc/scipy/reference", None),
    # "numpy": ("https://numpy.org/devdocs", None),
    # "matplotlib": ("https://matplotlib.org/stable", None),
    # "pandas": ("https://pandas.pydata.org/pandas-docs/stable", None),
    # "pyvista": ("https://docs.pyvista.org/", None),
    # "grpc": ("https://grpc.github.io/grpc/python/", None),
}

# numpydoc configuration
numpydoc_show_class_members = False
numpydoc_xref_param_type = True
numpydoc_xref_ignore = {"of", "optional", "or"}

# Consider enabling numpydoc validation. See:
# https://numpydoc.readthedocs.io/en/latest/validation.html#
numpydoc_validate = True
numpydoc_validation_checks = {
    "GL06",  # Found unknown section
    "GL07",  # Sections are in the wrong order.
    "GL08",  # The object does not have a docstring
    "GL09",  # Deprecation warning should precede extended summary
    "GL10",  # reST directives {directives} must be followed by two colons
    "SS01",  # No summary found
    "SS02",  # Summary does not start with a capital letter
    # "SS03", # Summary does not end with a period
    "SS04",  # Summary contains heading whitespaces
    # "SS05", # Summary must start with infinitive verb, not third person
    "RT02",  # The first line of the Returns section should contain only the
    # type, unless multiple values are being returned"
}

# static path
html_static_path = ["_static"]

# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]

# The suffix(es) of source filenames.
source_suffix = ".rst"

# The master toctree document.
master_doc = "index"

# Generate section labels up to four levels deep
autosectionlabel_maxdepth = 4

# -- Examples configuration --------------------------------------------------
def _copy_examples_and_convert_to_notebooks(source_dir, output_dir, ignored_files_regex=None):
    """
    Recursively copies all files from the source directory to the output directory.

    Creates any necessary subfolders in the process. Python scripts with the ".py" extension
    are also converted to Jupyter notebooks with the ".ipynb" extension using Jupytext.

    Parameters
    ----------
    source_dir : Path
        The source directory to copy files from.
    output_dir : Path
        The output directory to copy files to and convert Python scripts.
    ignored_files_regex : str, optional
        A regular expression pattern to match ignored files. Files whose names match
        the pattern will be skipped. If None (default), no files are ignored.

    Raises
    ------
    RuntimeError
        If the output directory or any necessary subfolders cannot be created.

    Notes
    -----
    - This function uses pathlib.Path methods for working with file paths.
    - Any existing files in the output directory will be overwritten.
    - If a Python script cannot be converted to a Jupyter notebook, an exception is raised.

    Examples
    --------
    >>> _copy_examples_and_convert_to_notebooks("my_project/examples", "docs/notebooks", ignored_files_regex=r"^test.*")

    """
    if not source_dir.exists():
        raise ValueError(f"Source directory {source_dir.name} does not exist.")
    if not output_dir.exists():
        output_dir.mkdir(parents=True)

    ignored_files_re = re.compile(ignored_files_regex) if ignored_files_regex else None

    for file_source_path in source_dir.rglob("*"):
        if not file_source_path.is_file():
            continue

        matches_regex = ignored_files_re and ignored_files_re.match(file_source_path.name)
        is_rst_file = file_source_path.name.endswith(".rst")

        if matches_regex and not is_rst_file:
            print(f"Ignoring {file_source_path.name}")
            exclude_patterns.append(str(file_source_path.relative_to(source_dir)))
            continue

        rel_path = file_source_path.relative_to(source_dir)
        file_output_path = output_dir / rel_path

        file_output_path.parent.mkdir(parents=True, exist_ok=True)
        print(f"Copying {file_source_path.name}")
        shutil.copy(file_source_path, file_output_path)

        if file_source_path.suffix == ".py":
            try:
                ntbk = jupytext.read(file_source_path)
                jupytext.write(ntbk, file_output_path.with_suffix(".ipynb"))
            except Exception as e:
                raise RuntimeError(f"Failed to convert {file_source_path} to notebook: {e}")

exclude_patterns = []

EXAMPLES_SOURCE_DIR = Path(__file__).parent.parent.parent.absolute() / "examples"
EXAMPLES_OUTPUT_DIR = Path(__file__).parent.absolute() / "examples"
BUILD_EXAMPLES = True if os.environ.get("BUILD_EXAMPLES", "false") == "true" else False
if BUILD_EXAMPLES:
    ipython_dir = Path("../../.ipython").absolute()
    os.environ["IPYTHONDIR"] = str(ipython_dir)

ignore_example_files = r"test.*" if BUILD_EXAMPLES else r"^(?!test)"

# Properly configure the table of contents for the examples/index.rst file
jinja_contexts = {
    "examples": {
        "build_examples": BUILD_EXAMPLES,
    }
}

_copy_examples_and_convert_to_notebooks(
    EXAMPLES_SOURCE_DIR,
    EXAMPLES_OUTPUT_DIR,
    ignore_example_files,
)

nbsphinx_prolog = """
Download this example as a :download:`Jupyter notebook </{{ env.docname }}.ipynb>` or a
:download:`Python script </{{ env.docname }}.py>`.

----
"""


import os
import time
import numpy as np
from collections import OrderedDict
import sys
import decimal
import sys
from rank_parser import *
import matplotlib.pyplot as plt

def main():
    DeepDataDir=sys.argv[1]

    tech = sys.argv[3]     # Tech names such as "DeepFL", "DeepFL-Spectrum"...
    model = sys.argv[4]    # model name, such as "mlp" "mlp2" "rnn" "birnn"
    loss = sys.argv[5]     # loss function
    epochnumber = int(sys.argv[6])   # epoch number (notice that this is a maximal number)
    sub = sys.argv[7]
    dump_step = int(sys.argv[8])
    figure_name = sys.argv[9]
    ResultDir = sys.argv[2] + '/'+figure_name   #directory of results
    subs=[]
    vers= []
    if(sub == 'all'):
        subs=['Chart','Lang','Math','Time','Closure','Mockito']
        vers=[26, 65, 106, 27, 133, 38]
    elif(sub=='noClosure'):
        subs=['Chart','Lang','Math','Time','Mockito']
        vers=[26, 65, 106, 27, 38]
    elif(sub=='Chart'):
        subs=['Chart']
        vers=[26]
    elif(sub=='Mockito'):
        subs=['Mockito']
        vers=[38]
    elif(sub=='10fold'):
        subs=['10fold']
        vers=[10]
    else:
        print('WRONG!'+sub)
        exit()

    #dnns=['mlp','mlp2','rnn','birnn','fc']
    #techsvector=[dnns[0],dnns[1],dnns[2],dnns[3],dnns[4]]
    dnns=['fc']
    techsvector=[dnns[0]]
    final_result = []
    for i in range(epochnumber/dump_step):
        step = (i+1)*dump_step
        #For RQ1,  current 3 libsvm and 4 deeplearning(SpectrumTestJawkByte)
        resultBysub=initializeResult(subs,techsvector)
        #read deep learning result and RQ2 line grpah trend compare mlp,mlp2,rnn,birnn
        truevers=readDeepResult(DeepDataDir,subs,tech,dnns,step,vers,resultBysub,techsvector,ResultDir)
        #Calculate overallresult
        CalculateOverall(resultBysub,truevers,techsvector)
        final_result.append(resultBysub)
    final_result = np.array(final_result)
    m = dnns.index(model)
    plt.figure(figsize=(8,14))
    for i in range(len(subs)):
        ax1 = plt.subplot(int((len(subs)+1)/2)+1,2,i+1)
        x = [(j+1)*dump_step for j in range(epochnumber/dump_step)]
        ax1.plot(x,final_result[:,i,m,0],marker='o',label='top1',markersize=2)
        ax1.plot(x,final_result[:,i,m,1],marker='*',label='top3',markersize=2)
        ax1.plot(x,final_result[:,i,m,2],marker='d',label='top5',markersize=2)
        ax1.set_xlabel('steps')
        ax2 = ax1.twinx()
        ax2.plot(x,final_result[:,i,m,3],marker='s',label='MFR',color='black',markersize=2)
        ax2.plot(x,final_result[:,i,m,4],marker='.',label='MAR',color = 'yellow',markersize=2)
        plt.title(subs[i],loc='right')

    #PLOT ALL subfigure which including MLP and RNN comparison
    ax1 = plt.subplot(int((len(subs)+1)/2)+1,1,int(len(subs)/2)+1)
    #x = [(j+1)*dump_step for j in range(epochnumber/dump_step)] + ['MLP','RNN']
    x = np.array([(j+1)*dump_step for j in range(epochnumber/dump_step)])
    #top1_y = np.concatenate((final_result[:,len(subs),m,0],[187,215]))
    print(np.shape(final_result[:,len(subs),m,0]))
    top1_y = final_result[:,len(subs),m,0]
    print("sum top1 y is",top1_y)
    max_y = max(top1_y)
    second_y = max(top1_y[:-1])
    max_x = x[np.argmax(top1_y)]
    second_x = x[np.argmax(top1_y[:-1])]
    print(np.argmax(second_y),second_x,second_y)
    ax1.plot(x,top1_y,marker='o',label='top1',markersize=2)
    #ax1.annotate(str(max_x)+","+str(max_y),xy=(max_x,max_y),color='red')
    #ax1.annotate(str(second_x)+","+str(second_y),xy=(str(second_x),second_y),color='blue')
    #ax1.plot(x,np.concatenate((final_result[:,len(subs),m,1],[271,282])),marker='*',label='top3',markersize=2)
    #ax1.plot(x,np.concatenate((final_result[:,len(subs),m,2],[305,304])),marker='d',label='top5',markersize=2)
    ax1.plot(x,final_result[:,len(subs),m,1],marker='*',label='top3',markersize=2)
    ax1.plot(x,final_result[:,len(subs),m,2],marker='d',label='top5',markersize=2)
    ax1.set_xlabel('steps')
    ax2 = ax1.twinx()
    ax2.plot(x,final_result[:,len(subs),m,3],label='MFR',marker='s',color='black',markersize=2)
    ax2.plot(x,final_result[:,len(subs),m,4],label='MAR',marker='.',color = 'yellow',markersize=2)
    #ax2.plot(x,np.concatenate((final_result[:,len(subs),m,3],[8.27,6.62])),label='MFR',marker='s',color='black',markersize=2)
    #ax2.plot(x,np.concatenate((final_result[:,len(subs),m,4],[10.67,8.54])),label='MAR',marker='.',color = 'yellow',markersize=2)
    plt.title('ALL',loc='right')
    plt.xlabel('steps')
    ax1.legend(loc='upper right')
    ax2.legend(loc='upper left')
    plt.subplots_adjust(wspace =0.3, hspace =0.3)
    #plt.show()
    plt.savefig("fig_result/"+figure_name)
main()


import json
import requests
from config import keys

""" Обработка исключений"""
class ConvertionException(Exception):
    pass

class CryptoConvertor:
    @staticmethod
    def convertor(quote: str, base: str, amount: str):
        if quote == base:
            raise ConvertionException(f"Невозможно перевести одинаковые валюты {base}.")
        try:
            quote_ticker = keys[quote]
        except KeyError:
            raise ConvertionException(f"Не удалось обработать валюту {quote}")
        try:
            base_ticker = keys[base]
        except KeyError:
            raise ConvertionException(f"Не удалось обработать валюту {base}")
        try:
            amount = float(amount)
        except ValueError:
            raise ConvertionException(f"Не удалось обработать количество {amount}")
      # получаем api конвертирующий одну валюту в другую
        r = requests.get(f'https://min-api.cryptocompare.com/data/price?fsym={quote_ticker}&tsyms={base_ticker}')
        # преобразовывем ответ формата json в объект python
        resp = json.loads(r.content)[keys[base]]
       # умножаем полученный объект на колисчество валюты
        return resp * amount


from flask import Flask, jsonify, request
from flask_cors import CORS
import mysql.connector
import json

app = Flask(__name__)
CORS(app)
posts = [
    {
      "message": "Hello world",
      "user": "David"
    },
    {
      "message": "Hello world2",
      "user": "Daniel"
    }
    ];

@app.route("/posts", methods = ['GET', 'POST'])
def handlePosts():
  mydb = mysql.connector.connect(
  host="0.0.0.0",
  user="ubuntu",
  password="",
  database="website"
  )

  if request.method == 'GET':
    mycursor = mydb.cursor()
    mycursor.execute("SELECT * FROM web")
    myresult = mycursor.fetchall()
    payload = []
    content = {}
    for result in myresult:
      content = {'message': result[1], 'user': result[0]}
      payload.append(content)
      content = {}
    print(jsonify(payload))
    return jsonify(payload)
  else:

    mycursor = mydb.cursor()
    sql = ("INSERT INTO web(name, message) VALUES (%s, %s)")
    data = request.get_json(force=True)
    val = (data["name"],data["message"])
    mycursor.execute(sql, val)

    mydb.commit()
    return "{\"status\": true}"

if __name__ == "__main__":
    app.run(host = "0.0.0.0")


# -*- coding: utf-8 -*-
"""
Created on Fri Mar 25 11:34:50 2016

@author: chc
"""

def transform_txt_matrix(filename,sp=',',perturbation=0):
	"""
	transform txt into matrix
	name1 \t name2 \t num
	…\t …\t …
	"""
	import numpy as np
	with open(filename,'rt') as f:
		txt=f.read()
	info= txt.split('\n')[:-1]
	name_dict={}
	name_list=[]
	num=0
	for item in info:
		temp=item.split(sp)
		temp1=temp[0]
		temp2=temp[1]
		if temp1 not in name_dict:
			name_dict[temp1]=num
			name_list.append(temp1)
			num+=1
		if temp2 not in name_dict:
			name_dict[temp2]=num
			name_list.append(temp1)
			num+=1
	matrix=np.zeros((num,num))
	for item in info:
		temp=item.split(sp)
		temp1=temp[0]
		temp2=temp[1]
		temp3=temp[2]
		matrix[name_dict[temp1],name_dict[temp2]]=float(temp3)+perturbation
	return matrix,name_list

def IOF_martix(matrix,way='forward'):
	"""
	if way is 'forward'
	B[i,j]=x[i,j]/X^j
	-------------------------------------
	if way is 'backward'
	B[i,j]=x[i,j]/X_i

	"""
	import numpy as np
	n=len(matrix)
	IOF_mat=np.zeros((n,n))
	if way=='forward':
		row_sum=np.sum(matrix,axis=1)
		for j in range(0,n):
			if row_sum[j]:
				IOF_mat[:,j]=matrix[:,j]/row_sum[j]
	if way=='backward':
		col_sum=np.sum(matrix,axis=0)
		for i in range(0,n):
			if col_sum[i]:
				IOF_mat[i,:]=matrix[i,:]/col_sum[i]
	return IOF_mat

def max_eigenvalue(matrix,types='sparse',perturbation=0.001):
	"""
	find largest eigenvalue of matrix
	"""
	import scipy
	import numpy as np
	import scipy.sparse.linalg
	matrix=matrix+np.eye(len(matrix))
	if types=='sparse':
		sp_matrix=scipy.sparse.coo_matrix(matrix)
	eig_value=scipy.sparse.linalg.eigs(sp_matrix,k=1, return_eigenvectors=False)
	eig_value=float(eig_value[0].real)-1
	return eig_value

def IOF_range(matrix,name_list):
	import copy
	n=len(matrix)
	#ev_0=max_eigenvalue(matrix,types='sparse')
	ev_0=1
	iof={}
	for i in range(0,n):
		print(i+1,'/',n)
		temp_matrix=copy.deepcopy(matrix)
		temp_matrix[i,:]=0
		temp_matrix[:,i]=0
		temp_ev=max_eigenvalue(temp_matrix,types='sparse')
		iof[name_list[i]]=ev_0-temp_ev
	from operator import itemgetter
	iof_range=sorted(iof.items(), key=itemgetter(1), reverse=True)
	return iof_range


"""
Rollups
"""
import copy
import logging
from typing import List, Dict

from nbdb.common.data_point import DataPoint, DATA_DROP
from nbdb.common.data_point import MISSING_POINT_VALUE, TOMBSTONE_VALUE
from nbdb.common.metric_parsers import MetricParsers
from nbdb.common.telemetry import Telemetry
from nbdb.common.tracing_config import TracingConfig
from nbdb.config.settings import Settings
from nbdb.schema.schema import Schema, ROLLUP_MEAN, ROLLUP_LAST, \
    ROLLUP_MAX, ROLLUP_SUM, \
    DASHBOARD_QUERIES, ROLLUP_NONE
from nbdb.store.sparse_algo_selector import SparseAlgoSelector
from nbdb.store.sparse_series_stats import SparseSeriesStats
from nbdb.store.sparse_store import SparseStore

from nbdb.schema.schema import SHARD_BY_CLUSTER, SHARD_BY_METRIC

logger = logging.getLogger()

# pylint: disable=too-few-public-methods
class Rollups:
    """
    Streaming rollup values computed as the raw points are processed
    The intermediate values are stored in the stats object.
    The rolled up value is pushed to the store after sparse check
    i.e. rolled up value is also sparse in nature
    """

    def __init__(self,
                 rollup_settings: Settings,
                 schema: Schema,
                 sparse_algo_selector: SparseAlgoSelector,
                 sparse_store: SparseStore,
                 past_message_lag: int=None,
                 future_message_lag: int=None):
        """
        Initialize the Rollups
        :param windows: List of window intervals for which to compute rollup
        :param schema: Schema object, is used to identify rollup function
        :param sparse_algo_selector: Sparse algorithm selector (for computing
        sparseness of the rolled up value)
        :param sparse_store: SparseStore where rolled up value is written after
        sparseness check
        """
        self.should_report_clock_skew: bool = rollup_settings.report_clock_skew
        self.windows: List[int] = schema.rollup_windows
        self.schema: Schema = schema
        self.sparse_algo_selector: SparseAlgoSelector = sparse_algo_selector
        self.sparse_store: SparseStore = sparse_store
        # maintain a count of rollups written per window
        self._telemetry_rollup_count: Dict[int, int] = dict()
        for window in self.windows:
            self._telemetry_rollup_count[window] = 0
        self.duplicated_data_points = 0
        self.past_message_lag = past_message_lag
        self.future_message_lag = future_message_lag

    def add(self, data_point: DataPoint, stat: SparseSeriesStats,
            replay_mode: bool) -> None:
        """
        # TODO: Handle the Tombstones, we will receive tombstone values on
        #  raw data points
        :param data_point:
        :param stat:
        :param replay_mode:
        :return:
        """
        # check if this is a collocated data point
        # no need for a rollup, collocated data point is a duplicate
        # datapoint that is also stored in a collocated datasource
        # same point will also be written to a regular datasource and
        # that will be rolled up
        if data_point.datasource == DASHBOARD_QUERIES:
            return

        # Check if we have any history to compute rollup with
        last_unrolled_epoch = stat.get_window_epoch(window=0)
        if last_unrolled_epoch <= 0:
            # This is the first datapoint received. This datapoint's epoch will
            # become the last unrolled epoch
            last_unrolled_epoch = data_point.epoch

        func_name = stat.get_rollup_function()
        if func_name == ROLLUP_NONE:
            func_name = self.schema.get_rollup_function(data_point.series_id)
            stat.set_rollup_function(func_name)
        for window in self.windows:
            self._add_to_window(window,
                                func_name,
                                data_point,
                                stat,
                                last_unrolled_epoch,
                                replay_mode)

    def _add_to_window(self,
                       window: int,
                       func_name: int,
                       data_point: DataPoint,
                       stat: SparseSeriesStats,
                       last_unrolled_epoch: int,
                       replay_mode: bool) -> None:
        """
        :param window: rollup window interval (in seconds)
        :param func_name: rollup aggregation function to use
        :param data_point: new data point received
        :param stat: stats object
        :param last_unrolled_epoch: Refers to the epoch of the last raw
        datapoint that was written (ie. not dropped due to sparseness)
        :param replay_mode: Boolean indicating whether datapoint was received
        in replay mode or not
        :return:
        """
        # get the last checkpoint for this window
        last_check_point = stat.get_check_point(window, -1)

        if last_check_point == -1:
            last_check_point = Rollups._init_checkpoint(window,
                                                        stat,
                                                        data_point.epoch)
        if data_point.epoch < last_check_point:
            # Don't process if the data point is older than last_check_point
            if TracingConfig.TRACE_ACTIVE:
                logger.info('TRACE: rollups: Skipping raw datapoint because '
                            'older than last checkpoint. data_point=%s '
                            'window=%s last_check_point=%s', data_point,
                            window, last_check_point)
            return

        # check if the new data point is within this rollup window
        next_check_point = last_check_point + window
        while next_check_point <= data_point.epoch:
            # new data point is beyond the checkpoint
            # so we can compute the rollup value for this window
            self._process_check_point(window,
                                      func_name,
                                      last_unrolled_epoch,
                                      last_check_point,
                                      next_check_point,
                                      stat,
                                      data_point,
                                      replay_mode)
            last_check_point = next_check_point
            next_check_point = next_check_point + window

        if data_point.value == TOMBSTONE_VALUE:
            # If we get a tombstone value, we generate a value with the
            # intermediate computations for the next check point, and insert a
            # tombstone right after that
            self._process_tombstone(window,
                                    func_name,
                                    last_unrolled_epoch,
                                    data_point.epoch,
                                    last_check_point,
                                    next_check_point,
                                    stat,
                                    data_point,
                                    replay_mode)
            # Nothing more should be done. Return
            return

        if data_point.epoch < next_check_point:
            # still within the current window, incrementally add the
            # value to the intermediate rollup value
            # last value received was valid for delta epoch
            start_epoch = max(last_check_point, last_unrolled_epoch)
            delta_epoch = data_point.epoch - start_epoch
            self._increment(window, func_name, stat, delta_epoch)
            if TracingConfig.TRACE_ACTIVE:
                logger.info('TRACE: rollups: Raw datapoint within '
                            'next_check_point. Incrementing. data_point=%s '
                            'window=%s next_check_point=%s delta_epoch=%s',
                            data_point, window, next_check_point, delta_epoch)

    @staticmethod
    def _init_checkpoint(window: int,
                         stat: SparseSeriesStats,
                         latest_unrolled_epoch: int) -> int:
        """
        Initialize the checkpoint for the window
        :param window:
        :param stat:
        :param latest_unrolled_epoch: Latest raw epoch to be added to a window.
        :return: checkpoint
        """
        # Initialize the checkpoint
        # checkpoint should be aligned to window boundary
        last_check_point = int(latest_unrolled_epoch / window) * window
        stat.set_check_point(window, last_check_point)
        return last_check_point

    def _process_check_point(self,
                             window: int,
                             func_name: int,
                             last_unrolled_epoch: int,
                             last_check_point: int,
                             next_check_point: int,
                             stat: SparseSeriesStats,
                             data_point: DataPoint,
                             replay_mode: bool) -> None:
        """

        --LC|---IV---NC|-------|-------|--------------

        :param window:
        :param func_name:
        :param last_unrolled_epoch:
        :param last_check_point:
        :param next_check_point:
        :param stat:
        :param data_point:
        :return:
        """
        # restrict the computation of delta epoch to current window only
        start_epoch = max(last_unrolled_epoch, last_check_point)
        delta_epoch = next_check_point - start_epoch
        # increment the rollup algo with last value which was valid for
        # delta epoch
        Rollups._increment(window, func_name, stat, delta_epoch)
        # compute the rolled up value now, and advance the rollup window
        rollup_value = Rollups._compute(window, func_name, stat)
        if rollup_value is None:
            # Create a missing point marker datapoint
            rollup_data_point = self._create_rollup_data_point(
                window, next_check_point, MISSING_POINT_VALUE, data_point,
                # This is a special value. Make sure it's NOT normalized before
                # storing
                is_special_value=True)
        else:
            # Create a data point with the computed rolled up value
            rollup_data_point = self._create_rollup_data_point(
                window, next_check_point, rollup_value, data_point)

        # Now verify if this is sparse
        result = self.sparse_algo_selector.process(rollup_data_point, stat,
                                                   replay_mode, window)
        if TracingConfig.TRACE_ACTIVE:
            logger.info('TRACE: rollups._process_check_point: '
                        'rollup_data_point=%s replay_mode=%s window=%s '
                        'result:%s', rollup_data_point, replay_mode, window,
                        result)

        if result != DATA_DROP:
            self._write_to_store(window, stat, rollup_data_point,
                                 replay_mode)

        # set the new checkpoint
        stat.set_check_point(window, next_check_point)

    # pylint: disable-msg=R0913  # Too Many Arguments
    # pylint: disable-msg=R0914  # Too Many Locals
    def _process_tombstone(self,
                           window: int,
                           func_name: int,
                           last_unrolled_epoch: int,
                           unrolled_tombstone_epoch: int,
                           last_check_point: int,
                           next_check_point: int,
                           stat: SparseSeriesStats,
                           data_point: DataPoint,
                           replay_mode: bool) -> None:
        """
        Process any partial computations and generate datapoints for it before
        generating a tombstone datapoint. This function should only be called
        if a tombstone value was received for the raw series

        Reference illustration:
        --LC|---IV-TB--NC|-------|-------|--------------

        :param window:
        :param func_name:
        :param last_unrolled_epoch:
        :param last_check_point:
        :param next_check_point:
        :param stat:
        :param data_point:
        :return:
        """
        # restrict the computation of delta epoch to current window only
        start_epoch = max(last_unrolled_epoch, last_check_point)
        delta_epoch = unrolled_tombstone_epoch - start_epoch
        # increment the rollup algo with last value which was valid for
        # delta epoch
        Rollups._increment(window, func_name, stat, delta_epoch)
        # compute the rolled up value now, and advance the rollup window
        rollup_value = Rollups._compute(window, func_name, stat)
        if rollup_value is None:
            # No need to create a missing point marker. We will just directly
            # create the tombstone epoch
            rollup_tombstone_epoch = next_check_point
            if TracingConfig.TRACE_ACTIVE:
                logger.info('TRACE: rollups._process_tombstone: Not creating '
                            'partial window because NULL value. '
                            'data_point=%s window=%s replay_mode=%s',
                            data_point, window, replay_mode)
        else:
            # Create a rollup data point using the intermediate state
            rollup_data_point = self._create_rollup_data_point(
                window, next_check_point, rollup_value, data_point)

            if TracingConfig.TRACE_ACTIVE:
                logger.info('TRACE: rollups._process_tombstone: Creating '
                            'partial window value. rollup_data_point=%s '
                            'window=%s replay_mode=%s', rollup_data_point,
                            window, replay_mode)

            self._write_to_store(window, stat, rollup_data_point,
                                 replay_mode)

            # Store the rollup tombstone 10m after the intermediate state
            # rollup datapoint
            rollup_tombstone_epoch = next_check_point + window

        # Create a rollup tombstone datapoint
        rollup_tombstone = self._create_rollup_data_point(
            window, rollup_tombstone_epoch, TOMBSTONE_VALUE, data_point,
            # This is a special value. Make sure it's NOT normalized before
            # storing
            is_special_value=True)
        self._write_to_store(window, stat, rollup_tombstone, replay_mode)

    def generate_data_point_shard_by_metric(self, data_point: DataPoint,
                                            window: int,
                                            shard: int) -> DataPoint:
        """
        Generate a duplicate datapoint for the rollup cross-cluster datasource.

        Duplicating to the cross-cluster datasources enables fast cross-cluster
        querying.

        NOTE: We assume that the cross-cluster pattern matching has already
        been done & verified before this method is called
        """
        datasource_name = self.schema.compose_crosscluster_datasource(
            shard, window)
        dup_data_point = copy.deepcopy(data_point)
        dup_data_point.datasource = datasource_name
        return dup_data_point

    def _write_to_store(self,
                        window: int,
                        stat: SparseSeriesStats,
                        rollup_data_point: DataPoint,
                        replay_mode: bool) -> None:
        """
        Write the data point to store
        :param window:
        :param rollup_data_point:
        """
        if not replay_mode:
            # We skip generating any writes for replay mode datapoints
            self.sparse_store.write(rollup_data_point)

            if TracingConfig.TRACE_ACTIVE:
                logger.info('TRACE: rollups: Writing to store'
                            'rollup_data_point=%s replay_mode=%s window=%s ',
                            rollup_data_point, replay_mode, window)

            # Check if datapoint matches one of the cross-cluster patterns. If
            # so, we duplicate a datapoint to the rollup cross-cluster
            # datasource to enable fast cross-cluster querying
            shard = stat.get_crosscluster_shard()
            if shard is not None:
                # Datapoint matched one of the cross-cluster patterns
                dup_data_point = self.generate_data_point_shard_by_metric(
                        rollup_data_point, window, shard)
                self.sparse_store.write(dup_data_point)

                if TracingConfig.TRACE_ACTIVE:
                    logger.info('TRACE: rollups: Writing dup_cc_point'
                                'dup_data_point=%s replay_mode=%s '
                                'window=%s', dup_data_point, replay_mode,
                                window)

                self.duplicated_data_points += 1
                # meter.mark calls are expensive, amortize it across 1000 calls
                if self.duplicated_data_points > 1000:
                    Telemetry.inst.registry.meter(
                        'MetricConsumer.duplicated_rollup_cc_data_points'
                    ).mark(self.duplicated_data_points)
                    self.duplicated_data_points = 0

        # Report the telemetry to so we can track rollups are generating data
        _rollups_written = self._telemetry_rollup_count[window]
        _rollups_written += 1
        # meter.mark calls are expensive, amortize it across 1000 counts
        if _rollups_written > 1000:
            Telemetry.inst.registry.meter(
                'SparseSeriesWriter.rollups.sparse_writes',
                tag_key_values=[f"Window={window}"]).mark(_rollups_written)
            _rollups_written = 0
        self._telemetry_rollup_count[window] = _rollups_written
        # Update the windows point that got written
        self.sparse_algo_selector.update_written_point(rollup_data_point,
                                                       stat,
                                                       window)

    @staticmethod
    def _increment(window: int,
                   func_name: int,
                   stat: SparseSeriesStats,
                   delta_epoch: int) -> None:
        """

        :param window:
        :param func_name:
        :param stat:
        :param delta_epoch:
        :return:
        """
        last_value = stat.get_window_value(0)
        if last_value < 0:
            # ignore missing data points
            return

        # Keep track of sum of delta epochs. This helps us determine the number
        # of non-NULL datapoints seen in the window
        delta_epoch_sum = stat.get_rollup_intermediate_delta_epoch(window)
        delta_epoch_sum += delta_epoch
        stat.set_rollup_intermediate_delta_epoch(window, delta_epoch_sum)

        if func_name == ROLLUP_MEAN:
            weighted_sum = stat.get_rollup_intermediate_value(window)
            weighted_sum += last_value * delta_epoch
            stat.set_rollup_intermediate_value(window, weighted_sum)
        elif func_name == ROLLUP_MAX:
            max_so_far = stat.get_rollup_intermediate_value(window)
            max_so_far = max(max_so_far, last_value)
            stat.set_rollup_intermediate_value(window, max_so_far)
        elif func_name == ROLLUP_LAST:
            stat.set_rollup_intermediate_value(window, last_value)
        elif func_name == ROLLUP_SUM:
            weighted_sum = stat.get_rollup_intermediate_value(window)
            weighted_sum += last_value * delta_epoch
            stat.set_rollup_intermediate_value(window, weighted_sum)
        else:
            raise ValueError('Unsupported rollup function: ' + str(func_name))

    @staticmethod
    def _compute(window: int,
                 func_name: int,
                 stat: SparseSeriesStats) -> float:
        """

        :param window:
        :param func_name:
        :param stat:
        :return:
        """
        try:
            intermediate_value = stat.get_rollup_intermediate_value(window)
            delta_epoch_sum = stat.get_rollup_intermediate_delta_epoch(window)
            if delta_epoch_sum == 0:
                # No datapoints were seen for this window. Return NULL
                # value
                return None

            if func_name == ROLLUP_MEAN:
                weighted_sum = intermediate_value
                return weighted_sum / delta_epoch_sum
            if func_name == ROLLUP_MAX:
                return intermediate_value
            if func_name == ROLLUP_LAST:
                return intermediate_value
            if func_name == ROLLUP_SUM:
                return intermediate_value
            raise ValueError('Unsupported rollup function: ' + str(func_name))
        finally:
            # reset the intermediate value & delta_epoch for next checkpoint
            Rollups._reset_intermediate_state(window, stat)

    @staticmethod
    def _reset_intermediate_state(window: int,
                                  stat: SparseSeriesStats) -> None:
        """
        Reset the intermediate value & delta_epoch for next checkpoint
        :param window:
        :param stat:
        """
        stat.set_rollup_intermediate_value(window, 0)
        stat.set_rollup_intermediate_delta_epoch(window, 0)

    def _create_rollup_data_point(self,
                                  window: int,
                                  check_point: int,
                                  value: float,
                                  data_point: DataPoint,
                                  is_special_value: bool = False) -> DataPoint:
        """

        :param window: Rollup window
        :param check_point: epoch at which the rollup is created
        :param value: rollup value
        :param data_point: sample data point for the original series
        :param is_special_value: Indicates whether the rollup value is a
        special reserved value. Eg. -1 and -3 are reserved values for missing
        point & tombstone markers
        :return:
        """
        rollup_data_source = self.schema.get_rollup_datasource(window, data_point.tags)

        if self.should_report_clock_skew:
            MetricParsers.detect_and_report_clock_skew(
                data_point.server_rx_epoch, check_point, 'rollups',
                rollup_data_source, '',
                self.past_message_lag, self.future_message_lag)
        return DataPoint(rollup_data_source,
                         data_point.field,
                         data_point.tags,
                         check_point,
                         data_point.server_rx_epoch,
                         value,
                         is_special_value=is_special_value)


#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8
"""
Fix num of submissions
"""

from django.core.management.base import BaseCommand
from django.db import transaction
from django.utils.translation import gettext_lazy

from onadata.apps.logger.models import Instance
from onadata.apps.logger.models import XForm
from onadata.apps.main.models import UserProfile

class Command(BaseCommand):
    """Fix num of submissions"""

    help = gettext_lazy("Fix num of submissions")

    def handle(self, *args, **kwargs):
        i = 0
        xform_count = XForm.objects.filter(downloadable=True).count()
        for xform in XForm.objects.filter(downloadable=True).iterator():
            with transaction.atomic():
                instance_count = xform.instances.filter(deleted_at=None).count()
                xform.num_of_submissions = instance_count
                xform.save(update_fields=["num_of_submissions"])
            i += 1
            self.stdout.write(
                f"Processing {i} of {xform_count}: {xform.id_string} ({instance_count})"
            )

        i = 0
        profile_count = UserProfile.objects.count()
        for profile in UserProfile.objects.select_related("user__username").iterator():
            with transaction.atomic():
                instance_count = Instance.objects.filter(
                    deleted_at=None, xform__user_id=profile.user_id
                ).count()
                profile.num_of_submissions = instance_count
                profile.save(update_fields=["num_of_submissions"])
            i += 1
            self.stdout.write(
                f"Processing {i} of {profile_count}: {profile.user.username} "
                f"({instance_count})"
            )


import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
import pickle
import re
import string
from sklearn.model_selection import train_test_split

def tokenize(s):
    regex = re.compile('[%s]' % re.escape(string.punctuation))
    out = regex.sub(' ', s).split()
    return out

def fit_logistic(x, y):
    y = y.values
    model = LogisticRegression(C=4, dual=True)
    return model.fit(x, y)

def training_code(PATH = "archive/labeled_data.csv"):
    COMMENT = 'tweet'
    label_cols = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']

    print("Load data...")
    train = pd.read_csv(PATH)

    print("Fill empty with unknown...")
    train[COMMENT].fillna('unknown', inplace=True)

    print("Train TFIDF vectorizer...")
    tfidfvectorizer = TfidfVectorizer(ngram_range=(1, 2), tokenizer=tokenize,
                                      min_df=3, max_df=0.9, strip_accents='unicode',
                                      use_idf=1, smooth_idf=True, sublinear_tf=1)

    train_term_doc = tfidfvectorizer.fit_transform(train[COMMENT])
    x = train_term_doc

    # joblib.dump(tfidfvectorizer, 'models/tfidf_vectorizer_train.pkl')
    with open('models/tfidf_vectorizer_train.pkl', 'wb') as tfidf_file:
        pickle.dump(tfidfvectorizer, tfidf_file)

    print("Fit logistic regression for each class...")
    for i, j in enumerate(label_cols):
        print("Fitting:", j)
        model = fit_logistic(x, train[j])

        # joblib.dump(model, 'models/logistic_{}.pkl'.format(j))
        with open('models/logistic_{}.pkl'.format(j), 'wb') as lg_file:
            pickle.dump(model, lg_file)

if __name__ == "__main__":
    training_code()


import json
import time
from datetime import datetime
import random
import datetime

from google.cloud import pubsub_v1

# --- Base variables and auth path
PROJECT_ID = "task-cf-370710"
TOPIC_ID = "df-pub_sub-topic"
MAX_MESSAGES = 5
MAX_ERRORS = 2

# --- PubSub Utils Classes
class PubSubPublisher:
    def __init__(self, project_id, topic_id):
        self.project_id = project_id
        self.topic_id = topic_id
        self.publisher = pubsub_v1.PublisherClient(transport=None)
        self.topic_path = self.publisher.topic_path(self.project_id, self.topic_id)

    def publish(self, data: str):
        result = self.publisher.publish(self.topic_path, data.encode("utf-8"))
        return result

# --- Main publishing script
def main():
    i = 0
    publisher = PubSubPublisher(PROJECT_ID, TOPIC_ID)

    err_idx = set()

    while len(err_idx) < MAX_ERRORS:
        err_idx.add(random.randint(0, MAX_MESSAGES-1))

    while i < MAX_MESSAGES:
        now = datetime.datetime.utcnow()
        if i in err_idx:
            data = {
                "message": f"message-{i}",
                "number": "not an int",
                "age": random.random()
            }
        else:
            data = {
                "message": f"message-{i}",
                "number": random.randint(0, 10),
                "age": random.randint(10, 30),
                "timestamp": now.strftime('%Y-%m-%d %H:%M:%S')
            }
        publisher.publish(json.dumps(data))
        time.sleep(1)
        i += 1

if __name__ == "__main__":
    main()


# search in Array can be performed using the `linearSearch algorithm `
# in LinearSearh we need to search each element of the array to the target value
# if the match found then we can return the index of the matched element or target value
# if not match not found then in that case search is unsuccessful and we can output the result as per the wish

from array import *
from typing import Union

arr1=array("i",[1,2,3,4,5,6])

# here we need to perform the linear search for getting the elements of the array

def searchElementUsingIndex(arr:array, target)->Union[int,str]:

    for index in range(len(arr)):  # here  the time complexity for the for loop is of O(N)

        if arr[index]==target: # for the if statement the time complecity is of O(1)

            return index  # return also a constant time complecioty hence the time complecity is of O(1) in here
    else:
        return "No match found" # here insted of the str we can also return -1 which means the elements not found in here

        # here also for this the time complecity is of O(1)

# hence the total time complxity will be of O(N) in this case

# the time complexity of len() is of O(1) which is the constant time complecity

# the time complexity of the range() is also of O(1) as it does not create the actual sequence of interger same as the array , but rather create a iterator object which is of range object
# which can be produced on demand based on the length of the Array

# the space complecity of the function will be of O(1) as we are just accessing the element of array which is already created  we are not changing the value

def searchElementUsingElement(arr:array, target)->Union[int,str]:

    for ind in arr:

        if ind==target:

            return arr.index(ind)
    else:
        return "No match found"

# print(searchElementUsingIndex(arr1,5))
# print(searchElementUsingIndex(arr1,9))

print(searchElementUsingElement(arr1,5))
print(searchElementUsingElement(arr1,9))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Dec 18 17:52:13 2020
@author: rfrazin

Anderson-Darling Test for univariate normality.
See the Wikipedia article.

This implements the single sample test.
The default is case 3 (i.e., unknown mean and var).
For cases 0, 1 and 2, in which the mean and/or var are known, simply set
   mu and/or var to their known values.
Note that the critical values of the test statistic (called A2 here) have
only a very weak dependence on the number of samples, making it particularly
good for non-independent samples for which the degrees of freedom are hard to
determine.  However, the effective sample size should be greater than 8.
Generally speaking, the normality is questionable when A2 is greater than
about 0.6 and should be rejected when greater than about 1.

"""

import numpy as np
from scipy.stats import norm

#x - is an array of samples from a univariate process
def ADtest(x, mu=None, var=None):
    assert x.ndim == 1
    u = np.sort(x)
    n = len(u)
    if mu is None: # (cases 0 and 2)
        mu = np.mean(u)
    if var is None: # (cases 0 and 1)
        var = np.var(u, ddof=1)
    y = (u - mu)/np.sqrt(var)
    s = 0.
    for k in range(n):
        cdfk = norm.cdf(y[k])
        s +=  (2*k - 1)*np.log(cdfk) + (2*(n-k) + 1)*np.log(1 - cdfk)
    s /= (-1*n)
    A2 = s - n
    return(A2)


#!/usr/bin/python3

# Convert all the objects in 1 category to YOLOV5 annotation files.
# The category ID is hard coded for the input category & output category.
# The COCO parser is copied from https://github.com/immersive-limit/coco-manager

# set max_objects for testing on a smaller data set

# python3 ~/truckcam/coco_to_yolo.py annotations/instances_val2017.json val2017/ val_person
# python3 ~/truckcam/coco_to_yolo.py annotations/instances_train2017.json train2017/ train_person

import os
import sys
import fileinput
import shutil
import json
from pathlib import Path

old_category = 1
new_category = 0
# limit the maximum number of objects for testing
#max_objects = -1
max_objects = 1000

class ImageObject:
    def __init__(self):
        self.category_id = -1
        self.box = [ 0.0, 0.0, 0.0, 0.0 ]

images = []

class Coco():
    def _process_info(self):
        self.info = self.coco['info']

    def _process_licenses(self):
        self.licenses = self.coco['licenses']

    def _process_categories(self):
        self.categories = dict()
        self.super_categories = dict()
        self.category_set = set()

        for category in self.coco['categories']:
            cat_id = category['id']
            super_category = category['supercategory']

            # Add category to categories dict
            if cat_id not in self.categories:
                self.categories[cat_id] = category
                self.category_set.add(category['name'])
            else:
                print(f'ERROR: Skipping duplicate category id: {category}')

            # Add category id to the super_categories dict
            if super_category not in self.super_categories:
                self.super_categories[super_category] = {cat_id}
            else:
                self.super_categories[super_category] |= {cat_id} # e.g. {1, 2, 3} |= {4} => {1, 2, 3, 4}

    def _process_images(self):
        self.images = dict()
        for image in self.coco['images']:
            image_id = image['id']
            if image_id not in self.images:
# create a new image record
                self.images[image_id] = image
            else:
                print(f'ERROR: Skipping duplicate image id: {image}')

    def _process_segmentations(self):
        self.segmentations = dict()
        for segmentation in self.coco['annotations']:
            image_id = segmentation['image_id']
            if image_id not in self.segmentations:
                self.segmentations[image_id] = []
            self.segmentations[image_id].append(segmentation)

    def process(self, input_json, input_dir, output_dir):
        # Open json
        self.input_json_path = Path(input_json)

        if not self.input_json_path.exists():
            print('Input JSON path not found.')
            print('Giving up & going to a movie.')
            quit()

        # Load the json into RAM
        print('Loading JSON file...')
        with open(self.input_json_path) as json_file:
            self.coco = json.load(json_file)

        print('Processing input JSON...')
        self._process_info()
        self._process_licenses()
        # category ID -> category
        self._process_categories()
        # image ID -> image file
        self._process_images()
        # bounding boxes -> image, category
        self._process_segmentations()

        # Want a .txt file for each image with the desired categories in it
        print("old images=%d" % len(self.images.items()))

# count the old objects
        count = 0
        for key, value in self.segmentations.items():
            count += len(value)
        print("old objects=%d" % count)

# read the old objects
        count = 0
        new_images = dict()
# 1 segmentation key for each image with multiple objects in it
        for key, value in self.segmentations.items():
            #print("seg=%s, %d" % (key, len(value)))
            for object in value:
                if object['category_id'] == old_category and object['iscrowd'] == 0:
                    image_id = object['image_id']
                    image = self.images[image_id]
                    image_w = image['width']
                    image_h = image['height']
                    # bbox is x, y, w, h
                    object_x = object['bbox'][0]
                    object_y = object['bbox'][1]
                    object_w = object['bbox'][2]
                    object_h = object['bbox'][3]

                    if image_id == 370210:
                        print("seg=%s" % object)
                        print("image=%s w=%d h=%d box=%d,%d,%d,%d" %
                            (image['file_name'], image_w, image_h, object_x, object_y, object_w, object_h))
                    image_object = ImageObject()
                    image_object.category_id = old_category
# YOLOV5 expects the center x, center y, w, h in fractions of the image size
                    image_object.box = [float(object_x + object_w / 2) / image_w,
                        float(object_y + object_h / 2) / image_h,
                        float(object_w) / image_w,
                        float(object_h) / image_h]
                    if image_id not in new_images:
                        new_images[image_id] = []
                    new_images[image_id].append(image_object)
                    count += 1
# debug
                if max_objects > 0 and count > max_objects:
                    break
            if max_objects > 0 and count > max_objects:
                break

        print("new images=%d" % len(new_images.items()))
        print("new objects=%d" % count)

        if True:
            for key, value in new_images.items():
                image = self.images[key]

                prefix = image['file_name'].split('.')[0]
                new_filename = prefix + '.txt'

    #            print('%s, %s:' % (image['file_name'], new_filename))
    # copy the image to the destination dir
                shutil.copy(input_dir + '/' + image['file_name'],
                    output_dir)
    # create the txt annotations
                file = open(output_dir + '/' + new_filename, 'w')
                for i in value:
                    file.write('%d %f %f %f %f\n' % (new_category, i.box[0], i.box[1], i.box[2], i.box[3]))
    #                print('%d %f %f %f %f' % (new_category, i.box[0], i.box[1], i.box[2], i.box[3]))
                file.close()

if __name__ == "__main__":
    input_json = sys.argv[1]
    input_dir = sys.argv[2]
    output_dir = sys.argv[3]

    print("JSON file: %s" % input_json)
    print("Source image dir: %s" % input_dir)
    print("Destination dir: %s" % output_dir)
    print("Source category ID: %d" % old_category)
    print("Dest category index: %d" % new_category)
    print("Max objects: %d" % max_objects)

    value = input('Press Enter to continue.')

    cf = Coco()
    cf.process(input_json, input_dir, output_dir)


import matplotlib.pyplot as plt
import networkx as nx

# В этой строчке задается случайное дерево на 20 вершинах. При каждом перезапуске блока будет создаваться новое дерево.
tree = nx.random_tree(12)

# Заводим множество посещенных вершин
visited = set()

# Здесь приведена краткая памятка по командам, которые могут потребоваться
# Следующая команда добавляет вершину v в множество посещенных
# visited.add(v)
# Следующая команда возвращает итератор по соседям вершины v
# tree[v]

# Функция должна возвращать глубину дерева с корнем v

v_root=0 # ввершина относительно которой считается

def count_depth(v_root):
    # Добавьте здесь ваше решение
    visited.add(v_root)
    depth = 0
    for u in tree[v_root]:
        if u not in visited:
            depth = max(depth, count_depth(u)+1)
    return depth

print(f'Глубина дерева: {count_depth(0)}')

nx.draw_networkx(tree)
plt.show()