import logging
import math
import re
import string
import sys
import unicodedata
from collections import defaultdict
from itertools import cycle
from pathlib import Path
from typing import IO, Dict, List, Optional, Pattern, Set, Union

from symspellpy import helpers
from symspellpy.composition import Composition
from symspellpy.editdistance import DistanceAlgorithm, EditDistance
from symspellpy.pickle_mixin import PickleMixin
from symspellpy.suggest_item import SuggestItem
from symspellpy.verbosity import Verbosity

logger = logging.getLogger(__name__)
WORD_PATTERN = re.compile(r"(([^\W_]|['’])+)")

class SymSpell(PickleMixin):
    """Symmetric Delete spelling correction algorithm.

    `initial_capacity` from the original code is omitted since python cannot
    preallocate memory. `compact_mask` from the original code is omitted since
    we're not mapping suggested corrections to hash codes.

    Args:
        max_dictionary_edit_distance: Maximum edit distance for doing lookups.
        prefix_length: The length of word prefixes used for spell checking.
        count_threshold: The minimum frequency count for dictionary words to be
            considered correct spellings.

    Attributes:
        _max_dictionary_edit_distance (int): Maximum dictionary term length.
        _prefix_length (int): The length of word prefixes used for spell
            checking.
        _count_threshold (int): A threshold may be specified, when a term occurs
            so frequently in the corpus that it is considered a valid word for
            spelling correction.
        _distance_algorithm (DistanceAlgorithm): Edit distance algorithms.
        _max_length (int): Length of longest word in the dictionary.

    Raises:
        ValueError: If `max_dictionary_edit_distance` is negative.
        ValueError: If `prefix_length` is less than 1 or not greater than
            `max_dictionary_edit_distance`.
        ValueError: If `count_threshold` is negative.
    """

    data_version = 3
    # Number of all words in the corpus used to generate the frequency
    # dictionary. This is used to calculate the word occurrence probability p
    # from word counts c : p=c/N. N equals the sum of all counts c in the
    # dictionary only if the dictionary is complete, but not if the dictionary is
    # truncated or filtered.
    N = 1024908267229
    bigram_count_min = sys.maxsize

    def __init__(
        self,
        max_dictionary_edit_distance: int = 2,
        prefix_length: int = 7,
        count_threshold: int = 1,
    ) -> None:
        if max_dictionary_edit_distance < 0:
            raise ValueError("max_dictionary_edit_distance cannot be negative")
        if prefix_length < 1:
            raise ValueError("prefix_length cannot be less than 1")
        if prefix_length <= max_dictionary_edit_distance:
            raise ValueError(
                "prefix_length must be greater than max_dictionary_edit_distance"
            )
        if count_threshold < 0:
            raise ValueError("count_threshold cannot be negative")
        self._words: Dict[str, int] = {}
        self._below_threshold_words: Dict[str, int] = {}
        self._bigrams: Dict[str, int] = {}
        self._deletes: Dict[str, List[str]] = defaultdict(list)
        self._replaced_words: Dict[str, SuggestItem] = {}

        self._max_dictionary_edit_distance = max_dictionary_edit_distance
        self._prefix_length = prefix_length
        self._count_threshold = count_threshold
        self._distance_algorithm = DistanceAlgorithm.DAMERAU_OSA_FAST
        self._max_length = 0

    @property
    def below_threshold_words(self) -> (Dict[str, int]):
        """Dictionary of unique words that are below the count threshold for
        being considered correct spellings.
        """
        return self._below_threshold_words

    @property
    def bigrams(self) -> Dict[str, int]:
        """Dictionary of unique correct spelling bigrams, and the frequency count
        for each word.
        """
        return self._bigrams

    @property
    def deletes(self) -> Dict[str, List[str]]:
        """Dictionary that contains a mapping of lists of suggested correction
        words to the original words and the deletes derived from them. A list of
        suggestions might have a single suggestion, or multiple suggestions.
        """
        return self._deletes

    @property
    def distance_algorithm(self) -> DistanceAlgorithm:
        """The current distance algorithm."""
        return self._distance_algorithm

    @distance_algorithm.setter
    def distance_algorithm(self, value: DistanceAlgorithm) -> None:
        if not isinstance(value, DistanceAlgorithm):
            raise TypeError(
                "can only assign DistanceAlgorithm type values to distance_algorithm"
            )
        self._distance_algorithm = value

    @property
    def entry_count(self) -> int:
        """Number of unique correct spelling words."""
        return len(self._deletes)

    @property
    def replaced_words(self) -> Dict[str, SuggestItem]:
        """Dictionary corrected/modified words."""
        return self._replaced_words

    @property
    def words(self) -> Dict[str, int]:
        """Dictionary of unique correct spelling words, and the frequency count
        for each word.
        """
        return self._words

    @property
    def word_count(self) -> int:
        """Number of unique correct spelling words."""
        return len(self._words)

    def create_dictionary(
        self, corpus: Union[Path, str, IO[str]], encoding: Optional[str] = None
    ) -> bool:
        """Loads multiple dictionary words from a file containing plain text.

        **NOTE**: Merges with any dictionary data already loaded.

        Args:
            corpus: The path+filename of the file or afile object of the
                dictionary.
            encoding: Text encoding of the corpus file.

        Returns:
            ``True`` if file loaded, or ``False`` if file not found.
        """
        if isinstance(corpus, (Path, str)):
            corpus = Path(corpus)
            if not corpus.exists():
                logger.error(f"Corpus not found at {corpus}.")
                return False
            for key in self._parse_words(corpus.read_text(encoding=encoding)):
                self.create_dictionary_entry(key, 1)
        else:
            for line in corpus:
                for key in self._parse_words(line):
                    self.create_dictionary_entry(key, 1)
        return True

    def create_dictionary_entry(self, key: str, count: int) -> bool:
        """Creates/updates an entry in the dictionary.

        For every word there are deletes with an edit distance of
        1..max_edit_distance created and added to the dictionary. Every delete
        entry has a suggestions list, which points to the original term(s) it was
        created from. The dictionary may be dynamically updated (word frequency
        and new words) at any time by calling create_dictionary_entry.

        Args:
            key: The word to add to dictionary.
            count: The frequency count for word.

        Returns:
            ``True`` if the word was added as a new correctly spelled word, or
            ``False`` if the word is added as a below threshold word, or updates
            an existing correctly spelled word.
        """
        if count <= 0:
            # Early return if count is zero, as it can't change anything
            if self._count_threshold > 0:
                return False
            count = 0

        # Look first in below threshold words, update count, and allow promotion
        # to correct spelling word if count reaches threshold threshold must be
        # >1 for there to be the possibility of low threshold words
        if self._count_threshold > 1 and key in self._below_threshold_words:
            count_previous = self._below_threshold_words[key]
            # calculate new count for below threshold word
            count = helpers.increment_count(count, count_previous)
            # has reached threshold - remove from below threshold collection (it
            # will be added to correct words below)
            if count >= self._count_threshold:
                self._below_threshold_words.pop(key)
            else:
                self._below_threshold_words[key] = count
                return False
        elif key in self._words:
            count_previous = self._words[key]
            # just update count if it's an already added above threshold word
            self._words[key] = helpers.increment_count(count, count_previous)
            return False
        elif count < self._count_threshold:
            # new or existing below threshold word
            self._below_threshold_words[key] = count
            return False

        # what we have at this point is a new, above threshold word
        self._words[key] = count

        # edits/suggestions are created only once, no matter how often word
        # occurs. edits/suggestions are created as soon as the word occurs in the
        # corpus, even if the same term existed before in the dictionary as an
        # edit from another word
        if len(key) > self._max_length:
            self._max_length = len(key)

        # create deletes
        edits = self._edits_prefix(key)
        for delete in edits:
            self._deletes[delete].append(key)
        return True

    def delete_dictionary_entry(self, key: str) -> bool:
        """Deletes an entry in the dictionary.

        If the deleted entry is the longest word, update :attr:`_max_length`
        with the next longest word.

        Args:
            key: The word to add to dictionary.

        Returns:
            ``True`` if the word is successfully deleted, or ``False`` if the
            word is not found.
        """
        if key not in self._words:
            return False
        del self._words[key]
        # look for the next longest word if we just deleted the longest word
        if len(key) == self._max_length:
            self._max_length = max(map(len, self._words.keys()))

        # create deletes
        edits = self._edits_prefix(key)
        for delete in edits:
            self._deletes[delete].remove(key)
        return True

    def load_bigram_dictionary(
        self,
        corpus: Union[Path, str],
        term_index: int,
        count_index: int,
        separator: Optional[str] = None,
        encoding: Optional[str] = None,
    ) -> bool:
        """Loads multiple dictionary entries from a file of word/frequency count
        pairs.

        **NOTE**: Merges with any dictionary data already loaded.

        Args:
            corpus: The path+filename of the file.
            term_index: The column position of the word.
            count_index: The column position of the frequency count.
            separator: Separator characters between term(s) and count.
            encoding: Text encoding of the dictionary file.

        Returns:
            ``True`` if file loaded, or ``False`` if file not found.
        """
        corpus = Path(corpus)
        if not corpus.exists():
            logger.error(f"Bigram dictionary file not found at {corpus}.")
            return False
        with open(corpus, "r", encoding=encoding) as infile:
            return self._load_bigram_dictionary_stream(
                infile, term_index, count_index, separator
            )

    def load_dictionary(
        self,
        corpus: Union[Path, str],
        term_index: int,
        count_index: int,
        separator: str = " ",
        encoding: Optional[str] = None,
    ):
        """Loads multiple dictionary entries from a file of word/frequency count
        pairs.

        **NOTE**: Merges with any dictionary data already loaded.

        Args:
            corpus: The path+filename of the file.
            term_index: The column position of the word.
            count_index: The column position of the frequency count.
            separator: Separator characters between term(s) and count.
            encoding: Text encoding of the dictionary file.

        Returns:
            ``True`` if file loaded, or ``False`` if file not found.
        """
        corpus = Path(corpus)
        if not corpus.exists():
            logger.error(f"Dictionary file not found at {corpus}.")
            return False
        with open(corpus, "r", encoding=encoding) as infile:
            return self._load_dictionary_stream(
                infile, term_index, count_index, separator
            )

    def lookup(
        self,
        phrase: str,
        verbosity: Verbosity,
        max_edit_distance: Optional[int] = None,
        include_unknown: bool = False,
        ignore_token: Optional[Pattern[str]] = None,
        transfer_casing: bool = False,
    ) -> List[SuggestItem]:
        """Finds suggested spellings for a given phrase word.

        Args:
            phrase: The word being spell checked.
            verbosity: The value controlling the quantity/closeness of the
                returned suggestions.
            max_edit_distance: The maximum edit distance between phrase and
                suggested words. Set to :attr:`_max_dictionary_edit_distance` by
                default.
            include_unknown: A flag to determine whether to include phrase word
                in suggestions, if no words within edit distance found.
            ignore_token: A regex pattern describing what words/phrases to ignore
                and leave unchanged.
            transfer_casing: A flag to determine whether the casing --- i.e.,
                uppercase vs lowercase --- should be carried over from `phrase`.

        Returns:
            A list of :class:`SuggestItem` objects representing suggested correct
            spellings for the phrase word, sorted by edit distance, and
            secondarily by count frequency.

        Raises:
            ValueError: If `max_edit_distance` is greater than
                :attr:`_max_dictionary_edit_distance`
        """
        if max_edit_distance is None:
            max_edit_distance = self._max_dictionary_edit_distance
        if max_edit_distance > self._max_dictionary_edit_distance:
            raise ValueError("distance too large")
        suggestions: List[SuggestItem] = []
        phrase_len = len(phrase)

        if transfer_casing:
            original_phrase = phrase
            phrase = phrase.lower()

        def early_exit():
            if include_unknown and not suggestions:
                suggestions.append(SuggestItem(phrase, max_edit_distance + 1, 0))
            return suggestions

        # early exit - word is too big to possibly match any words
        if phrase_len - max_edit_distance > self._max_length:
            return early_exit()

        # quick look for exact match
        if phrase in self._words:
            suggestion_count = self._words[phrase]
            if transfer_casing:
                suggestions.append(SuggestItem(original_phrase, 0, suggestion_count))
            else:
                suggestions.append(SuggestItem(phrase, 0, suggestion_count))
            # early exit - return exact match, unless caller wants all matches
            if verbosity != Verbosity.ALL:
                return early_exit()

        if ignore_token is not None and re.match(ignore_token, phrase) is not None:
            suggestion_count = 1
            suggestions.append(SuggestItem(phrase, 0, suggestion_count))
            # early exit - return exact match, unless caller wants all matches
            if verbosity != Verbosity.ALL:
                return early_exit()

        # early termination, if we only want to check if word in dictionary or
        # get its frequency e.g. for word segmentation
        if max_edit_distance == 0:
            return early_exit()

        considered_deletes = set()
        considered_suggestions = set()
        # we considered the phrase already in the 'phrase in self._words' above
        considered_suggestions.add(phrase)

        max_edit_distance_2 = max_edit_distance
        candidate_pointer = 0
        candidates = []

        # add original prefix
        phrase_prefix_len = phrase_len
        if phrase_prefix_len > self._prefix_length:
            phrase_prefix_len = self._prefix_length
            candidates.append(phrase[:phrase_prefix_len])
        else:
            candidates.append(phrase)
        distance_comparer = EditDistance(self._distance_algorithm)
        while candidate_pointer < len(candidates):
            candidate = candidates[candidate_pointer]
            candidate_pointer += 1
            candidate_len = len(candidate)
            len_diff = phrase_prefix_len - candidate_len

            # early termination: if candidate distance is already higher than
            # suggestion distance, then there are no better suggestions to be
            # expected
            if len_diff > max_edit_distance_2:
                # skip to next candidate if Verbosity.ALL, look no
                # further if Verbosity.TOP or CLOSEST (candidates are
                # ordered by delete distance, so none are closer than
                # current)
                if verbosity == Verbosity.ALL:  # pragma: no cover
                    # `max_edit_distance_2`` only updated when
                    # verbosity != ALL. New candidates are generated from
                    # deletes so it keeps getting shorter. This should never
                    # be reached.
                    continue
                break  # pragma: no cover, "peephole" optimization, http://bugs.python.org/issue2506

            if candidate in self._deletes:
                dict_suggestions = self._deletes[candidate]
                for suggestion in dict_suggestions:
                    if suggestion == phrase:
                        continue
                    suggestion_len = len(suggestion)
                    # phrase and suggestion lengths diff > allowed/current best
                    # distance
                    if (
                        abs(suggestion_len - phrase_len) > max_edit_distance_2
                        # suggestion must be for a different delete string, in
                        # same bin only because of hash collision
                        or suggestion_len < candidate_len
                        # if suggestion len = delete len, then it either equals
                        # delete or is in same bin only because of hash collision
                        or (suggestion_len == candidate_len and suggestion != candidate)
                    ):
                        continue  # pragma: no cover, "peephole" optimization, http://bugs.python.org/issue2506
                    suggestion_prefix_len = min(suggestion_len, self._prefix_length)
                    if (
                        suggestion_prefix_len > phrase_prefix_len
                        and suggestion_prefix_len - candidate_len > max_edit_distance_2
                    ):
                        continue
                    # True Damerau-Levenshtein Edit Distance: adjust distance,
                    # if both distances>0. We allow simultaneous edits (deletes)
                    # of max_edit_distance on on both the dictionary and the
                    # phrase term. For replaces and adjacent transposes the
                    # resulting edit distance stays <= max_edit_distance. For
                    # inserts and deletes the resulting edit distance might
                    # exceed max_edit_distance. To prevent suggestions of a
                    # higher edit distance, we need to calculate the resulting
                    # edit distance, if there are simultaneous edits on both
                    # sides. Example: (bank==bnak and bank==bink, but bank!=kanb
                    # and bank!=xban and bank!=baxn for max_edit_distance=1).
                    # Two deletes on each side of a pair makes them all equal,
                    # but the first two pairs have edit distance=1, the others
                    # edit distance=2.
                    distance = 0
                    min_distance = 0
                    if candidate_len == 0:
                        # suggestions which have no common chars with phrase
                        # (phrase_len<=max_edit_distance &&
                        # suggestion_len<=max_edit_distance)
                        distance = max(phrase_len, suggestion_len)
                        if (
                            distance > max_edit_distance_2
                            or suggestion in considered_suggestions
                        ):
                            continue
                    elif suggestion_len == 1:
                        # This should always be phrase_len - 1? Since
                        # suggestions are generated from deletes of the input
                        # phrase
                        distance = (
                            phrase_len
                            if phrase.index(suggestion[0]) < 0
                            else phrase_len - 1
                        )
                        # `suggestion` only gets added to
                        # `considered_suggestions` when `suggestion_len>1`.
                        # Given the max_dictionary_edit_distance and
                        # prefix_length restrictions, `distance`` should never
                        # be >max_edit_distance_2
                        if (
                            distance > max_edit_distance_2
                            or suggestion in considered_suggestions
                        ):  # pragma: no cover
                            continue
                    # number of edits in prefix ==maxeditdistance AND no
                    # identical suffix, then editdistance>max_edit_distance and
                    # no need for Levenshtein calculation
                    # (phraseLen >= prefixLength) &&
                    # (suggestionLen >= prefixLength)
                    else:
                        # handles the shortcircuit of min_distance assignment
                        # when first boolean expression evaluates to False
                        if self._prefix_length - max_edit_distance == candidate_len:
                            min_distance = (
                                min(phrase_len, suggestion_len) - self._prefix_length
                            )
                        else:
                            min_distance = 0
                        # pylint: disable=too-many-boolean-expressions
                        if (
                            self._prefix_length - max_edit_distance == candidate_len
                            and (
                                min_distance > 1
                                and phrase[phrase_len + 1 - min_distance :]
                                != suggestion[suggestion_len + 1 - min_distance :]
                            )
                            or (
                                min_distance > 0
                                and phrase[phrase_len - min_distance]
                                != suggestion[suggestion_len - min_distance]
                                and (
                                    phrase[phrase_len - min_distance - 1]
                                    != suggestion[suggestion_len - min_distance]
                                    or phrase[phrase_len - min_distance]
                                    != suggestion[suggestion_len - min_distance - 1]
                                )
                            )
                        ):
                            continue
                        # delete_in_suggestion_prefix is somewhat expensive, and
                        # only pays off when verbosity is TOP or CLOSEST
                        if suggestion in considered_suggestions:
                            continue
                        considered_suggestions.add(suggestion)
                        distance = distance_comparer.compare(
                            phrase, suggestion, max_edit_distance_2
                        )
                        if distance < 0:
                            continue
                    # do not process higher distances than those already found,
                    # if verbosity<ALL (note: max_edit_distance_2 will always
                    # equal max_edit_distance when Verbosity.ALL)
                    if distance <= max_edit_distance_2:  # pragma: no branch
                        suggestion_count = self._words[suggestion]
                        item = SuggestItem(suggestion, distance, suggestion_count)
                        if suggestions:
                            if verbosity == Verbosity.CLOSEST:
                                # we will calculate DamLev distance only to the
                                # smallest found distance so far
                                if distance < max_edit_distance_2:
                                    suggestions = []
                            elif verbosity == Verbosity.TOP:
                                if (  # pragma: no branch, "peephole" optimization, http://bugs.python.org/issue2506
                                    distance < max_edit_distance_2
                                    or suggestion_count > suggestions[0].count
                                ):
                                    max_edit_distance_2 = distance
                                    suggestions[0] = item
                                continue
                        if verbosity != Verbosity.ALL:
                            max_edit_distance_2 = distance
                        suggestions.append(item)
            # add edits: derive edits (deletes) from candidate (phrase) and add
            # them to candidates list. this is a recursive process until the
            # maximum edit distance has been reached
            if len_diff < max_edit_distance and candidate_len <= self._prefix_length:
                # do not create edits with edit distance smaller than
                # suggestions already found
                if verbosity != Verbosity.ALL and len_diff >= max_edit_distance_2:
                    continue
                for i in range(candidate_len):
                    delete = candidate[:i] + candidate[i + 1 :]
                    if delete not in considered_deletes:
                        considered_deletes.add(delete)
                        candidates.append(delete)
        if len(suggestions) > 1:
            suggestions.sort()

        if transfer_casing:
            suggestions = [
                SuggestItem(
                    helpers.case_transfer_similar(original_phrase, s.term),
                    s.distance,
                    s.count,
                )
                for s in suggestions
            ]

        early_exit()
        return suggestions

    def lookup_compound(
        self,
        phrase: str,
        max_edit_distance: int,
        ignore_non_words: bool = False,
        transfer_casing: bool = False,
        split_by_space: bool = False,
        ignore_term_with_digits: bool = False,
    ) -> List[SuggestItem]:
        """`lookup_compound` supports compound aware automatic spelling
        correction of multi-word input strings with three cases:

        1. mistakenly inserted space into a correct word led to two incorrect
           terms
        2. mistakenly omitted space between two correct words led to one
           incorrect combined term
        3. multiple independent input terms with/without spelling errors

        Find suggested spellings for a multi-word input string
        (supports word splitting/merging).

        Args:
            phrase: The string being spell checked.
            max_edit_distance: The maximum edit distance between input and
                suggested words.
            ignore_non_words: A flag to determine whether numbers and acronyms
                are left alone during the spell checking process.
            transfer_casing: A flag to determine whether the casing --- i.e.,
                uppercase vs lowercase --- should be carried over from `phrase`.
            split_by_space: Splits the phrase into words simply based on space.
            ignore_any_term_with_digits: A flag to determine whether any term
                with digits is left alone during the spell checking process. Only
                works when ``ignore_non_words` is also ``True``.

        Returns:
            A list of :class:`SuggestItem` objects representing suggested correct
            spellings for `phrase`.
        """
        # Parse input string into single terms
        terms_1 = helpers.parse_words(phrase, split_by_space=split_by_space)
        # Second list of single terms with preserved cases so we can ignore
        # acronyms (all cap words)
        if ignore_non_words:
            terms_2 = helpers.parse_words(phrase, True, split_by_space)
        suggestions = []
        suggestion_parts = []
        distance_comparer = EditDistance(self._distance_algorithm)

        # translate every item to its best suggestion, otherwise it remains
        # unchanged
        is_last_combi = False
        for i, _ in enumerate(terms_1):
            if ignore_non_words:
                if helpers.try_parse_int64(terms_1[i]) is not None:
                    suggestion_parts.append(SuggestItem(terms_1[i], 0, self.N))
                    continue
                if helpers.is_acronym(terms_2[i], ignore_term_with_digits):
                    suggestion_parts.append(SuggestItem(terms_2[i], 0, self.N))
                    continue
            suggestions = self.lookup(terms_1[i], Verbosity.TOP, max_edit_distance)
            # combi check, always before split
            if i > 0 and not is_last_combi:
                suggestions_combi = self.lookup(
                    terms_1[i - 1] + terms_1[i],
                    Verbosity.TOP,
                    max_edit_distance,
                )
                if suggestions_combi:
                    best_1 = suggestion_parts[-1]
                    if suggestions:
                        best_2 = suggestions[0]
                    else:
                        # estimated word occurrence probability
                        # P=10 / (N * 10^word length l)
                        best_2 = SuggestItem.create_with_probability(
                            terms_1[i], max_edit_distance + 1
                        )
                    # distance_1=edit distance between 2 split terms and their
                    # best corrections: als comparative value for the combination
                    distance_1 = best_1.distance + best_2.distance
                    if distance_1 >= 0 and (
                        suggestions_combi[0].distance + 1 < distance_1
                        or (
                            suggestions_combi[0].distance + 1 == distance_1
                            and (
                                suggestions_combi[0].count
                                > best_1.count / self.N * best_2.count
                            )
                        )
                    ):
                        suggestions_combi[0].distance += 1
                        suggestion_parts[-1] = suggestions_combi[0]
                        self._replaced_words[terms_1[i - 1]] = suggestions_combi[0]
                        is_last_combi = True
                        continue
            is_last_combi = False

            # alway split terms without suggestion / never split terms with
            # suggestion ed=0 / never split single char terms
            if suggestions and (suggestions[0].distance == 0 or len(terms_1[i]) == 1):
                # choose best suggestion
                suggestion_parts.append(suggestions[0])
            else:
                # if no perfect suggestion, split word into pairs
                suggestion_split_best = None
                # add original term
                if suggestions:
                    suggestion_split_best = suggestions[0]
                if len(terms_1[i]) > 1:
                    for j in range(1, len(terms_1[i])):
                        part_1 = terms_1[i][:j]
                        part_2 = terms_1[i][j:]
                        suggestions_1 = self.lookup(
                            part_1, Verbosity.TOP, max_edit_distance
                        )
                        if not suggestions_1:
                            continue
                        suggestions_2 = self.lookup(
                            part_2, Verbosity.TOP, max_edit_distance
                        )
                        if not suggestions_2:
                            continue
                        # select best suggestion for split pair
                        tmp_term = f"{suggestions_1[0].term} {suggestions_2[0].term}"
                        tmp_distance = distance_comparer.compare(
                            terms_1[i], tmp_term, max_edit_distance
                        )
                        if tmp_distance < 0:
                            tmp_distance = max_edit_distance + 1
                        if suggestion_split_best is not None:
                            if tmp_distance > suggestion_split_best.distance:
                                continue
                            if tmp_distance < suggestion_split_best.distance:
                                suggestion_split_best = None
                        if tmp_term in self._bigrams:
                            tmp_count = self._bigrams[tmp_term]
                            # increase count, if split corrections are part of
                            # or identical to input single term correction exists
                            if suggestions:
                                best_si = suggestions[0]
                                # alternatively remove the single term from
                                # suggestion_split, but then other splittings
                                # could win
                                if (
                                    suggestions_1[0].term + suggestions_2[0].term
                                    == terms_1[i]
                                ):
                                    # make count bigger than count of single
                                    # term correction
                                    tmp_count = max(tmp_count, best_si.count + 2)
                                elif best_si.term in (
                                    suggestions_1[0].term,
                                    suggestions_2[0].term,
                                ):
                                    # make count bigger than count of single
                                    # term correction
                                    tmp_count = max(tmp_count, best_si.count + 1)
                            # no single term correction exists
                            elif (
                                suggestions_1[0].term + suggestions_2[0].term
                                == terms_1[i]
                            ):
                                tmp_count = max(
                                    tmp_count,
                                    max(
                                        suggestions_1[0].count,
                                        suggestions_2[0].count,
                                    )
                                    + 2,
                                )
                        else:
                            # The Naive Bayes probability of the word
                            # combination is the product of the two word
                            # probabilities: P(AB)=P(A)*P(B) use it to estimate
                            # the frequency count of the combination, which then
                            # is used to rank/select the best splitting variant
                            tmp_count = min(
                                self.bigram_count_min,
                                int(
                                    suggestions_1[0].count
                                    / self.N
                                    * suggestions_2[0].count
                                ),
                            )
                        suggestion_split = SuggestItem(
                            tmp_term, tmp_distance, tmp_count
                        )
                        if (
                            suggestion_split_best is None
                            or suggestion_split.count > suggestion_split_best.count
                        ):
                            suggestion_split_best = suggestion_split

                    if suggestion_split_best is not None:
                        # select best suggestion for split pair
                        suggestion_parts.append(suggestion_split_best)
                        self._replaced_words[terms_1[i]] = suggestion_split_best
                    else:
                        item = SuggestItem.create_with_probability(
                            terms_1[i], max_edit_distance + 1
                        )
                        suggestion_parts.append(item)
                        self._replaced_words[terms_1[i]] = item
                else:
                    # estimated word occurrence probability
                    # P=10 / (N * 10^word length l)
                    item = SuggestItem.create_with_probability(
                        terms_1[i], max_edit_distance + 1
                    )
                    suggestion_parts.append(item)
                    self._replaced_words[terms_1[i]] = item
        joined_term = ""
        joined_count: float = self.N
        for item in suggestion_parts:
            joined_term += item.term + " "
            joined_count *= item.count / self.N
        joined_term = joined_term.rstrip()
        if transfer_casing:
            joined_term = helpers.case_transfer_similar(phrase, joined_term)
        suggestion = SuggestItem(
            joined_term,
            distance_comparer.compare(phrase, joined_term, 2 ** 31 - 1),
            int(joined_count),
        )
        return [suggestion]

    def word_segmentation(
        self,
        phrase: str,
        max_edit_distance: Optional[int] = None,
        max_segmentation_word_length: Optional[int] = None,
        ignore_token: Optional[Pattern] = None,
    ) -> Composition:
        """`word_segmentation` divides a string into words by inserting missing
        spaces at the appropriate positions misspelled words are corrected and do
        not affect segmentation existing spaces are allowed and considered for
        optimum segmentation.

        `word_segmentation` uses a novel approach *without* recursion.
        https://medium.com/@wolfgarbe/fast-word-segmentation-for-noisy-text-2c2c41f9e8da
        While each string of length n can be segmented in 2^n−1 possible
        compositions
        https://en.wikipedia.org/wiki/Composition_(combinatorics)
        `word_segmentation` has a linear runtime O(n) to find the optimum
        composition.

        Finds suggested spellings for a multi-word input string (supports word
        splitting/merging).

        Args:
            phrase: The string being spell checked.
            max_segmentation_word_length: The maximum word length that should be
                considered.
            max_edit_distance: The maximum edit distance between input and
                corrected words (0=no correction/segmentation only).
            ignore_token: A regex pattern describing what words/phrases to ignore
                and leave unchanged.

        Returns:
            The word segmented string, the word segmented and spelling corrected
            string, the edit distance sum between input string and corrected
            string, the sum of word occurence probabilities in log scale (a
            measure of how common and probable the corrected segmentation is).
        """
        # normalize ligatures: scientiﬁc -> scientific
        phrase = unicodedata.normalize("NFKC", phrase).replace("\u002D", "")

        if max_edit_distance is None:
            max_edit_distance = self._max_dictionary_edit_distance
        if max_segmentation_word_length is None:
            max_segmentation_word_length = self._max_length
        array_size = min(max_segmentation_word_length, len(phrase))
        compositions = [Composition()] * array_size
        circular_index = cycle(range(array_size))
        idx = -1

        # outer loop (column): all possible part start positions
        for j in range(len(phrase)):
            # inner loop (row): all possible part lengths (from start position):
            # part can't be bigger than longest word in dictionary (other than
            # long unknown word)
            imax = min(len(phrase) - j, max_segmentation_word_length)
            for i in range(1, imax + 1):
                # get top spelling correction/ed for part
                part = phrase[j : j + i]
                separator_len = 0
                top_ed = 0
                top_log_prob = 0.0
                top_result = ""

                if part[0].isspace():
                    # remove space for levensthein calculation
                    part = part[1:]
                else:
                    # add ed+1: space did not exist, had to be inserted
                    separator_len = 1

                # remove space from part1, add number of removed spaces to top_ed
                top_ed += len(part)
                # remove space. add number of removed spaces to ed
                part = part.replace(" ", "")
                top_ed -= len(part)

                # v6.7: Lookup against the lowercase term
                results = self.lookup(
                    part.lower(),
                    Verbosity.TOP,
                    max_edit_distance,
                    ignore_token=ignore_token,
                )
                if results:
                    top_result = results[0].term
                    # v6.7: retain/preserve upper case
                    if len(part) > 0 and part[0].isupper():
                        top_result = top_result.capitalize()

                    top_ed += results[0].distance
                    # Naive Bayes Rule. We assume the word probabilities of two
                    # words to be independent. Therefore the resulting
                    # probability of the word combination is the product of the
                    # two word probabilities. Instead of computing the product
                    # of probabilities we are computing the sum of the logarithm
                    # of probabilities because the probabilities of words are
                    # about 10^-10, the product of many such small numbers could
                    # exceed (underflow) the floating number range and become
                    # zero. log(ab)=log(a)+log(b)
                    top_log_prob = math.log10(float(results[0].count) / float(self.N))
                else:
                    top_result = part
                    # default, if word not found. otherwise long input text
                    # would win as long unknown word (with ed=edmax+1), although
                    # there there should many spaces inserted
                    top_ed += len(part)
                    top_log_prob = math.log10(10.0 / self.N / 10.0 ** len(part))

                dest = (i + idx) % array_size
                # set values in first loop
                if j == 0:
                    compositions[dest] = Composition(
                        part, top_result, top_ed, top_log_prob
                    )
                # pylint: disable=C0301,R0916
                elif (
                    i == max_segmentation_word_length
                    # replace values if better log_prob_sum, if same edit
                    # distance OR one space difference
                    or (
                        compositions[dest].distance_sum
                        in (
                            compositions[idx].distance_sum + top_ed,
                            compositions[idx].distance_sum + separator_len + top_ed,
                        )
                        and compositions[dest].log_prob_sum
                        < compositions[idx].log_prob_sum + top_log_prob
                    )
                    # replace values if smaller edit distance
                    or compositions[idx].distance_sum + separator_len + top_ed
                    < compositions[dest].distance_sum
                ):
                    if (
                        len(top_result) == 1 and top_result[0] in string.punctuation
                    ) or (len(top_result) == 2 and top_result.startswith("'")):
                        compositions[dest] = Composition.create(
                            compositions[idx], part, top_result, top_ed, top_log_prob
                        )
                    else:
                        compositions[dest] = Composition.create(
                            compositions[idx],
                            f" {part}",
                            f" {top_result}",
                            separator_len + top_ed,
                            top_log_prob,
                        )
            idx = next(circular_index)
        return compositions[idx]

    def _delete_in_suggestion_prefix(
        self, delete: str, delete_len: int, suggestion: str, suggestion_len: int
    ) -> bool:  # pragma: no cover
        """Checks whether all delete chars are present in the suggestion prefix
        in correct order, otherwise this is just a hash collision.

        **NOTE**: No longer used in the Python port.
        """
        if delete_len == 0:
            return True
        if self._prefix_length < suggestion_len:
            suggestion_len = self._prefix_length
        j = 0
        for i in range(delete_len):
            del_char = delete[i]
            while j < suggestion_len and del_char != suggestion[j]:
                j += 1
            if j == suggestion_len:
                return False
        return True

    def _edits(
        self,
        word: str,
        edit_distance: int,
        delete_words: Set[str],
        current_distance: int = 0,
    ) -> Set[str]:
        """Inexpensive and language independent: only deletes, no transposes +
        replaces + inserts replaces and inserts are expensive and language
        dependent.
        """
        edit_distance += 1
        if not word:
            return delete_words
        for i in range(current_distance, len(word)):
            delete = word[:i] + word[i + 1 :]
            if delete not in delete_words:
                delete_words.add(delete)
            # recursion, if maximum edit distance not yet reached
            if edit_distance < self._max_dictionary_edit_distance:
                self._edits(delete, edit_distance, delete_words, current_distance=i)
        return delete_words

    def _edits_prefix(self, key: str) -> Set[str]:
        hash_set = set()
        if len(key) <= self._max_dictionary_edit_distance:
            hash_set.add("")
        if len(key) > self._prefix_length:
            key = key[: self._prefix_length]
        hash_set.add(key)
        return self._edits(key, 0, hash_set)

    def _load_bigram_dictionary_stream(
        self,
        corpus_stream: IO[str],
        term_index: int,
        count_index: int,
        separator: Optional[str] = None,
    ):
        """Loads multiple dictionary entries from a stream of word/frequency
        count pairs.

        **NOTE**: Merges with any dictionary data already loaded.

        Args:
            corpus_stream: A file object of the dictionary.
            term_index: The column position of the word.
            count_index: The column position of the frequency count.
            separator: Separator characters between term(s) and count.

        Returns:
            ``True`` after file object is loaded.
        """
        min_parts = 3 if separator is None else 2
        for line in corpus_stream:
            parts = line.rstrip().split(separator)
            if len(parts) < min_parts:
                continue
            count = helpers.try_parse_int64(parts[count_index])
            if count is None:
                continue
            key = (
                f"{parts[term_index]} {parts[term_index + 1]}"
                if separator is None
                else parts[term_index]
            )
            self._bigrams[key] = count
            if count < self.bigram_count_min:
                self.bigram_count_min = count
        return True

    def _load_dictionary_stream(
        self,
        corpus_stream: IO[str],
        term_index: int,
        count_index: int,
        separator: str = " ",
    ) -> bool:
        """Loads multiple dictionary entries from a stream of word/frequency
        count pairs.

        **NOTE**: Merges with any dictionary data already loaded.

        Args:
            corpus_stream: A file object of the dictionary.
            term_index: The column position of the word.
            count_index: The column position of the frequency count.
            separator: Separator characters between term(s) and count.

        Returns:
            ``True`` after file object is loaded.
        """
        for line in corpus_stream:
            parts = line.rstrip().split(separator)
            if len(parts) < 2:
                continue
            count = helpers.try_parse_int64(parts[count_index])
            if count is None:
                continue
            key = parts[term_index]
            self.create_dictionary_entry(key, count)
        return True

    @staticmethod
    def _parse_words(text: str) -> List[str]:
        """Creates a non-unique wordlist from sample text language independent
        (e.g. works with Chinese characters).
        """
        # // \w Alphanumeric characters (including non-latin characters, umlaut
        # characters and digits) plus "_". [^\W_] is the equivalent of \w
        # excluding "_". Compatible with non-latin characters, does not split
        # words at apostrophes. Uses capturing groups to combine a negated set
        # with a character set.
        matches = WORD_PATTERN.findall(text.lower())
        # The above regex returns ("ghi'jkl", "l") for "ghi'jkl", so we extract
        # the first element
        matches = [match[0] for match in matches]
        return matches


from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^/$', views.index, name='index'),
    url(r'^fileGraph/$', views.fileGraph, name='fileGraph'),
    url(r'^procedureGraph/$', views.procedureGraph, name='procedureGraph'),

    url(r'^jsonGraph/$', views.jsonGraph, name='jsonGraph'),
    url(r'^jsonNodeName/$', views.jsonNodeName, name='jsonNodeName'),
]


# -*- coding:utf-8 -*-
import os
from time import time
from random import randint
import matplotlib.pyplot as plt

try:
    from primality_test_gmpy2 import get_random_prime
    from gmpy2 import ceil
    from gmpy2 import powmod as quick_mod
    from gmpy2 import gcdext as ext_gcd
except ModuleNotFoundError:
    from primality_test import get_random_prime, quick_mod
    from math import ceil
    # from custom_gcd import ext_gcd

# 判断 k 是否为 p 的原根，这里要求 p 是安全素数
def is_primitive_root(k, p) -> bool:
    a = (p - 1) // 2
    if quick_mod(k, 2, p) == 1:
        return False
    elif quick_mod(k, a, p) == 1:
        return False
    else:
        return True
    # r = 1
    # for _ in range((base-1)//2):
    #     r = (r * element) % base
    #     if r == 1:
    #         return False
    # if r == base - 1:  # a^{(\phi(p)-1)/2} equiv -1 mod p -> ord(a)=p-1
    #     return True
    # else:
    #     return False

class ElGamal:
    def __init__(self, key_info=None, nbits=None, only_encrypt=None):
        if key_info is not None:  # 指定密钥信息
            for element in ['p', 'alpha']:
                if key_info.get(element) is None:
                    print('incomplete key info, require p or alpha')
                    return
            self.p = key_info['p']
            self.alpha = key_info['alpha']
            if only_encrypt is None:  # 非仅加密，要求完整密钥信息
                if key_info.get('a') is None:
                    print('incomplete key info, require a')
                    return
                self.a = key_info['a']
                if key_info.get('beta') is not None:
                    self.beta = key_info['beta']
                else:
                    self.beta = quick_mod(self.alpha, self.a, self.p)
            else:  # 仅加密，不要求提供私钥信息
                self.a = None
                if key_info.get('beta') is None:
                    print('incomplete key info, require beta')
                    return
                self.beta = key_info['beta']
        elif nbits is not None:  # 指定密钥长度，随机生成密钥信息
            self.p = get_random_prime(nbits)
            self.alpha = randint(2, self.p - 1)
            while not is_primitive_root(self.alpha, self.p):
                self.alpha = randint(2, self.p - 1)
            self.a = randint((self.p - 1) // 2, self.p - 1)
            self.beta = quick_mod(self.alpha, self.a, self.p)
        else:
            print('no key info specified')
            pass

    def encrypt(self, message) -> tuple:
        x = randint(2, self.p - 1)
        y1 = quick_mod(self.alpha, x, self.p)
        y2 = message * quick_mod(self.beta, x, self.p) % self.p
        return y1, y2

    def decrypt(self, ciphertext: tuple):
        if self.a is None:
            print('no private key')
            return None
        if len(ciphertext) != 2:
            print('incomplete ciphertext pair')
            return None
        y1 = ciphertext[0]
        y2 = ciphertext[1]
        # reference: ext_gcd(a,b) -> (gcd(a,b), s, t) which satisfies as+bt=gcd(a,b)
        tmp = ext_gcd(quick_mod(y1, self.a, self.p), self.p)[1]
        message = y2 * tmp % self.p
        return message

    def show_key_info(self):
        public_key = {'p': self.p, 'alpha': self.alpha, 'beta': self.beta}
        private_key = {'p': self.p, 'alpha': self.alpha, 'a': self.a, 'beta': self.beta}
        return {'public_key': public_key, 'private_key': private_key}

if __name__ == '__main__':
    # e = ElGamal(key_info={'p': 2579, 'alpha': 2, 'a': 765, 'beta': 949})
    # 128bits 明文信息加解密
    e = ElGamal(nbits=129)
    r = randint(2 ** 128, 2 ** 129 - 1)
    c = e.encrypt(r)
    m = e.decrypt(c)
    print(e.show_key_info())
    print('ciphertext pair: {}'.format({'y1': hex(c[0]), 'y2': hex(c[1])}))
    print('message recovered from ciphertext: {}'.format(hex(m)))
    # 文本文件加密
    e = ElGamal(nbits=128)
    print(e.show_key_info())
    p = e.p
    try:
        os.mkdir('message_file')
        os.mkdir('cipher_file')
    except FileExistsError:
        print('folder exists')
    encrypt_time = 0
    decrypt_time = 0
    entire_time = time()
    for file_name in os.listdir('./OANC_data'):
        # 加密
        with open('./OANC_data/' + file_name, 'rb') as plaintext:
            plain_data = plaintext.read()
            plain_data_int = int.from_bytes(plain_data, byteorder='little', signed=False)
        with open('./cipher_file/' + file_name, 'w') as f:
            while plain_data_int != 0:
                plain_data_block = plain_data_int % p
                plain_data_int //= p
                t = time()
                cipher_data = e.encrypt(plain_data_block)
                encrypt_time += time() - t
                f.write(str(cipher_data[0]) + ' ' + str(cipher_data[1]) + '\n')
        # 解密
        plain_data_int = 0
        k = 0
        with open('./cipher_file/' + file_name, 'r') as f:
            cipher_data = f.readline()[:-1].split(' ')
            while cipher_data != ['']:
                y1, y2 = int(cipher_data[0]), int(cipher_data[1])
                t = time()
                plain_data_int += e.decrypt((y1, y2)) * p ** k
                decrypt_time += time() - t
                k += 1
                cipher_data = f.readline()[:-1].split(' ')
        with open('./message_file/' + file_name, 'wb') as f:
            f.write(
                int(plain_data_int).to_bytes(length=int(ceil(len(hex(plain_data_int)[2:]) / 2)), byteorder='little'))
    entire_time = time() - entire_time
    with open('./assets/crypto_time.txt', 'w') as f:
        f.write('{} {} {}'.format(encrypt_time, decrypt_time, entire_time))
    print('encrypt time: {}\ndecrypt time: {}\nentire time: {}'.
          format(encrypt_time, decrypt_time, entire_time))
    print('encrypt speed: {} KB/s\ndecrypt speed: {} KB/s'
          .format(27369401 / 1024 / encrypt_time, 27369401 / 1024 / decrypt_time))
    # 可视化
    file_size_set = []
    for file_name in os.listdir('./OANC_data'):
        with open('./OANC_data/' + file_name, 'rb') as plaintext:
            file_size_set.append(len(plaintext.read()) / 1024)
    plt.scatter(range(len(file_size_set)), file_size_set,
                color='#70a1ff', sizes=[1 for _ in range(len(file_size_set))])
    plt.axhline(y=sum(file_size_set) / len(file_size_set), label='average file size',
                xmin=0, xmax=len(file_size_set), linestyle='--', color='#ff6348')
    plt.title('file size distribution')
    plt.ylim([0, max(file_size_set) + 1])
    plt.ylabel('size(KB)')
    plt.legend()
    plt.savefig('./assets/file_size_distribution.svg', format='svg')
    plt.show()

    with open('./assets/crypto_time.txt', 'r') as f:
        t = f.read().split(' ')
        entire_time, decrypt_time, entire_time = [float(e) for e in t]
    plt.pie(x=[encrypt_time, decrypt_time, entire_time - encrypt_time - decrypt_time],
            labels=[str(int(encrypt_time * 100) // 100) + ' s',
                    str(int(decrypt_time * 100) // 100) + ' s',
                    str(int((entire_time - encrypt_time - decrypt_time) * 100) // 100) + ' s'],
            colors=['#7bed9f', '#70a1ff', '#eccc68'], autopct='%.1f%%')
    plt.title('ElGamal speed test')
    plt.legend(['encrypt', 'decrypt', 'file IO and others'], loc='lower left', bbox_to_anchor=(-0.3, 0))
    plt.savefig('./assets/crypto_time.svg', format='svg')
    plt.show()


# -*- coding: utf-8 -*-
# @Time   : 2019/4/29 11:51
# @Author :xdf_FCC
# @Email  :670992243@qq.com
# @File   :do_logging.py
'''

'''
import logging
from API_03.common.do_contants import fh_dir
from API_03.common.do_config import config

#
# class GetLogger:
#     def __init__(self, name):
#
#         # 设置自己的日志收集器
#         self.my_log = logging.getLogger(name)
#
#     def get_loggor(self, level, msg):
#         self.my_log.setLevel('DEBUG')
#         # 设置日志的输出格式
#         fmt = "%(asctime)s - %(name)s - %(levelname)s - %(message)s - %(filename)s:  %(lineno)d"
#         # fmt="%(levelno)s-%(asctime)s-%(message)s"
#         myfmt = logging.Formatter(fmt)
#         # 将日志输出到控制台中
#         kh = logging.StreamHandler()
#         # 读取配置文件中等级
#         kh_level = config.get_str('level', 'lv_1')
#         kh.setLevel(kh_level)
#         kh.setFormatter(myfmt)
#         # 输出到文件中，考虑到资源的有效利用，输出级别为info
#         fh = logging.FileHandler(fh_dir + '\case.log',encoding='UTF-8',mode='a')
#         fh_level = config.get_str('level', 'lv_2')
#         fh.setLevel(fh_level)
#         fh.setFormatter(myfmt)
#         # 日志收集器与输出渠道进行对接
#         self.my_log.addHandler(kh)
#         self.my_log.addHandler(fh)
#
#         if level == 'DEBUG':
#             self.my_log.debug(msg)
#         elif level == 'INFO':
#             self.my_log.info(msg)
#         elif level == 'WARNING':
#             self.my_log.warning(msg)
#         elif level == 'ERROR':
#             self.my_log.error(msg)
#         else:
#             self.my_log.critical(msg)
#         self.my_log.removeHandler(kh)
#         self.my_log.removeHandler(fh)
#
#     def debug(self, msg):
#         return self.get_loggor('DEBUG', msg)
#
#     def info(self, msg):
#         return self.get_loggor('INFO', msg)
#
#     def warning(self, msg):
#         return self.get_loggor('WARNING', msg)
#
#     def error(self, msg):
#         return self.get_loggor('ERROR', msg)
#
#     def critical(self, msg):
#         return self.get_loggor('CRITICAL', msg)

#
#
# my_log = GetLogger(__name__)
# my_log.info('输出测试info')
# my_log.debug('输出测试debug')
# my_log.error('输出测试error')
# my_log.warning('输出测试warning')

def GetLogger(name):
    my_log = logging.getLogger(name)
    my_log.setLevel('DEBUG')
    # 设置日志的输出格式
    fmt = "%(asctime)s - %(name)s - %(levelname)s - %(message)s - %(filename)s:  %(lineno)d"
    # fmt="%(levelno)s-%(asctime)s-%(message)s"
    myfmt = logging.Formatter(fmt)
    # 将日志输出到控制台中
    kh = logging.StreamHandler()
    # 读取配置文件中等级
    kh_level = config.get_str('level', 'lv_1')
    kh.setLevel(kh_level)
    kh.setFormatter(myfmt)
    # 输出到文件中，考虑到资源的有效利用，输出级别为info
    fh = logging.FileHandler(fh_dir + '\case.log', encoding='UTF-8')
    fh_level = config.get_str('level', 'lv_2')
    fh.setLevel(fh_level)
    fh.setFormatter(myfmt)
    # 日志收集器与输出渠道进行对接
    my_log.addHandler(kh)
    my_log.addHandler(fh)
    return my_log


import json

def transform_json_srx(srx, output_file):
    data = {
        "securityZone": srx.securityZone,
        "hostname": srx.hostname,
        "timezone": srx.timezone,
        "dns": srx.dns,
        "ntpserver": srx.ntpserver,
        "snat": srx.snat,
        "dnat": srx.dnat_list,
        "static_route": srx.static_route,
        "firewall_address": srx.firewall_address,
        "addrgrp": srx.addrgrp,
        "custom_services": srx.custom_services,
        "interfaces": srx.interfaces,
        "firewall_policy": srx.firewall_policy,
    }
    with open(output_file, "w") as f:
        json.dump(data, f, indent=4)

def transform_json_fgt(fgt, output_file, timezone):
    data = {
        "router_policy": fgt.router_policy,
        "timezone": timezone,
        "dnat_pool": fgt.dnat_pool,
        "dnat_rule_set": fgt.dnat_rule_set,
        "routing_instance": fgt.routing_instance,
        "policy_statement": fgt.policy_statement,
        "security_zone": fgt.security_zone,
        "ntp_list": fgt.ntp_list,
        "dns_list": fgt.dns_list,
        "hostname": fgt.hostname,
        "fw_ippool": fgt.fw_ippool,
        "fw_address": fgt.fw_address,
        "static_route": fgt.static_route,
        "interfaces": fgt.interfaces,
        "vlan": fgt.vlan,
        "fw_policy": fgt.fw_policy,
        "custom_services": fgt.custom_services,
        "security_policy": fgt.security_policy,
        "addr_group": fgt.addr_group,
    }
    with open(output_file, "w") as f:
        json.dump(data, f, indent=4)
    # output_fgt("./result/toFgtRes.txt", data)


# import the necessary packages
from skimage.measure import compare_ssim as ssim
from sklearn.metrics import mean_squared_error
import numpy as np
import cv2
import sys
import math
from PIL import Image

def mse(imageA, imageB):
	err = np.sum((imageA.astype("float") - imageB.astype("float")) ** 2)
	err /= float(imageA.shape[0] * imageA.shape[1])
	return err

def psnr(img1, img2):
    mse = np.mean( (img1 - img2) ** 2 )
    if mse == 0:
    	return 100
    PIXEL_MAX = 255.0
    return 20 * math.log10(PIXEL_MAX / math.sqrt(mse))

def bits_per_pixel(image_):
	mode_to_bpp = {'1':1, 'L':8, 'P':8, 'RGB':24, 'RGBA':32, 'CMYK':32, 'YCbCr':24, 'I':32, 'F':32}
	im = Image.open(image_)
	mode = im.mode
	print("bits per pixel for "+str(image_)+' : '+str(mode_to_bpp.get(mode)))

def compare_images(imageA, imageB):
	original = cv2.imread(imageA)
	contrast = cv2.imread(imageB)
	m = mse(original, contrast)
	s = ssim(original, contrast,multichannel=True)
	p = psnr(original, contrast)
	bits_per_pixel(imageA)
	bits_per_pixel(imageB)
	#print the result
	print("MSE: %.2f \nSSIM: %.2f" % (m, s))
	print("PSNR: "+str(p))

print("Images : "+sys.argv[1]+", "+sys.argv[2])

compare_images("images/"+sys.argv[1],"images/"+sys.argv[2])


import asyncio

# This example uses aiofile for asynchronous file reads.
# It's not a dependency of the project but can be installed
# with `pip install aiofile`.
import aiofile

from amazon_transcribe.client import TranscribeStreamingClient
from amazon_transcribe.handlers import TranscriptResultStreamHandler
from amazon_transcribe.model import TranscriptEvent
from amazon_transcribe.utils import apply_realtime_delay

"""
Here's an example of a custom event handler you can extend to
process the returned transcription results as needed. This
handler will simply print the text out to your interpreter.
"""

SAMPLE_RATE = 16000
BYTES_PER_SAMPLE = 2
CHANNEL_NUMS = 1

# An example file can be found at tests/integration/assets/test.wav
AUDIO_PATH = "tests/integration/assets/test.wav"
CHUNK_SIZE = 1024 * 8
REGION = "us-west-2"

class MyEventHandler(TranscriptResultStreamHandler):
    async def handle_transcript_event(self, transcript_event: TranscriptEvent):
        # This handler can be implemented to handle transcriptions as needed.
        # Here's an example to get started.
        results = transcript_event.transcript.results
        for result in results:
            for alt in result.alternatives:
                print(alt.transcript)

async def basic_transcribe():
    # Setup up our client with our chosen AWS region
    client = TranscribeStreamingClient(region=REGION)

    # Start transcription to generate our async stream
    stream = await client.start_stream_transcription(
        language_code="en-US",
        media_sample_rate_hz=SAMPLE_RATE,
        media_encoding="pcm",
    )

    async def write_chunks():
        # NOTE: For pre-recorded files longer than 5 minutes, the sent audio
        # chunks should be rate limited to match the realtime bitrate of the
        # audio stream to avoid signing issues.
        async with aiofile.AIOFile(AUDIO_PATH, "rb") as afp:
            reader = aiofile.Reader(afp, chunk_size=CHUNK_SIZE)
            await apply_realtime_delay(
                stream, reader, BYTES_PER_SAMPLE, SAMPLE_RATE, CHANNEL_NUMS
            )
        await stream.input_stream.end_stream()

    # Instantiate our handler and start processing events
    handler = MyEventHandler(stream.output_stream)
    await asyncio.gather(write_chunks(), handler.handle_events())

loop = asyncio.get_event_loop()
loop.run_until_complete(basic_transcribe())
loop.close()


from celery import shared_task
from redis import Redis
import json
from .tasks import TeacherSyncTask, StudentSyncTask

# Setup - START
redis_client = Redis(host='localhost', port=6379, db=0)

MODEL_SYNC_TASKS = {
    'datasync.Teacher': TeacherSyncTask,
    'datasync.Student': StudentSyncTask
}
# Setup - END

def get_sync_task_class(entity):
    if entity in MODEL_SYNC_TASKS:
        return MODEL_SYNC_TASKS[entity]
    else:
        raise ValueError("Unsupported entity type")

@shared_task
def enqueue_data_to_sync(record_id, entity, operation, delete=False):
    my_json = {
        'record_id': record_id,
        'entity': entity,
        'operation': operation,
        'delete': delete
    }
    my_json = json.dumps(my_json)
    redis_client.rpush('task_queue', my_json)

@shared_task
def dequeue_data_to_sync():
    # Logic to check for new records and enqueue tasks
    # based on the newly created records
    problematic_tasks = []
    while True:
        task = redis_client.lpop('task_queue')  # Fetch task from Redis list
        if task:
            # Task found in the queue
            task_dict = json.loads(task)
            try:
                SYNC_TASK_CLASS = get_sync_task_class(task_dict['entity'])
                task_obj = SYNC_TASK_CLASS(
                    task_dict['record_id'],
                    task_dict['entity'],
                    task_dict['operation'],
                    task_dict['delete'])
                task_obj.sync()

            except Exception as e:
                print(f"Error occurred during task execution: {e}")
                # Re-enqueue the task for later processing
                problematic_tasks.append(task_dict)
                continue
        else:
            # No task found in the queue
            break

    # Re-enqueue the problematic tasks - in case if any api calls failed
    for task in problematic_tasks:
        enqueue_data_to_sync.delay(task['record_id'], task['entity'], task['operation'], task['delete'])


import re
import pickle
import argparse
import pandas as pd
import numpy as np

from tqdm import tqdm
from typing import Dict, List, Tuple

# ML Models
import sklearn
from sklearn.feature_selection import RFE
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neural_network import MLPClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
import xgboost as xgb

# ML Utilities
from sklearn.model_selection import train_test_split, cross_validate, GridSearchCV
from sklearn.utils import shuffle
from sklearn.utils._testing import ignore_warnings
from sklearn.exceptions import ConvergenceWarning
from sklearn.metrics import max_error

import utils
import handcrafted_features as HF

class ClassificationModel:
    """
        Wrapper Class for Classification Model.

        Parameters
        ----------
            - model: sklearn.Model
                - model to wrap. defaults to LogisticRegression if None
            - class_weight: Dict[int, float]
                - class weights for imbalnced training dataset.

        Performs the following:
            - training model (self.train)
            - testing model (self.test)
            - predictions (self.__call__)
            - k-fold validation (self.kfold_validate)
            - train validation (self.train_validate)
            - saving model (self.save)
            - loading model (self.load, class method)
    """
    def __init__(self, model=None, class_weight=None):
        # Define model
        if model is None:
            print("No model provided ... Defaulting to Logistic Regression ...")
            self.model = LogisticRegression(penalty='l1',
                                            solver='saga',
                                            multi_class='multinomial',
                                            max_iter=10000)
        else:
            self.model = model
        assert self.model is not None
        # Provide class weight
        if class_weight is not None:
            self.model.class_weight = class_weight
        self.model_name = type(self.model).__name__
        self.metrics = ['accuracy', 'f1_weighted', 'precision_weighted', 'recall_weighted']

    def __call__(self, X):
        return self.model.predict(X)

    def train(self, X, Y):
        assert X.shape[0] == Y.shape[0], "There must be the same number of examples. Note that 0-dimension is batch ... "
        self.model.fit(X, Y)
        print("trained")

    def test(self, X, Y):
        scoring = ['accuracy', 'precision']
        assert X.shape[0] == Y.shape[0], "There must be the same number of examples. Note that 0-dimension is batch ..."
        pred = self.model.predict(X)
        pred = np.flipud(pred)
        print(pred)
        print(Y)
        acc = sklearn.metrics.accuracy_score(Y, pred)
        f1 = sklearn.metrics.f1_score(Y, pred, average='weighted')
        pre = sklearn.metrics.precision_score(Y, pred, average='weighted')
        rec = sklearn.metrics.recall_score(Y, pred, average='weighted')
        qwk = sklearn.metrics.cohen_kappa_score(Y, pred, weights="quadratic")
        #print(f"Test Accuracy: {acc * 100:.2f}%")
        #print(f"Test F1-score: {f1 * 100:.2f}%")
        #print(f"Test Precision: {pre * 100:.2f}%")
        #print(f"Test Recall: {rec * 100:.2f}%")
        #print(f"Test QWK: {qwk * 100:.2f}%")
        return acc,f1,pre,rec,qwk

    def save(self, path):
        print(f"Saving model to {path} ...")
        pickle.dump(self.model, open(path, 'wb'))

    @classmethod
    def load(cls, path):
        print(f"Loading model from {path} ...")
        cls.model = pickle.load(open(path, 'rb'))
    """
    def rf_randomsearch(self, X:np.array, Y:np.array):
        n_estimators = [100, 300, 500, 800, 1200]
        max_depth = [5, 8, 15, 25, 30]
        min_samples_split = [2, 5, 10, 15, 100]
        min_samples_leaf = [1, 2, 5, 10]
        random_grid = dict(n_estimators = n_estimators,
                    max_depth = max_depth,
                    min_samples_split = min_samples_split,
                    min_samples_leaf = min_samples_leaf)
        rf_random = GridSearchCV(self.model, random_grid,cv = 3, verbose=1, n_jobs = -1)
        rf_random.fit(X,Y)
        return rf_random.best_params_
    """

    def SVC_randomsearch(self, X:np.array, Y:np.array):
        param_grid = {'C': [0.1,1, 10, 100], 'gamma': [1,0.1,0.01,0.001],'kernel': ['rbf', 'poly', 'sigmoid']}
        grid = GridSearchCV(SVC(),param_grid,refit=True,verbose=2)
        grid.fit(X,Y)
        return grid.best_params_

    @ignore_warnings(category=ConvergenceWarning)
    def kfold_validate(self, X:np.array, Y:np.array, n_splits:int=10):
        """
            Performs a K-fold validation.

            Parameters
            --------------
                - X: np.array [num_samples, num_features]
                    - numpy array of inputs.
                - Y: np.array [num_samples, ]
                    - numpy array of labels. must not be 1-hot.
            Returns
            --------------
                - None.
        """
        X, Y = shuffle(X, Y)
        scores = cross_validate(self.model, X, Y, scoring=self.metrics, cv=n_splits, n_jobs=4)
        print("metrics over {}-fold validation:".format(n_splits))
        for k, v in scores.items():
            print(k, v.tolist())
            print("average {}:".format(k), sum(v.tolist()) / len(v.tolist()))
            print('*' * 50)

    @ignore_warnings(category=ConvergenceWarning)
    def train_validate(self, X:np.array, Y:np.array):
        """
            Performs a train - validation.

            Parameters
            --------------
                - X: np.array [num_samples, num_features]
                    - numpy array of inputs.
                - Y: np.array [num_samples, ]
                    - numpy array of labels. must not be 1-hot.
            Returns
            --------------
                - None.
        """
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)
        self.train(X_train, Y_train)
        score = self.test(X_test, Y_test)
        return score

def get_class_error(model, X, Y, mapper):
    """
        Returns the mean and standard deviation error of grade difference
        between prediction and labels.

        Parameters
        --------------
            - model: sklearn.Model
                - classification model to evaluate.
            - X: np.array
                - input features.
            - Y: np.array
                - labels.
            - mapper: Dict[float, int]
                - a dictionary that hashes from grade (float) to multi-class label (int).
        Returns
        --------------
            - None.
    """
    reverse = {v:k for k,v in mapper.items()}
    Y_grade = np.array([reverse[i] for i in Y])
    Y_pred  = np.array([reverse[i] for i in model(X)])
    print(max_error(Y_grade, Y_pred))
    avg = np.mean(np.abs(Y_grade - Y_pred))
    std = np.std(Y_grade - Y_pred)
    print(f"Error Mean: {avg}, Error St. Dev.: {std}")

def runner(args,k_fold,dataset,neural_model_to_use,this_handcrafted_features_list):
    #train
    df_train = pd.read_csv('Research_Data/'+str(dataset)+'.'+str(k_fold)+'.train.combined.csv')

    #test
    df_test = pd.read_csv('Research_Data/'+str(dataset)+'.'+str(k_fold)+'.test.combined.csv')

    X_train, Y_train, mapper = utils.prepare_classification_data(df_train,neural_model_to_use,this_handcrafted_features_list)
    print(len(X_train))
    X_test, Y_test, mapper = utils.prepare_classification_data(df_test,neural_model_to_use,this_handcrafted_features_list)
    print(len(X_test))

    MODELS = list()
    if args.logistic:
        MODELS.append(ClassificationModel
                     (LogisticRegression
                     (penalty='l1',
                     dual=False,
                     tol=1e-4,
                     C=1.0,
                     fit_intercept=True,
                     intercept_scaling=1,
                     class_weight=None,
                     random_state=None,
                     solver='saga',
                     max_iter=10000,
                     multi_class='multinomial',
                     verbose=0,
                     warm_start=False,
                     n_jobs=None,
                     l1_ratio=None)))
    if args.perceptron:
        MODELS.append(ClassificationModel(MLPClassifier()))
    if args.svc:
        MODELS.append(ClassificationModel
                     (SVC
                     (C=1,
                     kernel="poly",
                     degree=3,
                     gamma="scale",
                     coef0=0,
                     shrinking=True,
                     probability=False,
                     tol=1e-3,
                     cache_size=200,
                     class_weight=None,
                     verbose=False,
                     max_iter=-1,
                     decision_function_shape="ovr",
                     break_ties=False,
                     random_state=None)))
    if args.decision_tree:
        MODELS.append(ClassificationModel(DecisionTreeClassifier()))
    if args.naive_bayes:
        MODELS.append(ClassificationModel(GaussianNB()))
    if args.random_forest:
        MODELS.append(ClassificationModel
                     (RandomForestClassifier
                     (n_estimators=800,
                     criterion="gini",
                     max_depth=None,
                     min_samples_split=2,
                     min_samples_leaf=1,
                     min_weight_fraction_leaf=0.0,
                     max_features="auto",
                     max_leaf_nodes=None,
                     min_impurity_decrease=0.0,
                     min_impurity_split=None,
                     bootstrap=True,
                     oob_score=False,
                     n_jobs=None,
                     random_state=None,
                     verbose=0,
                     warm_start=False,
                     class_weight=None,
                     ccp_alpha=0.0,
                     max_samples=None)))
    if args.gradient_boost:
        MODELS.append(ClassificationModel
                     (xgb.XGBClassifier(
                     eta=0.05,
                     max_depth=9,
                     min_child_weight=0.05,
                     gamma=1,
                     subsample=0.8,
                     objective='softmax')))
    for model in MODELS:
        name = utils.CAMEL2SNAKE(model.model_name)
        print(f"{'=' * 25} RUNNING {name.replace('_', ' ').upper()} ... {'=' * 25}")

        model.train(X_train, Y_train)
        acc,f1,pre,rec,qwk = model.test(X_test, Y_test)
        return acc,f1,pre,rec,qwk
        #model.save(f'checkpoint/classification/{name.lower()}.pt')

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Specify models to run ...')
    parser.add_argument(
        '--logistic', '-l',
        action='store_true',
        help="Run logistic regression."
    )
    parser.add_argument(
        '--perceptron', '-p',
        action='store_true',
        help="Run perceptron regression."
    )
    parser.add_argument(
        '--svc', '-s',
        action='store_true',
        help="Run support vector regression."
    )
    parser.add_argument(
        '--decision-tree', '-d',
        action='store_true',
        help="Run decision tree regression."
    )
    parser.add_argument(
        '--naive-bayes', '-n',
        action='store_true',
        help="Run naive bayes."
    )
    parser.add_argument(
        '--random-forest', '-r',
        action='store_true',
        help="Run random forest regression."
    )
    parser.add_argument(
        '--gradient-boost', '-g',
        action='store_true',
        help="Run gradient boosted trees."
    )
    parser.add_argument(
        '--svc-kernel',
        type=str,
        default='rbf',
        help='SVC Kernel to use.'
    )
    args = parser.parse_args()

    # choose dataset
    # 'cambridge' 'weebit' 'onestop'
    dataset = "weebit50"

    # choose neural model to use
    # 'bert' 'roberta' 'bart' 'xlnet' 'none'
    neural_model_to_use = ["bert"]

    # choose handcrafted features to use
    # if FeatureSet_all_HF(), use below
    # handcrafted_features_to_use = HF.FeatureSet_all_as_list_HF()
    # if else, use below
    # handcrafted_features_to_use = [HF.FeatureSet_xxx_HF()]
    handcrafted_features_to_use = [HF.FeatureSet_Total_HF()]
    total_acc_list =[]
    for this_handcrafted_features_list in handcrafted_features_to_use:
        acc_list = []
        f1_list = []
        pre_list = []
        rec_list = []
        qwk_list = []
        if dataset == 'weebit':
            k_fold = [0,1,2,3,4]
            for fold in k_fold:
                acc,f1,pre,rec,qwk = runner(args,fold,'weebit',neural_model_to_use,this_handcrafted_features_list)
                acc_list.append(acc)
                f1_list.append(f1)
                pre_list.append(pre)
                rec_list.append(rec)
                qwk_list.append(qwk)

        if dataset == 'onestop':
            k_fold = [0,1,2]
            for fold in k_fold:
                acc,f1,pre,rec,qwk = runner(args,fold,'onestop',neural_model_to_use,this_handcrafted_features_list)
                acc_list.append(acc)
                f1_list.append(f1)
                pre_list.append(pre)
                rec_list.append(rec)
                qwk_list.append(qwk)

        if dataset == 'cambridge':
            k_fold = [0,1,2,3,4]
            for fold in k_fold:
                acc,f1,pre,rec,qwk = runner(args,fold,'cambridge',neural_model_to_use,this_handcrafted_features_list)
                acc_list.append(acc)
                f1_list.append(f1)
                pre_list.append(pre)
                rec_list.append(rec)
                qwk_list.append(qwk)

        if dataset == 'weebit50' or dataset == 'weebit150' or dataset == 'weebit60p' or dataset == 'weebit80p':
            k_fold = [0]
            number_list=[10,20,30,40,50,60,70,80,90,110,120,130,140,150]
            for number in number_list:
                print(number)
                for fold in k_fold:
                    acc,f1,pre,rec,qwk = runner(args,fold,'weebit'+str(number),neural_model_to_use,this_handcrafted_features_list)
                    acc_list.append(acc)
                    f1_list.append(f1)
                    pre_list.append(pre)
                    rec_list.append(rec)
                    qwk_list.append(qwk)

                print(acc_list)
                print(f"{'=' * 25} AVERAGING ... {'=' * 25}")
                print("avg acc:" +str(round(float(sum(acc_list) / len(acc_list)), 3)))
                print("avg f1:" +str(round(float(sum(f1_list) / len(f1_list)), 3)))
                print("avg pre:" +str(round(float(sum(pre_list) / len(pre_list)), 3)))
                print("avg rec:" +str(round(float(sum(rec_list) / len(rec_list)), 3)))
                print("avg qwk:" +str(round(float(sum(qwk_list) / len(qwk_list)), 3)))
        #total_acc_list.append((this_handcrafted_features_list[0],round(float(sum(acc_list) / len(acc_list)),3)))
    #total_acc_list.sort(key = lambda x: x[1])
    #print(total_acc_list)


import numpy as np
import sklearn
from sklearn.preprocessing import scale
from sklearn.datasets import load_digits
from sklearn.cluster import KMeans
from sklearn import metrics
import time

digits = load_digits()

#scale serve para transpor os valores em um intervalor de -1 até 1 --> Salvar tempo computacionals
data = scale(digits.data)

#Pegar as Labels do dataset
y = digits.target

#Criar os centroids, os pontos de separação
k = len(np.unique(y))

#Aqui ele irá pegar a quantidade de valores que queremos classificar e suas features
samples, features = data.shape

#Essa função será para pegar a acuracidade do código
#Iremos dar o classificador estimator, que irá pegar o valor data, e assim pegar um score de acuracidade
def bench_k_means(estimator, name, data):
    estimator.fit(data)
    print('%-9s\t%.2fs\t%i\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f'
          % (name, estimator.inertia_,
          metrics.homogeneity_score(y, estimator.labels_),
          metrics.completeness_score(y, estimator.labels_),
          metrics.v_measure_score(y, estimator.labels_),
          metrics.adjusted_rand_score(y, estimator.labels_),
          metrics.adjusted_mutual_info_score(y,  estimator.labels_),
          metrics.silhouette_score(data, estimator.labels_,metric='euclidean')))

#Criando o classifier
#n_cluster = Quantas centroids queremos inserir
# init = Aonde os centroids irão se posicionar inicialmente
#n_init = Quantas vezes o classifier vai rodar com diferentes centroids
clf = KMeans(n_clusters=k, init="random", n_init=10)

bench_k_means(clf, "1", data)


from numpy import array, hstack

# ============= enthought library imports =======================
from traits.api import HasTraits, Str, Int, Bool, Float, Property, Color, Array

# ============= local library imports  ==========================

class BaseDetector(HasTraits):
    name = Str
    kind = Str
    serial_id = Str
    intensity = Str
    std = Str
    intensities = Array
    nstd = Int(10)
    active = Bool(True)
    gain = Float
    software_gain = Float(1.0)

    color = Color
    series_id = Int
    isotope = Str
    mass = Float

    isotopes = Property

    index = Float
    ypadding = Str

    def set_intensity(self, v):
        if v is not None:
            n = self.nstd
            if self.intensities is None:
                self.intensities = array([])

            self.intensities = hstack((self.intensities[-n:], [v]))
            self.std = "{:0.5f}".format(self.intensities.std())
            self.intensity = "{:0.5f}".format(v)

    @property
    def gain_outdated(self):
        return abs(self.get_gain() - self.gain) < 1e-7

    def get_gain(self):
        v = self._read_gain()
        try:
            v = float(v)
        except (TypeError, ValueError):
            v = 1
        self.gain = v
        return v

    def set_gain(self):
        self._set_gain()

    def get_deflection_correction(self, *args, **kw):
        return 0

    def toyaml(self):
        attrs = (
            "name",
            "index",
            "software_gain",
            "serial_id",
            "relative_position",
            "use_deflection",
            "protection_threshold",
            "deflection_correction_sign",
            "deflection_name",
            "active",
            "isotope",
            "kind",
            "ypadding",
        )
        yd = {attr: getattr(self, attr) for attr in attrs}
        color = self.color
        yd["color"] = int("0x{:02X}{:02X}{:02X}".format(*color.getRgb()), 16)
        return yd

    # private
    def _set_gain(self):
        raise NotImplementedError

    def _read_gain(self):
        raise NotImplementedError

    def _get_isotopes(self):
        molweights = self.spectrometer.molecular_weights
        return sorted(molweights.keys())
        # return sorted(molweights.keys(), key=lambda x: int(x[2:]))

    def __repr__(self):
        return "Detector({})".format(self.name)

# ============= EOF =============================================


#!/usr/bin/env python3

import sys
import timeit

from scrubadub.comparison import make_fake_document

def main():
    doc, _ = make_fake_document(paragraphs=20, seed=1234)
    variables = {'doc': doc}
    setup_cmd = 'import scrubadub; scrubber = scrubadub.Scrubber()'
    cmd = 'scrubber.clean(doc)'

    print("Timing '{}':".format(cmd))
    repeats = 50
    timer = timeit.Timer(cmd, setup=setup_cmd, globals=variables)
    try:
        time = timer.timeit(number=repeats)
    except Exception:
        timer.print_exc()
        sys.exit(1)
    else:
        print("{: >8.4f}s total runtime".format(time))
        print("{: >8.4f}s per iteration".format(time/repeats))

    if time/repeats > 0.1:
        print("Usual runtimes for the default set of detectors is 0.02s per iteration.")
        sys.exit(1)

    sys.exit(0)

if __name__ == "__main__":
    main()


from collections import defaultdict
from tqdm import tqdm
import numpy as np

from .common import voc_ap, viou

def eval_detection_scores(gt_relations, pred_relations, viou_threshold, top_returns=None):
    pred_relations = sorted(pred_relations, key=lambda x: x['score'], reverse=True)
    if top_returns is not None:
        pred_relations = pred_relations[:top_returns]
    gt_detected = np.zeros((len(gt_relations),), dtype=bool)
    hit_scores = np.ones((len(pred_relations))) * -np.inf
    for pred_idx, pred_relation in enumerate(pred_relations):
        ov_max = -float('Inf')
        k_max = -1
        for gt_idx, gt_relation in enumerate(gt_relations):
            if not gt_detected[gt_idx]\
                    and tuple(pred_relation['triplet']) == tuple(gt_relation['triplet']):
                s_iou = viou(pred_relation['sub_traj'], pred_relation['duration'],
                        gt_relation['sub_traj'], gt_relation['duration'])
                o_iou = viou(pred_relation['obj_traj'], pred_relation['duration'],
                        gt_relation['obj_traj'], gt_relation['duration'])
                ov = min(s_iou, o_iou)
                if ov >= viou_threshold and ov > ov_max:
                    ov_max = ov
                    k_max = gt_idx
        if k_max >= 0:
            hit_scores[pred_idx] = pred_relation['score']
            gt_detected[k_max] = True
    tp = np.isfinite(hit_scores)
    fp = ~tp
    cum_tp = np.cumsum(tp).astype(np.float32)
    cum_fp = np.cumsum(fp).astype(np.float32)
    rec = cum_tp / np.maximum(len(gt_relations), np.finfo(np.float32).eps)
    prec = cum_tp / np.maximum(cum_tp + cum_fp, np.finfo(np.float32).eps)
    return prec, rec, hit_scores

def eval_tagging_scores(gt_relations, pred_relations, top_returns=None):
    pred_relations = sorted(pred_relations, key=lambda x: x['score'], reverse=True)
    if top_returns is not None:
        pred_relations = pred_relations[:top_returns]
    # ignore trajectories
    gt_triplets = set(tuple(r['triplet']) for r in gt_relations)
    pred_triplets = []
    hit_scores = []
    for r in pred_relations:
        triplet = tuple(r['triplet'])
        if not triplet in pred_triplets:
            pred_triplets.append(triplet)
            hit_scores.append(r['score'])
    hit_scores = np.asarray(hit_scores)
    for i, t in enumerate(pred_triplets):
        if not t in gt_triplets:
            hit_scores[i] = -np.inf
    tp = np.isfinite(hit_scores)
    fp = ~tp
    cum_tp = np.cumsum(tp).astype(np.float32)
    cum_fp = np.cumsum(fp).astype(np.float32)
    rec = cum_tp / np.maximum(len(gt_triplets), np.finfo(np.float32).eps)
    prec = cum_tp / np.maximum(cum_tp + cum_fp, np.finfo(np.float32).eps)
    return prec, rec, hit_scores

def evaluate(groundtruth, prediction, top_returns=200, viou_threshold=0.5, det_recall_at_n=[50, 100], tag_precision_at_n=[1, 5, 10]):
    """ evaluate visual relation detection and visual
    relation tagging.
    """
    video_ap = dict()
    tot_scores = defaultdict(list)
    tot_tp = defaultdict(list)
    prec_at_n = defaultdict(list)
    tot_gt_relations = 0
    print('Computing average precision AP over {} videos...'.format(len(groundtruth)))
    for vid, gt_relations in tqdm(groundtruth.items()):
        if len(gt_relations)==0:
            continue
        tot_gt_relations += len(gt_relations)
        predict_relations = prediction.get(vid, [])
        # compute average precision and recalls in detection setting
        det_prec, det_rec, det_scores = eval_detection_scores(
                gt_relations, predict_relations, viou_threshold, top_returns)
        video_ap[vid] = voc_ap(det_rec, det_prec)
        tp = np.isfinite(det_scores)
        for nre in det_recall_at_n:
            cut_off = min(nre, det_scores.size)
            tot_scores[nre].append(det_scores[:cut_off])
            tot_tp[nre].append(tp[:cut_off])
        # compute precisions in tagging setting
        tag_prec, _, _ = eval_tagging_scores(gt_relations, predict_relations, top_returns)
        for nre in tag_precision_at_n:
            cut_off = min(nre, tag_prec.size)
            if cut_off > 0:
                prec_at_n[nre].append(tag_prec[cut_off - 1])
            else:
                prec_at_n[nre].append(0.)
    # calculate mean ap for detection
    mean_ap = round(float(np.mean(list(video_ap.values()))), 4)
    # calculate recall for detection
    rec_at_n = dict()
    for nre in det_recall_at_n:
        scores = np.concatenate(tot_scores[nre])
        tps = np.concatenate(tot_tp[nre])
        sort_indices = np.argsort(scores)[::-1]
        tps = tps[sort_indices]
        cum_tp = np.cumsum(tps).astype(np.float32)
        rec = cum_tp / np.maximum(tot_gt_relations, np.finfo(np.float32).eps)
        if len(rec) > 0:
            rec_at_n[nre] = round(float(rec[-1]), 4)
        else:
            rec_at_n[nre] = 0.
    # calculate mean precision for tagging
    mprec_at_n = dict()
    for nre in tag_precision_at_n:
        mprec_at_n[nre] = round(float(np.mean(prec_at_n[nre])), 4)
    # print scores
    print('detection mean AP (used in challenge): {}'.format(mean_ap))
    print('detection recall@50: {}'.format(rec_at_n[50]))
    print('detection recall@100: {}'.format(rec_at_n[100]))
    print('tagging precision@1: {}'.format(mprec_at_n[1]))
    print('tagging precision@5: {}'.format(mprec_at_n[5]))
    print('tagging precision@10: {}'.format(mprec_at_n[10]))
    return mean_ap, rec_at_n, mprec_at_n


import pytest
import aiosocks
from aiohttp.test_utils import make_mocked_coro
from unittest import mock

async def test_create_connection_init():
    addr = aiosocks.Socks5Addr('localhost')
    auth = aiosocks.Socks5Auth('usr', 'pwd')
    dst = ('python.org', 80)

    # proxy argument
    with pytest.raises(AssertionError) as ct:
        await aiosocks.create_connection(None, None, auth, dst)
    assert 'proxy must be Socks4Addr() or Socks5Addr() tuple' in str(ct.value)

    with pytest.raises(AssertionError) as ct:
        await aiosocks.create_connection(None, auth, auth, dst)
    assert 'proxy must be Socks4Addr() or Socks5Addr() tuple' in str(ct.value)

    # proxy_auth
    with pytest.raises(AssertionError) as ct:
        await aiosocks.create_connection(None, addr, addr, dst)
    assert 'proxy_auth must be None or Socks4Auth()' in str(ct.value)

    # dst
    with pytest.raises(AssertionError) as ct:
        await aiosocks.create_connection(None, addr, auth, None)
    assert 'invalid dst format, tuple("dst_host", dst_port))' in str(ct.value)

    # addr and auth compatibility
    with pytest.raises(ValueError) as ct:
        await aiosocks.create_connection(
            None, addr, aiosocks.Socks4Auth(''), dst)
    assert 'proxy is Socks5Addr but proxy_auth is not Socks5Auth' \
           in str(ct.value)

    with pytest.raises(ValueError) as ct:
        await aiosocks.create_connection(
            None, aiosocks.Socks4Addr(''), auth, dst)
    assert 'proxy is Socks4Addr but proxy_auth is not Socks4Auth' \
           in str(ct.value)

    # test ssl, server_hostname
    with pytest.raises(ValueError) as ct:
        await aiosocks.create_connection(
            None, addr, auth, dst, server_hostname='python.org')
    assert 'server_hostname is only meaningful with ssl' in str(ct.value)

async def test_connection_fail():
    addr = aiosocks.Socks5Addr('localhost')
    auth = aiosocks.Socks5Auth('usr', 'pwd')
    dst = ('python.org', 80)

    loop_mock = mock.Mock()
    loop_mock.create_connection = make_mocked_coro(raise_exception=OSError())

    with pytest.raises(aiosocks.SocksConnectionError):
        await aiosocks.create_connection(
            None, addr, auth, dst, loop=loop_mock)

async def test_negotiate_fail():
    addr = aiosocks.Socks5Addr('localhost')
    auth = aiosocks.Socks5Auth('usr', 'pwd')
    dst = ('python.org', 80)

    loop_mock = mock.Mock()
    loop_mock.create_connection = make_mocked_coro((mock.Mock(), mock.Mock()))

    with mock.patch('aiosocks.asyncio.Future') as future_mock:
        future_mock.side_effect = make_mocked_coro(
            raise_exception=aiosocks.SocksError())

        with pytest.raises(aiosocks.SocksError):
            await aiosocks.create_connection(
                None, addr, auth, dst, loop=loop_mock)

async def test_open_connection():
    addr = aiosocks.Socks5Addr('localhost')
    auth = aiosocks.Socks5Auth('usr', 'pwd')
    dst = ('python.org', 80)

    transp, proto = mock.Mock(), mock.Mock()
    reader, writer = mock.Mock(), mock.Mock()

    proto.app_protocol.reader, proto.app_protocol.writer = reader, writer

    loop_mock = mock.Mock()
    loop_mock.create_connection = make_mocked_coro((transp, proto))

    with mock.patch('aiosocks.asyncio.Future') as future_mock:
        future_mock.side_effect = make_mocked_coro(True)
        r, w = await aiosocks.open_connection(addr, auth, dst, loop=loop_mock)

    assert reader is r
    assert writer is w


#python

import k3d
import testing

document = k3d.new_document()

grid = k3d.plugin.create("PolyGrid"), document;
grid.columns = 1
grid.rows = 1

modifier = k3d.plugin.create("SubdivideEdges", document)
cuda_modifier = k3d.plugin.create("CUDASubdivideEdges", document)

modifier.mesh_selection = k3d.geometry.selection.create(1)
modifier.vertices = 1

cuda_modifier.mesh_selection = modifier.mesh_selection
cuda_modifier.vertices = 1

k3d.property.connect(document, grid.get_property("output_mesh"), modifier.get_property("input_mesh"))
k3d.property.connect(document, grid.get_property("output_mesh"), cuda_modifier.get_property("input_mesh"))

testing.output_mesh_difference(cuda_modifier.output_mesh, modifier.output_mesh, 1)

difference = testing.get_mesh_difference(document, cuda_modifier.get_property("output_mesh"), modifier.get_property("output_mesh"), 1)

if not difference.equal:
    raise "Meshes not equal"